mittens 0.2.0 → 0.3.0

data/vendor/snowball/java/org/tartarus/snowball/SnowballStemmer.java

@@ -1,6 +1,9 @@
 
 package org.tartarus.snowball;
 
+/**
+ * Parent class of all snowball stemmers, which must implement <code>stem</code>
+ */
 public abstract class SnowballStemmer extends SnowballProgram {
     public abstract boolean stem();
 

data/vendor/snowball/java/org/tartarus/snowball/TestApp.java

@@ -1,7 +1,6 @@
 
 package org.tartarus.snowball;
 
-import java.lang.reflect.Method;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.FileInputStream;
@@ -13,6 +12,7 @@ import java.io.OutputStreamWriter;
 import java.io.Reader;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 
 public class TestApp {
     private static void usage()
@@ -20,56 +20,71 @@ public class TestApp {
         System.err.println("Usage: TestApp <algorithm> [<input file>] [-o <output file>]");
     }
 
-    public static void main(String [] args) throws Throwable {
-	if (args.length < 2) {
+    private static SnowballStemmer getStemmer(String lang) {
+        try {
+            String c = "org.tartarus.snowball.ext." + lang + "Stemmer";
+            return (SnowballStemmer) Class.forName(c).getDeclaredConstructor().newInstance();
+        } catch (ReflectiveOperationException e) {
+            return null;
+        }
+    }
+
+    public static void main(String[] args) throws Throwable {
+        if (args.length < 1) {
             usage();
             return;
         }
 
-	Class stemClass = Class.forName("org.tartarus.snowball.ext." +
-					args[0] + "Stemmer");
-        SnowballStemmer stemmer = (SnowballStemmer) stemClass.newInstance();
+        SnowballStemmer stemmer = getStemmer(args[0]);
+        if (stemmer == null) {
+            System.err.println("Stemmer " + args[0] + " not found");
+            return;
+        }
 
-	int arg = 1;
+        int arg = 1;
 
-	InputStream instream;
-	if (args.length > arg && !args[arg].equals("-o")) {
-	    instream = new FileInputStream(args[arg++]);
-	} else {
-	    instream = System.in;
-	}
+        InputStream instream;
+        if (args.length > arg && !args[arg].equals("-o")) {
+            instream = new FileInputStream(args[arg++]);
+        } else {
+            instream = System.in;
+        }
 
         OutputStream outstream;
-	if (args.length > arg) {
+        if (args.length > arg) {
             if (args.length != arg + 2 || !args[arg].equals("-o")) {
                 usage();
                 return;
             }
-	    outstream = new FileOutputStream(args[arg + 1]);
-	} else {
-	    outstream = System.out;
-	}
+            outstream = new FileOutputStream(args[arg + 1]);
+        } else {
+            outstream = System.out;
+        }
 
-	Reader reader = new InputStreamReader(instream, StandardCharsets.UTF_8);
-	reader = new BufferedReader(reader);
+        Reader reader = new InputStreamReader(instream, StandardCharsets.UTF_8);
+        reader = new BufferedReader(reader);
 
-	Writer output = new OutputStreamWriter(outstream, StandardCharsets.UTF_8);
-	output = new BufferedWriter(output);
+        Writer output = new OutputStreamWriter(outstream, StandardCharsets.UTF_8);
+        output = new BufferedWriter(output);
 
-	StringBuffer input = new StringBuffer();
-	int character;
-	while ((character = reader.read()) != -1) {
-	    char ch = (char) character;
-	    if (Character.isWhitespace(ch)) {
-		stemmer.setCurrent(input.toString());
-		stemmer.stem();
-		output.write(stemmer.getCurrent());
-		output.write('\n');
-		input.delete(0, input.length());
-	    } else {
-		input.append(ch < 127 ? Character.toLowerCase(ch) : ch);
-	    }
-	}
-	output.flush();
+        char[] input = new char[8];
+        int length = 0;
+        int character;
+        while ((character = reader.read()) != -1) {
+            char ch = (char) character;
+            if (Character.isWhitespace(ch)) {
+                stemmer.setCurrent(input, length);
+                stemmer.stem();
+                output.write(stemmer.getCurrentBuffer(), 0, stemmer.getCurrentBufferLength());
+                output.write('\n');
+                length = 0;
+            } else {
+                if (length == input.length) {
+                        input = Arrays.copyOf(input, length + 1);
+                }
+                input[length++] = ch < 127 ? Character.toLowerCase(ch) : ch;
+            }
+        }
+        output.close();
     }
 }

data/vendor/snowball/javascript/base-stemmer.js

@@ -1,5 +1,18 @@
+// @ts-check
+
 /**@constructor*/
-BaseStemmer = function() {
+const BaseStemmer = function() {
+    /** @protected */
+    this.current = '';
+    this.cursor = 0;
+    this.limit = 0;
+    this.limit_backward = 0;
+    this.bra = 0;
+    this.ket = 0;
+
+    /**
+     * @param {string} value
+     */
     this.setCurrent = function(value) {
         this.current = value;
         this.cursor = 0;
@@ -9,11 +22,18 @@ BaseStemmer = function() {
         this.ket = this.limit;
     };
 
+    /**
+     * @return {string}
+     */
     this.getCurrent = function() {
         return this.current;
     };
 
+    /**
+     * @param {BaseStemmer} other
+     */
     this.copy_from = function(other) {
+        /** @protected */
         this.current          = other.current;
         this.cursor           = other.cursor;
         this.limit            = other.limit;
@@ -22,7 +42,14 @@ BaseStemmer = function() {
         this.ket              = other.ket;
     };
 
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
     this.in_grouping = function(s, min, max) {
+        /** @protected */
         if (this.cursor >= this.limit) return false;
         var ch = this.current.charCodeAt(this.cursor);
         if (ch > max || ch < min) return false;
@@ -32,7 +59,34 @@ BaseStemmer = function() {
         return true;
     };
 
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_in_grouping = function(s, min, max) {
+        /** @protected */
+        while (this.cursor < this.limit) {
+            var ch = this.current.charCodeAt(this.cursor);
+            if (ch > max || ch < min)
+                return true;
+            ch -= min;
+            if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0)
+                return true;
+            this.cursor++;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
     this.in_grouping_b = function(s, min, max) {
+        /** @protected */
         if (this.cursor <= this.limit_backward) return false;
         var ch = this.current.charCodeAt(this.cursor - 1);
         if (ch > max || ch < min) return false;
@@ -42,7 +96,32 @@ BaseStemmer = function() {
         return true;
     };
 
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_in_grouping_b = function(s, min, max) {
+        /** @protected */
+        while (this.cursor > this.limit_backward) {
+            var ch = this.current.charCodeAt(this.cursor - 1);
+            if (ch > max || ch < min) return true;
+            ch -= min;
+            if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return true;
+            this.cursor--;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
     this.out_grouping = function(s, min, max) {
+        /** @protected */
         if (this.cursor >= this.limit) return false;
         var ch = this.current.charCodeAt(this.cursor);
         if (ch > max || ch < min) {
@@ -57,7 +136,35 @@ BaseStemmer = function() {
         return false;
     };
 
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_out_grouping = function(s, min, max) {
+        /** @protected */
+        while (this.cursor < this.limit) {
+            var ch = this.current.charCodeAt(this.cursor);
+            if (ch <= max && ch >= min) {
+                ch -= min;
+                if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) != 0) {
+                    return true;
+                }
+            }
+            this.cursor++;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
     this.out_grouping_b = function(s, min, max) {
+        /** @protected */
         if (this.cursor <= this.limit_backward) return false;
         var ch = this.current.charCodeAt(this.cursor - 1);
         if (ch > max || ch < min) {
@@ -72,8 +179,34 @@ BaseStemmer = function() {
         return false;
     };
 
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_out_grouping_b = function(s, min, max) {
+        /** @protected */
+        while (this.cursor > this.limit_backward) {
+            var ch = this.current.charCodeAt(this.cursor - 1);
+            if (ch <= max && ch >= min) {
+                ch -= min;
+                if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) != 0) {
+                    return true;
+                }
+            }
+            this.cursor--;
+        }
+        return false;
+    };
+
+    /**
+     * @param {string} s
+     * @return {boolean}
+     */
     this.eq_s = function(s)
     {
+        /** @protected */
         if (this.limit - this.cursor < s.length) return false;
         if (this.current.slice(this.cursor, this.cursor + s.length) != s)
         {
@@ -83,8 +216,13 @@ BaseStemmer = function() {
         return true;
     };
 
+    /**
+     * @param {string} s
+     * @return {boolean}
+     */
     this.eq_s_b = function(s)
     {
+        /** @protected */
         if (this.cursor - this.limit_backward < s.length) return false;
         if (this.current.slice(this.cursor - s.length, this.cursor) != s)
         {
@@ -94,8 +232,13 @@ BaseStemmer = function() {
         return true;
     };
 
-    /** @return {number} */ this.find_among = function(v)
+    /**
+     * @param {Among[]} v
+     * @return {number}
+     */
+    this.find_among = function(v)
     {
+        /** @protected */
         var i = 0;
         var j = v.length;
 
@@ -165,8 +308,13 @@ BaseStemmer = function() {
     };
 
     // find_among_b is for backwards processing. Same comments apply
+    /**
+     * @param {Among[]} v
+     * @return {number}
+     */
     this.find_among_b = function(v)
     {
+        /** @protected */
         var i = 0;
         var j = v.length
 
@@ -232,8 +380,15 @@ BaseStemmer = function() {
     /* to replace chars between c_bra and c_ket in this.current by the
      * chars in s.
      */
+    /**
+     * @param {number} c_bra
+     * @param {number} c_ket
+     * @param {string} s
+     * @return {number}
+     */
     this.replace_s = function(c_bra, c_ket, s)
     {
+        /** @protected */
         var adjustment = s.length - (c_ket - c_bra);
         this.current = this.current.slice(0, c_bra) + s + this.current.slice(c_ket);
         this.limit += adjustment;
@@ -242,8 +397,12 @@ BaseStemmer = function() {
         return adjustment;
     };
 
+    /**
+     * @return {boolean}
+     */
     this.slice_check = function()
     {
+        /** @protected */
         if (this.bra < 0 ||
             this.bra > this.ket ||
             this.ket > this.limit ||
@@ -254,8 +413,13 @@ BaseStemmer = function() {
         return true;
     };
 
+    /**
+     * @param {number} c_bra
+     * @return {boolean}
+     */
     this.slice_from = function(s)
     {
+        /** @protected */
         var result = false;
         if (this.slice_check())
         {
@@ -265,20 +429,34 @@ BaseStemmer = function() {
         return result;
     };
 
+    /**
+     * @return {boolean}
+     */
     this.slice_del = function()
     {
+        /** @protected */
         return this.slice_from("");
     };
 
+    /**
+     * @param {number} c_bra
+     * @param {number} c_ket
+     * @param {string} s
+     */
     this.insert = function(c_bra, c_ket, s)
     {
+        /** @protected */
         var adjustment = this.replace_s(c_bra, c_ket, s);
         if (c_bra <= this.bra) this.bra += adjustment;
         if (c_bra <= this.ket) this.ket += adjustment;
     };
 
+    /**
+     * @return {string}
+     */
     this.slice_to = function()
     {
+        /** @protected */
         var result = '';
         if (this.slice_check())
         {
@@ -287,8 +465,14 @@ BaseStemmer = function() {
         return result;
     };
 
+    /**
+     * @return {string}
+     */
     this.assign_to = function()
     {
+        /** @protected */
         return this.current.slice(0, this.limit);
     };
 };
+
+if (typeof module === 'object' && module.exports) module.exports = BaseStemmer;

data/vendor/snowball/javascript/stemwords.js

@@ -1,5 +1,3 @@
-const stemmer = require('base-stemmer.js');
-
 const fs = require('fs');
 const readline = require('readline');
 
@@ -81,7 +79,7 @@ else
 function stemming (lang, input, output, encoding) {
     const lines = readline.createInterface({
         input: fs.createReadStream(input, encoding),
-	terminal: false
+        terminal: false
     });
     var out = fs.createWriteStream(output, encoding);
     var stemmer = create(lang);
@@ -93,10 +91,9 @@ function stemming (lang, input, output, encoding) {
 function create (name) {
     var lc_name = name.toLowerCase();
     if (!lc_name.match('\\W') && lc_name != 'base') {
-        var algo = lc_name.substr(0, 1).toUpperCase() + lc_name.substr(1);
         try {
-            const stemmer = require(lc_name + '-stemmer.js');
-            return Function('return new ' + algo + 'Stemmer()')();
+            const Stemmer = require(lc_name + '-stemmer.js');
+            return new Stemmer();
         } catch (error) {
         }
     }

data/vendor/snowball/libstemmer/libstemmer_c.in

@@ -45,7 +45,7 @@ sb_stemmer_new(const char * algorithm, const char * charenc)
         if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
     }
     if (module->name == NULL) return NULL;
-    
+
     stemmer = (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
     if (stemmer == NULL) return NULL;
 

data/vendor/snowball/libstemmer/mkalgorithms.pl

@@ -73,16 +73,16 @@ EOS
     my @algorithms = sort keys(%algorithms);
     print OUT "\nlibstemmer_algorithms =";
     foreach $lang (@algorithms) {
-	print OUT "\\\n    ", $lang;
+        print OUT "\\\n    ", $lang;
     }
     print OUT "\n";
 
     for my $enc (qw(ISO_8859_1 ISO_8859_2 KOI8_R)) {
-	print OUT "\n${enc}_algorithms =";
-	foreach $lang (@algorithms) {
-	    print OUT "\\\n    ", $lang if exists $algorithm_encs{$lang}->{$enc};
-	}
-	print OUT "\n";
+        print OUT "\n${enc}_algorithms =";
+        foreach $lang (@algorithms) {
+            print OUT "\\\n    ", $lang if exists $algorithm_encs{$lang}->{$enc};
+        }
+        print OUT "\n";
     }
 }
 

data/vendor/snowball/libstemmer/mkmodules.pl

@@ -35,7 +35,7 @@ sub addalgenc($$) {
       my $norm_enc = lc $enc;
       $norm_enc =~ s/_//g;
       if ($norm_enc ne $enc_only) {
-	  return;
+          return;
       }
   }
 
@@ -146,7 +146,7 @@ EOS
 
 struct stemmer_modules {
   const char * name;
-  stemmer_encoding_t enc; 
+  stemmer_encoding_t enc;
   struct SN_env * (*create)(void);
   void (*close)(struct SN_env *);
   int (*stem)(struct SN_env *);

data/vendor/snowball/libstemmer/modules.txt

@@ -14,8 +14,10 @@ armenian        UTF_8                   armenian,hy,hye,arm
 basque          UTF_8,ISO_8859_1        basque,eu,eus,baq
 catalan         UTF_8,ISO_8859_1        catalan,ca,cat
 danish          UTF_8,ISO_8859_1        danish,da,dan
-dutch           UTF_8,ISO_8859_1        dutch,nl,dut,nld
+dutch           UTF_8,ISO_8859_1        dutch,nl,dut,nld,kraaij_pohlmann
 english         UTF_8,ISO_8859_1        english,en,eng
+esperanto       UTF_8                   esperanto,eo,epo
+estonian        UTF_8                   estonian,et,est
 finnish         UTF_8,ISO_8859_1        finnish,fi,fin
 french          UTF_8,ISO_8859_1        french,fr,fre,fra
 german          UTF_8,ISO_8859_1        german,de,ger,deu
@@ -29,7 +31,7 @@ lithuanian      UTF_8                   lithuanian,lt,lit
 nepali          UTF_8                   nepali,ne,nep
 norwegian       UTF_8,ISO_8859_1        norwegian,no,nor
 portuguese      UTF_8,ISO_8859_1        portuguese,pt,por
-romanian        UTF_8,ISO_8859_2        romanian,ro,rum,ron
+romanian        UTF_8                   romanian,ro,rum,ron
 russian         UTF_8,KOI8_R            russian,ru,rus
 serbian         UTF_8                   serbian,sr,srp
 spanish         UTF_8,ISO_8859_1        spanish,es,esl,spa
@@ -42,7 +44,14 @@ yiddish         UTF_8                   yiddish,yi,yid
 # The porter algorithm is included in the libstemmer distribution to assist
 # with backwards compatibility, but for new systems the english algorithm
 # should be used in preference.
-porter          UTF_8,ISO_8859_1        porter			english
+porter          UTF_8,ISO_8859_1        porter                  english
+
+# This is Martin Porter's Dutch stemmer.  It was the default Dutch stemming
+# in Snowball 2.2.0 and earlier, but after user feedback and careful evaluation
+# we concluded that the Kraaij-Pohlmann Dutch stemmer was a better default.
+# We still provide this to help people who have a lot of existing data indexed
+# using it.
+dutch_porter    UTF_8,ISO_8859_1        dutch_porter            dutch
 
 # Some other stemmers in the snowball project are not included in the standard
 # distribution. To compile a libstemmer with them in, add them to this list,
@@ -51,13 +60,7 @@ porter          UTF_8,ISO_8859_1        porter			english
 # intended for general use, and use of them is is not fully supported.  These
 # algorithms are:
 #
-# german2          - This is a slight modification of the german stemmer.
-#german2          UTF_8,ISO_8859_1        german2		german
-#
-# kraaij_pohlmann  - This is a different dutch stemmer.
-#kraaij_pohlmann  UTF_8,ISO_8859_1        kraaij_pohlmann	dutch
-#
 # lovins           - This is an english stemmer, but fairly outdated, and
 #                    only really applicable to a restricted type of input text
 #                    (keywords in academic publications).
-#lovins           UTF_8,ISO_8859_1        lovins		english
+#lovins           UTF_8,ISO_8859_1        lovins                english

data/vendor/snowball/libstemmer/test.c

@@ -13,7 +13,7 @@ int main () {
     struct sb_stemmer * s;
     const char ** list = sb_stemmer_list();
     if (*list == 0) error("TEST FAIL: empty list of stemmers");
-    
+
     s = sb_stemmer_new("e");
     if (s != 0) error("TEST FAIL: non zero return for unrecognised language");
     s = sb_stemmer_new("english");