mittens 0.1.0

data/vendor/snowball/java/org/tartarus/snowball/TestApp.java

@@ -0,0 +1,75 @@
+
+package org.tartarus.snowball;
+
+import java.lang.reflect.Method;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+
+public class TestApp {
+    private static void usage()
+    {
+        System.err.println("Usage: TestApp <algorithm> [<input file>] [-o <output file>]");
+    }
+
+    public static void main(String [] args) throws Throwable {
+	if (args.length < 2) {
+            usage();
+            return;
+        }
+
+	Class stemClass = Class.forName("org.tartarus.snowball.ext." +
+					args[0] + "Stemmer");
+        SnowballStemmer stemmer = (SnowballStemmer) stemClass.newInstance();
+
+	int arg = 1;
+
+	InputStream instream;
+	if (args.length > arg && !args[arg].equals("-o")) {
+	    instream = new FileInputStream(args[arg++]);
+	} else {
+	    instream = System.in;
+	}
+
+        OutputStream outstream;
+	if (args.length > arg) {
+            if (args.length != arg + 2 || !args[arg].equals("-o")) {
+                usage();
+                return;
+            }
+	    outstream = new FileOutputStream(args[arg + 1]);
+	} else {
+	    outstream = System.out;
+	}
+
+	Reader reader = new InputStreamReader(instream, StandardCharsets.UTF_8);
+	reader = new BufferedReader(reader);
+
+	Writer output = new OutputStreamWriter(outstream, StandardCharsets.UTF_8);
+	output = new BufferedWriter(output);
+
+	StringBuffer input = new StringBuffer();
+	int character;
+	while ((character = reader.read()) != -1) {
+	    char ch = (char) character;
+	    if (Character.isWhitespace(ch)) {
+		stemmer.setCurrent(input.toString());
+		stemmer.stem();
+		output.write(stemmer.getCurrent());
+		output.write('\n');
+		input.delete(0, input.length());
+	    } else {
+		input.append(ch < 127 ? Character.toLowerCase(ch) : ch);
+	    }
+	}
+	output.flush();
+    }
+}

data/vendor/snowball/javascript/base-stemmer.js

@@ -0,0 +1,294 @@
+/**@constructor*/
+BaseStemmer = function() {
+    this.setCurrent = function(value) {
+        this.current = value;
+        this.cursor = 0;
+        this.limit = this.current.length;
+        this.limit_backward = 0;
+        this.bra = this.cursor;
+        this.ket = this.limit;
+    };
+
+    this.getCurrent = function() {
+        return this.current;
+    };
+
+    this.copy_from = function(other) {
+        this.current          = other.current;
+        this.cursor           = other.cursor;
+        this.limit            = other.limit;
+        this.limit_backward   = other.limit_backward;
+        this.bra              = other.bra;
+        this.ket              = other.ket;
+    };
+
+    this.in_grouping = function(s, min, max) {
+        if (this.cursor >= this.limit) return false;
+        var ch = this.current.charCodeAt(this.cursor);
+        if (ch > max || ch < min) return false;
+        ch -= min;
+        if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return false;
+        this.cursor++;
+        return true;
+    };
+
+    this.in_grouping_b = function(s, min, max) {
+        if (this.cursor <= this.limit_backward) return false;
+        var ch = this.current.charCodeAt(this.cursor - 1);
+        if (ch > max || ch < min) return false;
+        ch -= min;
+        if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return false;
+        this.cursor--;
+        return true;
+    };
+
+    this.out_grouping = function(s, min, max) {
+        if (this.cursor >= this.limit) return false;
+        var ch = this.current.charCodeAt(this.cursor);
+        if (ch > max || ch < min) {
+            this.cursor++;
+            return true;
+        }
+        ch -= min;
+        if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) == 0) {
+            this.cursor++;
+            return true;
+        }
+        return false;
+    };
+
+    this.out_grouping_b = function(s, min, max) {
+        if (this.cursor <= this.limit_backward) return false;
+        var ch = this.current.charCodeAt(this.cursor - 1);
+        if (ch > max || ch < min) {
+            this.cursor--;
+            return true;
+        }
+        ch -= min;
+        if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) {
+            this.cursor--;
+            return true;
+        }
+        return false;
+    };
+
+    this.eq_s = function(s)
+    {
+        if (this.limit - this.cursor < s.length) return false;
+        if (this.current.slice(this.cursor, this.cursor + s.length) != s)
+        {
+            return false;
+        }
+        this.cursor += s.length;
+        return true;
+    };
+
+    this.eq_s_b = function(s)
+    {
+        if (this.cursor - this.limit_backward < s.length) return false;
+        if (this.current.slice(this.cursor - s.length, this.cursor) != s)
+        {
+            return false;
+        }
+        this.cursor -= s.length;
+        return true;
+    };
+
+    /** @return {number} */ this.find_among = function(v)
+    {
+        var i = 0;
+        var j = v.length;
+
+        var c = this.cursor;
+        var l = this.limit;
+
+        var common_i = 0;
+        var common_j = 0;
+
+        var first_key_inspected = false;
+
+        while (true)
+        {
+            var k = i + ((j - i) >>> 1);
+            var diff = 0;
+            var common = common_i < common_j ? common_i : common_j; // smaller
+            // w[0]: string, w[1]: substring_i, w[2]: result, w[3]: function (optional)
+            var w = v[k];
+            var i2;
+            for (i2 = common; i2 < w[0].length; i2++)
+            {
+                if (c + common == l)
+                {
+                    diff = -1;
+                    break;
+                }
+                diff = this.current.charCodeAt(c + common) - w[0].charCodeAt(i2);
+                if (diff != 0) break;
+                common++;
+            }
+            if (diff < 0)
+            {
+                j = k;
+                common_j = common;
+            }
+            else
+            {
+                i = k;
+                common_i = common;
+            }
+            if (j - i <= 1)
+            {
+                if (i > 0) break; // v->s has been inspected
+                if (j == i) break; // only one item in v
+
+                // - but now we need to go round once more to get
+                // v->s inspected. This looks messy, but is actually
+                // the optimal approach.
+
+                if (first_key_inspected) break;
+                first_key_inspected = true;
+            }
+        }
+        do {
+            var w = v[i];
+            if (common_i >= w[0].length)
+            {
+                this.cursor = c + w[0].length;
+                if (w.length < 4) return w[2];
+                var res = w[3](this);
+                this.cursor = c + w[0].length;
+                if (res) return w[2];
+            }
+            i = w[1];
+        } while (i >= 0);
+        return 0;
+    };
+
+    // find_among_b is for backwards processing. Same comments apply
+    this.find_among_b = function(v)
+    {
+        var i = 0;
+        var j = v.length
+
+        var c = this.cursor;
+        var lb = this.limit_backward;
+
+        var common_i = 0;
+        var common_j = 0;
+
+        var first_key_inspected = false;
+
+        while (true)
+        {
+            var k = i + ((j - i) >> 1);
+            var diff = 0;
+            var common = common_i < common_j ? common_i : common_j;
+            var w = v[k];
+            var i2;
+            for (i2 = w[0].length - 1 - common; i2 >= 0; i2--)
+            {
+                if (c - common == lb)
+                {
+                    diff = -1;
+                    break;
+                }
+                diff = this.current.charCodeAt(c - 1 - common) - w[0].charCodeAt(i2);
+                if (diff != 0) break;
+                common++;
+            }
+            if (diff < 0)
+            {
+                j = k;
+                common_j = common;
+            }
+            else
+            {
+                i = k;
+                common_i = common;
+            }
+            if (j - i <= 1)
+            {
+                if (i > 0) break;
+                if (j == i) break;
+                if (first_key_inspected) break;
+                first_key_inspected = true;
+            }
+        }
+        do {
+            var w = v[i];
+            if (common_i >= w[0].length)
+            {
+                this.cursor = c - w[0].length;
+                if (w.length < 4) return w[2];
+                var res = w[3](this);
+                this.cursor = c - w[0].length;
+                if (res) return w[2];
+            }
+            i = w[1];
+        } while (i >= 0);
+        return 0;
+    };
+
+    /* to replace chars between c_bra and c_ket in this.current by the
+     * chars in s.
+     */
+    this.replace_s = function(c_bra, c_ket, s)
+    {
+        var adjustment = s.length - (c_ket - c_bra);
+        this.current = this.current.slice(0, c_bra) + s + this.current.slice(c_ket);
+        this.limit += adjustment;
+        if (this.cursor >= c_ket) this.cursor += adjustment;
+        else if (this.cursor > c_bra) this.cursor = c_bra;
+        return adjustment;
+    };
+
+    this.slice_check = function()
+    {
+        if (this.bra < 0 ||
+            this.bra > this.ket ||
+            this.ket > this.limit ||
+            this.limit > this.current.length)
+        {
+            return false;
+        }
+        return true;
+    };
+
+    this.slice_from = function(s)
+    {
+        var result = false;
+        if (this.slice_check())
+        {
+            this.replace_s(this.bra, this.ket, s);
+            result = true;
+        }
+        return result;
+    };
+
+    this.slice_del = function()
+    {
+        return this.slice_from("");
+    };
+
+    this.insert = function(c_bra, c_ket, s)
+    {
+        var adjustment = this.replace_s(c_bra, c_ket, s);
+        if (c_bra <= this.bra) this.bra += adjustment;
+        if (c_bra <= this.ket) this.ket += adjustment;
+    };
+
+    this.slice_to = function()
+    {
+        var result = '';
+        if (this.slice_check())
+        {
+            result = this.current.slice(this.bra, this.ket);
+        }
+        return result;
+    };
+
+    this.assign_to = function()
+    {
+        return this.current.slice(0, this.limit);
+    };
+};

data/vendor/snowball/javascript/stemwords.js

@@ -0,0 +1,106 @@
+const stemmer = require('base-stemmer.js');
+
+const fs = require('fs');
+const readline = require('readline');
+
+function usage() {
+    console.log("usage: stemwords.js [-l <language>] -i <input file> -o <output file> [-c <character encoding>] [-h]\n");
+    console.log("The input file consists of a list of words to be stemmed, one per");
+    console.log("line. Words should be in lower case.\n");
+    console.log("If -c is given, the argument is the character encoding of the input");
+    console.log("and output files.  If it is omitted, the UTF-8 encoding is used.\n");
+    console.log("The output file consists of the stemmed words, one per line.\n");
+    console.log("-h displays this help");
+}
+
+if (process.argv.length < 5)
+{
+    usage();
+}
+else
+{
+    var input = '';
+    var output = '';
+    var encoding = 'utf8';
+    var language = 'English';
+    var show_help = false;
+    while (process.argv.length > 0)
+    {
+        var arg = process.argv.shift();
+        switch (arg)
+        {
+        case "-h":
+            show_help = true;
+            process.argv.length = 0;
+            break;
+        case "-l":
+            if (process.argv.length == 0)
+            {
+                show_help = true;
+                break;
+            }
+            language = process.argv.shift();
+            break;
+        case "-i":
+            if (process.argv.length == 0)
+            {
+                show_help = true;
+                break;
+            }
+            input = process.argv.shift();
+            break;
+        case "-o":
+            if (process.argv.length == 0)
+            {
+                show_help = true;
+                break;
+            }
+            output = process.argv.shift();
+            break;
+        case "-c":
+            if (process.argv.length == 0)
+            {
+                show_help = true;
+                break;
+            }
+            encoding = process.argv.shift();
+            break;
+        }
+    }
+    if (show_help || input == '' || output == '')
+    {
+        usage();
+    }
+    else
+    {
+        stemming(language, input, output, encoding);
+    }
+}
+
+// function stemming (lang : string, input : string, output : string, encoding : string) {
+function stemming (lang, input, output, encoding) {
+    const lines = readline.createInterface({
+        input: fs.createReadStream(input, encoding),
+	terminal: false
+    });
+    var out = fs.createWriteStream(output, encoding);
+    var stemmer = create(lang);
+    lines.on('line', (original) => {
+        out.write(stemmer.stemWord(original) + '\n');
+    });
+}
+
+function create (name) {
+    var lc_name = name.toLowerCase();
+    if (!lc_name.match('\\W') && lc_name != 'base') {
+        var algo = lc_name.substr(0, 1).toUpperCase() + lc_name.substr(1);
+        try {
+            const stemmer = require(lc_name + '-stemmer.js');
+            return Function('return new ' + algo + 'Stemmer()')();
+        } catch (error) {
+        }
+    }
+    console.log('Unknown stemming language: ' + name + '\n');
+    usage();
+    process.exit(1);
+}

data/vendor/snowball/libstemmer/libstemmer_c.in

@@ -0,0 +1,96 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "@MODULES_H@"
+
+struct sb_stemmer {
+    struct SN_env * (*create)(void);
+    void (*close)(struct SN_env *);
+    int (*stem)(struct SN_env *);
+
+    struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+    return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+    const struct stemmer_encoding * encoding;
+    if (charenc == NULL) return ENC_UTF_8;
+    for (encoding = encodings; encoding->name != 0; encoding++) {
+        if (strcmp(encoding->name, charenc) == 0) break;
+    }
+    if (encoding->name == NULL) return ENC_UNKNOWN;
+    return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+    stemmer_encoding_t enc;
+    const struct stemmer_modules * module;
+    struct sb_stemmer * stemmer;
+
+    enc = sb_getenc(charenc);
+    if (enc == ENC_UNKNOWN) return NULL;
+
+    for (module = modules; module->name != 0; module++) {
+        if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+    }
+    if (module->name == NULL) return NULL;
+    
+    stemmer = (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+    if (stemmer == NULL) return NULL;
+
+    stemmer->create = module->create;
+    stemmer->close = module->close;
+    stemmer->stem = module->stem;
+
+    stemmer->env = stemmer->create();
+    if (stemmer->env == NULL)
+    {
+        sb_stemmer_delete(stemmer);
+        return NULL;
+    }
+
+    return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+    if (stemmer == 0) return;
+    if (stemmer->close) {
+        stemmer->close(stemmer->env);
+        stemmer->close = 0;
+    }
+    free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+    int ret;
+    if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+    {
+        stemmer->env->l = 0;
+        return NULL;
+    }
+    ret = stemmer->stem(stemmer->env);
+    if (ret < 0) return NULL;
+    stemmer->env->p[stemmer->env->l] = 0;
+    return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+    return stemmer->env->l;
+}

data/vendor/snowball/libstemmer/mkalgorithms.pl

@@ -0,0 +1,90 @@
+#!/usr/bin/env perl
+use strict;
+use 5.006;
+use warnings;
+
+my $progname = $0;
+
+if (scalar @ARGV != 2) {
+  print "Usage: $progname <outfile> <modules description file>\n";
+  exit 1;
+}
+
+my $outname = shift(@ARGV);
+my $descfile = shift(@ARGV);
+
+my %aliases = ();
+my %algorithms = ();
+my %algorithm_encs = ();
+
+my %encs = ();
+
+sub addalgenc($$) {
+  my $alg = shift();
+  my $enc = shift();
+
+  if (defined $algorithm_encs{$alg}) {
+      my $hashref = $algorithm_encs{$alg};
+      $$hashref{$enc}=1;
+  } else {
+      my %newhash = ($enc => 1);
+      $algorithm_encs{$alg}=\%newhash;
+  }
+
+  $encs{$enc} = 1;
+}
+
+sub readinput()
+{
+    open DESCFILE, $descfile;
+    my $line;
+    while ($line = <DESCFILE>)
+    {
+        next if $line =~ m/^\s*#/;
+        next if $line =~ m/^\s*$/;
+        my ($alg,$encstr,$aliases) = split(/\s+/, $line);
+        my $enc;
+        my $alias;
+
+        $algorithms{$alg} = 1;
+        foreach $alias (split(/,/, $aliases)) {
+            foreach $enc (split(/,/, $encstr)) {
+                $aliases{$alias} = $alg;
+                addalgenc($alg, $enc);
+            }
+        }
+    }
+}
+
+sub printoutput()
+{
+    open (OUT, ">$outname") or die "Can't open output file `$outname': $!\n";
+
+    print OUT <<EOS;
+# $outname: Lists of stemming modules.
+#
+# This file is generated by mkalgorithms.pl from a list of module names.
+# Do not edit manually.
+EOS
+
+    my $need_sep = 0;
+    my $lang;
+    my $enc;
+    my @algorithms = sort keys(%algorithms);
+    print OUT "\nlibstemmer_algorithms =";
+    foreach $lang (@algorithms) {
+	print OUT "\\\n    ", $lang;
+    }
+    print OUT "\n";
+
+    for my $enc (qw(ISO_8859_1 ISO_8859_2 KOI8_R)) {
+	print OUT "\n${enc}_algorithms =";
+	foreach $lang (@algorithms) {
+	    print OUT "\\\n    ", $lang if exists $algorithm_encs{$lang}->{$enc};
+	}
+	print OUT "\n";
+    }
+}
+
+readinput();
+printoutput();