mittens 0.1.0

data/vendor/snowball/go/stemwords/generate.go

@@ -0,0 +1,68 @@
+// +build ignore
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+)
+
+// tool to register all algorithms built with the stemwords tool
+
+func main() {
+	flag.Parse()
+
+	if flag.NArg() < 1 {
+		log.Fatal("must specify algorithms directory")
+	}
+
+	var w io.Writer
+	if flag.NArg() > 1 {
+		var err error
+		w, err = os.Create(flag.Arg(1))
+		if err != nil {
+			log.Fatalf("error creating output file %v", err)
+		}
+	} else {
+		w = os.Stdout
+	}
+
+	fmt.Fprintf(w, "%s", header)
+
+	files, err := ioutil.ReadDir(flag.Arg(0))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, file := range files {
+		fmt.Fprintf(w, "  %s \"github.com/snowballstem/snowball/go/algorithms/%s\"\n",
+			file.Name(), file.Name())
+	}
+
+	fmt.Fprintf(w, closeImportStartInit)
+
+	for _, file := range files {
+		fmt.Fprintf(w, "  languages[\"%s\"] = %s.Stem\n",
+			file.Name(), file.Name())
+	}
+
+	fmt.Fprintf(w, "%s", footer)
+}
+
+var header = `// generated list of supported algorithms, DO NOT EDIT
+
+package main
+
+import (
+`
+
+var closeImportStartInit = `)
+
+func init() {`
+
+var footer = `}
+`

data/vendor/snowball/go/stemwords/main.go

@@ -0,0 +1,68 @@
+//go:generate go run generate.go ../algorithms algorithms.go
+//go:generate gofmt -s -w algorithms.go
+
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	snowballRuntime "github.com/snowballstem/snowball/go"
+)
+
+var language = flag.String("l", "", "language")
+var input = flag.String("i", "", "input file")
+var output = flag.String("o", "", "output file")
+
+func main() {
+	flag.Parse()
+
+	if *language == "" {
+		log.Fatal("must specify language")
+	}
+
+	stemmer, ok := languages[*language]
+	if !ok {
+		log.Fatalf("no language support for %s", *language)
+	}
+
+	var reader = os.Stdin
+	if *input != "" {
+		var err error
+		reader, err = os.Open(*input)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer reader.Close()
+	}
+
+	var writer = os.Stdout
+	if *output != "" {
+		var err error
+		writer, err = os.Create(*output)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer writer.Close()
+	}
+
+	var err error
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		word := scanner.Text()
+		env := snowballRuntime.NewEnv(word)
+		stemmer(env)
+		fmt.Fprintf(writer, "%s\n", env.Current())
+	}
+
+	if err = scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+type StemFunc func(env *snowballRuntime.Env) bool
+
+var languages = make(map[string]StemFunc)

data/vendor/snowball/go/util.go

@@ -0,0 +1,34 @@
+package snowball
+
+import (
+	"math"
+	"unicode/utf8"
+)
+
+const MaxInt = math.MaxInt32
+const MinInt = math.MinInt32
+
+func splitAt(str string, mid int) (string, string) {
+	return str[:mid], str[mid:]
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+func onCharBoundary(s string, pos int) bool {
+	if pos <= 0 || pos >= len(s) {
+		return true
+	}
+	return utf8.RuneStart(s[pos])
+}
+
+// RuneCountInString is a wrapper around utf8.RuneCountInString
+// this allows us to not have to conditionally include
+// the utf8 package into some stemmers and not others
+func RuneCountInString(str string) int {
+	return utf8.RuneCountInString(str)
+}

data/vendor/snowball/iconv.py

@@ -0,0 +1,50 @@
+#!env python
+# Simple (but slow) iconv replacement in Python.
+import sys
+
+in_cs = out_cs = in_file = out_file = pending = None
+for arg in sys.argv[1:]:
+    if pending != None:
+        arg = pending + arg
+        pending = None
+    if arg.startswith('-'):
+        if arg[1] in ('f', 't', 'o'):
+            if len(arg) == 2:
+                pending = arg
+                continue
+        if arg[1] == 'f':
+            in_cs = arg[2:]
+            continue
+        if arg[1] == 't':
+            out_cs = arg[2:]
+            continue
+        if arg[1] == 'o':
+            out_file = open(arg[2:], 'wb')
+            continue
+        print("Unknown option: '%s'" % arg)
+        sys.exit(1)
+    if in_file == None:
+        in_file = open(arg, 'rb')
+        continue
+    print("Too many arguments")
+    sys.exit(1)
+
+if in_cs == None:
+    print("Need to specify input cs with -f")
+    sys.exit(1)
+if out_cs == None:
+    print("Need to specify output cs with -t")
+    sys.exit(1)
+
+if in_file == None:
+    if hasattr(sys.stdin, 'buffer'):
+        in_file = sys.stdin.buffer
+    else:
+        in_file = sys.stdin
+if out_file == None:
+    if hasattr(sys.stdout, 'buffer'):
+        out_file = sys.stdout.buffer
+    else:
+        out_file = sys.stdout
+
+out_file.write(in_file.read().decode(in_cs).encode(out_cs))

data/vendor/snowball/include/libstemmer.h

@@ -0,0 +1,78 @@
+
+/* Make header file work when included from C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sb_stemmer;
+typedef unsigned char sb_symbol;
+
+/* FIXME - should be able to get a version number for each stemming
+ * algorithm (which will be incremented each time the output changes). */
+
+/** Returns an array of the names of the available stemming algorithms.
+ *  Note that these are the canonical names - aliases (ie, other names for
+ *  the same algorithm) will not be included in the list.
+ *  The list is terminated with a null pointer.
+ *
+ *  The list must not be modified in any way.
+ */
+const char ** sb_stemmer_list(void);
+
+/** Create a new stemmer object, using the specified algorithm, for the
+ *  specified character encoding.
+ *
+ *  All algorithms will usually be available in UTF-8, but may also be
+ *  available in other character encodings.
+ *
+ *  @param algorithm The algorithm name.  This is either the english
+ *  name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
+ *  language.  Note that case is significant in this parameter - the
+ *  value should be supplied in lower case.
+ *
+ *  @param charenc The character encoding.  NULL may be passed as
+ *  this value, in which case UTF-8 encoding will be assumed. Otherwise,
+ *  the argument may be one of "UTF_8", "ISO_8859_1" (i.e. Latin 1),
+ *  "ISO_8859_2" (i.e. Latin 2) or "KOI8_R" (Russian).  Note that case is
+ *  significant in this parameter.
+ *
+ *  @return NULL if the specified algorithm is not recognised, or the
+ *  algorithm is not available for the requested encoding.  Otherwise,
+ *  returns a pointer to a newly created stemmer for the requested algorithm.
+ *  The returned pointer must be deleted by calling sb_stemmer_delete().
+ *
+ *  @note NULL will also be returned if an out of memory error occurs.
+ */
+struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
+
+/** Delete a stemmer object.
+ *
+ *  This frees all resources allocated for the stemmer.  After calling
+ *  this function, the supplied stemmer may no longer be used in any way.
+ *
+ *  It is safe to pass a null pointer to this function - this will have
+ *  no effect.
+ */
+void                sb_stemmer_delete(struct sb_stemmer * stemmer);
+
+/** Stem a word.
+ *
+ *  The return value is owned by the stemmer - it must not be freed or
+ *  modified, and it will become invalid when the stemmer is called again,
+ *  or if the stemmer is freed.
+ *
+ *  The length of the return value can be obtained using sb_stemmer_length().
+ *
+ *  If an out-of-memory error occurs, this will return NULL.
+ */
+const sb_symbol *   sb_stemmer_stem(struct sb_stemmer * stemmer,
+                                    const sb_symbol * word, int size);
+
+/** Get the length of the result of the last stemmed word.
+ *  This should not be called before sb_stemmer_stem() has been called.
+ */
+int                 sb_stemmer_length(struct sb_stemmer * stemmer);
+
+#ifdef __cplusplus
+}
+#endif

data/vendor/snowball/java/org/tartarus/snowball/Among.java

@@ -0,0 +1,29 @@
+package org.tartarus.snowball;
+
+import java.lang.reflect.Method;
+
+public class Among {
+    public Among (String s, int substring_i, int result) {
+        this.s = s.toCharArray();
+        this.substring_i = substring_i;
+	this.result = result;
+	this.method = null;
+    }
+
+    public Among (String s, int substring_i, int result, String methodname,
+		  Class<? extends SnowballProgram> programclass) {
+        this.s = s.toCharArray();
+        this.substring_i = substring_i;
+	this.result = result;
+	try {
+	    this.method = programclass.getDeclaredMethod(methodname);
+	} catch (NoSuchMethodException e) {
+	    throw new RuntimeException(e);
+	}
+    }
+
+    public final char[] s; /* search string */
+    public final int substring_i; /* index to longest matching substring */
+    public final int result; /* result of the lookup */
+    public final Method method; /* method to use if substring matches */
+};

data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java

@@ -0,0 +1,381 @@
+
+package org.tartarus.snowball;
+import java.lang.reflect.InvocationTargetException;
+import java.io.Serializable;
+
+public class SnowballProgram implements Serializable {
+    protected SnowballProgram()
+    {
+	current = new StringBuilder();
+	init();
+    }
+
+    static final long serialVersionUID = 2016072500L;
+
+    private void init() {
+	cursor = 0;
+	limit = current.length();
+	limit_backward = 0;
+	bra = cursor;
+	ket = limit;
+    }
+
+    /**
+     * Set the current string.
+     */
+    public void setCurrent(String value)
+    {
+        // Make a new StringBuilder.  If we reuse the old one, and a user of
+        // the library keeps a reference to the buffer returned (for example,
+        // by converting it to a String in a way which doesn't force a copy),
+        // the buffer size will not decrease, and we will risk wasting a large
+        // amount of memory.
+        // Thanks to Wolfram Esser for spotting this problem.
+        current = new StringBuilder(value);
+	init();
+    }
+
+    /**
+     * Get the current string.
+     */
+    public String getCurrent()
+    {
+        return current.toString();
+    }
+
+    // current string
+    protected StringBuilder current;
+
+    protected int cursor;
+    protected int limit;
+    protected int limit_backward;
+    protected int bra;
+    protected int ket;
+
+    public SnowballProgram(SnowballProgram other) {
+	current          = other.current;
+	cursor           = other.cursor;
+	limit            = other.limit;
+	limit_backward   = other.limit_backward;
+	bra              = other.bra;
+	ket              = other.ket;
+    }
+
+    protected void copy_from(SnowballProgram other)
+    {
+	current          = other.current;
+	cursor           = other.cursor;
+	limit            = other.limit;
+	limit_backward   = other.limit_backward;
+	bra              = other.bra;
+	ket              = other.ket;
+    }
+
+    protected boolean in_grouping(char [] s, int min, int max)
+    {
+	if (cursor >= limit) return false;
+	char ch = current.charAt(cursor);
+	if (ch > max || ch < min) return false;
+	ch -= min;
+	if ((s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return false;
+	cursor++;
+	return true;
+    }
+
+    protected boolean in_grouping_b(char [] s, int min, int max)
+    {
+	if (cursor <= limit_backward) return false;
+	char ch = current.charAt(cursor - 1);
+	if (ch > max || ch < min) return false;
+	ch -= min;
+	if ((s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) return false;
+	cursor--;
+	return true;
+    }
+
+    protected boolean out_grouping(char [] s, int min, int max)
+    {
+	if (cursor >= limit) return false;
+	char ch = current.charAt(cursor);
+	if (ch > max || ch < min) {
+	    cursor++;
+	    return true;
+	}
+	ch -= min;
+	if ((s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) {
+	    cursor++;
+	    return true;
+	}
+	return false;
+    }
+
+    protected boolean out_grouping_b(char [] s, int min, int max)
+    {
+	if (cursor <= limit_backward) return false;
+	char ch = current.charAt(cursor - 1);
+	if (ch > max || ch < min) {
+	    cursor--;
+	    return true;
+	}
+	ch -= min;
+	if ((s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) {
+	    cursor--;
+	    return true;
+	}
+	return false;
+    }
+
+    protected boolean eq_s(CharSequence s)
+    {
+	if (limit - cursor < s.length()) return false;
+	int i;
+	for (i = 0; i != s.length(); i++) {
+	    if (current.charAt(cursor + i) != s.charAt(i)) return false;
+	}
+	cursor += s.length();
+	return true;
+    }
+
+    protected boolean eq_s_b(CharSequence s)
+    {
+	if (cursor - limit_backward < s.length()) return false;
+	int i;
+	for (i = 0; i != s.length(); i++) {
+	    if (current.charAt(cursor - s.length() + i) != s.charAt(i)) return false;
+	}
+	cursor -= s.length();
+	return true;
+    }
+
+    protected int find_among(Among v[])
+    {
+	int i = 0;
+	int j = v.length;
+
+	int c = cursor;
+	int l = limit;
+
+	int common_i = 0;
+	int common_j = 0;
+
+	boolean first_key_inspected = false;
+
+	while (true) {
+	    int k = i + ((j - i) >> 1);
+	    int diff = 0;
+	    int common = common_i < common_j ? common_i : common_j; // smaller
+	    Among w = v[k];
+	    int i2;
+	    for (i2 = common; i2 < w.s.length; i2++) {
+		if (c + common == l) {
+		    diff = -1;
+		    break;
+		}
+		diff = current.charAt(c + common) - w.s[i2];
+		if (diff != 0) break;
+		common++;
+	    }
+	    if (diff < 0) {
+		j = k;
+		common_j = common;
+	    } else {
+		i = k;
+		common_i = common;
+	    }
+	    if (j - i <= 1) {
+		if (i > 0) break; // v->s has been inspected
+		if (j == i) break; // only one item in v
+
+		// - but now we need to go round once more to get
+		// v->s inspected. This looks messy, but is actually
+		// the optimal approach.
+
+		if (first_key_inspected) break;
+		first_key_inspected = true;
+	    }
+	}
+	while (true) {
+	    Among w = v[i];
+	    if (common_i >= w.s.length) {
+		cursor = c + w.s.length;
+		if (w.method == null) return w.result;
+		boolean res;
+		try {
+		    Object resobj = w.method.invoke(this);
+		    res = resobj.toString().equals("true");
+		} catch (InvocationTargetException e) {
+		    res = false;
+		    // FIXME - debug message
+		} catch (IllegalAccessException e) {
+		    res = false;
+		    // FIXME - debug message
+		}
+		cursor = c + w.s.length;
+		if (res) return w.result;
+	    }
+	    i = w.substring_i;
+	    if (i < 0) return 0;
+	}
+    }
+
+    // find_among_b is for backwards processing. Same comments apply
+    protected int find_among_b(Among v[])
+    {
+	int i = 0;
+	int j = v.length;
+
+	int c = cursor;
+	int lb = limit_backward;
+
+	int common_i = 0;
+	int common_j = 0;
+
+	boolean first_key_inspected = false;
+
+	while (true) {
+	    int k = i + ((j - i) >> 1);
+	    int diff = 0;
+	    int common = common_i < common_j ? common_i : common_j;
+	    Among w = v[k];
+	    int i2;
+	    for (i2 = w.s.length - 1 - common; i2 >= 0; i2--) {
+		if (c - common == lb) {
+		    diff = -1;
+		    break;
+		}
+		diff = current.charAt(c - 1 - common) - w.s[i2];
+		if (diff != 0) break;
+		common++;
+	    }
+	    if (diff < 0) {
+		j = k;
+		common_j = common;
+	    } else {
+		i = k;
+		common_i = common;
+	    }
+	    if (j - i <= 1) {
+		if (i > 0) break;
+		if (j == i) break;
+		if (first_key_inspected) break;
+		first_key_inspected = true;
+	    }
+	}
+	while (true) {
+	    Among w = v[i];
+	    if (common_i >= w.s.length) {
+		cursor = c - w.s.length;
+		if (w.method == null) return w.result;
+
+		boolean res;
+		try {
+		    Object resobj = w.method.invoke(this);
+		    res = resobj.toString().equals("true");
+		} catch (InvocationTargetException e) {
+		    res = false;
+		    // FIXME - debug message
+		} catch (IllegalAccessException e) {
+		    res = false;
+		    // FIXME - debug message
+		}
+		cursor = c - w.s.length;
+		if (res) return w.result;
+	    }
+	    i = w.substring_i;
+	    if (i < 0) return 0;
+	}
+    }
+
+    /* to replace chars between c_bra and c_ket in current by the
+     * chars in s.
+     */
+    protected int replace_s(int c_bra, int c_ket, String s)
+    {
+	int adjustment = s.length() - (c_ket - c_bra);
+	current.replace(c_bra, c_ket, s);
+	limit += adjustment;
+	if (cursor >= c_ket) cursor += adjustment;
+	else if (cursor > c_bra) cursor = c_bra;
+	return adjustment;
+    }
+
+    protected void slice_check()
+    {
+	if (bra < 0 ||
+	    bra > ket ||
+	    ket > limit ||
+	    limit > current.length())   // this line could be removed
+	{
+	    System.err.println("faulty slice operation");
+	// FIXME: report error somehow.
+	/*
+	    fprintf(stderr, "faulty slice operation:\n");
+	    debug(z, -1, 0);
+	    exit(1);
+	    */
+	}
+    }
+
+    protected void slice_from(String s)
+    {
+	slice_check();
+	replace_s(bra, ket, s);
+    }
+
+    protected void slice_from(CharSequence s)
+    {
+        slice_from(s.toString());
+    }
+
+    protected void slice_del()
+    {
+	slice_from("");
+    }
+
+    protected void insert(int c_bra, int c_ket, String s)
+    {
+	int adjustment = replace_s(c_bra, c_ket, s);
+	if (c_bra <= bra) bra += adjustment;
+	if (c_bra <= ket) ket += adjustment;
+    }
+
+    protected void insert(int c_bra, int c_ket, CharSequence s)
+    {
+	insert(c_bra, c_ket, s.toString());
+    }
+
+    /* Copy the slice into the supplied StringBuilder */
+    protected void slice_to(StringBuilder s)
+    {
+	slice_check();
+	s.replace(0, s.length(), current.substring(bra, ket));
+    }
+
+    protected void assign_to(StringBuilder s)
+    {
+	s.replace(0, s.length(), current.substring(0, limit));
+    }
+
+/*
+extern void debug(struct SN_env * z, int number, int line_count)
+{   int i;
+    int limit = SIZE(z->p);
+    //if (number >= 0) printf("%3d (line %4d): '", number, line_count);
+    if (number >= 0) printf("%3d (line %4d): [%d]'", number, line_count,limit);
+    for (i = 0; i <= limit; i++)
+    {   if (z->lb == i) printf("{");
+        if (z->bra == i) printf("[");
+        if (z->c == i) printf("|");
+        if (z->ket == i) printf("]");
+        if (z->l == i) printf("}");
+        if (i < limit)
+        {   int ch = z->p[i];
+            if (ch == 0) ch = '#';
+            printf("%c", ch);
+        }
+    }
+    printf("'\n");
+}
+*/
+
+};

data/vendor/snowball/java/org/tartarus/snowball/SnowballStemmer.java

@@ -0,0 +1,8 @@
+
+package org.tartarus.snowball;
+
+public abstract class SnowballStemmer extends SnowballProgram {
+    public abstract boolean stem();
+
+    static final long serialVersionUID = 2016072500L;
+};