mittens 0.1.0

data/vendor/snowball/python/snowballstemmer/basestemmer.py

@@ -0,0 +1,323 @@
+class BaseStemmer(object):
+    def __init__(self):
+        self.set_current("")
+
+    def set_current(self, value):
+        '''
+        Set the self.current string.
+        '''
+        self.current = value
+        self.cursor = 0
+        self.limit = len(self.current)
+        self.limit_backward = 0
+        self.bra = self.cursor
+        self.ket = self.limit
+
+    def get_current(self):
+        '''
+        Get the self.current string.
+        '''
+        return self.current
+
+    def copy_from(self, other):
+        self.current          = other.current
+        self.cursor           = other.cursor
+        self.limit            = other.limit
+        self.limit_backward   = other.limit_backward
+        self.bra              = other.bra
+        self.ket              = other.ket
+
+    def in_grouping(self, s, min, max):
+        if self.cursor >= self.limit:
+            return False
+        ch = ord(self.current[self.cursor])
+        if ch > max or ch < min:
+            return False
+        ch -= min
+        if (s[ch >> 3] & (0x1 << (ch & 0x7))) == 0:
+            return False
+        self.cursor += 1
+        return True
+
+    def go_in_grouping(self, s, min, max):
+        while self.cursor < self.limit:
+            ch = ord(self.current[self.cursor])
+            if ch > max or ch < min:
+                return True
+            ch -= min
+            if (s[ch >> 3] & (0x1 << (ch & 0x7))) == 0:
+                return True
+            self.cursor += 1
+        return False
+
+    def in_grouping_b(self, s, min, max):
+        if self.cursor <= self.limit_backward:
+            return False
+        ch = ord(self.current[self.cursor - 1])
+        if ch > max or ch < min:
+            return False
+        ch -= min
+        if (s[ch >> 3] & (0x1 << (ch & 0x7))) == 0:
+            return False
+        self.cursor -= 1
+        return True
+
+    def go_in_grouping_b(self, s, min, max):
+        while self.cursor > self.limit_backward:
+            ch = ord(self.current[self.cursor - 1])
+            if ch > max or ch < min:
+                return True
+            ch -= min
+            if (s[ch >> 3] & (0x1 << (ch & 0x7))) == 0:
+                return True
+            self.cursor -= 1
+        return False
+
+    def out_grouping(self, s, min, max):
+        if self.cursor >= self.limit:
+            return False
+        ch = ord(self.current[self.cursor])
+        if ch > max or ch < min:
+            self.cursor += 1
+            return True
+        ch -= min
+        if (s[ch >> 3] & (0X1 << (ch & 0x7))) == 0:
+            self.cursor += 1
+            return True
+        return False
+
+    def go_out_grouping(self, s, min, max):
+        while self.cursor < self.limit:
+            ch = ord(self.current[self.cursor])
+            if ch <= max and ch >= min:
+                ch -= min
+                if (s[ch >> 3] & (0X1 << (ch & 0x7))):
+                    return True
+            self.cursor += 1
+        return False
+
+    def out_grouping_b(self, s, min, max):
+        if self.cursor <= self.limit_backward:
+            return False
+        ch = ord(self.current[self.cursor - 1])
+        if ch > max or ch < min:
+            self.cursor -= 1
+            return True
+        ch -= min
+        if (s[ch >> 3] & (0X1 << (ch & 0x7))) == 0:
+            self.cursor -= 1
+            return True
+        return False
+
+    def go_out_grouping_b(self, s, min, max):
+        while self.cursor > self.limit_backward:
+            ch = ord(self.current[self.cursor - 1])
+            if ch <= max and ch >= min:
+                ch -= min
+                if (s[ch >> 3] & (0X1 << (ch & 0x7))):
+                    return True
+            self.cursor -= 1
+        return False
+
+    def eq_s(self, s):
+        if self.limit - self.cursor < len(s):
+            return False
+        if self.current[self.cursor:self.cursor + len(s)] != s:
+            return False
+        self.cursor += len(s)
+        return True
+
+    def eq_s_b(self, s):
+        if self.cursor - self.limit_backward < len(s):
+            return False
+        if self.current[self.cursor - len(s):self.cursor] != s:
+            return False
+        self.cursor -= len(s)
+        return True
+
+    def find_among(self, v):
+        i = 0
+        j = len(v)
+
+        c = self.cursor
+        l = self.limit
+
+        common_i = 0
+        common_j = 0
+
+        first_key_inspected = False
+
+        while True:
+            k = i + ((j - i) >> 1)
+            diff = 0
+            common = min(common_i, common_j) # smaller
+            w = v[k]
+            for i2 in range(common, len(w.s)):
+                if c + common == l:
+                    diff = -1
+                    break
+                diff = ord(self.current[c + common]) - ord(w.s[i2])
+                if diff != 0:
+                    break
+                common += 1
+            if diff < 0:
+                j = k
+                common_j = common
+            else:
+                i = k
+                common_i = common
+            if j - i <= 1:
+                if i > 0:
+                    break # v->s has been inspected
+                if j == i:
+                    break # only one item in v
+                # - but now we need to go round once more to get
+                # v->s inspected. This looks messy, but is actually
+                # the optimal approach.
+                if first_key_inspected:
+                    break
+                first_key_inspected = True
+        while True:
+            w = v[i]
+            if common_i >= len(w.s):
+                self.cursor = c + len(w.s)
+                if w.method is None:
+                    return w.result
+                method = getattr(self, w.method)
+                res = method()
+                self.cursor = c + len(w.s)
+                if res:
+                    return w.result
+            i = w.substring_i
+            if i < 0:
+                return 0
+        return -1 # not reachable
+
+    def find_among_b(self, v):
+        '''
+        find_among_b is for backwards processing. Same comments apply
+        '''
+        i = 0
+        j = len(v)
+
+        c = self.cursor
+        lb = self.limit_backward
+
+        common_i = 0
+        common_j = 0
+
+        first_key_inspected = False
+
+        while True:
+            k = i + ((j - i) >> 1)
+            diff = 0
+            common = min(common_i, common_j)
+            w = v[k]
+            for i2 in range(len(w.s) - 1 - common, -1, -1):
+                if c - common == lb:
+                    diff = -1
+                    break
+                diff = ord(self.current[c - 1 - common]) - ord(w.s[i2])
+                if diff != 0:
+                    break
+                common += 1
+            if diff < 0:
+                j = k
+                common_j = common
+            else:
+                i = k
+                common_i = common
+            if j - i <= 1:
+                if i > 0:
+                    break
+                if j == i:
+                    break
+                if first_key_inspected:
+                    break
+                first_key_inspected = True
+        while True:
+            w = v[i]
+            if common_i >= len(w.s):
+                self.cursor = c - len(w.s)
+                if w.method is None:
+                    return w.result
+                method = getattr(self, w.method)
+                res = method()
+                self.cursor = c - len(w.s)
+                if res:
+                    return w.result
+            i = w.substring_i
+            if i < 0:
+                return 0
+        return -1 # not reachable
+
+    def replace_s(self, c_bra, c_ket, s):
+        '''
+        to replace chars between c_bra and c_ket in self.current by the
+        chars in s.
+
+        @type c_bra int
+        @type c_ket int
+        @type s: string
+        '''
+        adjustment = len(s) - (c_ket - c_bra)
+        self.current = self.current[0:c_bra] + s + self.current[c_ket:]
+        self.limit += adjustment
+        if self.cursor >= c_ket:
+            self.cursor += adjustment
+        elif self.cursor > c_bra:
+            self.cursor = c_bra
+        return adjustment
+
+    def slice_check(self):
+        if self.bra < 0 or self.bra > self.ket or self.ket > self.limit or self.limit > len(self.current):
+            return False
+        return True
+
+    def slice_from(self, s):
+        '''
+        @type s string
+        '''
+        result = False
+        if self.slice_check():
+            self.replace_s(self.bra, self.ket, s)
+            result = True
+        return result
+
+    def slice_del(self):
+        return self.slice_from("")
+
+    def insert(self, c_bra, c_ket, s):
+        '''
+        @type c_bra int
+        @type c_ket int
+        @type s: string
+        '''
+        adjustment = self.replace_s(c_bra, c_ket, s)
+        if c_bra <= self.bra:
+            self.bra += adjustment
+        if c_bra <= self.ket:
+            self.ket += adjustment
+
+    def slice_to(self):
+        '''
+        Return the slice as a string.
+        '''
+        result = ''
+        if self.slice_check():
+            result = self.current[self.bra:self.ket]
+        return result
+
+    def assign_to(self):
+        '''
+        Return the current string up to the limit.
+        '''
+        return self.current[0:self.limit]
+
+    def stemWord(self, word):
+        self.set_current(word)
+        self._stem()
+        return self.get_current()
+
+    def stemWords(self, words):
+        return [self.stemWord(word) for word in words]

data/vendor/snowball/python/stemwords.py

@@ -0,0 +1,101 @@
+import sys
+import codecs
+import snowballstemmer
+
+def usage():
+    print('''usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]
+
+The input file consists of a list of words to be stemmed, one per
+line. Words should be in lower case, but (for English) A-Z letters
+are mapped to their a-z equivalents anyway. If omitted, stdin is
+used.
+
+If -c is given, the argument is the character encoding of the input
+and output files.  If it is omitted, the UTF-8 encoding is used.
+
+If -p is given the output file consists of each word of the input
+file followed by \"->\" followed by its stemmed equivalent.
+If -p2 is given the output file is a two column layout containing
+the input words in the first column and the stemmed eqivalents in
+the second column.
+
+Otherwise, the output file consists of the stemmed words, one per
+line.
+
+-h displays this help''' % sys.argv[0])
+
+def main():
+    argv = sys.argv[1:]
+    if len(argv) < 5:
+        usage()
+    else:
+        pretty = 0
+        input = ''
+        output = ''
+        encoding = 'utf_8'
+        language = 'English'
+        show_help = False
+        while len(argv):
+            arg = argv[0]
+            argv = argv[1:]
+            if arg == '-h':
+                show_help = True
+                break
+            elif arg == "-p":
+                pretty = 1
+            elif arg == "-p2":
+                pretty = 2
+            elif arg == "-l":
+                if len(argv) == 0:
+                    show_help = True
+                    break
+                language = argv[0]
+                argv = argv[1:]
+            elif arg == "-i":
+                if len(argv) == 0:
+                    show_help = True
+                    break
+                input = argv[0]
+                argv = argv[1:]
+            elif arg == "-o":
+                if len(argv) == 0:
+                    show_help = True
+                    break
+                output = argv[0]
+                argv = argv[1:]
+            elif arg == "-c":
+                if len(argv) == 0:
+                    show_help = True
+                    break
+                encoding = argv[0]
+        if show_help or input == '' or output == '':
+            usage()
+        else:
+            stemming(language, input, output, encoding, pretty)
+
+
+def stemming(lang, input, output, encoding, pretty):
+    stemmer = snowballstemmer.stemmer(lang)
+    with codecs.open(output, "w", encoding) as outfile:
+        with codecs.open(input, "r", encoding) as infile:
+            for original in infile.readlines():
+                original = original.strip()
+                # Convert only ASCII-letters to lowercase, to match C behavior
+                original = ''.join((c.lower() if 'A' <= c <= 'Z' else c for c in original))
+                stemmed = stemmer.stemWord(original)
+                if pretty == 0:
+                    if stemmed != "":
+                        outfile.write(stemmed)
+                elif pretty == 1:
+                    outfile.write(original, " -> ", stemmed)
+                elif pretty == 2:
+                    outfile.write(original)
+                    if len(original) < 30:
+                        outfile.write(" " * (30 - len(original)))
+                    else:
+                        outfile.write("\n")
+                        outfile.write(" " * 30)
+                    outfile.write(stemmed)
+                outfile.write('\n')
+
+main()

data/vendor/snowball/python/testapp.py

@@ -0,0 +1,28 @@
+import sys
+import re
+import snowballstemmer
+
+
+def usage():
+    print("testapp.py <algorithm> \"sentence\"...")
+
+def main():
+    argv = sys.argv
+    if len(argv) < 1:
+        usage()
+        return
+    algorithm = 'english'
+    if len(argv) > 2:
+        algorithm = argv[1]
+        argv = argv[2:]
+    else:
+        argv = argv[1:]
+    stemmer = snowballstemmer.stemmer(algorithm)
+    splitter = re.compile(r"[\s\.-]")
+    for arg in argv:
+        for word in splitter.split(arg):
+            if word == '':
+                continue
+            original = word.lower()
+            print(original + " -> " + stemmer.stemWord(original))
+main()

data/vendor/snowball/runtime/api.c

@@ -0,0 +1,58 @@
+
+#include <stdlib.h> /* for calloc, free */
+#include "header.h"
+
+extern struct SN_env * SN_create_env(int S_size, int I_size)
+{
+    struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
+    if (z == NULL) return NULL;
+    z->p = create_s();
+    if (z->p == NULL) goto error;
+    if (S_size)
+    {
+        int i;
+        z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
+        if (z->S == NULL) goto error;
+
+        for (i = 0; i < S_size; i++)
+        {
+            z->S[i] = create_s();
+            if (z->S[i] == NULL) goto error;
+        }
+    }
+
+    if (I_size)
+    {
+        z->I = (int *) calloc(I_size, sizeof(int));
+        if (z->I == NULL) goto error;
+    }
+
+    return z;
+error:
+    SN_close_env(z, S_size);
+    return NULL;
+}
+
+extern void SN_close_env(struct SN_env * z, int S_size)
+{
+    if (z == NULL) return;
+    if (S_size)
+    {
+        int i;
+        for (i = 0; i < S_size; i++)
+        {
+            lose_s(z->S[i]);
+        }
+        free(z->S);
+    }
+    free(z->I);
+    if (z->p) lose_s(z->p);
+    free(z);
+}
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
+{
+    int err = replace_s(z, 0, z->l, size, s, NULL);
+    z->c = 0;
+    return err;
+}

data/vendor/snowball/runtime/api.h

@@ -0,0 +1,32 @@
+
+typedef unsigned char symbol;
+
+/* Or replace 'char' above with 'short' for 16 bit characters.
+
+   More precisely, replace 'char' with whatever type guarantees the
+   character width you need. Note however that sizeof(symbol) should divide
+   HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
+   there is an alignment problem. In the unlikely event of a problem here,
+   consult Martin Porter.
+
+*/
+
+struct SN_env {
+    symbol * p;
+    int c; int l; int lb; int bra; int ket;
+    symbol * * S;
+    int * I;
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * SN_create_env(int S_size, int I_size);
+extern void SN_close_env(struct SN_env * z, int S_size);
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
+
+#ifdef __cplusplus
+}
+#endif

data/vendor/snowball/runtime/header.h

@@ -0,0 +1,61 @@
+
+#include <limits.h>
+
+#include "api.h"
+
+#define MAXINT INT_MAX
+#define MININT INT_MIN
+
+#define HEAD 2*sizeof(int)
+
+#define SIZE(p)        ((int *)(p))[-1]
+#define SET_SIZE(p, n) ((int *)(p))[-1] = n
+#define CAPACITY(p)    ((int *)(p))[-2]
+
+struct among
+{   int s_size;     /* number of chars in string */
+    const symbol * s;       /* search string */
+    int substring_i;/* index to longest matching substring */
+    int result;     /* result of the lookup */
+    int (* function)(struct SN_env *);
+};
+
+extern symbol * create_s(void);
+extern void lose_s(symbol * p);
+
+extern int skip_utf8(const symbol * p, int c, int limit, int n);
+
+extern int skip_b_utf8(const symbol * p, int c, int limit, int n);
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_v(struct SN_env * z, const symbol * p);
+extern int eq_v_b(struct SN_env * z, const symbol * p);
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size);
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
+
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
+extern int slice_from_v(struct SN_env * z, const symbol * p);
+extern int slice_del(struct SN_env * z);
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
+
+extern symbol * slice_to(struct SN_env * z, symbol * p);
+extern symbol * assign_to(struct SN_env * z, symbol * p);
+
+extern int len_utf8(const symbol * p);
+
+extern void debug(struct SN_env * z, int number, int line_count);