mittens 0.3.4 → 0.3.5

data/vendor/snowball/libstemmer/libstemmer_c.in

@@ -22,13 +22,13 @@ sb_stemmer_list(void)
 static stemmer_encoding_t
 sb_getenc(const char * charenc)
 {
-    const struct stemmer_encoding * encoding;
+    size_t i;
     if (charenc == NULL) return ENC_UTF_8;
-    for (encoding = encodings; encoding->name != 0; encoding++) {
-        if (strcmp(encoding->name, charenc) == 0) break;
+    for (i = 0; i < sizeof(encodings) / sizeof(encodings[0]); ++i) {
+        if (strcmp(encodings[i].name, charenc) == 0)
+            return encodings[i].enc;
     }
-    if (encoding->name == NULL) return ENC_UNKNOWN;
-    return encoding->enc;
+    return ENC_UNKNOWN;
 }
 
 extern struct sb_stemmer *

data/vendor/snowball/libstemmer/mkmodules.pl

@@ -121,18 +121,21 @@ EOS
 typedef enum {
   ENC_UNKNOWN=0,
 EOS
+    my $enc_array_len = 0;
     my $neednl = 0;
     for $enc (sort keys %encs) {
+        length($enc) > $enc_array_len and $enc_array_len = length($enc);
         print OUT ",\n" if $neednl;
         print OUT "  ENC_${enc}";
         $neednl = 1;
     }
+    ++$enc_array_len; # Allow for terminating zero byte.
     print OUT <<EOS;
 
 } stemmer_encoding_t;
 
 struct stemmer_encoding {
-  const char * name;
+  char name[$enc_array_len];
   stemmer_encoding_t enc;
 };
 static const struct stemmer_encoding encodings[] = {
@@ -141,7 +144,6 @@ EOS
         print OUT "  {\"${enc}\", ENC_${enc}},\n";
     }
     print OUT <<EOS;
-  {0,ENC_UNKNOWN}
 };
 
 struct stemmer_modules {
@@ -231,7 +233,7 @@ EOS
 
     $need_sep = 0;
     for $srcfile ('runtime/api.c',
-                  'runtime/utilities.c',
+                  'runtime/snowball_runtime.c',
                   "libstemmer/libstemmer${extn}.c") {
         print OUT " \\\n" if $need_sep;
         print OUT "  $srcfile";

data/vendor/snowball/libstemmer/modules.txt

@@ -16,6 +16,7 @@ catalan         UTF_8,ISO_8859_1        catalan,ca,cat
 czech           UTF_8,ISO_8859_2        czech,cs,ces,cze
 danish          UTF_8,ISO_8859_1        danish,da,dan
 dutch           UTF_8,ISO_8859_1        dutch,nl,dut,nld,kraaij_pohlmann
+earlymodernenglish UTF_8                earlymodernenglish
 english         UTF_8,ISO_8859_1        english,en,eng
 esperanto       UTF_8                   esperanto,eo,epo
 estonian        UTF_8                   estonian,et,est
@@ -31,18 +32,19 @@ italian         UTF_8,ISO_8859_1        italian,it,ita
 lithuanian      UTF_8                   lithuanian,lt,lit
 nepali          UTF_8                   nepali,ne,nep
 norwegian       UTF_8,ISO_8859_1        norwegian,no,nor
+persian         UTF_8                   persian,fa,fas,per
 polish          UTF_8,ISO_8859_2        polish,pl,pol
-persian         UTF_8                   persian,fa,fas,pers
 portuguese      UTF_8,ISO_8859_1        portuguese,pt,por
 romanian        UTF_8                   romanian,ro,rum,ron
 russian         UTF_8,KOI8_R            russian,ru,rus
 serbian         UTF_8                   serbian,sr,srp
+sesotho         UTF_8                   sesotho,st,sot
 spanish         UTF_8,ISO_8859_1        spanish,es,esl,spa
 swedish         UTF_8,ISO_8859_1        swedish,sv,swe
 tamil           UTF_8                   tamil,ta,tam
 turkish         UTF_8                   turkish,tr,tur
+ukrainian       UTF_8                   ukrainian,uk,ukr
 yiddish         UTF_8                   yiddish,yi,yid
-sesotho         UTF_8                   sesotho,st,sot
 
 # Also include the traditional porter algorithm for english.
 # The porter algorithm is included in the libstemmer distribution to assist

data/vendor/snowball/pascal/SnowballProgram.pas

@@ -70,8 +70,8 @@ Begin
     FCursor  := 0;
     FLimit   := Length(Current);
     FBkLimit := 0;
-    FBra     := FCursor;
-    FKet     := FLimit;
+    FBra     := 0;
+    FKet     := 0;
 End;
 
 Function TSnowballProgram.InGrouping(s : array of char; min, max : Integer) : Boolean;

data/vendor/snowball/php/base-stemmer.php

@@ -445,8 +445,8 @@ abstract class SnowballStemmer {
         $this->cursor = 0;
         $this->limit = strlen($word);
         $this->limit_backward = 0;
-        $this->bra = $this->cursor;
-        $this->ket = $this->limit;
+        $this->bra = 0;
+        $this->ket = 0;
         $this->stem();
         return $this->current;
     }

data/vendor/snowball/python/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 import re
 
-SNOWBALL_VERSION = '3.1.1'
+SNOWBALL_VERSION = '3.2.0'
 
 n_stemmers = 0
 
@@ -45,6 +45,10 @@ for lang in langs:
     if lang_titlecase != 'Sesotho':
         classifiers.append('Natural Language :: ' + lang_titlecase)
 
+    # Portuguese stemmer also works for Brazilian Portuguese.
+    if lang_titlecase == 'Portuguese':
+        classifiers.append('Natural Language :: Portuguese (Brazilian)')
+
 classifiers.extend([
     'Operating System :: OS Independent',
     'Programming Language :: Python',

data/vendor/snowball/python/snowballstemmer/basestemmer.py

@@ -10,8 +10,7 @@ class BaseStemmer:
         self.cursor = 0
         self.limit = len(self.current)
         self.limit_backward = 0
-        self.bra = self.cursor
-        self.ket = self.limit
+        self.bra = self.ket = 0
 
     def get_current(self):
         '''

data/vendor/snowball/runtime/api.c

@@ -1,14 +1,17 @@
 
 #include <stdlib.h> /* for malloc, free */
+#include <string.h> /* for memset */
 #include "snowball_runtime.h"
 
-static const struct SN_env default_SN_env;
-
 extern struct SN_env * SN_new_env(int alloc_size)
 {
     struct SN_env * z = (struct SN_env *) malloc(alloc_size);
     if (z == NULL) return NULL;
-    *z = default_SN_env;
+    /* This memset() will initialise z->p to all-bits-zero which (at least
+     * theoretically) may not be the NULL pointer representation, but that's OK
+     * as immediately afterwards we assign to z->p.
+     */
+    memset(z, 0, alloc_size);
     z->p = create_s();
     if (z->p == NULL) {
         SN_delete_env(z);
@@ -17,6 +20,14 @@ extern struct SN_env * SN_new_env(int alloc_size)
     return z;
 }
 
+/* This is used for the stemmer-specific *_create_env() for stemmers where
+ * there are non non-localised variables.
+ */
+extern struct SN_env * SN_new_env_no_vars(void)
+{
+    return SN_new_env(sizeof(struct SN_env));
+}
+
 extern void SN_delete_env(struct SN_env * z)
 {
     if (z == NULL) return;

data/vendor/snowball/runtime/api.h

@@ -1,21 +1,24 @@
 #ifndef SNOWBALL_API_H_INCLUDED
 #define SNOWBALL_API_H_INCLUDED
 
+#ifdef SNOWBALL_WIDE
+# ifndef __cplusplus
+/* wchar_t is a built-in type in C++, but not in C. */
+#  include <stddef.h>
+# endif
+typedef wchar_t symbol;
+// Use a different typename - for C++ this then means that the runtime helper
+// functions that depend on sizeof(symbol) will be distinct thanks to
+// overloading on SN_env and/or symbol.
+# define SN_env SN_env_w
+#else
 typedef unsigned char symbol;
-
-/* Or replace 'char' above with 'short' for 16 bit characters.
-
-   More precisely, replace 'char' with whatever type guarantees the
-   character width you need.
-*/
+#endif
 
 struct SN_env {
     symbol * p;
     int c; int l; int lb; int bra; int ket;
     int af;
-#ifdef __cplusplus
-    SN_env() : p(), c(), l(), lb(), bra(), ket(), af() { }
-#endif
 };
 
 #ifdef __cplusplus
@@ -23,6 +26,7 @@ extern "C" {
 #endif
 
 extern struct SN_env * SN_new_env(int alloc_size);
+extern struct SN_env * SN_new_env_no_vars(void);
 extern void SN_delete_env(struct SN_env * z);
 
 extern int SN_set_current(struct SN_env * z, int size, const symbol * s);

data/vendor/snowball/runtime/{utilities.c → snowball_runtime.c}

@@ -1,10 +1,10 @@
+#include "snowball_runtime.h"
 
+#include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "snowball_runtime.h"
-
 #ifdef SNOWBALL_RUNTIME_THROW_EXCEPTIONS
 # include <new>
 # include <stdexcept>
@@ -29,7 +29,7 @@
  */
 typedef int sizeof_symbol_divides_head[(HEAD % sizeof(symbol) == 0) ? 1 : -1];
 
-#define CREATE_SIZE 1
+#define CREATE_SIZE 31
 
 extern symbol * create_s(void) {
     symbol * p;
@@ -349,204 +349,138 @@ extern int eq_v_b(struct SN_env * z, const symbol * p) {
     return eq_s_b(z, SIZE(p), p);
 }
 
-#ifdef SNOWBALL_COVERAGE
-/* Declare more entries than any real Snowball program will have. */
-static char among_seen[4096];
+#ifdef SNOWBALL_WIDE
+# define GET_A(X) ((X)[1])
+# define GET_B(X) ((X)[2])
+# define NWAY(X) ((X) + 3)
+# define SEG_RESULT(X) (X)[3]
+# define SEG_DATA(X) ((X) + 4)
+#else
+# define GET_A(X) ((X)[1] & 0xff)
+# define GET_B(X) ((X)[1] >> 8)
+# define NWAY(X) ((X) + 2)
+# define SEG_RESULT(X) (X)[2]
+# define SEG_DATA(X) ((X) + 3)
 #endif
 
-extern int find_among(struct SN_env * z, const struct among * v, int v_size,
-                      int (*call_among_func)(struct SN_env*)) {
-
-    int i = 0;
-    int j = v_size;
-
-    int c = z->c; int l = z->l;
-    const symbol * q = z->p + c;
-
-    const struct among * w;
-
-    int common_i = 0;
-    int common_j = 0;
-
-    int first_key_inspected = 0;
-
-#ifdef SNOWBALL_COVERAGE
-    int among_number = v[v_size].s_size;
-    if (among_number < (int)sizeof(among_seen) &&
-        among_seen[among_number] == 0) {
-        /* Report every entry once, then unused cases will appear (and we can
-         * decrement each count when generating the coverage report).
-         */
-        int k;
-        for (k = 0; k < v_size; ++k) {
-            w = v + k;
-            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-        }
-        /* If the among matches the empty string without a gating function then
-         * the "no match" case is impossible and so not useful to include in a
-         * coverage report.
-         */
-        if (v[v_size * 2].s_size != -1) {
-            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
-        }
-        among_seen[among_number] = 1;
+static int seg_matches(const symbol * p, const unsigned short * x, int len) {
+    int i;
+    x = SEG_DATA(x);
+    if (sizeof(symbol) == 1) {
+        return memcmp(p, x, len) == 0;
     }
-#endif
+    for (i = 0; i < len; ++i) {
+        if (p[i] != x[i]) return 0;
+    }
+    return 1;
+}
+
+extern int find_among(struct SN_env * z, const unsigned short * v) {
+    int c = z->c;
+    int l = z->l;
+    int o = 0;
+    int r = 0;
     while (1) {
-        int k = i + ((j - i) >> 1);
-        int diff = 0;
-        int common = common_i < common_j ? common_i : common_j; /* smaller */
-        w = v + k;
-        {
-            int i2; for (i2 = common; i2 < w->s_size; i2++) {
-                if (c + common == l) { diff = -1; break; }
-                diff = q[common] - w->s[i2];
-                if (diff != 0) break;
-                common++;
-            }
+        if (o < 0) {
+            z->c = c;
+            return -o;
         }
-        if (diff < 0) {
-            j = k;
-            common_j = common;
-        } else {
-            i = k;
-            common_i = common;
+        if (v[o]) {
+            r = v[o];
+            z->c = c;
         }
-        if (j - i <= 1) {
-            if (i > 0) break; /* v->s has been inspected */
-            if (j == i) break; /* only one item in v */
-
-            /* - but now we need to go round once more to get
-               v->s inspected. This looks messy, but is actually
-               the optimal approach.  */
-
-            if (first_key_inspected) break;
-            first_key_inspected = 1;
-        }
-    }
-    w = v + i;
-    while (1) {
-        if (common_i >= w->s_size) {
-            z->c = c + w->s_size;
-#ifdef SNOWBALL_COVERAGE
-            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-#endif
-            if (!w->function) return w->result;
-            z->af = w->function;
-            if (call_among_func(z)) {
-                z->c = c + w->s_size;
-#ifdef SNOWBALL_COVERAGE
-                fprintf(stderr, "%s: among %d : %d of %d func-t '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-#endif
-                return w->result;
+        if (c < l) {
+            symbol a = GET_A(v + o);
+            symbol b = GET_B(v + o);
+            if (b == 0) {
+                /* Substring segment. */
+                int old_c = c;
+                c += a;
+                if (c <= l && seg_matches(z->p + old_c, v + o, a)) {
+                    o = (short)SEG_RESULT(v + o);
+                    assert(o);
+                    continue;
+                }
+            } else {
+                symbol ch = z->p[c];
+                if (b < a) {
+                    /* 2-way dispatch. */
+                    if (ch == a || ch == b) {
+                        o = (short)NWAY(v + o)[(ch == a)];
+                        if (o) {
+                            ++c;
+                            continue;
+                        }
+                    }
+                } else {
+                    /* N-way dispatch. */
+                    if (ch >= a && ch <= b) {
+                        o = (short)NWAY(v + o)[ch - a];
+                        if (o) {
+                            ++c;
+                            continue;
+                        }
+                    }
+                }
             }
-#ifdef SNOWBALL_COVERAGE
-            fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-#endif
         }
-        if (!w->substring_i) {
-#ifdef SNOWBALL_COVERAGE
-            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
-#endif
-            return 0;
-        }
-        w += w->substring_i;
+        return r;
     }
 }
 
 /* find_among_b is for backwards processing. Same comments apply */
-
-extern int find_among_b(struct SN_env * z, const struct among * v, int v_size,
-                        int (*call_among_func)(struct SN_env*)) {
-
-    int i = 0;
-    int j = v_size;
-
-    int c = z->c; int lb = z->lb;
-    const symbol * q = z->p + c - 1;
-
-    const struct among * w;
-
-    int common_i = 0;
-    int common_j = 0;
-
-    int first_key_inspected = 0;
-
-#ifdef SNOWBALL_COVERAGE
-    int among_number = v[v_size].s_size;
-    if (among_number < (int)sizeof(among_seen) &&
-        among_seen[among_number] == 0) {
-        /* Report every entry once, then unused cases will appear (and we can
-         * decrement each count when generating the coverage report).
-         */
-        int k;
-        for (k = 0; k < v_size; ++k) {
-            w = v + k;
-            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-        }
-        /* If the among matches the empty string without a gating function then
-         * the "no match" case is impossible and so not useful to include in a
-         * coverage report.
-         */
-        if (v[v_size * 2].s_size != -1) {
-            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
-        }
-        among_seen[among_number] = 1;
-    }
-#endif
+extern int find_among_b(struct SN_env * z, const unsigned short * v) {
+    int c = z->c;
+    int lb = z->lb;
+    int o = 0;
+    int r = 0;
     while (1) {
-        int k = i + ((j - i) >> 1);
-        int diff = 0;
-        int common = common_i < common_j ? common_i : common_j;
-        w = v + k;
-        {
-            int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
-                if (c - common == lb) { diff = -1; break; }
-                diff = q[- common] - w->s[i2];
-                if (diff != 0) break;
-                common++;
-            }
+        if (o < 0) {
+            z->c = c;
+            return -o;
         }
-        if (diff < 0) { j = k; common_j = common; }
-                 else { i = k; common_i = common; }
-        if (j - i <= 1) {
-            if (i > 0) break;
-            if (j == i) break;
-            if (first_key_inspected) break;
-            first_key_inspected = 1;
+        if (v[o]) {
+            r = v[o];
+            z->c = c;
         }
-    }
-    w = v + i;
-    while (1) {
-        if (common_i >= w->s_size) {
-            z->c = c - w->s_size;
-#ifdef SNOWBALL_COVERAGE
-            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-#endif
-            if (!w->function) return w->result;
-            z->af = w->function;
-            if (call_among_func(z)) {
-#ifdef SNOWBALL_COVERAGE
-                fprintf(stderr, "%s: among %d : %d of %d func-t '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-#endif
-                z->c = c - w->s_size;
-                return w->result;
+        if (c > lb) {
+            symbol a = GET_A(v + o);
+            symbol b = GET_B(v + o);
+            if (b == 0) {
+                /* Substring segment. */
+                c -= a;
+                if (c >= lb && seg_matches(z->p + c, v + o, a)) {
+                    o = (short)SEG_RESULT(v + o);
+                    assert(o);
+                    continue;
+                }
+            } else {
+                symbol ch = z->p[c - 1];
+                if (b < a) {
+                    /* 2-way dispatch. */
+                    if (ch == a || ch == b) {
+                        o = (short)NWAY(v + o)[(ch == a)];
+                        if (o) {
+                            --c;
+                            continue;
+                        }
+                    }
+                } else {
+                    /* N-way dispatch. */
+                    if (ch >= a && ch <= b) {
+                        o = (short)NWAY(v + o)[ch - a];
+                        if (o) {
+                            --c;
+                            continue;
+                        }
+                    }
+                }
             }
-#ifdef SNOWBALL_COVERAGE
-            fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
-#endif
         }
-        if (!w->substring_i) {
-#ifdef SNOWBALL_COVERAGE
-            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
-#endif
-            return 0;
-        }
-        w += w->substring_i;
+        return r;
     }
 }
 
-
 /* Increase the size of the buffer pointed to by p to at least n symbols.
  * On success, returns 0.  If insufficient memory, returns -1.
  */

data/vendor/snowball/runtime/snowball_runtime.h

@@ -16,6 +16,32 @@
 # define CAPACITY(p)    ((int *)(p))[-2]
 #endif
 
+/* We need to know the endianness to correctly encode among tables when
+ * we aren't using wide characters.
+ */
+#ifndef SNOWBALL_WIDE
+# if !defined SNOWBALL_BIGENDIAN && !defined SNOWBALL_LITTLEENDIAN
+#  ifdef __BYTE_ORDER__ /* GCC, clang */
+#   if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#    define SNOWBALL_BIGENDIAN
+#   else
+#    define SNOWBALL_LITTLEENDIAN
+#   endif
+#  elif defined _MSC_VER && (defined _M_AMD64 || defined _M_ARM || defined _M_ARM64 || defined _M_IX86) /* MSVC */
+#   define SNOWBALL_LITTLEENDIAN
+#  elif defined HAVE_ENDIAN_H
+#   include <endian.h>
+#   if BYTE_ORDER == BIG_ENDIAN
+#    define SNOWBALL_BIGENDIAN
+#   else
+#    define SNOWBALL_LITTLEENDIAN
+#   endif
+#  else
+#   error Platform endianness unknown - define SNOWBALL_BIGENDIAN or SNOWBALL_LITTLEENDIAN
+#  endif
+# endif
+#endif
+
 #ifdef SNOWBALL_RUNTIME_THROW_EXCEPTIONS
 # define SNOWBALL_ERR void
 #else
@@ -29,35 +55,22 @@ static void debug(struct SN_env * z, int n, int line) {
     int len = SIZE(z->p);
     printf("%3d (line %4d): [%d]'", n, line, len);
     for (i = 0; i <= len; i++) {
-        if (z->lb == i) printf("{");
-        if (z->bra == i) printf("[");
-        if (z->c == i) printf("|");
-        if (z->ket == i) printf("]");
-        if (z->l == i) printf("}");
+        if (z->lb == i) putc('{', stdout);
+        if (z->bra == i) putc('[', stdout);
+        if (z->c == i) putc('|', stdout);
+        if (z->ket == i) putc(']', stdout);
+        if (z->l == i) putc('}', stdout);
         if (i < len) {
             int ch = z->p[i];
             if (ch == 0) ch = '#';
-            printf("%c", ch);
+            putc(ch, stdout);
         }
     }
     printf("'\n");
+    fflush(stdout);
 }
 #endif
 
-struct among
-{
-    /* Number of symbols in s. */
-    int s_size;
-    /* Search string. */
-    const symbol * s;
-    /* Delta of index to longest matching substring, or 0 if none. */
-    int substring_i;
-    /* Result of the lookup. */
-    int result;
-    /* Optional condition routine index, or 0 if none. */
-    int function;
-};
-
 /* MSVC doesn't like functions declared `extern "C"` throwing exceptions. */
 #if defined __cplusplus && !defined SNOWBALL_RUNTIME_THROW_EXCEPTIONS
 extern "C" {
@@ -85,10 +98,8 @@ extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
 extern int eq_v(struct SN_env * z, const symbol * p);
 extern int eq_v_b(struct SN_env * z, const symbol * p);
 
-extern int find_among(struct SN_env * z, const struct among * v, int v_size,
-                      int (*)(struct SN_env *));
-extern int find_among_b(struct SN_env * z, const struct among * v, int v_size,
-                        int (*)(struct SN_env *));
+extern int find_among(struct SN_env * z, const unsigned short * v);
+extern int find_among_b(struct SN_env * z, const unsigned short * v);
 
 extern SNOWBALL_ERR replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s);
 extern SNOWBALL_ERR slice_from_s(struct SN_env * z, int s_size, const symbol * s);

data/vendor/snowball/rust/src/snowball/snowball_env.rs

@@ -21,7 +21,7 @@ impl<'a> SnowballEnv<'a> {
             limit: len as i32,
             limit_backward: 0,
             bra: 0,
-            ket: len as i32,
+            ket: 0,
         }
     }