mittens 0.3.0 → 0.3.2

data/vendor/snowball/runtime/utilities.c

@@ -3,14 +3,39 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "header.h"
+#include "snowball_runtime.h"
+
+#ifdef SNOWBALL_RUNTIME_THROW_EXCEPTIONS
+# include <new>
+# include <stdexcept>
+# define SNOWBALL_RETURN_OK return
+# define SNOWBALL_RETURN_OR_THROW(R, E) throw E
+# define SNOWBALL_PROPAGATE_ERR(F) F
+#else
+# define SNOWBALL_RETURN_OK return 0
+# define SNOWBALL_RETURN_OR_THROW(R, E) return R
+# define SNOWBALL_PROPAGATE_ERR(F) do { \
+        int snowball_err = F; \
+        if (snowball_err < 0) return snowball_err; \
+    } while (0)
+#endif
+
+#define HEAD (2 * sizeof(int))
+
+/* Note that sizeof(symbol) should divide HEAD without remainder, otherwise
+ * there is an alignment problem.
+ *
+ * We support C90 here so use a typedef trick instead of static_assert.
+ */
+typedef int sizeof_symbol_divides_head[(HEAD % sizeof(symbol) == 0) ? 1 : -1];
 
 #define CREATE_SIZE 1
 
 extern symbol * create_s(void) {
     symbol * p;
     void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
-    if (mem == NULL) return NULL;
+    if (mem == NULL)
+        SNOWBALL_RETURN_OR_THROW(NULL, std::bad_alloc());
     p = (symbol *) (HEAD + (char *) mem);
     CAPACITY(p) = CREATE_SIZE;
     SET_SIZE(p, 0);
@@ -24,14 +49,15 @@ extern void lose_s(symbol * p) {
 
 /*
    new_p = skip_utf8(p, c, l, n); skips n characters forwards from p + c.
-   new_p is the new position, or -1 on failure.
+   new_p is the new position, or -1 on failure (if c would be > l).
+
+   Caller ensures n >= 0.
 
    -- used to implement hop and next in the utf8 case.
 */
 
 extern int skip_utf8(const symbol * p, int c, int limit, int n) {
     int b;
-    if (n < 0) return -1;
     for (; n > 0; n--) {
         if (c >= limit) return -1;
         b = p[c++];
@@ -49,14 +75,15 @@ extern int skip_utf8(const symbol * p, int c, int limit, int n) {
 
 /*
    new_p = skip_b_utf8(p, c, lb, n); skips n characters backwards from p + c - 1
-   new_p is the new position, or -1 on failure.
+   new_p is the new position, or -1 on failure (if c would be < lb).
+
+   Caller ensures n >= 0.
 
    -- used to implement hop and next in the utf8 case.
 */
 
 extern int skip_b_utf8(const symbol * p, int c, int limit, int n) {
     int b;
-    if (n < 0) return -1;
     for (; n > 0; n--) {
         if (c <= limit) return -1;
         b = p[--c];
@@ -119,13 +146,79 @@ static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
     return 4;
 }
 
+#ifdef SNOWBALL_COVERAGE
+/* The grouping number gets stored in a byte, clamped to 255. */
+static char grouping_seen[255];
+
+static void report_coverage(const unsigned char * s, int min, int max, int ch, const unsigned char * p, int w) {
+    int i = 0;
+    int j;
+    int outof = 0;
+    const unsigned char * loc = s + (max - min + 8) / 8;
+    int grouping_number = *loc++;
+    /* Adjust ch be an offset from min if it's past the end of the range.  If
+     * we already subtracted min then this will condition will be false.  Only
+     * needed for the "out" case but the condition can never be true for the
+     * "in" case.
+     */
+    if (ch > max) ch -= min;
+    /* Find the index of this character in the grouping. */
+    for (j = 0; j != max - min; ++j) {
+        if (s[j >> 3] & (0X1 << (j & 0X7))) {
+            ++outof;
+            if (j < ch) ++i;
+        }
+    }
+    if (grouping_number < (int)sizeof(grouping_seen) &&
+        grouping_seen[grouping_number] == 0) {
+        /* Report every entry once, then unused cases will appear (and we can
+         * decrement each count when generating the coverage report).
+         */
+        int k = 0;
+        for (j = 0; j != max - min; ++j) {
+            if (s[j >> 3] & (0X1 << (j & 0X7))) {
+                fprintf(stderr, "%s index %d of %d '", loc, k, outof + 1);
+                int codepoint = j + min;
+                if (codepoint < 0x80) {
+                    putc(codepoint, stderr);
+                } else if (codepoint < 0x800) {
+                    putc((codepoint >> 6) | 0xC0, stderr);
+                    putc((codepoint & 0x3F) | 0x80, stderr);
+                } else {
+                    putc((codepoint >> 12) | 0xE0, stderr);
+                    putc(((codepoint >> 6) & 0x3F) | 0x80, stderr);
+                    putc((codepoint & 0x3F) | 0x80, stderr);
+                }
+                fprintf(stderr, "'\n");
+                ++k;
+            }
+        }
+        grouping_seen[grouping_number] = 1;
+    }
+    fprintf(stderr, "%s index %d of %d '%.*s'\n", loc, i, outof + 1, w, p);
+}
+
+static void report_coverage_nomatch(const unsigned char * s, int min, int max) {
+    const unsigned char * loc = s + (max - min + 8) / 8;
+    ++loc;
+    fprintf(stderr, "%s no match\n", loc);
+}
+#endif
+
 extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
     do {
         int ch;
         int w = get_utf8(z->p, z->c, z->l, & ch);
         if (!w) return -1;
-        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) {
+#ifdef SNOWBALL_COVERAGE
+            report_coverage_nomatch(s, min, max);
+#endif
             return w;
+        }
+#ifdef SNOWBALL_COVERAGE
+        report_coverage(s, min, max, ch, z->p + z->c, w);
+#endif
         z->c += w;
     } while (repeat);
     return 0;
@@ -136,8 +229,15 @@ extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min,
         int ch;
         int w = get_b_utf8(z->p, z->c, z->lb, & ch);
         if (!w) return -1;
-        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+        if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0) {
+#ifdef SNOWBALL_COVERAGE
+            report_coverage_nomatch(s, min, max);
+#endif
             return w;
+        }
+#ifdef SNOWBALL_COVERAGE
+        report_coverage(s, min, max, ch, z->p + z->c - w, w);
+#endif
         z->c -= w;
     } while (repeat);
     return 0;
@@ -148,8 +248,15 @@ extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, i
         int ch;
         int w = get_utf8(z->p, z->c, z->l, & ch);
         if (!w) return -1;
-        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)) {
+#ifdef SNOWBALL_COVERAGE
+            report_coverage(s, min, max, ch, z->p + z->c, w);
+#endif
             return w;
+        }
+#ifdef SNOWBALL_COVERAGE
+        report_coverage_nomatch(s, min, max);
+#endif
         z->c += w;
     } while (repeat);
     return 0;
@@ -160,8 +267,15 @@ extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min,
         int ch;
         int w = get_b_utf8(z->p, z->c, z->lb, & ch);
         if (!w) return -1;
-        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0))
+        if (!(ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)) {
+#ifdef SNOWBALL_COVERAGE
+            report_coverage(s, min, max, ch, z->p + z->c - w, w);
+#endif
             return w;
+        }
+#ifdef SNOWBALL_COVERAGE
+        report_coverage_nomatch(s, min, max);
+#endif
         z->c -= w;
     } while (repeat);
     return 0;
@@ -235,7 +349,13 @@ extern int eq_v_b(struct SN_env * z, const symbol * p) {
     return eq_s_b(z, SIZE(p), p);
 }
 
-extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
+#ifdef SNOWBALL_COVERAGE
+/* Declare more entries than any real Snowball program will have. */
+static char among_seen[4096];
+#endif
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size,
+                      int (*call_among_func)(struct SN_env*)) {
 
     int i = 0;
     int j = v_size;
@@ -250,6 +370,28 @@ extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
 
     int first_key_inspected = 0;
 
+#ifdef SNOWBALL_COVERAGE
+    int among_number = v[v_size].s_size;
+    if (among_number < (int)sizeof(among_seen) &&
+        among_seen[among_number] == 0) {
+        /* Report every entry once, then unused cases will appear (and we can
+         * decrement each count when generating the coverage report).
+         */
+        int k;
+        for (k = 0; k < v_size; ++k) {
+            w = v + k;
+            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+        }
+        /* If the among matches the empty string without a gating function then
+         * the "no match" case is impossible and so not useful to include in a
+         * coverage report.
+         */
+        if (v[v_size * 2].s_size != -1) {
+            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
+        }
+        among_seen[among_number] = 1;
+    }
+#endif
     while (1) {
         int k = i + ((j - i) >> 1);
         int diff = 0;
@@ -286,21 +428,36 @@ extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
     while (1) {
         if (common_i >= w->s_size) {
             z->c = c + w->s_size;
-            if (w->function == NULL) return w->result;
-            {
-                int res = w->function(z);
+#ifdef SNOWBALL_COVERAGE
+            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+#endif
+            if (!w->function) return w->result;
+            z->af = w->function;
+            if (call_among_func(z)) {
                 z->c = c + w->s_size;
-                if (res) return w->result;
+#ifdef SNOWBALL_COVERAGE
+                fprintf(stderr, "%s: among %d : %d of %d func-t '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+#endif
+                return w->result;
             }
+#ifdef SNOWBALL_COVERAGE
+            fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+#endif
+        }
+        if (!w->substring_i) {
+#ifdef SNOWBALL_COVERAGE
+            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
+#endif
+            return 0;
         }
-        if (!w->substring_i) return 0;
         w += w->substring_i;
     }
 }
 
 /* find_among_b is for backwards processing. Same comments apply */
 
-extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size,
+                        int (*call_among_func)(struct SN_env*)) {
 
     int i = 0;
     int j = v_size;
@@ -315,6 +472,28 @@ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
 
     int first_key_inspected = 0;
 
+#ifdef SNOWBALL_COVERAGE
+    int among_number = v[v_size].s_size;
+    if (among_number < (int)sizeof(among_seen) &&
+        among_seen[among_number] == 0) {
+        /* Report every entry once, then unused cases will appear (and we can
+         * decrement each count when generating the coverage report).
+         */
+        int k;
+        for (k = 0; k < v_size; ++k) {
+            w = v + k;
+            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+        }
+        /* If the among matches the empty string without a gating function then
+         * the "no match" case is impossible and so not useful to include in a
+         * coverage report.
+         */
+        if (v[v_size * 2].s_size != -1) {
+            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
+        }
+        among_seen[among_number] = 1;
+    }
+#endif
     while (1) {
         int k = i + ((j - i) >> 1);
         int diff = 0;
@@ -341,55 +520,59 @@ extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
     while (1) {
         if (common_i >= w->s_size) {
             z->c = c - w->s_size;
-            if (w->function == NULL) return w->result;
-            {
-                int res = w->function(z);
+#ifdef SNOWBALL_COVERAGE
+            fprintf(stderr, "%s: among %d : %d of %d string '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+#endif
+            if (!w->function) return w->result;
+            z->af = w->function;
+            if (call_among_func(z)) {
+#ifdef SNOWBALL_COVERAGE
+                fprintf(stderr, "%s: among %d : %d of %d func-t '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+#endif
                 z->c = c - w->s_size;
-                if (res) return w->result;
+                return w->result;
             }
+#ifdef SNOWBALL_COVERAGE
+            fprintf(stderr, "%s: among %d : %d of %d func-f '%.*s'\n", w[v_size].s, among_number, w[v_size].result, v_size, w->s_size, w->s);
+#endif
+        }
+        if (!w->substring_i) {
+#ifdef SNOWBALL_COVERAGE
+            fprintf(stderr, "%s: among %d no match\n", v[v_size * 2].s, among_number);
+#endif
+            return 0;
         }
-        if (!w->substring_i) return 0;
         w += w->substring_i;
     }
 }
 
 
 /* Increase the size of the buffer pointed to by p to at least n symbols.
- * If insufficient memory, returns NULL and frees the old buffer.
+ * On success, returns 0.  If insufficient memory, returns -1.
  */
-static symbol * increase_size(symbol * p, int n) {
-    symbol * q;
+static int increase_size(symbol ** p, int n) {
     int new_size = n + 20;
-    void * mem = realloc((char *) p - HEAD,
+    void * mem = realloc((char *) *p - HEAD,
                          HEAD + (new_size + 1) * sizeof(symbol));
-    if (mem == NULL) {
-        lose_s(p);
-        return NULL;
-    }
+    symbol * q;
+    if (mem == NULL) return -1;
     q = (symbol *) (HEAD + (char *)mem);
     CAPACITY(q) = new_size;
-    return q;
+    *p = q;
+    return 0;
 }
 
 /* to replace symbols between c_bra and c_ket in z->p by the
    s_size symbols at s.
    Returns 0 on success, -1 on error.
-   Also, frees z->p (and sets it to NULL) on error.
 */
-extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
+extern SNOWBALL_ERR replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s)
 {
-    int adjustment;
-    int len;
-    if (z->p == NULL) {
-        z->p = create_s();
-        if (z->p == NULL) return -1;
-    }
-    adjustment = s_size - (c_ket - c_bra);
-    len = SIZE(z->p);
+    int adjustment = s_size - (c_ket - c_bra);
     if (adjustment != 0) {
+        int len = SIZE(z->p);
         if (adjustment + len > CAPACITY(z->p)) {
-            z->p = increase_size(z->p, adjustment + len);
-            if (z->p == NULL) return -1;
+            SNOWBALL_PROPAGATE_ERR(increase_size(&z->p, adjustment + len));
         }
         memmove(z->p + c_ket + adjustment,
                 z->p + c_ket,
@@ -402,82 +585,97 @@ extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const
             z->c = c_bra;
     }
     if (s_size) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
-    if (adjptr != NULL)
-        *adjptr = adjustment;
-    return 0;
+    SNOWBALL_RETURN_OK;
 }
 
-static int slice_check(struct SN_env * z) {
+# define REPLACE_S(Z, B, K, SIZE, S) \
+    SNOWBALL_PROPAGATE_ERR(replace_s(Z, B, K, SIZE, S))
+
+static SNOWBALL_ERR slice_check(struct SN_env * z) {
 
     if (z->bra < 0 ||
         z->bra > z->ket ||
         z->ket > z->l ||
-        z->p == NULL ||
         z->l > SIZE(z->p)) /* this line could be removed */
     {
 #if 0
         fprintf(stderr, "faulty slice operation:\n");
         debug(z, -1, 0);
 #endif
-        return -1;
+        SNOWBALL_RETURN_OR_THROW(-1, std::logic_error("Snowball slice invalid"));
     }
-    return 0;
+    SNOWBALL_RETURN_OK;
 }
 
-extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
-    if (slice_check(z)) return -1;
-    return replace_s(z, z->bra, z->ket, s_size, s, NULL);
+# define SLICE_CHECK(Z) SNOWBALL_PROPAGATE_ERR(slice_check(Z))
+
+extern SNOWBALL_ERR slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
+    SLICE_CHECK(z);
+    REPLACE_S(z, z->bra, z->ket, s_size, s);
+    z->ket = z->bra + s_size;
+    SNOWBALL_RETURN_OK;
 }
 
-extern int slice_from_v(struct SN_env * z, const symbol * p) {
+extern SNOWBALL_ERR slice_from_v(struct SN_env * z, const symbol * p) {
     return slice_from_s(z, SIZE(p), p);
 }
 
-extern int slice_del(struct SN_env * z) {
-    return slice_from_s(z, 0, NULL);
+extern SNOWBALL_ERR slice_del(struct SN_env * z) {
+    SLICE_CHECK(z);
+    {
+        int slice_size = z->ket - z->bra;
+        if (slice_size != 0) {
+            int len = SIZE(z->p);
+            memmove(z->p + z->bra,
+                    z->p + z->ket,
+                    (len - z->ket) * sizeof(symbol));
+            SET_SIZE(z->p, len - slice_size);
+            z->l -= slice_size;
+            if (z->c >= z->ket)
+                z->c -= slice_size;
+            else if (z->c > z->bra)
+                z->c = z->bra;
+            z->ket = z->bra;
+        }
+    }
+    SNOWBALL_RETURN_OK;
 }
 
-extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
-    int adjustment;
-    if (replace_s(z, bra, ket, s_size, s, &adjustment))
-        return -1;
-    if (bra <= z->bra) z->bra += adjustment;
-    if (bra <= z->ket) z->ket += adjustment;
-    return 0;
+extern SNOWBALL_ERR insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
+    REPLACE_S(z, bra, ket, s_size, s);
+    if (bra <= z->ket) {
+        int adjustment = s_size - (ket - bra);
+        z->ket += adjustment;
+        if (bra <= z->bra) z->bra += adjustment;
+    }
+    SNOWBALL_RETURN_OK;
 }
 
-extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
+extern SNOWBALL_ERR insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
     return insert_s(z, bra, ket, SIZE(p), p);
 }
 
-extern symbol * slice_to(struct SN_env * z, symbol * p) {
-    if (slice_check(z)) {
-        lose_s(p);
-        return NULL;
-    }
+extern SNOWBALL_ERR slice_to(struct SN_env * z, symbol ** p) {
+    SLICE_CHECK(z);
     {
         int len = z->ket - z->bra;
-        if (CAPACITY(p) < len) {
-            p = increase_size(p, len);
-            if (p == NULL)
-                return NULL;
+        if (CAPACITY(*p) < len) {
+            SNOWBALL_PROPAGATE_ERR(increase_size(p, len));
         }
-        memmove(p, z->p + z->bra, len * sizeof(symbol));
-        SET_SIZE(p, len);
+        memmove(*p, z->p + z->bra, len * sizeof(symbol));
+        SET_SIZE(*p, len);
     }
-    return p;
+    SNOWBALL_RETURN_OK;
 }
 
-extern symbol * assign_to(struct SN_env * z, symbol * p) {
+extern SNOWBALL_ERR assign_to(struct SN_env * z, symbol ** p) {
     int len = z->l;
-    if (CAPACITY(p) < len) {
-        p = increase_size(p, len);
-        if (p == NULL)
-            return NULL;
+    if (CAPACITY(*p) < len) {
+        SNOWBALL_PROPAGATE_ERR(increase_size(p, len));
     }
-    memmove(p, z->p, len * sizeof(symbol));
-    SET_SIZE(p, len);
-    return p;
+    memmove(*p, z->p, len * sizeof(symbol));
+    SET_SIZE(*p, len);
+    SNOWBALL_RETURN_OK;
 }
 
 extern int len_utf8(const symbol * p) {
@@ -489,25 +687,3 @@ extern int len_utf8(const symbol * p) {
     }
     return len;
 }
-
-#if 0
-extern void debug(struct SN_env * z, int number, int line_count) {
-    int i;
-    int limit = SIZE(z->p);
-    /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/
-    if (number >= 0) printf("%3d (line %4d): [%d]'", number, line_count,limit);
-    for (i = 0; i <= limit; i++) {
-        if (z->lb == i) printf("{");
-        if (z->bra == i) printf("[");
-        if (z->c == i) printf("|");
-        if (z->ket == i) printf("]");
-        if (z->l == i) printf("}");
-        if (i < limit)
-        {   int ch = z->p[i];
-            if (ch == 0) ch = '#';
-            printf("%c", ch);
-        }
-    }
-    printf("'\n");
-}
-#endif

data/vendor/snowball/rust/src/main.rs

@@ -85,9 +85,9 @@ impl Stemmer {
     pub fn create(lang: String) -> Self {
         // Have a look at ../build.rs
         // There we generate a file that is rust code for a closure that returns a stemmer.
-        // We match against all the algorithms in src/snowball/algoritms/ folder.
+        // We match against all the algorithms in src/snowball/algorithms/ folder.
         // Alas, this cannot be included as a match statement or function because of Rust's
-        // hygenic macros.
+        // hygienic macros.
         let match_language = include!(concat!(env!("OUT_DIR"), "/lang_matches.rs"));
         match_language(lang)
     }

data/vendor/snowball/rust/src/snowball/snowball_env.rs

@@ -93,10 +93,10 @@ impl<'a> SnowballEnv<'a> {
     }
 
     /// Replace string between `bra` and `ket` with s
-    pub fn slice_from(&mut self, s: &str) -> bool {
+    pub fn slice_from(&mut self, s: &str) {
         let (bra, ket) = (self.bra, self.ket);
         self.replace_s(bra, ket, s);
-        true
+        self.ket = bra + s.len() as i32;
     }
 
     /// Move cursor to next character
@@ -331,7 +331,7 @@ impl<'a> SnowballEnv<'a> {
 
 
     /// Helper function that removes the string slice between `bra` and `ket`
-    pub fn slice_del(&mut self) -> bool {
+    pub fn slice_del(&mut self) {
         self.slice_from("")
     }
 
@@ -407,9 +407,8 @@ impl<'a> SnowballEnv<'a> {
             if common_i >= w.0.len() as i32{
                 self.cursor = c + w.0.len() as i32;
                 if let Some(ref method) = w.3 {
-                    let res = method(self, context);
-                    self.cursor = c + w.0.len() as i32;
-                    if res {
+                    if method(self, context) {
+                        self.cursor = c + w.0.len() as i32;
                         return w.2;
                     }
                 } else {
@@ -481,9 +480,8 @@ impl<'a> SnowballEnv<'a> {
             if common_i >= w.0.len() as i32 {
                 self.cursor = c - w.0.len() as i32;
                 if let Some(ref method) = w.3 {
-                    let res = method(self, context);
-                    self.cursor = c - w.0.len() as i32;
-                    if res {
+                    if method(self, context) {
+                        self.cursor = c - w.0.len() as i32;
                         return w.2;
                     }
                 } else {

data/vendor/snowball/tests/compilertest

@@ -0,0 +1,62 @@
+#!/bin/sh
+set -e
+SNOWBALL='../snowball -o tmp'
+r=0
+
+# Tests for expected compiler errors.
+set x errors/*.sbl
+shift
+echo "1..$#"
+for t ; do
+  b=`echo "$t"|sed 's/\.sbl$//'`
+  f=
+  # Run with -syntax to avoid generating files if we get none of the expected
+  # errors.
+  if $SNOWBALL -syntax "$t" 2> tmp.stderr > tmp.syntax ; then
+    f="snowball compiler did not fail"
+  else
+    if ! diff "$b.stderr" tmp.stderr ; then
+      f="stderr output not as expected"
+    fi
+  fi
+  if [ -z "$f" ] ; then
+    echo "ok - $b"
+  else
+    echo "not ok - $b: $f"
+    r=1
+  fi
+done
+
+# Tests which check --syntax output.
+set x syntax/*.sbl
+shift
+echo "1..$#"
+for t ; do
+  b=`echo "$t"|sed 's/\.sbl$//'`
+  f=
+  if $SNOWBALL -syntax "$t" 2> tmp.stderr > tmp.syntax ; then
+    if [ -f "$b.stderr" ] ; then
+      if ! diff "$b.stderr" tmp.stderr ; then
+	f="stderr output not as expected"
+      fi
+    fi
+    if [ -z "$f" ] && [ -f "$b.syntax" ] ; then
+      if ! diff "$b.syntax" tmp.syntax ; then
+	f="syntax tree not as expected"
+      else
+	rm tmp.syntax
+      fi
+    fi
+  else
+    f="snowball compiler failed"
+  fi
+  if [ -z "$f" ] ; then
+    echo "ok - $b"
+    rm -f tmp.stderr tmp.syntax
+  else
+    echo "not ok - $b: $f"
+    r=1
+  fi
+done
+
+exit $r