mittens 0.1.1 → 0.3.0

data/vendor/snowball/compiler/tokeniser.c

@@ -12,7 +12,7 @@ struct system_word {
 };
 
 
-/* ASCII collating assumed in syswords.c */
+/* ASCII collating assumed in syswords.h */
 
 #include "syswords.h"
 
@@ -22,16 +22,16 @@ static int hex_to_num(int ch);
 
 static int smaller(int a, int b) { return a < b ? a : b; }
 
-extern symbol * get_input(const char * filename) {
+extern byte * get_input(const char * filename) {
     FILE * input = fopen(filename, "r");
-    if (input == 0) { return 0; }
+    if (input == NULL) { return NULL; }
     {
-        symbol * u = create_b(INITIAL_INPUT_BUFFER_SIZE);
+        byte * u = create_s(INITIAL_INPUT_BUFFER_SIZE);
         int size = 0;
         while (true) {
             int ch = getc(input);
             if (ch == EOF) break;
-            if (size >= CAPACITY(u)) u = increase_capacity(u, size);
+            if (size >= CAPACITY(u)) u = increase_capacity_s(u, size);
             u[size++] = ch;
         }
         fclose(input);
@@ -40,7 +40,7 @@ extern symbol * get_input(const char * filename) {
     }
 }
 
-static void error(struct tokeniser * t, const char * s1, int n, symbol * p, const char * s2) {
+static void error(struct tokeniser * t, const char * s1, byte * p, int n, const char * s2) {
     if (t->error_count == 20) { fprintf(stderr, "... etc\n"); exit(1); }
     fprintf(stderr, "%s:%d: ", t->file, t->line_number);
     if (s1) fprintf(stderr, "%s", s1);
@@ -54,25 +54,19 @@ static void error(struct tokeniser * t, const char * s1, int n, symbol * p, cons
 }
 
 static void error1(struct tokeniser * t, const char * s) {
-    error(t, s, 0,0, 0);
+    error(t, s, NULL, 0, NULL);
 }
 
 static void error2(struct tokeniser * t, const char * s) {
-    error(t, "unexpected end of text after ", 0,0, s);
+    error(t, "unexpected end of text after ", NULL, 0, s);
 }
 
-static int compare_words(int m, symbol * p, int n, const byte * q) {
+static int compare_words(int m, const byte * p, int n, const byte * q) {
     if (m != n) return m - n;
-    {
-        int i; for (i = 0; i < n; i++) {
-            int diff = p[i] - q[i];
-            if (diff) return diff;
-        }
-    }
-    return 0;
+    return memcmp(p, q, n);
 }
 
-static int find_word(int n, symbol * p) {
+static int find_word(int n, const byte * p) {
     int i = 0; int j = vocab->code;
     do {
         int k = i + (j - i)/2;
@@ -84,22 +78,6 @@ static int find_word(int n, symbol * p) {
     return -1;
 }
 
-static int get_number(int n, symbol * p) {
-    int x = 0;
-    int i; for (i = 0; i < n; i++) x = 10*x + p[i] - '0';
-    return x;
-}
-
-static int eq_s(struct tokeniser * t, const char * s) {
-    int l = strlen(s);
-    if (SIZE(t->p) - t->c < l) return false;
-    {
-        int i;
-        for (i = 0; i < l; i++) if (t->p[t->c + i] != s[i]) return false;
-    }
-    t->c += l; return true;
-}
-
 static int white_space(struct tokeniser * t, int ch) {
     switch (ch) {
         case '\n':
@@ -113,61 +91,65 @@ static int white_space(struct tokeniser * t, int ch) {
     return false;
 }
 
-static symbol * find_in_m(struct tokeniser * t, int n, symbol * p) {
+static symbol * find_in_m(struct tokeniser * t, int n, byte * p) {
     struct m_pair * q;
     for (q = t->m_pairs; q; q = q->next) {
-        symbol * name = q->name;
-        if (n == SIZE(name) && memcmp(name, p, n * sizeof(symbol)) == 0) return q->value;
+        byte * name = q->name;
+        if (n == SIZE(name) && memcmp(name, p, n) == 0) return q->value;
     }
-    return 0;
+    return NULL;
 }
 
 static int read_literal_string(struct tokeniser * t, int c) {
-    symbol * p = t->p;
+    byte * p = t->p;
     int ch;
     SIZE(t->b) = 0;
     while (true) {
-        if (c >= SIZE(p)) { error2(t, "'"); return c; }
+        if (c >= SIZE(p) || p[c] == '\n') {
+            error1(t, "string literal not terminated");
+            return c;
+        }
         ch = p[c];
-        if (ch == '\n') { error1(t, "string not terminated"); return c; }
         c++;
         if (ch == t->m_start) {
             /* Inside insert characters. */
             int c0 = c;
             int newlines = false; /* no newlines as yet */
-            int black_found = false; /* no printing chars as yet */
+            int all_whitespace = true; /* no printing chars as yet */
             while (true) {
-                if (c >= SIZE(p)) { error2(t, "'"); return c; }
-                ch = p[c]; c++;
-                if (ch == t->m_end) break;
-                if (!white_space(t, ch)) black_found = true;
-                if (ch == '\n') newlines = true;
-                if (newlines && black_found) {
-                    error1(t, "string not terminated");
+                if (c >= SIZE(p) || (p[c] == '\n' && !all_whitespace)) {
+                    error1(t, "string literal not terminated");
                     return c;
                 }
+                ch = p[c];
+                if (ch == '\n') {
+                    newlines = true;
+                }
+                c++;
+                if (ch == t->m_end) break;
+                if (!white_space(t, ch)) all_whitespace = false;
             }
             if (!newlines) {
                 int n = c - c0 - 1;    /* macro size */
                 int firstch = p[c0];
                 symbol * q = find_in_m(t, n, p + c0);
-                if (q == 0) {
+                if (q == NULL) {
                     if (n == 1 && (firstch == '\'' || firstch == t->m_start))
-                        t->b = add_to_b(t->b, 1, p + c0);
+                        t->b = add_symbol_to_b(t->b, p[c0]);
                     else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') {
                         int codepoint = 0;
                         int x;
                         if (t->uplusmode == UPLUS_DEFINED) {
                             /* See if found with xxxx upper-cased. */
-                            symbol * uc = create_b(n);
+                            byte * uc = create_s(n);
                             int i;
                             for (i = 0; i != n; ++i) {
                                 uc[i] = toupper(p[c0 + i]);
                             }
                             q = find_in_m(t, n, uc);
-                            lose_b(uc);
-                            if (q != 0) {
-                                t->b = add_to_b(t->b, SIZE(q), q);
+                            lose_s(uc);
+                            if (q != NULL) {
+                                t->b = add_to_b(t->b, q, SIZE(q));
                                 continue;
                             }
                             error1(t, "Some U+xxxx stringdefs seen but not this one");
@@ -189,15 +171,14 @@ static int read_literal_string(struct tokeniser * t, int c) {
                             /* Ensure there's enough space for a max length
                              * UTF-8 sequence. */
                             if (CAPACITY(t->b) < SIZE(t->b) + 3) {
-                                t->b = increase_capacity(t->b, 3);
+                                t->b = increase_capacity_b(t->b, 3);
                             }
                             SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b));
                         } else {
-                            symbol sym;
                             if (t->encoding == ENC_SINGLEBYTE) {
                                 /* Only ISO-8859-1 is handled this way - for
                                  * other single-byte character sets you need
-                                 * stringdef all the U+xxxx codes you use
+                                 * to stringdef all the U+xxxx codes you use
                                  * like - e.g.:
                                  *
                                  * stringdef U+0171   hex 'FB'
@@ -210,13 +191,14 @@ static int read_literal_string(struct tokeniser * t, int c) {
                                     error1(t, "character values exceed 64K");
                                 }
                             }
-                            sym = codepoint;
-                            t->b = add_to_b(t->b, 1, &sym);
+                            t->b = add_symbol_to_b(t->b, (symbol)codepoint);
                         }
-                    } else
-                        error(t, "string macro '", n, p + c0, "' undeclared");
-                } else
-                    t->b = add_to_b(t->b, SIZE(q), q);
+                    } else {
+                        error(t, "string macro '", p + c0, n, "' undeclared");
+                    }
+                } else {
+                    t->b = add_to_b(t->b, q, SIZE(q));
+                }
             }
         } else {
             if (ch == '\'') return c;
@@ -226,7 +208,7 @@ static int read_literal_string(struct tokeniser * t, int c) {
                      * strings, but historically it's worked for single-byte
                      * and UTF-8 if the source encoding matches what the
                      * generated stemmer works in and it seems unfair to just
-                     * suddenly make this a hard error.`
+                     * suddenly make this a hard error.
                      */
                     fprintf(stderr,
                             "%s:%d: warning: Non-ASCII literal strings aren't "
@@ -237,13 +219,13 @@ static int read_literal_string(struct tokeniser * t, int c) {
                               "portable - use stringdef instead");
                 }
             }
-            t->b = add_to_b(t->b, 1, p + c - 1);
+            t->b = add_symbol_to_b(t->b, p[c - 1]);
         }
     }
 }
 
 static int next_token(struct tokeniser * t) {
-    symbol * p = t->p;
+    byte * p = t->p;
     int c = t->c;
     int ch;
     int code = -1;
@@ -256,21 +238,21 @@ static int next_token(struct tokeniser * t) {
             while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++;
             code = find_word(c - c0, p + c0);
             if (code < 0 || t->token_disabled[code]) {
-                t->b = move_to_b(t->b, c - c0, p + c0);
+                SIZE(t->s) = 0;
+                t->s = add_s_to_s(t->s, (const char*)p + c0, c - c0);
                 code = c_name;
             }
-        } else
-        if (isdigit(ch)) {
-            int c0 = c;
-            while (c < SIZE(p) && isdigit(p[c])) c++;
-            t->number = get_number(c - c0, p + c0);
+        } else if (isdigit(ch)) {
+            int value = ch - '0';
+            while (++c < SIZE(p) && isdigit(p[c])) {
+                value = 10 * value + (p[c] - '0');
+            }
+            t->number = value;
             code = c_number;
-        } else
-        if (ch == '\'') {
+        } else if (ch == '\'') {
             c = read_literal_string(t, c + 1);
             code = c_literalstring;
-        } else
-        {
+        } else {
             int lim = smaller(2, SIZE(p) - c);
             int i;
             for (i = lim; i > 0; i--) {
@@ -282,7 +264,7 @@ static int next_token(struct tokeniser * t) {
             t->c = c;
             return code;
         }
-        error(t, "'", 1, p + c, "' unknown");
+        error(t, "'", p + c, 1, "' unknown");
         c++;
         continue;
     }
@@ -309,7 +291,8 @@ static void read_chars(struct tokeniser * t) {
             ch = next_char(t);
             if (white_space(t, ch) || ch < 0) break;
         }
-        t->b2 = move_to_b(t->b2, t->c - c0 - 1, t->p + c0);
+        SIZE(t->s) = 0;
+        t->s = add_s_to_s(t->s, (const char*)t->p + c0, t->c - c0 - 1);
     }
 }
 
@@ -372,28 +355,39 @@ static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
 }
 
 extern int read_token(struct tokeniser * t) {
-    symbol * p = t->p;
+    byte * p = t->p;
     int held = t->token_held;
     t->token_held = false;
     if (held) return t->token;
+    t->token_reported_as_unexpected = false;
     while (true) {
         int code = next_token(t);
         switch (code) {
             case c_comment1: /*  slash-slash comment */
                 while (t->c < SIZE(p) && p[t->c] != '\n') t->c++;
                 continue;
-            case c_comment2: /* slash-star comment */
+            case c_comment2: { /* slash-star comment */
+                // Scan for a '*' stopping one before the end since we need a
+                // '/' to follow it to close the comment.
+                int size_less_one = SIZE(p) - 1;
+                int c = t->c;
                 while (true) {
-                    if (t->c >= SIZE(p)) {
+                    if (c >= size_less_one) {
                         error1(t, "/* comment not terminated");
                         t->token = -1;
                         return -1;
                     }
-                    if (p[t->c] == '\n') t->line_number++;
-                    if (eq_s(t, "*/")) break;
-                    t->c++;
+                    if (p[c] == '\n') {
+                        t->line_number++;
+                    } else if (p[c] == '*' && p[c + 1] == '/') {
+                        // Found '*/' to end of comment.
+                        t->c = c + 2;
+                        break;
+                    }
+                    ++c;
                 }
                 continue;
+            }
             case c_stringescapes: {
                 int ch1 = next_real_char(t);
                 int ch2 = next_real_char(t);
@@ -422,11 +416,11 @@ extern int read_token(struct tokeniser * t) {
                 if (base > 0) convert_numeric_string(t, t->b, base);
                 {   NEW(m_pair, q);
                     q->next = t->m_pairs;
-                    q->name = copy_b(t->b2);
+                    q->name = copy_s(t->s);
                     q->value = copy_b(t->b);
                     t->m_pairs = q;
                     if (t->uplusmode != UPLUS_DEFINED &&
-                        (SIZE(t->b2) >= 3 && t->b2[0] == 'U' && t->b2[1] == '+')) {
+                        (SIZE(t->s) >= 3 && t->s[0] == 'U' && t->s[1] == '+')) {
                         if (t->uplusmode == UPLUS_UNICODE) {
                             error1(t, "U+xxxx already used with implicit meaning");
                         } else {
@@ -448,22 +442,28 @@ extern int read_token(struct tokeniser * t) {
                 }
                 {
                     NEW(input, q);
-                    char * file = b_to_s(t->b);
-                    symbol * u = get_input(file);
-                    if (u == 0) {
+                    char * file = b_to_sz(t->b);
+                    int file_owned = 1;
+                    byte * u = get_input(file);
+                    if (u == NULL) {
                         struct include * r;
                         for (r = t->includes; r; r = r->next) {
-                            symbol * b = copy_b(r->b);
-                            b = add_to_b(b, SIZE(t->b), t->b);
-                            free(file);
-                            file = b_to_s(b);
+                            byte * s = copy_s(r->s);
+                            s = add_sz_to_s(s, file);
+                            s[SIZE(s)] = 0;
+                            if (file_owned > 0) {
+                                free(file);
+                            } else {
+                                lose_s((byte *)file);
+                            }
+                            file = (char*)s;
+                            file_owned = -1;
                             u = get_input(file);
-                            lose_b(b);
-                            if (u != 0) break;
+                            if (u != NULL) break;
                         }
                     }
-                    if (u == 0) {
-                        error(t, "Can't get '", SIZE(t->b), t->b, "'");
+                    if (u == NULL) {
+                        error(t, "Can't get '", (byte *)file, strlen(file), "'");
                         exit(1);
                     }
                     memmove(q, t, sizeof(struct input));
@@ -471,14 +471,14 @@ extern int read_token(struct tokeniser * t) {
                     t->p = u;
                     t->c = 0;
                     t->file = file;
-                    t->file_needs_freeing = true;
+                    t->file_owned = file_owned;
                     t->line_number = 1;
                 }
                 p = t->p;
                 continue;
             case -1:
                 if (t->next) {
-                    lose_b(p);
+                    lose_s(p);
                     {
                         struct input * q = t->next;
                         memmove(t, q, sizeof(struct input)); p = t->p;
@@ -496,6 +496,12 @@ extern int read_token(struct tokeniser * t) {
     }
 }
 
+extern int peek_token(struct tokeniser * t) {
+    int token = read_token(t);
+    t->token_held = true;
+    return token;
+}
+
 extern const char * name_of_token(int code) {
     int i;
     for (i = 1; i < vocab->code; i++)
@@ -509,6 +515,13 @@ extern const char * name_of_token(int code) {
         case c_grouping:     return "grouping";
         case c_call:         return "call";
         case c_booltest:     return "Boolean test";
+        case c_functionend:  return "Function end";
+        case c_goto_grouping:
+                             return "goto grouping";
+        case c_gopast_grouping:
+                             return "gopast grouping";
+        case c_goto_non:     return "goto non";
+        case c_gopast_non:   return "gopast non";
         case -2:             return "start of text";
         case -1:             return "end of text";
         default:             return "?";
@@ -519,21 +532,22 @@ extern void disable_token(struct tokeniser * t, int code) {
     t->token_disabled[code] = 1;
 }
 
-extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
+extern struct tokeniser * create_tokeniser(byte * p, char * file) {
     NEW(tokeniser, t);
-    t->next = 0;
+    t->next = NULL;
     t->p = p;
     t->c = 0;
     t->file = file;
-    t->file_needs_freeing = false;
+    t->file_owned = 0;
     t->line_number = 1;
     t->b = create_b(0);
-    t->b2 = create_b(0);
+    t->s = create_s(0);
     t->m_start = -1;
-    t->m_pairs = 0;
+    t->m_pairs = NULL;
     t->get_depth = 0;
     t->error_count = 0;
     t->token_held = false;
+    t->token_reported_as_unexpected = false;
     t->token = -2;
     t->previous_token = -2;
     t->uplusmode = UPLUS_NONE;
@@ -543,12 +557,12 @@ extern struct tokeniser * create_tokeniser(symbol * p, char * file) {
 
 extern void close_tokeniser(struct tokeniser * t) {
     lose_b(t->b);
-    lose_b(t->b2);
+    lose_s(t->s);
     {
         struct m_pair * q = t->m_pairs;
         while (q) {
             struct m_pair * q_next = q->next;
-            lose_b(q->name);
+            lose_s(q->name);
             lose_b(q->value);
             FREE(q);
             q = q_next;
@@ -562,6 +576,10 @@ extern void close_tokeniser(struct tokeniser * t) {
             q = q_next;
         }
     }
-    if (t->file_needs_freeing) free(t->file);
+    if (t->file_owned > 0) {
+        free(t->file);
+    } else if (t->file_owned < 0) {
+        lose_s((byte *)t->file);
+    }
     FREE(t);
 }

data/vendor/snowball/csharp/Snowball/Among.cs

@@ -2,10 +2,10 @@
 // Copyright (c) 2002, Richard Boulton
 // Copyright (c) 2015, Cesar Souza
 // All rights reserved.
-// 
+//
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are met:
-// 
+//
 //     * Redistributions of source code must retain the above copyright notice,
 //     * this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above copyright
@@ -14,7 +14,7 @@
 //     * Neither the name of the copyright holders nor the names of its contributors
 //     * may be used to endorse or promote products derived from this software
 //     * without specific prior written permission.
-// 
+//
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -34,41 +34,41 @@ namespace Snowball
     /// <summary>
     ///   Snowball's among construction.
     /// </summary>
-    /// 
+    ///
     public sealed class Among
     {
         /// <summary>
         ///   Search string.
         /// </summary>
-        /// 
+        ///
         public string SearchString { get; private set; }
 
         /// <summary>
         ///   Index to longest matching substring.
         /// </summary>
-        /// 
+        ///
         public int MatchIndex { get; private set; }
 
         /// <summary>
         ///   Result of the lookup.
         /// </summary>
-        /// 
+        ///
         public int Result { get; private set; }
 
         /// <summary>
         ///   Action to be invoked.
         /// </summary>
-        /// 
+        ///
         public Func<bool> Action { get; private set; }
 
         /// <summary>
         ///   Initializes a new instance of the <see cref="Among"/> class.
         /// </summary>
-        /// 
+        ///
         /// <param name="str">The search string.</param>
         /// <param name="index">The index to the longest matching substring.</param>
         /// <param name="result">The result of the lookup.</param>
-        /// 
+        ///
         public Among(String str, int index, int result)
             : this(str, index, result, null)
         {
@@ -77,12 +77,12 @@ namespace Snowball
         /// <summary>
         ///   Initializes a new instance of the <see cref="Among"/> class.
         /// </summary>
-        /// 
+        ///
         /// <param name="str">The search string.</param>
         /// <param name="index">The index to the longest matching substring.</param>
         /// <param name="result">The result of the lookup.</param>
         /// <param name="action">The action to be performed, if any.</param>
-        /// 
+        ///
         public Among(String str, int index, int result, Func<bool> action)
         {
             this.SearchString = str;
@@ -94,11 +94,11 @@ namespace Snowball
         /// <summary>
         ///   Returns a <see cref="System.String" /> that represents this instance.
         /// </summary>
-        /// 
+        ///
         /// <returns>
         ///   A <see cref="System.String" /> that represents this instance.
         /// </returns>
-        /// 
+        ///
         public override string ToString()
         {
             return SearchString;

data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs

@@ -2,7 +2,7 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
-// General Information about an assembly is controlled through the following 
+// General Information about an assembly is controlled through the following
 // set of attributes. Change these attribute values to modify the information
 // associated with an assembly.
 [assembly: AssemblyTitle("Snowball")]
@@ -14,8 +14,8 @@ using System.Runtime.InteropServices;
 [assembly: AssemblyTrademark("")]
 [assembly: AssemblyCulture("")]
 
-// Setting ComVisible to false makes the types in this assembly not visible 
-// to COM components.  If you need to access a type in this assembly from 
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components.  If you need to access a type in this assembly from
 // COM, set the ComVisible attribute to true on that type.
 [assembly: ComVisible(false)]
 
@@ -25,12 +25,12 @@ using System.Runtime.InteropServices;
 // Version information for an assembly consists of the following four values:
 //
 //      Major Version
-//      Minor Version 
+//      Minor Version
 //      Build Number
 //      Revision
 //
-// You can specify all the values or you can default the Build and Revision Numbers 
+// You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion(/*SNOWBALL_VERSION*/"2.2.0.0")]
-[assembly: AssemblyFileVersion(/*SNOWBALL_VERSION*/"2.2.0.0")]
+[assembly: AssemblyVersion(/*SNOWBALL_VERSION*/"3.0.1.0")]
+[assembly: AssemblyFileVersion(/*SNOWBALL_VERSION*/"3.0.1.0")]