mittens 0.3.1 → 0.3.2

data/vendor/snowball/compiler/tokeniser.c

@@ -5,39 +5,42 @@
 #include <ctype.h>   /* isalpha etc */
 #include "header.h"
 
-struct system_word {
-    int s_size;      /* size of system word */
-    const byte * s;  /* pointer to the system word */
-    int code;        /* its internal code */
-};
-
-
-/* ASCII collating assumed in syswords.h */
-
-#include "syswords.h"
+#include "tokens.h"
 
 #define INITIAL_INPUT_BUFFER_SIZE 8192
 
 static int hex_to_num(int ch);
 
-static int smaller(int a, int b) { return a < b ? a : b; }
-
 extern byte * get_input(const char * filename) {
-    FILE * input = fopen(filename, "r");
+    FILE * input = strcmp(filename, "-") == 0 ? stdin : fopen(filename, "rb");
     if (input == NULL) { return NULL; }
-    {
-        byte * u = create_s(INITIAL_INPUT_BUFFER_SIZE);
-        int size = 0;
+    byte * u = NULL;
+    int size = fseek(input, 0, SEEK_END) == 0 ? ftell(input) : -1;
+    if (size >= 0 && fseek(input, 0, SEEK_SET) == 0) {
+        u = create_s(size);
+        if (fread(u, size, 1, input) != 1) {
+            fprintf(stderr, "%s: Read error\n", filename);
+            exit(1);
+        }
+    } else {
+        // Unseekable stream, e.g. piped stdin.
+        size = 0;
+        u = create_s(INITIAL_INPUT_BUFFER_SIZE);
         while (true) {
-            int ch = getc(input);
-            if (ch == EOF) break;
-            if (size >= CAPACITY(u)) u = increase_capacity_s(u, size);
-            u[size++] = ch;
+            int s = CAPACITY(u) - size;
+            int r = fread(u + size, 1, s, input);
+            if (r < 0) {
+                fprintf(stderr, "%s: Read error\n", filename);
+                exit(1);
+            }
+            size += r;
+            if (r < s) break;
+            u = increase_capacity_s(u, size);
         }
-        fclose(input);
-        SIZE(u) = size;
-        return u;
     }
+    if (input != stdin) fclose(input);
+    SET_SIZE(u, size);
+    return u;
 }
 
 static void error(struct tokeniser * t, const char * s1, byte * p, int n, const char * s2) {
@@ -67,11 +70,12 @@ static int compare_words(int m, const byte * p, int n, const byte * q) {
 }
 
 static int find_word(int n, const byte * p) {
-    int i = 0; int j = vocab->code;
+    int i = -1;
+    int j = NUM_ALPHA_TOKENS;
     do {
-        int k = i + (j - i)/2;
-        const struct system_word * w = vocab + k;
-        int diff = compare_words(n, p, w->s_size, w->s);
+        int k = i + (j - i) / 2;
+        const struct token * w = alpha_tokens + k;
+        int diff = compare_words(n, p, w->s_size, (const byte*)w->s);
         if (diff == 0) return w->code;
         if (diff < 0) j = k; else i = k;
     } while (j - i != 1);
@@ -102,20 +106,19 @@ static symbol * find_in_m(struct tokeniser * t, int n, byte * p) {
 
 static int read_literal_string(struct tokeniser * t, int c) {
     byte * p = t->p;
-    int ch;
-    SIZE(t->b) = 0;
+    SET_SIZE(t->b, 0);
     while (true) {
         if (c >= SIZE(p) || p[c] == '\n') {
             error1(t, "string literal not terminated");
             return c;
         }
-        ch = p[c];
+        int ch = p[c];
         c++;
         if (ch == t->m_start) {
             /* Inside insert characters. */
             int c0 = c;
-            int newlines = false; /* no newlines as yet */
-            int all_whitespace = true; /* no printing chars as yet */
+            bool newlines = false; /* no newlines as yet */
+            bool all_whitespace = true; /* no printing chars as yet */
             while (true) {
                 if (c >= SIZE(p) || (p[c] == '\n' && !all_whitespace)) {
                     error1(t, "string literal not terminated");
@@ -138,7 +141,6 @@ static int read_literal_string(struct tokeniser * t, int c) {
                         t->b = add_symbol_to_b(t->b, p[c0]);
                     else if (n >= 3 && firstch == 'U' && p[c0 + 1] == '+') {
                         int codepoint = 0;
-                        int x;
                         if (t->uplusmode == UPLUS_DEFINED) {
                             /* See if found with xxxx upper-cased. */
                             byte * uc = create_s(n);
@@ -156,7 +158,7 @@ static int read_literal_string(struct tokeniser * t, int c) {
                         } else {
                             t->uplusmode = UPLUS_UNICODE;
                         }
-                        for (x = c0 + 2; x != c - 1; ++x) {
+                        for (int x = c0 + 2; x != c - 1; ++x) {
                             int hex = hex_to_num(p[x]);
                             if (hex < 0) {
                                 error1(t, "Bad hex digit following U+");
@@ -170,10 +172,11 @@ static int read_literal_string(struct tokeniser * t, int c) {
                             }
                             /* Ensure there's enough space for a max length
                              * UTF-8 sequence. */
-                            if (CAPACITY(t->b) < SIZE(t->b) + 3) {
+                            int b_size = SIZE(t->b);
+                            if (CAPACITY(t->b) < b_size + 3) {
                                 t->b = increase_capacity_b(t->b, 3);
                             }
-                            SIZE(t->b) += put_utf8(codepoint, t->b + SIZE(t->b));
+                            SET_SIZE(t->b, b_size + put_utf8(codepoint, t->b + b_size));
                         } else {
                             if (t->encoding == ENC_SINGLEBYTE) {
                                 /* Only ISO-8859-1 is handled this way - for
@@ -227,43 +230,105 @@ static int read_literal_string(struct tokeniser * t, int c) {
 static int next_token(struct tokeniser * t) {
     byte * p = t->p;
     int c = t->c;
-    int ch;
     int code = -1;
     while (true) {
         if (c >= SIZE(p)) { t->c = c; return -1; }
-        ch = p[c];
+        int ch = p[c];
         if (white_space(t, ch)) { c++; continue; }
+
         if (isalpha(ch)) {
             int c0 = c;
             while (c < SIZE(p) && (isalnum(p[c]) || p[c] == '_')) c++;
+            t->c = c;
             code = find_word(c - c0, p + c0);
-            if (code < 0 || t->token_disabled[code]) {
-                SIZE(t->s) = 0;
-                t->s = add_s_to_s(t->s, (const char*)p + c0, c - c0);
-                code = c_name;
+            if (code >= 0 && !t->token_disabled[code]) {
+                return code;
             }
-        } else if (isdigit(ch)) {
+            SET_SIZE(t->s, 0);
+            t->s = add_slen_to_s(t->s, (const char*)p + c0, c - c0);
+            return c_name;
+        }
+
+        if (isdigit(ch)) {
             int value = ch - '0';
             while (++c < SIZE(p) && isdigit(p[c])) {
                 value = 10 * value + (p[c] - '0');
             }
+            t->c = c;
             t->number = value;
-            code = c_number;
-        } else if (ch == '\'') {
-            c = read_literal_string(t, c + 1);
-            code = c_literalstring;
-        } else {
-            int lim = smaller(2, SIZE(p) - c);
-            int i;
-            for (i = lim; i > 0; i--) {
-                code = find_word(i, p + c);
-                if (code >= 0) { c += i; break; }
-            }
+            return c_number;
         }
-        if (code >= 0) {
-            t->c = c;
-            return code;
+
+        byte ch1 = p[c];
+        byte ch2 = (SIZE(p) - c > 1) ? p[c + 1] : 0;
+        t->c = c + 2;
+        switch (ch1) {
+          case '\'':
+            t->c = read_literal_string(t, c + 1);
+            return c_literalstring;
+          case '!':
+            if (ch2 == '=') return c_ne;             // !=
+            break;
+          case '$':
+            --t->c;
+            return c_dollar;                         // $
+          case '(':
+            --t->c;
+            return c_bra;                            // (
+          case ')':
+            --t->c;
+            return c_ket;                            // )
+          case '*':
+            if (ch2 == '=') return c_multiplyassign; // *=
+            --t->c;
+            return c_multiply;                       // *
+          case '+':
+            if (ch2 == '=') return c_plusassign;     // +=
+            --t->c;
+            return c_plus;                           // +
+          case '-':
+            if (ch2 == '=') return c_minusassign;    // -=
+            if (ch2 == '>') return c_sliceto;        // ->
+            --t->c;
+            return c_minus;                          // -
+          case '/':
+            if (ch2 == '/') return c_comment1;       // //
+            if (ch2 == '*') return c_comment2;       // /*
+            if (ch2 == '=') return c_divideassign;   // /=
+            --t->c;
+            return c_divide;                         // /
+          case '<':
+            if (ch2 == '-') return c_slicefrom;      // <-
+            if (ch2 == '=') return c_le;             // <=
+            if (ch2 == '+') {                        // <+
+                fprintf(stderr,
+                        "%s:%d: warning: `<+` is a legacy feature - "
+                        "use `insert` instead\n",
+                        t->file, t->line_number);
+                return c_insert;
+            }
+            --t->c;
+            return c_lt;                             // <
+          case '=':
+            if (ch2 == '=') return c_eq;             // ==
+            if (ch2 == '>') return c_assignto;       // =>
+            --t->c;
+            return c_assign;                         // =
+          case '>':
+            if (ch2 == '=') return c_ge;             // >=
+            --t->c;
+            return c_gt;                             // >
+          case '?':
+            --t->c;
+            return c_debug;                          // ?
+          case '[':
+            --t->c;
+            return c_leftslice;                      // [
+          case ']':
+            --t->c;
+            return c_rightslice;                     // ]
         }
+
         error(t, "'", p + c, 1, "' unknown");
         c++;
         continue;
@@ -285,15 +350,13 @@ static int next_real_char(struct tokeniser * t) {
 static void read_chars(struct tokeniser * t) {
     int ch = next_real_char(t);
     if (ch < 0) { error2(t, "stringdef"); return; }
-    {
-        int c0 = t->c-1;
-        while (true) {
-            ch = next_char(t);
-            if (white_space(t, ch) || ch < 0) break;
-        }
-        SIZE(t->s) = 0;
-        t->s = add_s_to_s(t->s, (const char*)t->p + c0, t->c - c0 - 1);
+    int c0 = t->c-1;
+    while (true) {
+        ch = next_char(t);
+        if (white_space(t, ch) || ch < 0) break;
     }
+    SET_SIZE(t->s, 0);
+    t->s = add_slen_to_s(t->s, (const char*)t->p + c0, t->c - c0 - 1);
 }
 
 static int decimal_to_num(int ch) {
@@ -313,53 +376,53 @@ static void convert_numeric_string(struct tokeniser * t, symbol * p, int base) {
     while (true) {
         while (c < SIZE(p) && p[c] == ' ') c++;
         if (c == SIZE(p)) break;
-        {
-            int number = 0;
-            while (c != SIZE(p)) {
-                int ch = p[c];
-                if (ch == ' ') break;
-                if (base == 10) {
-                    ch = decimal_to_num(ch);
-                    if (ch < 0) {
-                        error1(t, "decimal string contains non-digits");
-                        return;
-                    }
-                } else {
-                    ch = hex_to_num(ch);
-                    if (ch < 0) {
-                        error1(t, "hex string contains non-hex characters");
-                        return;
-                    }
-                }
-                number = base * number + ch;
-                c++;
-            }
-            if (t->encoding == ENC_SINGLEBYTE) {
-                if (number < 0 || number > 0xff) {
-                    error1(t, "character values exceed 256");
+
+        int number = 0;
+        while (c != SIZE(p)) {
+            int ch = p[c];
+            if (ch == ' ') break;
+            if (base == 10) {
+                ch = decimal_to_num(ch);
+                if (ch < 0) {
+                    error1(t, "decimal string contains non-digits");
                     return;
                 }
             } else {
-                if (number < 0 || number > 0xffff) {
-                    error1(t, "character values exceed 64K");
+                ch = hex_to_num(ch);
+                if (ch < 0) {
+                    error1(t, "hex string contains non-hex characters");
                     return;
                 }
             }
-            if (t->encoding == ENC_UTF8)
-                d += put_utf8(number, p + d);
-            else
-                p[d++] = number;
+            number = base * number + ch;
+            c++;
         }
+        if (t->encoding == ENC_SINGLEBYTE) {
+            if (number < 0 || number > 0xff) {
+                error1(t, "character values exceed 256");
+                return;
+            }
+        } else {
+            if (number < 0 || number > 0xffff) {
+                error1(t, "character values exceed 64K");
+                return;
+            }
+        }
+        if (t->encoding == ENC_UTF8)
+            d += put_utf8(number, p + d);
+        else
+            p[d++] = number;
     }
-    SIZE(p) = d;
+    SET_SIZE(p, d);
 }
 
 extern int read_token(struct tokeniser * t) {
-    byte * p = t->p;
-    int held = t->token_held;
-    t->token_held = false;
-    if (held) return t->token;
+    if (t->token_held) {
+        t->token_held = false;
+        return t->token;
+    }
     t->token_reported_as_unexpected = false;
+    byte * p = t->p;
     while (true) {
         int code = next_token(t);
         switch (code) {
@@ -407,83 +470,105 @@ extern int read_token(struct tokeniser * t) {
                 int base = 0;
                 read_chars(t);
                 code = read_token(t);
-                if (code == c_hex) { base = 16; code = read_token(t); } else
-                if (code == c_decimal) { base = 10; code = read_token(t); }
+                if (code == c_hex) {
+                    // We use `hex` to define U+xxxx stringdefs for single-byte
+                    // character sets so suppress the warning there, e.g.:
+                    //
+                    //   stringdef U+02D9  hex 'FF'
+                    if (!(t->s[0] == 'U' && t->s[1] == '+')) {
+                        fprintf(stderr,
+                                "%s:%d: warning: `hex` is a legacy feature - "
+                                "use {U+1234} notation instead\n",
+                                t->file, t->line_number);
+                    }
+                    base = 16;
+                    code = read_token(t);
+                } else if (code == c_decimal) {
+                    fprintf(stderr,
+                            "%s:%d: warning: `decimal` is a legacy feature - "
+                            "use {U+1234} notation instead\n",
+                            t->file, t->line_number);
+                    base = 10;
+                    code = read_token(t);
+                }
                 if (code != c_literalstring) {
                     error1(t, "string omitted after stringdef");
                     continue;
                 }
                 if (base > 0) convert_numeric_string(t, t->b, base);
-                {   NEW(m_pair, q);
-                    q->next = t->m_pairs;
-                    q->name = copy_s(t->s);
-                    q->value = copy_b(t->b);
-                    t->m_pairs = q;
-                    if (t->uplusmode != UPLUS_DEFINED &&
-                        (SIZE(t->s) >= 3 && t->s[0] == 'U' && t->s[1] == '+')) {
-                        if (t->uplusmode == UPLUS_UNICODE) {
-                            error1(t, "U+xxxx already used with implicit meaning");
-                        } else {
-                            t->uplusmode = UPLUS_DEFINED;
-                        }
+
+                NEW(m_pair, q);
+                q->next = t->m_pairs;
+                q->name = copy_s(t->s);
+                q->value = copy_b(t->b);
+                t->m_pairs = q;
+                if (t->uplusmode != UPLUS_DEFINED &&
+                    (SIZE(t->s) >= 3 && t->s[0] == 'U' && t->s[1] == '+')) {
+                    if (t->uplusmode == UPLUS_UNICODE) {
+                        error1(t, "U+xxxx already used with implicit meaning");
+                    } else {
+                        t->uplusmode = UPLUS_DEFINED;
                     }
                 }
                 continue;
             }
-            case c_get:
+            case c_get: {
                 code = read_token(t);
                 if (code != c_literalstring) {
-                    error1(t, "string omitted after get"); continue;
+                    error1(t, "string omitted after get");
+                    continue;
                 }
                 t->get_depth++;
                 if (t->get_depth > 10) {
                     error1(t, "get directives go 10 deep. Looping?");
                     exit(1);
                 }
-                {
-                    NEW(input, q);
-                    char * file = b_to_sz(t->b);
-                    int file_owned = 1;
-                    byte * u = get_input(file);
-                    if (u == NULL) {
-                        struct include * r;
-                        for (r = t->includes; r; r = r->next) {
-                            byte * s = copy_s(r->s);
-                            s = add_sz_to_s(s, file);
-                            s[SIZE(s)] = 0;
-                            if (file_owned > 0) {
-                                free(file);
-                            } else {
-                                lose_s((byte *)file);
-                            }
-                            file = (char*)s;
-                            file_owned = -1;
-                            u = get_input(file);
-                            if (u != NULL) break;
+
+                NEW(input, q);
+                char * file = b_to_sz(t->b);
+                int file_owned = 1;
+                byte * u = get_input(file);
+                if (u == NULL) {
+                    struct include * r;
+                    for (r = t->includes; r; r = r->next) {
+                        byte * s = copy_s(r->s);
+                        s = add_sz_to_s(s, file);
+                        s[SIZE(s)] = 0;
+                        if (file_owned > 0) {
+                            free(file);
+                        } else {
+                            lose_s((byte *)file);
                         }
+                        file = (char*)s;
+                        file_owned = -1;
+                        u = get_input(file);
+                        if (u != NULL) break;
                     }
-                    if (u == NULL) {
-                        error(t, "Can't get '", (byte *)file, strlen(file), "'");
-                        exit(1);
-                    }
-                    memmove(q, t, sizeof(struct input));
-                    t->next = q;
-                    t->p = u;
-                    t->c = 0;
-                    t->file = file;
-                    t->file_owned = file_owned;
-                    t->line_number = 1;
                 }
+                if (u == NULL) {
+                    error(t, "Can't get '", (byte *)file, strlen(file), "'");
+                    exit(1);
+                }
+                memmove(q, t, sizeof(struct input));
+                t->next = q;
+                t->p = u;
+                t->c = 0;
+                t->file = file;
+                t->file_owned = file_owned;
+                t->line_number = 1;
+
                 p = t->p;
                 continue;
+            }
             case -1:
                 if (t->next) {
                     lose_s(p);
-                    {
-                        struct input * q = t->next;
-                        memmove(t, q, sizeof(struct input)); p = t->p;
-                        FREE(q);
-                    }
+
+                    struct input * q = t->next;
+                    memmove(t, q, sizeof(struct input));
+                    p = t->p;
+                    FREE(q);
+
                     t->get_depth--;
                     continue;
                 }
@@ -503,28 +588,55 @@ extern int peek_token(struct tokeniser * t) {
 }
 
 extern const char * name_of_token(int code) {
-    int i;
-    for (i = 1; i < vocab->code; i++)
-        if ((vocab + i)->code == code) return (const char *)(vocab + i)->s;
+    for (int i = 0; i < NUM_ALPHA_TOKENS; i++) {
+        if (alpha_tokens[i].code == code)
+            return alpha_tokens[i].s;
+    }
     switch (code) {
-        case c_mathassign:   return "=";
-        case c_name:         return "name";
-        case c_number:       return "number";
-        case c_literalstring:return "literal";
-        case c_neg:          return "neg";
-        case c_grouping:     return "grouping";
-        case c_call:         return "call";
-        case c_booltest:     return "Boolean test";
-        case c_functionend:  return "Function end";
-        case c_goto_grouping:
-                             return "goto grouping";
-        case c_gopast_grouping:
-                             return "gopast grouping";
-        case c_goto_non:     return "goto non";
-        case c_gopast_non:   return "gopast non";
-        case -2:             return "start of text";
-        case -1:             return "end of text";
-        default:             return "?";
+        case c_dollar:          return "$";
+        case c_bra:             return "(";
+        case c_ket:             return ")";
+        case c_multiply:        return "*";
+        case c_plus:            return "+";
+        case c_minus:           return "-";
+        case c_divide:          return "/";
+        case c_lt:              return "<";
+        case c_stringassign:
+        case c_assign:          return "=";
+        case c_gt:              return ">";
+        case c_debug:           return "?";
+        case c_leftslice:       return "[";
+        case c_rightslice:      return "]";
+        case c_ne:              return "!=";
+        case c_multiplyassign:  return "*=";
+        case c_plusassign:      return "+=";
+        case c_minusassign:     return "-=";
+        case c_sliceto:         return "->";
+        case c_comment2:        return "/*";
+        case c_comment1:        return "//";
+        case c_divideassign:    return "/=";
+        case c_insert:          return "<+";
+        case c_slicefrom:       return "<-";
+        case c_le:              return "<=";
+        case c_eq:              return "==";
+        case c_assignto:        return "=>";
+        case c_ge:              return ">=";
+        case c_name:            return "name";
+        case c_number:          return "number";
+        case c_literalstring:   return "literal";
+        case c_neg:             return "neg";
+        case c_grouping:        return "grouping";
+        case c_call:            return "call";
+        case c_booltest:        return "Boolean test";
+        case c_functionend:     return "Function end";
+        case c_goto_grouping:   return "goto grouping";
+        case c_gopast_grouping: return "gopast grouping";
+        case c_goto_non:        return "goto non";
+        case c_gopast_non:      return "gopast non";
+        case c_not_booltest:    return "Inverted boolean test";
+        case -2:                return "start of text";
+        case -1:                return "end of text";
+        default:                return "?";
     }
 }
 
@@ -534,24 +646,16 @@ extern void disable_token(struct tokeniser * t, int code) {
 
 extern struct tokeniser * create_tokeniser(byte * p, char * file) {
     NEW(tokeniser, t);
-    t->next = NULL;
+    *t = (struct tokeniser){0};
     t->p = p;
-    t->c = 0;
     t->file = file;
-    t->file_owned = 0;
     t->line_number = 1;
     t->b = create_b(0);
     t->s = create_s(0);
     t->m_start = -1;
-    t->m_pairs = NULL;
-    t->get_depth = 0;
-    t->error_count = 0;
-    t->token_held = false;
-    t->token_reported_as_unexpected = false;
     t->token = -2;
     t->previous_token = -2;
     t->uplusmode = UPLUS_NONE;
-    memset(t->token_disabled, 0, sizeof(t->token_disabled));
     return t;
 }