functionalscript 0.6.8 → 0.6.10

package/json/parser/module.f.js

@@ -59,16 +59,20 @@ const isValueToken = token => {
     }
 };
 const parseValueOp = token => state => {
-    if (isValueToken(token)) {
-        return pushValue(state)(tokenToValue(token));
-    }
-    if (token.kind === '[') {
-        return startArray(state);
-    }
-    if (token.kind === '{') {
-        return startObject(state);
+    switch (token.kind) {
+        case ']':
+            if (state.status === '[,') {
+                return endArray(state);
+            }
+            return { status: 'error', message: 'unexpected token' };
+        case '[': return startArray(state);
+        case '{': return startObject(state);
+        default:
+            if (isValueToken(token)) {
+                return pushValue(state)(tokenToValue(token));
+            }
+            return { status: 'error', message: 'unexpected token' };
     }
-    return { status: 'error', message: 'unexpected token' };
 };
 const parseArrayStartOp = token => state => {
     if (isValueToken(token)) {
@@ -119,12 +123,17 @@ const parseObjectNextOp = token => state => {
     return { status: 'error', message: 'unexpected token' };
 };
 const parseObjectCommaOp = token => state => {
+    if (token.kind === '}') {
+        return endObject(state);
+    }
     if (token.kind === 'string') {
         return pushKey(state)(token.value);
     }
     return { status: 'error', message: 'unexpected token' };
 };
 const foldOp = token => state => {
+    if (token.kind === 'eof')
+        return state;
     switch (state.status) {
         case 'result': return { status: 'error', message: 'unexpected token' };
         case 'error': return { status: 'error', message: state.message };

package/json/parser/test.f.js

@@ -121,6 +121,22 @@ export default {
             if (result !== '["ok",{"a":{"b":{"c":["d"]}}}]') {
                 throw result;
             }
+        },
+        () => {
+            const tokenList = tokenizeString('[1,]');
+            const obj = parser.parse(tokenList);
+            const result = stringify(obj);
+            if (result !== '["ok",[1]]') {
+                throw result;
+            }
+        },
+        () => {
+            const tokenList = tokenizeString('{"a":1,}');
+            const obj = parser.parse(tokenList);
+            const result = stringify(obj);
+            if (result !== '["ok",{"a":1}]') {
+                throw result;
+            }
         }
     ],
     invalid: [
@@ -180,14 +196,6 @@ export default {
                 throw result;
             }
         },
-        () => {
-            const tokenList = tokenizeString('[1,]');
-            const obj = parser.parse(tokenList);
-            const result = stringify(obj);
-            if (result !== '["error","unexpected token"]') {
-                throw result;
-            }
-        },
         () => {
             const tokenList = tokenizeString('[,1]');
             const obj = parser.parse(tokenList);
@@ -268,14 +276,6 @@ export default {
                 throw result;
             }
         },
-        () => {
-            const tokenList = tokenizeString('{"1":2,}');
-            const obj = parser.parse(tokenList);
-            const result = stringify(obj);
-            if (result !== '["error","unexpected token"]') {
-                throw result;
-            }
-        },
         () => {
             const tokenList = tokenizeString('{,"1":2}');
             const obj = parser.parse(tokenList);

package/json/tokenizer/module.f.d.ts

@@ -4,5 +4,5 @@ export type JsonToken = {
     readonly kind: 'true' | 'false' | 'null';
 } | {
     readonly kind: '{' | '}' | ':' | ',' | '[' | ']';
-} | jsTokenizer.StringToken | jsTokenizer.NumberToken | jsTokenizer.ErrorToken;
+} | jsTokenizer.StringToken | jsTokenizer.NumberToken | jsTokenizer.ErrorToken | jsTokenizer.EofToken;
 export declare const tokenize: (input: list.List<number>) => list.List<JsonToken>;

package/json/tokenizer/module.f.js

@@ -16,6 +16,7 @@ const mapToken = input => {
         case 'null':
         case 'string':
         case 'number':
+        case 'eof':
         case 'error': return [input];
         case 'ws':
         case 'nl': return empty;
@@ -25,18 +26,18 @@ const mapToken = input => {
 const parseDefaultState = input => {
     if (input === null)
         return [empty, { kind: 'def' }];
-    switch (input.kind) {
+    switch (input.token.kind) {
         case '-': return [empty, { kind: '-' }];
-        default: return [mapToken(input), { kind: 'def' }];
+        default: return [mapToken(input.token), { kind: 'def' }];
     }
 };
 const parseMinusState = input => {
     if (input === null)
         return [[{ kind: 'error', message: 'invalid token' }], { kind: 'def' }];
-    switch (input.kind) {
+    switch (input.token.kind) {
         case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }];
-        case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def' }];
-        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapToken(input) }, { kind: 'def' }];
+        case 'number': return [[{ kind: 'number', bf: multiply(input.token.bf)(-1n), value: `-${input.token.value}` }], { kind: 'def' }];
+        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapToken(input.token) }, { kind: 'def' }];
     }
 };
 const scanToken = state => input => {
@@ -46,6 +47,6 @@ const scanToken = state => input => {
     }
 };
 export const tokenize = (input) => {
-    const jsTokens = jsTokenizer.tokenize(input);
+    const jsTokens = jsTokenizer.tokenize(input)('');
     return flat(stateScan(scanToken)({ kind: 'def' })(list.concat(jsTokens)([null])));
 };

package/json/tokenizer/test.f.js

@@ -10,356 +10,356 @@ const stringify = serializer.stringifyAsTree(sort);
 export default {
     json: [
         () => {
-            const result = tokenizeString('');
-            if (result.length !== 0) {
+            const result = stringify(tokenizeString(''));
+            if (result !== '[{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('{'));
-            if (result !== '[{"kind":"{"}]') {
+            if (result !== '[{"kind":"{"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('}'));
-            if (result !== '[{"kind":"}"}]') {
+            if (result !== '[{"kind":"}"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString(':'));
-            if (result !== '[{"kind":":"}]') {
+            if (result !== '[{"kind":":"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString(','));
-            if (result !== '[{"kind":","}]') {
+            if (result !== '[{"kind":","},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('['));
-            if (result !== '[{"kind":"["}]') {
+            if (result !== '[{"kind":"["},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString(']'));
-            if (result !== '[{"kind":"]"}]') {
+            if (result !== '[{"kind":"]"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('ᄑ'));
-            if (result !== '[{"kind":"error","message":"unexpected character"}]') {
+            if (result !== '[{"kind":"error","message":"unexpected character"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('{ \t\n\r}'));
-            if (result !== '[{"kind":"{"},{"kind":"}"}]') {
+            if (result !== '[{"kind":"{"},{"kind":"}"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('""'));
-            if (result !== '[{"kind":"string","value":""}]') {
+            if (result !== '[{"kind":"string","value":""},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"value"'));
-            if (result !== '[{"kind":"string","value":"value"}]') {
+            if (result !== '[{"kind":"string","value":"value"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"value'));
-            if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
+            if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"value1" "value2"'));
-            if (result !== '[{"kind":"string","value":"value1"},{"kind":"string","value":"value2"}]') {
+            if (result !== '[{"kind":"string","value":"value1"},{"kind":"string","value":"value2"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"'));
-            if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
+            if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\\\"'));
-            if (result !== '[{"kind":"string","value":"\\\\"}]') {
+            if (result !== '[{"kind":"string","value":"\\\\"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\""'));
-            if (result !== '[{"kind":"string","value":"\\""}]') {
+            if (result !== '[{"kind":"string","value":"\\""},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\/"'));
-            if (result !== '[{"kind":"string","value":"/"}]') {
+            if (result !== '[{"kind":"string","value":"/"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\x"'));
-            if (result !== '[{"kind":"error","message":"unescaped character"},{"kind":"string","value":"x"}]') {
+            if (result !== '[{"kind":"error","message":"unescaped character"},{"kind":"string","value":"x"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\'));
-            if (result !== '[{"kind":"error","message":"\\" are missing"}]') {
+            if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\b\\f\\n\\r\\t"'));
-            if (result !== '[{"kind":"string","value":"\\b\\f\\n\\r\\t"}]') {
+            if (result !== '[{"kind":"string","value":"\\b\\f\\n\\r\\t"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\u1234"'));
-            if (result !== '[{"kind":"string","value":"ሴ"}]') {
+            if (result !== '[{"kind":"string","value":"ሴ"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\uaBcDEeFf"'));
-            if (result !== '[{"kind":"string","value":"ꯍEeFf"}]') {
+            if (result !== '[{"kind":"string","value":"ꯍEeFf"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('"\\uEeFg"'));
-            if (result !== '[{"kind":"error","message":"invalid hex value"},{"kind":"string","value":"g"}]') {
+            if (result !== '[{"kind":"error","message":"invalid hex value"},{"kind":"string","value":"g"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0'));
-            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"}]') {
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('[0]'));
-            if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"}]') {
+            if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('00'));
-            if (result !== '[{"kind":"error","message":"invalid number"}]') {
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0abc,'));
-            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"error","message":"invalid token"},{"kind":","}]') {
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"error","message":"invalid token"},{"kind":","},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('123456789012345678901234567890'));
-            if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"}]') {
+            if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('{90}'));
-            if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"}]') {
+            if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('1 2'));
-            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"bf":[2n,0],"kind":"number","value":"2"}]') {
+            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0. 2'));
-            if (result !== '[{"kind":"error","message":"invalid number"},{"bf":[2n,0],"kind":"number","value":"2"}]') {
+            if (result !== '[{"kind":"error","message":"invalid number"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('10-0'));
-            if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"bf":[0n,0],"kind":"number","value":"-0"}]') {
+            if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"bf":[0n,0],"kind":"number","value":"-0"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('9a:'));
-            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"error","message":"invalid token"},{"kind":":"}]') {
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"error","message":"invalid token"},{"kind":":"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-10'));
-            if (result !== '[{"bf":[-10n,0],"kind":"number","value":"-10"}]') {
+            if (result !== '[{"bf":[-10n,0],"kind":"number","value":"-10"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-'));
-            if (result !== '[{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('--'));
-            if (result !== '[{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('---'));
-            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-0'));
-            if (result !== '[{"bf":[0n,0],"kind":"number","value":"-0"}]') {
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"-0"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-00'));
-            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-.123'));
-            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"bf":[123n,0],"kind":"number","value":"123"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0.01'));
-            if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"}]') {
+            if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-0.9'));
-            if (result !== '[{"bf":[-9n,-1],"kind":"number","value":"-0.9"}]') {
+            if (result !== '[{"bf":[-9n,-1],"kind":"number","value":"-0.9"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-0.'));
-            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-0.]'));
-            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"]"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('12.34'));
-            if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"}]') {
+            if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-12.00'));
-            if (result !== '[{"bf":[-1200n,-2],"kind":"number","value":"-12.00"}]') {
+            if (result !== '[{"bf":[-1200n,-2],"kind":"number","value":"-12.00"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-12.'));
-            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('12.]'));
-            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"}]') {
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0e1'));
-            if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"}]') {
+            if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0e+2'));
-            if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"}]') {
+            if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0e-0'));
-            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"}]') {
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('12e0000'));
-            if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"}]') {
+            if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-12e-0001'));
-            if (result !== '[{"bf":[-12n,-1],"kind":"number","value":"-12e-0001"}]') {
+            if (result !== '[{"bf":[-12n,-1],"kind":"number","value":"-12e-0001"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('-12.34e1234'));
-            if (result !== '[{"bf":[-1234n,1232],"kind":"number","value":"-12.34e1234"}]') {
+            if (result !== '[{"bf":[-1234n,1232],"kind":"number","value":"-12.34e1234"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0e'));
-            if (result !== '[{"kind":"error","message":"invalid number"}]') {
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0e-'));
-            if (result !== '[{"kind":"error","message":"invalid number"}]') {
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('1234567890n'));
-            if (result !== '[{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('0n'));
-            if (result !== '[{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('[-1234567890n]'));
-            if (result !== '[{"kind":"["},{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"kind":"]"}]') {
+            if (result !== '[{"kind":"["},{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"kind":"]"},{"kind":"eof"}]') {
                 throw result;
             }
         },
@@ -367,25 +367,25 @@ export default {
     id: [
         () => {
             const result = stringify(tokenizeString('err'));
-            if (result !== '[{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('{e}'));
-            if (result !== '[{"kind":"{"},{"kind":"error","message":"invalid token"},{"kind":"}"}]') {
+            if (result !== '[{"kind":"{"},{"kind":"error","message":"invalid token"},{"kind":"}"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('tru'));
-            if (result !== '[{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('break'));
-            if (result !== '[{"kind":"error","message":"invalid token"}]') {
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') {
                 throw result;
             }
         },
@@ -393,25 +393,25 @@ export default {
     keywords: [
         () => {
             const result = stringify(tokenizeString('true'));
-            if (result !== '[{"kind":"true"}]') {
+            if (result !== '[{"kind":"true"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('false'));
-            if (result !== '[{"kind":"false"}]') {
+            if (result !== '[{"kind":"false"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('null'));
-            if (result !== '[{"kind":"null"}]') {
+            if (result !== '[{"kind":"null"},{"kind":"eof"}]') {
                 throw result;
             }
         },
         () => {
             const result = stringify(tokenizeString('[null]'));
-            if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"}]') {
+            if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"},{"kind":"eof"}]') {
                 throw result;
             }
         },