functionalscript 0.37.0 → 0.38.0

package/{fs/djs/tokenizer-new → fjs/djs/tokenizer}/proof.f.js

@@ -1,160 +1,13 @@
 import { descentParser } from '../../bnf/descent/module.f.js';
-import { backspace, ht, lf, ff, cr, quotationMark, solidus, reverseSolidus, digitRange, digit0, latinCapitalLetterA, latinSmallLetterA, latinSmallLetterB, latinSmallLetterF, latinSmallLetterN, latinSmallLetterR, latinSmallLetterT, latinSmallLetterU, range, } from '../../text/ascii/module.f.js';
-import { stringToCodePointList } from '../../text/utf16/module.f.js';
-import { contains } from '../../types/range/module.f.js';
-import { concat, filter, flat, flatMap, map, stateScan, toArray } from '../../types/list/module.f.js';
-import { jsGrammar, parse } from './module.f.js';
-const mapCodePoint = (cp) => [cp, undefined];
-const descentParserCpOnly = (m, name, cp) => {
-    const cpm = toArray(map(mapCodePoint)(cp));
-    return m(name, cpm);
-};
-const tokenizeString = s => {
-    const m = descentParser(jsGrammar());
-    const cp = toArray(stringToCodePointList(s));
-    const [ast, ok, len] = descentParserCpOnly(m, '', cp);
-    if (cp.length === 0) {
-        return JSON.stringify([{ kind: 'eof' }]);
-    }
-    if (!ok)
-        return 'error';
-    if (cp.length > 0 && len !== cp.length)
-        return 'error';
-    const flatTokens = getTokensFromAstRule(ast);
-    const filterTokens = concat(filter(filterFunc)(flatTokens))(['']);
-    const tokens = flat(stateScan(scanFunc)(['', []])(filterTokens));
-    const jsTokens = concat(map(toJsToken)(tokens))([{ kind: 'eof' }]);
-    const result = toArray(filter(v => v !== null)(jsTokens));
-    //return JSON.stringify(toArray(filterTokens))
-    return JSON.stringify(result);
-};
-const isNlTag = (tag) => tag === '\n' || tag === '\r';
-const isWsTag = (tag) => tag === ' ' || tag === '\t';
-const scanFunc = (input, state) => {
-    if (typeof input === 'string') {
-        if (isNlTag(input) && isNlTag(state[0]))
-            return [null, state];
-        if (isWsTag(input) && isWsTag(state[0]))
-            return [null, state];
-        if (isNlTag(input) && isWsTag(state[0]))
-            return [null, [input, []]];
-        if (isWsTag(input) && isNlTag(state[0]))
-            return [null, state];
-        const newState = [input, []];
-        if (state[0] === '') {
-            return [null, newState];
-        }
-        const tk = [state[0], toArray(state[1])];
-        return [[tk], newState];
-    }
-    return [null, [state[0], concat(state[1])([input])]];
-};
-// All operator tag strings produced by the grammar's operator rule
-const operatorTags = new Set([
-    '.', '=>', '===', '==', '=', '!==', '!=', '!',
-    '>>>=', '>>>', '>>=', '>>', '>=', '>',
-    '<<<=', '<<<', '<<=', '<<', '<=', '<',
-    '+=', '++', '+', '-=', '--', '-',
-    '**=', '**', '*=', '*', '/=', '/', '%=', '%', // TODO: '/' here causes multi-char line comments (e.g. //ab\n) to fall through as two '/' operator tokens because the oneline rule only consumes one non-newline character (option vs repeat0Plus). Fix the comment rule to use repeat0Plus, or make filterFunc distinguish operator-branch '/' from slash characters inside comments.
-    '&&=', '&&', '&=', '&', '||=', '||', '|=', '|',
-    '^=', '^', '~', '??=', '??', '?.', '?',
-    '[', ']', '{', '}', '(', ')', ',', ':'
-]);
-const filterFunc = tk => {
-    if (typeof tk === 'number')
-        return true;
-    switch (tk) {
-        case 'number':
-        case 'string':
-        case 'id':
-        case '\n':
-        case '\r':
-        case ' ':
-        case '\t':
-            return true;
-        default:
-            return operatorTags.has(tk);
-    }
-};
-const rangeCapitalAF = range('AF');
-const stringDecodeScan = (cp, state) => {
-    switch (state.kind) {
-        case 'escape':
-            switch (cp) {
-                case quotationMark: return [[quotationMark], { kind: 'normal' }]; // \" → "
-                case reverseSolidus: return [[reverseSolidus], { kind: 'normal' }]; // \\ → \
-                case solidus: return [[solidus], { kind: 'normal' }]; // \/ → /
-                case latinSmallLetterB: return [[backspace], { kind: 'normal' }]; // \b → backspace (BS)
-                case latinSmallLetterF: return [[ff], { kind: 'normal' }]; // \f → form feed (FF)
-                case latinSmallLetterN: return [[lf], { kind: 'normal' }]; // \n → line feed (LF)
-                case latinSmallLetterR: return [[cr], { kind: 'normal' }]; // \r → carriage return (CR)
-                case latinSmallLetterT: return [[ht], { kind: 'normal' }]; // \t → horizontal tab (HT)
-                case latinSmallLetterU: return [null, { kind: 'unicode', acc: 0, count: 0 }]; // \u → start 4 hex digits
-                default: return [[cp], { kind: 'normal' }];
-            }
-        case 'unicode': {
-            // convert hex digit char to its numeric value: '0'-'9', 'A'-'F', 'a'-'f'
-            const digit = contains(digitRange)(cp) ? cp - digit0
-                : contains(rangeCapitalAF)(cp) ? cp - (latinCapitalLetterA - 10)
-                    : cp - (latinSmallLetterA - 10);
-            const acc = (state.acc << 4) | digit;
-            return state.count === 3 ? [[acc], { kind: 'normal' }] : [null, { kind: 'unicode', acc, count: state.count + 1 }];
-        }
-        default:
-            return cp === reverseSolidus ? [null, { kind: 'escape' }] : [[cp], { kind: 'normal' }];
-    }
-};
-const decodeJsonString = codePoints => String.fromCodePoint(...toArray(flat(stateScan(stringDecodeScan)({ kind: 'normal' })(codePoints.slice(1, -1)))));
-const keywords = new Set([
-    'true', 'false', 'null', 'undefined',
-    'arguments', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
-    'debugger', 'default', 'delete', 'do', 'else', 'enum', 'eval', 'export',
-    'extends', 'finally', 'for', 'function', 'if', 'implements', 'import', 'in',
-    'instanceof', 'interface', 'let', 'new', 'package', 'private', 'protected',
-    'public', 'return', 'static', 'super', 'switch', 'this', 'throw', 'try',
-    'typeof', 'var', 'void', 'while', 'with', 'yield',
-]);
-const toJsToken = tk => {
-    switch (tk[0]) {
-        case '\n':
-        case '\r':
-            return { kind: 'nl' };
-        case ' ':
-        case '\t':
-            return { kind: 'ws' };
-        case 'string':
-            return { kind: 'string', value: decodeJsonString(tk[1]) };
-        case 'id': {
-            const value = String.fromCodePoint(...tk[1]);
-            if (keywords.has(value))
-                return { kind: value };
-            return { kind: 'id', value };
-        }
-        default:
-            return { kind: tk[0] };
-    }
-};
-const getTokensFromAstRuleOrCodePoint = value => {
-    if (value instanceof Array)
-        return [value[0]];
-    return getTokensFromAstRule(value);
-};
-const getTokensFromAstSequence = seq => {
-    return flatMap(getTokensFromAstRuleOrCodePoint)(seq);
-};
-const tagToToken = tag => {
-    switch (typeof tag) {
-        case 'string': return tag;
-        case 'undefined': return 'undefined';
-        default: return 'true';
-    }
-};
-const getTokensFromAstRule = ast => {
-    const token = tagToToken(ast.tag);
-    if (ast.sequence.length === 0)
-        return [token];
-    return { first: token, tail: getTokensFromAstSequence(ast.sequence) };
-};
+import { stringToCodePointList, stringToList } from '../../text/utf16/module.f.js';
+import { toArray } from '../../types/list/module.f.js';
+import { jsGrammar, tokenizeString, descentParserCpOnly, tokenizeJs, tokenize } from './module.f.js';
+import { assert, assertEq } from '../../asserts/module.f.js';
+import { stringifyAsTree } from '../serializer/module.f.js';
+import { sort } from '../../types/object/module.f.js';
+// DjsTokenWithMetadata carries bigint fields (bf, bigint value) that JSON.stringify can't
+// serialize — reuse the same djs stringifyAsTree serializer tokenizeString uses internally.
+const stringify = stringifyAsTree(sort);
 export const proof = {
     isValid: [() => {
             const m = descentParser(jsGrammar());
@@ -162,9 +15,7 @@ export const proof = {
                 const cp = toArray(stringToCodePointList(s));
                 const mr = descentParserCpOnly(m, '', cp);
                 const success = mr[1] && mr[2] === cp.length;
-                if (success !== expected) {
-                    throw JSON.stringify([s, mr]);
-                }
+                assertEq(success, expected, JSON.stringify([s, mr]));
             };
             expect('"', false);
             expect('   true   ', true);
@@ -201,150 +52,120 @@ export const proof = {
             const cp = toArray(stringToCodePointList('tr'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== 'id')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, 'id', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('"tr"'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== 'string')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, 'string', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('56.7e+5'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== 'number')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, 'number', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('56n'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== 'number')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, 'number', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('*'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '*')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, '*', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('**'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '**')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, '**', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('=>'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '=>')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, '=>', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('=='));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '==')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assert(!(seq.tag !== '=='), JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('==='));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '===')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assert(!(seq.tag !== '==='), JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('='));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '=')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, '=', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList(' '));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== ' ')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, ' ', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('\n'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '\n')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, '\n', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('/\n'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== '/')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, '/', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('//\n'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== 'comment')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, 'comment', JSON.stringify(mr));
         },
         () => {
             const m = descentParser(jsGrammar());
             const cp = toArray(stringToCodePointList('/*1*/'));
             const mr = descentParserCpOnly(m, '', cp);
             const seq = mr[0].sequence[0];
-            if (seq instanceof Array)
-                throw JSON.stringify(mr);
-            if (seq.tag !== 'comment')
-                throw JSON.stringify(mr);
+            assert(!(seq instanceof Array), JSON.stringify(mr));
+            assertEq(seq.tag, 'comment', JSON.stringify(mr));
         }
     ],
     djs: [
@@ -392,9 +213,7 @@ export const proof = {
         },
         () => {
             const result = tokenizeString('ᄑ');
-            if (result !== 'error') {
-                throw result;
-            }
+            assertEq(result, 'error');
         },
         () => {
             const result = tokenizeString('{ \t\n\r}');
@@ -416,9 +235,7 @@ export const proof = {
         },
         () => {
             const result = tokenizeString('"value');
-            if (result !== 'error') {
-                throw result;
-            }
+            assertEq(result, 'error');
         },
         () => {
             const result = tokenizeString('"value1" "value2"');
@@ -428,9 +245,7 @@ export const proof = {
         },
         () => {
             const result = tokenizeString('"');
-            if (result !== 'error') {
-                throw result;
-            }
+            assertEq(result, 'error');
         },
         () => {
             const result = tokenizeString('"\\\\"');
@@ -452,27 +267,19 @@ export const proof = {
         },
         () => {
             const result = tokenizeString('"\\x"');
-            if (result !== 'error') {
-                throw result;
-            }
+            assertEq(result, 'error');
         },
         () => {
             const result = tokenizeString('"\\');
-            if (result !== 'error') {
-                throw result;
-            }
+            assertEq(result, 'error');
         },
         () => {
             const result = tokenizeString('"\r"');
-            if (result !== 'error') {
-                throw result;
-            }
+            assertEq(result, 'error');
         },
         () => {
             const result = tokenizeString('"\n null');
-            if (result !== 'error') {
-                throw result;
-            }
+            assertEq(result, 'error');
         },
         () => {
             const result = tokenizeString('"\\b\\f\\n\\r\\t"');
@@ -494,130 +301,166 @@ export const proof = {
         },
         () => {
             const result = tokenizeString('"\\uEeFg"');
-            if (result !== 'error') {
-                throw result;
-            }
-        },
-        //     () => {
-        //         const result = tokenizeString('0')
-        //         if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') { throw result }
-        //     },
-        // () => {
-        //     const result = tokenizeString('[0]')
-        //     if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"},{"kind":"eof"}]') { throw result }
-        // },
-        // () => {
-        //     const result = tokenizeString('00')
-        //     if (result !== 'error') { throw result }
-        // },
-        // () => {
-        //     const result = tokenizeString('0abc,')
-        //     if (result !== 'error') { throw result }
-        // },
-        //     () => {
-        //         const result = tokenizeString('123456789012345678901234567890')
-        //         if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"},{"kind":"eof"}]') { throw result }
-        //     },
-        // () => {
-        //     const result = tokenizeString('{90}')
-        //     if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"},{"kind":"eof"}]') { throw result }
-        // },
-        //     () => {
-        //         const result = tokenizeString('1 2')
-        //         if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0. 2')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('10-0')
-        //         if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('9a:')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-10')
-        //         if (result !== '[{"kind":"-"},{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-0')
-        //         if (result !== '[{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-00')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-.123')
-        //         if (result !== '[{"kind":"-"},{"kind":"."},{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0.01')
-        //         if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-0.9')
-        //         if (result !== '[{"kind":"-"},{"bf":[9n,-1],"kind":"number","value":"0.9"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-0.')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-0.]')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('12.34')
-        //         if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-12.00')
-        //         if (result !== '[{"kind":"-"},{"bf":[1200n,-2],"kind":"number","value":"12.00"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-12.')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('12.]')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0e1')
-        //         if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0e+2')
-        //         if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0e-0')
-        //         if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('12e0000')
-        //         if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-12e-0001')
-        //         if (result !== '[{"kind":"-"},{"bf":[12n,-1],"kind":"number","value":"12e-0001"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('-12.34e1234')
-        //         if (result !== '[{"kind":"-"},{"bf":[1234n,1232],"kind":"number","value":"12.34e1234"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0e')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0e-')
-        //         if (result !== 'error') { throw result }
-        //     },
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('0');
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('[0]');
+            if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('00');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('0abc,');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('123456789012345678901234567890');
+            if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('{90}');
+            if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('1 2');
+            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"ws"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('0. 2');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('10-0');
+            if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('9a:');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('-10');
+            if (result !== '[{"kind":"-"},{"bf":[10n,0],"kind":"number","value":"10"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-0');
+            if (result !== '[{"kind":"-"},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-00');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('-.123');
+            if (result !== '[{"kind":"-"},{"kind":"."},{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('0.01');
+            if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-0.9');
+            if (result !== '[{"kind":"-"},{"bf":[9n,-1],"kind":"number","value":"0.9"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-0.');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('-0.]');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('12.34');
+            if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-12.00');
+            if (result !== '[{"kind":"-"},{"bf":[1200n,-2],"kind":"number","value":"12.00"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-12.');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('12.]');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('0e1');
+            if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('0e+2');
+            if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('0e-0');
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('12e0000');
+            if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-12e-0001');
+            if (result !== '[{"kind":"-"},{"bf":[12n,-1],"kind":"number","value":"12e-0001"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('-12.34e1234');
+            if (result !== '[{"kind":"-"},{"bf":[1234n,1232],"kind":"number","value":"12.34e1234"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('0e');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('0e-');
+            assertEq(result, 'error');
+        },
         () => {
             const result = tokenizeString('ABCdef1234567890$_');
             if (result !== '[{"kind":"id","value":"ABCdef1234567890$_"},{"kind":"eof"}]') {
@@ -630,50 +473,60 @@ export const proof = {
                 throw result;
             }
         },
-        //     () => {
-        //         const result = tokenizeString('123 _123')
-        //         if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"_123"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('123 $123')
-        //         if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"$123"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('123_123')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('123$123')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('1234567890n')
-        //         if (result !== '[{"kind":"bigint","value":1234567890n},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('0n')
-        //         if (result !== '[{"kind":"bigint","value":0n},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('[-1234567890n]')
-        //         if (result !== '[{"kind":"["},{"kind":"-"},{"kind":"bigint","value":1234567890n},{"kind":"]"},{"kind":"eof"}]') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('123.456n')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('123e456n')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('1234567890na')
-        //         if (result !== 'error') { throw result }
-        //     },
-        //     () => {
-        //         const result = tokenizeString('1234567890nn')
-        //         if (result !== 'error') { throw result }
-        //     },
+        () => {
+            const result = tokenizeString('123 _123');
+            if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"_123"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('123 $123');
+            if (result !== '[{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"ws"},{"kind":"id","value":"$123"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('123_123');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('123$123');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('1234567890n');
+            if (result !== '[{"kind":"bigint","value":1234567890n},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('0n');
+            if (result !== '[{"kind":"bigint","value":0n},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('[-1234567890n]');
+            if (result !== '[{"kind":"["},{"kind":"-"},{"kind":"bigint","value":1234567890n},{"kind":"]"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('123.456n');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('123e456n');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('1234567890na');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('1234567890nn');
+            assertEq(result, 'error');
+        },
     ],
     operators: [
         () => {
@@ -694,10 +547,12 @@ export const proof = {
                 throw result;
             }
         },
-        //     () => {
-        //         const result = tokenizeString('1*2')
-        //         if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"*"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') { throw result }
-        //     },
+        () => {
+            const result = tokenizeString('1*2');
+            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"kind":"*"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
         () => {
             const result = tokenizeString('( )');
             if (result !== '[{"kind":"("},{"kind":"ws"},{"kind":")"},{"kind":"eof"}]') {
@@ -734,6 +589,14 @@ export const proof = {
                 throw result;
             }
         },
+        () => {
+            // '<<<' and '<<<=' are not JS operators; maximal munch tokenizes them
+            // as '<<' followed by '<' / '<=', matching the old tokenizer's behavior.
+            const result = tokenizeString('<<< <<<=');
+            if (result !== '[{"kind":"<<"},{"kind":"<"},{"kind":"ws"},{"kind":"<<"},{"kind":"<="},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
         () => {
             const result = tokenizeString('&& || ! ??');
             if (result !== '[{"kind":"&&"},{"kind":"ws"},{"kind":"||"},{"kind":"ws"},{"kind":"!"},{"kind":"ws"},{"kind":"??"},{"kind":"eof"}]') {
@@ -1113,37 +976,175 @@ export const proof = {
             }
         },
     ],
-    throw: {
-        parse: () => { parse(''); }
-    },
-    // comments: [
-    //     () => {
-    //         const result = tokenizeString('//singleline comment')
-    //         if (result !== '[{"kind":"//","value":"singleline comment"},{"kind":"eof"}]') { throw result }
-    //     },
-    //     () => {
-    //         const result = tokenizeString('true//singleline comment\nfalse')
-    //         if (result !== '[{"kind":"true"},{"kind":"//","value":"singleline comment"},{"kind":"nl"},{"kind":"false"},{"kind":"eof"}]') { throw result }
-    //     },
-    //     () => {
-    //         const result = tokenizeString('/* multiline comment */')
-    //         if (result !== '[{"kind":"/*","value":" multiline comment "},{"kind":"eof"}]') { throw result }
-    //     },
-    //     () => {
-    //         const result = tokenizeString('/* multiline comment *')
-    //         if (result !== 'error') { throw result }
-    //     },
-    //     () => {
-    //         const result = tokenizeString('/* multiline comment ')
-    //         if (result !== 'error') { throw result }
-    //     },
-    //     () => {
-    //         const result = tokenizeString('/* multiline comment \n * **/')
-    //         if (result !== '[{"kind":"/*","value":" multiline comment \\n * *"},{"kind":"nl"},{"kind":"eof"}]') { throw result }
-    //     },
-    //     () => {
-    //         const result = tokenizeString('/* multiline comment *\n * **/')
-    //         if (result !== '[{"kind":"/*","value":" multiline comment *\\n * *"},{"kind":"nl"},{"kind":"eof"}]') { throw result }
-    //     },
-    // ],
+    comments: [
+        () => {
+            const result = tokenizeString('//singleline comment');
+            if (result !== '[{"kind":"//","value":"singleline comment"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('true//singleline comment\nfalse');
+            if (result !== '[{"kind":"true"},{"kind":"//","value":"singleline comment"},{"kind":"nl"},{"kind":"false"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('/* multiline comment */');
+            if (result !== '[{"kind":"/*","value":" multiline comment "},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('/* multiline comment *');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('/* multiline comment ');
+            assertEq(result, 'error');
+        },
+        () => {
+            const result = tokenizeString('/* multiline comment \n * **/');
+            if (result !== '[{"kind":"/*","value":" multiline comment \\n * *"},{"kind":"nl"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+        () => {
+            const result = tokenizeString('/* multiline comment *\n * **/');
+            if (result !== '[{"kind":"/*","value":" multiline comment *\\n * *"},{"kind":"nl"},{"kind":"eof"}]') {
+                throw result;
+            }
+        },
+    ],
+    metadata: [
+        () => {
+            const result = toArray(tokenizeJs(stringToList(''))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"eof"},"metadata":{"path":"a.js","line":1,"column":1}}]');
+        },
+        () => {
+            const result = toArray(tokenizeJs(stringToList('true'))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"true"},"metadata":{"path":"a.js","line":1,"column":1}},{"token":{"kind":"eof"},"metadata":{"path":"a.js","line":1,"column":5}}]');
+        },
+        () => {
+            const result = toArray(tokenizeJs(stringToList('true false'))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"true"},"metadata":{"path":"a.js","line":1,"column":1}},{"token":{"kind":"ws"},"metadata":{"path":"a.js","line":1,"column":5}},{"token":{"kind":"false"},"metadata":{"path":"a.js","line":1,"column":6}},{"token":{"kind":"eof"},"metadata":{"path":"a.js","line":1,"column":11}}]');
+        },
+        () => {
+            // a lone \n collapses into a single nl token, but line/column still advance
+            // correctly for whatever comes after it
+            const result = toArray(tokenizeJs(stringToList('a\nb'))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"id","value":"a"},"metadata":{"path":"a.js","line":1,"column":1}},{"token":{"kind":"nl"},"metadata":{"path":"a.js","line":1,"column":2}},{"token":{"kind":"id","value":"b"},"metadata":{"path":"a.js","line":2,"column":1}},{"token":{"kind":"eof"},"metadata":{"path":"a.js","line":2,"column":2}}]');
+        },
+        () => {
+            // two consecutive newlines still collapse into one nl token, but line still
+            // advances by two for the token after them
+            const result = toArray(tokenizeJs(stringToList('a\n\nb'))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"id","value":"a"},"metadata":{"path":"a.js","line":1,"column":1}},{"token":{"kind":"nl"},"metadata":{"path":"a.js","line":1,"column":2}},{"token":{"kind":"id","value":"b"},"metadata":{"path":"a.js","line":3,"column":1}},{"token":{"kind":"eof"},"metadata":{"path":"a.js","line":3,"column":2}}]');
+        },
+        () => {
+            // position after a multi-line block comment resumes on the comment's closing line
+            const result = toArray(tokenizeJs(stringToList('/* c\n */ x'))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"/*","value":" c\\n "},"metadata":{"path":"a.js","line":1,"column":1}},{"token":{"kind":"nl"},"metadata":{"path":"a.js","line":1,"column":1}},{"token":{"kind":"ws"},"metadata":{"path":"a.js","line":2,"column":4}},{"token":{"kind":"id","value":"x"},"metadata":{"path":"a.js","line":2,"column":5}},{"token":{"kind":"eof"},"metadata":{"path":"a.js","line":2,"column":6}}]');
+        },
+        () => {
+            const result = toArray(tokenizeJs(stringToList('"unterminated'))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"error","message":"invalid token"},"metadata":{"path":"a.js","line":1,"column":1}}]');
+        },
+        () => {
+            // position points at the poisoning trailing char ('0'), not the start of input
+            const result = toArray(tokenizeJs(stringToList('00'))('a.js'));
+            assertEq(JSON.stringify(result), '[{"token":{"kind":"error","message":"invalid token"},"metadata":{"path":"a.js","line":1,"column":2}}]');
+        },
+    ],
+    // DJS-level: keyword remapping and '-'-folding on top of tokenizeJs. Doesn't re-test
+    // JS-level token shapes already covered above/elsewhere in this file.
+    djsTokenize: [
+        () => {
+            // keywords other than true/false/null/undefined become plain ids
+            const result = toArray(tokenize(stringToList('break'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":1,"line":1,"path":""},"token":{"kind":"id","value":"break"}},{"metadata":{"column":6,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            const result = toArray(tokenize(stringToList('-10'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"bf":[-10n,0],"kind":"number","value":"-10"}},{"metadata":{"column":4,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            const result = toArray(tokenize(stringToList('-0'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"bf":[0n,0],"kind":"number","value":"-0"}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            const result = toArray(tokenize(stringToList('-1234567890n'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"bigint","value":-1234567890n}},{"metadata":{"column":13,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            // '-' followed by '-': one error, stays in minus-state waiting for what follows
+            const result = toArray(tokenize(stringToList('--'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":1,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            const result = toArray(tokenize(stringToList('---'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":1,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":4,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":4,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            // dangling '-' at eof
+            const result = toArray(tokenize(stringToList('-'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            const result = toArray(tokenize(stringToList('[-1234567890n]'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":1,"line":1,"path":""},"token":{"kind":"["}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"bigint","value":-1234567890n}},{"metadata":{"column":14,"line":1,"path":""},"token":{"kind":"]"}},{"metadata":{"column":15,"line":1,"path":""},"token":{"kind":"eof"}}]');
+        },
+        () => {
+            // grammar-level tokenizer error position flows through the DJS wrapper unchanged
+            const result = toArray(tokenize(stringToList('00'))(''));
+            assertEq(stringify(result), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}}]');
+        },
+    ],
+    // Regression coverage for large inputs: both a file with many short tokens and a
+    // single very long token used to overflow the JS call stack, because fjs/bnf/descent's
+    // matcher recursed once per grammar step. It now runs on an explicit frame stack, so
+    // the whole-file jsGrammar() match is safe at any input length.
+    largeInputs: [
+        () => {
+            // many short whitespace tokens: the reported repro (`' '.repeat(5000)`)
+            const result = tokenizeString(' '.repeat(5000));
+            assertEq(result, '[{"kind":"ws"},{"kind":"eof"}]');
+        },
+        () => {
+            // many distinct short tokens, well past the ~1000-1500 token crash threshold
+            // the old whole-file recursive match hit for this shape
+            const src = Array.from({ length: 3000 }, (_, i) => `a${i % 10}`).join(' ');
+            const result = tokenizeString(src);
+            assert(result !== 'error', result);
+            const parsed = JSON.parse(result);
+            // 3000 ids + 2999 separating ws + a trailing eof token
+            assertEq(parsed.length, 3000 + 2999 + 1);
+        },
+        () => {
+            // same many-short-tokens shape through the metadata-aware entry point
+            const src = 'x '.repeat(3000);
+            const result = toArray(tokenizeJs(stringToList(src))('a.js'));
+            assertEq(result.length, 3000 * 2 + 1);
+        },
+        () => {
+            // a single long token: a 5 KB string literal — overflows unless the descent
+            // matcher itself is iterative
+            const value = 'a'.repeat(5000);
+            const result = tokenizeString(`"${value}"`);
+            assertEq(result, `[{"kind":"string","value":"${value}"},{"kind":"eof"}]`);
+        },
+        () => {
+            // a single long token via the hand-written recursive multilineContent rule:
+            // a 20 KB block comment (the size the pre-grammar tokenizer handled)
+            const value = 'x'.repeat(20000);
+            const result = tokenizeString(`/*${value}*/`);
+            assertEq(result, `[{"kind":"/*","value":"${value}"},{"kind":"eof"}]`);
+        },
+        () => {
+            // a single long identifier (repeat0Plus(idChar) inside one token match)
+            const value = 'x'.repeat(10000);
+            const result = tokenizeString(value);
+            assertEq(result, `[{"kind":"id","value":"${value}"},{"kind":"eof"}]`);
+        },
+    ],
 };