functionalscript 0.37.0 → 0.38.0

package/fjs/djs/tokenizer/module.f.js

@@ -0,0 +1,472 @@
+/**
+ * Experimental DJS parser implementation.
+ *
+ * @module
+ */
+import { descentParser } from "../../bnf/descent/module.f.js";
+import { eof, none, notSet, option, range, remove, repeat, repeat0Plus, set, unicodeMax, unicodeRange } from "../../bnf/module.f.js";
+import { isKeywordToken, } from "../../js/tokenizer/module.f.js";
+import { multiply } from "../../types/bigfloat/module.f.js";
+import { asterisk, backspace, ht, lf, ff, cr, quotationMark, solidus, reverseSolidus, digitRange, digit0, latinCapitalLetterA, latinSmallLetterA, latinSmallLetterB, latinSmallLetterF, latinSmallLetterN, latinSmallLetterR, latinSmallLetterT, latinSmallLetterU, range as asciiRange, } from "../../text/ascii/module.f.js";
+import { codePointListToString, stringToCodePointList } from "../../text/utf16/module.f.js";
+import { contains } from "../../types/range/module.f.js";
+import { concat, empty, filter, flat, flatMap, fold, map, stateScan, toArray } from "../../types/list/module.f.js";
+import { stringifyAsTree } from "../serializer/module.f.js";
+import { sort } from "../../types/object/module.f.js";
+// Builds the single-token grammar that jsGrammar's whole-file `tokens` rule repeats.
+const buildToken = () => {
+    const onenine = range('19');
+    const digit = range('09');
+    const string = [
+        '"',
+        repeat0Plus({
+            ...remove(range(` ${unicodeMax}`), set('"\\')),
+            escape: [
+                '\\',
+                {
+                    ...set('"\\bfnrt'),
+                    solidus: '/',
+                    u: [
+                        'u',
+                        ...repeat(4)({
+                            digit,
+                            AF: range('AF'),
+                            af: range('af'),
+                        })
+                    ],
+                }
+            ],
+        }),
+        '"'
+    ];
+    const digits0 = repeat0Plus(digit);
+    const digits = [digit, digits0];
+    const ws = set(' \t');
+    const newLine = set('\n\r');
+    const idStart = {
+        smallLetter: range('az'),
+        bigLetter: range('AZ'),
+        lowLine: '_',
+        dollarSign: '$'
+    };
+    const idChar = {
+        ...idStart,
+        digit
+    };
+    // '.' and 'e'/'E' always succeed once seen, tagging missing digits as `numError`,
+    // so a malformed fraction/exponent (e.g. `0.`, `0e`) fails the whole number instead
+    // of silently ending it early. The sign is matched via string-literal branches
+    // (not `set('+-')`) because a range-variant branch's tag is the matched character
+    // itself, which would collide with the '+'/'-' operator tags in filterFunc.
+    const fracPart = { withDot: ['.', { valid: digits, numError: none }], noDot: none };
+    const expPart = { withExp: [set('Ee'), option({ plus: '+', minus: '-' }), { valid: digits, numError: none }], noExp: none };
+    // ECMAScript disallows a NumericLiteral immediately followed by an IdentifierStart
+    // or DecimalDigit (e.g. `00`, `123abc`). Consume that character into the number and
+    // tag it `numError` instead of leaving it to silently start a new token. idChar is
+    // wrapped in a sequence so this branch's own `numError` tag survives — a variant
+    // referenced directly as another variant's branch loses the outer tag to whichever
+    // of its own branches matches.
+    const number = [
+        {
+            0: '0',
+            onenine: [onenine, digits0],
+        },
+        option({
+            bigint: 'n',
+            frac: [fracPart, expPart]
+        }),
+        { numError: [idChar], ok: none }
+    ];
+    const id = [idStart, repeat0Plus(idChar)];
+    const operator = {
+        '.': '.',
+        '=>': '=>',
+        '===': '===',
+        '==': '==',
+        '=': '=',
+        '!==': '!==',
+        '!=': '!=',
+        '!': '!',
+        '>>>=': '>>>=',
+        '>>>': '>>>',
+        '>>=': '>>=',
+        '>>': '>>',
+        '>=': '>=',
+        '>': '>',
+        '<<=': '<<=',
+        '<<': '<<',
+        '<=': '<=',
+        '<': '<',
+        '+=': '+=',
+        '++': '++',
+        '+': '+',
+        '-=': '-=',
+        '--': '--',
+        '-': '-',
+        '**=': '**=',
+        '**': '**',
+        '*=': '*=',
+        '*': '*',
+        '/=': '/=',
+        '/': '/',
+        '%=': '%=',
+        '%': '%',
+        '&&=': '&&=',
+        '&&': '&&',
+        '&=': '&=',
+        '&': '&',
+        '||=': '||=',
+        '||': '||',
+        '|=': '|=',
+        '|': '|',
+        '^=': '^=',
+        '^': '^',
+        '~': '~',
+        '??=': '??=',
+        '??': '??',
+        '?.': '?.',
+        '?': '?',
+        '[': '[',
+        ']': ']',
+        '{': '{',
+        '}': '}',
+        '(': '(',
+        ')': ')',
+        ',': ',',
+        ':': ':'
+    };
+    // Recursive rule: tries end (*/) first at every position so **/  → content(*) + terminator(*/).
+    // Falls back to the empty `unterminated` alternative at EOF instead of failing, so `comment`
+    // always succeeds and the descent parser never backtracks into matching '/' and '*' as
+    // separate operators. Callers detect an unterminated comment by checking for this tag.
+    const multilineContent = () => {
+        const char = { na: notSet('*'), a: '*' };
+        const end = ['*', '/'];
+        const more = [char, multilineContent];
+        return { end, more, unterminated: none };
+    };
+    const comment = ['/', {
+            // TODO: investigate why `not(commentEnd)` instead of `remove(unicodeRange, newLine)` fail tests.
+            oneline: ['/', repeat0Plus(remove(unicodeRange, newLine)), option(newLine)],
+            multiline: ['*', multilineContent]
+        }
+    ];
+    const token = {
+        number,
+        string,
+        id,
+        comment,
+        operator,
+        ws,
+        newLine,
+        eof
+    };
+    return token;
+};
+// The whole file's token stream as one right-recursive grammar rule. Safe at any input
+// length: descentParser matches on an explicit frame stack, not the JS call stack
+// (see fjs/bnf/descent/module.f.ts).
+export const jsGrammar = () => repeat0Plus(buildToken());
+const stringify = stringifyAsTree(sort);
+const mapCodePoint = (cp) => [cp, undefined];
+export const descentParserCpOnly = (m, name, cp) => {
+    const cpm = toArray(map(mapCodePoint)(cp));
+    return m(name, cpm);
+};
+// Advances path/line/column by one code point, mirroring fjs/js/tokenizer's tokenizeWithPositionOp.
+const advanceMetadata = cp => metadata => cp === lf
+    ? { path: metadata.path, line: metadata.line + 1, column: 1 }
+    : { path: metadata.path, line: metadata.line, column: metadata.column + 1 };
+// Pairs each code point with the metadata of its position *before* it's consumed.
+const metadataScan = (cp, metadata) => [[[cp, metadata]], advanceMetadata(cp)(metadata)];
+const codePointsWithMetadata = path => cp => toArray(flat(stateScan(metadataScan)({ path, line: 1, column: 1 })(cp)));
+const isNlTag = (tag) => tag === '\n' || tag === '\r';
+const isWsTag = (tag) => tag === ' ' || tag === '\t';
+const scanFunc = (input, state) => {
+    const [stateTag, stateMetadata, stateCodePoints] = state;
+    if (typeof input === 'string') {
+        if (isNlTag(input) && isNlTag(stateTag))
+            return [null, state];
+        if (isWsTag(input) && isWsTag(stateTag))
+            return [null, state];
+        if (isNlTag(input) && isWsTag(stateTag))
+            return [null, [input, null, []]];
+        if (isWsTag(input) && isNlTag(stateTag))
+            return [null, state];
+        const newState = [input, null, []];
+        if (stateTag === '') {
+            return [null, newState];
+        }
+        const tk = [stateTag, stateMetadata, toArray(stateCodePoints)];
+        return [[tk], newState];
+    }
+    const [cp, codePointMetadata] = input;
+    const startMetadata = stateMetadata === null ? codePointMetadata : stateMetadata;
+    return [null, [stateTag, startMetadata, concat(stateCodePoints)([cp])]];
+};
+// All operator tag strings produced by the grammar's operator rule
+const operatorTags = new Set([
+    '.', '=>', '===', '==', '=', '!==', '!=', '!',
+    '>>>=', '>>>', '>>=', '>>', '>=', '>',
+    '<<=', '<<', '<=', '<',
+    '+=', '++', '+', '-=', '--', '-',
+    '**=', '**', '*=', '*', '/=', '/', '%=', '%', // TODO: '/' here causes multi-char line comments (e.g. //ab\n) to fall through as two '/' operator tokens because the oneline rule only consumes one non-newline character (option vs repeat0Plus). Fix the comment rule to use repeat0Plus, or make filterFunc distinguish operator-branch '/' from slash characters inside comments.
+    '&&=', '&&', '&=', '&', '||=', '||', '|=', '|',
+    '^=', '^', '~', '??=', '??', '?.', '?',
+    '[', ']', '{', '}', '(', ')', ',', ':'
+]);
+const filterFunc = tk => {
+    if (tk instanceof Array)
+        return true;
+    switch (tk) {
+        case 'number':
+        case 'string':
+        case 'id':
+        case 'comment':
+        case '\n':
+        case '\r':
+        case ' ':
+        case '\t':
+            return true;
+        default:
+            return operatorTags.has(tk);
+    }
+};
+const rangeCapitalAF = asciiRange('AF');
+const stringDecodeScan = (cp, state) => {
+    switch (state.kind) {
+        case 'escape':
+            switch (cp) {
+                case quotationMark: return [[quotationMark], { kind: 'normal' }]; // \" → "
+                case reverseSolidus: return [[reverseSolidus], { kind: 'normal' }]; // \\ → \
+                case solidus: return [[solidus], { kind: 'normal' }]; // \/ → /
+                case latinSmallLetterB: return [[backspace], { kind: 'normal' }]; // \b → backspace (BS)
+                case latinSmallLetterF: return [[ff], { kind: 'normal' }]; // \f → form feed (FF)
+                case latinSmallLetterN: return [[lf], { kind: 'normal' }]; // \n → line feed (LF)
+                case latinSmallLetterR: return [[cr], { kind: 'normal' }]; // \r → carriage return (CR)
+                case latinSmallLetterT: return [[ht], { kind: 'normal' }]; // \t → horizontal tab (HT)
+                case latinSmallLetterU: return [null, { kind: 'unicode', acc: 0, count: 0 }]; // \u → start 4 hex digits
+                default: return [[cp], { kind: 'normal' }];
+            }
+        case 'unicode': {
+            // convert hex digit char to its numeric value: '0'-'9', 'A'-'F', 'a'-'f'
+            const digit = contains(...digitRange)(cp) ? cp - digit0
+                : contains(...rangeCapitalAF)(cp) ? cp - (latinCapitalLetterA - 10)
+                    : cp - (latinSmallLetterA - 10);
+            const acc = (state.acc << 4) | digit;
+            return state.count === 3 ? [[acc], { kind: 'normal' }] : [null, { kind: 'unicode', acc, count: state.count + 1 }];
+        }
+        default:
+            return cp === reverseSolidus ? [null, { kind: 'escape' }] : [[cp], { kind: 'normal' }];
+    }
+};
+const decodeJsonString = codePoints => codePointListToString(flat(stateScan(stringDecodeScan)({ kind: 'normal' })(codePoints.slice(1, -1))));
+// value is grammar-validated: (0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? — no need to re-check its shape here.
+const decodeNumber = value => {
+    // at most one of 'e'/'E' can occur, so whichever indexOf finds it (if any) is the exponent marker
+    const lowerEIndex = value.indexOf('e');
+    const eIndex = lowerEIndex < 0 ? value.indexOf('E') : lowerEIndex;
+    const mantissaText = eIndex < 0 ? value : value.slice(0, eIndex);
+    const expText = eIndex < 0 ? '' : value.slice(eIndex + 1);
+    const dotIndex = mantissaText.indexOf('.');
+    const intDigits = dotIndex < 0 ? mantissaText : mantissaText.slice(0, dotIndex);
+    const fracDigits = dotIndex < 0 ? '' : mantissaText.slice(dotIndex + 1);
+    const mantissa = BigInt(intDigits + fracDigits);
+    const exp = (expText === '' ? 0 : Number(expText)) - fracDigits.length;
+    return [mantissa, exp];
+};
+const keywords = new Set([
+    'true', 'false', 'null', 'undefined',
+    'arguments', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
+    'debugger', 'default', 'delete', 'do', 'else', 'enum', 'eval', 'export',
+    'extends', 'finally', 'for', 'function', 'if', 'implements', 'import', 'in',
+    'instanceof', 'interface', 'let', 'new', 'package', 'private', 'protected',
+    'public', 'return', 'static', 'super', 'switch', 'this', 'throw', 'try',
+    'typeof', 'var', 'void', 'while', 'with', 'yield',
+]);
+const toJsToken = tk => {
+    const [tag, , codePoints] = tk;
+    switch (tag) {
+        case '\n':
+        case '\r':
+            return { kind: 'nl' };
+        case ' ':
+        case '\t':
+            return { kind: 'ws' };
+        case 'string':
+            return { kind: 'string', value: decodeJsonString(codePoints) };
+        case 'id': {
+            const value = codePointListToString(codePoints);
+            if (keywords.has(value))
+                return { kind: value };
+            return { kind: 'id', value };
+        }
+        case 'number': {
+            const value = codePointListToString(codePoints);
+            if (value.endsWith('n'))
+                return { kind: 'bigint', value: BigInt(value.slice(0, -1)) };
+            return { kind: 'number', value, bf: decodeNumber(value) };
+        }
+        case 'comment':
+            if (codePoints[1] === asterisk) // block comment /*...*/
+                return { kind: '/*', value: codePointListToString(codePoints.slice(2, -2)) };
+            return { kind: '//', value: codePointListToString(codePoints.slice(2)) };
+        default:
+            return { kind: tag };
+    }
+};
+const toJsTokens = tk => {
+    const token = toJsToken(tk);
+    if (token.kind === '/*') {
+        const hasNl = token.value.includes('\n') || token.value.includes('\r');
+        if (hasNl)
+            return [token, { kind: 'nl' }];
+    }
+    return [token];
+};
+// Same as toJsTokens, but pairs each emitted token with tk's start metadata instead of
+// discarding it — used by tokenize (the metadata-aware entry point), not tokenizeString.
+const toJsTokenWithMetadata = tk => {
+    const [, metadata] = tk;
+    const token = toJsToken(tk);
+    if (token.kind === '/*') {
+        const hasNl = token.value.includes('\n') || token.value.includes('\r');
+        if (hasNl)
+            return [{ token, metadata }, { token: { kind: 'nl' }, metadata }];
+    }
+    return [{ token, metadata }];
+};
+const getTokensFromAstRuleOrCodePoint = value => {
+    if (value instanceof Array)
+        return [value];
+    return getTokensFromAstRule(value);
+};
+const getTokensFromAstSequence = seq => {
+    return flatMap(getTokensFromAstRuleOrCodePoint)(seq);
+};
+const tagToToken = tag => {
+    switch (typeof tag) {
+        case 'string': return tag;
+        case 'undefined': return 'undefined';
+        default: return 'true';
+    }
+};
+const getTokensFromAstRule = ast => {
+    const token = tagToToken(ast.tag);
+    if (ast.sequence.length === 0)
+        return [token];
+    return { first: token, tail: getTokensFromAstSequence(ast.sequence) };
+};
+export const tokenizeString = s => {
+    const cp = toArray(stringToCodePointList(s));
+    if (cp.length === 0) {
+        return stringify([{ kind: 'eof' }]);
+    }
+    const m = descentParser(jsGrammar());
+    const cpm = codePointsWithMetadata('')(cp);
+    const [ast, ok, len] = m('', cpm);
+    if (!ok || len !== cp.length)
+        return 'error';
+    const flatTokens = toArray(getTokensFromAstRule(ast));
+    // multilineContent tags an unterminated block comment as 'unterminated', and number
+    // tags a malformed fraction/exponent or a disallowed trailing char as 'numError',
+    // rather than failing outright — detect them here instead.
+    if (flatTokens.includes('unterminated') || flatTokens.includes('numError'))
+        return 'error';
+    const filterTokens = concat(filter(filterFunc)(flatTokens))(['']);
+    const tokens = flat(stateScan(scanFunc)(['', null, []])(filterTokens));
+    const jsTokens = concat(flatMap(toJsTokens)(tokens))([{ kind: 'eof' }]);
+    const result = toArray(jsTokens);
+    return stringify(result);
+};
+// Finds `tag` in flatTokens and returns the metadata of the next code point after it.
+// numError/unterminated tags are followed by the poisoning char / EOF-hitting position
+// in the flattened AST walk order, so this pinpoints roughly where tokenization failed.
+const metadataAfterTag = (tag, flatTokens, fallback) => {
+    const idx = flatTokens.indexOf(tag);
+    const found = idx < 0 ? undefined : flatTokens.slice(idx + 1).find((t) => t instanceof Array);
+    return found === undefined ? fallback : found[1];
+};
+export const tokenizeJs = input => path => {
+    const cp = toArray(input);
+    const initial = { path, line: 1, column: 1 };
+    if (cp.length === 0)
+        return [{ token: { kind: 'eof' }, metadata: initial }];
+    const m = descentParser(jsGrammar());
+    const cpm = codePointsWithMetadata(path)(cp);
+    const [ast, ok, len] = m('', cpm);
+    const finalMetadata = fold(advanceMetadata)(initial)(cp);
+    if (!ok || len !== cp.length) {
+        const errorMetadata = len < cpm.length ? cpm[len][1] : finalMetadata;
+        return [{ token: { kind: 'error', message: 'invalid token' }, metadata: errorMetadata }];
+    }
+    const flatTokens = toArray(getTokensFromAstRule(ast));
+    const structuralError = flatTokens.includes('unterminated') ? 'unterminated'
+        : flatTokens.includes('numError') ? 'numError'
+            : null;
+    if (structuralError !== null) {
+        const errorMetadata = metadataAfterTag(structuralError, flatTokens, finalMetadata);
+        return [{ token: { kind: 'error', message: 'invalid token' }, metadata: errorMetadata }];
+    }
+    const filterTokens = concat(filter(filterFunc)(flatTokens))(['']);
+    const tokens = flat(stateScan(scanFunc)(['', null, []])(filterTokens));
+    const withMetadata = concat(flatMap(toJsTokenWithMetadata)(tokens));
+    return withMetadata([{ token: { kind: 'eof' }, metadata: finalMetadata }]);
+};
+const mapDjsToken = input => {
+    switch (input.kind) {
+        case 'id':
+        case 'bigint':
+        case '{':
+        case '}':
+        case ':':
+        case ',':
+        case '[':
+        case ']':
+        case '.':
+        case '=':
+        case 'true':
+        case 'false':
+        case 'null':
+        case 'string':
+        case 'number':
+        case 'ws':
+        case 'nl':
+        case 'undefined':
+        case '//':
+        case '/*':
+        case 'eof':
+        case 'error': return [input];
+        default: return isKeywordToken(input) ? [{ kind: 'id', value: input.kind }] : [{ kind: 'error', message: 'invalid token' }];
+    }
+};
+const parseDjsDefaultState = input => {
+    switch (input.kind) {
+        case 'eof': return [[{ kind: 'eof' }], { kind: 'def' }];
+        case '-': return [empty, { kind: '-' }];
+        default: return [mapDjsToken(input), { kind: 'def' }];
+    }
+};
+// Folds a leading '-' into the following number/bigint token, mirroring the old
+// fjs/djs/tokenizer's minus-state exactly.
+const parseDjsMinusState = input => {
+    switch (input.kind) {
+        case 'eof': return [[{ kind: 'error', message: 'invalid token' }, { kind: 'eof' }], { kind: 'def' }];
+        case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }];
+        case 'bigint': return [[{ kind: 'bigint', value: -1n * input.value }], { kind: 'def' }];
+        case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def' }];
+        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapDjsToken(input) }, { kind: 'def' }];
+    }
+};
+const scanDjsToken = (input, state) => {
+    switch (state.kind) {
+        case '-': return parseDjsMinusState(input);
+        default: return parseDjsDefaultState(input);
+    }
+};
+const mapDjsTokenWithMetadata = metadata => token => ({ token, metadata });
+const scanDjsTokenWithMetadata = (input, state) => {
+    const [djsTokens, newState] = scanDjsToken(input.token, state);
+    return [map(mapDjsTokenWithMetadata(input.metadata))(djsTokens), newState];
+};
+export const tokenize = input => path => flat(stateScan(scanDjsTokenWithMetadata)({ kind: 'def' })(tokenizeJs(input)(path)));

package/{fs/djs/tokenizer-new → fjs/djs/tokenizer}/proof.f.d.ts

@@ -6,7 +6,8 @@ export declare const proof: {
     ws: (() => void)[];
     id: (() => void)[];
     keywords: (() => void)[];
-    throw: {
-        parse: () => void;
-    };
+    comments: (() => void)[];
+    metadata: (() => void)[];
+    djsTokenize: (() => void)[];
+    largeInputs: (() => void)[];
 };