functionalscript 0.43.1 → 0.44.0

package/fjs/djs/tokenizer/module.f.mjs

@@ -0,0 +1,624 @@
+/**
+ * Experimental DJS parser implementation.
+ *
+ * @module
+ *
+ * @import {
+ *   AstRuleMeta,
+ *   AstSequenceMeta,
+ *   AstTag,
+ *   CodePointMeta,
+ *   DescentMatch,
+ *   DescentMatchResult
+ * } from '../../bnf/descent/types.ts'
+ * @import { DataRule, Rule } from '../../bnf/types.ts'
+ * @import {
+ *   BigIntToken,
+ *   CommentToken,
+ *   EofToken,
+ *   ErrorToken,
+ *   IdToken,
+ *   JsToken,
+ *   JsTokenWithMetadata,
+ *   NewLineToken,
+ *   NumberToken,
+ *   StringToken,
+ *   TokenMetadata,
+ *   WhitespaceToken,
+ * } from '../../js/tokenizer/types.ts'
+ * @import { CodePoint } from '../../text/utf16/types.ts'
+ * @import { StateScan } from '../../types/function/operator/types.ts'
+ * @import { List } from '../../types/list/types.ts'
+ * @import { Unknown } from '../types.ts'
+ * @import { DjsToken, DjsTokenWithMetadata } from './types.ts'
+ */
+
+import { descentParser } from '../../bnf/descent/module.f.mjs'
+import {
+    eof,
+    none,
+    notSet,
+    option,
+    range,
+    remove,
+    repeat,
+    repeat0Plus,
+    set,
+    unicodeMax,
+    unicodeRange,
+} from '../../bnf/module.f.mjs'
+import { isKeywordToken } from '../../js/tokenizer/module.f.mjs'
+import { multiply } from '../../types/bigfloat/module.f.mjs'
+import {
+    asterisk, backspace, ht, lf, ff, cr,
+    quotationMark, solidus, reverseSolidus,
+    digitRange, digit0,
+    latinCapitalLetterA,
+    latinSmallLetterA, latinSmallLetterB, latinSmallLetterF,
+    latinSmallLetterN, latinSmallLetterR, latinSmallLetterT, latinSmallLetterU,
+    range as asciiRange,
+} from '../../text/ascii/module.f.mjs'
+import { codePointListToString, stringToCodePointList } from '../../text/utf16/module.f.mjs'
+import { contains } from '../../types/range/module.f.mjs'
+import { concat, empty, filter, flat, flatMap, fold, map, stateScan, toArray } from '../../types/list/module.f.mjs'
+import { stringifyAsTree } from '../serializer/module.f.mjs'
+import { sort } from '../../types/object/module.f.mjs'
+
+// Builds the single-token grammar that jsGrammar's whole-file `tokens` rule repeats.
+/** @type {() => Rule} */
+const buildToken = () => {
+
+    const onenine = range('19')
+
+    /** @type {Rule} */
+    const digit = range('09')
+
+    const string = [
+        '"',
+        repeat0Plus({
+            ...remove(range(` ${unicodeMax}`), set('"\\')),
+            escape: [
+                '\\',
+                {
+                    ...set('"\\bfnrt'),
+                    solidus: '/',
+                    u: [
+                        'u',
+                        ...repeat(4)({
+                            digit,
+                            AF: range('AF'),
+                            af: range('af'),
+                        })
+                    ],
+                }
+            ],
+        }),
+        '"'
+    ]
+
+    const digits0 = repeat0Plus(digit)
+
+    const digits = [digit, digits0]
+
+    const ws = set(' \t')
+
+    const newLine = set('\n\r')
+
+    const idStart = {
+        smallLetter: range('az'),
+        bigLetter: range('AZ'),
+        lowLine: '_',
+        dollarSign: '$'
+    }
+
+    const idChar = {
+        ...idStart,
+        digit
+    }
+
+    // '.' and 'e'/'E' always succeed once seen, tagging missing digits as `numError`,
+    // so a malformed fraction/exponent (e.g. `0.`, `0e`) fails the whole number instead
+    // of silently ending it early. The sign is matched via string-literal branches
+    // (not `set('+-')`) because a range-variant branch's tag is the matched character
+    // itself, which would collide with the '+'/'-' operator tags in filterFunc.
+    const fracPart = { withDot: ['.', { valid: digits, numError: none }], noDot: none }
+    const expPart = { withExp: [set('Ee'), option({ plus: '+', minus: '-' }), { valid: digits, numError: none }], noExp: none }
+
+    // ECMAScript disallows a NumericLiteral immediately followed by an IdentifierStart
+    // or DecimalDigit (e.g. `00`, `123abc`). Consume that character into the number and
+    // tag it `numError` instead of leaving it to silently start a new token. idChar is
+    // wrapped in a sequence so this branch's own `numError` tag survives — a variant
+    // referenced directly as another variant's branch loses the outer tag to whichever
+    // of its own branches matches.
+    const number = [
+        {
+            0: '0',
+            onenine: [onenine, digits0],
+        },
+        option({
+            bigint: 'n',
+            frac: [fracPart, expPart]
+        }),
+        { numError: [idChar], ok: none }
+    ]
+
+    const id = [idStart, repeat0Plus(idChar)]
+
+    const operator = {
+        '.': '.',
+        '=>': '=>',
+        '===': '===',
+        '==': '==',
+        '=': '=',
+        '!==': '!==',
+        '!=': '!=',
+        '!': '!',
+        '>>>=': '>>>=',
+        '>>>': '>>>',
+        '>>=': '>>=',
+        '>>': '>>',
+        '>=': '>=',
+        '>': '>',
+        '<<=': '<<=',
+        '<<': '<<',
+        '<=': '<=',
+        '<': '<',
+        '+=': '+=',
+        '++': '++',
+        '+': '+',
+        '-=': '-=',
+        '--': '--',
+        '-': '-',
+        '**=': '**=',
+        '**': '**',
+        '*=': '*=',
+        '*': '*',
+        '/=': '/=',
+        '/': '/',
+        '%=': '%=',
+        '%': '%',
+        '&&=': '&&=',
+        '&&': '&&',
+        '&=': '&=',
+        '&': '&',
+        '||=': '||=',
+        '||': '||',
+        '|=': '|=',
+        '|': '|',
+        '^=': '^=',
+        '^': '^',
+        '~': '~',
+        '??=': '??=',
+        '??': '??',
+        '?.': '?.',
+        '?': '?',
+        '[': '[',
+        ']': ']',
+        '{': '{',
+        '}': '}',
+        '(': '(',
+        ')': ')',
+        ',': ',',
+        ':': ':'
+    }
+
+    // Recursive rule: tries end (*/) first at every position so **/  → content(*) + terminator(*/).
+    // Falls back to the empty `unterminated` alternative at EOF instead of failing, so `comment`
+    // always succeeds and the descent parser never backtracks into matching '/' and '*' as
+    // separate operators. Callers detect an unterminated comment by checking for this tag.
+    /** @type {() => DataRule} */
+    const multilineContent = () => {
+        /** @type {Rule} */
+        const char = { na: notSet('*'), a: '*' }
+        /** @type {Rule} */
+        const end = ['*', '/']
+        /** @type {Rule} */
+        const more = [char, multilineContent]
+        return { end, more, unterminated: none }
+    }
+
+    const comment = ['/', {
+            // TODO: investigate why `not(commentEnd)` instead of `remove(unicodeRange, newLine)` fail tests.
+            oneline: ['/', repeat0Plus(remove(unicodeRange, newLine)), option(newLine)],
+            multiline: ['*', multilineContent]
+        }
+    ]
+
+    const token = {
+        number,
+        string,
+        id,
+        comment,
+        operator,
+        ws,
+        newLine,
+        eof
+    }
+
+    return token
+}
+
+// The whole file's token stream as one right-recursive grammar rule. Safe at any input
+// length: descentParser matches on an explicit frame stack, not the JS call stack
+// (see fjs/bnf/descent/module.f.mjs).
+/** @type {() => Rule} */
+export const jsGrammar = () => repeat0Plus(buildToken())
+
+const stringify = stringifyAsTree(sort)
+
+/** @type {(cp: CodePoint) => CodePointMeta<unknown>} */
+const mapCodePoint = cp => [cp, undefined]
+
+/** @type {(m: DescentMatch<unknown>, name: string, cp: readonly CodePoint[]) => DescentMatchResult<unknown>} */
+export const descentParserCpOnly = (m, name, cp) => {
+    const cpm = toArray(map(mapCodePoint)(cp))
+    return m(name, cpm)
+}
+
+// Advances path/line/column by one code point, mirroring fjs/js/tokenizer's tokenizeWithPositionOp.
+/** @type {(cp: number) => (metadata: TokenMetadata) => TokenMetadata} */
+const advanceMetadata = cp => metadata => cp === lf
+    ? { path: metadata.path, line: metadata.line + 1, column: 1 }
+    : { path: metadata.path, line: metadata.line, column: metadata.column + 1 }
+
+// Pairs each code point with the metadata of its position *before* it's consumed.
+/** @type {StateScan<number, TokenMetadata, readonly [CodePointMeta<TokenMetadata>]>} */
+const metadataScan = (cp, metadata) => [[[cp, metadata]], advanceMetadata(cp)(metadata)]
+
+/** @type {(path: string) => (cp: readonly number[]) => readonly CodePointMeta<TokenMetadata>[]} */
+const codePointsWithMetadata = path => cp => toArray(flat(stateScan(metadataScan)({ path, line: 1, column: 1 })(cp)))
+
+// tag, the metadata of the token's first code point, and its code points.
+/** @typedef {[string, TokenMetadata, readonly number[]]} _Token */
+
+/** @typedef {string | CodePointMeta<TokenMetadata>} _FlatToken */
+
+/** @typedef {[string, TokenMetadata | null, List<number>]} _TokenScanState */
+
+/** @type {(tag: string) => boolean} */
+const isNlTag = tag => tag === '\n' || tag === '\r'
+/** @type {(tag: string) => boolean} */
+const isWsTag = tag => tag === ' ' || tag === '\t'
+
+/** @type {StateScan<_FlatToken, _TokenScanState, List<_Token>>} */
+const scanFunc = (input, state) => {
+    const [stateTag, stateMetadata, stateCodePoints] = state
+    if (typeof input === 'string') {
+        if (isNlTag(input) && isNlTag(stateTag)) return [null, state]
+        if (isWsTag(input) && isWsTag(stateTag)) return [null, state]
+        if (isNlTag(input) && isWsTag(stateTag)) return [null, [input, null, []]]
+        if (isWsTag(input) && isNlTag(stateTag)) return [null, state]
+        /** @type {_TokenScanState} */
+        const newState = [input, null, []]
+        if (stateTag === '') {
+            return [null, newState]
+        }
+        /** @type {_Token} */
+        const tk = [stateTag, /** @type {TokenMetadata} */ (stateMetadata), toArray(stateCodePoints)]
+        return [[tk], newState]
+    }
+    const [cp, codePointMetadata] = input
+    const startMetadata = stateMetadata === null ? codePointMetadata : stateMetadata
+    return [null, [stateTag, startMetadata, concat(stateCodePoints)([cp])]]
+}
+
+// All operator tag strings produced by the grammar's operator rule
+const operatorTags = /** @type {ReadonlySet<string>} */ (new Set([
+    '.', '=>', '===', '==', '=', '!==', '!=', '!',
+    '>>>=', '>>>', '>>=', '>>', '>=', '>',
+    '<<=', '<<', '<=', '<',
+    '+=', '++', '+', '-=', '--', '-',
+    '**=', '**', '*=', '*', '/=', '/', '%=', '%', // TODO: '/' here causes multi-char line comments (e.g. //ab\n) to fall through as two '/' operator tokens because the oneline rule only consumes one non-newline character (option vs repeat0Plus). Fix the comment rule to use repeat0Plus, or make filterFunc distinguish operator-branch '/' from slash characters inside comments.
+    '&&=', '&&', '&=', '&', '||=', '||', '|=', '|',
+    '^=', '^', '~', '??=', '??', '?.', '?',
+    '[', ']', '{', '}', '(', ')', ',', ':'
+]))
+
+/** @type {(tk: _FlatToken) => boolean} */
+const filterFunc = tk => {
+    if (tk instanceof Array)
+        return true
+    switch (tk) {
+        case 'number':
+        case 'string':
+        case 'id':
+        case 'comment':
+        case '\n':
+        case '\r':
+        case ' ':
+        case '\t':
+            return true
+        default:
+            return operatorTags.has(tk)
+    }
+}
+
+const rangeCapitalAF = asciiRange('AF')
+
+/** @typedef {
+ *   | { readonly kind: 'normal' }
+ *   | { readonly kind: 'escape' }
+ *   | { readonly kind: 'unicode', readonly acc: number, readonly count: number }
+ * } _StringDecodeState */
+
+/** @type {StateScan<number, _StringDecodeState, List<number>>} */
+const stringDecodeScan = (cp, state) => {
+    switch (state.kind) {
+        case 'escape':
+            switch (cp) {
+                case quotationMark:  return [[quotationMark],  { kind: 'normal' }]  // \" → "
+                case reverseSolidus: return [[reverseSolidus], { kind: 'normal' }]  // \\ → \
+                case solidus:        return [[solidus],        { kind: 'normal' }]  // \/ → /
+                case latinSmallLetterB: return [[backspace], { kind: 'normal' }]    // \b → backspace (BS)
+                case latinSmallLetterF: return [[ff],        { kind: 'normal' }]    // \f → form feed (FF)
+                case latinSmallLetterN: return [[lf],        { kind: 'normal' }]    // \n → line feed (LF)
+                case latinSmallLetterR: return [[cr],        { kind: 'normal' }]    // \r → carriage return (CR)
+                case latinSmallLetterT: return [[ht],        { kind: 'normal' }]    // \t → horizontal tab (HT)
+                case latinSmallLetterU: return [null, { kind: 'unicode', acc: 0, count: 0 }]  // \u → start 4 hex digits
+                default:  return [[cp], { kind: 'normal' }]
+            }
+        case 'unicode': {
+            // convert hex digit char to its numeric value: '0'-'9', 'A'-'F', 'a'-'f'
+            const digit = contains(...digitRange)(cp) ? cp - digit0
+                : contains(...rangeCapitalAF)(cp) ? cp - (latinCapitalLetterA - 10)
+                : cp - (latinSmallLetterA - 10)
+            const acc = (state.acc << 4) | digit
+            return state.count === 3 ? [[acc], { kind: 'normal' }] : [null, { kind: 'unicode', acc, count: state.count + 1 }]
+        }
+        default:
+            return cp === reverseSolidus ? [null, { kind: 'escape' }] : [[cp], { kind: 'normal' }]
+    }
+}
+
+/** @type {(codePoints: readonly number[]) => string} */
+const decodeJsonString = codePoints => codePointListToString(flat(stateScan(stringDecodeScan)({ kind: 'normal' })(codePoints.slice(1, -1))))
+
+// value is grammar-validated: (0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? — no need to re-check its shape here.
+/** @type {(value: string) => readonly [bigint, number]} */
+const decodeNumber = value => {
+    // at most one of 'e'/'E' can occur, so whichever indexOf finds it (if any) is the exponent marker
+    const lowerEIndex = value.indexOf('e')
+    const eIndex = lowerEIndex < 0 ? value.indexOf('E') : lowerEIndex
+    const mantissaText = eIndex < 0 ? value : value.slice(0, eIndex)
+    const expText = eIndex < 0 ? '' : value.slice(eIndex + 1)
+    const dotIndex = mantissaText.indexOf('.')
+    const intDigits = dotIndex < 0 ? mantissaText : mantissaText.slice(0, dotIndex)
+    const fracDigits = dotIndex < 0 ? '' : mantissaText.slice(dotIndex + 1)
+    const mantissa = BigInt(intDigits + fracDigits)
+    const exp = (expText === '' ? 0 : Number(expText)) - fracDigits.length
+    return [mantissa, exp]
+}
+
+const keywords = /** @type {ReadonlySet<string>} */ (new Set([
+    'true', 'false', 'null', 'undefined',
+    'arguments', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
+    'debugger', 'default', 'delete', 'do', 'else', 'enum', 'eval', 'export',
+    'extends', 'finally', 'for', 'function', 'if', 'implements', 'import', 'in',
+    'instanceof', 'interface', 'let', 'new', 'package', 'private', 'protected',
+    'public', 'return', 'static', 'super', 'switch', 'this', 'throw', 'try',
+    'typeof', 'var', 'void', 'while', 'with', 'yield',
+]))
+
+/** @type {(tk: _Token) => JsToken} */
+const toJsToken = tk => {
+    const [tag, , codePoints] = tk
+    switch (tag) {
+        case '\n':
+        case '\r':
+            return { kind: 'nl' }
+        case ' ':
+        case '\t':
+            return { kind: 'ws' }
+        case 'string':
+            return { kind: 'string', value: decodeJsonString(codePoints) }
+        case 'id': {
+            const value = codePointListToString(codePoints)
+            if (keywords.has(value)) return /** @type {JsToken} */ ({ kind: value })
+            return { kind: 'id', value }
+        }
+        case 'number': {
+            const value = codePointListToString(codePoints)
+            if (value.endsWith('n')) return { kind: 'bigint', value: BigInt(value.slice(0, -1)) }
+            return { kind: 'number', value, bf: decodeNumber(value) }
+        }
+        case 'comment':
+            if (codePoints[1] === asterisk) // block comment /*...*/
+                return { kind: '/*', value: codePointListToString(codePoints.slice(2, -2)) }
+            return { kind: '//', value: codePointListToString(codePoints.slice(2)) }
+        default:
+            return /** @type {JsToken} */ ({ kind: tag })
+    }
+}
+
+/** @type {(tk: _Token) => List<JsToken>} */
+const toJsTokens = tk => {
+    const token = toJsToken(tk)
+    if (token.kind === '/*') {
+        const hasNl = token.value.includes('\n') || token.value.includes('\r')
+        if (hasNl) return [token, { kind: 'nl' }]
+    }
+    return [token]
+}
+
+// Same as toJsTokens, but pairs each emitted token with tk's start metadata instead of
+// discarding it — used by tokenize (the metadata-aware entry point), not tokenizeString.
+/** @type {(tk: _Token) => List<JsTokenWithMetadata>} */
+const toJsTokenWithMetadata = tk => {
+    const [, metadata] = tk
+    const token = toJsToken(tk)
+    if (token.kind === '/*') {
+        const hasNl = token.value.includes('\n') || token.value.includes('\r')
+        if (hasNl) return [{ token, metadata }, { token: { kind: 'nl' }, metadata }]
+    }
+    return [{ token, metadata }]
+}
+
+/** @type {(value: AstRuleMeta<TokenMetadata>|CodePointMeta<TokenMetadata>) => List<_FlatToken>} */
+const getTokensFromAstRuleOrCodePoint = value => {
+    if (value instanceof Array)
+        return [value]
+
+    return getTokensFromAstRule(value)
+}
+
+/** @type {(seq: AstSequenceMeta<TokenMetadata>) => List<_FlatToken>} */
+const getTokensFromAstSequence = seq => {
+    return flatMap(getTokensFromAstRuleOrCodePoint)(seq)
+}
+
+/** @type {(tag: AstTag) => _FlatToken} */
+const tagToToken = tag => {
+    switch (typeof tag) {
+        case 'string': return tag
+        case 'undefined': return 'undefined'
+        default: return 'true'
+    }
+}
+
+/** @type {(ast: AstRuleMeta<TokenMetadata>) => List<_FlatToken>} */
+const getTokensFromAstRule = ast => {
+    const token = tagToToken(ast.tag)
+    if (ast.sequence.length === 0)
+        return [token]
+
+    return { first: token, tail: getTokensFromAstSequence(ast.sequence) }
+}
+
+/** @type {(s: string) => string} */
+export const tokenizeString = s => {
+    const cp = toArray(stringToCodePointList(s))
+    if (cp.length === 0) {
+        return stringify(/** @type {Unknown} */ ([{ kind: 'eof' }]))
+    }
+    const m = /** @type {DescentMatch<TokenMetadata>} */ (descentParser(jsGrammar()))
+    const cpm = codePointsWithMetadata('')(cp)
+    const { ast, success: ok, idx: len } = m('', cpm)
+    if (!ok || len !== cp.length)
+        return 'error'
+
+    const flatTokens = toArray(getTokensFromAstRule(ast))
+    // multilineContent tags an unterminated block comment as 'unterminated', and number
+    // tags a malformed fraction/exponent or a disallowed trailing char as 'numError',
+    // rather than failing outright — detect them here instead.
+    if (flatTokens.includes('unterminated') || flatTokens.includes('numError')) return 'error'
+    const filterTokens = concat(filter(filterFunc)(flatTokens))([''])
+    const tokens = flat(stateScan(scanFunc)(['', null, []])(filterTokens))
+    const jsTokens = concat(flatMap(toJsTokens)(tokens))([{ kind: 'eof' }])
+    const result = toArray(jsTokens)
+    return stringify(/** @type {Unknown} */ (result))
+}
+
+// Finds `tag` in flatTokens and returns the metadata of the next code point after it.
+// numError/unterminated tags are followed by the poisoning char / EOF-hitting position
+// in the flattened AST walk order, so this pinpoints roughly where tokenization failed.
+/** @type {(tag: string, flatTokens: readonly _FlatToken[], fallback: TokenMetadata) => TokenMetadata} */
+const metadataAfterTag = (tag, flatTokens, fallback) => {
+    const idx = flatTokens.indexOf(tag)
+    const found = idx < 0 ? undefined : flatTokens.slice(idx + 1).find((/** @type {_FlatToken} */ t) => t instanceof Array)
+    return found === undefined ? fallback : found[1]
+}
+
+/** @type {(input: List<number>) => (path: string) => List<JsTokenWithMetadata>} */
+export const tokenizeJs = input => path => {
+    const cp = toArray(input)
+    /** @type {TokenMetadata} */
+    const initial = { path, line: 1, column: 1 }
+    if (cp.length === 0) return [{ token: { kind: 'eof' }, metadata: initial }]
+
+    const m = /** @type {DescentMatch<TokenMetadata>} */ (descentParser(jsGrammar()))
+    const cpm = codePointsWithMetadata(path)(cp)
+    const { ast, success: ok, idx: len } = m('', cpm)
+    const finalMetadata = fold(advanceMetadata)(initial)(cp)
+
+    if (!ok || len !== cp.length) {
+        const errorMetadata = len < cpm.length ? cpm[len][1] : finalMetadata
+        return [{ token: { kind: 'error', message: 'invalid token' }, metadata: errorMetadata }]
+    }
+
+    const flatTokens = toArray(getTokensFromAstRule(ast))
+    const structuralError = flatTokens.includes('unterminated') ? 'unterminated'
+        : flatTokens.includes('numError') ? 'numError'
+        : null
+    if (structuralError !== null) {
+        const errorMetadata = metadataAfterTag(structuralError, flatTokens, finalMetadata)
+        return [{ token: { kind: 'error', message: 'invalid token' }, metadata: errorMetadata }]
+    }
+
+    const filterTokens = concat(filter(filterFunc)(flatTokens))([''])
+    const tokens = flat(stateScan(scanFunc)(['', null, []])(filterTokens))
+    const withMetadata = concat(flatMap(toJsTokenWithMetadata)(tokens))
+    return withMetadata([{ token: { kind: 'eof' }, metadata: finalMetadata }])
+}
+
+/** @typedef {{ readonly kind: 'def' | '-' }} _DjsScanState */
+
+/** @type {(input: JsToken) => List<DjsToken>} */
+const mapDjsToken = input => {
+    switch (input.kind) {
+        case 'id':
+        case 'bigint':
+        case '{':
+        case '}':
+        case ':':
+        case ',':
+        case '[':
+        case ']':
+        case '.':
+        case '=':
+        case 'true':
+        case 'false':
+        case 'null':
+        case 'string':
+        case 'number':
+        case 'ws':
+        case 'nl':
+        case 'undefined':
+        case '//':
+        case '/*':
+        case 'eof':
+        case 'error': return [input]
+        default: return isKeywordToken(input) ? [{ kind: 'id', value: input.kind }] : [{ kind: 'error', message: 'invalid token' }]
+    }
+}
+
+/** @type {(input: JsToken) => readonly [List<DjsToken>, _DjsScanState]} */
+const parseDjsDefaultState = input => {
+    switch (input.kind) {
+        case 'eof': return [[{ kind: 'eof' }], { kind: 'def' }]
+        case '-': return [empty, { kind: '-' }]
+        default: return [mapDjsToken(input), { kind: 'def' }]
+    }
+}
+
+// Folds a leading '-' into the following number/bigint token, mirroring the old
+// fjs/djs/tokenizer's minus-state exactly.
+/** @type {(input: JsToken) => readonly [List<DjsToken>, _DjsScanState]} */
+const parseDjsMinusState = input => {
+    switch (input.kind) {
+        case 'eof': return [[{ kind: 'error', message: 'invalid token' }, { kind: 'eof' }], { kind: 'def' }]
+        case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }]
+        case 'bigint': return [[{ kind: 'bigint', value: -1n * input.value }], { kind: 'def' }]
+        case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def' }]
+        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapDjsToken(input) }, { kind: 'def' }]
+    }
+}
+
+/** @type {StateScan<JsToken, _DjsScanState, List<DjsToken>>} */
+const scanDjsToken = (input, state) => {
+    switch (state.kind) {
+        case '-': return parseDjsMinusState(input)
+        default: return parseDjsDefaultState(input)
+    }
+}
+
+/** @type {(metadata: TokenMetadata) => (token: DjsToken) => DjsTokenWithMetadata} */
+const mapDjsTokenWithMetadata = metadata => token => ({ token, metadata })
+
+/** @type {StateScan<JsTokenWithMetadata, _DjsScanState, List<DjsTokenWithMetadata>>} */
+const scanDjsTokenWithMetadata = (input, state) => {
+    const [djsTokens, newState] = scanDjsToken(input.token, state)
+    return [map(mapDjsTokenWithMetadata(input.metadata))(djsTokens), newState]
+}
+
+/** @type {(input: List<number>) => (path: string) => List<DjsTokenWithMetadata>} */
+export const tokenize = input => path => flat(stateScan(scanDjsTokenWithMetadata)({ kind: 'def' })(tokenizeJs(input)(path)))