functionalscript 0.44.0 → 0.46.0

package/fjs/djs/tokenizer/module.f.mjs

@@ -3,13 +3,11 @@
  *
  * @module
  *
+ * @import { Ast, AstSequence, AstTag } from '../../bnf/matcher/types.ts'
  * @import {
- *   AstRuleMeta,
- *   AstSequenceMeta,
- *   AstTag,
  *   CodePointMeta,
  *   DescentMatch,
- *   DescentMatchResult
+ *   DescentMatchResult,
  * } from '../../bnf/descent/types.ts'
  * @import { DataRule, Rule } from '../../bnf/types.ts'
  * @import {
@@ -29,11 +27,14 @@
  * @import { CodePoint } from '../../text/utf16/types.ts'
  * @import { StateScan } from '../../types/function/operator/types.ts'
  * @import { List } from '../../types/list/types.ts'
- * @import { Unknown } from '../types.ts'
  * @import { DjsToken, DjsTokenWithMetadata } from './types.ts'
+ * @import { TriviaKind } from '../../js/tokenizer/types.ts'
+ * @import { Nullable } from '../../types/nullable/types.ts'
  */
 
-import { descentParser } from '../../bnf/descent/module.f.mjs'
+import { assert, assertEq } from '../../asserts/module.f.mjs'
+import { descentParserRuleSet } from '../../bnf/descent/module.f.mjs'
+import { toData } from '../../bnf/data/module.f.mjs'
 import {
     eof,
     none,
@@ -47,19 +48,17 @@ import {
     unicodeMax,
     unicodeRange,
 } from '../../bnf/module.f.mjs'
-import { isKeywordToken } from '../../js/tokenizer/module.f.mjs'
-import { multiply } from '../../types/bigfloat/module.f.mjs'
+import { keywords } from '../../js/keywords/module.f.mjs'
+import { isKeywordToken, mergeTrivia } from '../../js/tokenizer/module.f.mjs'
 import {
     asterisk, backspace, ht, lf, ff, cr,
     quotationMark, solidus, reverseSolidus,
-    digitRange, digit0,
-    latinCapitalLetterA,
-    latinSmallLetterA, latinSmallLetterB, latinSmallLetterF,
+    hexDigitValue,
+    latinSmallLetterB, latinSmallLetterF,
     latinSmallLetterN, latinSmallLetterR, latinSmallLetterT, latinSmallLetterU,
-    range as asciiRange,
 } from '../../text/ascii/module.f.mjs'
 import { codePointListToString, stringToCodePointList } from '../../text/utf16/module.f.mjs'
-import { contains } from '../../types/range/module.f.mjs'
+import { mapUnwrap } from '../../types/nullable/module.f.mjs'
 import { concat, empty, filter, flat, flatMap, fold, map, stateScan, toArray } from '../../types/list/module.f.mjs'
 import { stringifyAsTree } from '../serializer/module.f.mjs'
 import { sort } from '../../types/object/module.f.mjs'
@@ -244,6 +243,20 @@ const buildToken = () => {
 /** @type {() => Rule} */
 export const jsGrammar = () => repeat0Plus(buildToken())
 
+/**
+ * The whole-file matcher together with the name of the rule to start it at.
+ *
+ * `toData` generates rule names, so the entry name belongs to the conversion
+ * rather than to the grammar's spelling and is read back from it here. Building
+ * both from one conversion also keeps the grammar built once per matcher.
+ *
+ * @type {<T>() => readonly [DescentMatch<T>, string]}
+ */
+export const jsMatcher = () => {
+    const [ruleSet, entry] = toData(jsGrammar())
+    return [descentParserRuleSet(ruleSet), entry]
+}
+
 const stringify = stringifyAsTree(sort)
 
 /** @type {(cp: CodePoint) => CodePointMeta<unknown>} */
@@ -275,19 +288,29 @@ const codePointsWithMetadata = path => cp => toArray(flat(stateScan(metadataScan
 
 /** @typedef {[string, TokenMetadata | null, List<number>]} _TokenScanState */
 
-/** @type {(tag: string) => boolean} */
-const isNlTag = tag => tag === '\n' || tag === '\r'
-/** @type {(tag: string) => boolean} */
-const isWsTag = tag => tag === ' ' || tag === '\t'
+/**
+ * The grammar tag of a trivia code point, as the kind `mergeTrivia` speaks in;
+ * `null` for every other tag.
+ *
+ * @type {(tag: string) => Nullable<TriviaKind>}
+ */
+const triviaKind = tag =>
+    tag === '\n' || tag === '\r' ? 'nl' :
+    tag === ' ' || tag === '\t' ? 'ws' :
+    null
 
 /** @type {StateScan<_FlatToken, _TokenScanState, List<_Token>>} */
 const scanFunc = (input, state) => {
     const [stateTag, stateMetadata, stateCodePoints] = state
     if (typeof input === 'string') {
-        if (isNlTag(input) && isNlTag(stateTag)) return [null, state]
-        if (isWsTag(input) && isWsTag(stateTag)) return [null, state]
-        if (isNlTag(input) && isWsTag(stateTag)) return [null, [input, null, []]]
-        if (isWsTag(input) && isNlTag(stateTag)) return [null, state]
+        // A trivia run continues: `mergeTrivia` decides the run's kind, and
+        // the pending token only has to be restarted under the incoming tag
+        // when that kind is not the one it already has (ws followed by nl).
+        const inputKind = triviaKind(input)
+        const stateKind = triviaKind(stateTag)
+        if (inputKind !== null && stateKind !== null) {
+            return [null, mergeTrivia(stateKind, inputKind) === stateKind ? state : [input, null, []]]
+        }
         /** @type {_TokenScanState} */
         const newState = [input, null, []]
         if (stateTag === '') {
@@ -303,7 +326,7 @@ const scanFunc = (input, state) => {
 }
 
 // All operator tag strings produced by the grammar's operator rule
-const operatorTags = /** @type {ReadonlySet<string>} */ (new Set([
+const operatorTags = new Set([
     '.', '=>', '===', '==', '=', '!==', '!=', '!',
     '>>>=', '>>>', '>>=', '>>', '>=', '>',
     '<<=', '<<', '<=', '<',
@@ -312,7 +335,7 @@ const operatorTags = /** @type {ReadonlySet<string>} */ (new Set([
     '&&=', '&&', '&=', '&', '||=', '||', '|=', '|',
     '^=', '^', '~', '??=', '??', '?.', '?',
     '[', ']', '{', '}', '(', ')', ',', ':'
-]))
+])
 
 /** @type {(tk: _FlatToken) => boolean} */
 const filterFunc = tk => {
@@ -333,7 +356,13 @@ const filterFunc = tk => {
     }
 }
 
-const rangeCapitalAF = asciiRange('AF')
+/**
+ * A `\uXXXX` escape reaches the `unicode` state only after the grammar has
+ * accepted its four hex digits, so a non-hex code point here is a tokenizer
+ * bug rather than bad input — assert instead of decoding it to a garbage
+ * value, which is what the hand-rolled ternary chain used to do.
+ */
+const unwrapHexDigitValue = mapUnwrap(hexDigitValue)
 
 /** @typedef {
  *   | { readonly kind: 'normal' }
@@ -344,7 +373,17 @@ const rangeCapitalAF = asciiRange('AF')
 /** @type {StateScan<number, _StringDecodeState, List<number>>} */
 const stringDecodeScan = (cp, state) => {
     switch (state.kind) {
-        case 'escape':
+        case 'escape': {
+            // The grammar's own `escape` rule (`buildToken`'s `string`) only ever
+            // accepts `"`, `\`, `/`, `b`, `f`, `n`, `r`, `t`, or `u` right after a
+            // backslash — any other character fails to parse before a token
+            // reaches this scan at all, so narrowing to those nine is provable,
+            // not merely assumed.
+            assert(
+                cp === quotationMark || cp === reverseSolidus || cp === solidus ||
+                cp === latinSmallLetterB || cp === latinSmallLetterF || cp === latinSmallLetterN ||
+                cp === latinSmallLetterR || cp === latinSmallLetterT || cp === latinSmallLetterU,
+                cp)
             switch (cp) {
                 case quotationMark:  return [[quotationMark],  { kind: 'normal' }]  // \" → "
                 case reverseSolidus: return [[reverseSolidus], { kind: 'normal' }]  // \\ → \
@@ -354,15 +393,16 @@ const stringDecodeScan = (cp, state) => {
                 case latinSmallLetterN: return [[lf],        { kind: 'normal' }]    // \n → line feed (LF)
                 case latinSmallLetterR: return [[cr],        { kind: 'normal' }]    // \r → carriage return (CR)
                 case latinSmallLetterT: return [[ht],        { kind: 'normal' }]    // \t → horizontal tab (HT)
-                case latinSmallLetterU: return [null, { kind: 'unicode', acc: 0, count: 0 }]  // \u → start 4 hex digits
-                default:  return [[cp], { kind: 'normal' }]
+                // `\u` is the only case the assertion above leaves. It is a
+                // `default` rather than a `case latinSmallLetterU` because the
+                // ASCII constants are plain `number`s, so the switch can never
+                // be exhaustive to TypeScript and the clause would fall through
+                // into `unicode` below.
+                default: return [null, { kind: 'unicode', acc: 0, count: 0 }]  // \u → start 4 hex digits
             }
+        }
         case 'unicode': {
-            // convert hex digit char to its numeric value: '0'-'9', 'A'-'F', 'a'-'f'
-            const digit = contains(...digitRange)(cp) ? cp - digit0
-                : contains(...rangeCapitalAF)(cp) ? cp - (latinCapitalLetterA - 10)
-                : cp - (latinSmallLetterA - 10)
-            const acc = (state.acc << 4) | digit
+            const acc = (state.acc << 4) | unwrapHexDigitValue(cp)
             return state.count === 3 ? [[acc], { kind: 'normal' }] : [null, { kind: 'unicode', acc, count: state.count + 1 }]
         }
         default:
@@ -373,31 +413,8 @@ const stringDecodeScan = (cp, state) => {
 /** @type {(codePoints: readonly number[]) => string} */
 const decodeJsonString = codePoints => codePointListToString(flat(stateScan(stringDecodeScan)({ kind: 'normal' })(codePoints.slice(1, -1))))
 
-// value is grammar-validated: (0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)? — no need to re-check its shape here.
-/** @type {(value: string) => readonly [bigint, number]} */
-const decodeNumber = value => {
-    // at most one of 'e'/'E' can occur, so whichever indexOf finds it (if any) is the exponent marker
-    const lowerEIndex = value.indexOf('e')
-    const eIndex = lowerEIndex < 0 ? value.indexOf('E') : lowerEIndex
-    const mantissaText = eIndex < 0 ? value : value.slice(0, eIndex)
-    const expText = eIndex < 0 ? '' : value.slice(eIndex + 1)
-    const dotIndex = mantissaText.indexOf('.')
-    const intDigits = dotIndex < 0 ? mantissaText : mantissaText.slice(0, dotIndex)
-    const fracDigits = dotIndex < 0 ? '' : mantissaText.slice(dotIndex + 1)
-    const mantissa = BigInt(intDigits + fracDigits)
-    const exp = (expText === '' ? 0 : Number(expText)) - fracDigits.length
-    return [mantissa, exp]
-}
-
-const keywords = /** @type {ReadonlySet<string>} */ (new Set([
-    'true', 'false', 'null', 'undefined',
-    'arguments', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
-    'debugger', 'default', 'delete', 'do', 'else', 'enum', 'eval', 'export',
-    'extends', 'finally', 'for', 'function', 'if', 'implements', 'import', 'in',
-    'instanceof', 'interface', 'let', 'new', 'package', 'private', 'protected',
-    'public', 'return', 'static', 'super', 'switch', 'this', 'throw', 'try',
-    'typeof', 'var', 'void', 'while', 'with', 'yield',
-]))
+/** @type {ReadonlySet<string>} */
+const keywordSet = new Set(keywords)
 
 /** @type {(tk: _Token) => JsToken} */
 const toJsToken = tk => {
@@ -413,13 +430,13 @@ const toJsToken = tk => {
             return { kind: 'string', value: decodeJsonString(codePoints) }
         case 'id': {
             const value = codePointListToString(codePoints)
-            if (keywords.has(value)) return /** @type {JsToken} */ ({ kind: value })
+            if (keywordSet.has(value)) return /** @type {JsToken} */ ({ kind: value })
             return { kind: 'id', value }
         }
         case 'number': {
             const value = codePointListToString(codePoints)
             if (value.endsWith('n')) return { kind: 'bigint', value: BigInt(value.slice(0, -1)) }
-            return { kind: 'number', value, bf: decodeNumber(value) }
+            return { kind: 'number', value }
         }
         case 'comment':
             if (codePoints[1] === asterisk) // block comment /*...*/
@@ -453,7 +470,7 @@ const toJsTokenWithMetadata = tk => {
     return [{ token, metadata }]
 }
 
-/** @type {(value: AstRuleMeta<TokenMetadata>|CodePointMeta<TokenMetadata>) => List<_FlatToken>} */
+/** @type {(value: Ast<CodePointMeta<TokenMetadata>>|CodePointMeta<TokenMetadata>) => List<_FlatToken>} */
 const getTokensFromAstRuleOrCodePoint = value => {
     if (value instanceof Array)
         return [value]
@@ -461,7 +478,7 @@ const getTokensFromAstRuleOrCodePoint = value => {
     return getTokensFromAstRule(value)
 }
 
-/** @type {(seq: AstSequenceMeta<TokenMetadata>) => List<_FlatToken>} */
+/** @type {(seq: AstSequence<CodePointMeta<TokenMetadata>>) => List<_FlatToken>} */
 const getTokensFromAstSequence = seq => {
     return flatMap(getTokensFromAstRuleOrCodePoint)(seq)
 }
@@ -475,7 +492,7 @@ const tagToToken = tag => {
     }
 }
 
-/** @type {(ast: AstRuleMeta<TokenMetadata>) => List<_FlatToken>} */
+/** @type {(ast: Ast<CodePointMeta<TokenMetadata>>) => List<_FlatToken>} */
 const getTokensFromAstRule = ast => {
     const token = tagToToken(ast.tag)
     if (ast.sequence.length === 0)
@@ -488,11 +505,11 @@ const getTokensFromAstRule = ast => {
 export const tokenizeString = s => {
     const cp = toArray(stringToCodePointList(s))
     if (cp.length === 0) {
-        return stringify(/** @type {Unknown} */ ([{ kind: 'eof' }]))
+        return stringify([{ kind: 'eof' }])
     }
-    const m = /** @type {DescentMatch<TokenMetadata>} */ (descentParser(jsGrammar()))
+    const [m, entry] = jsMatcher()
     const cpm = codePointsWithMetadata('')(cp)
-    const { ast, success: ok, idx: len } = m('', cpm)
+    const { ast, success: ok, idx: len } = m(entry, cpm)
     if (!ok || len !== cp.length)
         return 'error'
 
@@ -505,16 +522,19 @@ export const tokenizeString = s => {
     const tokens = flat(stateScan(scanFunc)(['', null, []])(filterTokens))
     const jsTokens = concat(flatMap(toJsTokens)(tokens))([{ kind: 'eof' }])
     const result = toArray(jsTokens)
-    return stringify(/** @type {Unknown} */ (result))
+    return stringify(result)
 }
 
 // Finds `tag` in flatTokens and returns the metadata of the next code point after it.
 // numError/unterminated tags are followed by the poisoning char / EOF-hitting position
 // in the flattened AST walk order, so this pinpoints roughly where tokenization failed.
+//
+// The single call site only passes a `tag` it already confirmed via
+// `flatTokens.includes(tag)`, so `indexOf` here is never -1.
 /** @type {(tag: string, flatTokens: readonly _FlatToken[], fallback: TokenMetadata) => TokenMetadata} */
 const metadataAfterTag = (tag, flatTokens, fallback) => {
     const idx = flatTokens.indexOf(tag)
-    const found = idx < 0 ? undefined : flatTokens.slice(idx + 1).find((/** @type {_FlatToken} */ t) => t instanceof Array)
+    const found = flatTokens.slice(idx + 1).find((/** @type {_FlatToken} */ t) => t instanceof Array)
     return found === undefined ? fallback : found[1]
 }
 
@@ -525,14 +545,19 @@ export const tokenizeJs = input => path => {
     const initial = { path, line: 1, column: 1 }
     if (cp.length === 0) return [{ token: { kind: 'eof' }, metadata: initial }]
 
-    const m = /** @type {DescentMatch<TokenMetadata>} */ (descentParser(jsGrammar()))
+    const [m, entry] = jsMatcher()
     const cpm = codePointsWithMetadata(path)(cp)
-    const { ast, success: ok, idx: len } = m('', cpm)
+    const { ast, success: ok, idx: len } = m(entry, cpm)
     const finalMetadata = fold(advanceMetadata)(initial)(cp)
 
     if (!ok || len !== cp.length) {
-        const errorMetadata = len < cpm.length ? cpm[len][1] : finalMetadata
-        return [{ token: { kind: 'error', message: 'invalid token' }, metadata: errorMetadata }]
+        // `len` is always `< cpm.length` (`=== cp.length`) here: `ok` false only
+        // ever happens with `len === 0` (nothing matched at position 0), and a
+        // `len !== cp.length` failure with `ok` true is by definition a partial
+        // match (`len < cp.length`) — `repeat0Plus` never fails after
+        // successfully consuming the whole input, so there is no way to reach
+        // this branch with `len === cp.length`.
+        return [{ token: { kind: 'error', message: 'invalid token' }, metadata: cpm[len][1] }]
     }
 
     const flatTokens = toArray(getTokensFromAstRule(ast))
@@ -592,13 +617,20 @@ const parseDjsDefaultState = input => {
 
 // Folds a leading '-' into the following number/bigint token, mirroring the old
 // fjs/djs/tokenizer's minus-state exactly.
+//
+// No `case '-'` here: the underlying `js/tokenizer` always merges two adjacent
+// `-` characters into a single `'--'` token (the decrement operator), so this
+// state — entered only after a single, unmerged `-` — can never itself see
+// another `'-'`-kind input. Such an input falls through to `default`, which
+// handles it exactly like any other non-number/bigint/eof token.
 /** @type {(input: JsToken) => readonly [List<DjsToken>, _DjsScanState]} */
 const parseDjsMinusState = input => {
     switch (input.kind) {
         case 'eof': return [[{ kind: 'error', message: 'invalid token' }, { kind: 'eof' }], { kind: 'def' }]
-        case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }]
         case 'bigint': return [[{ kind: 'bigint', value: -1n * input.value }], { kind: 'def' }]
-        case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def' }]
+        // negation is lexical: the minus sign joins the lexeme, so the token
+        // stays the exact source text of the number.
+        case 'number': return [[{ kind: 'number', value: `-${input.value}` }], { kind: 'def' }]
         default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapDjsToken(input) }, { kind: 'def' }]
     }
 }
@@ -622,3 +654,18 @@ const scanDjsTokenWithMetadata = (input, state) => {
 
 /** @type {(input: List<number>) => (path: string) => List<DjsTokenWithMetadata>} */
 export const tokenize = input => path => flat(stateScan(scanDjsTokenWithMetadata)({ kind: 'def' })(tokenizeJs(input)(path)))
+
+export const proof = {
+    // `tagToToken`'s `true` arm fires only for an `AstTag` of literal `true`.
+    // `bnf/data`'s `emptyTagOf` is the only source of that value, and
+    // `bnf/descent/module.f.mjs` reads it in exactly one place (its variant
+    // arm), always wrapped in `mrFail` — a failure. A failed variant only
+    // propagates as the overall `result` when nothing in it matches, so it
+    // never survives into a successful sequence/AST; a `true` tag therefore
+    // can never appear on a successful `descentParser` match for any
+    // grammar, `jsGrammar` included, so `tokenizeJs` can't reach this arm.
+    // Call it directly to cover that branch.
+    tagToTokenTrueTag: () => {
+        assertEq(tagToToken(true), 'true')
+    },
+}

package/fjs/djs/tokenizer/proof.f.d.mts

@@ -1,4 +1,7 @@
+/**
+ */
 export declare const proof: {
+    jsGrammar: () => void;
     isValid: (() => void)[];
     tokenizer: (() => void)[];
     djs: (() => void)[];