functionalscript 0.44.0 → 0.46.0

package/fjs/js/tokenizer/module.f.mjs

@@ -1,20 +1,28 @@
 /**
  * JavaScript tokenizer built as a range-map state machine over code points,
  * producing tokens for keywords, identifiers, punctuators, comments, strings,
- * and numeric literals (including `BigFloat`).
+ * and numeric literals.
+ *
+ * Numeric scanning is lexeme-first: a `number` token carries the exact source
+ * text and no derived numeric value, so tokenization stays bounded by the
+ * input and never fails because a coefficient or exponent is too large for a
+ * runtime numeric type.
  *
  * @module
+ *
+ * @import { Reduce, Scan, StateScan } from '../../types/function/operator/types.ts'
+ * @import { RangeMerge } from '../../types/range_map/types.ts'
+ * @import { List } from '../../types/list/types.ts'
+ * @import { Entry } from '../../types/ordered_map/types.ts'
+ * @import { Range as NumberRange } from '../../types/range/types.ts'
+ * @import { JsToken, TokenMetadata, JsTokenWithMetadata, _TokenizerStateWithMetadata, _TokenizerState, _ErrorMessage, _InitialState, _ParseIdState, _ParseWhitespaceState, _ParseNewLineState, _ParseStringState, _ParseEscapeCharState, _ParseOperatorState, _ParseCommentState, _ParseUnicodeCharState, _ParseNumberState, _InvalidNumberState, _EofState, _CharCodeOrEof, _ToToken, _CreateToToken, _RangeFunc, _RangeMapToToken, TriviaKind, } from './types.ts'
  */
 
-/** @import { Scan, StateScan } from '../../types/function/operator/types.ts' */
 import { strictEqual } from '../../types/function/operator/module.f.mjs'
-/** @import { RangeMapArray, RangeMerge } from '../../types/range_map/types.ts' */
 import { merge, fromRange, get } from '../../types/range_map/module.f.mjs'
-/** @import { List } from '../../types/list/types.ts' */
 import { empty, stateScan, flat, toArray, reduce as listReduce, scan, map as listMap } from '../../types/list/module.f.mjs'
+import { keywords } from '../keywords/module.f.mjs'
 import { at, fromEntries } from '../../types/ordered_map/module.f.mjs'
-/** @import { Entry } from '../../types/ordered_map/types.ts' */
-/** @import { Range as NumberRange } from '../../types/range/types.ts' */
 import { one } from '../../types/range/module.f.mjs'
 import {
     range,
@@ -51,8 +59,9 @@ import {
     digit0,
     colon,
     //
+    hexDigitValue,
+    //
     latinCapitalLetterRange,
-    latinCapitalLetterA,
     latinCapitalLetterE,
     //
     leftSquareBracket,
@@ -61,7 +70,6 @@ import {
     lowLine,
     //
     latinSmallLetterRange,
-    latinSmallLetterA,
     latinSmallLetterB,
     latinSmallLetterE,
     latinSmallLetterF,
@@ -74,42 +82,7 @@ import {
     rightCurlyBracket,
     dollarSign
 }  from '../../text/ascii/module.f.mjs'
-import { todo, assertEq } from '../../asserts/module.f.mjs'
-/** @import {
-    StringToken,
-    NumberToken,
-    BigIntToken,
-    ErrorToken,
-    WhitespaceToken,
-    NewLineToken,
-    IdToken,
-    CommentToken,
-    EofToken,
-    JsToken,
-    TokenMetadata,
-    JsTokenWithMetadata,
-    _TokenizerStateWithMetadata,
-    _TokenizerState,
-    _ErrorMessage,
-    _InitialState,
-    _ParseIdState,
-    _ParseWhitespaceState,
-    _ParseNewLineState,
-    _ParseStringState,
-    _ParseEscapeCharState,
-    _ParseOperatorState,
-    _ParseCommentState,
-    _ParseUnicodeCharState,
-    _ParseNumberState,
-    _ParseNumberBuffer,
-    _InvalidNumberState,
-    _EofState,
-    _CharCodeOrEof,
-    _ToToken,
-    _CreateToToken,
-    _RangeFunc,
-    _RangeMapToToken,
-} from './types.ts' */
+import { todo, assertEq, assertStructurallySame } from '../../asserts/module.f.mjs'
 
 const { fromCharCode } = String
 
@@ -150,9 +123,6 @@ const rangeSetTerminalForNumber = [
     one(tilde),
 ]
 
-const rangeSmallAF = range('af')
-const rangeCapitalAF = range('AF')
-
 const rangeIdStart = [
     latinSmallLetterRange,
     latinCapitalLetterRange,
@@ -192,16 +162,19 @@ const rangeId = [digitRange, ...rangeIdStart]
 const appendChar = old => input => `${old}${fromCharCode(input)}`
 
 /**
- * @template T
- * @param {_CreateToToken<T>} def
- * @returns {(a: _CreateToToken<T>) => (b: _CreateToToken<T>) => _CreateToToken<T>}
+ * @type {<T>(a: T) => Reduce<T>}
  */
-const union = def => a => b => {
+const unionX = def => a => b => {
     if (a === def || a === b) { return b }
     if (b === def) { return a }
     throw [a, b]
 }
 
+/**
+ * @type {<T>(a: _CreateToToken<T>) => Reduce<_CreateToToken<T>>}
+ */
+const union = unionX
+
 /**
  * @template T
  * @param {_CreateToToken<T>} def
@@ -264,83 +237,32 @@ const create = def => a => {
     return v => c => x(c)(v)(c)
 }
 
-/** @type {(digit: number) => bigint} */
-const digitToBigInt = d => BigInt(d - digit0)
-
-/** @type {(digit: number) => _ParseNumberBuffer} */
-const startNumber = digit => ({ s: 1n, m: digitToBigInt(digit), f: 0, es: 1, e: 0 })
-
-/** @type {(digit: number) => (b: _ParseNumberBuffer) => _ParseNumberBuffer} */
-const addIntDigit = digit => b => ({ ...b, m: b.m * 10n + digitToBigInt(digit) })
-
-/** @type {(digit: number) => (b: _ParseNumberBuffer) => _ParseNumberBuffer} */
-const addFracDigit = digit => b => ({ ...b, m: b.m * 10n + digitToBigInt(digit), f: b.f - 1 })
-
-/** @type {(digit: number) => (b: _ParseNumberBuffer) => _ParseNumberBuffer} */
-const addExpDigit = digit => b => ({ ...b, e: b.e * 10 + digit - digit0 })
-
-/** @type {(s: _ParseNumberState) => JsToken} */
-const bufferToNumberToken = ({ numberKind, value, b }) => {
-    if (numberKind === 'bigint')
-        return { kind: 'bigint', value: b.s * b.m }
-    return { kind: 'number', value: value, bf: [b.s * b.m, b.f + b.es * b.e] }
-}
+/**
+ * Turns a completed numeric scanning state into its token.
+ *
+ * A `number` token carries the lexeme and nothing else — deriving a numeric
+ * value is each consumer's own policy, so no valid literal can fail to
+ * tokenize because its coefficient or exponent exceeds a runtime numeric
+ * limit. A `bigint` literal is the one case where the value *is* the token:
+ * `123n` means that bigint, so it is constructed here from the same lexeme.
+ *
+ * @type {(s: _ParseNumberState) => JsToken}
+ */
+const stateToNumberToken = ({ numberKind, value }) =>
+    numberKind === 'bigint'
+        ? { kind: 'bigint', value: BigInt(value) }
+        : { kind: 'number', value }
 
 /**
- * @link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar#keywords
+ * Derived from the one source of truth for JavaScript keywords,
+ * `fjs/js/keywords` — FunctionalScript is a strict subset of JavaScript, so
+ * the tokenizer recognizes exactly that module's `keywords`.
  */
 /** @type {List<Entry<JsToken>>} */
-const keywordEntries = [
-    ['arguments', { kind: 'arguments' }],
-    ['await', { kind: 'await' }],
-    ['break', { kind: 'break' }],
-    ['case', { kind: 'case' }],
-    ['catch', { kind: 'catch' }],
-    ['class', { kind: 'class' }],
-    ['const', { kind: 'const' }],
-    ['continue', { kind: 'continue' }],
-    ['debugger', { kind: 'debugger' }],
-    ['default', { kind: 'default' }],
-    ['delete', { kind: 'delete' }],
-    ['do', { kind: 'do' }],
-    ['else', { kind: 'else' }],
-    ['enum', { kind: 'enum' }],
-    ['eval', { kind: 'eval' }],
-    ['export', { kind: 'export' }],
-    ['extends', { kind: 'extends' }],
-    ['false', { kind: 'false' }],
-    ['finally', { kind: 'finally' }],
-    ['for', { kind: 'for' }],
-    ['function', { kind: 'function' }],
-    ['if', { kind: 'if' }],
-    ['implements', { kind: 'implements' }],
-    ['import', { kind: 'import' }],
-    ['in', { kind: 'in' }],
-    ['instanceof', { kind: 'instanceof' }],
-    ['interface', { kind: 'interface' }],
-    ['let', { kind: 'let' }],
-    ['new', { kind: 'new' }],
-    ['null', { kind: 'null' }],
-    ['package', { kind: 'package' }],
-    ['private', { kind: 'private' }],
-    ['protected', { kind: 'protected' }],
-    ['public', { kind: 'public' }],
-    ['return', { kind: 'return' }],
-    ['static', { kind: 'static' }],
-    ['super', { kind: 'super' }],
-    ['switch', { kind: 'switch' }],
-    ['this', { kind: 'this' }],
-    ['throw', { kind: 'throw' }],
-    ['true', { kind: 'true' }],
-    ['try', { kind: 'try' }],
-    ['typeof', { kind: 'typeof' }],
-    ['undefined', { kind: 'undefined' }],
-    ['var', { kind: 'var' }],
-    ['void', { kind: 'void' }],
-    ['while', { kind: 'while' }],
-    ['with', { kind: 'with' }],
-    ['yield', { kind: 'yield' }],
-]
+const keywordEntries = keywords.map(kind =>
+    // every keyword kind is a `JsToken` kind by construction: `_KeywordToken`
+    // derives its kinds from this same `keywords` list
+    [kind, /** @type {JsToken} */ ({ kind })])
 
 const keywordMap = fromEntries(keywordEntries)
 
@@ -419,20 +341,20 @@ const hasOperatorToken = op => at(op)(operatorMap) !== null
 
 /** @type {(state: _InitialState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const initialStateOp = create(
-    /** @type {_CreateToToken<_TokenizerState>} */ (state => () => [[{ kind: 'error', message: 'unexpected character' }], state])
+    state => () => [[{ kind: 'error', message: 'unexpected character' }], state]
 )([
-    rangeFunc(rangeOneNine)(/** @type {_CreateToToken<_TokenizerState>} */ (() => input => [empty, { kind: 'number', value: fromCharCode(input), b: startNumber(input), numberKind: 'int' }])),
-    rangeSetFunc(rangeIdStart)(/** @type {_CreateToToken<_TokenizerState>} */ (() => input => [empty, { kind: 'id', value: fromCharCode(input) }])),
-    rangeSetFunc(rangeSetWhiteSpace)(/** @type {_CreateToToken<_TokenizerState>} */ (() => () => [empty, { kind: 'ws' }])),
-    rangeSetFunc(rangeSetNewLine)(/** @type {_CreateToToken<_TokenizerState>} */ (() => () => [empty, { kind: 'nl' }])),
-    rangeFunc(one(quotationMark))(/** @type {_CreateToToken<_TokenizerState>} */ (() => () => [empty, { kind: 'string', value: '' }])),
-    rangeFunc(one(digit0))(/** @type {_CreateToToken<_TokenizerState>} */ (() => input => [empty, { kind: 'number', value: fromCharCode(input), b: startNumber(input), numberKind: '0' }])),
-    rangeSetFunc(rangeOpStart)(/** @type {_CreateToToken<_TokenizerState>} */ (() => input => [empty, { kind: 'op', value: fromCharCode(input) }]))
+    rangeFunc(rangeOneNine)(() => input => [empty, { kind: 'number', value: fromCharCode(input), numberKind: 'int' }]),
+    rangeSetFunc(rangeIdStart)(() => input => [empty, { kind: 'id', value: fromCharCode(input) }]),
+    rangeSetFunc(rangeSetWhiteSpace)(() => () => [empty, { kind: 'ws' }]),
+    rangeSetFunc(rangeSetNewLine)(() => () => [empty, { kind: 'nl' }]),
+    rangeFunc(one(quotationMark))(() => () => [empty, { kind: 'string', value: '' }]),
+    rangeFunc(one(digit0))(() => input => [empty, { kind: 'number', value: fromCharCode(input), numberKind: '0' }]),
+    rangeSetFunc(rangeOpStart)(() => input => [empty, { kind: 'op', value: fromCharCode(input) }])
 ])
 
 /** @type {_CreateToToken<_ParseNumberState>} */
 const invalidNumberToToken = () => input => {
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
 }
 
@@ -440,36 +362,36 @@ const invalidNumberToToken = () => input => {
 const fullStopToToken = state => input => {
     switch (state.numberKind) {
         case '0':
-        case 'int': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: state.b, numberKind: '.' }]
-        default: return tokenizeOp(input, { kind: 'invalidNumber' })
+        case 'int': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: '.' }]
+        default: return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
     }
 }
 
 /** @type {_CreateToToken<_ParseNumberState>} */
 const digit0ToToken = state => input => {
     switch (state.numberKind) {
-        case '0': return tokenizeOp(input, { kind: 'invalidNumber' })
+        case '0': return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
         case '.':
-        case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addFracDigit(input)(state.b), numberKind: 'fractional' }]
+        case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'fractional' }]
         case 'e':
         case 'e+':
         case 'e-':
-        case 'expDigits': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addExpDigit(input)(state.b), numberKind: 'expDigits' }]
-        default: return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addIntDigit(input)(state.b), numberKind: state.numberKind }]
+        case 'expDigits': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'expDigits' }]
+        default: return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: state.numberKind }]
     }
 }
 
 /** @type {_CreateToToken<_ParseNumberState>} */
 const digit19ToToken = state => input => {
     switch (state.numberKind) {
-        case '0': return tokenizeOp(input, { kind: 'invalidNumber' })
+        case '0': return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
         case '.':
-        case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addFracDigit(input)(state.b), numberKind: 'fractional' }]
+        case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'fractional' }]
         case 'e':
         case 'e+':
         case 'e-':
-        case 'expDigits': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addExpDigit(input)(state.b), numberKind: 'expDigits' }]
-        default: return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addIntDigit(input)(state.b), numberKind: 'int' }]
+        case 'expDigits': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'expDigits' }]
+        default: return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'int' }]
     }
 }
 
@@ -478,15 +400,15 @@ const expToToken = state => input => {
     switch (state.numberKind) {
         case '0':
         case 'int':
-        case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: state.b, numberKind: 'e' }]
-        default: return tokenizeOp(input, { kind: 'invalidNumber' })
+        case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e' }]
+        default: return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
     }
 }
 
 /** @type {_CreateToToken<_ParseNumberState>} */
 const hyphenMinusToToken = state => input => {
     switch (state.numberKind) {
-        case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: { ...state.b, es: -1 }, numberKind: 'e-' }]
+        case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e-' }]
         default: return terminalToToken(state)(input)
     }
 }
@@ -494,8 +416,8 @@ const hyphenMinusToToken = state => input => {
 /** @type {_CreateToToken<_ParseNumberState>} */
 const plusSignToToken = state => input => {
     switch (state.numberKind) {
-        case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: state.b, numberKind: 'e+' }]
-        default: return tokenizeOp(input, { kind: 'invalidNumber' })
+        case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), numberKind: 'e+' }]
+        default: return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
     }
 }
 
@@ -507,13 +429,13 @@ const terminalToToken = state => input => {
         case 'e+':
         case 'e-':
             {
-                const next = tokenizeOp(input, { kind: 'initial' })
+                const next = tokenizeCharCodeOp(input, { kind: 'initial' })
                 return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
             }
         default:
             {
-                const next = tokenizeOp(input, { kind: 'initial' })
-                return [{ first: bufferToNumberToken(state), tail: next[0] }, next[1]]
+                const next = tokenizeCharCodeOp(input, { kind: 'initial' })
+                return [{ first: stateToNumberToken(state), tail: next[0] }, next[1]]
             }
     }
 }
@@ -524,11 +446,11 @@ const bigintToToken = state => input => {
         case '0':
         case 'int':
             {
-                return [empty, { kind: 'number', value: state.value, b: state.b, numberKind: 'bigint' }]
+                return [empty, { kind: 'number', value: state.value, numberKind: 'bigint' }]
             }
         default:
             {
-                const next = tokenizeOp(input, { kind: 'initial' })
+                const next = tokenizeCharCodeOp(input, { kind: 'initial' })
                 return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
             }
     }
@@ -548,12 +470,12 @@ const parseNumberStateOp = create(invalidNumberToToken)([
 
 /** @type {(state: _InvalidNumberState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const invalidNumberStateOp = create(
-    /** @type {_CreateToToken<_InvalidNumberState>} */ (() => () => [empty, { kind: 'invalidNumber' }])
+    () => () => [empty, { kind: 'invalidNumber' }]
 )([
-    rangeSetFunc(rangeSetTerminalForNumber)(/** @type {_CreateToToken<_InvalidNumberState>} */ (() => input => {
-        const next = tokenizeOp(input, { kind: 'initial' })
+    rangeSetFunc(rangeSetTerminalForNumber)(() => input => {
+        const next = tokenizeCharCodeOp(input, { kind: 'initial' })
         return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
-    }))
+    })
 ])
 
 /** @type {readonly NumberRange[]} */
@@ -565,66 +487,66 @@ const rangeSetStringControl = [
 
 /** @type {(state: _ParseStringState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseStringStateOp = create(
-    /** @type {_CreateToToken<_ParseStringState>} */ (state => input => [empty, { kind: 'string', value: appendChar(state.value)(input) }])
+    state => input => [empty, { kind: 'string', value: appendChar(state.value)(input) }]
 )([
-    rangeFunc(one(quotationMark))(/** @type {_CreateToToken<_ParseStringState>} */ (state => () => [[{ kind: 'string', value: state.value }], { kind: 'initial' }])),
-    rangeFunc(one(reverseSolidus))(/** @type {_CreateToToken<_ParseStringState>} */ (state => () => [empty, { kind: 'escapeChar', value: state.value }])),
-    rangeSetFunc(rangeSetNewLine)(/** @type {_CreateToToken<_ParseStringState>} */ (() => () => [[{ kind: 'error', message: 'unterminated string literal' }], { kind: 'nl' }])),
-    rangeSetFunc(rangeSetStringControl)(/** @type {_CreateToToken<_ParseStringState>} */ (state => () => [[{ kind: 'error', message: 'unescaped control character in string' }], { kind: 'string', value: state.value }]))
+    rangeFunc(one(quotationMark))(state => () => [[{ kind: 'string', value: state.value }], { kind: 'initial' }]),
+    rangeFunc(one(reverseSolidus))(state => () => [empty, { kind: 'escapeChar', value: state.value }]),
+    rangeSetFunc(rangeSetNewLine)(() => () => [[{ kind: 'error', message: 'unterminated string literal' }], { kind: 'nl' }]),
+    rangeSetFunc(rangeSetStringControl)(state => () => [[{ kind: 'error', message: 'unescaped control character in string' }], { kind: 'string', value: state.value }])
 ])
 
 /** @type {_CreateToToken<_ParseEscapeCharState>} */
 const parseEscapeDefault = state => input => {
-    const next = tokenizeOp(input, { kind: 'string', value: state.value })
+    const next = tokenizeCharCodeOp(input, { kind: 'string', value: state.value })
     return [{ first: { kind: 'error', message: 'unescaped character' }, tail: next[0] }, next[1]]
 }
 
 /** @type {(state: _ParseEscapeCharState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseEscapeCharStateOp = create(parseEscapeDefault)([
-    rangeSetFunc([one(quotationMark), one(reverseSolidus), one(solidus)])(/** @type {_CreateToToken<_ParseEscapeCharState>} */ (state => input => [empty, { kind: 'string', value: appendChar(state.value)(input) }])),
-    rangeFunc(one(latinSmallLetterB))(/** @type {_CreateToToken<_ParseEscapeCharState>} */ (state => () => [empty, { kind: 'string', value: appendChar(state.value)(backspace) }])),
-    rangeFunc(one(latinSmallLetterF))(/** @type {_CreateToToken<_ParseEscapeCharState>} */ (state => () => [empty, { kind: 'string', value: appendChar(state.value)(ff) }])),
-    rangeFunc(one(latinSmallLetterN))(/** @type {_CreateToToken<_ParseEscapeCharState>} */ (state => () => [empty, { kind: 'string', value: appendChar(state.value)(lf) }])),
-    rangeFunc(one(latinSmallLetterR))(/** @type {_CreateToToken<_ParseEscapeCharState>} */ (state => () => [empty, { kind: 'string', value: appendChar(state.value)(cr) }])),
-    rangeFunc(one(latinSmallLetterT))(/** @type {_CreateToToken<_ParseEscapeCharState>} */ (state => () => [empty, { kind: 'string', value: appendChar(state.value)(ht) }])),
-    rangeFunc(one(latinSmallLetterU))(/** @type {_CreateToToken<_ParseEscapeCharState>} */ (state => () => [empty, { kind: 'unicodeChar', value: state.value, unicode: 0, hexIndex: 0 }])),
+    rangeSetFunc([one(quotationMark), one(reverseSolidus), one(solidus)])(state => input => [empty, { kind: 'string', value: appendChar(state.value)(input) }]),
+    rangeFunc(one(latinSmallLetterB))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(backspace) }]),
+    rangeFunc(one(latinSmallLetterF))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(ff) }]),
+    rangeFunc(one(latinSmallLetterN))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(lf) }]),
+    rangeFunc(one(latinSmallLetterR))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(cr) }]),
+    rangeFunc(one(latinSmallLetterT))(state => () => [empty, { kind: 'string', value: appendChar(state.value)(ht) }]),
+    rangeFunc(one(latinSmallLetterU))(state => () => [empty, { kind: 'unicodeChar', value: state.value, unicode: 0, hexIndex: 0 }]),
 ])
 
 /** @type {_CreateToToken<_ParseUnicodeCharState>} */
 const parseUnicodeCharDefault = state => input => {
-    const next = tokenizeOp(input, { kind: 'string', value: state.value })
+    const next = tokenizeCharCodeOp(input, { kind: 'string', value: state.value })
     return [{ first: { kind: 'error', message: 'invalid hex value' }, tail: next[0] }, next[1]]
 }
 
-/** @type {(offset: number) => _CreateToToken<_ParseUnicodeCharState>} */
-const parseUnicodeCharHex = offset => state => input => {
-    const hexValue = input - offset
+/**
+ * `hexDigitValue` classifies the code point and decodes it in one step, so this
+ * state needs no range-map dispatch: a `null` value is exactly the non-hex
+ * input the default handler rejects.
+ *
+ * @type {(state: _ParseUnicodeCharState) => (input: number) => readonly [List<JsToken>, _TokenizerState]}
+ */
+const parseUnicodeCharStateOp = state => input => {
+    const hexValue = hexDigitValue(input)
+    if (hexValue === null) { return parseUnicodeCharDefault(state)(input) }
     const newUnicode = state.unicode | (hexValue << (3 - state.hexIndex) * 4)
     return [empty, state.hexIndex === 3 ?
         { kind: 'string', value: appendChar(state.value)(newUnicode) } :
         { kind: 'unicodeChar', value: state.value, unicode: newUnicode, hexIndex: state.hexIndex + 1 }]
 }
 
-/** @type {(state: _ParseUnicodeCharState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
-const parseUnicodeCharStateOp = create(parseUnicodeCharDefault)([
-    rangeFunc(digitRange)(parseUnicodeCharHex(digit0)),
-    rangeFunc(rangeSmallAF)(parseUnicodeCharHex(latinSmallLetterA - 10)),
-    rangeFunc(rangeCapitalAF)(parseUnicodeCharHex(latinCapitalLetterA - 10))
-])
-
 /** @type {(s: string) => JsToken} */
 const idToToken = s => at(s)(keywordMap) ?? { kind: 'id', value: s }
 
 /** @type {_CreateToToken<_ParseIdState>} */
 const parseIdDefault = state => input => {
     const keyWordToken = idToToken(state.value)
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: keyWordToken, tail: next[0] }, next[1]]
 }
 
 /** @type {(state: _ParseIdState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseIdStateOp = create(parseIdDefault)([
-    rangeSetFunc(rangeId)(/** @type {_CreateToToken<_ParseIdState>} */ (state => input => [empty, { kind: 'id', value: appendChar(state.value)(input) }]))
+    rangeSetFunc(rangeId)(state => input => [empty, { kind: 'id', value: appendChar(state.value)(input) }])
 ])
 
 /** @type {(state: _ParseOperatorState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
@@ -636,7 +558,7 @@ const parseOperatorStateOp = state => input => {
         default: {
             if (hasOperatorToken(nextStateValue))
                 return [empty, { kind: 'op', value: nextStateValue }]
-            const next = tokenizeOp(input, { kind: 'initial' })
+            const next = tokenizeCharCodeOp(input, { kind: 'initial' })
             return [{ first: getOperatorToken(state.value), tail: next[0] }, next[1]]
         }
     }
@@ -644,59 +566,81 @@ const parseOperatorStateOp = state => input => {
 
 /** @type {(state: _ParseCommentState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseSinglelineCommentStateOp = create(
-    /** @type {_CreateToToken<_ParseCommentState>} */ (state => input => [empty, { ...state, value: appendChar(state.value)(input) }])
+    state => input => [empty, { ...state, value: appendChar(state.value)(input) }]
 )([
-    rangeSetFunc(rangeSetNewLine)(/** @type {_CreateToToken<_ParseCommentState>} */ (state => () => [[{ kind: '//', value: state.value }], { kind: 'nl' }]))
+    rangeSetFunc(rangeSetNewLine)(state => () => [[{ kind: '//', value: state.value }], { kind: 'nl' }])
 ])
 
 /** @type {(state: _ParseCommentState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseMultilineCommentStateOp = create(
-    /** @type {_CreateToToken<_ParseCommentState>} */ (state => input => [empty, { ...state, value: appendChar(state.value)(input) }])
+    state => input => [empty, { ...state, value: appendChar(state.value)(input) }]
 )([
-    rangeFunc(one(asterisk))(/** @type {_CreateToToken<_ParseCommentState>} */ (state => () => [empty, { ...state, kind: '/**' }])),
-    rangeSetFunc(rangeSetNewLine)(/** @type {_CreateToToken<_ParseCommentState>} */ (state => input => [empty, { ...state, value: appendChar(state.value)(input), newLine: true }])),
+    rangeFunc(one(asterisk))(state => () => [empty, { ...state, kind: '/**' }]),
+    rangeSetFunc(rangeSetNewLine)(state => input => [empty, { ...state, value: appendChar(state.value)(input), newLine: true }]),
 ])
 
 /** @type {(state: _ParseCommentState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseMultilineCommentAsteriskStateOp = create(
-    /** @type {_CreateToToken<_ParseCommentState>} */ (state => input => [empty, { ...state, kind: '/*', value: appendChar(appendChar(state.value)(asterisk))(input) }])
+    state => input => [empty, { ...state, kind: '/*', value: appendChar(appendChar(state.value)(asterisk))(input) }]
 )([
-    rangeFunc(one(asterisk))(/** @type {_CreateToToken<_ParseCommentState>} */ (state => () => [empty, { ...state, value: appendChar(state.value)(asterisk) }])),
-    rangeSetFunc(rangeSetNewLine)(/** @type {_CreateToToken<_ParseCommentState>} */ (state => input => [empty, { kind: '/*', value: appendChar(appendChar(state.value)(asterisk))(input), newLine: true }])),
-    rangeFunc(one(solidus))(/** @type {_CreateToToken<_ParseCommentState>} */ (state => () => {
+    rangeFunc(one(asterisk))(state => () => [empty, { ...state, value: appendChar(state.value)(asterisk) }]),
+    rangeSetFunc(rangeSetNewLine)(state => input => [empty, { kind: '/*', value: appendChar(appendChar(state.value)(asterisk))(input), newLine: true }]),
+    rangeFunc(one(solidus))(state => () => {
         /** @type {List<JsToken>} */
         const tokens = state.newLine ? [{ kind: '/*', value: state.value }, { kind: 'nl' }] : [{ kind: '/*', value: state.value }]
         return [tokens, { kind: 'initial' }]
-    }))
+    })
 ])
 
+/**
+ * The coalescing rule for whitespace/newline trivia: a maximal run collapses
+ * to a single token, and a run containing any newline is an `nl`. Equal kinds
+ * coalesce; `nl` absorbs `ws`.
+ *
+ * Exported because `fjs/djs/tokenizer` produces the same token stream and must
+ * agree byte for byte — its scanner reaches the same four decisions from
+ * grammar tags. This module defines `JsToken`, so the rule is stated here once
+ * rather than re-derived on each side with only the proofs to catch a drift.
+ *
+ * @type {(a: TriviaKind, b: TriviaKind) => TriviaKind}
+ */
+export const mergeTrivia = (a, b) => a === 'nl' || b === 'nl' ? 'nl' : 'ws'
+
+/**
+ * The two trivia states, shared rather than rebuilt, so a run of trivia
+ * allocates nothing per character.
+ *
+ * @type {{ readonly ws: _ParseWhitespaceState, readonly nl: _ParseNewLineState }}
+ */
+const triviaState = { ws: { kind: 'ws' }, nl: { kind: 'nl' } }
+
 /** @type {_CreateToToken<_ParseWhitespaceState>} */
 const parseWhitespaceDefault = () => input => {
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: { kind: 'ws' }, tail: next[0] }, next[1]]
 }
 
 /** @type {(state: _ParseWhitespaceState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseWhitespaceStateOp = create(parseWhitespaceDefault)([
-    rangeSetFunc(rangeSetWhiteSpace)(/** @type {_CreateToToken<_ParseWhitespaceState>} */ (state => () => [empty, state])),
-    rangeSetFunc(rangeSetNewLine)(/** @type {_CreateToToken<_ParseWhitespaceState>} */ (() => () => [empty, { kind: 'nl' }]))
+    rangeSetFunc(rangeSetWhiteSpace)(({ kind }) => () => [empty, triviaState[mergeTrivia(kind, 'ws')]]),
+    rangeSetFunc(rangeSetNewLine)(({ kind }) => () => [empty, triviaState[mergeTrivia(kind, 'nl')]])
 ])
 
 /** @type {_CreateToToken<_ParseNewLineState>} */
 const parseNewLineDefault = () => input => {
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: { kind: 'nl' }, tail: next[0] }, next[1]]
 }
 
 /** @type {(state: _ParseNewLineState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const parseNewLineStateOp = create(parseNewLineDefault)([
-    rangeSetFunc(rangeSetWhiteSpace)(/** @type {_CreateToToken<_ParseNewLineState>} */ (state => () => [empty, state])),
-    rangeSetFunc(rangeSetNewLine)(/** @type {_CreateToToken<_ParseNewLineState>} */ (state => () => [empty, state]))
+    rangeSetFunc(rangeSetWhiteSpace)(({ kind }) => () => [empty, triviaState[mergeTrivia(kind, 'ws')]]),
+    rangeSetFunc(rangeSetNewLine)(({ kind }) => () => [empty, triviaState[mergeTrivia(kind, 'nl')]])
 ])
 
 /** @type {(state: _EofState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
 const eofStateOp = create(
-    /** @type {_CreateToToken<_EofState>} */ (state => () => [[{ kind: 'error', message: 'eof' }], state])
+    state => () => [[{ kind: 'error', message: 'eof' }], state]
 )([])
 
 /** @type {StateScan<number, _TokenizerState, List<JsToken>>} */
@@ -735,7 +679,7 @@ const tokenizeEofOp = state => {
                 case 'e+':
                 case 'e-': return [[{ kind: 'error', message: 'invalid number' }, { kind: 'eof' }], { kind: 'eof', }]
             }
-            return [[bufferToNumberToken(state), { kind: 'eof' }], { kind: 'eof' }]
+            return [[stateToNumberToken(state), { kind: 'eof' }], { kind: 'eof' }]
         case 'op': return [[getOperatorToken(state.value), { kind: 'eof' }], { kind: 'eof' }]
         case '//': return [[{ kind: '//', value: state.value }, { kind: 'eof' }], { kind: 'eof' }]
         case '/*':
@@ -746,9 +690,6 @@ const tokenizeEofOp = state => {
     }
 }
 
-/** @type {StateScan<_CharCodeOrEof, _TokenizerState, List<JsToken>>} */
-const tokenizeOp = (input, state) => input === null ? tokenizeEofOp(state) : tokenizeCharCodeOp(input, state)
-
 /** @type {(metadata: TokenMetadata) => (token: JsToken) => JsTokenWithMetadata} */
 const mapTokenWithMetadata = metadata => token => { return { token, metadata } }
 
@@ -782,14 +723,23 @@ export const proof = {
         const result = getOperatorToken('@')
         assertEq(result.kind, 'error')
     },
+    // `tokenize` appends exactly one trailing `null` after its input, so the
+    // scan reaches `{ kind: 'eof' }` only on that final step — nothing ever
+    // runs tokenizeCharCodeOp/tokenizeEofOp again afterward with that state.
+    // Call each directly to cover their otherwise-unreachable `'eof'` arms.
+    tokenizeCharCodeOpAfterEof: () => {
+        const [tokens, state] = tokenizeCharCodeOp('a'.charCodeAt(0), { kind: 'eof' })
+        assertStructurallySame(toArray(tokens), [{ kind: 'error', message: 'eof' }])
+        assertStructurallySame(state, { kind: 'eof' })
+    },
+    tokenizeEofOpAfterEof: () => {
+        const [tokens, state] = tokenizeEofOp({ kind: 'eof' })
+        assertStructurallySame(toArray(tokens), [{ kind: 'error', message: 'eof' }, { kind: 'eof' }])
+        assertStructurallySame(state, { kind: 'eof' })
+    },
     throw: {
         // union throws when two distinct non-default handlers are merged for the same range;
         // this path is unreachable through the public API (no overlapping ranges in practice).
-        unionConflict: () => {
-            const def = (/** @type {undefined} */ _) => todo
-            const a = (/** @type {undefined} */ _) => todo
-            const b = (/** @type {undefined} */ _) => todo
-            union(def)(a)(b)
-        }
+        unionConflict: () => unionX(0)(1)(2)
     }
 }