functionalscript 0.43.0 → 0.44.0

package/fjs/media/json/serializer/module.f.mjs

@@ -0,0 +1,131 @@
+/**
+ * JSON serializer for deterministic string output.
+ *
+ * `stringSerialize` is FunctionalScript, not the host's `JSON.stringify`: it
+ * escapes over this repository's own UTF-16 decoder and reproduces the
+ * ECMAScript `QuoteJSONString` result exactly, lone surrogates included.
+ *
+ * @module
+ */
+/** @import { List } from '../../../types/list/types.ts' */
+import { flat, map, reduce, empty } from '../../../types/list/module.f.mjs'
+/** @import { Reduce } from '../../../types/function/operator/types.ts' */
+import { concat } from '../../../types/string/module.f.mjs'
+/** @import { CodePoint } from '../../../text/utf16/types.ts' */
+import { codePointToString, stringToCodePointList } from '../../../text/utf16/module.f.mjs'
+import { errorMask } from '../../../text/code_point/module.f.mjs'
+import {
+    backspace,
+    cr,
+    digit0,
+    ff,
+    ht,
+    latinSmallLetterA,
+    lf,
+    quotationMark,
+    reverseSolidus,
+    space,
+} from '../../../text/ascii/module.f.mjs'
+
+const jsonStringify = JSON.stringify
+
+const { fromCharCode } = String
+
+/**
+ * The code points JSON gives a two-character escape. Every other code point
+ * below `space` has no short form and goes through `unicodeEscape` instead.
+ */
+const escapeTable = /** @type {const} */ ({
+    [backspace]: '\\b',
+    [ht]: '\\t',
+    [lf]: '\\n',
+    [ff]: '\\f',
+    [cr]: '\\r',
+    [quotationMark]: '\\"',
+    [reverseSolidus]: '\\\\',
+})
+
+/** @type {(value: number) => string} */
+const hexDigit = value =>
+    fromCharCode(value < 10 ? digit0 + value : latinSmallLetterA + value - 10)
+
+/**
+ * `\uXXXX` with lowercase hex digits, matching ECMAScript's `UnicodeEscape`.
+ *
+ * @type {(unit: number) => string}
+ */
+const unicodeEscape = unit =>
+    `\\u${hexDigit(unit >> 12 & 0xf)}${hexDigit(unit >> 8 & 0xf)}${hexDigit(unit >> 4 & 0xf)}${hexDigit(unit & 0xf)}`
+
+/**
+ * Escapes one decoded code point. A code point tagged with `errorMask` is an
+ * unpaired surrogate, which well-formed JSON stringification (ES2019) emits as
+ * its `\uXXXX` escape rather than as a code unit; everything else is either a
+ * named escape, a `\u00XX` control escape, or the character itself.
+ *
+ * @type {(codePoint: CodePoint) => string}
+ */
+const escapeCodePoint = codePoint =>
+    (codePoint & errorMask) !== 0
+        ? unicodeEscape(codePoint & 0xffff)
+        : escapeTable[/** @type {keyof typeof escapeTable} */ (codePoint)]
+            ?? (codePoint < space ? unicodeEscape(codePoint) : codePointToString(codePoint))
+
+/**
+ * Serializes a string as a JSON string literal.
+ *
+ * @type {(_: string) => List<string>}
+ */
+export const stringSerialize
+    = input => [`"${concat(map(escapeCodePoint)(stringToCodePointList(input)))}"`]
+
+/**
+ * Serializes a number as a JSON number literal.
+ *
+ * @type {(_: number) => List<string>}
+ */
+export const numberSerialize
+    = input => [jsonStringify(input)]
+
+/**
+ * Shared serialized representation for `null`.
+ */
+export const nullSerialize = ['null']
+
+const trueSerialize = ['true']
+
+const falseSerialize = ['false']
+
+/** @type {(_: boolean) => List<string>} */
+export const boolSerialize
+    = value => value ? trueSerialize : falseSerialize
+
+const comma = [',']
+
+/** @type {Reduce<List<string>>} */
+const joinOp
+    = b => prior => flat([prior, comma, b])
+
+/** @type {(input: List<List<string>>) => List<string>} */
+const join
+    = reduce(joinOp)(empty)
+
+/** @type {(open: string) => (close: string) => (input: List<List<string>>) => List<string>} */
+const wrap
+    = open => close => {
+        const seqOpen = [open]
+        const seqClose = [close]
+        return input => flat([seqOpen, join(input), seqClose])
+    }
+
+/** @type {(input: List<List<string>>) => List<string>} */
+export const objectWrap
+    = wrap('{')('}')
+
+/**
+ * Wraps serialized entries into a JSON array.
+ *
+ * @type {(input: List<List<string>>) => List<string>}
+ */
+export const arrayWrap
+    = wrap('[')(']')

package/fjs/media/json/serializer/proof.f.mjs

@@ -0,0 +1,92 @@
+import { arrayWrap, boolSerialize, numberSerialize, objectWrap, stringSerialize } from './module.f.mjs'
+import * as list from '../../../types/list/module.f.mjs'
+import { concat } from '../../../types/string/module.f.mjs'
+import { assertEq } from '../../../asserts/module.f.mjs'
+
+const { toArray } = list
+
+// The expected literals below are what the host's `JSON.stringify` produces for
+// the same input; `stringSerialize` has to reproduce them exactly, so any
+// divergence in the FunctionalScript escaping shows up here as a failure.
+/** @type {(input: string) => string} */
+const serialized = input => concat(stringSerialize(input))
+
+export const proof = {
+    arrayWrap: [
+        () => {
+            const result = JSON.stringify(toArray(arrayWrap(null)))
+            assertEq(result, '["[","]"]')
+        },
+        () => {
+            const result = JSON.stringify(toArray(arrayWrap([['a','b']])))
+            assertEq(result, '["[","a","b","]"]')
+        },
+        () => {
+            const result = JSON.stringify(toArray(arrayWrap([['a'],['b']])))
+            assertEq(result, '["[","a",",","b","]"]')
+        }
+    ],
+    objectWrap: [
+        () => {
+            const result = JSON.stringify(toArray(objectWrap(null)))
+            if (result !== '["{","}"]') { throw result }
+        },
+        () => {
+            const result = JSON.stringify(toArray(objectWrap([['a','b']])))
+            if (result !== '["{","a","b","}"]') { throw result }
+        },
+        () => {
+            const result = JSON.stringify(toArray(objectWrap([['a'],['b']])))
+            if (result !== '["{","a",",","b","}"]') { throw result }
+        }
+    ],
+    stringSerialize: [
+        () => { assertEq(serialized('abc'), '"abc"') },
+        () => { assertEq(serialized('123'), '"123"') },
+        () => { assertEq(serialized(''), '""') },
+        // one chunk, like every other leaf serializer in this module
+        () => { assertEq(toArray(stringSerialize('a"')).length, 1) },
+        // the two escapes JSON requires outside the control block
+        () => { assertEq(serialized('a"b'), '"a\\"b"') },
+        () => { assertEq(serialized('a\\b'), '"a\\\\b"') },
+        // every named control escape
+        () => { assertEq(serialized('\b\f\n\r\t'), '"\\b\\f\\n\\r\\t"') },
+        // control code points without a named escape, covering both hex-digit
+        // halves: '0'-'9' and 'a'-'f'
+        () => { assertEq(serialized('\u0000'), '"\\u0000"') },
+        () => { assertEq(serialized('\u000b'), '"\\u000b"') },
+        () => { assertEq(serialized('\u001f'), '"\\u001f"') },
+        // `space` is the first code point copied through unescaped, and DEL is
+        // not a JSON escape at all
+        () => { assertEq(serialized(' \u007f'), '" \u007f"') },
+        // a surrogate pair decodes to one code point and survives unchanged
+        () => { assertEq(serialized('😀'), '"😀"') },
+        // unpaired surrogates, well-formed-stringify escaped: leading, trailing,
+        // in the middle, doubled, and left over at end of input
+        () => { assertEq(serialized('\ud800'), '"\\ud800"') },
+        () => { assertEq(serialized('\udfff'), '"\\udfff"') },
+        () => { assertEq(serialized('a\udc00b'), '"a\\udc00b"') },
+        () => { assertEq(serialized('\ud800\ud800'), '"\\ud800\\ud800"') },
+        () => { assertEq(serialized('a\ud83d'), '"a\\ud83d"') },
+    ],
+    numberSerialize: [
+        () => {
+            const result = JSON.stringify(toArray(numberSerialize(123)))
+            assertEq(result, '["123"]')
+        },
+        () => {
+            const result = JSON.stringify(toArray(numberSerialize(10e20)))
+            assertEq(result, '["1e+21"]')
+        }
+    ],
+    boolSerialize: [
+        () => {
+            const result = JSON.stringify(toArray(boolSerialize(false)))
+            assertEq(result, '["false"]')
+        },
+        () => {
+            const result = JSON.stringify(toArray(boolSerialize(true)))
+            assertEq(result, '["true"]')
+        }
+    ]
+}

package/fjs/media/json/tokenizer/module.f.d.mts

@@ -0,0 +1,17 @@
+/**
+ * Tokenizer for JSON lexical analysis.
+ *
+ * @module
+ */
+import type { List } from '../../../types/list/types.ts';
+import type { JsonToken } from './types.ts';
+/**
+ * Converts a stream of UTF-8 bytes into JSON tokens.
+ *
+ * The tokenizer accepts only JSON-compatible JavaScript tokens, ignores
+ * whitespace/newline tokens, and reports invalid token sequences as
+ * `{ kind: 'error' }` tokens.
+ *
+ * @type {(input: List<number>) => List<JsonToken>}
+ */
+export declare const tokenize: (input: List<number>) => List<JsonToken>;

package/fjs/media/json/tokenizer/module.f.mjs

@@ -0,0 +1,77 @@
+/**
+ * Tokenizer for JSON lexical analysis.
+ *
+ * @module
+ */
+
+/** @import { StateScan } from '../../../types/function/operator/types.ts' */
+/** @import { List } from '../../../types/list/types.ts' */
+import { concat, empty, flat, stateScan } from '../../../types/list/module.f.mjs'
+import { multiply } from '../../../types/bigfloat/module.f.mjs'
+import { tokenize as jsTokenize } from '../../../js/tokenizer/module.f.mjs'
+/** @import { JsToken } from '../../../js/tokenizer/types.ts' */
+/** @import { JsonToken, _ScanState, _ScanInput } from './types.ts' */
+
+/** @type {(input: JsToken) => List<JsonToken>} */
+const mapToken = input => {
+    switch (input.kind) {
+        case '{':
+        case '}':
+        case ':':
+        case ',':
+        case '[':
+        case ']':
+        case 'true':
+        case 'false':
+        case 'null':
+        case 'string':
+        case 'number':
+        case 'eof':
+        case 'error': return [input]
+        case 'ws':
+        case 'nl': return empty
+        default: return [{ kind: 'error', message: 'invalid token' }]
+    }
+}
+
+/** @type {(input: _ScanInput) => readonly [List<JsonToken>, _ScanState]} */
+const parseDefaultState = input => {
+    if (input === null) return [empty, { kind: 'def' }]
+    switch (input.token.kind) {
+        case '-': return [empty, { kind: '-' }]
+        default: return [mapToken(input.token), { kind: 'def' }]
+    }
+}
+
+/** @type {(input: _ScanInput) => readonly [List<JsonToken>, _ScanState]} */
+const parseMinusState = input => {
+    if (input === null) return [[{ kind: 'error', message: 'invalid token' }], { kind: 'def' }]
+    switch (input.token.kind) {
+        case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }]
+        case 'number': return [[{ kind: 'number', bf: multiply(input.token.bf)(-1n), value: `-${input.token.value}` }], { kind: 'def' }]
+        default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapToken(input.token) }, { kind: 'def' }]
+    }
+}
+
+/** @type {StateScan<_ScanInput, _ScanState, List<JsonToken>>} */
+const scanToken = (input, state) => {
+    switch (state.kind) {
+        case '-': return parseMinusState(input)
+        default: return parseDefaultState(input)
+    }
+}
+
+/**
+ * Converts a stream of UTF-8 bytes into JSON tokens.
+ *
+ * The tokenizer accepts only JSON-compatible JavaScript tokens, ignores
+ * whitespace/newline tokens, and reports invalid token sequences as
+ * `{ kind: 'error' }` tokens.
+ *
+ * @type {(input: List<number>) => List<JsonToken>}
+ */
+export const tokenize = input => {
+    /** @type {List<_ScanInput>} */
+    const jsTokens = jsTokenize(input)('')
+    return flat(stateScan(scanToken)({ kind: 'def' })(concat(jsTokens)([null])))
+}

package/fjs/media/json/tokenizer/proof.f.d.mts

@@ -0,0 +1,8 @@
+/**
+ * @import { JsonToken } from './types.ts'
+ */
+export declare const proof: {
+    json: (() => void)[];
+    id: (() => void)[];
+    keywords: (() => void)[];
+};

package/fjs/media/json/tokenizer/proof.f.mjs

@@ -0,0 +1,291 @@
+/**
+ * @import { JsonToken } from './types.ts'
+ */
+
+import { tokenize } from './module.f.mjs'
+import { toArray } from '../../../types/list/module.f.mjs'
+import { stringifyAsTree } from '../../../djs/serializer/module.f.mjs'
+import { sort } from '../../../types/object/module.f.mjs'
+import { stringToList } from '../../../text/utf16/module.f.mjs'
+
+/** @type {(s: string) => readonly JsonToken[]} */
+const tokenizeString = s => toArray(tokenize(stringToList(s)))
+
+const stringify = stringifyAsTree(sort)
+
+export const proof = {
+    json: [
+        () => {
+            const result = stringify(tokenizeString(''))
+            if (result !== '[{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('{'))
+            if (result !== '[{"kind":"{"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('}'))
+            if (result !== '[{"kind":"}"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString(':'))
+            if (result !== '[{"kind":":"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString(','))
+            if (result !== '[{"kind":","},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('['))
+            if (result !== '[{"kind":"["},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString(']'))
+            if (result !== '[{"kind":"]"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('ᄑ'))
+            if (result !== '[{"kind":"error","message":"unexpected character"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('{ \t\n\r}'))
+            if (result !== '[{"kind":"{"},{"kind":"}"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('""'))
+            if (result !== '[{"kind":"string","value":""},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"value"'))
+            if (result !== '[{"kind":"string","value":"value"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"value'))
+            if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"value1" "value2"'))
+            if (result !== '[{"kind":"string","value":"value1"},{"kind":"string","value":"value2"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"'))
+            if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\\\"'))
+            if (result !== '[{"kind":"string","value":"\\\\"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\""'))
+            if (result !== '[{"kind":"string","value":"\\""},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\/"'))
+            if (result !== '[{"kind":"string","value":"/"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\x"'))
+            if (result !== '[{"kind":"error","message":"unescaped character"},{"kind":"string","value":"x"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\'))
+            if (result !== '[{"kind":"error","message":"\\" are missing"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\b\\f\\n\\r\\t"'))
+            if (result !== '[{"kind":"string","value":"\\b\\f\\n\\r\\t"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\u1234"'))
+            if (result !== '[{"kind":"string","value":"ሴ"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\uaBcDEeFf"'))
+            if (result !== '[{"kind":"string","value":"ꯍEeFf"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('"\\uEeFg"'))
+            if (result !== '[{"kind":"error","message":"invalid hex value"},{"kind":"string","value":"g"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0'))
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('[0]'))
+            if (result !== '[{"kind":"["},{"bf":[0n,0],"kind":"number","value":"0"},{"kind":"]"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('00'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0abc,'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"error","message":"invalid token"},{"kind":","},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('123456789012345678901234567890'))
+            if (result !== '[{"bf":[123456789012345678901234567890n,0],"kind":"number","value":"123456789012345678901234567890"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('{90}'))
+            if (result !== '[{"kind":"{"},{"bf":[90n,0],"kind":"number","value":"90"},{"kind":"}"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('1 2'))
+            if (result !== '[{"bf":[1n,0],"kind":"number","value":"1"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0. 2'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"bf":[2n,0],"kind":"number","value":"2"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('10-0'))
+            if (result !== '[{"bf":[10n,0],"kind":"number","value":"10"},{"bf":[0n,0],"kind":"number","value":"-0"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('9a:'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"error","message":"invalid token"},{"kind":":"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-10'))
+            if (result !== '[{"bf":[-10n,0],"kind":"number","value":"-10"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('--'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('---'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-0'))
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"-0"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-00'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-.123'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"bf":[123n,0],"kind":"number","value":"123"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0.01'))
+            if (result !== '[{"bf":[1n,-2],"kind":"number","value":"0.01"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-0.9'))
+            if (result !== '[{"bf":[-9n,-1],"kind":"number","value":"-0.9"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-0.'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-0.]'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('12.34'))
+            if (result !== '[{"bf":[1234n,-2],"kind":"number","value":"12.34"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-12.00'))
+            if (result !== '[{"bf":[-1200n,-2],"kind":"number","value":"-12.00"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-12.'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('12.]'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"]"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0e1'))
+            if (result !== '[{"bf":[0n,1],"kind":"number","value":"0e1"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0e+2'))
+            if (result !== '[{"bf":[0n,2],"kind":"number","value":"0e+2"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0e-0'))
+            if (result !== '[{"bf":[0n,0],"kind":"number","value":"0e-0"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('12e0000'))
+            if (result !== '[{"bf":[12n,0],"kind":"number","value":"12e0000"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-12e-0001'))
+            if (result !== '[{"bf":[-12n,-1],"kind":"number","value":"-12e-0001"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('-12.34e1234'))
+            if (result !== '[{"bf":[-1234n,1232],"kind":"number","value":"-12.34e1234"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0e'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0e-'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('1234567890n'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('0n'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('[-1234567890n]'))
+            if (result !== '[{"kind":"["},{"kind":"error","message":"invalid token"},{"kind":"error","message":"invalid token"},{"kind":"]"},{"kind":"eof"}]') { throw result }
+        },
+    ],
+    id: [
+        () => {
+            const result = stringify(tokenizeString('err'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('{e}'))
+            if (result !== '[{"kind":"{"},{"kind":"error","message":"invalid token"},{"kind":"}"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('tru'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('break'))
+            if (result !== '[{"kind":"error","message":"invalid token"},{"kind":"eof"}]') { throw result }
+        },
+    ],
+    keywords: [
+        () => {
+            const result = stringify(tokenizeString('true'))
+            if (result !== '[{"kind":"true"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('false'))
+            if (result !== '[{"kind":"false"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('null'))
+            if (result !== '[{"kind":"null"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            const result = stringify(tokenizeString('[null]'))
+            if (result !== '[{"kind":"["},{"kind":"null"},{"kind":"]"},{"kind":"eof"}]') { throw result }
+        },
+    ]
+}

package/fjs/media/json/tokenizer/types.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Types for the JSON tokenizer.
+ *
+ * @module
+ */
+import type { StringToken, NumberToken, ErrorToken, EofToken, JsTokenWithMetadata } from '../../../js/tokenizer/types.ts';
+export type JsonToken = {
+    readonly kind: 'true' | 'false' | 'null';
+} | {
+    readonly kind: '{' | '}' | ':' | ',' | '[' | ']';
+} | StringToken | NumberToken | ErrorToken | EofToken;
+/** @internal */
+export type _ScanState = {
+    readonly kind: 'def' | '-';
+};
+/** @internal */
+export type _ScanInput = JsTokenWithMetadata | null;

package/fjs/media/json/tokenizer/types.js

@@ -0,0 +1,6 @@
+/**
+ * Types for the JSON tokenizer.
+ *
+ * @module
+ */
+export {};