functionalscript 0.46.0 → 0.47.0

package/fjs/djs/parser/proof.f.d.mts

@@ -2,6 +2,12 @@
  * @import { DjsTokenWithMetadata } from '../tokenizer/types.ts'
  */
 export declare const proof: {
+    parseCorpus: (() => void)[];
+    stackSafety: (() => void)[];
+    syntaxBeforeSemantic: (() => void)[];
+    eofContract: (() => void)[];
+    lexicalErrorStreamShape: (() => void)[];
+    framingKeywordsAsIdentifiers: (() => void)[];
     valid: (() => void)[];
     computedKey: (() => void)[];
     invalidComputedKey: (() => void)[];

package/fjs/djs/parser/proof.f.mjs

@@ -9,14 +9,316 @@ import { sort } from '../../types/object/module.f.mjs'
 import { stringToList } from '../../text/utf16/module.f.mjs'
 import { stringifyAsTree } from '../serializer/module.f.mjs'
 import { stringify } from '../../media/json/module.f.mjs'
-import { assert, assertEq } from '../../asserts/module.f.mjs'
+import { assert, assertEq, assertStructurallySame } from '../../asserts/module.f.mjs'
 
 /** @type {(s: string) => readonly DjsTokenWithMetadata[]} */
 const tokenizeString = s => toArray(tokenize(stringToList(s))(''))
 
 const stringifyDjsModule = stringifyAsTree(sort)
 
+/**
+ * `count` copies of `element`, comma-joined.
+ *
+ * Built by repeating the *string*, which allocates the result directly, rather
+ * than allocating an array and then writing over it: `Array(n).fill(x)` mutates
+ * what it just made, and proof code is held to the same immutability rule as
+ * everything else here.
+ *
+ * @type {(element: string) => (count: number) => string}
+ */
+const repeated = element => count => `${`${element},`.repeat(count - 1)}${element}`
+
+/**
+ * `count` distinct `k<i>:<i>` properties, comma-joined — the object form of
+ * {@link repeated}, where each entry has to differ.
+ *
+ * @type {(count: number) => string}
+ */
+const numberedMembers = count =>
+    Array.from({ length: count }, (_, i) => `k${i}:${i}`).join(',')
+
+/** @type {(kind: 'ws' | 'nl' | 'null' | 'true' | 'false' | 'undefined' | 'eof', line: number) => DjsTokenWithMetadata} */
+const proofKind = (kind, line) => ({ token: { kind }, metadata: { path: 'a.js', line, column: 1 } })
+
+/** @type {(value: string, line: number) => DjsTokenWithMetadata} */
+const proofId = (value, line) => ({ token: { kind: 'id', value }, metadata: { path: 'a.js', line, column: 1 } })
+
 export const proof = {
+    // The corpus that proved parity against the hand-written state machine,
+    // kept as fixed expectations now that the state machine is gone.
+    //
+    // Every value below was **recorded from that parser** before it was deleted,
+    // not written by hand and not read off the replacement — so these still say
+    // "the parser behaves as it always did", which a test written against the
+    // new implementation could not.
+    parseCorpus: [
+        () => {
+            for (const [source, expected] of [
+                ["export default null", "[[],[null]]"],
+                ["export default true", "[[],[true]]"],
+                ["export default false", "[[],[false]]"],
+                ["export default undefined", "[[],[undefined]]"],
+                ["export default 0.1", "[[],[0.1]]"],
+                ["export default 1.1e+2", "[[],[110]]"],
+                ["export default \"abc\"", "[[],[\"abc\"]]"],
+                ["export default 1234567890n", "[[],[1234567890n]]"],
+                ["export default []", "[[],[[\"array\",[]]]]"],
+                ["export default [1]", "[[],[[\"array\",[1]]]]"],
+                ["export default [1,]", "[[],[[\"array\",[1]]]]"],
+                ["export default [[]]", "[[],[[\"array\",[[\"array\",[]]]]]]"],
+                ["export default [0,[1,[2,[]]],3]", "[[],[[\"array\",[0,[\"array\",[1,[\"array\",[2,[\"array\",[]]]]]],3]]]]"],
+                ["export default [1234567890n]", "[[],[[\"array\",[1234567890n]]]]"],
+                ["export default {}", "[[],[{}]]"],
+                ["export default {\"a\":1}", "[[],[{\"a\":1}]]"],
+                ["export default {a: 1}", "[[],[{\"a\":1}]]"],
+                ["export default {\"a\":1,}", "[[],[{\"a\":1}]]"],
+                ["export default {[\"a\"]:1}", "[[],[{\"a\":1}]]"],
+                ["export default {a:1,\"b\":2,[\"c\"]:3,}", "[[],[{\"a\":1,\"b\":2,\"c\":3}]]"],
+                ["export default {\"a\":{\"b\":{\"c\":[\"d\"]}}}", "[[],[{\"a\":{\"b\":{\"c\":[\"array\",[\"d\"]]}}}]]"],
+                ["export default {\"a\":true,\"b\":false,\"c\":null,\"d\":undefined}", "[[],[{\"a\":true,\"b\":false,\"c\":null,\"d\":undefined}]]"],
+                ["export default {a:1,a:2}", "[[],[{\"a\":2}]]"],
+                ["export default {[\"__proto__\"]: 1}", "[[],[{\"__proto__\":1}]]"],
+                ["const a = 1\nexport default a", "[[],[1,[\"cref\",0]]]"],
+                ["const a = 1\nconst b = 2\nexport default [a,b]", "[[],[1,2,[\"array\",[[\"cref\",0],[\"cref\",1]]]]]"],
+                ["const a = a\nexport default a", "[[],[[\"cref\",0],[\"cref\",0]]]"],
+                ["import x from \"m\"\nexport default x", "[[\"m\"],[[\"aref\",0]]]"],
+                ["import x from \"m\"\nconst a = 1\nexport default a", "[[\"m\"],[1,[\"cref\",0]]]"],
+                ["import x from \"m\"\nimport y from \"n\"\nexport default [x,y]", "[[\"m\",\"n\"],[[\"array\",[[\"aref\",0],[\"aref\",1]]]]]"],
+                ["// c\nexport default 1", "[[],[1]]"],
+                ["/* c */ export default 1", "[[],[1]]"],
+                ["\n\n export default 1 \n\n", "[[],[1]]"],
+                ["const export = 1\nexport default export", "[[],[1,[\"cref\",0]]]"],
+                ["export default { from: 2, default: 3 }", "[[],[{\"default\":3,\"from\":2}]]"],
+            ]) {
+                const [tag, value] = parseFromTokens(tokenizeString(source))
+                assert(tag === 'ok', [source, tag])
+                assertEq(stringifyDjsModule(value), expected, source)
+            }
+        },
+        () => {
+            /** @type {readonly(readonly[string, string, readonly[number, number] | null])[]} */
+            const cases = [
+                ["42", "unexpected token", [1, 1]],
+                ["", "unexpected end", [1, 1]],
+                ["export default", "unexpected end", [1, 15]],
+                ["[1,2]", "unexpected token", [1, 1]],
+                ["{\"a\":1}", "unexpected token", [1, 1]],
+                ["export default [", "unexpected end", [1, 17]],
+                ["export default {", "unexpected end", [1, 17]],
+                ["const a = 1 export default a", "unexpected token", [1, 13]],
+                ["export default 1 2", "unexpected token", [1, 18]],
+                ["export default {a}", "unexpected token", [1, 18]],
+                ["export default {:1}", "unexpected token", [1, 17]],
+                ["export default [,]", "unexpected token", [1, 17]],
+                ["import x from y\nexport default x", "unexpected token", [1, 15]],
+                ["export x from \"m\"\nexport default 1", "unexpected token", [1, 8]],
+                ["const = 1\nexport default 1", "unexpected token", [1, 7]],
+                ["export default {[1]:2}", "unexpected token", [1, 18]],
+                ["const a = 1;\nexport default a", "unexpected token", [1, 12]],
+                ["export default 1\nconst b = 2", "unexpected token", [2, 1]],
+                ["import x from \"m\"", "unexpected end", [1, 18]],
+                ["const a = 1", "unexpected end", [1, 12]],
+                ["const a = 1\nconst a = 2\nexport default a", "duplicate id", [2, 7]],
+                ["import x from \"m\"\nimport x from \"n\"\nexport default x", "duplicate id", [2, 8]],
+                ["import x from \"m\"\nconst x = 1\nexport default x", "duplicate id", [2, 7]],
+                ["export default zzz", "const not found", [1, 16]],
+                ["const a = zzz\nexport default a", "const not found", [1, 11]],
+                ["export default [zzz]", "const not found", [1, 17]],
+                ["export default {a: zzz}", "const not found", [1, 20]],
+                ["export default {__proto__: 1}", "__proto__ requires the computed key form", [1, 17]],
+                ["export default {\"__proto__\": 1}", "__proto__ requires the computed key form", [1, 17]],
+                ["import x from \"m\"\nimport x from \"n\"\nimport y from \"o\"\nexport default y", "duplicate id", [2, 8]],
+                ["const a = 1\nconst a = 2\nconst b = 3\nexport default b", "duplicate id", [2, 7]],
+            ]
+            for (const [source, message, position] of cases) {
+                const [tag, value] = parseFromTokens(tokenizeString(source))
+                assert(tag === 'error', [source, tag])
+                assertEq(value.message, message, source)
+                const { metadata } = value
+                assertStructurallySame(
+                    metadata === null ? null : [metadata.line, metadata.column],
+                    position,
+                    source)
+            }
+        },
+    ],
+    // Wide and deep values, which cost the BNF path a stack frame each until
+    // both its traversals were made iterative.
+    //
+    // A repetition is only *flat* in the AST when `toData` recognizes the
+    // right-recursive shape, and nested inside `delimited`'s option scaffolding
+    // it does not — so 5,000 siblings are 5,000 levels of tree, and a recursive
+    // walk over them overflows exactly as deep nesting would. The state machine
+    // was fixed for its own version of this; see `containerStackCost`.
+    stackSafety: [
+        () => {
+            const source = `export default [${repeated('null')(5000)}]`
+            const tokens = tokenizeString(source)
+            assertEq(parseFromTokens(tokens)[0], 'ok')
+        },
+        () => {
+            const source = `export default [${repeated('{}')(5000)}]`
+            const tokens = tokenizeString(source)
+            assertEq(parseFromTokens(tokens)[0], 'ok')
+        },
+        () => {
+            const source = `export default ${'['.repeat(5000)}${']'.repeat(5000)}`
+            const tokens = tokenizeString(source)
+            assertEq(parseFromTokens(tokens)[0], 'ok')
+        },
+        () => {
+            // wide objects walk the member list rather than the element list
+            const source = `export default {${numberedMembers(5000)}}`
+            const tokens = tokenizeString(source)
+            assertEq(parseFromTokens(tokens)[0], 'ok')
+        },
+    ],
+    // A syntax error is reported ahead of a semantic one, wherever each sits.
+    //
+    // The grammar matches the whole module before the fold runs, so a malformed
+    // suffix is found before any name is resolved. The parser this replaced
+    // streamed, so it met an unresolved name first and said so. Both are true of
+    // the input; they answer different questions about it.
+    //
+    // This is the design's stated failure-parity bar rather than a departure
+    // from it — the error must "identify the furthest relevant token or EOF",
+    // and the syntax failure is the furthest relevant token. It is pinned here
+    // because it is a *change*, and because the reasoning is not recoverable
+    // from the positions alone.
+    syntaxBeforeSemantic: [
+        () => {
+            /** @type {readonly(readonly[string, string, readonly[number, number]])[]} */
+            const cases = [
+                // was: const not found @1:11
+                ['const a = missing x', 'unexpected token', [1, 19]],
+                // was: const not found @1:11
+                ['const a = missing', 'unexpected end', [1, 18]],
+                // was: const not found @1:16
+                ['export default missing 1', 'unexpected token', [1, 24]],
+                // was: duplicate id @2:7
+                ['const a = 1\nconst a = 2 x', 'unexpected token', [2, 13]],
+            ]
+            for (const [source, message, position] of cases) {
+                const [tag, value] = parseFromTokens(tokenizeString(source))
+                assert(tag === 'error', [source, tag])
+                assertEq(value.message, message, source)
+                const { metadata } = value
+                assertStructurallySame(
+                    metadata === null ? null : [metadata.line, metadata.column],
+                    position,
+                    source)
+            }
+        },
+        () => {
+            // with nothing malformed after it, the semantic error is still the
+            // one reported — the change is which error wins, not whether names
+            // are checked
+            const [tag, value] = parseFromTokens(tokenizeString('export default missing'))
+            assert(tag === 'error', tag)
+            assertEq(value.message, 'const not found')
+            assertEq(value.metadata?.column, 16)
+        },
+    ],
+    // The tokenizer's EOF contract, checked through the parser rather than
+    // through `splitEof` alone.
+    //
+    // Two of these are a **deliberate behaviour change at cutover**: the state
+    // machine accepts a stream with no `eof`, and one with two, because it only
+    // ever asks what the next token is. The BNF path requires exactly one, in
+    // final position, because a backend synthesizes its own logical end and a
+    // second marker would be a symbol the grammar has no rule for. Neither
+    // stream can come from the tokenizer, so this tightens a contract only a
+    // hand-built list could break — but it does tighten it, and the cutover
+    // should not be where that is discovered.
+    eofContract: [
+        () => {
+            const wellFormed = [
+                proofId('export', 1), proofKind('ws', 1), proofId('default', 1),
+                proofKind('ws', 1), proofKind('null', 1), proofKind('eof', 1),
+            ]
+            assertEq(parseFromTokens(wellFormed)[0], 'ok')
+        },
+        () => {
+            // no `eof`: the state machine read this as a complete module
+            const noEof = [
+                proofId('export', 1), proofKind('ws', 1), proofId('default', 1),
+                proofKind('ws', 1), proofKind('null', 1),
+            ]
+            const [tag, value] = parseFromTokens(noEof)
+            assert(tag === 'error', tag)
+            assertEq(value.message, 'missing end-of-input token')
+        },
+        () => {
+            // a second `eof`: likewise invisible to the state machine
+            const twoEof = [
+                proofId('export', 1), proofKind('ws', 1), proofId('default', 1),
+                proofKind('ws', 1), proofKind('null', 1), proofKind('eof', 1), proofKind('eof', 2),
+            ]
+            const [tag, value] = parseFromTokens(twoEof)
+            assert(tag === 'error', tag)
+            assertEq(value.message, 'end-of-input token is not final')
+        },
+        () => {
+            const [tag, value] = parseFromTokens([])
+            assert(tag === 'error', tag)
+            assertEq(value.message, 'missing end-of-input token')
+        },
+    ],
+    // A lexical failure ends the token stream at an `error` token and emits no
+    // `eof`. `splitEof` reads a missing `eof` that way rather than as a broken
+    // tokenizer contract, and reports the error where it happened — so that
+    // reading is pinned here against the real tokenizer, alongside the position
+    // the current parser reports for the same input.
+    lexicalErrorStreamShape: [
+        () => {
+            const tokens = tokenizeString('const a = "abc')
+            assertEq(tokens.length, 1)
+            assertEq(tokens[0].token.kind, 'error')
+            assertEq(tokens[0].metadata.column, 11)
+        },
+        () => {
+            const [tag, value] = parseFromTokens(tokenizeString('const a = "abc'))
+            assert(tag === 'error', tag)
+            assertEq(value.metadata?.line, 1)
+            assertEq(value.metadata?.column, 11)
+        },
+        () => {
+            const [tag, value] = parseFromTokens(tokenizeString('const a = /* x'))
+            assert(tag === 'error', tag)
+            assertEq(value.metadata?.column, 15)
+        },
+    ],
+    // None of the framing keywords is reserved: outside the positions that frame
+    // a module, the parser accepts them as ordinary identifiers. Pinned here
+    // because the BNF replacement gives each its own token symbol, and a grammar
+    // whose identifier rule accepted only the `id` symbol would stop parsing
+    // every case below without any other test noticing.
+    framingKeywordsAsIdentifiers: [
+        () => {
+            const [tag, value] = parseFromTokens(tokenizeString('const export = 1\nexport default export'))
+            assert(tag === 'ok', tag)
+            // `export` bound as a const and then referenced: the exported value
+            // resolves back to that binding, so the word acted as an identifier
+            // in both positions.
+            assertStructurallySame(value[1], [1, ['cref', 0]])
+        },
+        () => {
+            const [tag] = parseFromTokens(tokenizeString('const from = 1\nexport default from'))
+            assert(tag === 'ok', tag)
+        },
+        () => {
+            const [tag] = parseFromTokens(tokenizeString('const import = 1\nexport default import'))
+            assert(tag === 'ok', tag)
+        },
+        () => {
+            const [tag] = parseFromTokens(tokenizeString('export default { from: 2, default: 3 }'))
+            assert(tag === 'ok', tag)
+        },
+        () => {
+            const [tag] = parseFromTokens(tokenizeString('export default { export: 1 }'))
+            assert(tag === 'ok', tag)
+        },
+    ],
     valid: [
         () => {
             const tokenList = tokenizeString('export default null')
@@ -501,15 +803,14 @@ export const proof = {
             assertEq(obj[1].message, 'unexpected end')
         },
         () => {
-            // `parseFromTokens` itself, called with no tokens at all (the
+            // `parseFromTokens` itself, called with no tokens at all. The
             // tokenizer never produces this — it always emits at least an
-            // `eof` token — but the exported function's own contract must
-            // still handle it: the initial state is neither 'result' nor
-            // 'error', so it falls through to the same "unexpected end".
-            const obj = parseFromTokens(null)
+            // `eof` — but the exported function's contract must still answer.
+            // Since the BNF cutover it is named for what is missing rather
+            // than for running out of input.
+            const obj = parseFromTokens([])
             assert(obj[0] === 'error', obj)
-            assertEq(obj[1].message, 'unexpected end')
-            assertEq(obj[1].metadata, null)
+            assertEq(obj[1].message, 'missing end-of-input token')
         },
     ],
     errorMetadata: [
@@ -617,7 +918,10 @@ export const proof = {
             const tokenList = tokenizeString('const b = 1 \n import a from "a.f.js" \n export default [a,b]')
             const obj = parseFromTokens(tokenList)
             assert(obj[0] === 'error', obj)
-            assertEq(obj[1].message, 'import must come before const')
+            // reworded at the BNF cutover: statement ordering is shape now, so
+            // a late `import` is a token the grammar cannot use rather than a
+            // rule about ordering it could name. The position is unchanged.
+            assertEq(obj[1].message, 'unexpected token')
         },
         () => {
             // nothing follows `export default` but trivia
@@ -793,13 +1097,13 @@ export const proof = {
     containerStackCost: [
         () => {
             const [tag, value] = parseFromTokens(tokenizeString(
-                `export default [${Array(20000).fill('{}').join(',')}]`))
+                `export default [${repeated('{}')(20000)}]`))
             assert(tag === 'ok', tag)
             assertEq(value[1].length, 1)
         },
         () => {
             const [tag] = parseFromTokens(tokenizeString(
-                `export default [${Array(20000).fill('[]').join(',')}]`))
+                `export default [${repeated('[]')(20000)}]`))
             assert(tag === 'ok', tag)
         },
         () => {

package/fjs/djs/parser/types.d.ts

@@ -1,7 +1,7 @@
 /**
  * Type-level API for `fjs/djs/parser/module.f.mjs`: the `ParseError` shape
- * `parseFromTokens` reports, and the `_ValueToken` subset `tokenToValue`
- * accepts.
+ * `parseFromTokens` reports, the `_ValueToken` subset `tokenToValue` accepts,
+ * and the parser layer's token alphabet.
  *
  * @module
  */
@@ -11,6 +11,33 @@ export type ParseError = {
     readonly message: string;
     readonly metadata: TokenMetadata | null;
 };
+/**
+ * The words that frame a module, which the grammar has to tell apart from an
+ * ordinary identifier.
+ *
+ * The tokenizer emits all five as `{ kind: 'id' }` with the word in `value`, so
+ * a parser layer keyed on `kind` alone would give them the same symbol as any
+ * other identifier — and a grammar over that alphabet could not distinguish
+ * `export default` from two arbitrary names. They therefore get terminals of
+ * their own, which is what a registered alphabet allows: a name's symbol comes
+ * from its position in the list, so a name has no length limit.
+ *
+ * This is the same distinction the hand-written parser makes by comparing
+ * `token.value`, lifted from control flow into the alphabet.
+ */
+export type _FramingKeyword = 'import' | 'const' | 'export' | 'default' | 'from';
+/**
+ * A token name the parser layer's grammar can name as a terminal: every
+ * `DjsToken` kind except `eof`, plus the framing keywords.
+ *
+ * `eof` is excluded because a BNF backend synthesizes its own logical
+ * end-of-input; the tokenizer's physical `eof` token is split off the stream
+ * before any name is mapped, so it never reaches this alphabet.
+ *
+ * The kinds are derived from `DjsToken` rather than listed, so a token kind
+ * added there cannot silently go unrepresented at the parser layer.
+ */
+export type _OrdinaryTokenName = Exclude<DjsToken['kind'], 'eof'> | _FramingKeyword;
 /** DJS tokens that carry a directly-usable value. */
 export type _ValueToken = Extract<DjsToken, {
     readonly kind: 'null' | 'false' | 'true' | 'undefined' | 'string' | 'number' | 'bigint';