functionalscript 0.45.0 → 0.46.1

package/fjs/fsm/proof.f.mjs

@@ -1,17 +1,26 @@
 /**
  * @import { Grammar } from './module.f.mjs'
+ * @import { ByteSet } from '../types/byte_set/types.ts'
  */
 
-import { dfa, run, toRange, toUnion } from './module.f.mjs'
-import { union } from '../types/byte_set/module.f.mjs'
-import { sort, fromEntries } from '../types/object/module.f.mjs'
-import { stringify } from '../media/json/module.f.mjs'
-import { identity } from '../types/function/module.f.mjs'
-import { toArray } from '../types/list/module.f.mjs'
+import { dfa, run, toRange } from './module.f.mjs'
+import { one, union, empty, range as byteSetRange } from '../types/byte_set/module.f.mjs'
+import { toKey } from '../types/sorted_set/module.f.mjs'
+import { fold, toArray } from '../types/list/module.f.mjs'
 import { stringToList } from '../text/utf16/module.f.mjs'
-import { assertEq } from '../asserts/module.f.mjs'
+import { assertEq, assertStructurallySame } from '../asserts/module.f.mjs'
 
-const stringifyIdentity = stringify(identity)
+
+/**
+ * The byte set of a string's characters, used to spell a grammar's punctuation
+ * alphabets below. It lives here rather than in `fjs/fsm` or `types/byte_set`:
+ * `fjs/fsm` has no use for it, and a `types` leaf taking a *string* would have
+ * to depend on `fjs/text` to read one.
+ *
+ * @type {(s: string) => ByteSet}
+ */
+const toUnion = s =>
+    fold((/** @type {number} */i) => (/** @type {ByteSet} */bs) => union(bs)(one(i)))(empty)(stringToList(s))
 
 const buildDfa = () => {
     const lowercaseAlpha = toRange('az')
@@ -39,111 +48,71 @@ const buildDfa = () => {
 }
 
 export const proof = {
-    dfa: () => {
-        const dfa = buildDfa()
-        const entries = Object.entries(dfa)
-        const sortedEntries = sort(entries)
-        const obj = fromEntries(sortedEntries)
-        const result = stringifyIdentity(obj)
-
-        const expectedObj = {
-            '[""]': [
-                [ '[]', 35 ],
-                [ '["id"]', 36 ],
-                [ '[]', 47 ],
-                [ '["floatBegin","int"]', 57 ],
-                [ '[]', 64 ],
-                [ '["id"]', 90 ],
-                [ '[]', 94 ],
-                [ '["id"]', 95 ],
-                [ '[]', 96 ],
-                [ '["id"]', 122 ]
-            ],
-            '["float"]': [ [ '[]', 47 ], [ '["float"]', 57 ] ],
-            '["floatBegin","int"]': [
-                [ '[]', 45 ],
-                [ '["floatDot"]', 46 ],
-                [ '[]', 47 ],
-                [ '["floatBegin","int"]', 57 ]
-            ],
-            '["floatDot"]': [ [ '[]', 47 ], [ '["float"]', 57 ] ],
-            '["id"]': [
-                [ '[]', 35 ],
-                [ '["id"]', 36 ],
-                [ '[]', 47 ],
-                [ '["id"]', 57 ],
-                [ '[]', 64 ],
-                [ '["id"]', 90 ],
-                [ '[]', 94 ],
-                [ '["id"]', 95 ],
-                [ '[]', 96 ],
-                [ '["id"]', 122 ]
-            ],
-            '[]': []
-        };
-        const expectedResult = stringifyIdentity(expectedObj)
-
-        if (result !== expectedResult) {throw result }
-    },
+    toRange: [
+        // Two characters name the inclusive range's endpoints.
+        () => assertEq(toRange('az'), byteSetRange([0x61, 0x7a])),
+        // One character is the singleton range. This threw `RangeError: The
+        // number NaN cannot be converted to a BigInt` while `toRange` read a
+        // second character that was not there.
+        () => assertEq(toRange('a'), byteSetRange([0x61, 0x61])),
+        () => assertEq(toRange('a'), one(0x61)),
+        () => assertEq(toRange('\0'), one(0)),
+    ],
+    // The expected states are written as the sets they are, keyed through
+    // `toKey`, so this proof asserts the automaton and not the key encoding.
+    dfa: () => assertStructurallySame(buildDfa(), {
+        [toKey([''])]: [
+            [toKey([]), 35],
+            [toKey(['id']), 36],
+            [toKey([]), 47],
+            [toKey(['floatBegin', 'int']), 57],
+            [toKey([]), 64],
+            [toKey(['id']), 90],
+            [toKey([]), 94],
+            [toKey(['id']), 95],
+            [toKey([]), 96],
+            [toKey(['id']), 122],
+        ],
+        [toKey(['float'])]: [[toKey([]), 47], [toKey(['float']), 57]],
+        [toKey(['floatBegin', 'int'])]: [
+            [toKey([]), 45],
+            [toKey(['floatDot']), 46],
+            [toKey([]), 47],
+            [toKey(['floatBegin', 'int']), 57],
+        ],
+        [toKey(['floatDot'])]: [[toKey([]), 47], [toKey(['float']), 57]],
+        [toKey(['id'])]: [
+            [toKey([]), 35],
+            [toKey(['id']), 36],
+            [toKey([]), 47],
+            [toKey(['id']), 57],
+            [toKey([]), 64],
+            [toKey(['id']), 90],
+            [toKey([]), 94],
+            [toKey(['id']), 95],
+            [toKey([]), 96],
+            [toKey(['id']), 122],
+        ],
+        [toKey([])]: [],
+    }),
     run: [
-        () => {
-            const dfa = buildDfa()
-            const input = stringToList('a1')
-            const result = stringifyIdentity(toArray(run(dfa)(input)))
-
-            const expectedOutput = [
-                '["id"]',
-                '["id"]'
-            ]
-            const expectedResult = stringifyIdentity(expectedOutput)
-            assertEq(result, expectedResult)
-        },
-        () => {
-            const dfa = buildDfa()
-            const input = stringToList('0.1')
-            const result = stringifyIdentity(toArray(run(dfa)(input)))
-
-            const expectedOutput = [
-                '["floatBegin","int"]',
-                '["floatDot"]',
-                '["float"]'
-            ]
-            const expectedResult = stringifyIdentity(expectedOutput)
-            assertEq(result, expectedResult)
-        },
-        () => {
-            const dfa = buildDfa()
-            const input = stringToList('//')
-            const result = stringifyIdentity(toArray(run(dfa)(input)))
-
-            const expectedOutput = [
-                '[]',
-                '[]'
-            ]
-            const expectedResult = stringifyIdentity(expectedOutput)
-            assertEq(result, expectedResult)
-        },
-        () => {
-            const dfa = buildDfa()
-            const input = stringToList('::')
-            const result = stringifyIdentity(toArray(run(dfa)(input)))
-
-            const expectedOutput = [
-                '[]',
-                '[]'
-            ]
-            const expectedResult = stringifyIdentity(expectedOutput)
-            assertEq(result, expectedResult)
-        },
-        () => {
-            // `run` accepts any `Dfa` (a `StringMap`, so entries may be
-            // missing), not only one built by `dfa()`. A state absent from
-            // the map falls back to the empty transition table.
-            const result = stringifyIdentity(toArray(run({})(stringToList('a'))))
-
-            const expectedOutput = ['[]']
-            const expectedResult = stringifyIdentity(expectedOutput)
-            assertEq(result, expectedResult)
-        }
-    ]
+        () => assertStructurallySame(
+            toArray(run(buildDfa())(stringToList('a1'))),
+            [toKey(['id']), toKey(['id'])]),
+        () => assertStructurallySame(
+            toArray(run(buildDfa())(stringToList('0.1'))),
+            [toKey(['floatBegin', 'int']), toKey(['floatDot']), toKey(['float'])]),
+        () => assertStructurallySame(
+            toArray(run(buildDfa())(stringToList('//'))),
+            [toKey([]), toKey([])]),
+        () => assertStructurallySame(
+            toArray(run(buildDfa())(stringToList('::'))),
+            [toKey([]), toKey([])]),
+        // `run` accepts any `Dfa` (a `StringMap`, so entries may be missing),
+        // not only one built by `dfa()`. A state absent from the map falls
+        // back to the empty transition table.
+        () => assertStructurallySame(
+            toArray(run({})(stringToList('a'))),
+            [toKey([])]),
+    ],
 }

package/fjs/js/keywords/module.f.d.mts

@@ -0,0 +1,52 @@
+/**
+ * The JavaScript keywords — one source of truth.
+ *
+ * FunctionalScript is a strict subset of JavaScript: any FunctionalScript
+ * program must run the same on JavaScript. Every consumer that decides
+ * whether a name is a keyword — the JavaScript and DJS tokenizers, printers
+ * that emit identifiers — derives its set from this module instead of
+ * keeping a copy, so the sets cannot drift apart.
+ *
+ * @module
+ *
+ * @import { Assert } from '../../asserts/types.ts'
+ * @import { Equal } from '../../types/ts/types.ts'
+ */
+import type { Assert } from '../../asserts/types.ts';
+import type { Equal } from '../../types/ts/types.ts';
+/**
+ * The ECMAScript `ReservedWord` production
+ * ([ECMA-262 §12.7.2](https://tc39.es/ecma262/#prod-ReservedWord)) — never
+ * usable as identifiers.
+ */
+export declare const reservedWords: readonly ['await', 'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger', 'default', 'delete', 'do', 'else', 'enum', 'export', 'extends', 'false', 'finally', 'for', 'function', 'if', 'import', 'in', 'instanceof', 'new', 'null', 'return', 'super', 'switch', 'this', 'throw', 'true', 'try', 'typeof', 'var', 'void', 'while', 'with', 'yield'];
+/**
+ * Reserved only in strict-mode code — and every module is strict-mode code,
+ * so FunctionalScript treats them exactly like {@link reservedWords}.
+ */
+export declare const strictModeReservedWords: readonly ['implements', 'interface', 'let', 'package', 'private', 'protected', 'public', 'static'];
+/**
+ * Not reserved words, but strict-mode code cannot bind, assign, or shadow
+ * them.
+ */
+export declare const restrictedNames: readonly ['arguments', 'eval'];
+/**
+ * Every name FunctionalScript treats as a keyword, alphabetically: the
+ * {@link reservedWords}, the {@link strictModeReservedWords}, the
+ * {@link restrictedNames}, and `undefined` — an ordinary global in
+ * JavaScript that FunctionalScript keeps as a literal keyword.
+ *
+ * The proof verifies this list is exactly the sorted union of the groups,
+ * and `_KeywordsPinned` ties the two type-level unions together.
+ */
+export declare const keywords: readonly ['arguments', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger', 'default', 'delete', 'do', 'else', 'enum', 'eval', 'export', 'extends', 'false', 'finally', 'for', 'function', 'if', 'implements', 'import', 'in', 'instanceof', 'interface', 'let', 'new', 'null', 'package', 'private', 'protected', 'public', 'return', 'static', 'super', 'switch', 'this', 'throw', 'true', 'try', 'typeof', 'undefined', 'var', 'void', 'while', 'with', 'yield'];
+export type _KeywordsPinned = Assert<Equal<typeof keywords[number], typeof reservedWords[number] | typeof strictModeReservedWords[number] | typeof restrictedNames[number] | 'undefined'>>;
+/**
+ * @typedef {Assert<Equal<
+ *  typeof keywords[number],
+ *  | typeof reservedWords[number]
+ *  | typeof strictModeReservedWords[number]
+ *  | typeof restrictedNames[number]
+ *  | 'undefined'
+ * >>} _KeywordsPinned
+ */

package/fjs/js/keywords/module.f.mjs

@@ -0,0 +1,72 @@
+/**
+ * The JavaScript keywords — one source of truth.
+ *
+ * FunctionalScript is a strict subset of JavaScript: any FunctionalScript
+ * program must run the same on JavaScript. Every consumer that decides
+ * whether a name is a keyword — the JavaScript and DJS tokenizers, printers
+ * that emit identifiers — derives its set from this module instead of
+ * keeping a copy, so the sets cannot drift apart.
+ *
+ * @module
+ *
+ * @import { Assert } from '../../asserts/types.ts'
+ * @import { Equal } from '../../types/ts/types.ts'
+ */
+
+/**
+ * The ECMAScript `ReservedWord` production
+ * ([ECMA-262 §12.7.2](https://tc39.es/ecma262/#prod-ReservedWord)) — never
+ * usable as identifiers.
+ */
+export const reservedWords = /** @type {const} */ ([
+    'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
+    'debugger', 'default', 'delete', 'do', 'else', 'enum', 'export',
+    'extends', 'false', 'finally', 'for', 'function', 'if', 'import', 'in',
+    'instanceof', 'new', 'null', 'return', 'super', 'switch', 'this',
+    'throw', 'true', 'try', 'typeof', 'var', 'void', 'while', 'with',
+    'yield',
+])
+
+/**
+ * Reserved only in strict-mode code — and every module is strict-mode code,
+ * so FunctionalScript treats them exactly like {@link reservedWords}.
+ */
+export const strictModeReservedWords = /** @type {const} */ ([
+    'implements', 'interface', 'let', 'package', 'private', 'protected',
+    'public', 'static',
+])
+
+/**
+ * Not reserved words, but strict-mode code cannot bind, assign, or shadow
+ * them.
+ */
+export const restrictedNames = /** @type {const} */ (['arguments', 'eval'])
+
+/**
+ * Every name FunctionalScript treats as a keyword, alphabetically: the
+ * {@link reservedWords}, the {@link strictModeReservedWords}, the
+ * {@link restrictedNames}, and `undefined` — an ordinary global in
+ * JavaScript that FunctionalScript keeps as a literal keyword.
+ *
+ * The proof verifies this list is exactly the sorted union of the groups,
+ * and `_KeywordsPinned` ties the two type-level unions together.
+ */
+export const keywords = /** @type {const} */ ([
+    'arguments', 'await', 'break', 'case', 'catch', 'class', 'const',
+    'continue', 'debugger', 'default', 'delete', 'do', 'else', 'enum',
+    'eval', 'export', 'extends', 'false', 'finally', 'for', 'function',
+    'if', 'implements', 'import', 'in', 'instanceof', 'interface', 'let',
+    'new', 'null', 'package', 'private', 'protected', 'public', 'return',
+    'static', 'super', 'switch', 'this', 'throw', 'true', 'try', 'typeof',
+    'undefined', 'var', 'void', 'while', 'with', 'yield',
+])
+
+/**
+ * @typedef {Assert<Equal<
+ *  typeof keywords[number],
+ *  | typeof reservedWords[number]
+ *  | typeof strictModeReservedWords[number]
+ *  | typeof restrictedNames[number]
+ *  | 'undefined'
+ * >>} _KeywordsPinned
+ */

package/fjs/js/keywords/proof.f.d.mts

@@ -0,0 +1,3 @@
+export declare const proof: {
+    aggregate: () => void;
+};

package/fjs/js/keywords/proof.f.mjs

@@ -0,0 +1,13 @@
+import { assertEq } from '../../asserts/module.f.mjs'
+import { keywords, reservedWords, restrictedNames, strictModeReservedWords } from './module.f.mjs'
+
+export const proof = {
+    // `keywords` is exactly the sorted union of the groups plus `undefined`
+    aggregate: () => {
+        /** @type {readonly string[]} */
+        const union = [...reservedWords, ...strictModeReservedWords, ...restrictedNames, 'undefined']
+        // the names are unique, so the comparator never sees an equal pair
+        assertEq(keywords.join(), union.toSorted((a, b) => a < b ? -1 : 1).join())
+        assertEq(keywords.length, new Set(keywords).size)
+    },
+}

package/fjs/js/tokenizer/module.f.d.mts

@@ -1,26 +1,46 @@
 /**
  * JavaScript tokenizer built as a range-map state machine over code points,
  * producing tokens for keywords, identifiers, punctuators, comments, strings,
- * and numeric literals (including `BigFloat`).
+ * and numeric literals.
+ *
+ * Numeric scanning is lexeme-first: a `number` token carries the exact source
+ * text and no derived numeric value, so tokenization stays bounded by the
+ * input and never fails because a coefficient or exponent is too large for a
+ * runtime numeric type.
  *
  * @module
  *
- * @import { Scan, StateScan } from '../../types/function/operator/types.ts'
- * @import { RangeMapArray, RangeMerge } from '../../types/range_map/types.ts'
+ * @import { Reduce, Scan, StateScan } from '../../types/function/operator/types.ts'
+ * @import { RangeMerge } from '../../types/range_map/types.ts'
  * @import { List } from '../../types/list/types.ts'
  * @import { Entry } from '../../types/ordered_map/types.ts'
  * @import { Range as NumberRange } from '../../types/range/types.ts'
- * @import { StringToken, NumberToken, BigIntToken, ErrorToken, WhitespaceToken, NewLineToken, IdToken, CommentToken, EofToken, JsToken, TokenMetadata, JsTokenWithMetadata, _TokenizerStateWithMetadata, _TokenizerState, _ErrorMessage, _InitialState, _ParseIdState, _ParseWhitespaceState, _ParseNewLineState, _ParseStringState, _ParseEscapeCharState, _ParseOperatorState, _ParseCommentState, _ParseUnicodeCharState, _ParseNumberState, _ParseNumberBuffer, _InvalidNumberState, _EofState, _CharCodeOrEof, _ToToken, _CreateToToken, _RangeFunc, _RangeMapToToken, } from './types.ts'
+ * @import { JsToken, TokenMetadata, JsTokenWithMetadata, _TokenizerStateWithMetadata, _TokenizerState, _ErrorMessage, _InitialState, _ParseIdState, _ParseWhitespaceState, _ParseNewLineState, _ParseStringState, _ParseEscapeCharState, _ParseOperatorState, _ParseCommentState, _ParseUnicodeCharState, _ParseNumberState, _InvalidNumberState, _EofState, _CharCodeOrEof, _ToToken, _CreateToToken, _RangeFunc, _RangeMapToToken, TriviaKind, } from './types.ts'
  */
 import type { List } from '../../types/list/types.ts';
-import type { JsToken, JsTokenWithMetadata } from './types.ts';
+import type { JsToken, JsTokenWithMetadata, TriviaKind } from './types.ts';
 /** @type {(token: JsToken) => boolean} */
 export declare const isKeywordToken: (token: JsToken) => boolean;
+/**
+ * The coalescing rule for whitespace/newline trivia: a maximal run collapses
+ * to a single token, and a run containing any newline is an `nl`. Equal kinds
+ * coalesce; `nl` absorbs `ws`.
+ *
+ * Exported because `fjs/djs/tokenizer` produces the same token stream and must
+ * agree byte for byte — its scanner reaches the same four decisions from
+ * grammar tags. This module defines `JsToken`, so the rule is stated here once
+ * rather than re-derived on each side with only the proofs to catch a drift.
+ *
+ * @type {(a: TriviaKind, b: TriviaKind) => TriviaKind}
+ */
+export declare const mergeTrivia: (a: TriviaKind, b: TriviaKind) => TriviaKind;
 /** @type {(input: List<number>) => (path: string) => List<JsTokenWithMetadata>} */
 export declare const tokenize: (input: List<number>) => (path: string) => List<JsTokenWithMetadata>;
 export declare const proof: {
     getOperatorTokenInvalid: () => void;
+    tokenizeCharCodeOpAfterEof: () => void;
+    tokenizeEofOpAfterEof: () => void;
     throw: {
-        unionConflict: () => void;
+        unionConflict: () => number;
     };
 };