functionalscript 0.38.0 → 0.40.0

package/README.md

@@ -60,7 +60,7 @@ The CAS is also exposed as an [MCP](https://modelcontextprotocol.io/) server so
 claude mcp add cas -- npx functionalscript m
 ```
 
-See [`fjs/cas/mcp/README.md`](fjs/cas/mcp/README.md) for details on the `cas_add`, `cas_get`, and `cas_list` tools.
+See [`fjs/mcp/README.md`](fjs/mcp/README.md) for details on the `cas_add`, `cas_get`, and `cas_list` tools.
 
 ## Vision
 

package/fjs/bnf/descent/module.f.d.ts

@@ -7,9 +7,14 @@
  * AST ({@link AstRuleMeta}). Nullability (which rule can match empty input) is
  * computed once by {@link emptyTagMap} in `fjs/bnf/data`.
  *
+ * A failed result also carries a {@link DescentFailure}: the furthest position a
+ * terminal was rejected at, which — unlike the result's own index — never
+ * rewinds and is what diagnostics should be built from.
+ *
  * @module
  */
 import { type CodePoint } from '../../text/utf16/module.f.ts';
+import { type TerminalRange } from '../module.f.ts';
 import { type Rule as FRule } from '../module.f.ts';
 export type AstTag = string | true | undefined;
 /**
@@ -17,9 +22,45 @@ export type AstTag = string | true | undefined;
  */
 export type DescentMatchRule<T> = (name: string, tag: AstTag, s: readonly CodePointMeta<T>[], idx: number) => DescentMatchResult<T>;
 /**
- * Result tuple of a descent match operation: AST node, success flag, and next index.
+ * Where a match ran out of road, for diagnostics.
+ *
+ * `idx` is the furthest position any terminal was tried at and rejected, and
+ * `expected` holds the terminals that would have allowed progress there, in the
+ * order the grammar tried them and without repeats.
+ *
+ * Unlike a failed result's own index, this never rewinds: a failing sequence
+ * item rewinds the result to the sequence's start, while the furthest failure is
+ * a high-water mark over the whole match — including branches the grammar
+ * backtracked out of. That is what makes "expected X or Y at N" possible.
+ *
+ * `idx` is `0` with an empty `expected` when the match failed without ever
+ * rejecting a terminal, as an empty variant does.
  */
-export type DescentMatchResult<T> = readonly [AstRuleMeta<T>, boolean, number];
+export type DescentFailure = {
+    readonly idx: number;
+    readonly expected: readonly TerminalRange[];
+};
+/**
+ * Result of a descent match operation.
+ *
+ * `failure` is present exactly when `success` is `false`: a successful match has
+ * nothing to diagnose, and its `idx` already says where matching stopped. Note
+ * the consequence for a match that succeeds *without consuming all input* —
+ * `idx` still locates the position it stopped at, but the terminals that would
+ * have let it continue are not reported.
+ *
+ * On failure `idx` has rewound to the start of the enclosing sequence and
+ * locates nothing; read `failure.idx` instead.
+ *
+ * The same type describes a match in progress, where `failure` is likewise
+ * absent until the match ends.
+ */
+export type DescentMatchResult<T> = {
+    readonly ast: AstRuleMeta<T>;
+    readonly success: boolean;
+    readonly idx: number;
+    readonly failure?: DescentFailure;
+};
 /**
  * Entry-point recursive descent matcher.
  */

package/fjs/bnf/descent/module.f.js

@@ -7,6 +7,10 @@
  * AST ({@link AstRuleMeta}). Nullability (which rule can match empty input) is
  * computed once by {@link emptyTagMap} in `fjs/bnf/data`.
  *
+ * A failed result also carries a {@link DescentFailure}: the furthest position a
+ * terminal was rejected at, which — unlike the result's own index — never
+ * rewinds and is what diagnostics should be built from.
+ *
  * @module
  */
 import {} from '../../text/utf16/module.f.js';
@@ -15,6 +19,20 @@ import { contains as rangeContains } from '../../types/range/module.f.js';
 import { definedEntries } from '../../types/object/module.f.js';
 import { emptyTagMap, toData } from '../data/module.f.js';
 import {} from '../module.f.js';
+/**
+ * Folds one rejected terminal into the furthest-failure record: further along
+ * replaces, the same position accumulates (ignoring repeats), earlier is
+ * discarded.
+ */
+const recordFailure = (failure, idx, terminal) => {
+    if (idx > failure.idx) {
+        return { idx, expected: [terminal] };
+    }
+    if (idx < failure.idx || failure.expected.includes(terminal)) {
+        return failure;
+    }
+    return { idx, expected: [...failure.expected, terminal] };
+};
 /**
  * Creates a recursive descent parser that preserves metadata for each consumed
  * code point.
@@ -29,11 +47,14 @@ export const descentParser = (fr) => {
     // grammar recursion depth — right-recursive rules (e.g. repeat0Plus chains) no longer
     // overflow on long input (see the longInput proof group).
     const f = (name, tag, cp, idx) => {
-        const mrSuccess = (tag, sequence, idx) => [{ tag, sequence }, true, idx];
-        const mrFail = (tag, sequence, idx) => [{ tag, sequence }, false, idx];
+        const mrSuccess = (tag, sequence, idx) => ({ ast: { tag, sequence }, success: true, idx });
+        const mrFail = (tag, sequence, idx) => ({ ast: { tag, sequence }, success: false, idx });
         let stack = null;
         let task = { name, tag, idx };
         let result = mrFail(undefined, [], idx);
+        // High-water mark across the whole match, so it survives the rewinds a
+        // failing sequence item does to `result`.
+        let furthest = { idx: 0, expected: [] };
         while (true) {
             if (task !== null) {
                 const { name, tag, idx } = task;
@@ -43,14 +64,17 @@ export const descentParser = (fr) => {
                 // later `task` assignments that `name`'s narrowing depends on.
                 const rule = data[0][name];
                 if (typeof rule === 'number') {
-                    const emptyTag = emptyTags[name];
-                    if (idx >= cp.length) {
-                        result = emptyTag === undefined ? mrFail(emptyTag, [], idx) : mrSuccess(emptyTag, [], idx);
+                    // No nullable case: `emptyTagOf` in `bnf/data` returns `undefined`
+                    // for every terminal, so `emptyTags[name]` here is always
+                    // `undefined` and a terminal either consumes one symbol or fails.
+                    if (idx < cp.length && rangeContains(...rangeDecode(rule))(cp[idx][0])) {
+                        result = mrSuccess(tag, [cp[idx]], idx + 1);
                     }
                     else {
-                        const cpi = cp[idx];
-                        const range = rangeDecode(rule);
-                        result = rangeContains(...range)(cpi[0]) ? mrSuccess(tag, [cpi], idx + 1) : mrFail(emptyTag, [], idx);
+                        // The only place a terminal is rejected, so the only place
+                        // the furthest failure can advance.
+                        furthest = recordFailure(furthest, idx, rule);
+                        result = mrFail(undefined, [], idx);
                     }
                 }
                 else if (rule instanceof Array) {
@@ -78,12 +102,13 @@ export const descentParser = (fr) => {
                 continue;
             }
             if (stack === null) {
-                return result;
+                // A success has nothing to diagnose; only a failure carries it.
+                return result.success ? result : { ...result, failure: furthest };
             }
             const frame = stack.top;
             stack = stack.rest;
             if (frame.kind === 'seq') {
-                const [astRule, success, nidx] = result;
+                const { ast: astRule, success, idx: nidx } = result;
                 if (success === false) {
                     result = mrFail(frame.tag, [], frame.startIdx);
                 }
@@ -102,8 +127,8 @@ export const descentParser = (fr) => {
             else {
                 // success that consumed input wins immediately: the frame stays popped and
                 // `result` propagates to the frame below, matching the recursive `return m`.
-                if (!(result[1] && frame.idx !== result[2])) {
-                    const emptyResult = result[1] ? result : frame.emptyResult;
+                if (!(result.success && frame.idx !== result.idx)) {
+                    const emptyResult = result.success ? result : frame.emptyResult;
                     const entryIndex = frame.entryIndex + 1;
                     if (entryIndex < frame.entries.length) {
                         stack = { top: { ...frame, entryIndex, emptyResult }, rest: stack };

package/fjs/bnf/descent/proof.f.d.ts

@@ -3,4 +3,5 @@ export declare const proof: {
     descentParser: (() => void)[];
     longInput: (() => void)[];
     descentParserWithMeta: (() => void)[];
+    furthestFailure: (() => void)[];
 };

package/fjs/bnf/descent/proof.f.js

@@ -4,8 +4,12 @@ import { commaJoin0Plus, option, range, repeat0Plus, set } from '../module.f.js'
 import { deterministic } from '../testlib.f.js';
 import { emptyTagMap, toData } from '../data/module.f.js';
 import { descentParser } from './module.f.js';
-import { assertEq } from '../../asserts/module.f.js';
+import { assertEq, assertNotNullish } from '../../asserts/module.f.js';
 const mapCodePoint = (cp) => [cp, undefined];
+// The code point of a one-character string, for expectations that would
+// otherwise spell it as a bare number. Goes through the module's own
+// conversion, the same one that builds the parser's input.
+const cp1 = (s) => toArray(stringToCodePointList(s))[0];
 const descentParserCpOnly = (m, name, cp) => {
     const cpm = toArray(map(mapCodePoint)(cp));
     return m(name, cpm);
@@ -83,52 +87,59 @@ export const proof = {
             const m = descentParser(emptyRule);
             const mr = m("", []);
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[]},true,0]') {
+            if (result !== '{"ast":{"sequence":[]},"success":true,"idx":0}') {
                 throw result;
             }
         },
         () => {
             const emptyRule = '';
             const m = descentParser(emptyRule);
-            const mr = descentParserCpOnly(m, "", [65, 70]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('AF')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[]},true,0]') {
+            if (result !== '{"ast":{"sequence":[]},"success":true,"idx":0}') {
                 throw result;
             }
         },
         () => {
+            // Literal code point on purpose. Elsewhere both sides of the
+            // assertion come from `stringToCodePointList` — the input is built
+            // with it and the expectation interpolates `cp1`, which calls it
+            // too — so a change to the conversion would move both sides
+            // together and the test would still pass. Pinning `A` to 65 here
+            // covers the conversion itself.
             const terminalRangeRule = range('AF');
             const m = descentParser(terminalRangeRule);
-            const mr = descentParserCpOnly(m, "", [65]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('A')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[[65,null]]},true,1]') {
+            if (result !== '{"ast":{"sequence":[[65,null]]},"success":true,"idx":1}') {
                 throw result;
             }
         },
         () => {
             const terminalRangeRule = range('AF');
             const m = descentParser(terminalRangeRule);
-            const mr = descentParserCpOnly(m, "", [64]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('@')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[]},false,0]') {
+            if (result !== `{"ast":{"sequence":[]},"success":false,"idx":0,"failure":{"idx":0,"expected":[${range('AF')}]}}`) {
                 throw result;
             }
         },
         () => {
             const variantRule = { 'a': range('AA'), 'b': range('BB') };
             const m = descentParser(variantRule);
-            const mr = descentParserCpOnly(m, "", [65]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('A')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"tag":"a","sequence":[[65,null]]},true,1]') {
+            if (result !== `{"ast":{"tag":"a","sequence":[[${cp1('A')},null]]},"success":true,"idx":1}`) {
                 throw result;
             }
         },
         () => {
             const variantRule = { 'a': range('AA'), 'b': range('BB') };
             const m = descentParser(variantRule);
-            const mr = descentParserCpOnly(m, "", [64]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('@')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[]},false,0]') {
+            // Both branches were rejected at 0, so both terminals are expected there.
+            if (result !== `{"ast":{"sequence":[]},"success":false,"idx":0,"failure":{"idx":0,"expected":[${range('AA')},${range('BB')}]}}`) {
                 throw result;
             }
         },
@@ -138,7 +149,7 @@ export const proof = {
             const m = descentParser(variantRule);
             const mr = m("", []);
             const result = JSON.stringify(mr);
-            if (result !== '[{"tag":"e","sequence":[]},true,0]') {
+            if (result !== '{"ast":{"tag":"e","sequence":[]},"success":true,"idx":0}') {
                 throw result;
             }
         },
@@ -146,9 +157,9 @@ export const proof = {
             const emptyRule = '';
             const variantRule = { 'e': emptyRule, 'a': range('AA') };
             const m = descentParser(variantRule);
-            const mr = descentParserCpOnly(m, "", [64]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('@')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"tag":"e","sequence":[]},true,0]') {
+            if (result !== '{"ast":{"tag":"e","sequence":[]},"success":true,"idx":0}') {
                 throw result;
             }
         },
@@ -157,25 +168,29 @@ export const proof = {
             const m = descentParser(emptyVariantRule);
             const mr = m("", []);
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[]},false,0]') {
+            // A variant with no branches fails without ever trying a terminal,
+            // so there is nothing to expect.
+            if (result !== '{"ast":{"sequence":[]},"success":false,"idx":0,"failure":{"idx":0,"expected":[]}}') {
                 throw result;
             }
         },
         () => {
             const stringRule = 'AB';
             const m = descentParser(stringRule);
-            const mr = descentParserCpOnly(m, "", [65, 66]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('AB')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[{"sequence":[[65,null]]},{"sequence":[[66,null]]}]},true,2]') {
+            if (result !== `{"ast":{"sequence":[{"sequence":[[${cp1('A')},null]]},{"sequence":[[${cp1('B')},null]]}]},"success":true,"idx":2}`) {
                 throw result;
             }
         },
         () => {
             const stringRule = 'AB';
             const m = descentParser(stringRule);
-            const mr = descentParserCpOnly(m, "", [65, 67]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('AC')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[]},false,0]') {
+            // The result index rewound to the sequence's start, but the furthest
+            // failure kept the position where 'B' was actually rejected.
+            if (result !== `{"ast":{"sequence":[]},"success":false,"idx":0,"failure":{"idx":1,"expected":[${range('BB')}]}}`) {
                 throw result;
             }
         },
@@ -186,9 +201,9 @@ export const proof = {
             const digitRule = range('09');
             const numberRule = [optionalMinusRule, digitRule];
             const m = descentParser(numberRule);
-            const mr = descentParserCpOnly(m, "", [50]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('2')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[{"tag":"none","sequence":[]},{"sequence":[[50,null]]}]},true,1]') {
+            if (result !== `{"ast":{"sequence":[{"tag":"none","sequence":[]},{"sequence":[[${cp1('2')},null]]}]},"success":true,"idx":1}`) {
                 throw result;
             }
         },
@@ -199,9 +214,9 @@ export const proof = {
             const digitRule = range('09');
             const numberRule = [optionalMinusRule, digitRule];
             const m = descentParser(numberRule);
-            const mr = descentParserCpOnly(m, "", [45, 50]);
+            const mr = descentParserCpOnly(m, "", toArray(stringToCodePointList('-2')));
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[{"tag":"minus","sequence":[[45,null]]},{"sequence":[[50,null]]}]},true,2]') {
+            if (result !== `{"ast":{"sequence":[{"tag":"minus","sequence":[[${cp1('-')},null]]},{"sequence":[[${cp1('2')},null]]}]},"success":true,"idx":2}`) {
                 throw result;
             }
         },
@@ -214,7 +229,8 @@ export const proof = {
             const m = descentParser(numberRule);
             const mr = m("", []);
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[]},false,0]') {
+            // Past the end: '-' and then the digit range were both rejected at 0.
+            if (result !== `{"ast":{"sequence":[]},"success":false,"idx":0,"failure":{"idx":0,"expected":[${range('--')},${range('09')}]}}`) {
                 throw result;
             }
         },
@@ -223,7 +239,7 @@ export const proof = {
             const expect = (s, expected) => {
                 const cp = toArray(stringToCodePointList(s));
                 const mr = descentParserCpOnly(m, '', cp);
-                const success = mr[1] && mr[2] === cp.length;
+                const success = mr.success && mr.idx === cp.length;
                 assertEq(success, expected, mr);
             };
             expect('a', true);
@@ -243,7 +259,7 @@ export const proof = {
             const expect = (s, expected) => {
                 const cp = toArray(stringToCodePointList(s));
                 const mr = descentParserCpOnly(m, 'value', cp);
-                const success = mr[1] && mr[2] === cp.length;
+                const success = mr.success && mr.idx === cp.length;
                 assertEq(success, expected, mr);
             };
             expect('', false);
@@ -257,7 +273,7 @@ export const proof = {
             const expect = (s, expected) => {
                 const cp = toArray(stringToCodePointList(s));
                 const mr = descentParserCpOnly(m, '', cp);
-                const success = mr[1] && mr[2] === cp.length;
+                const success = mr.success && mr.idx === cp.length;
                 assertEq(success, expected, mr);
             };
             expect('   true   ', true);
@@ -293,7 +309,7 @@ export const proof = {
             const name = toData(rule)[1];
             const m = descentParser(rule);
             const cp = toArray(stringToCodePointList(' '.repeat(10000)));
-            const [, ok, idx] = descentParserCpOnly(m, name, cp);
+            const { success: ok, idx } = descentParserCpOnly(m, name, cp);
             assertEq(ok, true);
             assertEq(idx, 10000);
         },
@@ -304,7 +320,7 @@ export const proof = {
             const m = descentParser(deterministic());
             const n = 5000;
             const cp = toArray(stringToCodePointList('['.repeat(n) + ']'.repeat(n)));
-            const [, ok, idx] = descentParserCpOnly(m, '', cp);
+            const { success: ok, idx } = descentParserCpOnly(m, '', cp);
             assertEq(ok, true);
             assertEq(idx, n * 2);
         },
@@ -317,11 +333,36 @@ export const proof = {
             const digitRule = range('09');
             const numberRule = [optionalMinusRule, digitRule];
             const m = descentParser(numberRule);
-            const mr = m("", [[45, 'minus'], [50, 'two']]);
+            const mr = m("", [[cp1('-'), 'minus'], [cp1('2'), 'two']]);
             const result = JSON.stringify(mr);
-            if (result !== '[{"sequence":[{"tag":"minus","sequence":[[45,"minus"]]},{"sequence":[[50,"two"]]}]},true,2]') {
+            if (result !== `{"ast":{"sequence":[{"tag":"minus","sequence":[[${cp1('-')},"minus"]]},{"sequence":[[${cp1('2')},"two"]]}]},"success":true,"idx":2}`) {
                 throw result;
             }
         },
     ],
+    furthestFailure: [
+        () => {
+            // A branch rejected *before* the high-water mark must not pull it back:
+            // `x` gets to index 1 before failing, then `y` fails at 0.
+            const m = descentParser({ x: ['A', 'B'], y: 'B' });
+            const { success: ok, idx, failure } = descentParserCpOnly(m, '', toArray(stringToCodePointList('AC')));
+            assertEq(ok, false);
+            assertEq(idx, 0);
+            // A failed match always carries a failure; that is the contract.
+            const f = assertNotNullish(failure);
+            assertEq(f.idx, 1);
+            assertEq(f.expected.length, 1);
+            assertEq(f.expected[0], range('BB'));
+        },
+        () => {
+            // The same terminal rejected at the same index by two branches is
+            // expected once, not twice.
+            const m = descentParser({ x: ['A', 'B'], y: ['A', 'B', 'C'] });
+            const { failure } = descentParserCpOnly(m, '', toArray(stringToCodePointList('AC')));
+            const f = assertNotNullish(failure);
+            assertEq(f.idx, 1);
+            assertEq(f.expected.length, 1);
+            assertEq(f.expected[0], range('BB'));
+        },
+    ],
 };

package/fjs/bnf/ll1/module.f.js

@@ -55,7 +55,7 @@ export const dispatchMap = (ruleSet) => {
         const rule = ruleSet[name];
         if (typeof rule === 'number') {
             const range = rangeDecode(rule);
-            const dispatch = dispatchOp.fromRange(range)({ tag: undefined, rules: [] });
+            const dispatch = dispatchOp.fromRange({ tag: undefined, rules: [] })(range);
             const dr = { emptyTag: undefined, rangeMap: dispatch };
             return { ...dm, [name]: dr };
         }
@@ -126,7 +126,7 @@ export const parserRuleSet = (ruleSet) => {
             return mrSuccess(emptyTag, [], emptyTag === undefined ? null : cp);
         }
         const [cp0] = cp;
-        const dr = dispatchOp.get(cp0)(rangeMap);
+        const dr = dispatchOp.get(rangeMap)(cp0);
         if (dr === null) {
             return emptyTag === undefined
                 ? mrFail(emptyTag, [], cp)

package/fjs/bnf/token_symbol/module.f.d.ts

@@ -0,0 +1,38 @@
+import { type Nullable } from '../../types/nullable/module.f.ts';
+/**
+ * How many names one encoding holds: every symbol from {@link start} up to but
+ * not including `eof` (`0xFFFFFF`, the top of the 24-bit symbol space).
+ */
+export declare const capacity: number;
+/**
+ * A bidirectional map between a fixed alphabet of token names and the symbol
+ * range reserved for them.
+ */
+export type Encoding<T extends string> = {
+    /**
+     * The input symbol standing for `name`.
+     *
+     * The result is a bare symbol, the form a tokenizer emits. Wrap it in
+     * `oneEncode` to use it as a terminal of a grammar rule — a symbol and a
+     * `TerminalRange` are both plain numbers, so passing one where the other
+     * belongs is not a type error.
+     */
+    readonly encode: (name: T) => number;
+    /**
+     * The name a symbol stands for, or `null` when the symbol belongs to no
+     * registered name — a code point, `eof`, or a symbol past the end of the
+     * alphabet.
+     */
+    readonly decode: (symbol: number) => Nullable<T>;
+};
+/**
+ * Builds an encoding over the complete list of token names.
+ *
+ * A name takes the symbol at its index in `names`, so the list is append-only:
+ * inserting or reordering names changes the symbols of everything after the
+ * edit.
+ *
+ * @throws When `names` holds more than {@link capacity} entries, or when a name
+ * repeats — a repeated name has no single symbol to decode back to.
+ */
+export declare const encoding: <T extends string>(names: readonly T[]) => Encoding<T>;

package/fjs/bnf/token_symbol/module.f.js

@@ -0,0 +1,48 @@
+/**
+ * Encoding of multi-character token names as single BNF input symbols.
+ *
+ * A parser built on top of a tokenizer consumes one input symbol per token, so
+ * multi-character operators (`>>>=`) and keywords (`instanceof`) each need a
+ * symbol of their own. Names are registered as one fixed alphabet and get a
+ * symbol from their position in it, above the Unicode range and below `eof`.
+ *
+ * @module
+ */
+import { assert } from '../../asserts/module.f.js';
+import { fromUndefined } from '../../types/nullable/module.f.js';
+import { eof, rangeDecode, unicodeRange } from '../module.f.js';
+const [, unicodeLast] = rangeDecode(unicodeRange);
+const [eofSymbol] = rangeDecode(eof);
+/**
+ * The symbol of the first registered name: `0x110000`, one past the last
+ * Unicode scalar value, so a token symbol can never be mistaken for a code
+ * point of the layer below.
+ */
+const start = unicodeLast + 1;
+/**
+ * How many names one encoding holds: every symbol from {@link start} up to but
+ * not including `eof` (`0xFFFFFF`, the top of the 24-bit symbol space).
+ */
+export const capacity = eofSymbol - start;
+/**
+ * Builds an encoding over the complete list of token names.
+ *
+ * A name takes the symbol at its index in `names`, so the list is append-only:
+ * inserting or reordering names changes the symbols of everything after the
+ * edit.
+ *
+ * @throws When `names` holds more than {@link capacity} entries, or when a name
+ * repeats — a repeated name has no single symbol to decode back to.
+ */
+export const encoding = (names) => {
+    assert(names.length <= capacity, ['too many token names', names.length]);
+    assert(new Set(names).size === names.length, ['duplicate token name', names]);
+    return {
+        encode: name => {
+            const index = names.indexOf(name);
+            assert(index !== -1, ['unregistered token name', name]);
+            return start + index;
+        },
+        decode: symbol => fromUndefined(names[symbol - start]),
+    };
+};

package/fjs/bnf/token_symbol/proof.f.d.ts

@@ -0,0 +1,10 @@
+export declare const proof: {
+    encode: () => void;
+    decode: (() => void)[];
+    roundTrip: () => void;
+    throw: {
+        duplicateName: () => void;
+        unregisteredName: () => void;
+        tooManyNames: () => void;
+    };
+};

package/fjs/bnf/token_symbol/proof.f.js

@@ -0,0 +1,37 @@
+import { assertEq } from '../../asserts/module.f.js';
+import { capacity, encoding } from './module.f.js';
+const names = ['>>', '>>>=', 'instanceof'];
+export const proof = {
+    encode: () => {
+        const { encode } = encoding(names);
+        assertEq(encode('>>'), 0x110000);
+        assertEq(encode('instanceof'), 0x110002);
+    },
+    decode: [
+        () => {
+            const { decode } = encoding(names);
+            assertEq(decode(0x110000), '>>');
+            assertEq(decode(0x110002), 'instanceof');
+        },
+        () => {
+            const { decode } = encoding(names);
+            // past the end of the alphabet, a code point, and `eof`
+            assertEq(decode(0x110003), null);
+            assertEq(decode(0x10FFFF), null);
+            assertEq(decode(0xFFFFFF), null);
+        },
+    ],
+    roundTrip: () => {
+        const { encode, decode } = encoding(names);
+        for (const name of names) {
+            assertEq(decode(encode(name)), name);
+        }
+    },
+    throw: {
+        duplicateName: () => { encoding(['a', 'b', 'a']); },
+        unregisteredName: () => { encoding(['a']).encode('b'); },
+        // `capacity` names fit, so one more is the smallest list that doesn't.
+        // The array is sparse, so this costs a length, not the strings.
+        tooManyNames: () => { encoding(new Array(capacity + 1)); },
+    },
+};

package/fjs/cas/cli/module.f.js

@@ -5,7 +5,7 @@
  */
 import { sha256 } from '../../crypto/sha2/module.f.js';
 import { cBase32ToVec, vecToCBase32 } from '../../basen/cbase32/module.f.js';
-import { forEachStep, pure } from '../../effects/module.f.js';
+import { forEachStep, pure, step } from '../../effects/module.f.js';
 import { errorExit, log, writeFromStream } from '../../effects/node/module.f.js';
 import { dispatch } from '../../cli/module.f.js';
 import {} from '../../effects/memory/module.f.js';
@@ -19,10 +19,9 @@ export const commands = [
                 return errorExit("'cas add' expects one parameter");
             }
             const c = fileCas(sha256)(home);
-            return casAddFile(c)(path)
-                .step(hashResult => hashResult[0] === 'error'
+            return step(casAddFile(c)(path), hashResult => hashResult[0] === 'error'
                 ? pure(1)
-                : log(vecToCBase32(hashResult[1])).step(() => pure(0)));
+                : step(log(vecToCBase32(hashResult[1])), () => pure(0)));
         },
     },
     {
@@ -38,8 +37,7 @@ export const commands = [
             }
             const c = fileCas(sha256)(home);
             const x = c.read(hash);
-            return writeFromStream(path, x)
-                .step(([r, v]) => r === 'error' ? errorExit(`e: ` + String(v)) : pure(0));
+            return step(writeFromStream(path, x), ([r, v]) => r === 'error' ? errorExit(`e: ` + String(v)) : pure(0));
         },
     },
     {
@@ -47,9 +45,8 @@ export const commands = [
         description: 'List all stored content hashes',
         handler: ({ home }) => {
             const c = fileCas(sha256)(home);
-            return c.list()
-                .step(forEachStep(j => log(vecToCBase32(j))))
-                .step(() => pure(0));
+            const x0 = forEachStep(c.list(), j => log(vecToCBase32(j)));
+            return step(x0, () => pure(0));
         },
     },
 ];