functionalscript 0.45.0 → 0.46.0

package/fjs/bnf/descent/module.f.mjs

@@ -4,52 +4,49 @@
  * Built from the serializable IR in `fjs/bnf/data`, this is a sibling of the
  * LL(1) dispatch builder (`fjs/bnf/ll1`). It walks the grammar by recursive
  * descent and preserves per-code-point metadata, producing a metadata-aware
- * AST ({@link AstRuleMeta}). Nullability (which rule can match empty input) is
- * computed once by {@link emptyTagMap} in `fjs/bnf/data`.
+ * AST over leaves that carry it — `Ast<CodePointMeta<T>>`, built with the
+ * shared layer in `fjs/bnf/matcher`. Nullability (which rule can match empty
+ * input) is computed once by {@link emptyTagMap} in `fjs/bnf/data`.
  *
  * The caller passes physical symbols only; the matcher synthesizes the one
- * logical EOF ({@link eofSymbol}) after them, so a grammar can require the end
- * of input with the `eof` terminal.
+ * logical EOF after them, so a grammar can require the end of input with the
+ * `eof` terminal.
  *
  * A failed result also carries a {@link DescentFailure}: the furthest position a
  * terminal was rejected at, which — unlike the result's own index — never
  * rewinds and is what diagnostics should be built from.
  *
+ * A `Repeat` rule is matched iteratively rather than by descending into itself
+ * once per item, so a repetition produces one AST node holding a flat sequence
+ * of the items it matched instead of the right-recursive chain its functional
+ * spelling would otherwise build.
+ *
  * See `./types.ts` for the type-level API.
  *
  * @module
  *
  * @import { TerminalRange } from '../types.ts'
- * @import { Rule as DataRule, Sequence } from '../data/types.ts'
+ * @import { Rule as DataRule, RuleSet, Sequence } from '../data/types.ts'
  * @import { Rule as FRule } from '../types.ts'
- * @import { AstRuleMeta, AstTag, AstSequenceMeta, CodePointMeta, DescentFailure, DescentMatch, DescentMatchResult, DescentMatchRule } from './types.ts'
+ * @import { List } from '../../types/list/types.ts'
+ * @import { Ast, AstResult, AstSequence, AstTag, Cursor } from '../matcher/types.ts'
+ * @import { CodePointMeta, DescentFailure, DescentMatch, DescentMatchResult, DescentMatchRule } from './types.ts'
  */
 
-import { eofSymbol, rangeDecode } from '../module.f.mjs'
+import { rangeDecode } from '../module.f.mjs'
 import { contains as rangeContains } from '../../types/range/module.f.mjs'
+import { concat, toArray } from '../../types/list/module.f.mjs'
 import { definedEntries } from '../../types/object/module.f.mjs'
-import { emptyTagMap, toData } from '../data/module.f.mjs'
-
-/**
- * A match position that includes EOF consumption: `0 .. cp.length` are the
- * physical positions, and `cp.length + 1` is where the one synthesized EOF has
- * been consumed.
- *
- * This is the complete cursor the design calls `(idx, eofConsumed)`, written as
- * one number: `eofConsumed` can only be true at the physical end, so the pair
- * and the extended position hold the same information, and comparing cursors
- * compares progress — consuming EOF *is* progress even though the public index
- * does not move.
- *
- * @typedef {number} _Cursor
- */
+import { emptyTagMap, isRepeat, toData } from '../data/module.f.mjs'
+import { leafAt, mrFail, mrSuccess, physicalIdx, symbolAt } from '../matcher/module.f.mjs'
 
 /**
  * The furthest-failure record while matching, positioned by the complete
- * cursor. {@link DescentFailure} is its public, physically-positioned form.
+ * {@link Cursor}. {@link DescentFailure} is its public, physically-positioned
+ * form.
  *
  * @typedef {{
- *     readonly pos: _Cursor
+ *     readonly pos: Cursor
  *     readonly expected: readonly TerminalRange[]
  * }} _Failure
  */
@@ -57,41 +54,22 @@ import { emptyTagMap, toData } from '../data/module.f.mjs'
 /**
  * The machine's own result: a {@link DescentMatchResult} positioned by the
  * complete cursor, and with no failure record — that one is tracked per match
- * rather than per frame.
+ * rather than per frame. This backend always has a position, so it needs no
+ * `null` case.
  *
  * @template T
- * @typedef {{
- *     readonly ast: AstRuleMeta<T>
- *     readonly success: boolean
- *     readonly pos: _Cursor
- * }} _Result
- */
-
-/**
- * The public, physical index of a cursor. Consuming EOF moves the cursor past
- * the physical end, and both cursors report `input.length`.
- *
- * @type {(length: number) => (pos: _Cursor) => number}
+ * @typedef {AstResult<CodePointMeta<T>, Cursor>} _Result
  */
-const physicalIdx = length => pos => Math.min(pos, length)
 
 /**
- * The semantic symbol a cursor points at: a code point inside the physical
- * input, and the synthesized {@link eofSymbol} at its end. Only meaningful
- * where the cursor still has a symbol, `pos <= cp.length`.
+ * A leaf here is a code point with its metadata, so its symbol is the first
+ * half. This is the only thing {@link symbolAt} needs to know about a leaf.
  *
- * @type {<T>(cp: readonly CodePointMeta<T>[], pos: _Cursor) => number}
+ * @type {<T>(leaf: CodePointMeta<T>) => number}
  */
-const symbolAt = (cp, pos) => pos < cp.length ? cp[pos][0] : eofSymbol
+const symbolOf = ([symbol]) => symbol
 
-/**
- * What consuming the symbol at a cursor contributes to the AST: the code point
- * with its metadata, and nothing for the synthesized EOF — it has no physical
- * source element.
- *
- * @type {<T>(cp: readonly CodePointMeta<T>[], pos: _Cursor) => AstSequenceMeta<T>}
- */
-const leafAt = (cp, pos) => pos < cp.length ? [cp[pos]] : []
+const symbolAtCp = symbolAt(symbolOf)
 
 /**
  * Folds one rejected terminal into the furthest-failure record: further along
@@ -99,7 +77,7 @@ const leafAt = (cp, pos) => pos < cp.length ? [cp[pos]] : []
  * discarded. The comparison is on the complete cursor, so a failure after EOF
  * was consumed is further than one at the same physical index before it.
  *
- * @type {(failure: _Failure, pos: _Cursor, terminal: TerminalRange) => _Failure}
+ * @type {(failure: _Failure, pos: Cursor, terminal: TerminalRange) => _Failure}
  */
 const recordFailure = (failure, pos, terminal) => {
     if (pos > failure.pos) { return { pos, expected: [terminal] } }
@@ -108,21 +86,15 @@ const recordFailure = (failure, pos, terminal) => {
 }
 
 /**
- * Creates a recursive descent parser that preserves metadata for each consumed
- * code point.
+ * Creates a recursive descent parser from an already materialized
+ * {@link RuleSet}, preserving metadata for each consumed code point.
  *
  * @template T
- * @param {FRule} fr
+ * @param {RuleSet} ruleSet
  * @returns {DescentMatch<T>}
  */
-export const descentParser = fr => {
-    const data = toData(fr)
-    const emptyTags = emptyTagMap(data[0])
-
-    /** @type {(tag: AstTag, sequence: AstSequenceMeta<T>, pos: _Cursor) => _Result<T>} */
-    const mrSuccess = (tag, sequence, pos) => ({ ast: {tag, sequence}, success: true, pos })
-    /** @type {(tag: AstTag, sequence: AstSequenceMeta<T>, pos: _Cursor) => _Result<T>} */
-    const mrFail = (tag, sequence, pos) => ({ ast: {tag, sequence}, success: false, pos })
+export const descentParserRuleSet = ruleSet => {
+    const emptyTags = emptyTagMap(ruleSet)
 
     // A suspended sequence match: items[itemIndex] is being matched by the current
     // task; `seq` holds the ASTs of the items already matched.
@@ -132,8 +104,8 @@ export const descentParser = fr => {
      *     readonly tag: AstTag
      *     readonly items: Sequence
      *     readonly itemIndex: number
-     *     readonly startPos: _Cursor
-     *     readonly seq: AstSequenceMeta<T>
+     *     readonly startPos: Cursor
+     *     readonly seq: AstSequence<CodePointMeta<T>>
      * }} _SeqFrame
      */
 
@@ -145,12 +117,30 @@ export const descentParser = fr => {
      *     readonly kind: 'variant'
      *     readonly entries: readonly (readonly [string, string])[]
      *     readonly entryIndex: number
-     *     readonly pos: _Cursor
+     *     readonly pos: Cursor
      *     readonly emptyResult: _Result<T>
      * }} _VariantFrame
      */
 
-    /** @typedef {_SeqFrame | _VariantFrame} _Frame */
+    // A suspended repetition: `item` is being matched by the current task for the
+    // round that began at `roundStart`, and `items` holds the ASTs of the rounds
+    // that already completed. They accumulate as a list rather than an array
+    // because a repetition is as long as its input: appending to an array per
+    // round would copy the whole prefix each time and make one repetition
+    // quadratic in the number of items it matched.
+    /**
+     * @typedef {{
+     *     readonly kind: 'repeat'
+     *     readonly tag: AstTag
+     *     readonly item: string
+     *     readonly items: _Items
+     *     readonly roundStart: Cursor
+     * }} _RepeatFrame
+     */
+
+    /** @typedef {List<Ast<CodePointMeta<T>>>} _Items */
+
+    /** @typedef {_SeqFrame | _VariantFrame | _RepeatFrame} _Frame */
 
     // Immutable cons-cell stack: O(1) push/pop, no array copying per step.
     /**
@@ -164,12 +154,28 @@ export const descentParser = fr => {
     // tuple), or null when a result is ready to resume the innermost frame.
     /**
      * @typedef {{
+     *     readonly kind: 'rule'
      *     readonly name: string
      *     readonly tag: AstTag
-     *     readonly pos: _Cursor
-     * }} _Task
+     *     readonly pos: Cursor
+     * }} _RuleTask
      */
 
+    // The next round of a repetition, about to be started. Both the rule that
+    // introduces a repetition and the frame that finishes one of its rounds go
+    // through this, so a round is set up in exactly one place.
+    /**
+     * @typedef {{
+     *     readonly kind: 'repeat'
+     *     readonly tag: AstTag
+     *     readonly item: string
+     *     readonly items: _Items
+     *     readonly pos: Cursor
+     * }} _RepeatTask
+     */
+
+    /** @typedef {_RuleTask | _RepeatTask} _Task */
+
     // The recursive-descent matcher as an explicit-stack machine: each iteration either
     // starts the current task (pushing a frame for a sequence/variant and descending into
     // its first child) or feeds the pending result into the innermost frame. Semantics are
@@ -183,7 +189,7 @@ export const descentParser = fr => {
         /** @type {_Stack} */
         let stack = null
         /** @type {_Task | null} */
-        let task = { name, tag, pos: startPos }
+        let task = { kind: 'rule', name, tag, pos: startPos }
         /** @type {_Result<T>} */
         let result = mrFail(undefined, [], startPos)
         // High-water mark across the whole match, so it survives the rewinds a
@@ -196,16 +202,25 @@ export const descentParser = fr => {
                 // The explicit annotation cuts a control-flow inference cycle (TS7022):
                 // `name`'s narrowed type feeds `rule`, whose type would otherwise feed the
                 // later `task` assignments that `name`'s narrowing depends on.
-                const { name, tag, pos } = /** @type {_Task} */ (task)
+                const current = /** @type {_Task} */ (task)
                 task = null
+                if (current.kind === 'repeat') {
+                    // One round of a repetition is one match of its item, so the
+                    // frame below carries the rounds already collected and the
+                    // position this round may have to rewind to.
+                    stack = { top: { kind: 'repeat', tag: current.tag, item: current.item, items: current.items, roundStart: current.pos }, rest: stack }
+                    task = { kind: 'rule', name: current.item, tag: undefined, pos: current.pos }
+                    continue
+                }
+                const { name, tag, pos } = current
                 /** @type {DataRule} */
-                const rule = data[0][name]
+                const rule = ruleSet[name]
                 if (typeof rule === 'number') {
                     // No nullable case: `emptyTagOf` in `bnf/data` returns `undefined`
                     // for every terminal, so `emptyTags[name]` here is always
                     // `undefined` and a terminal either consumes one symbol or fails.
                     // Past the synthesized EOF there is no symbol left to consume.
-                    if (pos <= cp.length && rangeContains(...rangeDecode(rule))(symbolAt(cp, pos))) {
+                    if (pos <= cp.length && rangeContains(...rangeDecode(rule))(symbolAtCp(cp, pos))) {
                         result = mrSuccess(tag, leafAt(cp, pos), pos + 1)
                     } else {
                         // The only place a terminal is rejected, so the only place
@@ -218,8 +233,10 @@ export const descentParser = fr => {
                         result = mrSuccess(tag, [], pos)
                     } else {
                         stack = { top: { kind: 'seq', tag, items: rule, itemIndex: 0, startPos: pos, seq: [] }, rest: stack }
-                        task = { name: rule[0], tag: undefined, pos }
+                        task = { kind: 'rule', name: rule[0], tag: undefined, pos }
                     }
+                } else if (isRepeat(rule)) {
+                    task = { kind: 'repeat', tag, item: rule, items: null, pos }
                 } else {
                     const entries = definedEntries(rule)
                     const emptyTag = emptyTags[name]
@@ -229,7 +246,7 @@ export const descentParser = fr => {
                     } else {
                         stack = { top: { kind: 'variant', entries, entryIndex: 0, pos, emptyResult }, rest: stack }
                         const [entryTag, entryName] = entries[0]
-                        task = { name: entryName, tag: entryTag, pos }
+                        task = { kind: 'rule', name: entryName, tag: entryTag, pos }
                     }
                 }
                 continue
@@ -256,11 +273,29 @@ export const descentParser = fr => {
                     const itemIndex = frame.itemIndex + 1
                     if (itemIndex < frame.items.length) {
                         stack = { top: { ...frame, itemIndex, seq }, rest: stack }
-                        task = { name: frame.items[itemIndex], tag: undefined, pos }
+                        task = { kind: 'rule', name: frame.items[itemIndex], tag: undefined, pos }
                     } else {
                         result = mrSuccess(frame.tag, seq, pos)
                     }
                 }
+            } else if (frame.kind === 'repeat') {
+                if (result.success === false) {
+                    // A round that fails ends the repetition instead of failing
+                    // it: the rounds before it stand, and the whole match rewinds
+                    // to where the failed round began.
+                    result = mrSuccess(frame.tag, toArray(frame.items), frame.roundStart)
+                } else {
+                    const items = concat(frame.items)([result.ast])
+                    if (result.pos === frame.roundStart) {
+                        // A round that consumed nothing would repeat forever. Only
+                        // a hand-written `repeat` over a nullable rule reaches
+                        // this — `toData` never derives one — so keep the one
+                        // empty round and stop.
+                        result = mrSuccess(frame.tag, toArray(items), result.pos)
+                    } else {
+                        task = { kind: 'repeat', tag: frame.tag, item: frame.item, items, pos: result.pos }
+                    }
+                }
             } else {
                 // success that consumed input wins immediately: the frame stays popped and
                 // `result` propagates to the frame below, matching the recursive `return m`.
@@ -271,7 +306,7 @@ export const descentParser = fr => {
                     if (entryIndex < frame.entries.length) {
                         stack = { top: { ...frame, entryIndex, emptyResult }, rest: stack }
                         const [entryTag, entryName] = frame.entries[entryIndex]
-                        task = { name: entryName, tag: entryTag, pos: frame.pos }
+                        task = { kind: 'rule', name: entryName, tag: entryTag, pos: frame.pos }
                     } else {
                         result = emptyResult
                     }
@@ -287,3 +322,13 @@ export const descentParser = fr => {
 
     return match
 }
+
+/**
+ * Creates a recursive descent parser that preserves metadata for each consumed
+ * code point.
+ *
+ * @template T
+ * @param {FRule} fr
+ * @returns {DescentMatch<T>}
+ */
+export const descentParser = fr => descentParserRuleSet(toData(fr)[0])

package/fjs/bnf/descent/proof.f.d.mts

@@ -2,6 +2,7 @@
  * @module
  *
  * @import { CodePoint } from '../../text/utf16/types.ts'
+ * @import { RuleSet } from '../data/types.ts'
  * @import { DescentMatch, CodePointMeta, DescentMatchResult } from './types.ts'
  */
 export declare const proof: {
@@ -10,5 +11,6 @@ export declare const proof: {
     longInput: (() => void)[];
     descentParserWithMeta: (() => void)[];
     logicalEof: (() => void)[];
+    repeat: (() => void)[];
     furthestFailure: (() => void)[];
 };

package/fjs/bnf/descent/proof.f.mjs

@@ -2,6 +2,7 @@
  * @module
  *
  * @import { CodePoint } from '../../text/utf16/types.ts'
+ * @import { RuleSet } from '../data/types.ts'
  * @import { DescentMatch, CodePointMeta, DescentMatchResult } from './types.ts'
  */
 
@@ -9,9 +10,9 @@ import { stringToCodePointList } from '../../text/utf16/module.f.mjs'
 import { map, toArray } from '../../types/list/module.f.mjs'
 import { commaJoin0Plus, eof, option, range, repeat0Plus, set } from '../module.f.mjs'
 import { emptyTagMap, toData } from '../data/module.f.mjs'
-import { descentParser } from './module.f.mjs'
+import { descentParser, descentParserRuleSet } from './module.f.mjs'
 import { assertEq, assertNotNullish } from '../../asserts/module.f.mjs'
-import { deterministic } from '../testlib.f.mjs'
+import { deterministic, showAst } from '../testlib.f.mjs'
 
 /** @type {(cp: CodePoint) => CodePointMeta<unknown>} */
 const mapCodePoint = cp => [cp, undefined]
@@ -77,11 +78,14 @@ export const proof = {
             if (result !== '{"0":"none","3":true,"":true,"r":"none"}') { throw result }
         },
         () => {
+            // The whole repetition is one `repeat` rule, and a repetition always
+            // matches empty — with no tag, because it is a list of items rather
+            // than a choice between branches.
             const repeatRule = repeat0Plus(set(' \n\r\t'))
             const data = toData(repeatRule)
             const emptyTags = emptyTagMap(data[0])
             const result = JSON.stringify(emptyTags)
-            if (result !== '{"5":true,"r":"none"}') { throw result }
+            assertEq(result, '{"r":true}')
         }
     ],
     descentParser: [
@@ -254,33 +258,62 @@ export const proof = {
         () => {
             const m = descentParser(deterministic())
 
-            const expect = (/** @type {string} */s, /** @type {boolean} */expected) => {
+            // A match is pinned by the AST it built, not just by the fact that it
+            // matched: the shape is what a repetition changes, and asserting it
+            // subsumes success — only a full match produces one. `showAst` writes
+            // a node as `tag(children)` with the code points it consumed as one
+            // quoted string, `*` for the tagless empty match.
+            //
+            // The leading three spaces are the `repeat` to read first: they are
+            // three siblings of one node here, and three levels of nesting in the
+            // LL(1) backend's copy of these expectations.
+            /** @type {(s: string, expected: string) => void} */
+            const expectAst = (s, expected) => {
                 const cp = toArray(stringToCodePointList(s))
                 const mr = descentParserCpOnly(m, '', cp)
-                const success = mr.success && mr.idx === cp.length
-                assertEq(success, expected, mr)
+                assertEq(mr.success, true, s)
+                assertEq(mr.idx, cp.length, s)
+                assertEq(showAst(mr.ast), expected, s)
+            }
+
+            /** @type {(s: string) => void} */
+            const expectNoMatch = s => {
+                const cp = toArray(stringToCodePointList(s))
+                const mr = descentParserCpOnly(m, '', cp)
+                assertEq(mr.success && mr.idx === cp.length, false, s)
             }
 
-            expect('   true   ', true)
-            expect('   tr2ue   ', false)
-            expect('   true"   ', false)
-            expect('   "Hello"   ', true)
-            expect('   "Hello   ', false)
-            expect('   "Hello\\n\\r\\""   ', true)
-            expect('   -56.7e+5  ', true)
-            expect('   h-56.7e+5   ', false)
-            expect('   -56.7e+5   3', false)
-            expect('   [] ', true)
-            expect('   {} ', true)
-            expect('   [[[]]] ', true)
-            expect('   [1] ', true)
-            expect('   [ 12, false, "a"]  ', true)
-            expect('   [ 12, false2, "a"]  ', false)
-            expect('   { "q": [ 12, false, [{"b" : "c"}], "a"] }  ', true)
-            expect('   { "q": [ 12, false, [{}], "a"] }  ', true)
-            expect('   { "q": [ 12, false, [}], "a"] }  ', false)
-            expect('   [{ "q": [ 12, false, [{}], "a"] }]  ', true)
-            expect('   [{ "q": [ 12, false, [}], "a"] }]  ', false)
+            expectAst('   true   ', '((" "(" ") " "(" ") " "(" ")) "true"(("t") ("r") ("u") ("e")) (" "(" ") " "(" ") " "(" ")))')
+            expectAst('   "Hello"   ', '((" "(" ") " "(" ") " "(" ")) "string"((("\\"")) ("0x2300005b"("H") "0x5d10ffff"("e") "0x5d10ffff"("l") "0x5d10ffff"("l") "0x5d10ffff"("o")) (("\\""))) (" "(" ") " "(" ") " "(" ")))')
+            expectAst('   "Hello\\n\\r\\""   ', '((" "(" ") " "(" ") " "(" ")) "string"((("\\"")) ("0x2300005b"("H") "0x5d10ffff"("e") "0x5d10ffff"("l") "0x5d10ffff"("l") "0x5d10ffff"("o") "escape"((("\\\\")) "n"("n")) "escape"((("\\\\")) "r"("r")) "escape"((("\\\\")) "\\""("\\""))) (("\\""))) (" "(" ") " "(" ") " "(" ")))')
+            expectAst('   -56.7e+5  ', '((" "(" ") " "(" ") " "(" ")) "number"("some"(("-")) "onenine"(("5") (("6"))) "some"(((".")) (("7") ())) "some"("e"("e") "+"("+") (("5") ()))) (" "(" ") " "(" ")))')
+            expectAst('   [] ', '((" "(" ") " "(" ") " "(" ")) "array"((("[")) () "none"() (("]"))) (" "(" ")))')
+            expectAst('   {} ', '((" "(" ") " "(" ") " "(" ")) "object"((("{")) () "none"() (("}"))) (" "(" ")))')
+            expectAst('   [[[]]] ', '((" "(" ") " "(" ") " "(" ")) "array"((("[")) () "some"(("array"((("[")) () "some"(("array"((("[")) () "none"() (("]"))) ()) "none"()) (("]"))) ()) "none"()) (("]"))) (" "(" ")))')
+            expectAst('   [1] ', '((" "(" ") " "(" ") " "(" ")) "array"((("[")) () "some"(("number"("none"() "onenine"(("1") ()) "none"() "none"()) ()) "none"()) (("]"))) (" "(" ")))')
+            expectAst('   [ 12, false, "a"]  ', '((" "(" ") " "(" ") " "(" ")) "array"((("[")) (" "(" ")) "some"(("number"("none"() "onenine"(("1") (("2"))) "none"() "none"()) ()) "some"(((((",")) (" "(" "))) ("false"(("f") ("a") ("l") ("s") ("e")) ())) "some"(((((",")) (" "(" "))) ("string"((("\\"")) ("0x5d10ffff"("a")) (("\\""))) ())) "none"()))) (("]"))) (" "(" ") " "(" ")))')
+            expectAst('   { "q": [ 12, false, [{"b" : "c"}], "a"] }  ', '((" "(" ") " "(" ") " "(" ")) "object"((("{")) (" "(" ")) "some"(((((("\\"")) ("0x5d10ffff"("q")) (("\\""))) () ((":")) (" "(" ")) "array"((("[")) (" "(" ")) "some"(("number"("none"() "onenine"(("1") (("2"))) "none"() "none"()) ()) "some"(((((",")) (" "(" "))) ("false"(("f") ("a") ("l") ("s") ("e")) ())) "some"(((((",")) (" "(" "))) ("array"((("[")) () "some"(("object"((("{")) () "some"(((((("\\"")) ("0x5d10ffff"("b")) (("\\""))) (" "(" ")) ((":")) (" "(" ")) "string"((("\\"")) ("0x5d10ffff"("c")) (("\\"")))) ()) "none"()) (("}"))) ()) "none"()) (("]"))) ())) "some"(((((",")) (" "(" "))) ("string"((("\\"")) ("0x5d10ffff"("a")) (("\\""))) ())) "none"())))) (("]")))) (" "(" "))) "none"()) (("}"))) (" "(" ") " "(" ")))')
+            expectAst('   { "q": [ 12, false, [{}], "a"] }  ', '((" "(" ") " "(" ") " "(" ")) "object"((("{")) (" "(" ")) "some"(((((("\\"")) ("0x5d10ffff"("q")) (("\\""))) () ((":")) (" "(" ")) "array"((("[")) (" "(" ")) "some"(("number"("none"() "onenine"(("1") (("2"))) "none"() "none"()) ()) "some"(((((",")) (" "(" "))) ("false"(("f") ("a") ("l") ("s") ("e")) ())) "some"(((((",")) (" "(" "))) ("array"((("[")) () "some"(("object"((("{")) () "none"() (("}"))) ()) "none"()) (("]"))) ())) "some"(((((",")) (" "(" "))) ("string"((("\\"")) ("0x5d10ffff"("a")) (("\\""))) ())) "none"())))) (("]")))) (" "(" "))) "none"()) (("}"))) (" "(" ") " "(" ")))')
+            expectAst('   [{ "q": [ 12, false, [{}], "a"] }]  ', '((" "(" ") " "(" ") " "(" ")) "array"((("[")) () "some"(("object"((("{")) (" "(" ")) "some"(((((("\\"")) ("0x5d10ffff"("q")) (("\\""))) () ((":")) (" "(" ")) "array"((("[")) (" "(" ")) "some"(("number"("none"() "onenine"(("1") (("2"))) "none"() "none"()) ()) "some"(((((",")) (" "(" "))) ("false"(("f") ("a") ("l") ("s") ("e")) ())) "some"(((((",")) (" "(" "))) ("array"((("[")) () "some"(("object"((("{")) () "none"() (("}"))) ()) "none"()) (("]"))) ())) "some"(((((",")) (" "(" "))) ("string"((("\\"")) ("0x5d10ffff"("a")) (("\\""))) ())) "none"())))) (("]")))) (" "(" "))) "none"()) (("}"))) ()) "none"()) (("]"))) (" "(" ") " "(" ")))')
+
+            expectNoMatch('   tr2ue   ')
+            expectNoMatch('   true"   ')
+            expectNoMatch('   "Hello   ')
+            expectNoMatch('   h-56.7e+5   ')
+            expectNoMatch('   -56.7e+5   3')
+            expectNoMatch('   [ 12, false2, "a"]  ')
+            expectNoMatch('   { "q": [ 12, false, [}], "a"] }  ')
+
+            // The invalid input in detail. A failed match's own index rewound to
+            // the start and locates nothing; the furthest failure is the
+            // high-water mark, and it lands on the `}` that closes nothing.
+            const bad = '   [{ "q": [ 12, false, [}], "a"] }]  '
+            const badMr = descentParserCpOnly(m, '', toArray(stringToCodePointList(bad)))
+            assertEq(badMr.success, false, bad)
+            assertEq(badMr.idx, 0, bad)
+            const failure = assertNotNullish(badMr.failure)
+            assertEq(failure.idx, 25, bad)
+            assertEq(bad[failure.idx], '}', bad)
         }
     ],
     // Regression for the stack-recursive matcher bug: the matcher used to recurse
@@ -374,11 +407,12 @@ export const proof = {
         },
         () => {
             // Repetition terminates on EOF: consuming it moves the complete
-            // cursor, so the repeat makes exactly one round and then takes its
-            // empty branch — with `idx` alone this would never stop.
-            const m = descentParser(repeat0Plus(eof))
-            const mr = m('', [])
-            assertEq(JSON.stringify(mr), '{"ast":{"sequence":[{"sequence":[]},{"tag":"none","sequence":[]}]},"success":true,"idx":0}')
+            // cursor, so the repeat makes exactly one round and then stops —
+            // with `idx` alone this would never stop.
+            const rule = repeat0Plus(eof)
+            const m = descentParser(rule)
+            const mr = m(toData(rule)[1], [])
+            assertEq(JSON.stringify(mr), '{"ast":{"sequence":[{"sequence":[]}]},"success":true,"idx":0}')
         },
         () => {
             // Backtracking restores the complete cursor: `x` consumes EOF and
@@ -399,6 +433,58 @@ export const proof = {
             assertEq(f.expected[0], range('BB'))
         },
     ],
+    repeat: [
+        () => {
+            // The whole repetition is one node holding a flat sequence of the
+            // items it matched — not the right-recursive chain of `some`/`none`
+            // nodes its functional spelling builds.
+            const rule = repeat0Plus(set(' \n\r\t'))
+            const m = descentParser(rule)
+            const name = toData(rule)[1]
+            const mr = descentParserCpOnly(m, name, toArray(stringToCodePointList('  ')))
+            assertEq(
+                JSON.stringify(mr),
+                `{"ast":{"sequence":[{"tag":" ","sequence":[[${cp1(' ')},null]]},{"tag":" ","sequence":[[${cp1(' ')},null]]}]},"success":true,"idx":2}`)
+        },
+        () => {
+            // Zero items is a match, and the node is empty rather than tagged.
+            const rule = repeat0Plus(set(' \n\r\t'))
+            const m = descentParser(rule)
+            const mr = m(toData(rule)[1], [])
+            assertEq(JSON.stringify(mr), '{"ast":{"sequence":[]},"success":true,"idx":0}')
+        },
+        () => {
+            // A round that fails ends the repetition rather than failing it: the
+            // rounds before it stand and the match stops where the failed one
+            // began.
+            const rule = repeat0Plus(set(' \n\r\t'))
+            const m = descentParser(rule)
+            const name = toData(rule)[1]
+            const mr = descentParserCpOnly(m, name, toArray(stringToCodePointList(' x')))
+            assertEq(
+                JSON.stringify(mr),
+                `{"ast":{"sequence":[{"tag":" ","sequence":[[${cp1(' ')},null]]}]},"success":true,"idx":1}`)
+        },
+        () => {
+            // `toData` never folds a nullable item into a `repeat`, but a
+            // hand-written rule set can hold one. A round that consumes nothing
+            // would repeat forever, so it is kept once and ends the repetition.
+            /** @type {RuleSet} */
+            const ruleSet = {
+                repeated: 'optionalA',
+                optionalA: { some: 'a', none: 'e' },
+                a: range('aa'),
+                e: [],
+            }
+            const m = descentParserRuleSet(ruleSet)
+            assertEq(
+                JSON.stringify(m('repeated', [])),
+                '{"ast":{"sequence":[{"tag":"none","sequence":[]}]},"success":true,"idx":0}')
+            assertEq(
+                JSON.stringify(m('repeated', [[cp1('a'), null]])),
+                `{"ast":{"sequence":[{"tag":"some","sequence":[[${cp1('a')},null]]},{"tag":"none","sequence":[]}]},"success":true,"idx":1}`)
+        },
+    ],
     furthestFailure: [
         () => {
             // A branch rejected *before* the high-water mark must not pull it back:

package/fjs/bnf/descent/types.d.ts

@@ -1,11 +1,16 @@
 /**
  * Types for the recursive descent matcher backend.
  *
+ * The AST it builds is the shared one, over leaves that carry metadata:
+ * `Ast<CodePointMeta<T>>` from [`../matcher`](../matcher). What is declared
+ * here is what belongs to *this* backend — the metadata-carrying leaf, the
+ * diagnostics a backtracking matcher can report, and its public result.
+ *
  * @module
  */
 import type { CodePoint } from '../../text/utf16/types.ts';
+import type { Ast, AstTag } from '../matcher/types.ts';
 import type { TerminalRange } from '../types.ts';
-export type AstTag = string | true | undefined;
 /**
  * Recursive descent matcher for a single named rule, starting from `startPos` —
  * a position that counts the synthesized end-of-input symbol, unlike the public
@@ -53,7 +58,7 @@ export type DescentFailure = {
  * stopped just before it.
  */
 export type DescentMatchResult<T> = {
-    readonly ast: AstRuleMeta<T>;
+    readonly ast: Ast<CodePointMeta<T>>;
     readonly success: boolean;
     readonly idx: number;
     readonly failure?: DescentFailure;
@@ -63,17 +68,10 @@ export type DescentMatchResult<T> = {
  */
 export type DescentMatch<T> = (name: string, s: readonly CodePointMeta<T>[]) => DescentMatchResult<T>;
 /**
- * Code point value paired with metadata.
+ * Code point value paired with metadata: this backend's AST leaf.
+ *
+ * Preserving metadata per consumed code point is what distinguishes this
+ * backend, so the leaf type stays here rather than moving to the shared layer,
+ * which is parameterized by it.
  */
 export type CodePointMeta<T> = readonly [CodePoint, T];
-/**
- * AST sequence for the metadata-aware parser.
- */
-export type AstSequenceMeta<T> = readonly (AstRuleMeta<T> | CodePointMeta<T>)[];
-/**
- * Metadata-aware AST node.
- */
-export type AstRuleMeta<T> = {
-    readonly tag: AstTag;
-    readonly sequence: AstSequenceMeta<T>;
-};