functionalscript 0.43.1 → 0.45.0

package/fjs/bnf/data/module.f.d.mts

@@ -0,0 +1,47 @@
+/**
+ * The serializable BNF intermediate representation (IR) and the
+ * {@link toData} conversion from the functional grammar.
+ *
+ * This module is the pure, parser-agnostic substrate: it defines the
+ * {@link RuleSet} data form (`Rule = Variant | Sequence | TerminalRange`) and
+ * converts a functional grammar into it. The automaton builders that consume a
+ * {@link RuleSet} live in their own sibling modules (`fjs/bnf/ll1`,
+ * `fjs/bnf/descent`, …), so the IR stays free of any one parser's machinery.
+ *
+ * See `./types.ts` for the type-level API.
+ *
+ * @module
+ *
+ * @import { DataRule, Rule as FRule, Sequence as FSequence } from '../types.ts'
+ * @import { StringMap } from '../../types/object/types.ts'
+ * @import { EmptyTag, Rule, RuleSet, Sequence, Variant } from './types.ts'
+ */
+import type { Rule as FRule } from '../types.ts';
+import type { StringMap } from '../../types/object/types.ts';
+import type { EmptyTag, Rule, RuleSet } from './types.ts';
+export type _EmptyTagMap = StringMap<EmptyTag>;
+/**
+ * Computes, for every rule in the set, whether it can match empty input, by
+ * the standard nullable-set fixpoint: a sequence is nullable iff all of its
+ * items are (AND semantics), a variant iff at least one branch is (its tag is
+ * that branch's). Rules may reference each other cyclically (e.g. a `repeat`
+ * rule referring to itself), so this starts every rule as non-nullable and
+ * relaxes every rule, one round at a time, until a full round changes
+ * nothing. A round only ever grows the nullable set or moves a variant's
+ * chosen tag to a later (still-nullable) branch, both bounded, so this always
+ * terminates — but a rule's tag can still change for rounds *after* its own
+ * nullable/non-nullable status has already settled, while a cyclic
+ * dependency's tag catches up, so a fixed round count isn't enough.
+ *
+ * @type {(ruleSet: RuleSet) => _EmptyTagMap}
+ */
+export declare const emptyTagMap: (ruleSet: RuleSet) => _EmptyTagMap;
+export type _FRuleMap = StringMap<FRule>;
+export type _NewRule = (m: _FRuleMap) => readonly [_FRuleMap, RuleSet, Rule];
+/**
+ * Converts a functional grammar rule into serializable BNF data and returns
+ * the generated rule set with the entry rule identifier.
+ *
+ * @type {(fr: FRule) => readonly [RuleSet, string]}
+ */
+export declare const toData: (fr: FRule) => readonly [RuleSet, string];

package/fjs/bnf/data/module.f.mjs

@@ -0,0 +1,188 @@
+/**
+ * The serializable BNF intermediate representation (IR) and the
+ * {@link toData} conversion from the functional grammar.
+ *
+ * This module is the pure, parser-agnostic substrate: it defines the
+ * {@link RuleSet} data form (`Rule = Variant | Sequence | TerminalRange`) and
+ * converts a functional grammar into it. The automaton builders that consume a
+ * {@link RuleSet} live in their own sibling modules (`fjs/bnf/ll1`,
+ * `fjs/bnf/descent`, …), so the IR stays free of any one parser's machinery.
+ *
+ * See `./types.ts` for the type-level API.
+ *
+ * @module
+ *
+ * @import { DataRule, Rule as FRule, Sequence as FSequence } from '../types.ts'
+ * @import { StringMap } from '../../types/object/types.ts'
+ * @import { EmptyTag, Rule, RuleSet, Sequence, Variant } from './types.ts'
+ */
+
+import { stringToCodePointList } from '../../text/utf16/module.f.mjs'
+import { map, toArray } from '../../types/list/module.f.mjs'
+import { oneEncode } from '../module.f.mjs'
+import { definedEntries } from '../../types/object/module.f.mjs'
+
+/** @typedef {StringMap<EmptyTag>} _EmptyTagMap */
+
+/** @type {(map: _EmptyTagMap) => (rule: Rule) => EmptyTag} */
+const emptyTagOf = map => rule => {
+    if (typeof rule === 'number') {
+        return undefined
+    } else if (rule instanceof Array) {
+        return rule.every(item => map[item] !== undefined) ? true : undefined
+    } else {
+        /** @type {EmptyTag} */
+        let tag = undefined
+        for (const [k, item] of definedEntries(rule)) {
+            if (map[item] !== undefined) {
+                tag = k
+            }
+        }
+        return tag
+    }
+}
+
+/** @type {(ruleSet: RuleSet) => (map: _EmptyTagMap) => readonly [_EmptyTagMap, boolean]} */
+const emptyTagStep = ruleSet => map => {
+    let next = map
+    let changed = false
+    for (const name in ruleSet) {
+        const tag = emptyTagOf(map)(ruleSet[name])
+        if (tag !== next[name]) {
+            changed = true
+        }
+        next = { ...next, [name]: tag }
+    }
+    return [next, changed]
+}
+
+/**
+ * Computes, for every rule in the set, whether it can match empty input, by
+ * the standard nullable-set fixpoint: a sequence is nullable iff all of its
+ * items are (AND semantics), a variant iff at least one branch is (its tag is
+ * that branch's). Rules may reference each other cyclically (e.g. a `repeat`
+ * rule referring to itself), so this starts every rule as non-nullable and
+ * relaxes every rule, one round at a time, until a full round changes
+ * nothing. A round only ever grows the nullable set or moves a variant's
+ * chosen tag to a later (still-nullable) branch, both bounded, so this always
+ * terminates — but a rule's tag can still change for rounds *after* its own
+ * nullable/non-nullable status has already settled, while a cyclic
+ * dependency's tag catches up, so a fixed round count isn't enough.
+ *
+ * @type {(ruleSet: RuleSet) => _EmptyTagMap}
+ */
+export const emptyTagMap = ruleSet => {
+    const step = emptyTagStep(ruleSet)
+    /** @type {(map: _EmptyTagMap) => _EmptyTagMap} */
+    const relax = map => {
+        const [next, changed] = step(map)
+        return changed ? relax(next) : next
+    }
+    return relax({})
+}
+
+//
+
+/** @typedef {StringMap<FRule>} _FRuleMap */
+
+const { entries } = Object
+
+/** @type {(map: _FRuleMap) => (fr: FRule) => string | undefined} */
+const find = map => fr => {
+    for (const [k, v] of entries(map)) {
+        if (v === fr) {
+            return k
+        }
+    }
+    return undefined
+}
+
+/** @type {(map: _FRuleMap, name: string) => string} */
+const newName = (map, name) => {
+    let i = 0
+    let result = name
+    while (result in map) {
+        result = name + i
+        ++i
+    }
+    return result
+}
+
+/** @typedef {(m: _FRuleMap) => readonly [_FRuleMap, RuleSet, Rule]} _NewRule */
+
+/** @type {(list: FSequence) => _NewRule} */
+const sequence = list => map => {
+    /** @type {Sequence} */
+    let result = []
+    /** @type {RuleSet} */
+    let set = {}
+    for (const fr of list) {
+        const [map1, set1, id] = toDataAdd(map)(fr)
+        map = map1
+        set = { ...set, ...set1 }
+        result = [...result, id]
+    }
+    return [map, set, result]
+}
+
+/** @type {(fr: FRule) => _NewRule} */
+const variant = fr => map => {
+    /** @type {RuleSet} */
+    let set = {}
+    /** @type {Variant} */
+    let rule = {}
+    for (const [k, v] of entries(fr)) {
+        const [m1, s, id] = toDataAdd(map)(v)
+        map = m1
+        set = { ...set, ...s }
+        rule = { ...rule, [k]: id }
+    }
+    return [map, set, rule]
+}
+
+const mapOneEncode = map(oneEncode)
+
+/** @type {(dr: DataRule) => _NewRule} */
+const data = dr => {
+    switch (typeof dr) {
+        case 'string': {
+            return sequence(toArray(mapOneEncode(stringToCodePointList(dr))))
+        }
+        case 'number':
+            return m => [m, {}, dr]
+        default:
+            if (dr instanceof Array) {
+                return sequence(dr)
+            }
+            return variant(dr)
+    }
+}
+
+/** @type {(map: _FRuleMap) => (fr: FRule) => readonly [_FRuleMap, RuleSet, string]} */
+const toDataAdd = map => fr => {
+    {
+        const id = find(map)(fr)
+        if (id !== undefined) {
+            return [map, {}, id]
+        }
+    }
+    /** @type {readonly [DataRule, string]} */
+    const [dr, tmpId] =
+        typeof fr === 'function' ? [fr(), fr.name] : [fr, '']
+    const newRule = data(dr)
+    const id = newName(map, tmpId)
+    const map1 = { ...map, [id]: fr }
+    const [map2, set, rule] = newRule(map1)
+    return [map2, { ...set, [id]: rule }, id]
+}
+
+/**
+ * Converts a functional grammar rule into serializable BNF data and returns
+ * the generated rule set with the entry rule identifier.
+ *
+ * @type {(fr: FRule) => readonly [RuleSet, string]}
+ */
+export const toData = fr => {
+    const [, ruleSet, id] = toDataAdd({})(fr)
+    return [ruleSet, id]
+}

package/fjs/bnf/data/proof.f.d.mts

@@ -0,0 +1,13 @@
+/**
+ * @module
+ *
+ * @import { RuleSet } from './types.ts'
+ */
+export declare const proof: {
+    rangeDecode: () => void;
+    rangeEncode: () => void;
+    toData: (() => void)[];
+    variantTest: () => void;
+    emptyTagMap: (() => void)[];
+    example: () => void;
+};

package/fjs/bnf/data/proof.f.mjs

@@ -0,0 +1,159 @@
+/**
+ * @module
+ *
+ * @import { RuleSet } from './types.ts'
+ */
+
+import { identity } from '../../types/function/module.f.mjs'
+import { sort } from '../../types/object/module.f.mjs'
+import { oneEncode, option, range, rangeDecode, repeat0Plus, set } from '../module.f.mjs'
+import { emptyTagMap, toData } from './module.f.mjs'
+import { assertEq } from '../../asserts/module.f.mjs'
+import { stringify } from '../../media/json/module.f.mjs'
+import { classic, deterministic } from '../testlib.f.mjs'
+
+export const proof = {
+    rangeDecode: () => {
+        const decoded1 = stringify(sort)(rangeDecode(0x000079_000087))
+        assertEq(decoded1, '[121,135]')
+
+        const decoded2 = stringify(sort)(rangeDecode(0x000080_000087))
+        assertEq(decoded2, '[128,135]')
+
+        const decoded3 = stringify(sort)(rangeDecode(0x10FFFF_10FFFF))
+        assertEq(decoded3, '[1114111,1114111]')
+    },
+    rangeEncode: () => {
+        const encoded1 = oneEncode(0x79)
+        if (encoded1 !== 0x000079_000079) {throw encoded1}
+
+        const encoded2 = oneEncode(0x80)
+        if (encoded2 !== 0x000080_000080) {throw encoded2}
+
+        const encoded3 = oneEncode(0x10FFFF)
+        if (encoded3 !== 0x10FFFF_10FFFF) {throw encoded3}
+    },
+    toData: [
+        () => {
+            const c = toData(classic())
+            const d = toData(deterministic())
+        },
+        () => {
+            const stringRule = 'true'
+            const result = stringify(sort)(toData(stringRule))
+            if (result !== '[{"":["0","1","2","3"],"0":1946157172,"1":1912602738,"2":1962934389,"3":1694498917},""]') { throw result }
+        },
+        () => {
+            const terminalRangeRule = range('AF')
+            const result = stringify(sort)(toData(terminalRangeRule))
+            if (result !== '[{"":1090519110},""]') { throw result } //1090519110 = 65 * 2^24 + 70
+        },
+        () => {
+            const sequenceRangeRule = [range('AF'), range('af')]
+            const result = stringify(sort)(toData(sequenceRangeRule))
+            if (result !== '[{"":["0","1"],"0":1090519110,"1":1627390054},""]') { throw result }
+        },
+        () => {
+            const lazyRule = () => 'true'
+            const result = stringify(sort)(toData(lazyRule))
+            if (result !== '[{"":1946157172,"0":1912602738,"1":1962934389,"2":1694498917,"lazyRule":["","0","1","2"]},"lazyRule"]') { throw result }
+        },
+        () => {
+            const varintRule = { true: 'true', false: 'false'}
+            const result = stringify(sort)(toData(varintRule))
+            const expected = '[{"":{"false":"5","true":"0"},"0":["1","2","3","4"],"1":1946157172,"2":1912602738,"3":1962934389,"4":1694498917,"5":["6","7","8","9","4"],"6":1711276134,"7":1627390049,"8":1811939436,"9":1929379955},""]'
+            assertEq(result, expected, [result, expected])
+        },
+        () => {
+            const lazyRule = () => 'true'
+            const lazyRule0 = () => 'false'
+            const result = stringify(sort)(toData([lazyRule, lazyRule0]))
+            const expected = '[{"":["lazyRule","lazyRule0"],"0":1946157172,"1":1912602738,"2":1962934389,"3":1694498917,"4":1711276134,"5":1627390049,"6":1811939436,"7":1929379955,"lazyRule":["0","1","2","3"],"lazyRule0":["4","5","6","7","3"]},""]'
+            assertEq(result, expected, [result, expected])
+        },
+        () => {
+            const emptyRule = ''
+            const result = stringify(sort)(toData(emptyRule))
+            const expected = '[{"":[]},""]'
+            assertEq(result, expected, [result, expected])
+        },
+        () => {
+            const optionRule = option('a')
+            const result = stringify(identity)(toData(optionRule))
+            if (result !== '[{"0":["1"],"1":1627390049,"2":[],"":{"some":"0","none":"2"}},""]') { throw result }
+        },
+        () => {
+            const repeatRule = repeat0Plus(option('a'))
+            const result = stringify(identity)(toData(repeatRule))
+            if (result !== '[{"0":{"some":"1","none":"3"},"1":["2"],"2":1627390049,"3":[],"":["0","r"],"r":{"some":"","none":"3"}},"r"]') { throw result }
+        },
+        () => {
+            const repeatRule = repeat0Plus(set(' \n\r\t'))
+            const result = stringify(identity)(toData(repeatRule))
+            if (result !== '[{"0":{" ":"1","\\n":"2","\\r":"3","\\t":"4"},"1":536870944,"2":167772170,"3":218103821,"4":150994953,"5":[],"":["0","r"],"r":{"some":"","none":"5"}},"r"]') { throw result }
+        }
+    ],
+    variantTest: () => {
+        const varintRule = { a: 'a', b: 'b'}
+        const result = stringify(sort)(toData(varintRule))
+        if (result !== '[{"":{"a":"0","b":"2"},"0":["1"],"1":1627390049,"2":["3"],"3":1644167266},""]') { throw result }
+    },
+    emptyTagMap: [
+        () => {
+            const stringRule = 'true'
+            const result = JSON.stringify(emptyTagMap(toData(stringRule)[0]))
+            assertEq(result, '{}')
+        },
+        () => {
+            const emptyRule = ''
+            const result = JSON.stringify(emptyTagMap(toData(emptyRule)[0]))
+            assertEq(result, '{"":true}')
+        },
+        () => {
+            const emptyRule = ''
+            const varintRule = { true: 'true', e: emptyRule }
+            const result = JSON.stringify(emptyTagMap(toData(varintRule)[0]))
+            assertEq(result, '{"5":true,"":"e"}')
+        },
+        () => {
+            const repeatRule = repeat0Plus(option('a'))
+            const result = JSON.stringify(emptyTagMap(toData(repeatRule)[0]))
+            assertEq(result, '{"0":"none","3":true,"":true,"r":"none"}')
+        },
+        () => {
+            // Regression for the sequence case: a sequence is nullable iff
+            // *all* of its items are, not just the first one. The old
+            // descent-only implementation pinned the result to nullable as
+            // soon as it saw the leading empty item, and reported this
+            // 2-item sequence as nullable even though its second item is a
+            // mandatory terminal.
+            const data = toData(['', range('AA')])
+            const emptyTags = emptyTagMap(data[0])
+            assertEq(emptyTags[data[1]], undefined, emptyTags)
+        },
+        () => {
+            // Regression: a fixed round count (one round per rule) settles
+            // the nullable/non-nullable *set* — a standard result — but a
+            // variant's chosen *tag* can still change for rounds after that,
+            // while a cyclic dependency's own tag catches up. Here A is
+            // nullable via 'x' (A -> E, both settle in 2 rounds), but B (a
+            // 1-item sequence over A) only becomes nullable once A does, and
+            // once B is nullable, A's true winning branch is 'y' (A -> B) —
+            // reachable only by iterating to an actual fixpoint.
+            /** @type {RuleSet} */
+            const rs = { A: { x: 'E', y: 'B' }, B: ['A'], E: [] }
+            assertEq(emptyTagMap(rs).A, 'y')
+        },
+    ],
+    example: () => {
+        const grammar = {
+            space: 0x000020_000020,
+            digit: 0x000030_000039,
+            sequence: ['space', 'digit'],
+            spaceOrDigit: {
+                'whiteSpace': 'space',
+                'digit': 'digit',
+            }
+        }
+    },
+}

package/fjs/bnf/data/types.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Types for the serializable BNF intermediate representation (IR).
+ *
+ * @module
+ */
+import type { StringMap } from '../../types/object/types.ts';
+/**
+ * Encoded terminal range value used by BNF data rules.
+ *
+ * The same as the functional TerminalRange.
+ */
+export type TerminalRange = number;
+/**
+ * Ordered list of grammar rule names.
+ */
+export type Sequence = readonly string[];
+/** A variant of rule names. */
+export type Variant = StringMap<string>;
+/**
+ * Grammar rule definition.
+ *
+ * It can be one of:
+ * - a tagged variant map,
+ * - a sequence of referenced rule names,
+ * - an encoded terminal range.
+ */
+export type Rule = Variant | Sequence | TerminalRange;
+/** The full grammar */
+export type RuleSet = Readonly<Record<string, Rule>>;
+/**
+ * Whether a rule can match empty input: `undefined` if it never can, `true`
+ * if it can with no tag (a nullable sequence), or the tag of the nullable
+ * variant branch.
+ */
+export type EmptyTag = string | true | undefined;

package/fjs/bnf/descent/module.f.d.mts

@@ -0,0 +1,48 @@
+/**
+ * Recursive descent matcher backend over the BNF data {@link RuleSet}.
+ *
+ * Built from the serializable IR in `fjs/bnf/data`, this is a sibling of the
+ * LL(1) dispatch builder (`fjs/bnf/ll1`). It walks the grammar by recursive
+ * descent and preserves per-code-point metadata, producing a metadata-aware
+ * AST ({@link AstRuleMeta}). Nullability (which rule can match empty input) is
+ * computed once by {@link emptyTagMap} in `fjs/bnf/data`.
+ *
+ * The caller passes physical symbols only; the matcher synthesizes the one
+ * logical EOF ({@link eofSymbol}) after them, so a grammar can require the end
+ * of input with the `eof` terminal.
+ *
+ * A failed result also carries a {@link DescentFailure}: the furthest position a
+ * terminal was rejected at, which — unlike the result's own index — never
+ * rewinds and is what diagnostics should be built from.
+ *
+ * See `./types.ts` for the type-level API.
+ *
+ * @module
+ *
+ * @import { TerminalRange } from '../types.ts'
+ * @import { Rule as DataRule, Sequence } from '../data/types.ts'
+ * @import { Rule as FRule } from '../types.ts'
+ * @import { AstRuleMeta, AstTag, AstSequenceMeta, CodePointMeta, DescentFailure, DescentMatch, DescentMatchResult, DescentMatchRule } from './types.ts'
+ */
+import type { TerminalRange } from '../types.ts';
+import type { Rule as FRule } from '../types.ts';
+import type { AstRuleMeta, DescentMatch } from './types.ts';
+export type _Cursor = number;
+export type _Failure = {
+    readonly pos: _Cursor;
+    readonly expected: readonly TerminalRange[];
+};
+export type _Result<T> = {
+    readonly ast: AstRuleMeta<T>;
+    readonly success: boolean;
+    readonly pos: _Cursor;
+};
+/**
+ * Creates a recursive descent parser that preserves metadata for each consumed
+ * code point.
+ *
+ * @template T
+ * @param {FRule} fr
+ * @returns {DescentMatch<T>}
+ */
+export declare const descentParser: <T>(fr: FRule) => DescentMatch<T>;