functionalscript 0.43.1 → 0.45.0

package/fjs/bnf/module.f.mjs

@@ -0,0 +1,339 @@
+/**
+ * Core BNF grammar primitives and helpers for describing parser rules.
+ *
+ * The module provides terminal-range encoding utilities, rule composition
+ * types, and set/range helpers used by FunctionalScript grammar definitions.
+ * See `./types.ts` for the type-level API.
+ *
+ * @module
+ *
+ * @import { Tuple } from '../types/array/types.ts'
+ * @import { TerminalRange, Sequence, Rule, RangeVariant, None, Option, Repeat0Plus, Repeat1Plus, Join1Plus, Repeat } from './types.ts'
+ */
+
+import { codePointListToString, stringToCodePointList } from '../text/utf16/module.f.mjs'
+import { definedValues } from '../types/object/module.f.mjs'
+import { isTuple } from '../types/array/module.f.mjs'
+import { map, toArray, repeat as listRepeat } from '../types/list/module.f.mjs'
+import { contains } from '../types/range/module.f.mjs'
+import { assert } from '../asserts/module.f.mjs'
+
+// Internals:
+
+const { fromEntries } = Object
+
+const { fromCodePoint } = String
+
+/**
+ * Two 24 bit numbers can be fit into one JS number (53 bit).
+ */
+const offset = 24
+
+const mask = (1 << offset) - 1
+
+/**
+ * How many stored endpoint codes one 24-bit half holds: `2 ** 24`.
+ */
+const terminalSize = mask + 1
+
+/**
+ * The logical end-of-input symbol.
+ *
+ * `-1` is outside the non-negative physical-symbol domain, so no alphabet
+ * adapter can produce it and no ordinary symbol has to be reserved for it.
+ * Parser backends synthesize it exactly once, after the physical input.
+ */
+export const eofSymbol = -1
+
+/**
+ * The largest ordinary symbol, `2 ** 24 - 2`: the semantic domain is
+ * `[-1] | [0, 2 ** 24 - 2]`, exactly `2 ** 24` terminals, one per stored code.
+ */
+const maxSymbol = mask - 1
+
+const isValid = contains(eofSymbol, maxSymbol)
+
+/**
+ * Semantic terminal to its stored 24-bit endpoint code. Branchless: EOF wraps
+ * to `2 ** 24 - 1` (the top of the stored space) and every ordinary symbol
+ * keeps its own value as its code.
+ *
+ * @type {(value: number) => number}
+ */
+const encodeTerminal = value => (value + terminalSize) & mask
+
+/**
+ * The inverse of `encodeTerminal`, also branchless.
+ *
+ * Stored codes are an implementation representation, not semantic terminal
+ * ordering — `2 ** 24 - 1` is the largest code but the smallest terminal — so
+ * range operations that care about ordering must compare decoded values.
+ *
+ * @type {(value: number) => number}
+ */
+const decodeTerminal = value => ((value + 1) & mask) - 1
+
+/** @type {(a: number, b: number) => TerminalRange} */
+export const rangeEncode = (a, b) => {
+    assert(isValid(a) && isValid(b) && a <= b)
+    return Number((BigInt(encodeTerminal(a)) << BigInt(offset)) | BigInt(encodeTerminal(b)))
+}
+
+/**
+ * Encodes a single symbol as a {@link TerminalRange}.
+ *
+ * @type {(a: number) => TerminalRange}
+ */
+export const oneEncode = a => rangeEncode(a, a)
+
+/**
+ * End-of-file marker: the singleton range of {@link eofSymbol}. Deliberately
+ * outside the ordinary symbol domain — and so outside Unicode — so it can
+ * never collide with a real code point.
+ *
+ * @type {TerminalRange}
+ */
+export const eof = oneEncode(eofSymbol)
+
+/**
+ * Every ordinary symbol packed into a single {@link TerminalRange}. EOF is not
+ * one of them, so complements over this range never include it.
+ *
+ * @type {TerminalRange}
+ */
+export const fullRange = rangeEncode(0, maxSymbol)
+
+/**
+ * Unicode scalar value range packed into a single {@link TerminalRange}.
+ * @type {TerminalRange}
+ */
+export const unicodeRange = 0x000000_10FFFF
+
+/**
+ * Maximal Unicode code point encoded as a string value.
+ */
+export const unicodeMax = codePointListToString([0x10FFFF])
+
+/**
+ * Decodes a packed range into `[start, end]` semantic symbols.
+ *
+ * @type {(r: number) => Tuple<2, number>}
+ */
+export const rangeDecode = r =>
+    [decodeTerminal(Number(BigInt(r) >> BigInt(offset))), decodeTerminal(r & mask)]
+
+const mapOneEncode = map(oneEncode)
+
+/** @type {(s: string) => readonly TerminalRange[]} */
+export const toSequence = s =>
+    toArray(mapOneEncode(stringToCodePointList(s)))
+
+/**
+ * Converts the whole string into one rule:
+ * - a single {@link TerminalRange} when the string has one symbol,
+ * - a sequence of {@link TerminalRange} values when the string has multiple symbols.
+ *
+ * @type {(s: string) => readonly TerminalRange[] | TerminalRange}
+ */
+export const str = s => {
+    const x = toSequence(s)
+    return x.length === 1 ? x[0] : x
+}
+
+const mapEntry = map((/** @type {number} */ v) => [fromCodePoint(v), oneEncode(v)])
+
+/**
+ * Converts a string into a variant that maps each character to its symbol range.
+ *
+ * @type {(s: string) => RangeVariant}
+ */
+export const set = s =>
+    fromEntries(toArray(mapEntry(stringToCodePointList(s))))
+
+const isTuple2 = isTuple(2)
+
+/**
+ * Encodes a two-symbol string into a terminal range.
+ *
+ * @throws If `ab` does not contain exactly two unicode code points.
+ *
+ * @type {(ab: string) => TerminalRange}
+ */
+export const range = ab => {
+    const a = toArray(stringToCodePointList(ab))
+    if (!isTuple2(a)) {
+        throw `Invalid range ${ab}.`
+    }
+    return rangeEncode(...a)
+}
+
+/** @typedef {readonly TerminalRange[]} _RangeList */
+
+/** @type {(r: TerminalRange) => readonly [string, TerminalRange]} */
+const rangeToEntry = r =>
+    ['0x' + r.toString(16), r]
+
+/** @type {(r: _RangeList) => RangeVariant} */
+const toVariantRangeSet = r =>
+    fromEntries(r.map(rangeToEntry))
+
+/** @type {(list: _RangeList, ab: number) => _RangeList} */
+const removeOne = (list, ab) => {
+    const [a, b] = rangeDecode(ab)
+    /** @type {_RangeList} */
+    let result = []
+    for (const ab0 of list) {
+        const [a0, b0] = rangeDecode(ab0)
+        if (a0 < a) {
+            // [a0
+            //     ]a
+            result = [...result, rangeEncode(a0, Math.min(b0, a - 1))]
+        }
+        if (b < b0) {
+            //    b0]
+            // b[
+            result = [...result, rangeEncode(Math.max(b + 1, a0), b0)]
+        }
+    }
+    return result
+}
+
+/** @type {(range: TerminalRange, v: RangeVariant) => RangeVariant} */
+export const remove = (range, v) => {
+    /** @type {_RangeList} */
+    let result = [range]
+    for (const r of definedValues(v)) {
+        result = removeOne(result, r)
+    }
+    return toVariantRangeSet(result)
+}
+
+/**
+ * Returns the complement set of the provided ranges over {@link fullRange}.
+ *
+ * @type {(v: RangeVariant) => RangeVariant}
+ */
+export const not = v =>
+    remove(fullRange, v)
+
+/**
+ * Returns the complement set of a character set over {@link fullRange}.
+ *
+ * @type {(s: string) => RangeVariant}
+ */
+export const notSet = s =>
+    not(set(s))
+
+/**
+ * Shared empty sequence literal.
+ *
+ * @type {None}
+ */
+export const none = []
+
+/**
+ * Creates an option value from a required branch.
+ */
+export const option =
+    /**
+     * @template {Rule} S
+     * @param {S} some
+     * @returns {Option<S>}
+     */
+    some => ({
+        some,
+        none,
+    })
+
+/**
+ * Repeat zero or more times.
+ *
+ * https://english.stackexchange.com/questions/506480/single-word-quantifiers-for-zero-or-more-like-cardinalities
+ * - zero or more - any, 0Plus
+ * - one or more - several, 1Plus
+ *
+ * Also see: https://arbs.nzcer.org.nz/types-numbers
+ */
+export const repeat0Plus =
+    /**
+     * @template {Rule} T
+     * @param {T} some
+     * @returns {Repeat0Plus<T>}
+     */
+    some => {
+        const r = () => option(/** @type {const} */ ([some, r]))
+        return r
+    }
+
+/**
+ * Repeat one or more times.
+ */
+export const repeat1Plus =
+    /**
+     * @template {Rule} T
+     * @param {T} some
+     * @returns {Repeat1Plus<T>}
+     */
+    some =>
+        [some, repeat0Plus(some)]
+
+/**
+ * Repeats `some` one or more times separated by `separator`.
+ */
+export const join1Plus =
+    /**
+     * @template {Rule} T
+     * @template {Rule} S
+     * @param {T} some
+     * @param {S} separator
+     * @returns {Join1Plus<T, S>}
+     */
+    (some, separator) =>
+        [some, repeat0Plus([separator, some])]
+
+/**
+ * Repeats `some` zero or more times separated by `separator`.
+ */
+export const join0Plus =
+    /**
+     * @template {Rule} T
+     * @template {Rule} S
+     * @param {T} some
+     * @param {S} separator
+     * @returns {Rule}
+     */
+    (some, separator) =>
+        option(join1Plus(some, separator))
+
+/**
+ * A delimited, comma-separated list with whitespace between tokens:
+ * `open ws (item ws (',' ws item ws)*)? close`.
+ *
+ * `ws` is the per-grammar whitespace rule and is curried so callers can
+ * partially apply it once (`const cj = commaJoin0Plus(ws)`) before reusing
+ * the same combinator at multiple bracket pairs. The two-character string
+ * supplies the bracket pair via destructuring (e.g. `'[]'`, `'{}'`).
+ *
+ * @type {(ws: Rule) => (bracketPair: string, item: Rule) => Sequence}
+ */
+export const commaJoin0Plus = ws =>
+    ([open, close], item) =>
+        [open, ws, join0Plus([item, ws], [',', ws]), close]
+
+/**
+ * Repeats a rule a fixed number of times.
+ *
+ * @type {(n: number) => <T extends Rule>(some: T) => Repeat<T>}
+ */
+export const repeat = n => some =>
+    toArray(listRepeat(some)(n))
+
+/**
+ * Determines whether the rule is an empty rule.
+ *
+ * @type {(rule: Rule) => boolean}
+ */
+export const isEmpty = rule => {
+    const d = typeof rule === 'function' ? rule() : rule
+    return d === '' || (d instanceof Array && d.length === 0)
+}

package/fjs/bnf/proof.f.d.mts

@@ -0,0 +1,19 @@
+/**
+ * @module
+ *
+ * @import { Rule } from './types.ts'
+ */
+export declare const proof: {
+    test: () => void;
+    throw: {
+        rangeEncodeInvalid: (() => void)[];
+        rangeInvalid: (() => void)[];
+    };
+    terminal: (() => void)[];
+    complement: (() => void)[];
+    str: (() => void)[];
+    set: () => void;
+    commaJoin0Plus: () => void;
+    isEmpty: (() => void)[];
+    repeat1Plus: (() => void)[];
+};

package/fjs/bnf/proof.f.mjs

@@ -0,0 +1,165 @@
+/**
+ * @module
+ *
+ * @import { Rule } from './types.ts'
+ */
+
+import {
+    assert,
+    assertEq,
+    assertStructurallySame,
+} from '../asserts/module.f.mjs'
+import {
+    eof,
+    eofSymbol,
+    fullRange,
+    not,
+    notSet,
+    rangeDecode,
+    rangeEncode,
+    str,
+    set,
+    range,
+    commaJoin0Plus,
+    isEmpty,
+    oneEncode,
+    repeat1Plus,
+} from './module.f.mjs'
+import { definedValues } from '../types/object/module.f.mjs'
+import { classic, deterministic } from './testlib.f.mjs'
+
+// The last ordinary symbol, `2 ** 24 - 2`. Written out because the range codec
+// is what these tests check: deriving it from the codec would move both sides
+// of every assertion together.
+const maxSymbol = 0xFFFFFE
+
+export const proof = {
+    test: () => {
+        classic()
+        deterministic()
+    },
+    throw: {
+        rangeEncodeInvalid: [
+            // one below `eofSymbol`, the smallest semantic terminal
+            () => { rangeEncode(-2, 0) },
+            () => { rangeEncode(0, -2) },
+            // one above the largest ordinary symbol: `2 ** 24 - 1` is EOF's
+            // stored code, not a terminal of its own
+            () => { rangeEncode(0, maxSymbol + 1) },
+            // semantic ordering, so EOF is below every ordinary symbol
+            () => { rangeEncode(0, eofSymbol) },
+            () => { rangeEncode(5, 3) },
+        ],
+        rangeInvalid: [
+            () => { range('a') },
+            () => { range('abc') },
+        ],
+    },
+    terminal: [
+        () => {
+            // EOF is the singleton `[-1, -1]` and keeps the stored code
+            // `2 ** 24 - 1` at both endpoints.
+            assertEq(eof, 0xFFFFFF_FFFFFF)
+            const [a, b] = rangeDecode(eof)
+            assertEq(a, eofSymbol)
+            assertEq(b, eofSymbol)
+        },
+        () => {
+            // `fullRange` holds the ordinary symbols only.
+            const [a, b] = rangeDecode(fullRange)
+            assertEq(a, 0)
+            assertEq(b, maxSymbol)
+        },
+        () => {
+            // Round trips over the whole semantic domain: EOF, the ordinary
+            // minimum, an ordinary symbol, and the ordinary maximum.
+            for (const v of [eofSymbol, 0, 0x10FFFF, maxSymbol]) {
+                const [a, b] = rangeDecode(oneEncode(v))
+                assertEq(a, v)
+                assertEq(b, v)
+            }
+        },
+        () => {
+            // Ordinary symbols keep their own value as their stored code, so a
+            // packed literal still reads as its endpoints.
+            assertEq(oneEncode(0x20), 0x000020_000020)
+            const [a, b] = rangeDecode(0x000030_000039)
+            assertEq(a, 0x30)
+            assertEq(b, 0x39)
+        },
+        () => {
+            // A range may span EOF and ordinary symbols; the endpoints are
+            // ordered semantically, not by stored code.
+            const [a, b] = rangeDecode(rangeEncode(eofSymbol, maxSymbol))
+            assertEq(a, eofSymbol)
+            assertEq(b, maxSymbol)
+        },
+    ],
+    complement: [
+        () => {
+            // The complement of nothing over `fullRange` is `fullRange`: EOF is
+            // not an ordinary symbol, so no complement can produce it.
+            const r = definedValues(not({}))
+            assertEq(r.length, 1)
+            assertEq(r[0], fullRange)
+        },
+        () => {
+            const r = definedValues(notSet('a'))
+            const decoded = r.map(rangeDecode)
+            assertStructurallySame(decoded, [[0, 0x60], [0x62, maxSymbol]])
+        },
+    ],
+    str: [
+        () => {
+            const result = str('a')
+            assertEq(typeof result, 'number', result)
+            assertEq(result, oneEncode(0x61))
+        },
+        () => {
+            const result = str('ab')
+            assert(Array.isArray(result), result)
+            assertEq(result.length, 2, result)
+        },
+    ],
+    set: () => {
+        const result = set('abc')
+        assert(!(typeof result !== 'object' || result === null), result)
+        assert('a' in result, result)
+        assert('b' in result, result)
+        assert('c' in result, result)
+    },
+    commaJoin0Plus: () => {
+        /** @type {Rule} */
+        const ws = ''
+        /** @type {Rule} */
+        const item = 'x'
+        const result = commaJoin0Plus(ws)('[]', item)
+        assert(Array.isArray(result), result)
+        assert(result[0] === '[', result)
+    },
+    isEmpty: [
+        () => { assert(isEmpty(''), 'empty string should be empty') },
+        () => { assert(isEmpty([]), 'empty array should be empty') },
+        () => { assert(!(isEmpty('a')), 'non-empty string should not be empty') },
+        () => {
+            /** @type {Rule} */
+            const f = () => ''
+            assert(isEmpty(f), 'function returning empty string should be empty')
+        },
+    ],
+    repeat1Plus: [
+        () => {
+            const result = repeat1Plus('x')
+            assert(result[0] === 'x')
+            assertEq(typeof result[1], 'function', 'expected repeat0Plus function')
+        },
+        () => {
+            /** @type {Rule} */
+            const rule = 'ab'
+            const result = repeat1Plus(rule)
+            assertEq(result[0], rule)
+            const inner = result[1]()
+            assert(!(!('some' in inner) || !('none' in inner)), 'expected Option shape')
+        },
+    ],
+}

package/fjs/bnf/testlib.f.d.mts

@@ -0,0 +1,10 @@
+/**
+ * @module
+ *
+ * @import { Rule } from './types.ts'
+ */
+import type { Rule } from './types.ts';
+/** @type {() => Rule} */
+export declare const classic: () => Rule;
+/** @type {() => Rule} */
+export declare const deterministic: () => Rule;