functionalscript 0.43.1 → 0.44.0

package/fjs/bnf/module.f.mjs

@@ -0,0 +1,298 @@
+/**
+ * Core BNF grammar primitives and helpers for describing parser rules.
+ *
+ * The module provides terminal-range encoding utilities, rule composition
+ * types, and set/range helpers used by FunctionalScript grammar definitions.
+ * See `./types.ts` for the type-level API.
+ *
+ * @module
+ *
+ * @import { Tuple } from '../types/array/types.ts'
+ * @import { TerminalRange, Sequence, Rule, RangeVariant, None, Option, Repeat0Plus, Repeat1Plus, Join1Plus, Repeat } from './types.ts'
+ */
+
+import { codePointListToString, stringToCodePointList } from '../text/utf16/module.f.mjs'
+import { definedValues } from '../types/object/module.f.mjs'
+import { isTuple } from '../types/array/module.f.mjs'
+import { map, toArray, repeat as listRepeat } from '../types/list/module.f.mjs'
+import { contains } from '../types/range/module.f.mjs'
+import { assert } from '../asserts/module.f.mjs'
+
+// Internals:
+
+const { fromEntries } = Object
+
+const { fromCodePoint } = String
+
+/**
+ * Two 24 bit numbers can be fit into one JS number (53 bit).
+ */
+const offset = 24
+
+const mask = (1 << offset) - 1
+
+const isValid = contains(0, mask)
+
+/** @type {(a: number, b: number) => TerminalRange} */
+export const rangeEncode = (a, b) => {
+    assert(isValid(a) && isValid(b) && a <= b)
+    return Number((BigInt(a) << BigInt(offset)) | BigInt(b))
+}
+
+/**
+ * Encodes a single symbol as a {@link TerminalRange}.
+ *
+ * @type {(a: number) => TerminalRange}
+ */
+export const oneEncode = a => rangeEncode(a, a)
+
+/**
+ * End-of-file marker: `mask`, the single largest value the 24-bit symbol
+ * space can hold (the top of `fullRange`). Deliberately outside Unicode
+ * (`unicodeRange` tops out at `0x10FFFF`) so it can never collide with a
+ * real code point.
+ *
+ * @type {TerminalRange}
+ */
+export const eof = oneEncode(mask)
+
+/**
+ * Full 24-bit symbol range packed into a single {@link TerminalRange}.
+ * @type {TerminalRange}
+ */
+export const fullRange = 0x000000_FFFFFF
+
+/**
+ * Unicode scalar value range packed into a single {@link TerminalRange}.
+ * @type {TerminalRange}
+ */
+export const unicodeRange = 0x000000_10FFFF
+
+/**
+ * Maximal Unicode code point encoded as a string value.
+ */
+export const unicodeMax = codePointListToString([0x10FFFF])
+
+/**
+ * Decodes a packed range into `[start, end]` symbols.
+ *
+ * @type {(r: number) => Tuple<2, number>}
+ */
+export const rangeDecode = r =>
+    [Number(BigInt(r) >> BigInt(offset)), r & mask]
+
+const mapOneEncode = map(oneEncode)
+
+/** @type {(s: string) => readonly TerminalRange[]} */
+export const toSequence = s =>
+    toArray(mapOneEncode(stringToCodePointList(s)))
+
+/**
+ * Converts the whole string into one rule:
+ * - a single {@link TerminalRange} when the string has one symbol,
+ * - a sequence of {@link TerminalRange} values when the string has multiple symbols.
+ *
+ * @type {(s: string) => readonly TerminalRange[] | TerminalRange}
+ */
+export const str = s => {
+    const x = toSequence(s)
+    return x.length === 1 ? x[0] : x
+}
+
+const mapEntry = map((/** @type {number} */ v) => [fromCodePoint(v), oneEncode(v)])
+
+/**
+ * Converts a string into a variant that maps each character to its symbol range.
+ *
+ * @type {(s: string) => RangeVariant}
+ */
+export const set = s =>
+    fromEntries(toArray(mapEntry(stringToCodePointList(s))))
+
+const isTuple2 = isTuple(2)
+
+/**
+ * Encodes a two-symbol string into a terminal range.
+ *
+ * @throws If `ab` does not contain exactly two unicode code points.
+ *
+ * @type {(ab: string) => TerminalRange}
+ */
+export const range = ab => {
+    const a = toArray(stringToCodePointList(ab))
+    if (!isTuple2(a)) {
+        throw `Invalid range ${ab}.`
+    }
+    return rangeEncode(...a)
+}
+
+/** @typedef {readonly TerminalRange[]} _RangeList */
+
+/** @type {(r: TerminalRange) => readonly [string, TerminalRange]} */
+const rangeToEntry = r =>
+    ['0x' + r.toString(16), r]
+
+/** @type {(r: _RangeList) => RangeVariant} */
+const toVariantRangeSet = r =>
+    fromEntries(r.map(rangeToEntry))
+
+/** @type {(list: _RangeList, ab: number) => _RangeList} */
+const removeOne = (list, ab) => {
+    const [a, b] = rangeDecode(ab)
+    /** @type {_RangeList} */
+    let result = []
+    for (const ab0 of list) {
+        const [a0, b0] = rangeDecode(ab0)
+        if (a0 < a) {
+            // [a0
+            //     ]a
+            result = [...result, rangeEncode(a0, Math.min(b0, a - 1))]
+        }
+        if (b < b0) {
+            //    b0]
+            // b[
+            result = [...result, rangeEncode(Math.max(b + 1, a0), b0)]
+        }
+    }
+    return result
+}
+
+/** @type {(range: TerminalRange, v: RangeVariant) => RangeVariant} */
+export const remove = (range, v) => {
+    /** @type {_RangeList} */
+    let result = [range]
+    for (const r of definedValues(v)) {
+        result = removeOne(result, r)
+    }
+    return toVariantRangeSet(result)
+}
+
+/**
+ * Returns the complement set of the provided ranges over {@link fullRange}.
+ *
+ * @type {(v: RangeVariant) => RangeVariant}
+ */
+export const not = v =>
+    remove(fullRange, v)
+
+/**
+ * Returns the complement set of a character set over {@link fullRange}.
+ *
+ * @type {(s: string) => RangeVariant}
+ */
+export const notSet = s =>
+    not(set(s))
+
+/**
+ * Shared empty sequence literal.
+ *
+ * @type {None}
+ */
+export const none = []
+
+/**
+ * Creates an option value from a required branch.
+ */
+export const option =
+    /**
+     * @template {Rule} S
+     * @param {S} some
+     * @returns {Option<S>}
+     */
+    some => ({
+        some,
+        none,
+    })
+
+/**
+ * Repeat zero or more times.
+ *
+ * https://english.stackexchange.com/questions/506480/single-word-quantifiers-for-zero-or-more-like-cardinalities
+ * - zero or more - any, 0Plus
+ * - one or more - several, 1Plus
+ *
+ * Also see: https://arbs.nzcer.org.nz/types-numbers
+ */
+export const repeat0Plus =
+    /**
+     * @template {Rule} T
+     * @param {T} some
+     * @returns {Repeat0Plus<T>}
+     */
+    some => {
+        const r = () => option(/** @type {const} */ ([some, r]))
+        return r
+    }
+
+/**
+ * Repeat one or more times.
+ */
+export const repeat1Plus =
+    /**
+     * @template {Rule} T
+     * @param {T} some
+     * @returns {Repeat1Plus<T>}
+     */
+    some =>
+        [some, repeat0Plus(some)]
+
+/**
+ * Repeats `some` one or more times separated by `separator`.
+ */
+export const join1Plus =
+    /**
+     * @template {Rule} T
+     * @template {Rule} S
+     * @param {T} some
+     * @param {S} separator
+     * @returns {Join1Plus<T, S>}
+     */
+    (some, separator) =>
+        [some, repeat0Plus([separator, some])]
+
+/**
+ * Repeats `some` zero or more times separated by `separator`.
+ */
+export const join0Plus =
+    /**
+     * @template {Rule} T
+     * @template {Rule} S
+     * @param {T} some
+     * @param {S} separator
+     * @returns {Rule}
+     */
+    (some, separator) =>
+        option(join1Plus(some, separator))
+
+/**
+ * A delimited, comma-separated list with whitespace between tokens:
+ * `open ws (item ws (',' ws item ws)*)? close`.
+ *
+ * `ws` is the per-grammar whitespace rule and is curried so callers can
+ * partially apply it once (`const cj = commaJoin0Plus(ws)`) before reusing
+ * the same combinator at multiple bracket pairs. The two-character string
+ * supplies the bracket pair via destructuring (e.g. `'[]'`, `'{}'`).
+ *
+ * @type {(ws: Rule) => (bracketPair: string, item: Rule) => Sequence}
+ */
+export const commaJoin0Plus = ws =>
+    ([open, close], item) =>
+        [open, ws, join0Plus([item, ws], [',', ws]), close]
+
+/**
+ * Repeats a rule a fixed number of times.
+ *
+ * @type {(n: number) => <T extends Rule>(some: T) => Repeat<T>}
+ */
+export const repeat = n => some =>
+    toArray(listRepeat(some)(n))
+
+/**
+ * Determines whether the rule is an empty rule.
+ *
+ * @type {(rule: Rule) => boolean}
+ */
+export const isEmpty = rule => {
+    const d = typeof rule === 'function' ? rule() : rule
+    return d === '' || (d instanceof Array && d.length === 0)
+}

package/fjs/bnf/proof.f.d.mts

@@ -0,0 +1,17 @@
+/**
+ * @module
+ *
+ * @import { Rule } from './types.ts'
+ */
+export declare const proof: {
+    test: () => void;
+    throw: {
+        rangeEncodeInvalid: (() => void)[];
+        rangeInvalid: (() => void)[];
+    };
+    str: (() => void)[];
+    set: () => void;
+    commaJoin0Plus: () => void;
+    isEmpty: (() => void)[];
+    repeat1Plus: (() => void)[];
+};

package/fjs/bnf/proof.f.mjs

@@ -0,0 +1,89 @@
+/**
+ * @module
+ *
+ * @import { Rule } from './types.ts'
+ */
+
+import { assert, assertEq } from '../asserts/module.f.mjs'
+import {
+    rangeEncode,
+    str,
+    set,
+    range,
+    commaJoin0Plus,
+    isEmpty,
+    oneEncode,
+    repeat1Plus,
+} from './module.f.mjs'
+import { classic, deterministic } from './testlib.f.mjs'
+
+export const proof = {
+    test: () => {
+        classic()
+        deterministic()
+    },
+    throw: {
+        rangeEncodeInvalid: [
+            () => { rangeEncode(-1, 0) },
+            () => { rangeEncode(0, -1) },
+            () => { rangeEncode(5, 3) },
+        ],
+        rangeInvalid: [
+            () => { range('a') },
+            () => { range('abc') },
+        ],
+    },
+    str: [
+        () => {
+            const result = str('a')
+            assertEq(typeof result, 'number', result)
+            assertEq(result, oneEncode(0x61))
+        },
+        () => {
+            const result = str('ab')
+            assert(Array.isArray(result), result)
+            assertEq(result.length, 2, result)
+        },
+    ],
+    set: () => {
+        const result = set('abc')
+        assert(!(typeof result !== 'object' || result === null), result)
+        assert('a' in result, result)
+        assert('b' in result, result)
+        assert('c' in result, result)
+    },
+    commaJoin0Plus: () => {
+        /** @type {Rule} */
+        const ws = ''
+        /** @type {Rule} */
+        const item = 'x'
+        const result = commaJoin0Plus(ws)('[]', item)
+        assert(Array.isArray(result), result)
+        assert(result[0] === '[', result)
+    },
+    isEmpty: [
+        () => { assert(isEmpty(''), 'empty string should be empty') },
+        () => { assert(isEmpty([]), 'empty array should be empty') },
+        () => { assert(!(isEmpty('a')), 'non-empty string should not be empty') },
+        () => {
+            /** @type {Rule} */
+            const f = () => ''
+            assert(isEmpty(f), 'function returning empty string should be empty')
+        },
+    ],
+    repeat1Plus: [
+        () => {
+            const result = repeat1Plus('x')
+            assert(result[0] === 'x')
+            assertEq(typeof result[1], 'function', 'expected repeat0Plus function')
+        },
+        () => {
+            /** @type {Rule} */
+            const rule = 'ab'
+            const result = repeat1Plus(rule)
+            assertEq(result[0], rule)
+            const inner = result[1]()
+            assert(!(!('some' in inner) || !('none' in inner)), 'expected Option shape')
+        },
+    ],
+}

package/fjs/bnf/testlib.f.d.mts

@@ -0,0 +1,10 @@
+/**
+ * @module
+ *
+ * @import { Rule } from './types.ts'
+ */
+import type { Rule } from './types.ts';
+/** @type {() => Rule} */
+export declare const classic: () => Rule;
+/** @type {() => Rule} */
+export declare const deterministic: () => Rule;

package/fjs/bnf/testlib.f.mjs

@@ -0,0 +1,196 @@
+/**
+ * @module
+ *
+ * @import { Rule } from './types.ts'
+ */
+
+import {
+    commaJoin0Plus,
+    none,
+    option,
+    range,
+    remove,
+    repeat,
+    repeat0Plus,
+    set,
+    unicodeMax,
+} from './module.f.mjs'
+
+/** @type {() => Rule} */
+export const classic = () => {
+
+    // https://www.json.org/json-en.html
+
+    const json = () => [element]
+
+    const value = () => ({
+        object,
+        array,
+        string,
+        number,
+        true: 'true',
+        false: 'false',
+        null: 'null'
+    })
+
+    const object = () => ({
+        ws: ['{', ws, '}'],
+        members: ['{', members, '}'],
+    })
+
+    const members = () => ({
+        member,
+        members: [member, ',', members],
+    })
+
+    const member = () => [ws, string, ws, ':', element]
+
+    const array = () => ({
+        ws: ['[', ws, ']'],
+        elements: ['[', elements, ']'],
+    })
+
+    const elements = () => ({
+        element,
+        elements: [element, ',', elements],
+    })
+
+    const element = () => [ws, value, ws]
+
+    const string = () => ['"', characters, '"']
+
+    const characters = () => ({
+        none,
+        characters: [character, characters],
+    })
+
+    /** @type {Rule} */
+    const character = () => ({
+        0: 0x20_000021,
+        1: 0x23_00005B,
+        2: 0x5D_10FFFF,
+        escape: ['\\', escape],
+    })
+
+    const escape = () => ({
+        ...set('"\\/bfnrt'),
+        u: ['u', hex, hex, hex, hex],
+    })
+
+    const hex = () => ({
+        digit,
+        AF: range('AF'),
+        af: range('af'),
+    })
+
+    const number = () => [integer, fraction, exponent]
+
+    const integer = () => ({
+        digit,
+        onenine: [onenine, digits],
+        negDigit: ['-', digit],
+        negOnenine: ['-', onenine, digits],
+    })
+
+    const digits = () => ({
+        digit,
+        digits: [digit, digits],
+    })
+
+    /** @type {Rule} */
+    const digit = () => ({
+        '0': '0',
+        onenine,
+    })
+
+    const onenine = range('19')
+
+    const fraction = () => ({
+        none,
+        digits: ['.', digits],
+    })
+
+    const exponent = () => ({
+        none,
+        E: ['E', sign, digits],
+        e: ['e', sign, digits],
+    })
+
+    const sign = {
+        none,
+        ...set('+-'),
+    }
+
+    const ws = () => ({
+        none,
+        ' ': [' ', ws],
+        '\n': ['\n', ws],
+        '\r': ['\r', ws],
+        '\t': ['\t', ws],
+    })
+
+    return json
+}
+
+/** @type {() => Rule} */
+export const deterministic = () => {
+
+    const onenine = range('19')
+
+    /** @type {Rule} */
+    const digit = range('09')
+
+    const string = [
+        '"',
+        repeat0Plus({
+            ...remove(range(` ${unicodeMax}`), set('"\\')),
+            escape: [
+                '\\',
+                {
+                    ...set('"\\/bfnrt'),
+                    u: [
+                        'u',
+                        ...repeat(4)({
+                            digit,
+                            AF: range('AF'),
+                            af: range('af'),
+                        })
+                    ],
+                }
+            ],
+        }),
+        '"'
+    ]
+
+    const digits0 = repeat0Plus(digit)
+
+    const digits = [digit, digits0]
+
+    const number = [
+        option('-'),
+        {
+            0: '0',
+            onenine: [onenine, digits0],
+        },
+        option(['.', digits]),
+        option([set('Ee'), option(set('+-')), digits])
+    ]
+
+    const ws = repeat0Plus(set(' \n\r\t'))
+
+    const cj = commaJoin0Plus(ws)
+
+    const value = () => ({
+        array: cj('[]', value),
+        object: cj('{}', [string, ws, ':', ws, value]),
+        string,
+        number,
+        true: 'true',
+        false: 'false',
+        null: 'null'
+    })
+
+    const json = [ws, value, ws]
+
+    return json
+}

package/fjs/bnf/token_symbol/module.f.d.mts

@@ -0,0 +1,35 @@
+/**
+ * Encoding of multi-character token names as single BNF input symbols.
+ *
+ * A parser built on top of a tokenizer consumes one input symbol per token, so
+ * multi-character operators (`>>>=`) and keywords (`instanceof`) each need a
+ * symbol of their own. Names are registered as one fixed alphabet and get a
+ * symbol from their position in it, above the Unicode range and below `eof`.
+ *
+ * See `./types.ts` for the `Encoding<T>` type-level API.
+ *
+ * @module
+ *
+ * @import { Encoding } from './types.ts'
+ */
+import type { Encoding } from './types.ts';
+/**
+ * How many names one encoding holds: every symbol from {@link start} up to but
+ * not including `eof` (`0xFFFFFF`, the top of the 24-bit symbol space).
+ *
+ * @type {number}
+ */
+export declare const capacity: number;
+/**
+ * Builds an encoding over the complete list of token names.
+ *
+ * A name takes the symbol at its index in `names`, so the list is append-only:
+ * inserting or reordering names changes the symbols of everything after the
+ * edit.
+ *
+ * @throws When `names` holds more than {@link capacity} entries, or when a name
+ * repeats — a repeated name has no single symbol to decode back to.
+ *
+ * @type {<T extends string>(names: readonly T[]) => Encoding<T>}
+ */
+export declare const encoding: <T extends string>(names: readonly T[]) => Encoding<T>;

package/fjs/bnf/token_symbol/module.f.mjs

@@ -0,0 +1,62 @@
+/**
+ * Encoding of multi-character token names as single BNF input symbols.
+ *
+ * A parser built on top of a tokenizer consumes one input symbol per token, so
+ * multi-character operators (`>>>=`) and keywords (`instanceof`) each need a
+ * symbol of their own. Names are registered as one fixed alphabet and get a
+ * symbol from their position in it, above the Unicode range and below `eof`.
+ *
+ * See `./types.ts` for the `Encoding<T>` type-level API.
+ *
+ * @module
+ *
+ * @import { Encoding } from './types.ts'
+ */
+
+import { assert } from '../../asserts/module.f.mjs'
+import { fromUndefined } from '../../types/nullable/module.f.mjs'
+import { eof, rangeDecode, unicodeRange } from '../module.f.mjs'
+
+const [, unicodeLast] = rangeDecode(unicodeRange)
+
+const [eofSymbol] = rangeDecode(eof)
+
+/**
+ * The symbol of the first registered name: `0x110000`, one past the last
+ * Unicode scalar value, so a token symbol can never be mistaken for a code
+ * point of the layer below.
+ */
+const start = unicodeLast + 1
+
+/**
+ * How many names one encoding holds: every symbol from {@link start} up to but
+ * not including `eof` (`0xFFFFFF`, the top of the 24-bit symbol space).
+ *
+ * @type {number}
+ */
+export const capacity = eofSymbol - start
+
+/**
+ * Builds an encoding over the complete list of token names.
+ *
+ * A name takes the symbol at its index in `names`, so the list is append-only:
+ * inserting or reordering names changes the symbols of everything after the
+ * edit.
+ *
+ * @throws When `names` holds more than {@link capacity} entries, or when a name
+ * repeats — a repeated name has no single symbol to decode back to.
+ *
+ * @type {<T extends string>(names: readonly T[]) => Encoding<T>}
+ */
+export const encoding = names => {
+    assert(names.length <= capacity, ['too many token names', names.length])
+    assert(new Set(names).size === names.length, ['duplicate token name', names])
+    return {
+        encode: name => {
+            const index = names.indexOf(name)
+            assert(index !== -1, ['unregistered token name', name])
+            return start + index
+        },
+        decode: symbol => fromUndefined(names[symbol - start]),
+    }
+}