functionalscript 0.43.1 → 0.45.0

package/fjs/text/utf16/proof.f.mjs

@@ -0,0 +1,175 @@
+/**
+ * @import { Unknown } from '../../media/json/types.ts'
+ */
+
+import {
+    toCodePointList,
+    fromCodePointList,
+    stringToList,
+    listToString,
+    stringToCodePointList,
+    codePointListToString,
+    codePointToString
+} from './module.f.mjs'
+import { stringify as jsonStringify } from '../../media/json/module.f.mjs'
+import { sort } from '../../types/object/module.f.mjs'
+import { toArray } from '../../types/list/module.f.mjs'
+import { assertEq } from '../../asserts/module.f.mjs'
+
+/** @type {(a: readonly Unknown[]) => string} */
+const stringify = a =>
+    jsonStringify(sort)(a)
+
+export const proof = {
+    toCodePointList: [
+        () => {
+            const result = stringify(toArray(toCodePointList([-1, 65536])))
+            assertEq(result, '[4294967295,4294967295]')
+        },
+        () => {
+            const result = stringify(toArray(toCodePointList([0, 36, 8364, 55295, 57344, 65535])))
+            assertEq(result, '[0,36,8364,55295,57344,65535]')
+        },
+        () => {
+            const result = stringify(toArray(toCodePointList([56320, 57343])))
+            assertEq(result, '[-2147427328,-2147426305]')
+        },
+        () => {
+            const result = stringify(toArray(toCodePointList([55296, 56320, 55297, 56375, 55378, 57186, 56319, 57343])))
+            assertEq(result, '[65536,66615,150370,1114111]')
+        },
+        () => {
+            const result = stringify(toArray(toCodePointList([55296, 55296])))
+            assertEq(result, '[-2147428352,-2147428352]')
+        },
+        () => {
+            const result = stringify(toArray(toCodePointList([55296, 0])))
+            assertEq(result, '[-2147428352,0]')
+        },
+        () => {
+            const result = stringify(toArray(toCodePointList([56320])))
+            assertEq(result, '[-2147427328]')
+        },
+        () => {
+            const result = stringify(toArray(toCodePointList([56320, 0])))
+            assertEq(result, '[-2147427328,0]')
+        },
+        // `U16` is just `number`, so a non-integer in [0x0000, 0xFFFF] is a
+        // possible (if malformed) input. It must be rejected as invalid, not
+        // misclassified by the surrogate/BMP range checks, which only
+        // partition the integers in that range.
+        () => {
+            const result = stringify(toArray(toCodePointList([56319.5])))
+            assertEq(result, '[4294967295]')
+        },
+        // A non-integer word doesn't disturb a pending high surrogate: it is
+        // reported invalid on its own, and the surrogate is still flagged
+        // unpaired at EOF.
+        () => {
+            const result = stringify(toArray(toCodePointList([55296, 56319.5])))
+            assertEq(result, '[4294967295,-2147428352]')
+        }
+    ],
+    fromCodePointList: [
+        () => {
+            const result = stringify(toArray(fromCodePointList([0])))
+            assertEq(result, '[0]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0x24])))
+            assertEq(result, '[36]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0x20AC])))
+            assertEq(result, '[8364]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0xd7ff])))
+            assertEq(result, '[55295]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0xe000])))
+            assertEq(result, '[57344]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0xffff])))
+            assertEq(result, '[65535]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0x10000])))
+            assertEq(result, '[55296,56320]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0x10437])))
+            assertEq(result, '[55297,56375]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0x24B62])))
+            assertEq(result, '[55378,57186]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([0x10ffff])))
+            assertEq(result, '[56319,57343]')
+        },
+        () => {
+            const result = stringify(toArray(fromCodePointList([-1, 0xd800, 0xdfff, 0x110000])))
+            assertEq(result, '[65535,55296,57343,0]')
+        }
+    ],
+    string: [
+        () => {
+            const utf16List = stringToList("Hello world!😂🚜🚲")
+            const result = listToString(utf16List)
+            assertEq(result, "Hello world!😂🚜🚲")
+        },
+        () => {
+            const cpList = stringToCodePointList("Hello world!😂🚜🚲")
+            const result = codePointListToString(cpList)
+            assertEq(result, "Hello world!😂🚜🚲")
+        },
+        () => {
+            const a = stringToList("Hello world!😂🚜🚲")
+            const b = toCodePointList(a)
+            const c = fromCodePointList(b)
+            const result = listToString(c)
+            assertEq(result, "Hello world!😂🚜🚲")
+        }
+    ],
+    stringToList: [
+        () => {
+            const inputString = "Hello, i like js"
+            const utf16List = stringToList(inputString)
+        },
+        () => {
+            const inputString = "😇🤬🫥😑🫠"
+            const utf16List = stringToList(inputString)
+        }
+    ],
+    listToString: [
+        () => {
+            const utf16List = [0x0041, 0x0042, 0x0043]
+            const outputString = listToString(utf16List)
+        }
+    ],
+    stringToCodePointList: [
+        () => {
+            const inputString = "Hello, 😀"
+            const codePoints = stringToCodePointList(inputString)
+        }
+    ],
+    codePointListToString: [
+        () => {
+            const codePoints = [0x48, 0x65, 0x6C, 0x6C, 0x6F]
+            const outputString = codePointListToString(codePoints)
+        }
+    ],
+    codePointToString: [
+        () => { assertEq(codePointToString(0x48), 'H') },
+        // supplementary plane: one code point, two code units
+        () => { assertEq(codePointToString(0x1f600), '😀') },
+        // a surrogate is not a code point; it round-trips as its own code unit
+        () => { assertEq(codePointToString(0xd800), '\ud800') },
+        // above the maximum code point, the low 16 bits are kept
+        () => { assertEq(codePointToString(0x110000), '\u0000') },
+    ]
+}

package/fjs/text/utf16/types.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Types for UTF-16 code units and Unicode code points.
+ *
+ * @module
+ */
+/**
+ * Represent an unsigned UTF16, used to store one word UTF-16 (code unit).
+ */
+export type U16 = number;
+/**
+ * [0, 0x10_FFFF]: 16+5 = 21 bits
+ *
+ * 121_0000_0000: 16+16+9 = 41 bits
+ */
+/**
+ * Represent an Unicode code point.
+ * Has range: from 0x0000 to 0x10_FFFF (21 bits).
+ */
+export type CodePoint = number;

package/fjs/text/utf8/module.f.d.mts

@@ -0,0 +1,73 @@
+/**
+ * UTF-8 byte-level encoding and decoding utilities for FunctionalScript streams.
+ *
+ * @module
+ *
+ * @import { List, Thunk } from '../../types/list/types.ts'
+ * @import { StateScan } from '../../types/function/operator/types.ts'
+ * @import { Vec } from '../../types/bit_vec/types.ts'
+ * @import { ByteOrEof, I32, U8, Utf8NonEmptyState, Utf8State, } from './types.ts'
+ */
+import type { List, Thunk } from '../../types/list/types.ts';
+import type { StateScan } from '../../types/function/operator/types.ts';
+import type { Vec } from '../../types/bit_vec/types.ts';
+import type { I32, U8, Utf8NonEmptyState, Utf8State } from './types.ts';
+/**
+ * Maps a list of Unicode code points to a stream of UTF-8 bytes.
+ *
+ * @param input - A list of Unicode code points to be converted.
+ * @returns A thunk that lazily produces a sequence of UTF-8 bytes.
+ *
+ * @type {(input: List<number>) => Thunk<U8>}
+ */
+export declare const fromCodePointList: (input: List<number>) => Thunk<U8>;
+/**
+ * Converts a non-empty UTF-8 decoding state to an error code.
+ *
+ * @param {Utf8NonEmptyState} state A non-empty UTF-8 decoding state.
+ * @returns {I32} An I32 error code derived from the invalid UTF-8 state.
+ */
+export declare const utf8StateToError: (state: Utf8NonEmptyState) => I32;
+/**
+ * Decodes a byte into a Unicode code point, using a given UTF-8 state.
+ *
+ * Rejects overlong 3-/4-byte encodings (Unicode Table 3-7): a lead `E0` must
+ * be followed by a continuation `>= 0xA0`, and a lead `F0` by a continuation
+ * `>= 0x90`. It does not itself reject surrogates (`ED A0..BF`) or code
+ * points above `U+10FFFF` (`F4 90..BF`); {@link fromVec}'s
+ * `isValidCodePoint` pass filters those out of the raw code-point stream.
+ *
+ * @param state - The current UTF-8 decoding state.
+ * @param byte - A single byte to decode.
+ * @returns A tuple containing:
+ *   - A list of decoded Unicode code points or error codes.
+ *   - The updated UTF-8 state.
+ *
+ * @type {StateScan<number, Utf8State, readonly I32[]>}
+ */
+export declare const utf8ByteToCodePointOp: StateScan<number, Utf8State, readonly I32[]>;
+/**
+ * Handles the end-of-file (EOF) case for UTF-8 decoding: a leftover incomplete
+ * sequence is flushed as a single error code and the state resets to `null`.
+ * The flush itself is `eofFlush` from `code_point`, shared with UTF-16.
+ *
+ * @type {(state: Utf8State) => readonly [List<I32>, Utf8State]}
+ */
+export declare const utf8EofToCodePointOp: (state: Utf8State) => readonly [List<I32>, Utf8State];
+/**
+ * Converts a list of UTF-8 bytes into a list of Unicode code points.
+ *
+ * @param input - A list of UTF-8 bytes.
+ * @returns A list of Unicode code points or error codes.
+ *
+ * @type {(input: List<U8>) => List<I32>}
+ */
+export declare const toCodePointList: (input: List<U8>) => List<I32>;
+/**
+ * Returns the decoded string if `v` is valid UTF-8, or `null` otherwise.
+ * Rejects non-octet Vecs, invalid byte sequences, surrogates, and out-of-range
+ * code points.
+ *
+ * @type {(v: Vec) => string | null}
+ */
+export declare const fromVec: (v: Vec) => string | null;

package/fjs/text/utf8/module.f.mjs

@@ -0,0 +1,278 @@
+/**
+ * UTF-8 byte-level encoding and decoding utilities for FunctionalScript streams.
+ *
+ * @module
+ *
+ * @import { List, Thunk } from '../../types/list/types.ts'
+ * @import { StateScan } from '../../types/function/operator/types.ts'
+ * @import { Vec } from '../../types/bit_vec/types.ts'
+ * @import { ByteOrEof, I32, U8, Utf8NonEmptyState, Utf8State, } from './types.ts'
+ */
+
+import { flatMap, toArray } from '../../types/list/module.f.mjs'
+import {
+    decoder,
+    eofFlush,
+    errorMask,
+    isValidCodePoint,
+} from '../code_point/module.f.mjs'
+import { msb, u8List, length } from '../../types/bit_vec/module.f.mjs'
+import { codePointListToString } from '../utf16/module.f.mjs'
+
+/**
+ * UTF-8 byte-format constants. Each byte kind is defined by a tag — the fixed
+ * high bits identifying the kind — and a payload mask selecting the code-point
+ * bits the byte carries. The encoder and the decoder are exact inverses, so
+ * both read their bit patterns from this single set of definitions:
+ *
+ * | byte kind   | pattern      | tag       | payload mask |
+ * |-------------|--------------|-----------|--------------|
+ * | continuation| `10xx_xxxx`  | `contTag` | `contMask`   |
+ * | 2-byte lead | `110x_xxxx`  | `lead2Tag`| `lead2Mask`  |
+ * | 3-byte lead | `1110_xxxx`  | `lead3Tag`| `lead3Mask`  |
+ * | 4-byte lead | `1111_0xxx`  | `lead4Tag`| `lead4Mask`  |
+ */
+const contTag = 0b1000_0000
+const contMask = 0b0011_1111
+const lead2Tag = 0b1100_0000
+const lead2Mask = 0b0001_1111
+const lead3Tag = 0b1110_0000
+const lead3Mask = 0b0000_1111
+const lead4Tag = 0b1111_0000
+const lead4Mask = 0b0000_0111
+
+/**
+ * Encodes the low six bits of `x` as a UTF-8 continuation byte.
+ *
+ * @type {(x: number) => number}
+ */
+const contByte = x => x & contMask | contTag
+
+/**
+ * Reads the six payload bits of a continuation byte.
+ *
+ * @type {(b: number) => number}
+ */
+const contPayload = b => b & contMask
+
+/**
+ * The valid lead-byte range for 2-, 3-, and 4-byte sequences (RFC 3629);
+ * excludes overlong 2-byte leads (`C0`, `C1`) and leads above `U+10FFFF` (`F5`-`F7`).
+ */
+const leadMin = 0b1100_0010
+const leadMax = 0b1111_0100
+/** @type {(b: number) => boolean} */
+const isLeadByte = b => b >= leadMin && b <= leadMax
+
+/**
+ * Dispatches a fresh-state byte, emitting `prefix` ahead of whatever the byte
+ * itself produces. Shared by the `state === null` arm and by error recovery
+ * after {@link utf8StateToError}, which differ only in `prefix`.
+ *
+ * @type {(prefix: readonly I32[]) => (byte: number) => readonly [readonly I32[], Utf8State]}
+ */
+const restart = prefix =>
+    byte =>
+        byte < contTag ? [[...prefix, byte], null]
+        : isLeadByte(byte) ? [[...prefix], [byte]]
+        : [[...prefix, byte | errorMask], null]
+
+/**
+ * Converts a Unicode code point to a sequence of UTF-8 bytes.
+ * @param {number} input The Unicode code point to be converted. Valid range:
+ *   - 0x0000 to 0x007F for 1-byte sequences.
+ *   - 0x0080 to 0x07FF for 2-byte sequences.
+ *   - 0x0800 to 0xFFFF for 3-byte sequences.
+ *   - 0x10000 to 0x10FFFF for 4-byte sequences.
+ * @returns {readonly U8[]} A readonly array of UTF-8 bytes representing the input code point.
+ *   - Returns `[errorMask]` if the input does not match valid UTF-8 encoding rules.
+ */
+const codePointToUtf8 = input => {
+    if (input >= 0x0000 && input <= 0x007f) {
+        return [input & 0b01111_1111]
+    }
+    if (input >= 0x0080 && input <= 0x07ff) {
+        return [input >> 6 | lead2Tag, contByte(input)]
+    }
+    if (input >= 0x0800 && input <= 0xffff) {
+        return [
+            input >> 12 | lead3Tag,
+            contByte(input >> 6),
+            contByte(input),
+        ]
+    }
+    if (input >= 0x10000 && input <= 0x10ffff) {
+        return [
+            input >> 18 | lead4Tag,
+            contByte(input >> 12),
+            contByte(input >> 6),
+            contByte(input),
+        ]
+    }
+    if ((input & errorMask) !== 0) {
+        if ((input & 0b1000_0000_0000_0000) !== 0) {
+            return [
+                input >> 12 & lead4Mask | lead4Tag,
+                contByte(input >> 6),
+                contByte(input),
+            ]
+        }
+        if ((input & 0b0000_0100_0000_0000) !== 0) {
+            return [
+                input >> 6 & lead3Mask | lead3Tag,
+                contByte(input),
+            ]
+        }
+        if ((input & 0b0000_0010_0000_0000) !== 0) {
+            return [
+                input >> 6 & lead4Mask | lead4Tag,
+                contByte(input),
+            ]
+        }
+        if ((input & 0b0000_0000_1000_0000) !== 0) return [input & 0b1111_1111]
+    }
+    return [errorMask]
+}
+
+/**
+ * Maps a list of Unicode code points to a stream of UTF-8 bytes.
+ *
+ * @param input - A list of Unicode code points to be converted.
+ * @returns A thunk that lazily produces a sequence of UTF-8 bytes.
+ *
+ * @type {(input: List<number>) => Thunk<U8>}
+ */
+export const fromCodePointList = flatMap(
+    codePointToUtf8,
+)
+
+/**
+ * Converts a non-empty UTF-8 decoding state to an error code.
+ *
+ * @param {Utf8NonEmptyState} state A non-empty UTF-8 decoding state.
+ * @returns {I32} An I32 error code derived from the invalid UTF-8 state.
+ */
+export const utf8StateToError = state => {
+    let x
+    switch (state.length) {
+        case 1: {
+            [x] = state
+            break
+        }
+        case 2: {
+            const [s0, s1] = state
+            x = s0 < lead4Tag
+                ? ((s0 & lead3Mask) << 6) + contPayload(s1) + 0b0000_0100_0000_0000
+                : ((s0 & lead4Mask) << 6) + contPayload(s1) +
+                    0b0000_0010_0000_0000
+            break
+        }
+        case 3: {
+            const [s0, s1, s2] = state
+            x = ((s0 & lead4Mask) << 12) + (contPayload(s1) << 6) +
+                contPayload(s2) + 0b1000_0000_0000_0000
+            break
+        }
+        //default:
+        //    throw 'invalid state'
+    }
+    return x | errorMask
+}
+
+/**
+ * Decodes a byte into a Unicode code point, using a given UTF-8 state.
+ *
+ * Rejects overlong 3-/4-byte encodings (Unicode Table 3-7): a lead `E0` must
+ * be followed by a continuation `>= 0xA0`, and a lead `F0` by a continuation
+ * `>= 0x90`. It does not itself reject surrogates (`ED A0..BF`) or code
+ * points above `U+10FFFF` (`F4 90..BF`); {@link fromVec}'s
+ * `isValidCodePoint` pass filters those out of the raw code-point stream.
+ *
+ * @param state - The current UTF-8 decoding state.
+ * @param byte - A single byte to decode.
+ * @returns A tuple containing:
+ *   - A list of decoded Unicode code points or error codes.
+ *   - The updated UTF-8 state.
+ *
+ * @type {StateScan<number, Utf8State, readonly I32[]>}
+ */
+export const utf8ByteToCodePointOp = (byte, state) => {
+    if (byte < 0x00 || byte > 0xff) {
+        return [[errorMask], state]
+    }
+    if (state === null) return restart([])(byte)
+    if (byte >= contTag && byte < lead2Tag) {
+        switch (state.length) {
+            case 1: {
+                const [s0] = state
+                if (s0 < lead3Tag) {
+                    return [[((s0 & lead2Mask) << 6) + contPayload(byte)], null]
+                }
+                if (s0 < 0b1111_1000) {
+                    // Reject overlong 3-/4-byte encodings: after lead `E0` the
+                    // first continuation must be >= 0xA0, after lead `F0` it
+                    // must be >= 0x90 (Unicode Table 3-7).
+                    const overlong = s0 === lead3Tag && byte < 0b1010_0000 ||
+                        s0 === lead4Tag && byte < 0b1001_0000
+                    if (!overlong) return [[], [s0, byte]]
+                }
+                break
+            }
+            case 2: {
+                const [s0, s1] = state
+                if (s0 < lead4Tag) {
+                    return [[
+                        ((s0 & lead3Mask) << 12) + (contPayload(s1) << 6) +
+                        contPayload(byte),
+                    ], null]
+                }
+                if (s0 < 0b1111_1000) return [[], [s0, s1, byte]]
+                break
+            }
+            case 3: {
+                const [s0, s1, s2] = state
+                return [[
+                    ((s0 & lead4Mask) << 18) + (contPayload(s1) << 12) +
+                    (contPayload(s2) << 6) + contPayload(byte),
+                ], null]
+            }
+        }
+    }
+    return restart([utf8StateToError(state)])(byte)
+}
+
+/**
+ * Handles the end-of-file (EOF) case for UTF-8 decoding: a leftover incomplete
+ * sequence is flushed as a single error code and the state resets to `null`.
+ * The flush itself is `eofFlush` from `code_point`, shared with UTF-16.
+ *
+ * @type {(state: Utf8State) => readonly [List<I32>, Utf8State]}
+ */
+export const utf8EofToCodePointOp = eofFlush(utf8StateToError)
+
+/**
+ * Converts a list of UTF-8 bytes into a list of Unicode code points.
+ *
+ * @param input - A list of UTF-8 bytes.
+ * @returns A list of Unicode code points or error codes.
+ *
+ * @type {(input: List<U8>) => List<I32>}
+ */
+export const toCodePointList =
+    decoder(utf8ByteToCodePointOp, utf8EofToCodePointOp)
+
+/**
+ * Returns the decoded string if `v` is valid UTF-8, or `null` otherwise.
+ * Rejects non-octet Vecs, invalid byte sequences, surrogates, and out-of-range
+ * code points.
+ *
+ * @type {(v: Vec) => string | null}
+ */
+export const fromVec = v => {
+    if ((length(v) & 0b111n) !== 0n) { return null }
+    const arr = toArray(toCodePointList(u8List(msb)(v)))
+    for (const cp of arr) {
+        if (!isValidCodePoint(cp)) { return null }
+    }
+    return codePointListToString(arr)
+}