functionalscript 0.43.1 → 0.45.0

package/fjs/text/utf16/module.f.js

@@ -1,311 +0,0 @@
-/**
- * UTF-16 utilities: convert between strings, UTF-16 code units, and Unicode
- * code points, including surrogate-pair encoding/decoding via streaming
- * `stateScan` operations.
- *
- * @module
- */
-import { map, reduce, flatMap, empty, } from '../../types/list/module.f.js';
-import { concat } from '../../types/function/operator/module.f.js';
-import { contains } from '../../types/range/module.f.js';
-import { fn } from '../../types/function/module.f.js';
-import { decoder, errorMask, isBmpCodePoint, isHighSurrogate, isLowSurrogate, isSupplementaryPlane, } from '../code_point/module.f.js';
-/**
- * The BMP / surrogate / supplementary-plane predicates used below live in
- * `code_point`, the shared Unicode contract; see that module for their ranges.
- *
- * Converts a Unicode code point to its corresponding UTF-16 representation.
- *
- * This function handles:
- * 1. Code points in the Basic Multilingual Plane (BMP) [0x0000–0xFFFF],
- *    which map directly to a single 16-bit value (U16).
- * 2. Supplementary Plane code points [0x10000–0x10FFFF],
- *    which are represented as surrogate pairs in UTF-16.
- *
- * @param codePoint - A valid Unicode code point.
- * @returns A list of 16-bit unsigned integers (U16) representing the UTF-16 encoding
- *          of the input code point. If the code point is in the BMP range, a single U16
- *          value is returned. For code points in the supplementary planes, two U16
- *          values (a high and a low surrogate) are returned.
- * @example
- *
- * ```ts
- * const exampleCodePoints: List<CodePoint> = [
- *     0x0041,       // 'A' (BMP, single U16)
- *     0x1F600,      // 😀 (Emoji, supplementary plane, surrogate pair)
- *     0xD7FF,       // Last code point in the low BMP range
- *     0xE000,       // First code point in the high BMP range
- *     0x10FFFF,     // Maximum valid code point in Unicode
- *     0x110000,     // Invalid code point (outside Unicode range)
- * ]
- * exampleCodePoints.forEach((codePoint) => {
- *     const utf16Result = codePointToUtf16(codePoint)
- *     console.log(`Code Point: U+${codePoint.toString(16).toUpperCase()}`)
- *     console.log(`UTF-16: ${utf16Result.map(u16 => u16.toString(16).toUpperCase()).join(' ')}`)
- * })
- * ```
- */
-const codePointToUtf16 = (codePoint) => {
-    if (isBmpCodePoint(codePoint)) {
-        return [codePoint];
-    }
-    if (isSupplementaryPlane(codePoint)) {
-        const n = codePoint - 0x1_0000;
-        const high = (n >> 10) + 0xd800;
-        const low = (n & 0b0011_1111_1111) + 0xdc00;
-        return [high, low];
-    }
-    return [codePoint & 0xffff];
-};
-/**
- * Converts a UTF-16 sequence to its corresponding Unicode code points.
- *
- * This function handles:
- * 1. Single U16 values in the Basic Multilingual Plane (BMP) [0x0000–0xFFFF].
- * 2. Surrogate pairs representing code points in the Supplementary Plane [0x10000–0x10FFFF].
- * 3. Invalid input sequences by applying an error mask to the resulting code point.
- *
- * @param utf16 - A list of UTF-16 code units (U16) to convert.
- * @returns A list of Unicode code points. Each code point corresponds to one or more U16
- *          values in the input. Invalid sequences are marked with the `errorMask`.
- * @example
- *
- * ```ts
- * const exampleUtf16: List<U16> = [
- *     0x0041,       // 'A' (BMP, single U16)
- *     0xD83D, 0xDE00, // 😀 (Emoji, surrogate pair)
- *     0xD800,       // Unpaired high surrogate
- *     0xDC00,       // Unpaired low surrogate
- * ]
- *
- * const codePoints = toCodePointList(exampleUtf16)
- * codePoints.forEach((codePoint) => {
- *     if (codePoint & errorMask) {
- *         console.log(`Invalid sequence detected: ${codePoint.toString(16).toUpperCase()}`)
- *     } else {
- *         console.log(`Code Point: U+${codePoint.toString(16).toUpperCase()}`)
- *     }
- * })
- * ```
- */
-export const fromCodePointList = flatMap(codePointToUtf16);
-/**
- * Validates whether a given 16-bit unsigned integer (U16) falls within the valid range for UTF-16 code units.
- *
- * UTF-16 uses 16-bit code units to encode characters. The valid range for these code units is [0x0000, 0xFFFF].
- * This function is used to verify that a number is within this range.
- *
- * @param i - A 16-bit unsigned integer (U16) to validate.
- * @returns A boolean value indicating whether the input is a valid UTF-16 code unit.
- *
- * @example
- *
- * ```ts
- * const validU16 = u16(0x0041)  // true: U+0041 ('A')
- * const invalidU16 = u16(0x110000)  // false: Value is outside the valid range
- * const edgeCaseLow = u16(0x0000)  // true: Minimum valid value for UTF-16
- * const edgeCaseHigh = u16(0xFFFF)  // true: Maximum valid value for UTF-16
- * ```
- */
-const u16 = contains(0x0000, 0xFFFF);
-/**
- * A stateful operation that converts a UTF-16 word (U16) to a list of Unicode code points (CodePoint),
- * while maintaining the state of surrogate pair decoding.
- *
- * This operation processes UTF-16 code units and decodes them into Unicode code points. It handles:
- * 1. BMP code points (single U16).
- * 2. Surrogate pairs (two U16s representing a single code point in the supplementary planes).
- * 3. Invalid or malformed code units.
- *
- * It also manages the internal state, which is necessary to handle surrogate pairs. If the state is null,
- * it expects a valid BMP code point or a high surrogate. If the state is not null, it processes a low surrogate
- * and combines the pair into a single supplementary code point.
- *
- * @param state - The current state of the UTF-16 decoding operation.
- *               This can be either `null` (no state) or the previous high surrogate value.
- * @param word - The current UTF-16 word (U16) to process.
- * @returns A tuple where the first element is a list of decoded Unicode code points (`CodePoint`), and
- *          the second element is the updated state. If the word is invalid, an error code `0xffffffff` is returned.
- *
- * @example
- *
- * ```ts
- * const state: Utf16State = null;
- * const word: U16 = 0xD83D;  // High surrogate for 😀 emoji
- * const [decodedCodePoints, newState] = utf16ByteToCodePointOp(word, state);
- * ```
- *
- * @example
- *
- * ```ts
- * const state: Utf16State = 0xD83D;  // High surrogate already stored
- * const word: U16 = 0xDC00;  // Low surrogate for 😀 emoji
- * const [decodedCodePoints, newState] = utf16ByteToCodePointOp(word, state);
- * ```
- */
-const utf16ByteToCodePointOp = (word, state) => {
-    if (!u16(word)) {
-        return [[0xffffffff], state];
-    }
-    if (state === null) {
-        if (isBmpCodePoint(word)) {
-            return [[word], null];
-        }
-        if (isHighSurrogate(word)) {
-            return [[], word];
-        }
-        return [[word | errorMask], null];
-    }
-    if (isLowSurrogate(word)) {
-        const high = state - 0xd800;
-        const low = word - 0xdc00;
-        return [[(high << 10) + low + 0x10000], null];
-    }
-    if (isBmpCodePoint(word)) {
-        return [[state | errorMask, word], null];
-    }
-    if (isHighSurrogate(word)) {
-        return [[state | errorMask], word];
-    }
-    return [[state | errorMask, word | errorMask], null];
-};
-/**
- * Handles the EOF (end-of-file) condition during UTF-16 decoding.
- *
- * If there is no pending state (`state === null`), it simply returns an empty list
- * of code points, indicating no further input to process. If there is a pending state,
- * it is treated as an unpaired surrogate, and the `errorMask` is applied to flag
- * the invalid sequence.
- *
- * @param state - The current UTF-16 decoding state. This can be:
- *   - `null`: No pending surrogate to process.
- *   - A high surrogate (0xD800–0xDBFF) left from an earlier input, waiting for a low surrogate.
- * @returns A tuple:
- *   - The first element is a list of code points. If there’s a pending state, it is returned
- *     with the `errorMask` applied.
- *   - The second element is the next state, which will always be `null` because EOF means no
- *     further processing.
- *
- * @example
- *
- * ```js
- * const eofState = utf16EofToCodePointOp(0xD800) // Unpaired high surrogate
- * const validState = utf16EofToCodePointOp(null) // No pending state
- * ```
- */
-const utf16EofToCodePointOp = (state) => [state === null ? empty : [state | errorMask], null];
-/**
- * Converts a list of UTF-16 code units to a list of Unicode code points (CodePoint).
- * This function processes each UTF-16 code unit, decoding them into their corresponding Unicode code points.
- * The input list of `U16` values may represent characters in the Basic Multilingual Plane (BMP) or supplementary planes,
- * with surrogate pairs handled correctly. The function also handles EOF (`null`).
- * @param input - A list of UTF-16 code units (`U16`), possibly containing surrogate pairs.
- * @returns A list of Unicode code points (`CodePoint`), one for each valid code unit or surrogate pair.
- *
- * @example
- *
- * ```ts
- * const utf16List: List<U16> = [0x0041, 0xD83D, 0xDE00] // 'A' and 😀 (surrogate pair)
- * const codePoints = toCodePointList(utf16List)
- * ```
- */
-export const toCodePointList = decoder(utf16ByteToCodePointOp, utf16EofToCodePointOp);
-/**
- * Converts a string to a list of UTF-16 code units (U16).
- *
- * This function processes each character in the input string and converts it to its corresponding UTF-16 code unit(s).
- * Characters in the Basic Multilingual Plane (BMP) will produce a single `U16`, while supplementary plane characters
- * (those requiring surrogate pairs) will produce two `U16` values.
- * @param s - The input string to convert to UTF-16 code units.
- * @returns A list of UTF-16 code units (`U16`) representing the string.
- *
- * @example
- *
- * ```js
- * const inputString = "Hello, 😀"
- * const utf16List = stringToList(inputString)
- * ```
- */
-export const stringToList = (s) => {
-    const at = (i) => {
-        const first = s.charCodeAt(i);
-        return isNaN(first) ? empty : { first, tail: () => at(i + 1) };
-    };
-    return at(0);
-};
-/**
- * Converts a string to a list of Unicode code points (CodePoint).
- * This function first converts the string to a list of UTF-16 code units (U16) using `stringToList`,
- * then it converts the UTF-16 code units to Unicode code points using `toCodePointList`. This is useful for handling
- * Unicode characters, including supplementary characters represented by surrogate pairs in UTF-16.
- *
- * @param input - The input string to convert.
- * @returns A list of Unicode code points (`CodePoint`) corresponding to the characters in the string.
- *
- * @example
- *
- * ```js
- * const inputString = "Hello, 😀"
- * const codePoints = stringToCodePointList(inputString)
- * ```
- */
-export const stringToCodePointList = (input) => toCodePointList(stringToList(input));
-/**
- * Converts a list of UTF-16 code units (U16) to a string.
- * This function takes a list of `U16` values (UTF-16 code units) and reconstructs the original string by mapping
- * each code unit back to its character using `String.fromCharCode`. The resulting characters are concatenated
- * to form the final string.
- *
- * @param input - A list of UTF-16 code units (`U16`).
- * @returns A string representing the characters encoded by the input UTF-16 code units.
- *
- * @example
- *
- * ```ts
- * const utf16List: List<U16> = [0x0041, 0x0042, 0x0043] // 'ABC'
- * const outputString = listToString(utf16List)
- * ```
- */
-export const listToString = fn(map(String.fromCharCode))
-    .map(reduce(concat)(''))
-    .result;
-/**
- * Converts a list of Unicode code points (CodePoint) to a string.
- * This function first converts the list of Unicode code points to a list of UTF-16 code units using `fromCodePointList`,
- * then it uses `listToString` to reconstruct the string from the UTF-16 code units.
- *
- * @param input - A list of Unicode code points (`CodePoint`).
- * @returns A string representing the characters encoded by the input code points.
- *
- * @example
- *
- * ```ts
- * const codePoints: List<CodePoint> = [0x48, 0x65, 0x6C, 0x6C, 0x6F]
- * const outputString = codePointListToString(codePoints)
- * ```
- */
-export const codePointListToString = (input) => listToString(fromCodePointList(input));
-/**
- * Converts a single Unicode code point to a string — the scalar counterpart of
- * {@link codePointListToString}, for callers that hold one code point rather
- * than a list of them and would otherwise wrap it in a single-element list.
- *
- * Input outside the assignable range — a surrogate, a value above
- * `0x10FFFF`, or a negative one — is truncated to its low 16 bits and encoded
- * as that code unit, rather than rejected. This matches
- * {@link codePointListToString}, which reports malformed input through the
- * decoder's `errorMask` on the way *in* rather than through a failure on the
- * way out.
- *
- * @param codePoint - A Unicode code point (`CodePoint`).
- * @returns The one- or two-code-unit string encoding it.
- *
- * @example
- *
- * ```ts
- * codePointToString(0x48)      // 'H'
- * codePointToString(0x1F600)   // the 😀 surrogate pair
- * codePointToString(0x110000)  // '\u0000' — out of range, low 16 bits kept
- * ```
- */
-export const codePointToString = (codePoint) => String.fromCharCode(...codePointToUtf16(codePoint));

package/fjs/text/utf16/proof.f.d.ts

@@ -1,10 +0,0 @@
-export declare const proof: {
-    toCodePointList: (() => void)[];
-    fromCodePointList: (() => void)[];
-    string: (() => void)[];
-    stringToList: (() => void)[];
-    listToString: (() => void)[];
-    stringToCodePointList: (() => void)[];
-    codePointListToString: (() => void)[];
-    codePointToString: (() => void)[];
-};

package/fjs/text/utf16/proof.f.js

@@ -1,144 +0,0 @@
-import { toCodePointList, fromCodePointList, stringToList, listToString, stringToCodePointList, codePointListToString, codePointToString } from './module.f.js';
-import { stringify as jsonStringify } from '../../media/json/module.f.js';
-import { sort } from '../../types/object/module.f.js';
-import { toArray } from '../../types/list/module.f.js';
-import { assertEq } from '../../asserts/module.f.js';
-const stringify = (a) => jsonStringify(sort)(a);
-export const proof = {
-    toCodePointList: [
-        () => {
-            const result = stringify(toArray(toCodePointList([-1, 65536])));
-            assertEq(result, '[4294967295,4294967295]');
-        },
-        () => {
-            const result = stringify(toArray(toCodePointList([0, 36, 8364, 55295, 57344, 65535])));
-            assertEq(result, '[0,36,8364,55295,57344,65535]');
-        },
-        () => {
-            const result = stringify(toArray(toCodePointList([56320, 57343])));
-            assertEq(result, '[-2147427328,-2147426305]');
-        },
-        () => {
-            const result = stringify(toArray(toCodePointList([55296, 56320, 55297, 56375, 55378, 57186, 56319, 57343])));
-            assertEq(result, '[65536,66615,150370,1114111]');
-        },
-        () => {
-            const result = stringify(toArray(toCodePointList([55296, 55296])));
-            assertEq(result, '[-2147428352,-2147428352]');
-        },
-        () => {
-            const result = stringify(toArray(toCodePointList([55296, 0])));
-            assertEq(result, '[-2147428352,0]');
-        },
-        () => {
-            const result = stringify(toArray(toCodePointList([56320])));
-            assertEq(result, '[-2147427328]');
-        },
-        () => {
-            const result = stringify(toArray(toCodePointList([56320, 0])));
-            assertEq(result, '[-2147427328,0]');
-        }
-    ],
-    fromCodePointList: [
-        () => {
-            const result = stringify(toArray(fromCodePointList([0])));
-            assertEq(result, '[0]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0x24])));
-            assertEq(result, '[36]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0x20AC])));
-            assertEq(result, '[8364]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0xd7ff])));
-            assertEq(result, '[55295]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0xe000])));
-            assertEq(result, '[57344]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0xffff])));
-            assertEq(result, '[65535]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0x10000])));
-            assertEq(result, '[55296,56320]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0x10437])));
-            assertEq(result, '[55297,56375]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0x24B62])));
-            assertEq(result, '[55378,57186]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([0x10ffff])));
-            assertEq(result, '[56319,57343]');
-        },
-        () => {
-            const result = stringify(toArray(fromCodePointList([-1, 0xd800, 0xdfff, 0x110000])));
-            assertEq(result, '[65535,55296,57343,0]');
-        }
-    ],
-    string: [
-        () => {
-            const utf16List = stringToList("Hello world!😂🚜🚲");
-            const result = listToString(utf16List);
-            assertEq(result, "Hello world!😂🚜🚲");
-        },
-        () => {
-            const cpList = stringToCodePointList("Hello world!😂🚜🚲");
-            const result = codePointListToString(cpList);
-            assertEq(result, "Hello world!😂🚜🚲");
-        },
-        () => {
-            const a = stringToList("Hello world!😂🚜🚲");
-            const b = toCodePointList(a);
-            const c = fromCodePointList(b);
-            const result = listToString(c);
-            assertEq(result, "Hello world!😂🚜🚲");
-        }
-    ],
-    stringToList: [
-        () => {
-            const inputString = "Hello, i like js";
-            const utf16List = stringToList(inputString);
-        },
-        () => {
-            const inputString = "😇🤬🫥😑🫠";
-            const utf16List = stringToList(inputString);
-        }
-    ],
-    listToString: [
-        () => {
-            const utf16List = [0x0041, 0x0042, 0x0043];
-            const outputString = listToString(utf16List);
-        }
-    ],
-    stringToCodePointList: [
-        () => {
-            const inputString = "Hello, 😀";
-            const codePoints = stringToCodePointList(inputString);
-        }
-    ],
-    codePointListToString: [
-        () => {
-            const codePoints = [0x48, 0x65, 0x6C, 0x6C, 0x6F];
-            const outputString = codePointListToString(codePoints);
-        }
-    ],
-    codePointToString: [
-        () => { assertEq(codePointToString(0x48), 'H'); },
-        // supplementary plane: one code point, two code units
-        () => { assertEq(codePointToString(0x1f600), '😀'); },
-        // a surrogate is not a code point; it round-trips as its own code unit
-        () => { assertEq(codePointToString(0xd800), '\ud800'); },
-        // above the maximum code point, the low 16 bits are kept
-        () => { assertEq(codePointToString(0x110000), '\u0000'); },
-    ]
-};

package/fjs/text/utf8/module.f.d.ts

@@ -1,83 +0,0 @@
-/**
- * UTF-8 byte-level encoding and decoding utilities for FunctionalScript streams.
- *
- * @module
- */
-import { type List, type Thunk } from '../../types/list/module.f.ts';
-import type { StateScan } from '../../types/function/operator/module.f.ts';
-import type { Array1, Array2, Array3 } from '../../types/array/module.f.ts';
-import { type Vec } from '../../types/bit_vec/module.f.ts';
-/**
- * An unsigned 8-bit integer, represents a single byte.
- */
-export type U8 = number;
-/**
- * A singed 32-bit integer.
- */
-export type I32 = number;
-/**
- * Represents an unsigned 8-bit type - U8 or the end-of-file indicator.
- * The U8 represents the byte itself, and null indicates that reading does not return anything else.
- */
-export type ByteOrEof = U8 | null;
-/**
- * Represents the state of a UTF-8 decoding operation that contains at least one byte.
- */
-export type Utf8NonEmptyState = Array1<number> | Array2<number> | Array3<number>;
-/**
- * Represents the state of a UTF-8 decoding operation, which can be either `null` (no state)
- * or a non-empty state containing one or more bytes.
- */
-export type Utf8State = null | Utf8NonEmptyState;
-/**
- * Maps a list of Unicode code points to a stream of UTF-8 bytes.
- *
- * @param input - A list of Unicode code points to be converted.
- * @returns A thunk that lazily produces a sequence of UTF-8 bytes.
- */
-export declare const fromCodePointList: (input: List<number>) => Thunk<U8>;
-/**
- * Converts a non-empty UTF-8 decoding state to an error code.
- *
- * @param state - A non-empty UTF-8 decoding state.
- * @returns An I32 error code derived from the invalid UTF-8 state.
- */
-export declare const utf8StateToError: (state: Utf8NonEmptyState) => I32;
-/**
- * Decodes a byte into a Unicode code point, using a given UTF-8 state.
- *
- * Rejects overlong 3-/4-byte encodings (Unicode Table 3-7): a lead `E0` must
- * be followed by a continuation `>= 0xA0`, and a lead `F0` by a continuation
- * `>= 0x90`. It does not itself reject surrogates (`ED A0..BF`) or code
- * points above `U+10FFFF` (`F4 90..BF`); {@link fromVec}'s
- * `isValidCodePoint` pass filters those out of the raw code-point stream.
- *
- * @param state - The current UTF-8 decoding state.
- * @param byte - A single byte to decode.
- * @returns A tuple containing:
- *   - A list of decoded Unicode code points or error codes.
- *   - The updated UTF-8 state.
- */
-export declare const utf8ByteToCodePointOp: StateScan<number, Utf8State, readonly I32[]>;
-/**
- * Handles the end-of-file (EOF) case for UTF-8 decoding.
- *
- * @param state - The current UTF-8 decoding state.
- * @returns A tuple containing:
- *   - A list of decoded Unicode code points or error codes.
- *   - The reset UTF-8 state (`null`).
- */
-export declare const utf8EofToCodePointOp: (state: Utf8State) => readonly [List<I32>, Utf8State];
-/**
- * Converts a list of UTF-8 bytes into a list of Unicode code points.
- *
- * @param input - A list of UTF-8 bytes.
- * @returns A list of Unicode code points or error codes.
- */
-export declare const toCodePointList: (input: List<U8>) => List<I32>;
-/**
- * Returns the decoded string if `v` is valid UTF-8, or `null` otherwise.
- * Rejects non-octet Vecs, invalid byte sequences, surrogates, and out-of-range
- * code points.
- */
-export declare const fromVec: (v: Vec) => string | null;