@woosh/meep-engine 3.14.1 → 3.14.3

package/src/core/binary/zstd/zstd_decompress.js

@@ -0,0 +1,628 @@
+import { ZstdReverseBitReader } from "./ZstdReverseBitReader.js";
+import { zstd_build_fse_table, zstd_read_fse_table, zstd_rle_fse_table } from "./zstd_fse.js";
+import { zstd_decode_huffman_stream, zstd_read_huffman_table } from "./zstd_huffman.js";
+import {
+    LITERALS_LENGTH_CODE,
+    LITERALS_LENGTH_DEFAULT,
+    LITERALS_LENGTH_DEFAULT_LOG,
+    MATCH_LENGTH_CODE,
+    MATCH_LENGTH_DEFAULT,
+    MATCH_LENGTH_DEFAULT_LOG,
+    OFFSET_DEFAULT,
+    OFFSET_DEFAULT_LOG,
+} from "./zstd_sequence_tables.js";
+
+/**
+ * The four bytes every Zstandard frame opens with.
+ *
+ * @readonly
+ * @type {number}
+ */
+const MAGIC = 0xFD2FB528;
+
+/**
+ * Frames in this range are skippable: a decoder steps over them without reading their content.
+ *
+ * @readonly
+ * @type {number}
+ */
+const SKIPPABLE_MAGIC_LOW = 0x184D2A50;
+
+/**
+ * @readonly
+ * @type {number}
+ */
+const SKIPPABLE_MAGIC_HIGH = 0x184D2A5F;
+
+/**
+ * How the three sequence symbol types may be coded.
+ *
+ * @readonly
+ * @enum {number}
+ */
+const CompressionMode = {
+    Predefined: 0,
+    Rle: 1,
+    Fse: 2,
+    Repeat: 3,
+};
+
+/**
+ * Decompress a Zstandard frame.
+ *
+ * **Implemented from RFC 8878, rather than by taking a dependency.** The same argument that put
+ * meshopt, AVIF and Basis in this tree: a decompressor is a specification, and one written here is
+ * one that can be read, tested against its own absence and fixed. What it costs is this file.
+ *
+ * Zstandard is a Lempel-Ziv codec with two entropy stages over it. A block carries *literals* —
+ * bytes that appear outright, usually Huffman-coded — and *sequences*, each of which says how many
+ * literals to emit and then what to copy from what has already been produced. Both entropy stages
+ * are read backwards, because Finite State Entropy carries a state between symbols and so has to be
+ * decoded in the opposite direction to encoding.
+ *
+ * **What it does not do.** No dictionaries: none of the containers here use one, and a dictionary is
+ * a second format on top of this one. The content checksum is skipped rather than verified — the
+ * containers that carry these streams check their own payload sizes, and implementing XXH64 to
+ * re-check a length is not worth a file.
+ *
+ * @param {Uint8Array} bytes the whole frame
+ * @param {number} [expected_size] the decompressed size, when the caller knows it. A frame usually
+ * declares its own; this is for the ones that do not, and is checked against a declared size when
+ * both are present
+ * @returns {Uint8Array}
+ * @throws {Error} naming the field, for a frame this cannot read or that is not self-consistent
+ *
+ * @see https://www.rfc-editor.org/rfc/rfc8878.txt
+ * @author Alex Goldring
+ * @copyright Company Named Limited (c) 2026
+ */
+export function zstd_decompress(bytes, expected_size = undefined) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+
+    let offset = 0;
+
+    /*
+     A stream may open with skippable frames — user metadata a decoder steps over. KTX2 never writes
+     one, and stepping over them is four lines against a confusing failure if one ever appears.
+    */
+    for (; ;) {
+        if (offset + 8 > bytes.byteLength) {
+            break;
+        }
+
+        const magic = view.getUint32(offset, true);
+
+        if (magic < SKIPPABLE_MAGIC_LOW || magic > SKIPPABLE_MAGIC_HIGH) {
+            break;
+        }
+
+        offset += 8 + view.getUint32(offset + 4, true);
+    }
+
+    if (offset + 4 > bytes.byteLength || view.getUint32(offset, true) !== MAGIC) {
+        throw new Error(
+            "zstd: the input does not start with a Zstandard frame — expected the magic number"
+            + " FD2FB528"
+        );
+    }
+
+    offset += 4;
+
+    const descriptor = bytes[offset++];
+
+    const content_size_flag = descriptor >> 6;
+    const single_segment = (descriptor >> 5) & 1;
+    const has_checksum = (descriptor >> 2) & 1;
+    const dictionary_id_flag = descriptor & 3;
+
+    if ((descriptor >> 3) & 1) {
+        throw new Error(
+            "zstd: the frame header's reserved bit is set, which a later revision of the format may"
+            + " define and this decoder cannot guess at"
+        );
+    }
+
+    if (single_segment === 0) {
+        // the window descriptor, which bounds how far back a match may reach
+        offset++;
+    }
+
+    if (dictionary_id_flag !== 0) {
+        throw new Error(
+            "zstd: the frame names a dictionary, and dictionaries are not supported — the containers"
+            + " read here do not use them"
+        );
+    }
+
+    let declared_size;
+
+    const content_size_bytes = content_size_flag === 0
+        ? (single_segment ? 1 : 0)
+        : 1 << content_size_flag;
+
+    if (content_size_bytes === 1) {
+        declared_size = bytes[offset];
+    } else if (content_size_bytes === 2) {
+        // the two-byte form is biased by 256, since a smaller size would have used one byte
+        declared_size = view.getUint16(offset, true) + 256;
+    } else if (content_size_bytes === 4) {
+        declared_size = view.getUint32(offset, true);
+    } else if (content_size_bytes === 8) {
+        const low = view.getUint32(offset, true);
+        const high = view.getUint32(offset + 4, true);
+
+        declared_size = high * 4294967296 + low;
+    }
+
+    offset += content_size_bytes;
+
+    if (declared_size !== undefined && expected_size !== undefined
+        && declared_size !== expected_size) {
+        throw new Error(
+            `zstd: the frame declares ${declared_size} bytes of content and the caller expected`
+            + ` ${expected_size}`
+        );
+    }
+
+    const size = declared_size ?? expected_size;
+
+    if (size === undefined) {
+        throw new Error(
+            "zstd: the frame does not declare its content size and the caller did not supply one, so"
+            + " there is no bound on what it decompresses to"
+        );
+    }
+
+    const out = new Uint8Array(size);
+
+    const state = {
+        out,
+        written: 0,
+        literals_table: null,
+        offset_table: null,
+        match_table: null,
+        huffman: null,
+        repeat: [1, 4, 8],
+    };
+
+    for (; ;) {
+        if (offset + 3 > bytes.byteLength) {
+            throw new Error("zstd: the frame ends in the middle of a block header");
+        }
+
+        const header = bytes[offset] | (bytes[offset + 1] << 8) | (bytes[offset + 2] << 16);
+
+        offset += 3;
+
+        const last = header & 1;
+        const type = (header >> 1) & 3;
+        const block_size = header >> 3;
+
+        if (offset + (type === 1 ? 1 : block_size) > bytes.byteLength) {
+            throw new Error(
+                `zstd: a block claims ${block_size} bytes and only ${bytes.byteLength - offset}`
+                + ` remain in the frame`
+            );
+        }
+
+        if (type === 0) {
+            out.set(bytes.subarray(offset, offset + block_size), state.written);
+
+            state.written += block_size;
+            offset += block_size;
+        } else if (type === 1) {
+            out.fill(bytes[offset], state.written, state.written + block_size);
+
+            state.written += block_size;
+            offset += 1;
+        } else if (type === 2) {
+            decompress_block(bytes, offset, block_size, state);
+
+            offset += block_size;
+        } else {
+            throw new Error("zstd: a block declares the reserved block type 3");
+        }
+
+        if (last) {
+            break;
+        }
+    }
+
+    if (has_checksum) {
+        // four bytes of XXH64, which this does not verify — see the note on the function
+        offset += 4;
+    }
+
+    if (state.written !== size) {
+        throw new Error(
+            `zstd: the frame decompressed to ${state.written} bytes and declared ${size}`
+        );
+    }
+
+    return out;
+}
+
+/**
+ * Decompress one compressed block, appending to `state.out`.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset of the block's content
+ * @param {number} block_size
+ * @param {object} state
+ */
+function decompress_block(bytes, offset, block_size, state) {
+    const end = offset + block_size;
+
+    const literals = read_literals(bytes, offset, end, state);
+
+    execute_sequences(bytes, literals.end, end, literals.data, literals.count, state);
+}
+
+/**
+ * Read a block's literals section.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @param {number} end
+ * @param {object} state
+ * @returns {{data: Uint8Array, count: number, end: number}}
+ */
+function read_literals(bytes, offset, end, state) {
+    const first = bytes[offset];
+
+    const type = first & 3;
+    const size_format = (first >> 2) & 3;
+
+    let regenerated;
+    let compressed;
+    let header_bytes;
+    let streams = 1;
+
+    if (type === 0 || type === 1) {
+        if (size_format === 0 || size_format === 2) {
+            regenerated = first >> 3;
+            header_bytes = 1;
+        } else if (size_format === 1) {
+            regenerated = (first >> 4) | (bytes[offset + 1] << 4);
+            header_bytes = 2;
+        } else {
+            regenerated = (first >> 4) | (bytes[offset + 1] << 4) | (bytes[offset + 2] << 12);
+            header_bytes = 3;
+        }
+    } else if (size_format === 0 || size_format === 1) {
+        const packed = (first >> 4) | (bytes[offset + 1] << 4) | (bytes[offset + 2] << 12);
+
+        regenerated = packed & 0x3FF;
+        compressed = packed >> 10;
+        header_bytes = 3;
+        streams = size_format === 0 ? 1 : 4;
+    } else if (size_format === 2) {
+        const packed = (first >> 4) | (bytes[offset + 1] << 4) | (bytes[offset + 2] << 12)
+            | (bytes[offset + 3] << 20);
+
+        regenerated = packed & 0x3FFF;
+        compressed = Math.floor(packed / 16384) & 0x3FFF;
+        header_bytes = 4;
+        streams = 4;
+    } else {
+        const packed = (first >> 4) + bytes[offset + 1] * 16 + bytes[offset + 2] * 4096
+            + bytes[offset + 3] * 1048576 + bytes[offset + 4] * 268435456;
+
+        regenerated = packed % 262144;
+        compressed = Math.floor(packed / 262144) % 262144;
+        header_bytes = 5;
+        streams = 4;
+    }
+
+    const content = offset + header_bytes;
+
+    if (type === 0) {
+        return {
+            data: bytes.subarray(content, content + regenerated),
+            count: regenerated,
+            end: content + regenerated,
+        };
+    }
+
+    if (type === 1) {
+        const data = new Uint8Array(regenerated);
+
+        data.fill(bytes[content]);
+
+        return { data, count: regenerated, end: content + 1 };
+    }
+
+    let stream_offset = content;
+
+    if (type === 2) {
+        const { table, byte_length } = zstd_read_huffman_table(bytes, content, content + compressed);
+
+        state.huffman = table;
+        stream_offset += byte_length;
+    } else if (state.huffman === null) {
+        throw new Error(
+            "zstd: a treeless literals block reuses the previous block's Huffman table, and no"
+            + " previous block in this frame described one"
+        );
+    }
+
+    const data = new Uint8Array(regenerated);
+
+    const streams_end = content + compressed;
+
+    if (streams === 1) {
+        zstd_decode_huffman_stream(
+            state.huffman, bytes, stream_offset, streams_end - stream_offset, data, 0, regenerated
+        );
+
+        return { data, count: regenerated, end: streams_end };
+    }
+
+    /*
+     Four streams, so a decoder could run them in parallel. The first three sizes are written out and
+     the fourth is whatever is left; the output is split into four equal parts with the last taking
+     the remainder.
+    */
+    const sizes = [
+        bytes[stream_offset] | (bytes[stream_offset + 1] << 8),
+        bytes[stream_offset + 2] | (bytes[stream_offset + 3] << 8),
+        bytes[stream_offset + 4] | (bytes[stream_offset + 5] << 8),
+    ];
+
+    let at = stream_offset + 6;
+
+    const quarter = Math.ceil(regenerated / 4);
+
+    for (let i = 0; i < 4; i++) {
+        const length = i < 3 ? sizes[i] : streams_end - at;
+
+        const produced = i < 3 ? quarter : regenerated - quarter * 3;
+
+        zstd_decode_huffman_stream(
+            state.huffman, bytes, at, length, data, quarter * i, produced
+        );
+
+        at += length;
+    }
+
+    return { data, count: regenerated, end: streams_end };
+}
+
+/**
+ * Read a block's sequences and execute them against its literals.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset of the sequences section
+ * @param {number} end of the block
+ * @param {Uint8Array} literals
+ * @param {number} literal_count
+ * @param {object} state
+ */
+function execute_sequences(bytes, offset, end, literals, literal_count, state) {
+    const out = state.out;
+
+    if (offset >= end) {
+        // no sequences section at all: the block is its literals
+        out.set(literals.subarray(0, literal_count), state.written);
+
+        state.written += literal_count;
+
+        return;
+    }
+
+    const first = bytes[offset++];
+
+    let count;
+
+    if (first === 0) {
+        out.set(literals.subarray(0, literal_count), state.written);
+
+        state.written += literal_count;
+
+        return;
+    }
+
+    if (first < 128) {
+        count = first;
+    } else if (first < 255) {
+        count = ((first - 128) << 8) + bytes[offset++];
+    } else {
+        count = bytes[offset] + (bytes[offset + 1] << 8) + 0x7F00;
+        offset += 2;
+    }
+
+    const modes = bytes[offset++];
+
+    const literals_mode = modes >> 6;
+    const offsets_mode = (modes >> 4) & 3;
+    const match_mode = (modes >> 2) & 3;
+
+    if ((modes & 3) !== 0) {
+        throw new Error("zstd: the sequences header's reserved bits are set");
+    }
+
+    offset = read_sequence_table(
+        bytes, offset, end, literals_mode, state, "literals_table",
+        LITERALS_LENGTH_DEFAULT, LITERALS_LENGTH_DEFAULT_LOG, 35, 9
+    );
+
+    offset = read_sequence_table(
+        bytes, offset, end, offsets_mode, state, "offset_table",
+        OFFSET_DEFAULT, OFFSET_DEFAULT_LOG, 31, 8
+    );
+
+    offset = read_sequence_table(
+        bytes, offset, end, match_mode, state, "match_table",
+        MATCH_LENGTH_DEFAULT, MATCH_LENGTH_DEFAULT_LOG, 52, 9
+    );
+
+    const reader = new ZstdReverseBitReader().init(bytes, offset, end - offset);
+
+    /*
+     The three states are initialised in this order and then updated in a different one — literals,
+     match, offset — which is the sort of asymmetry that is invisible until the third sequence comes
+     out wrong.
+    */
+    let literals_state = reader.read(state.literals_table.accuracy_log);
+    let offset_state = reader.read(state.offset_table.accuracy_log);
+    let match_state = reader.read(state.match_table.accuracy_log);
+
+    let literal_at = 0;
+
+    for (let i = 0; i < count; i++) {
+        const offset_code = state.offset_table.symbol[offset_state];
+        const match_code = state.match_table.symbol[match_state];
+        const literals_code = state.literals_table.symbol[literals_state];
+
+        // read in this order: offset, then match length, then literals length
+        const offset_value = (2 ** offset_code) + reader.read(offset_code);
+
+        const match_length = MATCH_LENGTH_CODE.baseline[match_code]
+            + reader.read(MATCH_LENGTH_CODE.bits[match_code]);
+
+        const literals_length = LITERALS_LENGTH_CODE.baseline[literals_code]
+            + reader.read(LITERALS_LENGTH_CODE.bits[literals_code]);
+
+        const match_offset = resolve_offset(offset_value, literals_length, state.repeat);
+
+        // the literals this sequence emits, then the match copied from what is already written
+        for (let k = 0; k < literals_length; k++) {
+            out[state.written++] = literals[literal_at++];
+        }
+
+        if (match_offset > state.written) {
+            throw new Error(
+                `zstd: a sequence copies from ${match_offset} bytes back and only`
+                + ` ${state.written} have been produced`
+            );
+        }
+
+        let from = state.written - match_offset;
+
+        for (let k = 0; k < match_length; k++) {
+            out[state.written++] = out[from++];
+        }
+
+        if (i + 1 < count) {
+            literals_state = state.literals_table.baseline[literals_state]
+                + reader.read(state.literals_table.bits[literals_state]);
+
+            match_state = state.match_table.baseline[match_state]
+                + reader.read(state.match_table.bits[match_state]);
+
+            offset_state = state.offset_table.baseline[offset_state]
+                + reader.read(state.offset_table.bits[offset_state]);
+        }
+    }
+
+    // whatever literals the sequences did not consume are emitted after the last one
+    while (literal_at < literal_count) {
+        out[state.written++] = literals[literal_at++];
+    }
+}
+
+/**
+ * Resolve an offset value against the recent-offset history, and update it.
+ *
+ * **The rule is stranger than it looks and the strangeness is load-bearing.** Values one to three
+ * name one of the three most recently used offsets rather than a distance — and when the sequence
+ * emits no literals the meaning shifts by one, so that the combination which would otherwise be
+ * redundant instead names "the most recent offset, less a byte". Getting this wrong produces output
+ * that is right until the first repeat and then diverges.
+ *
+ * @param {number} offset_value
+ * @param {number} literals_length
+ * @param {number[]} repeat three offsets, most recent first, updated in place
+ * @returns {number} the distance to copy from
+ */
+function resolve_offset(offset_value, literals_length, repeat) {
+    if (offset_value > 3) {
+        const offset = offset_value - 3;
+
+        repeat[2] = repeat[1];
+        repeat[1] = repeat[0];
+        repeat[0] = offset;
+
+        return offset;
+    }
+
+    const index = literals_length === 0 ? offset_value : offset_value - 1;
+
+    if (index === 3) {
+        // "the most recent offset, less a byte", which only this combination can say
+        const offset = repeat[0] - 1;
+
+        repeat[2] = repeat[1];
+        repeat[1] = repeat[0];
+        repeat[0] = offset;
+
+        return offset;
+    }
+
+    const offset = repeat[index];
+
+    /*
+     Using one of the three moves it to the front and pushes the ones above it back, so the list
+     stays in order of recency. Using the first one changes nothing.
+    */
+    if (index === 1) {
+        repeat[1] = repeat[0];
+        repeat[0] = offset;
+    } else if (index === 2) {
+        repeat[2] = repeat[1];
+        repeat[1] = repeat[0];
+        repeat[0] = offset;
+    }
+
+    return offset;
+}
+
+/**
+ * Read one of the three sequence tables, in whichever of the four ways the block chose.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @param {number} end
+ * @param {number} mode see {@link CompressionMode}
+ * @param {object} state
+ * @param {string} slot which of `state`'s tables this is
+ * @param {number[]} predefined
+ * @param {number} predefined_log
+ * @param {number} max_symbol
+ * @param {number} max_accuracy_log
+ * @returns {number} the offset past the table description
+ */
+function read_sequence_table(
+    bytes, offset, end, mode, state, slot, predefined, predefined_log, max_symbol, max_accuracy_log
+) {
+    if (mode === CompressionMode.Predefined) {
+        state[slot] = zstd_build_fse_table(predefined, predefined.length, predefined_log);
+
+        return offset;
+    }
+
+    if (mode === CompressionMode.Rle) {
+        state[slot] = zstd_rle_fse_table(bytes[offset]);
+
+        return offset + 1;
+    }
+
+    if (mode === CompressionMode.Fse) {
+        const { table, byte_length } = zstd_read_fse_table(
+            bytes, offset, end, max_symbol, max_accuracy_log
+        );
+
+        state[slot] = table;
+
+        return offset + byte_length;
+    }
+
+    if (state[slot] === null) {
+        throw new Error(
+            `zstd: a block reuses the previous ${slot.replace("_table", "")} table and no previous`
+            + ` block in this frame described one`
+        );
+    }
+
+    return offset;
+}

package/src/core/binary/zstd/zstd_fse.d.ts

@@ -0,0 +1,83 @@
+/**
+ * One Finite State Entropy decoding table.
+ *
+ * Three parallel arrays rather than an array of objects: a table has up to 512 entries and every
+ * symbol decoded reads all three, so the shape that avoids an object per state is worth the noise.
+ *
+ * @typedef {object} ZstdFseTable
+ * @property {number} accuracy_log base-2 logarithm of the table size, and the width of an initial
+ * state read
+ * @property {Uint8Array} symbol what each state decodes to
+ * @property {Uint8Array} bits how many to read to reach the next state
+ * @property {Uint16Array} baseline what to add them to
+ */
+/**
+ * Build a decoding table from a normalised distribution.
+ *
+ * **The allocation order is the format, not a choice.** States are handed out by stepping a position
+ * through the table with a stride that visits every slot exactly once, so a symbol's states are
+ * spread rather than contiguous. Symbols with the "less than one" probability are placed first, from
+ * the end of the table backwards, and skipped over by the stepping. Getting either wrong builds a
+ * table that decodes plausible garbage rather than failing.
+ *
+ * @param {Int16Array|number[]} counts normalised probability per symbol, `-1` meaning "less than one"
+ * @param {number} symbol_count how many of `counts` are present
+ * @param {number} accuracy_log
+ * @returns {ZstdFseTable}
+ *
+ * @see RFC 8878, section 4.1.1
+ * @author Alex Goldring
+ * @copyright Company Named Limited (c) 2026
+ */
+export function zstd_build_fse_table(counts: Int16Array | number[], symbol_count: number, accuracy_log: number): ZstdFseTable;
+/**
+ * Read a normalised distribution and build its table.
+ *
+ * The distribution is read *forward* in little-endian order, unlike the streams the table then
+ * decodes — it is a header rather than an entropy-coded body.
+ *
+ * @param {Uint8Array} data
+ * @param {number} offset
+ * @param {number} end one past the last readable byte
+ * @param {number} max_symbol the largest symbol the context allows
+ * @param {number} max_accuracy_log the largest the context allows
+ * @returns {{table: ZstdFseTable, byte_length: number}}
+ * @throws {Error} for a distribution that overruns its scale or its symbol range
+ */
+export function zstd_read_fse_table(data: Uint8Array, offset: number, end: number, max_symbol: number, max_accuracy_log: number): {
+    table: ZstdFseTable;
+    byte_length: number;
+};
+/**
+ * A table for a stream that carries one symbol over and over.
+ *
+ * @param {number} value
+ * @returns {ZstdFseTable}
+ */
+export function zstd_rle_fse_table(value: number): ZstdFseTable;
+/**
+ * One Finite State Entropy decoding table.
+ *
+ * Three parallel arrays rather than an array of objects: a table has up to 512 entries and every
+ * symbol decoded reads all three, so the shape that avoids an object per state is worth the noise.
+ */
+export type ZstdFseTable = {
+    /**
+     * base-2 logarithm of the table size, and the width of an initial
+     * state read
+     */
+    accuracy_log: number;
+    /**
+     * what each state decodes to
+     */
+    symbol: Uint8Array;
+    /**
+     * how many to read to reach the next state
+     */
+    bits: Uint8Array;
+    /**
+     * what to add them to
+     */
+    baseline: Uint16Array;
+};
+//# sourceMappingURL=zstd_fse.d.ts.map

package/src/core/binary/zstd/zstd_fse.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"zstd_fse.d.ts","sourceRoot":"","sources":["../../../../../src/core/binary/zstd/zstd_fse.js"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH;;;;;;;;;;;;;;;;;GAiBG;AACH,6CATW,UAAU,GAAC,MAAM,EAAE,gBACnB,MAAM,gBACN,MAAM,GACJ,YAAY,CAgFxB;AAED;;;;;;;;;;;;;GAaG;AACH,0CARW,UAAU,UACV,MAAM,OACN,MAAM,cACN,MAAM,oBACN,MAAM,GACJ;IAAC,KAAK,EAAE,YAAY,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAC,CAkItD;AAED;;;;;GAKG;AACH,0CAHW,MAAM,GACJ,YAAY,CASxB;;;;;;;;;;;;kBAjQa,MAAM;;;;YAEN,UAAU;;;;UACV,UAAU;;;;cACV,WAAW"}