@volter/world-core 3.0.155 → 3.0.157

package/dist/src/ancestry.js

@@ -4,7 +4,7 @@ import { createHash, randomUUID } from 'node:crypto';
 import { volterHome } from "./volter-home.js";
 import { basename, dirname, join, relative, resolve, sep } from 'node:path';
 import { stateDirName } from "./storage.js";
-import { getActiveWorldStore } from "./world-store.js";
+import { getActiveWorldStore, storageLockScope, withStorageLockOperation } from "./world-store.js";
 let entered;
 export const canonicalStatePath = (path) => getActiveWorldStore().canonicalPath?.(path) ?? resolve(path);
 const metaPath = (dir) => join(dir, 'ancestry.json');
@@ -14,7 +14,8 @@ const inside = (base, path) => { const rel = relative(canonicalStatePath(base),
  * bookkeeping (generations, pins, branch pointers, removal receipts) and the not-being-removed check: every World
  * sharing the VOLTER_HOME waits on it, so no append, fold or caller's critical section runs inside it, and no state lock
  * is taken inside it (`withStateLock` takes its lock first). */
-export function withAncestryLock(fn) {
+export function withAncestryLock(fn, options = {}) {
+    const scope = storageLockScope(options);
     const store = getActiveWorldStore();
     const active = entered ??= new WeakSet();
     if (active.has(store))
@@ -23,13 +24,13 @@ export function withAncestryLock(fn) {
     return store.withLock(path, () => {
         active.add(store);
         try {
-            recoverRemovals();
+            recoverRemovals(path);
             return fn();
         }
         finally {
             active.delete(store);
         }
-    }, { retainLiveOwner: true });
+    }, { ...scope, retainLiveOwner: true });
 }
 /** The state locks this process holds, per store: taken again inside one, a lock is already ours. */
 let heldLocks;
@@ -39,15 +40,27 @@ const heldBy = (store) => { const map = heldLocks ??= new WeakMap(); let set = m
 } return set; };
 /** Run `fn` holding the cross-process lock at `lockPath`, a lock of the directory it sits in: taken first, then the
  *  coordinator only to check that the directory is not being removed. Re-entrant within a process. */
-export function withStateLock(lockPath, fn) {
+export function withStateLock(lockPath, fn, options = {}) {
+    const scope = storageLockScope(options);
     const store = getActiveWorldStore();
     const held = heldBy(store);
     if (held.has(lockPath))
         return fn();
     if (entered?.has(store))
         throw new Error(`State lock ${lockPath} taken inside the ancestry coordinator: a state lock is taken before it, never inside it`);
+    // A lock taken inside a held lock of the SAME directory asks the coordinator nothing: the outer take checked that
+    // the directory is not being removed, and a removal recorded since waits on that outer lock before it deletes
+    // anything, so this operation finishes first, as a holder that checked before the receipt does. Asked again, each
+    // write took the machine-wide coordinator three times for one directory.
+    const directory = dirname(lockPath);
+    const nested = [...held].some((path) => dirname(path) === directory);
     return store.withLock(lockPath, () => {
-        withAncestryLock(() => assertNotBeingRemoved(dirname(lockPath)));
+        // The removal check reads the receipts without the coordinator, and takes it only where one is there (S2 R17;
+        // architecture, the history lock): this holder already has the directory's lock, and a removal writes its receipt
+        // under the coordinator before it waits on every lock inside the tree, so a receipt not yet visible here belongs to
+        // a removal that waits for this holder, which finishes first. A visible receipt is checked, recovered, as before.
+        if (!nested && removalRecorded())
+            withAncestryLock(() => assertNotBeingRemoved(directory), { operation: 'removal preflight' });
         held.add(lockPath);
         try {
             return fn();
@@ -55,12 +68,12 @@ export function withStateLock(lockPath, fn) {
         finally {
             held.delete(lockPath);
         }
-    });
+    }, scope);
 }
 /** A state directory's history lock: its appends, captures, observation batches and folds. A branch's is taken before
  *  its ancestors'. */
-export function withHistoryLock(directory, fn) {
-    return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn);
+export function withHistoryLock(directory, fn, options = {}) {
+    return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn, storageLockScope(options));
 }
 /** A lock file a holder has recorded itself in (`{pid, hostname}`, written before the holder runs anything): a file
  *  that only ends in `.lock` (a yarn.lock) is not one. */
@@ -74,9 +87,10 @@ function isHeldLock(text) {
     }
 }
 /** Several sibling directories' history locks at once (a World's services, a changeset's targets), in one order. */
-export function withHistoryLocks(directories, fn) {
+export function withHistoryLocks(directories, fn, options = {}) {
+    const scope = storageLockScope(options);
     const sorted = [...new Set(directories.map((d) => canonicalStatePath(d)))].sort();
-    const take = (i) => (i === sorted.length ? fn() : withHistoryLock(sorted[i], () => take(i + 1)));
+    const take = (i) => (i === sorted.length ? fn() : withHistoryLock(sorted[i], () => take(i + 1), scope));
     return take(0);
 }
 /** The locks inside a tree a removal waits on before deleting it. */
@@ -125,7 +139,7 @@ export function stateGeneration(dir) {
             store.writeAtomic(metaPath(dir), JSON.stringify(meta));
         }
         return meta.generation;
-    });
+    }, { operation: 'state generation' });
 }
 /** Must be called in the same critical section as child pointer publication. A crash before
  * publication leaves a pending pin; it can be ignored only once its writer is provably dead. */
@@ -254,23 +268,34 @@ function releaseOutgoing(removal) {
 }
 /** A crash after deleting files cannot strand committed child pins. Receipts are replayed
  * before any subsequent ancestry publication, so path reuse cannot hide completed removal. */
-function recoverRemovals() {
+function recoverRemovals(lockPath) {
     const store = getActiveWorldStore();
+    const receipts = store.list(removalDir()).filter(n => n.endsWith('.json'));
+    if (!receipts.length) {
+        beingRemoved = [];
+        return;
+    }
     const pending = [];
-    for (const name of store.list(removalDir()).filter(n => n.endsWith('.json'))) {
-        const file = join(removalDir(), name);
-        const removal = JSON.parse(store.read(file));
-        if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing))
-            throw new Error('Invalid branch removal receipt; retained');
-        if (!store.exists(removal.path)) {
-            releaseOutgoing(removal);
-            store.remove(file);
+    withStorageLockOperation(lockPath, 'ancestry removal recovery', () => {
+        for (const name of receipts) {
+            const file = join(removalDir(), name);
+            const removal = JSON.parse(store.read(file));
+            if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing))
+                throw new Error('Invalid branch removal receipt; retained');
+            if (!store.exists(removal.path)) {
+                releaseOutgoing(removal);
+                store.remove(file);
+            }
+            else
+                pending.push(removal.path);
         }
-        else
-            pending.push(removal.path);
-    }
+    });
     beingRemoved = pending;
 }
+/** Whether any removal receipt is recorded now: read without the coordinator (withStateLock). */
+function removalRecorded() {
+    return getActiveWorldStore().list(removalDir()).some((name) => name.endsWith('.json'));
+}
 /** State whose removal is recorded and not yet complete: its deletion runs outside the coordinator, so nothing may pin
  * it as a parent or write into it meanwhile. Read under the coordinator, where recoverRemovals has just refreshed it. */
 let beingRemoved = [];
@@ -298,21 +323,21 @@ export function withStateRemoval(path, remove, check) {
         store.writeAtomic(receipt, JSON.stringify(removal));
         beingRemoved = [...beingRemoved, removal.path];
         return { removal, receipt };
-    });
+    }, { operation: 'removal preflight' });
     // Each lock is waited on alone, taken and let go: a holder that checked before the receipt finishes, and a taker
     // after it refuses once it holds the lock (withStateLock), so the removal never holds one lock while waiting on
     // another and no lock order can deadlock it. A wait that fails leaves nothing deleted: the receipt is released.
     const held = heldBy(store);
     try {
         for (const lock of locksUnder(path).filter((l) => !held.has(l)))
-            store.withLock(lock, () => undefined);
+            store.withLock(lock, () => undefined, { operation: 'removal preflight' });
     }
     catch (error) {
         withAncestryLock(() => {
             if (store.exists(receipt))
                 store.remove(receipt);
             beingRemoved = beingRemoved.filter((p) => p !== removal.path);
-        });
+        }, { operation: 'ancestry removal recovery' });
         throw error;
     }
     const result = remove();
@@ -323,6 +348,6 @@ export function withStateRemoval(path, remove, check) {
         releaseOutgoing(removal);
         if (store.exists(receipt))
             store.remove(receipt);
-    });
+    }, { operation: 'ancestry removal recovery' });
     return result;
 }

package/dist/src/blob-store.d.ts

@@ -19,7 +19,22 @@ export interface BlobStore {
      *  whole. Optional: `readBlobRange` falls back to `get` (a whole read) for a backend without it —
      *  the hosted World's R2 store today. */
     getRange?(key: string, start: number, endInclusive: number): Promise<Uint8Array | null>;
+    /** A body's chunks written to `key` as they arrive and hashed as they are written, kept (atomically, as `put`)
+     *  only when `accept` holds for their size and SHA-256; otherwise nothing is kept. True when kept. So an upload
+     *  of a large object (a model's weights) never holds more than a chunk. Optional: `writeBlobStream` collects
+     *  the chunks and `put`s them for a backend without it. */
+    putStream?(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean>;
 }
+/** What a streamed write wrote: its byte length and its SHA-256 (hex). */
+export type BlobWritten = {
+    size: number;
+    sha256: string;
+};
+/** A body's chunks to `key` through the active store, kept when `accept` holds for what was written (the store's
+ *  own streamed write where it has one, else the chunks collected and `put`). True when kept. */
+export declare function writeBlobStream(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean>;
+/** A web stream's chunks, read by its reader (a browser's ReadableStream need not be async-iterable). */
+export declare function streamChunks(body: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array>;
 /** A range of the blob at `key` through the active store: native when the store has it, else sliced. */
 export declare function readBlobRange(key: string, start: number, endInclusive: number): Promise<Uint8Array | null>;
 /** Today's layout, verbatim: keys are filesystem paths. */
@@ -30,6 +45,7 @@ export declare class FsBlobStore implements BlobStore {
     });
     get(key: string): Promise<Uint8Array | null>;
     put(key: string, bytes: Uint8Array): Promise<void>;
+    putStream(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean>;
     exists(key: string): Promise<boolean>;
     remove(key: string): Promise<void>;
     list(prefix: string): Promise<string[]>;

package/dist/src/blob-store.js

@@ -19,11 +19,50 @@
 // local worlds change NOTHING — scrub, backup, and inspection all still see the same tree.
 // A remote adapter maps the path to an object key (namespace prefix + path).
 import { createHash, randomUUID } from 'node:crypto';
-import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { volterHome } from "./volter-home.js";
 import { sharedBlobIndex } from "./shared-blob-index.js";
 import { refuseReadOnlyWrite, writesRefused } from "./request-scope.js";
+/** A body's chunks to `key` through the active store, kept when `accept` holds for what was written (the store's
+ *  own streamed write where it has one, else the chunks collected and `put`). True when kept. */
+export async function writeBlobStream(key, chunks, accept) {
+    const store = getActiveBlobStore();
+    if (store.putStream)
+        return store.putStream(key, chunks, accept);
+    const parts = [];
+    let size = 0;
+    for await (const chunk of chunks) {
+        parts.push(chunk);
+        size += chunk.byteLength;
+    }
+    const bytes = new Uint8Array(size);
+    let at = 0;
+    for (const part of parts) {
+        bytes.set(part, at);
+        at += part.byteLength;
+    }
+    if (!accept({ size, sha256: blobDigest(bytes) }))
+        return false;
+    await store.put(key, bytes);
+    return true;
+}
+/** A web stream's chunks, read by its reader (a browser's ReadableStream need not be async-iterable). */
+export async function* streamChunks(body) {
+    const reader = body.getReader();
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                return;
+            if (value)
+                yield value;
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
 /** A range of the blob at `key` through the active store: native when the store has it, else sliced. */
 export async function readBlobRange(key, start, endInclusive) {
     // one clamping for every backend: a negative or non-finite start reads from 0, a range that ends
@@ -72,6 +111,40 @@ export class FsBlobStore {
             rmSync(tmp, { force: true });
         }
     }
+    async putStream(key, chunks, accept) {
+        mkdirSync(dirname(key), { recursive: true });
+        const tmp = `${key}.tmp.${randomUUID()}`;
+        const hash = createHash('sha256');
+        let size = 0;
+        try {
+            const fd = openSync(tmp, 'wx');
+            try {
+                for await (const chunk of chunks) {
+                    hash.update(chunk);
+                    size += chunk.byteLength;
+                    for (let at = 0; at < chunk.byteLength;)
+                        at += writeSync(fd, chunk, at, chunk.byteLength - at);
+                }
+            }
+            finally {
+                closeSync(fd);
+            }
+            const sha256 = hash.digest('hex');
+            if (!accept({ size, sha256 }))
+                return false;
+            renameSync(tmp, key);
+            if (this.options.sharedIndexRoot !== false) {
+                try {
+                    sharedBlobIndex(this.options.sharedIndexRoot ?? join(volterHome(), 'blob-index-v1'), statSync(dirname(key)).dev, sha256).record(key);
+                }
+                catch { /* The index is an optimization, never a prerequisite for a blob write. */ }
+            }
+            return true;
+        }
+        finally {
+            rmSync(tmp, { force: true });
+        }
+    }
     async exists(key) {
         return existsSync(key);
     }
@@ -180,7 +253,9 @@ function readOnlyBlobs(store) {
         exists: (key) => store.exists(key),
         list: (prefix) => store.list(prefix),
         size: (key) => store.size(key),
+        ...(store.getRange ? { getRange: (key, start, endInclusive) => store.getRange(key, start, endInclusive) } : {}),
         put: async () => { refuseReadOnlyWrite('blobs'); },
+        putStream: async () => { refuseReadOnlyWrite('blobs'); return false; },
         remove: async () => { refuseReadOnlyWrite('blobs'); },
     };
 }

package/dist/src/changeset.js

@@ -727,7 +727,7 @@ export function narrationDrift(changeset) {
     return expected === changeset.narration ? null : { recorded: changeset.narration, expected };
 }
 export function rebaseChangeset(changeset, opts) {
-    return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts));
+    return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts), { operation: 'branch-pointer adjustment' });
 }
 function rebaseChangesetLocked(changeset, opts) {
     const at = opts.at ?? new Date().toISOString();

package/dist/src/clickhouse/compression.d.ts

@@ -0,0 +1,28 @@
+type Pair = [bigint, bigint];
+/** The CityHash 1.0.2 checksum used by ClickHouse native compression frames. */
+export declare function cityHash128(bytes: Uint8Array): Pair;
+export declare const MAX_COMPRESSED_BLOCK_SIZE = 1073741824;
+export declare const MAX_DECOMPRESSED_BLOCK_SIZE = 1073741824;
+/** LZ4 block sequences. Validate all lengths before allocating the output buffer. */
+export declare function decodeLz4Block(input: Uint8Array, size: number): Uint8Array;
+/** A valid literal-only LZ4 block. */
+export declare function encodeLz4Literals(input: Uint8Array): Uint8Array;
+export declare class ClickHouseCompressionError extends Error {
+    readonly code: number;
+    constructor(code: number, message: string);
+}
+/** ClickHouse compressed frame, including its sixteen-byte checksum.
+ * ICompressionCodec.cpp:62–69,102–116 (SHA2562cfbdbd7…) writes the9-byte header.
+ * CompressedReadBufferBase.cpp:193–194,255–256 (SHA2566f5af3a8…) reads low/high
+ * little-endian checksum words and verifies the complete header+payload before decompressing.
+ */
+export declare function encodeFrame(input: Uint8Array, method?: 0x82 | 0x02): Uint8Array;
+/** Decode one frame; undefined means the input has not yet supplied the whole frame.
+ * Sizes are validated from the header before waiting for its body, and the checksum
+ * precedes decompression. bytesRead allows the caller to retain subsequent frames.
+ */
+export declare function decodeFrame(input: Uint8Array): {
+    data: Uint8Array;
+    bytesRead: number;
+} | undefined;
+export {};

package/dist/src/clickhouse/compression.js

@@ -0,0 +1,281 @@
+// CityHashv1.0.2 arithmetic, ported from the vendor-carried Google source.
+// Source: ClickHouse/ClickHouse contrib/cityhash102/src/city.cc, SHA256
+// 4bcc8f0c09e3ef38210a0c568efc0550e4cb426719d8ad74a466f152296d0df7.
+// Copyright (c) 2011 Google, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+const k0 = 0xc3a5c85c97cb3127n, k1 = 0xb492b66fbe98f273n, k2 = 0x9ae16a3b2f90404fn, k3 = 0xc949d7c7509e6557n;
+const u = (value) => BigInt.asUintN(64, value);
+const rotate = (value, shift) => shift === 0 ? u(value) : u((u(value) >> BigInt(shift)) | (u(value) << BigInt(64 - shift)));
+const mix = (value) => u(value) ^ (u(value) >> 47n);
+function hash16(first, second) {
+    const mul = 0x9ddfea08eb382d69n;
+    let a = u((u(first) ^ u(second)) * mul);
+    a ^= a >> 47n;
+    let b = u((u(second) ^ a) * mul);
+    b ^= b >> 47n;
+    return u(b * mul);
+}
+/** The CityHash 1.0.2 checksum used by ClickHouse native compression frames. */
+export function cityHash128(bytes) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const fetch = (at) => view.getBigUint64(at, true);
+    const short = (at, length) => {
+        if (length > 8) {
+            const a = fetch(at), b = fetch(at + length - 8);
+            return hash16(a, rotate(u(b + BigInt(length)), length)) ^ b;
+        }
+        if (length >= 4)
+            return hash16(BigInt(length) + (BigInt(view.getUint32(at, true)) << 3n), BigInt(view.getUint32(at + length - 4, true)));
+        if (length) {
+            const a = bytes[at], b = bytes[at + (length >> 1)], c = bytes[at + length - 1];
+            return u(mix(u(BigInt(a + (b << 8)) * k2) ^ u(BigInt(length + (c << 2)) * k3)) * k2);
+        }
+        return k2;
+    };
+    const weak = (at, first, second) => {
+        const w = fetch(at), x = fetch(at + 8), y = fetch(at + 16), z = fetch(at + 24);
+        let a = u(first + w), b = rotate(u(second + a + z), 21);
+        const c = a;
+        a = u(a + x + y);
+        b = u(b + rotate(a, 44));
+        return [u(a + z), u(b + c)];
+    };
+    const murmur = (at, length, seed) => {
+        let [a, b] = seed, c = 0n, d = 0n, left = length - 16;
+        if (left <= 0) {
+            a = u(mix(u(a * k1)) * k1);
+            c = u(b * k1 + short(at, length));
+            d = mix(u(a + (length >= 8 ? fetch(at) : c)));
+        }
+        else {
+            c = hash16(u(fetch(at + length - 8) + k1), a);
+            d = hash16(u(b + BigInt(length)), u(c + fetch(at + length - 16)));
+            a = u(a + d);
+            do {
+                a ^= u(mix(u(fetch(at) * k1)) * k1);
+                a = u(a * k1);
+                b ^= a;
+                c ^= u(mix(u(fetch(at + 8) * k1)) * k1);
+                c = u(c * k1);
+                d ^= c;
+                at += 16;
+                left -= 16;
+            } while (left > 0);
+        }
+        a = hash16(a, c);
+        b = hash16(d, b);
+        return [a ^ b, hash16(b, a)];
+    };
+    const seeded = (at, length, seed) => {
+        if (length < 128)
+            return murmur(at, length, seed);
+        let [x, y] = seed, z = u(BigInt(length) * k1);
+        let v = [u(rotate(y ^ k1, 49) * k1 + fetch(at)), 0n];
+        v[1] = u(rotate(v[0], 42) * k1 + fetch(at + 8));
+        let w = [u(rotate(u(y + z), 35) * k1 + x), u(rotate(u(x + fetch(at + 88)), 53) * k1)];
+        const round = () => {
+            x = u(rotate(u(x + y + v[0] + fetch(at + 16)), 37) * k1);
+            y = u(rotate(u(y + v[1] + fetch(at + 48)), 42) * k1);
+            x ^= w[1];
+            y ^= v[0];
+            z = rotate(z ^ w[0], 33);
+            v = weak(at, u(v[1] * k1), u(x + w[0]));
+            w = weak(at + 32, u(z + w[1]), y);
+            [z, x] = [x, z];
+            at += 64;
+        };
+        do {
+            round();
+            round();
+            length -= 128;
+        } while (length >= 128);
+        y = u(y + rotate(w[0], 37) * k0 + z);
+        x = u(x + rotate(u(v[0] + z), 49) * k0);
+        for (let done = 0; done < length;) {
+            done += 32;
+            y = u(rotate(u(y - x), 42) * k0 + v[1]);
+            w[0] = u(w[0] + fetch(at + length - done + 16));
+            x = u(rotate(x, 49) * k0 + w[0]);
+            w[0] = u(w[0] + v[0]);
+            v = weak(at + length - done, v[0], v[1]);
+        }
+        x = hash16(x, v[0]);
+        y = hash16(y, w[0]);
+        return [u(hash16(u(x + v[1]), w[1]) + y), hash16(u(x + w[1]), u(y + v[1]))];
+    };
+    const length = bytes.length;
+    if (length >= 16)
+        return seeded(16, length - 16, [fetch(0) ^ k3, fetch(8)]);
+    if (length >= 8)
+        return seeded(0, 0, [fetch(0) ^ u(BigInt(length) * k0), fetch(length - 8) ^ k1]);
+    return seeded(0, length, [k0, k1]);
+}
+// ClickHouse src/Compression/CompressionInfo.h:7,14 (SHA25614b39ae2…): both limits are1GiB.
+export const MAX_COMPRESSED_BLOCK_SIZE = 0x40000000;
+export const MAX_DECOMPRESSED_BLOCK_SIZE = 0x40000000;
+/** LZ4 block sequences. Validate all lengths before allocating the output buffer. */
+export function decodeLz4Block(input, size) {
+    if (!Number.isSafeInteger(size) || size < 0 || size > MAX_DECOMPRESSED_BLOCK_SIZE || input.length > MAX_COMPRESSED_BLOCK_SIZE)
+        throw new RangeError('Invalid LZ4 block size');
+    // lz4_Block_format.md:114–126 requires a final literal sequence, five final literals
+    // (except a short all-literal block), and the last match at least12 bytes before its end.
+    // This is the format specification's strictness; ClickHouse's decoder is not retained
+    // here and may accept malformed end sequences that this decoder rejects.
+    const walk = (output) => {
+        let source = 0, target = 0, sequences = 0, finalLiterals = -1, lastMatchStart = -1;
+        const length = (initial) => {
+            let result = initial;
+            if (initial === 15) {
+                let value;
+                do {
+                    if (source >= input.length)
+                        throw new RangeError('Truncated LZ4 length');
+                    value = input[source++];
+                    result += value;
+                } while (value === 255);
+            }
+            return result;
+        };
+        while (source < input.length) {
+            sequences++;
+            const token = input[source++], literals = length(token >> 4);
+            if (literals > input.length - source || literals > size - target)
+                throw new RangeError('Invalid LZ4 literal range');
+            output?.set(input.subarray(source, source + literals), target);
+            source += literals;
+            target += literals;
+            if (source === input.length) {
+                finalLiterals = literals;
+                break;
+            }
+            if (source + 2 > input.length)
+                throw new RangeError('Truncated LZ4 offset');
+            const offset = input[source++] | (input[source++] << 8);
+            if (!offset || offset > target)
+                throw new RangeError('Invalid LZ4 offset');
+            const match = length(token & 15) + 4;
+            if (match > size - target)
+                throw new RangeError('Invalid LZ4 match range');
+            lastMatchStart = target;
+            if (output)
+                for (let i = 0; i < match; i++) {
+                    output[target] = output[target - offset];
+                    target++;
+                }
+            else
+                target += match;
+        }
+        if (finalLiterals < 0 || (size >= 5 && finalLiterals < 5) || (size < 5 && sequences !== 1) || (lastMatchStart >= 0 && size - lastMatchStart < 12))
+            throw new RangeError('Invalid LZ4 end sequence');
+        if (target !== size)
+            throw new RangeError('LZ4 decoded size differs');
+    };
+    walk();
+    const output = new Uint8Array(size);
+    walk(output);
+    return output;
+}
+/** A valid literal-only LZ4 block. */
+export function encodeLz4Literals(input) {
+    if (input.length > MAX_DECOMPRESSED_BLOCK_SIZE)
+        throw new RangeError('LZ4 block is too large');
+    const extension = input.length >= 15 ? Math.floor((input.length - 15) / 255) + 1 : 0;
+    const output = new Uint8Array(1 + extension + input.length);
+    let at = 0;
+    output[at++] = Math.min(input.length, 15) << 4;
+    if (extension) {
+        let left = input.length - 15;
+        while (left >= 255) {
+            output[at++] = 255;
+            left -= 255;
+        }
+        output[at++] = left;
+    }
+    output.set(input, at);
+    return output;
+}
+export class ClickHouseCompressionError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = 'ClickHouseCompressionError';
+    }
+}
+/** ClickHouse compressed frame, including its sixteen-byte checksum.
+ * ICompressionCodec.cpp:62–69,102–116 (SHA2562cfbdbd7…) writes the9-byte header.
+ * CompressedReadBufferBase.cpp:193–194,255–256 (SHA2566f5af3a8…) reads low/high
+ * little-endian checksum words and verifies the complete header+payload before decompressing.
+ */
+export function encodeFrame(input, method = 0x82) {
+    if (!input.length || input.length > MAX_DECOMPRESSED_BLOCK_SIZE)
+        throw new ClickHouseCompressionError(39, 'Invalid decompressed block size');
+    if (method !== 0x82 && method !== 0x02)
+        throw new ClickHouseCompressionError(89, 'Unknown compression method');
+    const bodyLength = method === 0x82 ? 1 + (input.length >= 15 ? Math.floor((input.length - 15) / 255) + 1 : 0) + input.length : input.length;
+    if (bodyLength + 9 > MAX_COMPRESSED_BLOCK_SIZE)
+        throw new ClickHouseCompressionError(39, 'Compressed block is too large');
+    const body = method === 0x82 ? encodeLz4Literals(input) : input;
+    const output = new Uint8Array(25 + body.length), view = new DataView(output.buffer);
+    output[16] = method;
+    view.setUint32(17, 9 + body.length, true);
+    view.setUint32(21, input.length, true);
+    output.set(body, 25);
+    const [low, high] = cityHash128(output.subarray(16));
+    view.setBigUint64(0, low, true);
+    view.setBigUint64(8, high, true);
+    return output;
+}
+/** Decode one frame; undefined means the input has not yet supplied the whole frame.
+ * Sizes are validated from the header before waiting for its body, and the checksum
+ * precedes decompression. bytesRead allows the caller to retain subsequent frames.
+ */
+export function decodeFrame(input) {
+    if (input.length < 25)
+        return undefined;
+    const view = new DataView(input.buffer, input.byteOffset, input.byteLength), method = input[16];
+    // CompressedReadBufferBase.cpp:134–138 selects/refuses the codec before header sizes.
+    if (method !== 0x82 && method !== 0x02)
+        throw new ClickHouseCompressionError(89, 'Unknown compression method');
+    const compressed = view.getUint32(17, true), size = view.getUint32(21, true);
+    if (compressed > MAX_COMPRESSED_BLOCK_SIZE || size > MAX_DECOMPRESSED_BLOCK_SIZE)
+        throw new ClickHouseCompressionError(39, 'Compressed block size exceeds the vendor limit');
+    if (compressed < 9 || !size)
+        throw new ClickHouseCompressionError(246, 'Corrupt compressed block header');
+    const bytesRead = 16 + compressed;
+    if (input.length < bytesRead)
+        return undefined;
+    const frame = input.subarray(16, bytesRead), [low, high] = cityHash128(frame);
+    if (low !== view.getBigUint64(0, true) || high !== view.getBigUint64(8, true))
+        throw new ClickHouseCompressionError(40, 'Checksum does not match');
+    const body = frame.subarray(9);
+    if (method === 0x02) {
+        if (body.length !== size)
+            throw new ClickHouseCompressionError(246, 'Uncompressed block size differs');
+        // Own the returned bytes, as the LZ4 branch does.
+        return { data: body.slice(), bytesRead };
+    }
+    try {
+        return { data: decodeLz4Block(body, size), bytesRead };
+    }
+    catch (error) {
+        throw new ClickHouseCompressionError(246, error instanceof Error ? error.message : 'Invalid LZ4 block');
+    }
+}

package/dist/src/clickhouse/index.d.ts

@@ -1,3 +1,4 @@
 export * from './values.js';
 export * from './types.js';
 export * from './sql.js';
+export * from './compression.js';

package/dist/src/clickhouse/index.js

@@ -4,3 +4,4 @@
 export * from "./values.js";
 export * from "./types.js";
 export * from "./sql.js";
+export * from "./compression.js";