@volter/world-core 3.0.155 → 3.0.157

package/src/ancestry.ts

@@ -4,7 +4,7 @@ import { createHash, randomUUID } from 'node:crypto';
 import { volterHome } from './volter-home.ts';
 import { basename, dirname, join, relative, resolve, sep } from 'node:path';
 import { stateDirName } from './storage.ts';
-import { getActiveWorldStore, type WorldStore } from './world-store.ts';
+import { getActiveWorldStore, storageLockScope, withStorageLockOperation, type WorldLockOptions, type WorldStore } from './world-store.ts';
 
 type Pin = { child: string; ownerPid: number; committed?: boolean };
 type Ancestry = { generation: string; children: Pin[] };
@@ -18,16 +18,17 @@ const inside = (base: string, path: string) => { const rel = relative(canonicalS
  * bookkeeping (generations, pins, branch pointers, removal receipts) and the not-being-removed check: every World
  * sharing the VOLTER_HOME waits on it, so no append, fold or caller's critical section runs inside it, and no state lock
  * is taken inside it (`withStateLock` takes its lock first). */
-export function withAncestryLock<T>(fn: () => T): T {
+export function withAncestryLock<T>(fn: () => T, options: WorldLockOptions = {}): T {
+  const scope = storageLockScope(options);
   const store = getActiveWorldStore();
   const active = entered ??= new WeakSet<WorldStore>();
   if (active.has(store)) return fn();
   const path = join(volterHome(), 'branch-ancestry.lock');
   return store.withLock(path, () => {
     active.add(store);
-    try { recoverRemovals(); return fn(); }
+    try { recoverRemovals(path); return fn(); }
     finally { active.delete(store); }
-  }, { retainLiveOwner: true });
+  }, { ...scope, retainLiveOwner: true });
 }
 
 /** The state locks this process holds, per store: taken again inside one, a lock is already ours. */
@@ -36,22 +37,33 @@ const heldBy = (store: WorldStore): Set<string> => { const map = heldLocks ??= n
 
 /** Run `fn` holding the cross-process lock at `lockPath`, a lock of the directory it sits in: taken first, then the
  *  coordinator only to check that the directory is not being removed. Re-entrant within a process. */
-export function withStateLock<T>(lockPath: string, fn: () => T): T {
+export function withStateLock<T>(lockPath: string, fn: () => T, options: WorldLockOptions = {}): T {
+  const scope = storageLockScope(options);
   const store = getActiveWorldStore();
   const held = heldBy(store);
   if (held.has(lockPath)) return fn();
   if (entered?.has(store)) throw new Error(`State lock ${lockPath} taken inside the ancestry coordinator: a state lock is taken before it, never inside it`);
+  // A lock taken inside a held lock of the SAME directory asks the coordinator nothing: the outer take checked that
+  // the directory is not being removed, and a removal recorded since waits on that outer lock before it deletes
+  // anything, so this operation finishes first, as a holder that checked before the receipt does. Asked again, each
+  // write took the machine-wide coordinator three times for one directory.
+  const directory = dirname(lockPath);
+  const nested = [...held].some((path) => dirname(path) === directory);
   return store.withLock(lockPath, () => {
-    withAncestryLock(() => assertNotBeingRemoved(dirname(lockPath)));
+    // The removal check reads the receipts without the coordinator, and takes it only where one is there (S2 R17;
+    // architecture, the history lock): this holder already has the directory's lock, and a removal writes its receipt
+    // under the coordinator before it waits on every lock inside the tree, so a receipt not yet visible here belongs to
+    // a removal that waits for this holder, which finishes first. A visible receipt is checked, recovered, as before.
+    if (!nested && removalRecorded()) withAncestryLock(() => assertNotBeingRemoved(directory), { operation: 'removal preflight' });
     held.add(lockPath);
     try { return fn(); } finally { held.delete(lockPath); }
-  });
+  }, scope);
 }
 
 /** A state directory's history lock: its appends, captures, observation batches and folds. A branch's is taken before
  *  its ancestors'. */
-export function withHistoryLock<T>(directory: string, fn: () => T): T {
-  return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn);
+export function withHistoryLock<T>(directory: string, fn: () => T, options: WorldLockOptions = {}): T {
+  return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn, storageLockScope(options));
 }
 
 /** A lock file a holder has recorded itself in (`{pid, hostname}`, written before the holder runs anything): a file
@@ -61,9 +73,10 @@ function isHeldLock(text: string | null): boolean {
 }
 
 /** Several sibling directories' history locks at once (a World's services, a changeset's targets), in one order. */
-export function withHistoryLocks<T>(directories: string[], fn: () => T): T {
+export function withHistoryLocks<T>(directories: string[], fn: () => T, options: WorldLockOptions = {}): T {
+  const scope = storageLockScope(options);
   const sorted = [...new Set(directories.map((d) => canonicalStatePath(d)))].sort();
-  const take = (i: number): T => (i === sorted.length ? fn() : withHistoryLock(sorted[i]!, () => take(i + 1)));
+  const take = (i: number): T => (i === sorted.length ? fn() : withHistoryLock(sorted[i]!, () => take(i + 1), scope));
   return take(0);
 }
 
@@ -105,7 +118,7 @@ export function stateGeneration(dir: string): string {
     let meta = metadata(dir);
     if (!meta) { meta = { generation: randomUUID(), children: [] }; store.writeAtomic(metaPath(dir), JSON.stringify(meta)); }
     return meta.generation;
-  });
+  }, { operation: 'state generation' });
 }
 
 /** Must be called in the same critical section as child pointer publication. A crash before
@@ -220,19 +233,28 @@ function releaseOutgoing(removal: Removal): void {
 
 /** A crash after deleting files cannot strand committed child pins. Receipts are replayed
  * before any subsequent ancestry publication, so path reuse cannot hide completed removal. */
-function recoverRemovals(): void {
+function recoverRemovals(lockPath: string): void {
   const store = getActiveWorldStore();
+  const receipts = store.list(removalDir()).filter(n => n.endsWith('.json'));
+  if (!receipts.length) { beingRemoved = []; return; }
   const pending: string[] = [];
-  for (const name of store.list(removalDir()).filter(n => n.endsWith('.json'))) {
-    const file = join(removalDir(), name);
-    const removal = JSON.parse(store.read(file)!) as Removal;
-    if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing)) throw new Error('Invalid branch removal receipt; retained');
-    if (!store.exists(removal.path)) { releaseOutgoing(removal); store.remove(file); }
-    else pending.push(removal.path);
-  }
+  withStorageLockOperation(lockPath, 'ancestry removal recovery', () => {
+    for (const name of receipts) {
+      const file = join(removalDir(), name);
+      const removal = JSON.parse(store.read(file)!) as Removal;
+      if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing)) throw new Error('Invalid branch removal receipt; retained');
+      if (!store.exists(removal.path)) { releaseOutgoing(removal); store.remove(file); }
+      else pending.push(removal.path);
+    }
+  });
   beingRemoved = pending;
 }
 
+/** Whether any removal receipt is recorded now: read without the coordinator (withStateLock). */
+function removalRecorded(): boolean {
+  return getActiveWorldStore().list(removalDir()).some((name) => name.endsWith('.json'));
+}
+
 /** State whose removal is recorded and not yet complete: its deletion runs outside the coordinator, so nothing may pin
  * it as a parent or write into it meanwhile. Read under the coordinator, where recoverRemovals has just refreshed it. */
 let beingRemoved: string[] = [];
@@ -260,18 +282,18 @@ export function withStateRemoval<T>(path: string, remove: () => T, check?: () =>
     store.writeAtomic(receipt, JSON.stringify(removal));
     beingRemoved = [...beingRemoved, removal.path];
     return { removal, receipt };
-  });
+  }, { operation: 'removal preflight' });
   // Each lock is waited on alone, taken and let go: a holder that checked before the receipt finishes, and a taker
   // after it refuses once it holds the lock (withStateLock), so the removal never holds one lock while waiting on
   // another and no lock order can deadlock it. A wait that fails leaves nothing deleted: the receipt is released.
   const held = heldBy(store);
   try {
-    for (const lock of locksUnder(path).filter((l) => !held.has(l))) store.withLock(lock, () => undefined);
+    for (const lock of locksUnder(path).filter((l) => !held.has(l))) store.withLock(lock, () => undefined, { operation: 'removal preflight' });
   } catch (error) {
     withAncestryLock(() => {
       if (store.exists(receipt)) store.remove(receipt);
       beingRemoved = beingRemoved.filter((p) => p !== removal.path);
-    });
+    }, { operation: 'ancestry removal recovery' });
     throw error;
   }
   const result = remove();
@@ -280,6 +302,6 @@ export function withStateRemoval<T>(path: string, remove: () => T, check?: () =>
     // Another process's recovery may have released it already; both steps are idempotent.
     releaseOutgoing(removal);
     if (store.exists(receipt)) store.remove(receipt);
-  });
+  }, { operation: 'ancestry removal recovery' });
   return result;
 }

package/src/blob-store.ts

@@ -19,7 +19,7 @@
 // local worlds change NOTHING — scrub, backup, and inspection all still see the same tree.
 // A remote adapter maps the path to an object key (namespace prefix + path).
 import { createHash, randomUUID } from 'node:crypto';
-import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync, writeSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { volterHome } from './volter-home.ts';
 import { sharedBlobIndex } from './shared-blob-index.ts';
@@ -46,6 +46,38 @@ export interface BlobStore {
    *  whole. Optional: `readBlobRange` falls back to `get` (a whole read) for a backend without it —
    *  the hosted World's R2 store today. */
   getRange?(key: string, start: number, endInclusive: number): Promise<Uint8Array | null>;
+  /** A body's chunks written to `key` as they arrive and hashed as they are written, kept (atomically, as `put`)
+   *  only when `accept` holds for their size and SHA-256; otherwise nothing is kept. True when kept. So an upload
+   *  of a large object (a model's weights) never holds more than a chunk. Optional: `writeBlobStream` collects
+   *  the chunks and `put`s them for a backend without it. */
+  putStream?(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean>;
+}
+
+/** What a streamed write wrote: its byte length and its SHA-256 (hex). */
+export type BlobWritten = { size: number; sha256: string };
+
+/** A body's chunks to `key` through the active store, kept when `accept` holds for what was written (the store's
+ *  own streamed write where it has one, else the chunks collected and `put`). True when kept. */
+export async function writeBlobStream(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean> {
+  const store = getActiveBlobStore();
+  if (store.putStream) return store.putStream(key, chunks, accept);
+  const parts: Uint8Array[] = [];
+  let size = 0;
+  for await (const chunk of chunks) { parts.push(chunk); size += chunk.byteLength; }
+  const bytes = new Uint8Array(size);
+  let at = 0;
+  for (const part of parts) { bytes.set(part, at); at += part.byteLength; }
+  if (!accept({ size, sha256: blobDigest(bytes) })) return false;
+  await store.put(key, bytes);
+  return true;
+}
+
+/** A web stream's chunks, read by its reader (a browser's ReadableStream need not be async-iterable). */
+export async function* streamChunks(body: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {
+  const reader = body.getReader();
+  try {
+    for (;;) { const { done, value } = await reader.read(); if (done) return; if (value) yield value; }
+  } finally { reader.releaseLock(); }
 }
 
 /** A range of the blob at `key` through the active store: native when the store has it, else sliced. */
@@ -86,6 +118,30 @@ export class FsBlobStore implements BlobStore {
       index?.record(key);
     } finally { rmSync(tmp, { force: true }); }
   }
+  async putStream(key: string, chunks: AsyncIterable<Uint8Array>, accept: (written: BlobWritten) => boolean): Promise<boolean> {
+    mkdirSync(dirname(key), { recursive: true });
+    const tmp = `${key}.tmp.${randomUUID()}`;
+    const hash = createHash('sha256');
+    let size = 0;
+    try {
+      const fd = openSync(tmp, 'wx');
+      try {
+        for await (const chunk of chunks) {
+          hash.update(chunk);
+          size += chunk.byteLength;
+          for (let at = 0; at < chunk.byteLength;) at += writeSync(fd, chunk, at, chunk.byteLength - at);
+        }
+      } finally { closeSync(fd); }
+      const sha256 = hash.digest('hex');
+      if (!accept({ size, sha256 })) return false;
+      renameSync(tmp, key);
+      if (this.options.sharedIndexRoot !== false) {
+        try { sharedBlobIndex(this.options.sharedIndexRoot ?? join(volterHome(), 'blob-index-v1'), statSync(dirname(key)).dev, sha256).record(key); }
+        catch { /* The index is an optimization, never a prerequisite for a blob write. */ }
+      }
+      return true;
+    } finally { rmSync(tmp, { force: true }); }
+  }
   async exists(key: string): Promise<boolean> {
     return existsSync(key);
   }
@@ -178,7 +234,9 @@ function readOnlyBlobs(store: BlobStore): BlobStore {
     exists: (key) => store.exists(key),
     list: (prefix) => store.list(prefix),
     size: (key) => store.size(key),
+    ...(store.getRange ? { getRange: (key: string, start: number, endInclusive: number) => store.getRange!(key, start, endInclusive) } : {}),
     put: async () => { refuseReadOnlyWrite('blobs'); },
+    putStream: async () => { refuseReadOnlyWrite('blobs'); return false; },
     remove: async () => { refuseReadOnlyWrite('blobs'); },
   };
 }

package/src/changeset.ts

@@ -1042,7 +1042,7 @@ export type RebaseActionResult = {
 export type RebaseReport = { changeset: string; from: string; to: string | null; at: string; outcome: 'unchanged' | 'rebased' | 'conflict'; actions: RebaseActionResult[] };
 
 export function rebaseChangeset(changeset: Changeset, opts: { targets: RebaseTarget[]; at?: string }): { changeset: Changeset; report: RebaseReport } {
-  return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts));
+  return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts), { operation: 'branch-pointer adjustment' });
 }
 function rebaseChangesetLocked(changeset: Changeset, opts: { targets: RebaseTarget[]; at?: string }): { changeset: Changeset; report: RebaseReport } {
   const at = opts.at ?? new Date().toISOString();

package/src/clickhouse/compression.ts

@@ -0,0 +1,172 @@
+// CityHashv1.0.2 arithmetic, ported from the vendor-carried Google source.
+// Source: ClickHouse/ClickHouse contrib/cityhash102/src/city.cc, SHA256
+// 4bcc8f0c09e3ef38210a0c568efc0550e4cb426719d8ad74a466f152296d0df7.
+// Copyright (c) 2011 Google, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+const k0 = 0xc3a5c85c97cb3127n, k1 = 0xb492b66fbe98f273n, k2 = 0x9ae16a3b2f90404fn, k3 = 0xc949d7c7509e6557n;
+type Pair = [bigint, bigint];
+const u = (value: bigint): bigint => BigInt.asUintN(64, value);
+const rotate = (value: bigint, shift: number): bigint => shift === 0 ? u(value) : u((u(value) >> BigInt(shift)) | (u(value) << BigInt(64 - shift)));
+const mix = (value: bigint): bigint => u(value) ^ (u(value) >> 47n);
+function hash16(first: bigint, second: bigint): bigint {
+  const mul = 0x9ddfea08eb382d69n;
+  let a = u((u(first) ^ u(second)) * mul); a ^= a >> 47n;
+  let b = u((u(second) ^ a) * mul); b ^= b >> 47n;
+  return u(b * mul);
+}
+/** The CityHash 1.0.2 checksum used by ClickHouse native compression frames. */
+export function cityHash128(bytes: Uint8Array): Pair {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const fetch = (at: number): bigint => view.getBigUint64(at, true);
+  const short = (at: number, length: number): bigint => {
+    if (length > 8) { const a = fetch(at), b = fetch(at + length - 8); return hash16(a, rotate(u(b + BigInt(length)), length)) ^ b; }
+    if (length >= 4) return hash16(BigInt(length) + (BigInt(view.getUint32(at, true)) << 3n), BigInt(view.getUint32(at + length - 4, true)));
+    if (length) { const a = bytes[at]!, b = bytes[at + (length >> 1)]!, c = bytes[at + length - 1]!; return u(mix(u(BigInt(a + (b << 8)) * k2) ^ u(BigInt(length + (c << 2)) * k3)) * k2); }
+    return k2;
+  };
+  const weak = (at: number, first: bigint, second: bigint): Pair => {
+    const w = fetch(at), x = fetch(at + 8), y = fetch(at + 16), z = fetch(at + 24);
+    let a = u(first + w), b = rotate(u(second + a + z), 21); const c = a;
+    a = u(a + x + y); b = u(b + rotate(a, 44)); return [u(a + z), u(b + c)];
+  };
+  const murmur = (at: number, length: number, seed: Pair): Pair => {
+    let [a, b] = seed, c = 0n, d = 0n, left = length - 16;
+    if (left <= 0) { a = u(mix(u(a * k1)) * k1); c = u(b * k1 + short(at, length)); d = mix(u(a + (length >= 8 ? fetch(at) : c))); }
+    else {
+      c = hash16(u(fetch(at + length - 8) + k1), a); d = hash16(u(b + BigInt(length)), u(c + fetch(at + length - 16))); a = u(a + d);
+      do { a ^= u(mix(u(fetch(at) * k1)) * k1); a = u(a * k1); b ^= a; c ^= u(mix(u(fetch(at + 8) * k1)) * k1); c = u(c * k1); d ^= c; at += 16; left -= 16; } while (left > 0);
+    }
+    a = hash16(a, c); b = hash16(d, b); return [a ^ b, hash16(b, a)];
+  };
+  const seeded = (at: number, length: number, seed: Pair): Pair => {
+    if (length < 128) return murmur(at, length, seed);
+    let [x, y] = seed, z = u(BigInt(length) * k1);
+    let v: Pair = [u(rotate(y ^ k1, 49) * k1 + fetch(at)), 0n]; v[1] = u(rotate(v[0], 42) * k1 + fetch(at + 8));
+    let w: Pair = [u(rotate(u(y + z), 35) * k1 + x), u(rotate(u(x + fetch(at + 88)), 53) * k1)];
+    const round = (): void => {
+      x = u(rotate(u(x + y + v[0] + fetch(at + 16)), 37) * k1); y = u(rotate(u(y + v[1] + fetch(at + 48)), 42) * k1);
+      x ^= w[1]; y ^= v[0]; z = rotate(z ^ w[0], 33);
+      v = weak(at, u(v[1] * k1), u(x + w[0])); w = weak(at + 32, u(z + w[1]), y);
+      [z, x] = [x, z]; at += 64;
+    };
+    do { round(); round(); length -= 128; } while (length >= 128);
+    y = u(y + rotate(w[0], 37) * k0 + z); x = u(x + rotate(u(v[0] + z), 49) * k0);
+    for (let done = 0; done < length;) { done += 32; y = u(rotate(u(y - x), 42) * k0 + v[1]); w[0] = u(w[0] + fetch(at + length - done + 16)); x = u(rotate(x, 49) * k0 + w[0]); w[0] = u(w[0] + v[0]); v = weak(at + length - done, v[0], v[1]); }
+    x = hash16(x, v[0]); y = hash16(y, w[0]); return [u(hash16(u(x + v[1]), w[1]) + y), hash16(u(x + w[1]), u(y + v[1]))];
+  };
+  const length = bytes.length;
+  if (length >= 16) return seeded(16, length - 16, [fetch(0) ^ k3, fetch(8)]);
+  if (length >= 8) return seeded(0, 0, [fetch(0) ^ u(BigInt(length) * k0), fetch(length - 8) ^ k1]);
+  return seeded(0, length, [k0, k1]);
+}
+
+// ClickHouse src/Compression/CompressionInfo.h:7,14 (SHA25614b39ae2…): both limits are1GiB.
+export const MAX_COMPRESSED_BLOCK_SIZE = 0x40000000;
+export const MAX_DECOMPRESSED_BLOCK_SIZE = 0x40000000;
+
+/** LZ4 block sequences. Validate all lengths before allocating the output buffer. */
+export function decodeLz4Block(input: Uint8Array, size: number): Uint8Array {
+  if (!Number.isSafeInteger(size) || size < 0 || size > MAX_DECOMPRESSED_BLOCK_SIZE || input.length > MAX_COMPRESSED_BLOCK_SIZE)
+    throw new RangeError('Invalid LZ4 block size');
+  // lz4_Block_format.md:114–126 requires a final literal sequence, five final literals
+  // (except a short all-literal block), and the last match at least12 bytes before its end.
+  // This is the format specification's strictness; ClickHouse's decoder is not retained
+  // here and may accept malformed end sequences that this decoder rejects.
+  const walk = (output?: Uint8Array): void => {
+    let source = 0, target = 0, sequences = 0, finalLiterals = -1, lastMatchStart = -1;
+    const length = (initial: number): number => {
+      let result = initial;
+      if (initial === 15) { let value: number; do { if (source >= input.length) throw new RangeError('Truncated LZ4 length'); value = input[source++]!; result += value; } while (value === 255); }
+      return result;
+    };
+    while (source < input.length) {
+      sequences++;
+      const token = input[source++]!, literals = length(token >> 4);
+      if (literals > input.length - source || literals > size - target) throw new RangeError('Invalid LZ4 literal range');
+      output?.set(input.subarray(source, source + literals), target); source += literals; target += literals;
+      if (source === input.length) { finalLiterals = literals; break; }
+      if (source + 2 > input.length) throw new RangeError('Truncated LZ4 offset');
+      const offset = input[source++]! | (input[source++]! << 8); if (!offset || offset > target) throw new RangeError('Invalid LZ4 offset');
+      const match = length(token & 15) + 4; if (match > size - target) throw new RangeError('Invalid LZ4 match range');
+      lastMatchStart = target;
+      if (output) for (let i = 0; i < match; i++) { output[target] = output[target - offset]!; target++; }
+      else target += match;
+    }
+    if (finalLiterals < 0 || (size >= 5 && finalLiterals < 5) || (size < 5 && sequences !== 1) || (lastMatchStart >= 0 && size - lastMatchStart < 12))
+      throw new RangeError('Invalid LZ4 end sequence');
+    if (target !== size) throw new RangeError('LZ4 decoded size differs');
+  };
+  walk();
+  const output = new Uint8Array(size); walk(output); return output;
+}
+/** A valid literal-only LZ4 block. */
+export function encodeLz4Literals(input: Uint8Array): Uint8Array {
+  if (input.length > MAX_DECOMPRESSED_BLOCK_SIZE) throw new RangeError('LZ4 block is too large');
+  const extension = input.length >= 15 ? Math.floor((input.length - 15) / 255) + 1 : 0;
+  const output = new Uint8Array(1 + extension + input.length); let at = 0;
+  output[at++] = Math.min(input.length, 15) << 4;
+  if (extension) { let left = input.length - 15; while (left >= 255) { output[at++] = 255; left -= 255; } output[at++] = left; }
+  output.set(input, at); return output;
+}
+
+export class ClickHouseCompressionError extends Error {
+  constructor(readonly code: number, message: string) { super(message); this.name = 'ClickHouseCompressionError'; }
+}
+/** ClickHouse compressed frame, including its sixteen-byte checksum.
+ * ICompressionCodec.cpp:62–69,102–116 (SHA2562cfbdbd7…) writes the9-byte header.
+ * CompressedReadBufferBase.cpp:193–194,255–256 (SHA2566f5af3a8…) reads low/high
+ * little-endian checksum words and verifies the complete header+payload before decompressing.
+ */
+export function encodeFrame(input: Uint8Array, method: 0x82 | 0x02 = 0x82): Uint8Array {
+  if (!input.length || input.length > MAX_DECOMPRESSED_BLOCK_SIZE) throw new ClickHouseCompressionError(39, 'Invalid decompressed block size');
+  if (method !== 0x82 && method !== 0x02) throw new ClickHouseCompressionError(89, 'Unknown compression method');
+  const bodyLength = method === 0x82 ? 1 + (input.length >= 15 ? Math.floor((input.length - 15) / 255) + 1 : 0) + input.length : input.length;
+  if (bodyLength + 9 > MAX_COMPRESSED_BLOCK_SIZE) throw new ClickHouseCompressionError(39, 'Compressed block is too large');
+  const body = method === 0x82 ? encodeLz4Literals(input) : input;
+  const output = new Uint8Array(25 + body.length), view = new DataView(output.buffer);
+  output[16] = method; view.setUint32(17, 9 + body.length, true); view.setUint32(21, input.length, true); output.set(body, 25);
+  const [low, high] = cityHash128(output.subarray(16)); view.setBigUint64(0, low, true); view.setBigUint64(8, high, true);
+  return output;
+}
+/** Decode one frame; undefined means the input has not yet supplied the whole frame.
+ * Sizes are validated from the header before waiting for its body, and the checksum
+ * precedes decompression. bytesRead allows the caller to retain subsequent frames.
+ */
+export function decodeFrame(input: Uint8Array): { data: Uint8Array; bytesRead: number } | undefined {
+  if (input.length < 25) return undefined;
+  const view = new DataView(input.buffer, input.byteOffset, input.byteLength), method = input[16]!;
+  // CompressedReadBufferBase.cpp:134–138 selects/refuses the codec before header sizes.
+  if (method !== 0x82 && method !== 0x02) throw new ClickHouseCompressionError(89, 'Unknown compression method');
+  const compressed = view.getUint32(17, true), size = view.getUint32(21, true);
+  if (compressed > MAX_COMPRESSED_BLOCK_SIZE || size > MAX_DECOMPRESSED_BLOCK_SIZE) throw new ClickHouseCompressionError(39, 'Compressed block size exceeds the vendor limit');
+  if (compressed < 9 || !size) throw new ClickHouseCompressionError(246, 'Corrupt compressed block header');
+  const bytesRead = 16 + compressed; if (input.length < bytesRead) return undefined;
+  const frame = input.subarray(16, bytesRead), [low, high] = cityHash128(frame);
+  if (low !== view.getBigUint64(0, true) || high !== view.getBigUint64(8, true)) throw new ClickHouseCompressionError(40, 'Checksum does not match');
+  const body = frame.subarray(9);
+  if (method === 0x02) {
+    if (body.length !== size) throw new ClickHouseCompressionError(246, 'Uncompressed block size differs');
+    // Own the returned bytes, as the LZ4 branch does.
+    return { data: body.slice(), bytesRead };
+  }
+  try { return { data: decodeLz4Block(body, size), bytesRead }; }
+  catch (error) { throw new ClickHouseCompressionError(246, error instanceof Error ? error.message : 'Invalid LZ4 block'); }
+}

package/src/clickhouse/index.ts

@@ -4,3 +4,5 @@
 export * from './values.ts';
 export * from './types.ts';
 export * from './sql.ts';
+
+export * from './compression.ts';