@volter/world-core 3.0.155 → 3.0.156

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/world-core",
-  "version": "3.0.155",
+  "version": "3.0.156",
   "description": "The kernel of Volter World: one log per twin, branches as pointers, checkpoints, the fold that keeps a twin current, the head that performs a write against the vendor, references, and the git plane. A twin package builds on it; the runtime serves it.",
   "keywords": [
     "twin",
@@ -128,9 +128,25 @@
       "types": "./dist/src/live/index.d.ts",
       "default": "./dist/src/live/index.js"
     },
+    "./mongodb": {
+      "types": "./dist/src/mongodb/index.d.ts",
+      "default": "./dist/src/mongodb/index.js"
+    },
     "./clickhouse": {
       "types": "./dist/src/clickhouse/index.d.ts",
       "default": "./dist/src/clickhouse/index.js"
+    },
+    "./runtime-state": {
+      "types": "./dist/src/runtime-state.d.ts",
+      "default": "./dist/src/runtime-state.js"
+    },
+    "./browser-state": {
+      "types": "./dist/src/browser-state.d.ts",
+      "default": "./dist/src/browser-state.js"
+    },
+    "./managed-original-inputs": {
+      "types": "./dist/src/managed-original-inputs.d.ts",
+      "default": "./dist/src/managed-original-inputs.js"
     }
   },
   "scripts": {
@@ -156,5 +172,8 @@
   },
   "engines": {
     "node": ">=22.3"
+  },
+  "optionalDependencies": {
+    "@xmldom/xmldom": "0.9.12"
   }
 }

package/src/actions.ts

@@ -12,7 +12,7 @@ import { parseExactJson } from './exact-json.ts';
 //
 // Projection = observed mirror, then apply each `set` transaction in order,
 // skipping any transaction that was reverted or confirmed.
-import { withHistoryLock } from './ancestry.ts';
+import { canonicalStatePath, withHistoryLock } from './ancestry.ts';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { randomUUID } from 'node:crypto';
 import { dirname, join } from 'node:path';
@@ -20,10 +20,10 @@ import { canonicalJson, hashFieldValue, subjectKey } from './hash.ts';
 import type { SubjectFields } from './hash.ts';
 import { appendDurable, eventsLockPath, ownerStoreRoots, projectionLockPath, twinLog, withFileLock, withoutIdentityClaim, worldPaths } from './storage.ts';
 import { landAsPlaceholder, placeholderPullActive } from './placeholder-remote.ts';
-import { aliasesFrom, dropCheckpoint, landedCopy, landedIds, parentEntries, readTree, treeStamp, type Receipt } from './log.ts';
+import { aliasesFrom, appendToBranchLog, dropCheckpoint, landedCopy, landedIds, parentEntries, readTree, treeStamp, type Receipt } from './log.ts';
 import { WorldServiceEventSchema } from './schemas.ts';
 import { inVendorMove, refuseReadOnlyWrite } from './request-scope.ts';
-import { getActiveWorldStore } from './world-store.ts';
+import { getActiveWorldStore, storageLockScope, type WorldLockOptions } from './world-store.ts';
 import { currentTraceparent, traceparentForDelivery } from './trace-context.ts';
 import type { WorldServiceEvent } from './types.ts';
 import type { TwinResource } from './serve.ts';
@@ -124,8 +124,11 @@ export class TwinActionPreconditionError extends Error {
 function actionsPath(service: string, root?: string): string {
   return join(dirname(worldPaths(service, root).events), 'actions.jsonl');
 }
-const actionsLock = (service: string, root?: string): string => `${actionsPath(service, root)}.lock`;
-const projectionLock = (service: string, root?: string): string => projectionLockPath(worldPaths(service, root));
+// Canonical, as the history lock's path is (ancestry.ts withHistoryLock), so a take inside it is seen as nested in the same
+// directory whatever the path it was asked by (/var or /private/var). The file is the same one, so a take by another
+// process through the other spelling still excludes this one.
+const actionsLock = (service: string, root?: string): string => join(canonicalStatePath(worldPaths(service, root).dir), 'actions.jsonl.lock');
+const projectionLock = (service: string, root?: string): string => { const paths = worldPaths(service, root); return projectionLockPath({ ...paths, dir: canonicalStatePath(paths.dir) }); };
 
 /** Every appended action carries a correlationId — generate one when the caller
  *  hasn't supplied it, so downstream joins (push ledger, logs) always have an id
@@ -186,20 +189,33 @@ function actionIds(service: string, root?: string): ReadonlySet<string> {
 
 /** A service's projection lock, under its history lock: one directory's locks are taken history, projection,
  *  actions, events, in that order (architecture, the history lock). */
-function withProjection<T>(service: string, root: string | undefined, fn: () => T): T {
-  return withHistoryLock(worldPaths(service, root).dir, () => withFileLock(projectionLock(service, root), fn));
+function withProjection<T>(service: string, root: string | undefined, fn: () => T, options: WorldLockOptions = {}): T {
+  const scope = storageLockScope(options);
+  return withHistoryLock(worldPaths(service, root).dir, () => withFileLock(projectionLock(service, root), fn, scope), scope);
 }
 
 function appendActionRaw(action: TwinAction, root?: string): void {
-  const path = actionsPath(action.service, root);
-  getActiveWorldStore().mkdir(dirname(path));
+  appendActionsRaw([action], root);
+}
+
+/** One service's actions appended as ONE store append, in order: the same lines `appendActionRaw` writes one at a time. */
+function appendActionsRaw(actions: readonly TwinAction[], root?: string): void {
+  if (!actions.length) return;
+  const service = actions[0]!.service;
+  const path = actionsPath(service, root);
+  // No mkdir: every append is made holding the projection lock (withProjection), whose file is in this same directory
+  // and was made there by the take (S2 R17 review).
   const memos = idMemoSlot(path);
   const held = memos.get(path);
   const current = held !== undefined && held.key === logKey(path);
-  withHistoryLock(worldPaths(action.service, root).dir, () => appendDurable(path, `${JSON.stringify(action)}\n`));
-  if (current) { held.ids.add(action.id); held.key = logKey(path); } else memos.delete(path);
-  twinLog('action.append', { service: action.service, id: action.id, op: action.op, correlationId: action.correlationId });
-  appendObserver?.(action, root);
+  const data = actions.map((action) => `${JSON.stringify(action)}\n`).join('');
+  // the log is the branch log a read's tree is folded from: the append extends that memo too (log.ts appendToBranchLog)
+  const after = withHistoryLock(worldPaths(service, root).dir, () => appendToBranchLog(service, root, data, () => appendDurable(path, data)), { operation: 'action append' });
+  if (current) { for (const action of actions) held.ids.add(action.id); held.key = after ?? logKey(path); } else memos.delete(path);
+  for (const action of actions) {
+    twinLog('action.append', { service: action.service, id: action.id, op: action.op, correlationId: action.correlationId });
+    appendObserver?.(action, root);
+  }
 }
 
 /** ONE observer of appended entries (state-system.ts: the runtime's `deploy: auto` hook). Called
@@ -264,7 +280,7 @@ export function appendAction(action: TwinAction, root?: string): TwinAction {
     if (placeholder) return placeholder;
     appendActionRaw(stamped, root);
     return stamped;
-  }));
+  }, { operation: 'action append' }), { operation: 'action append' });
 }
 
 /** Append `action` only if no exact action with the same id already exists — the whole
@@ -286,7 +302,7 @@ export function appendActionIfAbsent(action: TwinAction, root?: string): { actio
     if (placeholder) return { action: placeholder, appended: true, placeholder: true };
     appendActionRaw(stamped, root);
     return { action: stamped, appended: true };
-  }));
+  }, { operation: 'action append' }), { operation: 'action append' });
 }
 
 /**
@@ -314,12 +330,53 @@ export function appendActionOccurrence(base: TwinAction, root?: string): { actio
     if (placeholder) return { action: placeholder, appended: true, placeholder: true };
     appendActionRaw(stamped, root);
     return { action: stamped, appended: true };
-  }));
+  }, { operation: 'action append' }), { operation: 'action append' });
+}
+
+/**
+ * Append several writes of ONE service as occurrences, in order, under one projection + actions lock and as one store
+ * append: each lands with the id, ordinal and correlation `appendActionOccurrence` would give it called once per write
+ * in the same order (a write's ordinal counts the batch's earlier writes as taken). A Redis run's flush made one
+ * append per changed member and key, each taking the history, projection and actions locks and appending alone, one
+ * after another under the twin's single command queue (S2 R19: Twenty's queue scripts spent up to 2.1 s of a 2 s
+ * window in flush while 38 commands waited). A write carrying preconditions is not taken here: each is checked against
+ * the state the earlier ones made, which only the one-at-a-time path sees. While a placeholder pull is open a write
+ * lands as observed events, as `appendActionOccurrence`'s does.
+ */
+export function appendActionOccurrences(bases: readonly TwinAction[], root?: string): Array<{ action: TwinAction; appended: boolean; placeholder?: true }> {
+  if (!bases.length) return [];
+  const service = bases[0]!.service;
+  if (bases.some((base) => base.service !== service)) throw new Error(`appendActionOccurrences: one service per batch, got ${[...new Set(bases.map((base) => base.service))].join(', ')}`);
+  if (bases.some((base) => base.preconditions?.length)) throw new Error('appendActionOccurrences: a write with preconditions is appended alone');
+  refuseReadOnlyWrite(service);
+  return withProjection(service, root, () => withFileLock(actionsLock(service, root), () => {
+    const logged = actionIds(service, root);
+    const batch = new Set<string>();
+    const taken = { has: (id: string) => batch.has(id) || logged.has(id) };
+    const landed: Array<{ action: TwinAction; appended: boolean; placeholder?: true }> = [];
+    let unwritten: TwinAction[] = [];
+    for (const base of bases) {
+      const stamped = occurrenceOf(base, taken, root);
+      const placeholder = landIfPlaceholderPull(stamped, root);
+      if (placeholder) {
+        // no action row: its id is not taken, as it is not in the log the one-at-a-time path reads
+        // the lines before it are written first, so the log and the pull keep the writes' order
+        appendActionsRaw(unwritten, root); unwritten = [];
+        landed.push({ action: placeholder, appended: true, placeholder: true });
+        continue;
+      }
+      batch.add(stamped.id);
+      unwritten.push(stamped);
+      landed.push({ action: stamped, appended: true });
+    }
+    appendActionsRaw(unwritten, root);
+    return landed;
+  }, { operation: 'action append' }), { operation: 'action append' });
 }
 
 /** The occurrence rule, shared by both appenders that use it — the ordinal AND the merge base.
  *  Pure over (base, the log as it stands): the caller holds the lock. */
-function occurrenceOf(base: TwinAction, taken: ReadonlySet<string>, root?: string): TwinAction {
+function occurrenceOf(base: TwinAction, taken: Pick<ReadonlySet<string>, 'has'>, root?: string): TwinAction {
   {
     let ordinal = 0;
     while (taken.has(occurrenceId(base.id, ordinal))) ordinal += 1;
@@ -393,7 +450,7 @@ export function decideAndAppendAction<T>(
     if (placeholder) return { value: decision.value, action: placeholder, appended: true, placeholder: true };
     appendActionRaw(stamped, root);
     return { value: decision.value, action: stamped, appended: true };
-  }));
+  }, { operation: 'action decision' }), { operation: 'action decision' });
 }
 
 export function appendTransactionCommit(commit: TwinTransactionCommit, root?: string): TwinTransactionCommit {
@@ -565,9 +622,9 @@ export function confirmAction(opts: {
     withFileLock(eventsLockPath(paths), () => {
       const held = landedIds(parentEntries(opts.service, opts.root));
       for (const copy of copies) if (!held.has(copy.id)) appendDurable(paths.events, `${JSON.stringify(copy)}\n`);
-    });
+    }, { operation: 'action confirmation' });
     dropCheckpoint(opts.service, opts.root);
-  });
+  }, { operation: 'action confirmation' });
   const observedEventIds = copies.map((c) => c.id);
   return { observedEventId: observedEventIds.at(-1)!, observedEventIds };
 }
@@ -607,7 +664,7 @@ export function revertAction(opts: { service: string; actionId: string; occurred
     });
     appendActionRaw(revert, opts.root);
     return { status: 'reverted', revertId: revert.id };
-  }));
+  }, { operation: 'action revert' }), { operation: 'action revert' });
 }
 
 /** The LOCAL OVERLAY: pending actions (set, not reverted, not yet confirmed) — the divergence

package/src/ancestry.ts

@@ -4,7 +4,7 @@ import { createHash, randomUUID } from 'node:crypto';
 import { volterHome } from './volter-home.ts';
 import { basename, dirname, join, relative, resolve, sep } from 'node:path';
 import { stateDirName } from './storage.ts';
-import { getActiveWorldStore, type WorldStore } from './world-store.ts';
+import { getActiveWorldStore, storageLockScope, withStorageLockOperation, type WorldLockOptions, type WorldStore } from './world-store.ts';
 
 type Pin = { child: string; ownerPid: number; committed?: boolean };
 type Ancestry = { generation: string; children: Pin[] };
@@ -18,16 +18,17 @@ const inside = (base: string, path: string) => { const rel = relative(canonicalS
  * bookkeeping (generations, pins, branch pointers, removal receipts) and the not-being-removed check: every World
  * sharing the VOLTER_HOME waits on it, so no append, fold or caller's critical section runs inside it, and no state lock
  * is taken inside it (`withStateLock` takes its lock first). */
-export function withAncestryLock<T>(fn: () => T): T {
+export function withAncestryLock<T>(fn: () => T, options: WorldLockOptions = {}): T {
+  const scope = storageLockScope(options);
   const store = getActiveWorldStore();
   const active = entered ??= new WeakSet<WorldStore>();
   if (active.has(store)) return fn();
   const path = join(volterHome(), 'branch-ancestry.lock');
   return store.withLock(path, () => {
     active.add(store);
-    try { recoverRemovals(); return fn(); }
+    try { recoverRemovals(path); return fn(); }
     finally { active.delete(store); }
-  }, { retainLiveOwner: true });
+  }, { ...scope, retainLiveOwner: true });
 }
 
 /** The state locks this process holds, per store: taken again inside one, a lock is already ours. */
@@ -36,22 +37,33 @@ const heldBy = (store: WorldStore): Set<string> => { const map = heldLocks ??= n
 
 /** Run `fn` holding the cross-process lock at `lockPath`, a lock of the directory it sits in: taken first, then the
  *  coordinator only to check that the directory is not being removed. Re-entrant within a process. */
-export function withStateLock<T>(lockPath: string, fn: () => T): T {
+export function withStateLock<T>(lockPath: string, fn: () => T, options: WorldLockOptions = {}): T {
+  const scope = storageLockScope(options);
   const store = getActiveWorldStore();
   const held = heldBy(store);
   if (held.has(lockPath)) return fn();
   if (entered?.has(store)) throw new Error(`State lock ${lockPath} taken inside the ancestry coordinator: a state lock is taken before it, never inside it`);
+  // A lock taken inside a held lock of the SAME directory asks the coordinator nothing: the outer take checked that
+  // the directory is not being removed, and a removal recorded since waits on that outer lock before it deletes
+  // anything, so this operation finishes first, as a holder that checked before the receipt does. Asked again, each
+  // write took the machine-wide coordinator three times for one directory.
+  const directory = dirname(lockPath);
+  const nested = [...held].some((path) => dirname(path) === directory);
   return store.withLock(lockPath, () => {
-    withAncestryLock(() => assertNotBeingRemoved(dirname(lockPath)));
+    // The removal check reads the receipts without the coordinator, and takes it only where one is there (S2 R17;
+    // architecture, the history lock): this holder already has the directory's lock, and a removal writes its receipt
+    // under the coordinator before it waits on every lock inside the tree, so a receipt not yet visible here belongs to
+    // a removal that waits for this holder, which finishes first. A visible receipt is checked, recovered, as before.
+    if (!nested && removalRecorded()) withAncestryLock(() => assertNotBeingRemoved(directory), { operation: 'removal preflight' });
     held.add(lockPath);
     try { return fn(); } finally { held.delete(lockPath); }
-  });
+  }, scope);
 }
 
 /** A state directory's history lock: its appends, captures, observation batches and folds. A branch's is taken before
  *  its ancestors'. */
-export function withHistoryLock<T>(directory: string, fn: () => T): T {
-  return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn);
+export function withHistoryLock<T>(directory: string, fn: () => T, options: WorldLockOptions = {}): T {
+  return withStateLock(join(canonicalStatePath(directory), 'history.lock'), fn, storageLockScope(options));
 }
 
 /** A lock file a holder has recorded itself in (`{pid, hostname}`, written before the holder runs anything): a file
@@ -61,9 +73,10 @@ function isHeldLock(text: string | null): boolean {
 }
 
 /** Several sibling directories' history locks at once (a World's services, a changeset's targets), in one order. */
-export function withHistoryLocks<T>(directories: string[], fn: () => T): T {
+export function withHistoryLocks<T>(directories: string[], fn: () => T, options: WorldLockOptions = {}): T {
+  const scope = storageLockScope(options);
   const sorted = [...new Set(directories.map((d) => canonicalStatePath(d)))].sort();
-  const take = (i: number): T => (i === sorted.length ? fn() : withHistoryLock(sorted[i]!, () => take(i + 1)));
+  const take = (i: number): T => (i === sorted.length ? fn() : withHistoryLock(sorted[i]!, () => take(i + 1), scope));
   return take(0);
 }
 
@@ -105,7 +118,7 @@ export function stateGeneration(dir: string): string {
     let meta = metadata(dir);
     if (!meta) { meta = { generation: randomUUID(), children: [] }; store.writeAtomic(metaPath(dir), JSON.stringify(meta)); }
     return meta.generation;
-  });
+  }, { operation: 'state generation' });
 }
 
 /** Must be called in the same critical section as child pointer publication. A crash before
@@ -220,19 +233,28 @@ function releaseOutgoing(removal: Removal): void {
 
 /** A crash after deleting files cannot strand committed child pins. Receipts are replayed
  * before any subsequent ancestry publication, so path reuse cannot hide completed removal. */
-function recoverRemovals(): void {
+function recoverRemovals(lockPath: string): void {
   const store = getActiveWorldStore();
+  const receipts = store.list(removalDir()).filter(n => n.endsWith('.json'));
+  if (!receipts.length) { beingRemoved = []; return; }
   const pending: string[] = [];
-  for (const name of store.list(removalDir()).filter(n => n.endsWith('.json'))) {
-    const file = join(removalDir(), name);
-    const removal = JSON.parse(store.read(file)!) as Removal;
-    if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing)) throw new Error('Invalid branch removal receipt; retained');
-    if (!store.exists(removal.path)) { releaseOutgoing(removal); store.remove(file); }
-    else pending.push(removal.path);
-  }
+  withStorageLockOperation(lockPath, 'ancestry removal recovery', () => {
+    for (const name of receipts) {
+      const file = join(removalDir(), name);
+      const removal = JSON.parse(store.read(file)!) as Removal;
+      if (typeof removal.path !== 'string' || !Array.isArray(removal.outgoing)) throw new Error('Invalid branch removal receipt; retained');
+      if (!store.exists(removal.path)) { releaseOutgoing(removal); store.remove(file); }
+      else pending.push(removal.path);
+    }
+  });
   beingRemoved = pending;
 }
 
+/** Whether any removal receipt is recorded now: read without the coordinator (withStateLock). */
+function removalRecorded(): boolean {
+  return getActiveWorldStore().list(removalDir()).some((name) => name.endsWith('.json'));
+}
+
 /** State whose removal is recorded and not yet complete: its deletion runs outside the coordinator, so nothing may pin
  * it as a parent or write into it meanwhile. Read under the coordinator, where recoverRemovals has just refreshed it. */
 let beingRemoved: string[] = [];
@@ -260,18 +282,18 @@ export function withStateRemoval<T>(path: string, remove: () => T, check?: () =>
     store.writeAtomic(receipt, JSON.stringify(removal));
     beingRemoved = [...beingRemoved, removal.path];
     return { removal, receipt };
-  });
+  }, { operation: 'removal preflight' });
   // Each lock is waited on alone, taken and let go: a holder that checked before the receipt finishes, and a taker
   // after it refuses once it holds the lock (withStateLock), so the removal never holds one lock while waiting on
   // another and no lock order can deadlock it. A wait that fails leaves nothing deleted: the receipt is released.
   const held = heldBy(store);
   try {
-    for (const lock of locksUnder(path).filter((l) => !held.has(l))) store.withLock(lock, () => undefined);
+    for (const lock of locksUnder(path).filter((l) => !held.has(l))) store.withLock(lock, () => undefined, { operation: 'removal preflight' });
   } catch (error) {
     withAncestryLock(() => {
       if (store.exists(receipt)) store.remove(receipt);
       beingRemoved = beingRemoved.filter((p) => p !== removal.path);
-    });
+    }, { operation: 'ancestry removal recovery' });
     throw error;
   }
   const result = remove();
@@ -280,6 +302,6 @@ export function withStateRemoval<T>(path: string, remove: () => T, check?: () =>
     // Another process's recovery may have released it already; both steps are idempotent.
     releaseOutgoing(removal);
     if (store.exists(receipt)) store.remove(receipt);
-  });
+  }, { operation: 'ancestry removal recovery' });
   return result;
 }

package/src/changeset.ts

@@ -1042,7 +1042,7 @@ export type RebaseActionResult = {
 export type RebaseReport = { changeset: string; from: string; to: string | null; at: string; outcome: 'unchanged' | 'rebased' | 'conflict'; actions: RebaseActionResult[] };
 
 export function rebaseChangeset(changeset: Changeset, opts: { targets: RebaseTarget[]; at?: string }): { changeset: Changeset; report: RebaseReport } {
-  return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts));
+  return withHistoryLocks(opts.targets.map((t) => worldPaths(t.stateService, t.controlRoot).dir), () => rebaseChangesetLocked(changeset, opts), { operation: 'branch-pointer adjustment' });
 }
 function rebaseChangesetLocked(changeset: Changeset, opts: { targets: RebaseTarget[]; at?: string }): { changeset: Changeset; report: RebaseReport } {
   const at = opts.at ?? new Date().toISOString();

package/src/clickhouse/compression.ts

@@ -0,0 +1,172 @@
+// CityHashv1.0.2 arithmetic, ported from the vendor-carried Google source.
+// Source: ClickHouse/ClickHouse contrib/cityhash102/src/city.cc, SHA256
+// 4bcc8f0c09e3ef38210a0c568efc0550e4cb426719d8ad74a466f152296d0df7.
+// Copyright (c) 2011 Google, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+const k0 = 0xc3a5c85c97cb3127n, k1 = 0xb492b66fbe98f273n, k2 = 0x9ae16a3b2f90404fn, k3 = 0xc949d7c7509e6557n;
+type Pair = [bigint, bigint];
+const u = (value: bigint): bigint => BigInt.asUintN(64, value);
+const rotate = (value: bigint, shift: number): bigint => shift === 0 ? u(value) : u((u(value) >> BigInt(shift)) | (u(value) << BigInt(64 - shift)));
+const mix = (value: bigint): bigint => u(value) ^ (u(value) >> 47n);
+function hash16(first: bigint, second: bigint): bigint {
+  const mul = 0x9ddfea08eb382d69n;
+  let a = u((u(first) ^ u(second)) * mul); a ^= a >> 47n;
+  let b = u((u(second) ^ a) * mul); b ^= b >> 47n;
+  return u(b * mul);
+}
+/** The CityHash 1.0.2 checksum used by ClickHouse native compression frames. */
+export function cityHash128(bytes: Uint8Array): Pair {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const fetch = (at: number): bigint => view.getBigUint64(at, true);
+  const short = (at: number, length: number): bigint => {
+    if (length > 8) { const a = fetch(at), b = fetch(at + length - 8); return hash16(a, rotate(u(b + BigInt(length)), length)) ^ b; }
+    if (length >= 4) return hash16(BigInt(length) + (BigInt(view.getUint32(at, true)) << 3n), BigInt(view.getUint32(at + length - 4, true)));
+    if (length) { const a = bytes[at]!, b = bytes[at + (length >> 1)]!, c = bytes[at + length - 1]!; return u(mix(u(BigInt(a + (b << 8)) * k2) ^ u(BigInt(length + (c << 2)) * k3)) * k2); }
+    return k2;
+  };
+  const weak = (at: number, first: bigint, second: bigint): Pair => {
+    const w = fetch(at), x = fetch(at + 8), y = fetch(at + 16), z = fetch(at + 24);
+    let a = u(first + w), b = rotate(u(second + a + z), 21); const c = a;
+    a = u(a + x + y); b = u(b + rotate(a, 44)); return [u(a + z), u(b + c)];
+  };
+  const murmur = (at: number, length: number, seed: Pair): Pair => {
+    let [a, b] = seed, c = 0n, d = 0n, left = length - 16;
+    if (left <= 0) { a = u(mix(u(a * k1)) * k1); c = u(b * k1 + short(at, length)); d = mix(u(a + (length >= 8 ? fetch(at) : c))); }
+    else {
+      c = hash16(u(fetch(at + length - 8) + k1), a); d = hash16(u(b + BigInt(length)), u(c + fetch(at + length - 16))); a = u(a + d);
+      do { a ^= u(mix(u(fetch(at) * k1)) * k1); a = u(a * k1); b ^= a; c ^= u(mix(u(fetch(at + 8) * k1)) * k1); c = u(c * k1); d ^= c; at += 16; left -= 16; } while (left > 0);
+    }
+    a = hash16(a, c); b = hash16(d, b); return [a ^ b, hash16(b, a)];
+  };
+  const seeded = (at: number, length: number, seed: Pair): Pair => {
+    if (length < 128) return murmur(at, length, seed);
+    let [x, y] = seed, z = u(BigInt(length) * k1);
+    let v: Pair = [u(rotate(y ^ k1, 49) * k1 + fetch(at)), 0n]; v[1] = u(rotate(v[0], 42) * k1 + fetch(at + 8));
+    let w: Pair = [u(rotate(u(y + z), 35) * k1 + x), u(rotate(u(x + fetch(at + 88)), 53) * k1)];
+    const round = (): void => {
+      x = u(rotate(u(x + y + v[0] + fetch(at + 16)), 37) * k1); y = u(rotate(u(y + v[1] + fetch(at + 48)), 42) * k1);
+      x ^= w[1]; y ^= v[0]; z = rotate(z ^ w[0], 33);
+      v = weak(at, u(v[1] * k1), u(x + w[0])); w = weak(at + 32, u(z + w[1]), y);
+      [z, x] = [x, z]; at += 64;
+    };
+    do { round(); round(); length -= 128; } while (length >= 128);
+    y = u(y + rotate(w[0], 37) * k0 + z); x = u(x + rotate(u(v[0] + z), 49) * k0);
+    for (let done = 0; done < length;) { done += 32; y = u(rotate(u(y - x), 42) * k0 + v[1]); w[0] = u(w[0] + fetch(at + length - done + 16)); x = u(rotate(x, 49) * k0 + w[0]); w[0] = u(w[0] + v[0]); v = weak(at + length - done, v[0], v[1]); }
+    x = hash16(x, v[0]); y = hash16(y, w[0]); return [u(hash16(u(x + v[1]), w[1]) + y), hash16(u(x + w[1]), u(y + v[1]))];
+  };
+  const length = bytes.length;
+  if (length >= 16) return seeded(16, length - 16, [fetch(0) ^ k3, fetch(8)]);
+  if (length >= 8) return seeded(0, 0, [fetch(0) ^ u(BigInt(length) * k0), fetch(length - 8) ^ k1]);
+  return seeded(0, length, [k0, k1]);
+}
+
+// ClickHouse src/Compression/CompressionInfo.h:7,14 (SHA25614b39ae2…): both limits are1GiB.
+export const MAX_COMPRESSED_BLOCK_SIZE = 0x40000000;
+export const MAX_DECOMPRESSED_BLOCK_SIZE = 0x40000000;
+
+/** LZ4 block sequences. Validate all lengths before allocating the output buffer. */
+export function decodeLz4Block(input: Uint8Array, size: number): Uint8Array {
+  if (!Number.isSafeInteger(size) || size < 0 || size > MAX_DECOMPRESSED_BLOCK_SIZE || input.length > MAX_COMPRESSED_BLOCK_SIZE)
+    throw new RangeError('Invalid LZ4 block size');
+  // lz4_Block_format.md:114–126 requires a final literal sequence, five final literals
+  // (except a short all-literal block), and the last match at least12 bytes before its end.
+  // This is the format specification's strictness; ClickHouse's decoder is not retained
+  // here and may accept malformed end sequences that this decoder rejects.
+  const walk = (output?: Uint8Array): void => {
+    let source = 0, target = 0, sequences = 0, finalLiterals = -1, lastMatchStart = -1;
+    const length = (initial: number): number => {
+      let result = initial;
+      if (initial === 15) { let value: number; do { if (source >= input.length) throw new RangeError('Truncated LZ4 length'); value = input[source++]!; result += value; } while (value === 255); }
+      return result;
+    };
+    while (source < input.length) {
+      sequences++;
+      const token = input[source++]!, literals = length(token >> 4);
+      if (literals > input.length - source || literals > size - target) throw new RangeError('Invalid LZ4 literal range');
+      output?.set(input.subarray(source, source + literals), target); source += literals; target += literals;
+      if (source === input.length) { finalLiterals = literals; break; }
+      if (source + 2 > input.length) throw new RangeError('Truncated LZ4 offset');
+      const offset = input[source++]! | (input[source++]! << 8); if (!offset || offset > target) throw new RangeError('Invalid LZ4 offset');
+      const match = length(token & 15) + 4; if (match > size - target) throw new RangeError('Invalid LZ4 match range');
+      lastMatchStart = target;
+      if (output) for (let i = 0; i < match; i++) { output[target] = output[target - offset]!; target++; }
+      else target += match;
+    }
+    if (finalLiterals < 0 || (size >= 5 && finalLiterals < 5) || (size < 5 && sequences !== 1) || (lastMatchStart >= 0 && size - lastMatchStart < 12))
+      throw new RangeError('Invalid LZ4 end sequence');
+    if (target !== size) throw new RangeError('LZ4 decoded size differs');
+  };
+  walk();
+  const output = new Uint8Array(size); walk(output); return output;
+}
+/** A valid literal-only LZ4 block. */
+export function encodeLz4Literals(input: Uint8Array): Uint8Array {
+  if (input.length > MAX_DECOMPRESSED_BLOCK_SIZE) throw new RangeError('LZ4 block is too large');
+  const extension = input.length >= 15 ? Math.floor((input.length - 15) / 255) + 1 : 0;
+  const output = new Uint8Array(1 + extension + input.length); let at = 0;
+  output[at++] = Math.min(input.length, 15) << 4;
+  if (extension) { let left = input.length - 15; while (left >= 255) { output[at++] = 255; left -= 255; } output[at++] = left; }
+  output.set(input, at); return output;
+}
+
+export class ClickHouseCompressionError extends Error {
+  constructor(readonly code: number, message: string) { super(message); this.name = 'ClickHouseCompressionError'; }
+}
+/** ClickHouse compressed frame, including its sixteen-byte checksum.
+ * ICompressionCodec.cpp:62–69,102–116 (SHA2562cfbdbd7…) writes the9-byte header.
+ * CompressedReadBufferBase.cpp:193–194,255–256 (SHA2566f5af3a8…) reads low/high
+ * little-endian checksum words and verifies the complete header+payload before decompressing.
+ */
+export function encodeFrame(input: Uint8Array, method: 0x82 | 0x02 = 0x82): Uint8Array {
+  if (!input.length || input.length > MAX_DECOMPRESSED_BLOCK_SIZE) throw new ClickHouseCompressionError(39, 'Invalid decompressed block size');
+  if (method !== 0x82 && method !== 0x02) throw new ClickHouseCompressionError(89, 'Unknown compression method');
+  const bodyLength = method === 0x82 ? 1 + (input.length >= 15 ? Math.floor((input.length - 15) / 255) + 1 : 0) + input.length : input.length;
+  if (bodyLength + 9 > MAX_COMPRESSED_BLOCK_SIZE) throw new ClickHouseCompressionError(39, 'Compressed block is too large');
+  const body = method === 0x82 ? encodeLz4Literals(input) : input;
+  const output = new Uint8Array(25 + body.length), view = new DataView(output.buffer);
+  output[16] = method; view.setUint32(17, 9 + body.length, true); view.setUint32(21, input.length, true); output.set(body, 25);
+  const [low, high] = cityHash128(output.subarray(16)); view.setBigUint64(0, low, true); view.setBigUint64(8, high, true);
+  return output;
+}
+/** Decode one frame; undefined means the input has not yet supplied the whole frame.
+ * Sizes are validated from the header before waiting for its body, and the checksum
+ * precedes decompression. bytesRead allows the caller to retain subsequent frames.
+ */
+export function decodeFrame(input: Uint8Array): { data: Uint8Array; bytesRead: number } | undefined {
+  if (input.length < 25) return undefined;
+  const view = new DataView(input.buffer, input.byteOffset, input.byteLength), method = input[16]!;
+  // CompressedReadBufferBase.cpp:134–138 selects/refuses the codec before header sizes.
+  if (method !== 0x82 && method !== 0x02) throw new ClickHouseCompressionError(89, 'Unknown compression method');
+  const compressed = view.getUint32(17, true), size = view.getUint32(21, true);
+  if (compressed > MAX_COMPRESSED_BLOCK_SIZE || size > MAX_DECOMPRESSED_BLOCK_SIZE) throw new ClickHouseCompressionError(39, 'Compressed block size exceeds the vendor limit');
+  if (compressed < 9 || !size) throw new ClickHouseCompressionError(246, 'Corrupt compressed block header');
+  const bytesRead = 16 + compressed; if (input.length < bytesRead) return undefined;
+  const frame = input.subarray(16, bytesRead), [low, high] = cityHash128(frame);
+  if (low !== view.getBigUint64(0, true) || high !== view.getBigUint64(8, true)) throw new ClickHouseCompressionError(40, 'Checksum does not match');
+  const body = frame.subarray(9);
+  if (method === 0x02) {
+    if (body.length !== size) throw new ClickHouseCompressionError(246, 'Uncompressed block size differs');
+    // Own the returned bytes, as the LZ4 branch does.
+    return { data: body.slice(), bytesRead };
+  }
+  try { return { data: decodeLz4Block(body, size), bytesRead }; }
+  catch (error) { throw new ClickHouseCompressionError(246, error instanceof Error ? error.message : 'Invalid LZ4 block'); }
+}

package/src/clickhouse/index.ts

@@ -4,3 +4,5 @@
 export * from './values.ts';
 export * from './types.ts';
 export * from './sql.ts';
+
+export * from './compression.ts';