@statewalker/webrun-content-store 0.2.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-present StateWalker Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,68 @@
+# @statewalker/webrun-content-store
+
+Domain-neutral, content-addressed large-object store with content-defined chunking and deduplication.
+
+## Overview
+
+`content-store` persists large byte streams as deduplicated, content-defined chunks over the `@statewalker/webrun-storage` `BlobStore` seam. It is **algorithm-agnostic**: the whole-object and per-chunk identities are computed by an **injected `hashContent`**, and every id is treated as an opaque string (it carries whatever prefix the hasher produces). The content-defined chunker lives in-package; content below a threshold is stored as a single direct blob.
+
+It is shared infrastructure for both axes: it backs Axis A's "ask which chunks exist → send only the missing → assemble" transfer flow (via `@statewalker/webrun-content-transfer`) and Axis B's LFS-pointer-to-object indirection (`@statewalker/vcs-transport-lfs` / `-xet`). It knows nothing of git, LFS, SHA-256, commits, or sync — those live in the skins above it.
+
+## Installation
+
+```bash
+pnpm add @statewalker/webrun-content-store
+```
+
+## Quick Start
+
+```typescript
+import { createHash } from "node:crypto";
+import { memBlobStore } from "@statewalker/webrun-storage";
+import { createContentStore } from "@statewalker/webrun-content-store";
+
+async function sha256(bytes: AsyncIterable<Uint8Array>): Promise<string> {
+  const h = createHash("sha256");
+  for await (const chunk of bytes) h.update(chunk);
+  return `sha256:${h.digest("hex")}`;
+}
+
+const store = createContentStore(
+  { chunks: memBlobStore(), manifests: memBlobStore(), hashContent: sha256 },
+  { chunkThreshold: 64, cdc: { min: 64, avg: 256, max: 1024 } },
+);
+
+async function* streamOf(bytes: Uint8Array) { yield bytes; }
+
+// Object level: chunk + hash in one bounded-memory pass.
+const d = await store.put(streamOf(payload)); // ObjectDescriptor { id, size, chunks }
+const whole = store.read(d.id);               // reassembled stream
+const slice = store.read(d.id, { offset: 3000, length: 2500 }); // arbitrary range across chunks
+
+// Chunk level: negotiate and move only what a peer is missing.
+const missing = await store.hasChunks(d.chunks.map((c) => c.id)); // ids NOT present
+await store.gc([d.id]); // sweep everything not reachable from the live roots
+```
+
+## API
+
+**Object level**
+- `put(content): Promise<ObjectDescriptor>` — chunk + hash in a bounded-memory pass, store chunks + manifest.
+- `read(id, range?)` — stream the object's bytes, optionally sliced by `{ offset?, length? }`; empty stream if absent.
+- `has(id)`, `getManifest(id)`, `remove(id)`.
+
+**Chunk level** (transfer / dedup)
+- `hasChunks(ids): Promise<ChunkId[]>` — return exactly the ids **not** present.
+- `putChunk(id, bytes)`, `getChunk(id)`.
+
+**Maintenance**
+- `gc(liveRoots): Promise<{ removedObjects, removedChunks }>` — mark-sweep from caller-supplied roots.
+
+**Factory:** `createContentStore(deps, opts?)`, where `deps` is `{ chunks: BlobStore, manifests: BlobStore, hashContent }` and `opts` is `{ chunkThreshold?, cdc? }`. Types: `ContentStore`, `ObjectDescriptor`, `ChunkRef`, `ObjectId`, `ChunkId`, `CdcParams`.
+
+## Notes
+
+- **Injected hash, opaque ids.** The contract sketch spoke of intrinsic BLAKE3; the shipped code takes an injected `hashContent` and treats every id as opaque — the store commits to no algorithm, and object/chunk ids simply carry the hasher's own prefix.
+- **CDC + direct-blob threshold.** Content-defined chunking (so dedup survives inserts/shifts) is the default; content strictly below `chunkThreshold` is stored as a single blob with no chunking overhead. `hasChunks`/`getChunk`/`putChunk` let `content-transfer` move only missing chunks.
+- **Immutable + external liveness.** Chunks and objects are immutable and content-addressed; there are no refcounts — `gc(liveRoots)` reaches objects→chunks from caller roots and sweeps the rest.
+- Built red/green TDD.

package/dist/chunker.d.ts

@@ -0,0 +1,8 @@
+import type { ByteStream, CdcParams } from "./types.js";
+/**
+ * Split `source` into content-defined chunks. Content strictly smaller than
+ * `threshold` is emitted as a single direct blob (no CDC). Bounded memory: at
+ * most one pending chunk (plus one incoming array) is buffered at a time.
+ */
+export declare function chunkContent(source: ByteStream, cdc: CdcParams, threshold: number): AsyncIterable<Uint8Array>;
+//# sourceMappingURL=chunker.d.ts.map

package/dist/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AA6ExD;;;;GAIG;AACH,wBAAuB,YAAY,CACjC,MAAM,EAAE,UAAU,EAClB,GAAG,EAAE,SAAS,EACd,SAAS,EAAE,MAAM,GAChB,aAAa,CAAC,UAAU,CAAC,CAiD3B"}

package/dist/content-store.d.ts

@@ -0,0 +1,3 @@
+import type { ContentStore, ContentStoreDeps, ContentStoreOptions } from "./types.js";
+export declare function createContentStore(deps: ContentStoreDeps, opts?: ContentStoreOptions): ContentStore;
+//# sourceMappingURL=content-store.d.ts.map

package/dist/content-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-store.d.ts","sourceRoot":"","sources":["../src/content-store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAIV,YAAY,EACZ,gBAAgB,EAChB,mBAAmB,EAGpB,MAAM,YAAY,CAAC;AAuBpB,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,gBAAgB,EACtB,IAAI,CAAC,EAAE,mBAAmB,GACzB,YAAY,CAkGd"}

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * @statewalker/webrun-content-store
+ *
+ * Domain-neutral, content-addressed large-object store. It stores large byte
+ * streams as deduplicated, content-defined chunks over the `@statewalker/webrun-storage`
+ * BlobStore seam. It is algorithm-agnostic — object and chunk identities are
+ * computed by an injected `hashContent`, and every id is an opaque string. It
+ * knows nothing of git, LFS, SHA-256, commits, or sync.
+ */
+export { createContentStore } from "./content-store.js";
+export type { ByteStream, CdcParams, ChunkId, ChunkRef, ContentStore, ContentStoreDeps, ContentStoreOptions, ObjectDescriptor, ObjectId, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EACV,UAAU,EACV,SAAS,EACT,OAAO,EACP,QAAQ,EACR,YAAY,EACZ,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,QAAQ,GACT,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,238 @@
+//#region src/chunker.ts
+/**
+* Minimal Gear-hash FastCDC content-defined chunker.
+*
+* A rolling Gear fingerprint (`fp = (fp << 1) + GEAR[byte]`) restarts at every
+* chunk boundary; a cut is taken when the low bits of `fp` are zero. Normalized
+* chunking (Xia et al., FastCDC) uses a stricter mask before the average size
+* and a looser one after, tightening the size distribution. Because the cut for
+* a chunk depends only on the bytes since its own start, boundaries re-sync
+* shortly after an edit — so a mid-file insert leaves surrounding chunks intact.
+*/
+const GEAR = buildGearTable();
+/** 256 deterministic 32-bit gear values (xorshift32 from a fixed seed). */
+function buildGearTable() {
+	const g = /* @__PURE__ */ new Uint32Array(256);
+	let x = 2654435769;
+	for (let i = 0; i < 256; i++) {
+		x ^= x << 13;
+		x ^= x >>> 17;
+		x ^= x << 5;
+		x >>>= 0;
+		g[i] = x;
+	}
+	return g;
+}
+function masksFor(cdc) {
+	const bits = Math.round(Math.log2(cdc.avg));
+	const maskS = (1 << Math.min(bits + 2, 31)) - 1;
+	const maskL = (1 << Math.max(bits - 2, 1)) - 1;
+	return {
+		min: cdc.min,
+		avg: cdc.avg,
+		max: cdc.max,
+		maskS,
+		maskL
+	};
+}
+/**
+* Find the cut index (exclusive end of the first chunk) within `buf`, whose
+* chunk starts at index 0. Returns `-1` when more bytes are needed before a cut
+* can be decided (only when `!final`).
+*/
+function findCut(buf, m, final) {
+	const n = buf.length;
+	if (n <= m.min) return final ? n : -1;
+	const limit = Math.min(n, m.max);
+	let fp = 0;
+	let i = m.min;
+	const avgStop = Math.min(m.avg, limit);
+	for (; i < avgStop; i++) {
+		fp = (fp << 1) + GEAR[buf[i]] >>> 0;
+		if ((fp & m.maskS) === 0) return i;
+	}
+	for (; i < limit; i++) {
+		fp = (fp << 1) + GEAR[buf[i]] >>> 0;
+		if ((fp & m.maskL) === 0) return i;
+	}
+	if (limit === m.max) return m.max;
+	return final ? n : -1;
+}
+function concat(a, b) {
+	const out = new Uint8Array(a.length + b.length);
+	out.set(a, 0);
+	out.set(b, a.length);
+	return out;
+}
+/**
+* Split `source` into content-defined chunks. Content strictly smaller than
+* `threshold` is emitted as a single direct blob (no CDC). Bounded memory: at
+* most one pending chunk (plus one incoming array) is buffered at a time.
+*/
+async function* chunkContent(source, cdc, threshold) {
+	const iter = source[Symbol.asyncIterator]();
+	const prefix = [];
+	let prefixLen = 0;
+	let done = false;
+	while (prefixLen < threshold) {
+		const next = await iter.next();
+		if (next.done) {
+			done = true;
+			break;
+		}
+		prefix.push(next.value);
+		prefixLen += next.value.length;
+	}
+	if (done && prefixLen < threshold) {
+		let blob = /* @__PURE__ */ new Uint8Array(0);
+		for (const p of prefix) blob = concat(blob, p);
+		yield blob;
+		return;
+	}
+	const m = masksFor(cdc);
+	let buf = /* @__PURE__ */ new Uint8Array(0);
+	const feed = async function* () {
+		for (const p of prefix) yield p;
+		if (!done) for (let n = await iter.next(); !n.done; n = await iter.next()) yield n.value;
+	};
+	for await (const incoming of feed()) {
+		buf = concat(buf, incoming);
+		for (let cut = findCut(buf, m, false); cut >= 0; cut = findCut(buf, m, false)) {
+			yield buf.subarray(0, cut);
+			buf = buf.subarray(cut);
+		}
+	}
+	while (buf.length > 0) {
+		let cut = findCut(buf, m, true);
+		if (cut <= 0) cut = buf.length;
+		yield buf.subarray(0, cut);
+		buf = buf.subarray(cut);
+	}
+}
+//#endregion
+//#region src/manifest.ts
+/** Serialize a descriptor to JSON bytes for the manifests BlobStore. */
+function serializeManifest(descriptor) {
+	return new TextEncoder().encode(JSON.stringify(descriptor));
+}
+/** Parse manifest bytes back into a descriptor. */
+function parseManifest(bytes) {
+	return JSON.parse(new TextDecoder().decode(bytes));
+}
+//#endregion
+//#region src/content-store.ts
+const DEFAULT_THRESHOLD = 4096;
+const DEFAULT_CDC = {
+	min: 2048,
+	avg: 8192,
+	max: 32768
+};
+/** A fresh single-shot stream over one buffer (streams are consumed once). */
+async function* one(bytes) {
+	yield bytes;
+}
+async function collect(stream) {
+	const parts = [];
+	for await (const chunk of stream) parts.push(chunk);
+	const total = parts.reduce((n, p) => n + p.length, 0);
+	const out = new Uint8Array(total);
+	let off = 0;
+	for (const p of parts) {
+		out.set(p, off);
+		off += p.length;
+	}
+	return out;
+}
+function createContentStore(deps, opts) {
+	const { chunks, manifests, hashContent } = deps;
+	const threshold = opts?.chunkThreshold ?? DEFAULT_THRESHOLD;
+	const cdc = opts?.cdc ?? DEFAULT_CDC;
+	async function getManifest(id) {
+		if (!await manifests.has(id)) return void 0;
+		return parseManifest(await collect(manifests.get(id)));
+	}
+	/** Re-stream stored chunks in order — feeds the whole-object hash without buffering. */
+	async function* restream(refs) {
+		for (const ref of refs) yield* chunks.get(ref.id);
+	}
+	return {
+		async put(content) {
+			const refs = [];
+			let offset = 0;
+			for await (const bytes of chunkContent(content, cdc, threshold)) {
+				const id = await hashContent(one(bytes));
+				if (!await chunks.has(id)) await chunks.put(id, one(bytes));
+				refs.push({
+					id,
+					offset,
+					size: bytes.length
+				});
+				offset += bytes.length;
+			}
+			const objectId = await hashContent(restream(refs));
+			const descriptor = {
+				id: objectId,
+				size: offset,
+				chunks: refs
+			};
+			await manifests.put(objectId, one(serializeManifest(descriptor)));
+			return descriptor;
+		},
+		has(id) {
+			return manifests.has(id);
+		},
+		read(id, range) {
+			return (async function* () {
+				const descriptor = await getManifest(id);
+				if (!descriptor) return;
+				const start = range?.offset ?? 0;
+				const end = range?.length === void 0 ? descriptor.size : start + range.length;
+				for (const ref of descriptor.chunks) {
+					if (ref.offset + ref.size <= start) continue;
+					if (ref.offset >= end) break;
+					let pos = ref.offset;
+					for await (const piece of chunks.get(ref.id)) {
+						const pieceStart = pos;
+						pos += piece.length;
+						const from = Math.max(start, pieceStart);
+						const to = Math.min(end, pos);
+						if (from < to) yield piece.subarray(from - pieceStart, to - pieceStart);
+					}
+				}
+			})();
+		},
+		getManifest,
+		remove(id) {
+			return manifests.remove(id);
+		},
+		async hasChunks(ids) {
+			const missing = [];
+			for (const id of ids) if (!await chunks.has(id)) missing.push(id);
+			return missing;
+		},
+		async putChunk(id, bytes) {
+			await chunks.put(id, bytes);
+		},
+		getChunk(id) {
+			return chunks.get(id);
+		},
+		async gc(liveRoots) {
+			const live = new Set(liveRoots);
+			const liveChunks = /* @__PURE__ */ new Set();
+			for (const root of liveRoots) {
+				const d = await getManifest(root);
+				if (d) for (const ref of d.chunks) liveChunks.add(ref.id);
+			}
+			let removedObjects = 0;
+			let removedChunks = 0;
+			for await (const objId of manifests.list()) if (!live.has(objId) && await manifests.remove(objId)) removedObjects++;
+			for await (const chunkId of chunks.list()) if (!liveChunks.has(chunkId) && await chunks.remove(chunkId)) removedChunks++;
+			return {
+				removedObjects,
+				removedChunks
+			};
+		}
+	};
+}
+//#endregion
+export { createContentStore };

package/dist/manifest.d.ts

@@ -0,0 +1,6 @@
+import type { ObjectDescriptor } from "./types.js";
+/** Serialize a descriptor to JSON bytes for the manifests BlobStore. */
+export declare function serializeManifest(descriptor: ObjectDescriptor): Uint8Array;
+/** Parse manifest bytes back into a descriptor. */
+export declare function parseManifest(bytes: Uint8Array): ObjectDescriptor;
+//# sourceMappingURL=manifest.d.ts.map

package/dist/manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../src/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD,wEAAwE;AACxE,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,gBAAgB,GAAG,UAAU,CAE1E;AAED,mDAAmD;AACnD,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,gBAAgB,CAEjE"}

package/dist/types.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Public contract for the content-addressed large-object store.
+ *
+ * The store persists large byte streams as deduplicated, content-defined chunks
+ * over the domain-neutral {@link BlobStore} seam. It is algorithm-agnostic: the
+ * whole-object and per-chunk identities are computed by an injected
+ * `hashContent`, and every id is treated as an OPAQUE string (it carries
+ * whatever prefix the hasher produces). The store knows nothing of git, LFS,
+ * SHA-256, commits, or sync.
+ */
+import type { BlobStore, ByteStream } from "@statewalker/webrun-storage";
+export type { ByteStream };
+/** Opaque id of a single content-defined chunk (produced by `hashContent`). */
+export type ChunkId = string;
+/** Opaque id of a whole stored object (produced by `hashContent`). */
+export type ObjectId = string;
+/** Where one chunk sits inside its object. Refs tile `[0, size)` exactly. */
+export interface ChunkRef {
+    id: ChunkId;
+    offset: number;
+    size: number;
+}
+/** Serialized recipe for reassembling an object from its chunks. */
+export interface ObjectDescriptor {
+    id: ObjectId;
+    size: number;
+    /** One entry for a direct blob; many for a CDC-chunked object. */
+    chunks: ChunkRef[];
+}
+/** Injected backing stores + identity function. */
+export interface ContentStoreDeps {
+    /** Stores chunk bytes, keyed by {@link ChunkId}. */
+    chunks: BlobStore;
+    /** Stores serialized {@link ObjectDescriptor}s, keyed by {@link ObjectId}. */
+    manifests: BlobStore;
+    /** REQUIRED — the injected, algorithm-agnostic identity. Ids are opaque. */
+    hashContent: (bytes: ByteStream) => Promise<string>;
+}
+/** Content-defined chunker tuning. */
+export interface CdcParams {
+    min: number;
+    avg: number;
+    max: number;
+}
+export interface ContentStoreOptions {
+    /** Content strictly smaller than this is stored as a single direct blob (no CDC). */
+    chunkThreshold?: number;
+    /** Gear/FastCDC parameters. */
+    cdc?: CdcParams;
+}
+export interface ContentStore {
+    /** Chunk + hash `content` in a bounded-memory pass, storing chunks + manifest. */
+    put(content: ByteStream): Promise<ObjectDescriptor>;
+    has(id: ObjectId): Promise<boolean>;
+    /** Stream the object's bytes, optionally sliced; empty stream if absent. */
+    read(id: ObjectId, range?: {
+        offset?: number;
+        length?: number;
+    }): ByteStream;
+    getManifest(id: ObjectId): Promise<ObjectDescriptor | undefined>;
+    /** Drop the object entry; its chunks are reclaimed later by {@link ContentStore.gc}. */
+    remove(id: ObjectId): Promise<boolean>;
+    /** Return exactly the ids NOT present in the chunk store. */
+    hasChunks(ids: ChunkId[]): Promise<ChunkId[]>;
+    putChunk(id: ChunkId, bytes: ByteStream): Promise<void>;
+    getChunk(id: ChunkId): ByteStream;
+    /** Sweep every object not in `liveRoots` and every chunk they alone referenced. */
+    gc(liveRoots: ObjectId[]): Promise<{
+        removedObjects: number;
+        removedChunks: number;
+    }>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAEzE,YAAY,EAAE,UAAU,EAAE,CAAC;AAE3B,+EAA+E;AAC/E,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC;AAE7B,sEAAsE;AACtE,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC;AAE9B,6EAA6E;AAC7E,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oEAAoE;AACpE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,QAAQ,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,MAAM,EAAE,QAAQ,EAAE,CAAC;CACpB;AAED,mDAAmD;AACnD,MAAM,WAAW,gBAAgB;IAC/B,oDAAoD;IACpD,MAAM,EAAE,SAAS,CAAC;IAClB,8EAA8E;IAC9E,SAAS,EAAE,SAAS,CAAC;IACrB,4EAA4E;IAC5E,WAAW,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACrD;AAED,sCAAsC;AACtC,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,mBAAmB;IAClC,qFAAqF;IACrF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+BAA+B;IAC/B,GAAG,CAAC,EAAE,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAE3B,kFAAkF;IAClF,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACpD,GAAG,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACpC,4EAA4E;IAC5E,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,UAAU,CAAC;IAC7E,WAAW,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAC;IACjE,wFAAwF;IACxF,MAAM,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEvC,6DAA6D;IAC7D,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9C,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,QAAQ,CAAC,EAAE,EAAE,OAAO,GAAG,UAAU,CAAC;IAElC,mFAAmF;IACnF,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvF"}

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@statewalker/webrun-content-store",
+  "version": "0.2.2",
+  "description": "Domain-neutral, content-addressed large-object store with content-defined chunking over the storage BlobStore",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/statewalker/webrun-vcs.git",
+    "directory": "packages/webrun-content-store"
+  },
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@statewalker/webrun-files": "^0.10.0",
+    "@statewalker/webrun-storage": "^0.2.1"
+  },
+  "devDependencies": {
+    "@statewalker/webrun-files-mem": "^0.10.0",
+    "rolldown": "^1.2.12",
+    "vitest": "^5.0.3",
+    "typescript": "^7.0.2",
+    "@types/node": "^26.6.4"
+  },
+  "scripts": {
+    "build": "rm -rf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
+    "test": "vitest run",
+    "lint": "biome check --write .",
+    "lint:check": "biome check .",
+    "lint:fix": "biome check --write --unsafe .",
+    "format": "biome format --write .",
+    "format:check": "biome format .",
+    "format:fix": "biome check --write --unsafe .",
+    "typecheck": "tsc --noEmit"
+  }
+}

package/src/chunker.ts

@@ -0,0 +1,136 @@
+import type { ByteStream, CdcParams } from "./types.js";
+
+/**
+ * Minimal Gear-hash FastCDC content-defined chunker.
+ *
+ * A rolling Gear fingerprint (`fp = (fp << 1) + GEAR[byte]`) restarts at every
+ * chunk boundary; a cut is taken when the low bits of `fp` are zero. Normalized
+ * chunking (Xia et al., FastCDC) uses a stricter mask before the average size
+ * and a looser one after, tightening the size distribution. Because the cut for
+ * a chunk depends only on the bytes since its own start, boundaries re-sync
+ * shortly after an edit — so a mid-file insert leaves surrounding chunks intact.
+ */
+
+const GEAR = buildGearTable();
+
+/** 256 deterministic 32-bit gear values (xorshift32 from a fixed seed). */
+function buildGearTable(): Uint32Array {
+  const g = new Uint32Array(256);
+  let x = 0x9e3779b9;
+  for (let i = 0; i < 256; i++) {
+    x ^= x << 13;
+    x ^= x >>> 17;
+    x ^= x << 5;
+    x >>>= 0;
+    g[i] = x;
+  }
+  return g;
+}
+
+interface Masks {
+  min: number;
+  avg: number;
+  max: number;
+  maskS: number;
+  maskL: number;
+}
+
+function masksFor(cdc: CdcParams): Masks {
+  const bits = Math.round(Math.log2(cdc.avg));
+  // Normalization = 2: harder mask (more low bits ⇒ rarer cut) before avg,
+  // easier mask after, clamped so both stay valid low-bit masks.
+  const maskS = (1 << Math.min(bits + 2, 31)) - 1;
+  const maskL = (1 << Math.max(bits - 2, 1)) - 1;
+  return { min: cdc.min, avg: cdc.avg, max: cdc.max, maskS, maskL };
+}
+
+/**
+ * Find the cut index (exclusive end of the first chunk) within `buf`, whose
+ * chunk starts at index 0. Returns `-1` when more bytes are needed before a cut
+ * can be decided (only when `!final`).
+ */
+function findCut(buf: Uint8Array, m: Masks, final: boolean): number {
+  const n = buf.length;
+  if (n <= m.min) return final ? n : -1; // too small to cut yet
+  const limit = Math.min(n, m.max);
+  let fp = 0;
+  let i = m.min;
+  const avgStop = Math.min(m.avg, limit);
+  for (; i < avgStop; i++) {
+    fp = ((fp << 1) + GEAR[buf[i]]) >>> 0;
+    if ((fp & m.maskS) === 0) return i;
+  }
+  for (; i < limit; i++) {
+    fp = ((fp << 1) + GEAR[buf[i]]) >>> 0;
+    if ((fp & m.maskL) === 0) return i;
+  }
+  if (limit === m.max) return m.max; // forced cut at max size
+  return final ? n : -1; // remainder is a final chunk, or await more bytes
+}
+
+function concat(a: Uint8Array, b: Uint8Array): Uint8Array<ArrayBuffer> {
+  const out = new Uint8Array(a.length + b.length);
+  out.set(a, 0);
+  out.set(b, a.length);
+  return out;
+}
+
+/**
+ * Split `source` into content-defined chunks. Content strictly smaller than
+ * `threshold` is emitted as a single direct blob (no CDC). Bounded memory: at
+ * most one pending chunk (plus one incoming array) is buffered at a time.
+ */
+export async function* chunkContent(
+  source: ByteStream,
+  cdc: CdcParams,
+  threshold: number,
+): AsyncIterable<Uint8Array> {
+  const iter = source[Symbol.asyncIterator]();
+
+  // Buffer up to `threshold` bytes to decide direct-blob vs CDC.
+  const prefix: Uint8Array[] = [];
+  let prefixLen = 0;
+  let done = false;
+  while (prefixLen < threshold) {
+    const next = await iter.next();
+    if (next.done) {
+      done = true;
+      break;
+    }
+    prefix.push(next.value);
+    prefixLen += next.value.length;
+  }
+
+  if (done && prefixLen < threshold) {
+    // Whole content is below threshold → one direct blob.
+    let blob = new Uint8Array(0);
+    for (const p of prefix) blob = concat(blob, p);
+    yield blob;
+    return;
+  }
+
+  // At or above threshold → CDC over the buffered prefix, then the rest.
+  const m = masksFor(cdc);
+  let buf = new Uint8Array(0);
+  const feed = async function* (): AsyncIterable<Uint8Array> {
+    for (const p of prefix) yield p;
+    if (!done) {
+      for (let n = await iter.next(); !n.done; n = await iter.next()) yield n.value;
+    }
+  };
+
+  for await (const incoming of feed()) {
+    buf = concat(buf, incoming);
+    for (let cut = findCut(buf, m, false); cut >= 0; cut = findCut(buf, m, false)) {
+      yield buf.subarray(0, cut);
+      buf = buf.subarray(cut);
+    }
+  }
+  // Flush the tail.
+  while (buf.length > 0) {
+    let cut = findCut(buf, m, true);
+    if (cut <= 0) cut = buf.length;
+    yield buf.subarray(0, cut);
+    buf = buf.subarray(cut);
+  }
+}

package/src/content-store.ts

@@ -0,0 +1,136 @@
+import { chunkContent } from "./chunker.js";
+import { parseManifest, serializeManifest } from "./manifest.js";
+import type {
+  ByteStream,
+  ChunkId,
+  ChunkRef,
+  ContentStore,
+  ContentStoreDeps,
+  ContentStoreOptions,
+  ObjectDescriptor,
+  ObjectId,
+} from "./types.js";
+
+const DEFAULT_THRESHOLD = 4096;
+const DEFAULT_CDC = { min: 2048, avg: 8192, max: 32768 };
+
+/** A fresh single-shot stream over one buffer (streams are consumed once). */
+async function* one(bytes: Uint8Array): ByteStream {
+  yield bytes;
+}
+
+async function collect(stream: ByteStream): Promise<Uint8Array> {
+  const parts: Uint8Array[] = [];
+  for await (const chunk of stream) parts.push(chunk);
+  const total = parts.reduce((n, p) => n + p.length, 0);
+  const out = new Uint8Array(total);
+  let off = 0;
+  for (const p of parts) {
+    out.set(p, off);
+    off += p.length;
+  }
+  return out;
+}
+
+export function createContentStore(
+  deps: ContentStoreDeps,
+  opts?: ContentStoreOptions,
+): ContentStore {
+  const { chunks, manifests, hashContent } = deps;
+  const threshold = opts?.chunkThreshold ?? DEFAULT_THRESHOLD;
+  const cdc = opts?.cdc ?? DEFAULT_CDC;
+
+  async function getManifest(id: ObjectId): Promise<ObjectDescriptor | undefined> {
+    if (!(await manifests.has(id))) return undefined;
+    return parseManifest(await collect(manifests.get(id)));
+  }
+
+  /** Re-stream stored chunks in order — feeds the whole-object hash without buffering. */
+  async function* restream(refs: ChunkRef[]): ByteStream {
+    for (const ref of refs) yield* chunks.get(ref.id);
+  }
+
+  const store: ContentStore = {
+    async put(content) {
+      const refs: ChunkRef[] = [];
+      let offset = 0;
+      for await (const bytes of chunkContent(content, cdc, threshold)) {
+        const id = await hashContent(one(bytes));
+        if (!(await chunks.has(id))) await chunks.put(id, one(bytes));
+        refs.push({ id, offset, size: bytes.length });
+        offset += bytes.length;
+      }
+      const objectId = await hashContent(restream(refs));
+      const descriptor: ObjectDescriptor = { id: objectId, size: offset, chunks: refs };
+      await manifests.put(objectId, one(serializeManifest(descriptor)));
+      return descriptor;
+    },
+
+    has(id) {
+      return manifests.has(id);
+    },
+
+    read(id, range) {
+      return (async function* (): ByteStream {
+        const descriptor = await getManifest(id);
+        if (!descriptor) return;
+        const start = range?.offset ?? 0;
+        const end = range?.length === undefined ? descriptor.size : start + range.length;
+        for (const ref of descriptor.chunks) {
+          if (ref.offset + ref.size <= start) continue; // entirely before the range
+          if (ref.offset >= end) break; // chunks are ordered — nothing left overlaps
+          let pos = ref.offset;
+          for await (const piece of chunks.get(ref.id)) {
+            const pieceStart = pos;
+            pos += piece.length;
+            const from = Math.max(start, pieceStart);
+            const to = Math.min(end, pos);
+            if (from < to) yield piece.subarray(from - pieceStart, to - pieceStart);
+          }
+        }
+      })();
+    },
+
+    getManifest,
+
+    remove(id) {
+      return manifests.remove(id);
+    },
+
+    async hasChunks(ids) {
+      const missing: ChunkId[] = [];
+      for (const id of ids) {
+        if (!(await chunks.has(id))) missing.push(id);
+      }
+      return missing;
+    },
+
+    async putChunk(id, bytes) {
+      await chunks.put(id, bytes);
+    },
+
+    getChunk(id) {
+      return chunks.get(id);
+    },
+
+    async gc(liveRoots) {
+      const live = new Set(liveRoots);
+      const liveChunks = new Set<ChunkId>();
+      for (const root of liveRoots) {
+        const d = await getManifest(root);
+        if (d) for (const ref of d.chunks) liveChunks.add(ref.id);
+      }
+      let removedObjects = 0;
+      let removedChunks = 0;
+      for await (const objId of manifests.list()) {
+        if (!live.has(objId) && (await manifests.remove(objId))) removedObjects++;
+      }
+      for await (const chunkId of chunks.list()) {
+        if (!liveChunks.has(chunkId) && (await chunks.remove(chunkId))) removedChunks++;
+      }
+      return { removedObjects, removedChunks };
+    },
+  };
+
+  return store;
+}

package/src/index.ts

@@ -0,0 +1,22 @@
+/**
+ * @statewalker/webrun-content-store
+ *
+ * Domain-neutral, content-addressed large-object store. It stores large byte
+ * streams as deduplicated, content-defined chunks over the `@statewalker/webrun-storage`
+ * BlobStore seam. It is algorithm-agnostic — object and chunk identities are
+ * computed by an injected `hashContent`, and every id is an opaque string. It
+ * knows nothing of git, LFS, SHA-256, commits, or sync.
+ */
+
+export { createContentStore } from "./content-store.js";
+export type {
+  ByteStream,
+  CdcParams,
+  ChunkId,
+  ChunkRef,
+  ContentStore,
+  ContentStoreDeps,
+  ContentStoreOptions,
+  ObjectDescriptor,
+  ObjectId,
+} from "./types.js";

package/src/manifest.ts

@@ -0,0 +1,11 @@
+import type { ObjectDescriptor } from "./types.js";
+
+/** Serialize a descriptor to JSON bytes for the manifests BlobStore. */
+export function serializeManifest(descriptor: ObjectDescriptor): Uint8Array {
+  return new TextEncoder().encode(JSON.stringify(descriptor));
+}
+
+/** Parse manifest bytes back into a descriptor. */
+export function parseManifest(bytes: Uint8Array): ObjectDescriptor {
+  return JSON.parse(new TextDecoder().decode(bytes)) as ObjectDescriptor;
+}

package/src/types.ts

@@ -0,0 +1,79 @@
+/**
+ * Public contract for the content-addressed large-object store.
+ *
+ * The store persists large byte streams as deduplicated, content-defined chunks
+ * over the domain-neutral {@link BlobStore} seam. It is algorithm-agnostic: the
+ * whole-object and per-chunk identities are computed by an injected
+ * `hashContent`, and every id is treated as an OPAQUE string (it carries
+ * whatever prefix the hasher produces). The store knows nothing of git, LFS,
+ * SHA-256, commits, or sync.
+ */
+
+import type { BlobStore, ByteStream } from "@statewalker/webrun-storage";
+
+export type { ByteStream };
+
+/** Opaque id of a single content-defined chunk (produced by `hashContent`). */
+export type ChunkId = string;
+
+/** Opaque id of a whole stored object (produced by `hashContent`). */
+export type ObjectId = string;
+
+/** Where one chunk sits inside its object. Refs tile `[0, size)` exactly. */
+export interface ChunkRef {
+  id: ChunkId;
+  offset: number;
+  size: number;
+}
+
+/** Serialized recipe for reassembling an object from its chunks. */
+export interface ObjectDescriptor {
+  id: ObjectId;
+  size: number;
+  /** One entry for a direct blob; many for a CDC-chunked object. */
+  chunks: ChunkRef[];
+}
+
+/** Injected backing stores + identity function. */
+export interface ContentStoreDeps {
+  /** Stores chunk bytes, keyed by {@link ChunkId}. */
+  chunks: BlobStore;
+  /** Stores serialized {@link ObjectDescriptor}s, keyed by {@link ObjectId}. */
+  manifests: BlobStore;
+  /** REQUIRED — the injected, algorithm-agnostic identity. Ids are opaque. */
+  hashContent: (bytes: ByteStream) => Promise<string>;
+}
+
+/** Content-defined chunker tuning. */
+export interface CdcParams {
+  min: number;
+  avg: number;
+  max: number;
+}
+
+export interface ContentStoreOptions {
+  /** Content strictly smaller than this is stored as a single direct blob (no CDC). */
+  chunkThreshold?: number;
+  /** Gear/FastCDC parameters. */
+  cdc?: CdcParams;
+}
+
+export interface ContentStore {
+  // object level
+  /** Chunk + hash `content` in a bounded-memory pass, storing chunks + manifest. */
+  put(content: ByteStream): Promise<ObjectDescriptor>;
+  has(id: ObjectId): Promise<boolean>;
+  /** Stream the object's bytes, optionally sliced; empty stream if absent. */
+  read(id: ObjectId, range?: { offset?: number; length?: number }): ByteStream;
+  getManifest(id: ObjectId): Promise<ObjectDescriptor | undefined>;
+  /** Drop the object entry; its chunks are reclaimed later by {@link ContentStore.gc}. */
+  remove(id: ObjectId): Promise<boolean>;
+  // chunk level (transfer / dedup)
+  /** Return exactly the ids NOT present in the chunk store. */
+  hasChunks(ids: ChunkId[]): Promise<ChunkId[]>;
+  putChunk(id: ChunkId, bytes: ByteStream): Promise<void>;
+  getChunk(id: ChunkId): ByteStream;
+  // maintenance
+  /** Sweep every object not in `liveRoots` and every chunk they alone referenced. */
+  gc(liveRoots: ObjectId[]): Promise<{ removedObjects: number; removedChunks: number }>;
+}