@statewalker/webrun-content-transfer 0.2.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-present StateWalker Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,69 @@
+# @statewalker/webrun-content-transfer
+
+Resumable, chunk-aware byte mover over the `@statewalker/webrun-streams` duplex substrate.
+
+## Overview
+
+Given the object ids a destination needs, `content-transfer` transfers only the chunks the destination is **missing** from a source `ContentStore` to a destination store, over the domain-neutral `webrun-streams` `Duplex` substrate. It is **symmetric** — either endpoint may be a wire `remoteStore` proxy in front of a `serveStore` handler (upload = remote `to`, download = remote `from`). Every received chunk is verified by re-hash before it is written, and a serializable checkpoint makes an interrupted transfer resumable.
+
+It is shared infrastructure: the Phase-4 chunk-dedup `Transfer` that plugs into `@statewalker/webrun-files-sync`, and the engine `@statewalker/vcs-transport-xet` wraps as a Git LFS custom transfer agent. It builds on `@statewalker/webrun-content-store` and knows nothing of git, LFS, or sync — it moves opaque chunks between two content stores.
+
+## Installation
+
+```bash
+pnpm add @statewalker/webrun-content-transfer
+```
+
+## Quick Start
+
+```typescript
+import { createHash } from "node:crypto";
+import { memBlobStore } from "@statewalker/webrun-storage";
+import { createContentStore } from "@statewalker/webrun-content-store";
+import { transfer } from "@statewalker/webrun-content-transfer";
+
+async function sha256(bytes: AsyncIterable<Uint8Array>): Promise<string> {
+  const h = createHash("sha256");
+  for await (const chunk of bytes) h.update(chunk);
+  return `sha256:${h.digest("hex")}`;
+}
+const mkStore = () =>
+  createContentStore({ chunks: memBlobStore(), manifests: memBlobStore(), hashContent: sha256 });
+
+const from = mkStore();
+const to = mkStore();
+async function* streamOf(b: Uint8Array) { yield b; }
+const { id } = await from.put(streamOf(payload));
+
+// Move only the chunks `to` is missing; re-hash-verify each; emit progress.
+for await (const event of transfer([id], from, to, { hashContent: sha256 })) {
+  if (event.type === "object-done") console.log("done", event.objectId);
+}
+// to.read(id) now reconstructs the identical object.
+```
+
+For a remote peer, wrap a `webrun-streams` `Duplex`:
+
+```typescript
+import { remoteStore, serveStore } from "@statewalker/webrun-content-transfer";
+
+const handler = serveStore(serverStore);   // server: Duplex handler over a ContentStore
+const remote = remoteStore(clientDuplex);  // client: an AssemblingStore proxy
+await drain(transfer([id], from, remote, { hashContent: sha256 })); // upload to the remote
+```
+
+## API
+
+- **`transfer(objectIds, from, to, opts?): AsyncGenerator<TransferEvent>`** — the mover. Negotiates missing chunks per object, streams them, verifies by re-hash, and assembles. Events: `resumed` / `negotiated` / `chunk-sent` / `object-done`.
+- **`remoteStore(call: Duplex): AssemblingStore`** — client-side proxy `ContentStore` over a duplex (adds `putManifest` / `capabilities`).
+- **`serveStore(store): Duplex`** — server-side handler exposing a `ContentStore` over a duplex.
+- **`chunkTransfer(local, remote): FileTransfer`** — the `files-sync`-shaped `Transfer` adapter.
+
+`TransferOptions` carries optional `limits` (`TransferLimits`: `concurrency`, `maxBufferedBytes`, `batchSize`), `checkpoint` (`TransferCheckpoint`), and `hashContent`. Types: `AssemblingStore`, `Capabilities`, `TransferEvent`, `HashContent`.
+
+## Notes
+
+- **Integrity by re-hash.** The re-hash of a received chunk must equal its id (the same injected `hashContent` both stores use) or the chunk is rejected and the transfer fails loudly.
+- **Resumable + bounded.** A serializable `TransferCheckpoint` (done / pending chunk sets) resumes an interrupted transfer after re-verifying the source still holds every object; the pipeline is bounded by `TransferLimits` + `maxBufferedBytes` backpressure.
+- **Symmetric wire.** `hasChunks` negotiation is batched; the `remoteStore` proxy can `putManifest` so assemble costs only the manifest, not the object's bytes.
+- Built red/green TDD.

package/dist/chunk-transfer.d.ts

@@ -0,0 +1,22 @@
+/**
+ * files-sync interop: adapt content-store dedup transfer to the files-sync
+ * {@link https://github.com/statewalker/vcs `Transfer`} seam. The returned object
+ * is structurally a files-sync `Transfer` (kept structural so `src/` stays
+ * domain-neutral and never imports files-sync). For a copy/update it ingests the
+ * source file into the `local` store, moves the missing chunks to `remote` via
+ * {@link transfer}, then writes the reassembled bytes to the destination — so a
+ * file whose chunks the remote already holds costs nothing to move.
+ */
+import type { ContentStore } from "@statewalker/webrun-content-store";
+import type { FilesApi } from "@statewalker/webrun-files";
+/** The subset of a files-sync `SyncAction` this adapter reads (structural). */
+export interface FileSyncAction {
+    kind: string;
+    path?: string;
+}
+/** Structurally compatible with files-sync's `Transfer`. */
+export interface FileTransfer {
+    run(action: FileSyncAction, from: FilesApi, to: FilesApi): Promise<void>;
+}
+export declare function chunkTransfer(local: ContentStore, remote: ContentStore): FileTransfer;
+//# sourceMappingURL=chunk-transfer.d.ts.map

package/dist/chunk-transfer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-transfer.d.ts","sourceRoot":"","sources":["../src/chunk-transfer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAG1D,+EAA+E;AAC/E,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,4DAA4D;AAC5D,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,GAAG,YAAY,CAsBrF"}

package/dist/index.d.ts

@@ -0,0 +1,19 @@
+/**
+ * @statewalker/webrun-content-transfer
+ *
+ * Resumable, chunk-aware byte mover. Given the object ids a destination needs,
+ * it transfers only the chunks the destination is MISSING from a source
+ * {@link ContentStore} to a destination store, over the domain-neutral
+ * `@statewalker/webrun-streams` {@link Duplex} substrate. It is symmetric —
+ * either endpoint may be a wire {@link remoteStore} proxy in front of a
+ * {@link serveStore} handler. Every received chunk is verified by re-hash before
+ * it is written; a serializable checkpoint makes an interrupted transfer
+ * resumable. It knows nothing of git, LFS, or sync.
+ */
+export type { FileSyncAction, FileTransfer } from "./chunk-transfer.js";
+export { chunkTransfer } from "./chunk-transfer.js";
+export { remoteStore } from "./remote-store.js";
+export { serveStore } from "./serve-store.js";
+export { transfer } from "./transfer.js";
+export type { AssemblingStore, Capabilities, HashContent, TransferCheckpoint, TransferEvent, TransferLimits, TransferOptions, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,YAAY,EACV,eAAe,EACf,YAAY,EACZ,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,cAAc,EACd,eAAe,GAChB,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,309 @@
+//#region src/transfer.ts
+const DEFAULT_LIMITS = {
+	concurrency: 4,
+	maxBufferedBytes: 8388608,
+	batchSize: 256
+};
+/** A fresh single-shot stream over one buffer (streams are consumed once). */
+async function* one(bytes) {
+	yield bytes;
+}
+async function collect(stream) {
+	const parts = [];
+	for await (const chunk of stream) parts.push(chunk);
+	const total = parts.reduce((n, p) => n + p.length, 0);
+	const out = new Uint8Array(total);
+	let off = 0;
+	for (const p of parts) {
+		out.set(p, off);
+		off += p.length;
+	}
+	return out;
+}
+/** Re-stream a store's own chunks in manifest order — feeds `put` without buffering. */
+async function* reassembleFromStore(store, manifest) {
+	for (const ref of manifest.chunks) yield* store.getChunk(ref.id);
+}
+/**
+* Register `manifest` at `to`, so `to.read(id)` reconstructs the object. When
+* `to` is a wire proxy it assembles server-side from chunks it already holds
+* (only the manifest crosses the wire); otherwise it re-`put`s a local
+* reassembly (cheap, same-process) and verifies the resulting id.
+*/
+async function registerManifest(to, manifest) {
+	const proxy = to;
+	if (typeof proxy.putManifest === "function") {
+		await proxy.putManifest(manifest);
+		return;
+	}
+	const descriptor = await to.put(reassembleFromStore(to, manifest));
+	if (descriptor.id !== manifest.id) throw new Error(`content-transfer: assemble id mismatch ${descriptor.id} != ${manifest.id}`);
+}
+async function maybeCapabilities(to) {
+	const proxy = to;
+	if (typeof proxy.capabilities === "function") return proxy.capabilities();
+}
+/**
+* Run `work` over `refs` with at most `concurrency` in flight and never more
+* than `maxBufferedBytes` of chunk bytes buffered (a single over-budget chunk is
+* allowed alone, to guarantee progress). Yields each result as it settles.
+*/
+async function* pipeline(refs, limits, work) {
+	const inflight = /* @__PURE__ */ new Set();
+	let buffered = 0;
+	let idx = 0;
+	const canLaunch = () => idx < refs.length && inflight.size < limits.concurrency && (inflight.size === 0 || buffered + refs[idx].size <= limits.maxBufferedBytes);
+	while (idx < refs.length || inflight.size > 0) {
+		while (canLaunch()) {
+			const ref = refs[idx++];
+			buffered += ref.size;
+			const entry = { ref };
+			entry.promise = work(ref).then((value) => {
+				entry.value = value;
+				return entry;
+			});
+			inflight.add(entry.promise);
+		}
+		const settled = await Promise.race(inflight);
+		inflight.delete(settled.promise);
+		buffered -= settled.ref.size;
+		yield settled.value;
+	}
+}
+async function* transfer(objectIds, from, to, opts) {
+	const limits = {
+		...DEFAULT_LIMITS,
+		...opts?.limits
+	};
+	const hashContent = opts?.hashContent;
+	const caps = await maybeCapabilities(to);
+	if (opts?.checkpoint) yield {
+		type: "resumed",
+		done: opts.checkpoint.done.length
+	};
+	const manifests = /* @__PURE__ */ new Map();
+	const refById = /* @__PURE__ */ new Map();
+	for (const id of objectIds) {
+		const manifest = await from.getManifest(id);
+		if (!manifest) throw new Error(`content-transfer: source is missing object ${id}`);
+		manifests.set(id, manifest);
+		for (const ref of manifest.chunks) if (!refById.has(ref.id)) refById.set(ref.id, ref);
+	}
+	const allIds = [...refById.keys()];
+	const missing = [];
+	for (let i = 0; i < allIds.length; i += limits.batchSize) {
+		const batch = allIds.slice(i, i + limits.batchSize);
+		for (const cid of await to.hasChunks(batch)) {
+			const ref = refById.get(cid);
+			if (ref) missing.push(ref);
+		}
+	}
+	yield {
+		type: "negotiated",
+		objects: objectIds.length,
+		chunks: allIds.length,
+		missing: missing.length,
+		protocol: caps?.protocol
+	};
+	for await (const sent of pipeline(missing, limits, async (ref) => {
+		const bytes = await collect(from.getChunk(ref.id));
+		if (hashContent) {
+			const actual = await hashContent(one(bytes));
+			if (actual !== ref.id) throw new Error(`content-transfer: integrity check failed for chunk ${ref.id} (re-hash ${actual})`);
+		}
+		await to.putChunk(ref.id, one(bytes));
+		return {
+			id: ref.id,
+			size: bytes.length
+		};
+	})) yield {
+		type: "chunk-sent",
+		chunkId: sent.id,
+		size: sent.size
+	};
+	for (const id of objectIds) {
+		const manifest = manifests.get(id);
+		if (!manifest) continue;
+		await registerManifest(to, manifest);
+		yield {
+			type: "object-done",
+			objectId: id
+		};
+	}
+}
+//#endregion
+//#region src/chunk-transfer.ts
+function chunkTransfer(local, remote) {
+	return { async run(action, from, to) {
+		if (action.kind !== "copy" && action.kind !== "update") throw new Error(`content-transfer: chunkTransfer only handles copy/update, got ${action.kind}`);
+		const path = action.path;
+		if (!path) throw new Error(`content-transfer: ${action.kind} action has no path`);
+		const descriptor = await local.put(from.read(path));
+		const iterator = transfer([descriptor.id], local, remote)[Symbol.asyncIterator]();
+		while (!(await iterator.next()).done);
+		await to.write(path, remote.read(descriptor.id));
+	} };
+}
+//#endregion
+//#region src/protocol.ts
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+/** Encode `[len][header][body?]` as a byte stream. */
+async function* frameMessage(header, body) {
+	const json = encoder.encode(JSON.stringify(header));
+	const prefix = /* @__PURE__ */ new Uint8Array(4);
+	prefix[0] = json.length >>> 24 & 255;
+	prefix[1] = json.length >>> 16 & 255;
+	prefix[2] = json.length >>> 8 & 255;
+	prefix[3] = json.length & 255;
+	yield prefix;
+	yield json;
+	if (body) {
+		for await (const b of body) if (b.byteLength) yield b;
+	}
+}
+/** Read one framed message: its parsed header + a body byte stream. Accepts
+* either side of a {@link Duplex} (sync or async byte iterable). */
+async function readFrame(input) {
+	const iter = Symbol.asyncIterator in input ? input[Symbol.asyncIterator]() : (() => {
+		const sync = input[Symbol.iterator]();
+		return { next: async () => sync.next() };
+	})();
+	let buf = /* @__PURE__ */ new Uint8Array(0);
+	async function fill(n) {
+		while (buf.length < n) {
+			const { value, done } = await iter.next();
+			if (done) throw new Error("content-transfer protocol: unexpected end of stream");
+			const next = new Uint8Array(buf.length + value.length);
+			next.set(buf, 0);
+			next.set(value, buf.length);
+			buf = next;
+		}
+	}
+	await fill(4);
+	const headerLen = (buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3]) >>> 0;
+	await fill(4 + headerLen);
+	const header = JSON.parse(decoder.decode(buf.subarray(4, 4 + headerLen)));
+	const rest = buf.subarray(4 + headerLen);
+	async function* body() {
+		if (rest.byteLength) yield rest;
+		while (true) {
+			const { value, done } = await iter.next();
+			if (done) return;
+			if (value.byteLength) yield value;
+		}
+	}
+	return {
+		header,
+		body: body()
+	};
+}
+//#endregion
+//#region src/remote-store.ts
+const unsupported = (name) => {
+	throw new Error(`content-transfer: remote proxy does not support "${name}"`);
+};
+function remoteStore(call) {
+	async function request(header, body) {
+		const { header: resp, body: respBody } = await readFrame(call(frameMessage(header, body)));
+		if (!resp.ok) throw new Error(resp.error ?? "content-transfer: remote error");
+		return {
+			header: resp,
+			body: respBody
+		};
+	}
+	return {
+		async capabilities() {
+			const { header } = await request({ op: "capabilities" });
+			if (!header.capabilities) throw new Error("content-transfer: no capabilities in response");
+			return header.capabilities;
+		},
+		async getManifest(id) {
+			const { header } = await request({
+				op: "getManifest",
+				id
+			});
+			return header.manifest ?? void 0;
+		},
+		async hasChunks(ids) {
+			const { header } = await request({
+				op: "hasChunks",
+				ids
+			});
+			return header.missing ?? [];
+		},
+		getChunk(id) {
+			return (async function* () {
+				const { body } = await request({
+					op: "getChunk",
+					id
+				});
+				yield* body;
+			})();
+		},
+		async putChunk(id, bytes) {
+			await request({
+				op: "putChunk",
+				id
+			}, bytes);
+		},
+		async putManifest(manifest) {
+			await request({
+				op: "putManifest",
+				manifest
+			});
+		},
+		put: () => unsupported("put"),
+		has: () => unsupported("has"),
+		read: () => unsupported("read"),
+		remove: () => unsupported("remove"),
+		gc: () => unsupported("gc")
+	};
+}
+//#endregion
+//#region src/serve-store.ts
+const PROTOCOL = "content-transfer/1";
+function serveStore(store) {
+	return async function* handler(input) {
+		const { header, body } = await readFrame(input);
+		switch (header.op) {
+			case "capabilities":
+				yield* frameMessage({
+					ok: true,
+					capabilities: { protocol: PROTOCOL }
+				});
+				return;
+			case "getManifest":
+				yield* frameMessage({
+					ok: true,
+					manifest: await store.getManifest(header.id) ?? null
+				});
+				return;
+			case "hasChunks":
+				yield* frameMessage({
+					ok: true,
+					missing: await store.hasChunks(header.ids)
+				});
+				return;
+			case "getChunk":
+				yield* frameMessage({ ok: true }, store.getChunk(header.id));
+				return;
+			case "putChunk":
+				await store.putChunk(header.id, body);
+				yield* frameMessage({ ok: true });
+				return;
+			case "putManifest":
+				await registerManifest(store, header.manifest);
+				yield* frameMessage({ ok: true });
+				return;
+			default:
+				yield* frameMessage({
+					ok: false,
+					error: `unknown op ${header.op}`
+				});
+				return;
+		}
+	};
+}
+//#endregion
+export { chunkTransfer, remoteStore, serveStore, transfer };

package/dist/protocol.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Wire framing for the content-transfer RPC. One {@link Duplex} call carries one
+ * request→response. Every message is `[4-byte big-endian header length][header
+ * JSON][body bytes...]`: the JSON header names the op and its scalar args, and
+ * the remaining bytes are the streamed body (a chunk's bytes, or none).
+ *
+ * Small op surface, one per content-store transfer primitive:
+ *   - `capabilities`  → protocol handshake (BATCHED handshake, no body)
+ *   - `getManifest`   → object recipe (needed for the download direction)
+ *   - `hasChunks`     → BATCHED: many ids per request → the missing ids
+ *   - `getChunk`      → STREAMED: response body is the chunk bytes
+ *   - `putChunk`      → STREAMED: request body is the chunk bytes
+ *   - `putManifest`   → server-side assemble from already-present chunks
+ *
+ * (A Bloom prefilter for `hasChunks` is intentionally deferred — the plain
+ * batched id list is enough; add it only when a payload budget demands it.)
+ */
+import type { ChunkId, ObjectDescriptor, ObjectId } from "@statewalker/webrun-content-store";
+export type Request = {
+    op: "capabilities";
+} | {
+    op: "getManifest";
+    id: ObjectId;
+} | {
+    op: "hasChunks";
+    ids: ChunkId[];
+} | {
+    op: "getChunk";
+    id: ChunkId;
+} | {
+    op: "putChunk";
+    id: ChunkId;
+} | {
+    op: "putManifest";
+    manifest: ObjectDescriptor;
+};
+/** Encode `[len][header][body?]` as a byte stream. */
+export declare function frameMessage(header: unknown, body?: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): AsyncGenerator<Uint8Array>;
+/** Read one framed message: its parsed header + a body byte stream. Accepts
+ * either side of a {@link Duplex} (sync or async byte iterable). */
+export declare function readFrame<H>(input: AsyncIterable<Uint8Array> | Iterable<Uint8Array>): Promise<{
+    header: H;
+    body: AsyncGenerator<Uint8Array>;
+}>;
+//# sourceMappingURL=protocol.d.ts.map

package/dist/protocol.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../src/protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,mCAAmC,CAAC;AAE7F,MAAM,MAAM,OAAO,GACf;IAAE,EAAE,EAAE,cAAc,CAAA;CAAE,GACtB;IAAE,EAAE,EAAE,aAAa,CAAC;IAAC,EAAE,EAAE,QAAQ,CAAA;CAAE,GACnC;IAAE,EAAE,EAAE,WAAW,CAAC;IAAC,GAAG,EAAE,OAAO,EAAE,CAAA;CAAE,GACnC;IAAE,EAAE,EAAE,UAAU,CAAC;IAAC,EAAE,EAAE,OAAO,CAAA;CAAE,GAC/B;IAAE,EAAE,EAAE,UAAU,CAAC;IAAC,EAAE,EAAE,OAAO,CAAA;CAAE,GAC/B;IAAE,EAAE,EAAE,aAAa,CAAC;IAAC,QAAQ,EAAE,gBAAgB,CAAA;CAAE,CAAC;AAKtD,sDAAsD;AACtD,wBAAuB,YAAY,CACjC,MAAM,EAAE,OAAO,EACf,IAAI,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,GACtD,cAAc,CAAC,UAAU,CAAC,CAU5B;AAED;oEACoE;AACpE,wBAAsB,SAAS,CAAC,CAAC,EAC/B,KAAK,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,GACtD,OAAO,CAAC;IAAE,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAA;CAAE,CAAC,CAqC1D"}

package/dist/remote-store.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Client half of the wire protocol: wraps a {@link Duplex} as a
+ * {@link ContentStore}-shaped proxy. Each method opens one call, frames the
+ * request, and reads the framed response. The proxy exposes exactly the
+ * chunk-transfer surface a {@link transfer} needs from a remote peer —
+ * `capabilities` / `getManifest` / `hasChunks` / `getChunk` / `putChunk`, plus
+ * `putManifest` for the server-side assemble. Whole-object operations that would
+ * need the store's chunker (`put`) or have no transfer meaning (`has` / `read` /
+ * `remove` / `gc`) are not carried over the wire and throw.
+ */
+import type { Duplex } from "@statewalker/webrun-streams";
+import type { AssemblingStore } from "./types.js";
+export declare function remoteStore(call: Duplex): AssemblingStore;
+//# sourceMappingURL=remote-store.d.ts.map

package/dist/remote-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-store.d.ts","sourceRoot":"","sources":["../src/remote-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAQH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAC;AAE1D,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,YAAY,CAAC;AAkBhE,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,CAmDzD"}

package/dist/serve-store.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Server half of the wire protocol: turns a {@link ContentStore} into a
+ * {@link Duplex} handler. Each incoming call is one framed request; the handler
+ * dispatches it to the store and frames the response. Symmetric with
+ * {@link remoteStore} — an in-process test wires this Duplex straight into
+ * `remoteStore(...)` with no transport.
+ */
+import type { ContentStore } from "@statewalker/webrun-content-store";
+import type { Duplex } from "@statewalker/webrun-streams";
+export declare function serveStore(store: ContentStore): Duplex;
+//# sourceMappingURL=serve-store.d.ts.map

package/dist/serve-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serve-store.d.ts","sourceRoot":"","sources":["../src/serve-store.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAC;AAM1D,wBAAgB,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAmCtD"}

package/dist/transfer.d.ts

@@ -0,0 +1,20 @@
+/**
+ * The transfer pipeline: negotiate → queue → bounded workers → verify → assemble.
+ *
+ * For the whole set of object ids it (1) fetches each manifest from the source,
+ * (2) negotiates the missing chunks against the destination in bounded batches,
+ * (3) streams each missing chunk through a concurrency- and byte-bounded pool,
+ * re-hashing it before it is written, and (4) registers each object's manifest
+ * at the destination once all its chunks are present.
+ */
+import type { ContentStore, ObjectDescriptor } from "@statewalker/webrun-content-store";
+import type { TransferEvent, TransferOptions } from "./types.js";
+/**
+ * Register `manifest` at `to`, so `to.read(id)` reconstructs the object. When
+ * `to` is a wire proxy it assembles server-side from chunks it already holds
+ * (only the manifest crosses the wire); otherwise it re-`put`s a local
+ * reassembly (cheap, same-process) and verifies the resulting id.
+ */
+export declare function registerManifest(to: ContentStore, manifest: ObjectDescriptor): Promise<void>;
+export declare function transfer(objectIds: string[], from: ContentStore, to: ContentStore, opts?: TransferOptions): AsyncGenerator<TransferEvent>;
+//# sourceMappingURL=transfer.d.ts.map

package/dist/transfer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transfer.d.ts","sourceRoot":"","sources":["../src/transfer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAY,YAAY,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAClG,OAAO,KAAK,EAIV,aAAa,EAEb,eAAe,EAChB,MAAM,YAAY,CAAC;AA+BpB;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,YAAY,EAChB,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,IAAI,CAAC,CAUf;AAkDD,wBAAuB,QAAQ,CAC7B,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,EAAE,YAAY,EAClB,EAAE,EAAE,YAAY,EAChB,IAAI,CAAC,EAAE,eAAe,GACrB,cAAc,CAAC,aAAa,CAAC,CA+D/B"}

package/dist/types.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Public contract for the resumable, chunk-aware content transfer.
+ *
+ * The mover copies the chunks a destination is MISSING from a source
+ * {@link ContentStore} to a destination store, over the domain-neutral
+ * `@statewalker/webrun-streams` {@link Duplex} substrate. It is symmetric:
+ * either endpoint may be a {@link remoteStore} proxy (upload = remote `to`,
+ * download = remote `from`). It knows nothing of git, LFS, Xet, or sync — it
+ * moves opaque chunks between two content stores and verifies them by re-hash.
+ */
+import type { ByteStream, ChunkId, ContentStore, ObjectDescriptor, ObjectId } from "@statewalker/webrun-content-store";
+export type { ByteStream, ChunkId, ContentStore, ObjectId };
+/** Bounds on the transfer pipeline. */
+export interface TransferLimits {
+    /** Max chunks fetched + verified + written concurrently. Default 4. */
+    concurrency?: number;
+    /** Ceiling on the bytes of in-flight (fetched-but-not-yet-written) chunks. Default 8 MiB. */
+    maxBufferedBytes?: number;
+    /** Max chunk ids per `hasChunks` negotiation request. Default 256. */
+    batchSize?: number;
+}
+/**
+ * Serializable resume point. `done` are chunk ids already delivered to the
+ * destination; `pending` is advisory (the still-missing set at checkpoint time).
+ * Resume re-verifies the source still holds every object before continuing.
+ */
+export interface TransferCheckpoint {
+    objectIds: ObjectId[];
+    done: ChunkId[];
+    pending: ChunkId[];
+}
+/** Injected identity — the SAME function the content stores use. Re-hash of a
+ * received chunk must equal its id, or the chunk is rejected and the transfer
+ * fails loudly. */
+export type HashContent = (bytes: ByteStream) => Promise<string>;
+/** Progress events streamed by {@link transfer}. */
+export type TransferEvent = {
+    type: "resumed";
+    done: number;
+} | {
+    type: "negotiated";
+    objects: number;
+    chunks: number;
+    missing: number;
+    protocol?: string;
+} | {
+    type: "chunk-sent";
+    chunkId: ChunkId;
+    size: number;
+} | {
+    type: "object-done";
+    objectId: ObjectId;
+};
+export interface TransferOptions {
+    limits?: TransferLimits;
+    checkpoint?: TransferCheckpoint;
+    hashContent?: HashContent;
+}
+/** Peer capabilities exchanged before transfer over a {@link remoteStore}. */
+export interface Capabilities {
+    protocol: string;
+}
+/**
+ * A {@link ContentStore} that can register an object's manifest from chunks it
+ * already holds — the "assemble" step, without shipping the object's bytes.
+ * A local store lacks it (transfer falls back to {@link ContentStore.put}); the
+ * wire {@link remoteStore} proxy provides it so assemble costs only the manifest.
+ */
+export interface AssemblingStore extends ContentStore {
+    putManifest(manifest: ObjectDescriptor): Promise<void>;
+    capabilities(): Promise<Capabilities>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EACV,UAAU,EACV,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,QAAQ,EACT,MAAM,mCAAmC,CAAC;AAE3C,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;AAE5D,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6FAA6F;IAC7F,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,IAAI,EAAE,OAAO,EAAE,CAAC;IAChB,OAAO,EAAE,OAAO,EAAE,CAAC;CACpB;AAED;;mBAEmB;AACnB,MAAM,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAEjE,oDAAoD;AACpD,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3F;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACtD;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,QAAQ,EAAE,QAAQ,CAAA;CAAE,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAgB,SAAQ,YAAY;IACnD,WAAW,CAAC,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvD,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;CACvC"}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@statewalker/webrun-content-transfer",
+  "version": "0.2.2",
+  "description": "Resumable, chunk-aware content transfer over webrun-streams",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/statewalker/webrun-vcs.git",
+    "directory": "packages/webrun-content-transfer"
+  },
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@statewalker/webrun-files": "^0.10.0",
+    "@statewalker/webrun-streams": "^0.2.0",
+    "@statewalker/webrun-content-store": "^0.2.2"
+  },
+  "devDependencies": {
+    "@statewalker/webrun-files-mem": "^0.10.0",
+    "rolldown": "^1.2.12",
+    "vitest": "^5.0.3",
+    "typescript": "^7.0.2",
+    "@types/node": "^26.6.4",
+    "@statewalker/webrun-storage": "^0.2.1",
+    "@statewalker/webrun-files-sync": "^0.2.2"
+  },
+  "scripts": {
+    "build": "rm -rf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
+    "test": "vitest run",
+    "lint": "biome check --write .",
+    "lint:check": "biome check .",
+    "lint:fix": "biome check --write --unsafe .",
+    "format": "biome format --write .",
+    "format:check": "biome format .",
+    "format:fix": "biome check --write --unsafe .",
+    "typecheck": "tsc --noEmit"
+  }
+}

package/src/chunk-transfer.ts

@@ -0,0 +1,48 @@
+/**
+ * files-sync interop: adapt content-store dedup transfer to the files-sync
+ * {@link https://github.com/statewalker/vcs `Transfer`} seam. The returned object
+ * is structurally a files-sync `Transfer` (kept structural so `src/` stays
+ * domain-neutral and never imports files-sync). For a copy/update it ingests the
+ * source file into the `local` store, moves the missing chunks to `remote` via
+ * {@link transfer}, then writes the reassembled bytes to the destination — so a
+ * file whose chunks the remote already holds costs nothing to move.
+ */
+
+import type { ContentStore } from "@statewalker/webrun-content-store";
+import type { FilesApi } from "@statewalker/webrun-files";
+import { transfer } from "./transfer.js";
+
+/** The subset of a files-sync `SyncAction` this adapter reads (structural). */
+export interface FileSyncAction {
+  kind: string;
+  path?: string;
+}
+
+/** Structurally compatible with files-sync's `Transfer`. */
+export interface FileTransfer {
+  run(action: FileSyncAction, from: FilesApi, to: FilesApi): Promise<void>;
+}
+
+export function chunkTransfer(local: ContentStore, remote: ContentStore): FileTransfer {
+  return {
+    async run(action, from, to) {
+      if (action.kind !== "copy" && action.kind !== "update") {
+        throw new Error(
+          `content-transfer: chunkTransfer only handles copy/update, got ${action.kind}`,
+        );
+      }
+      const path = action.path;
+      if (!path) throw new Error(`content-transfer: ${action.kind} action has no path`);
+
+      const descriptor = await local.put(from.read(path));
+
+      const events = transfer([descriptor.id], local, remote);
+      const iterator = events[Symbol.asyncIterator]();
+      while (!(await iterator.next()).done) {
+        /* drain the transfer to completion */
+      }
+
+      await to.write(path, remote.read(descriptor.id));
+    },
+  };
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+/**
+ * @statewalker/webrun-content-transfer
+ *
+ * Resumable, chunk-aware byte mover. Given the object ids a destination needs,
+ * it transfers only the chunks the destination is MISSING from a source
+ * {@link ContentStore} to a destination store, over the domain-neutral
+ * `@statewalker/webrun-streams` {@link Duplex} substrate. It is symmetric —
+ * either endpoint may be a wire {@link remoteStore} proxy in front of a
+ * {@link serveStore} handler. Every received chunk is verified by re-hash before
+ * it is written; a serializable checkpoint makes an interrupted transfer
+ * resumable. It knows nothing of git, LFS, or sync.
+ */
+
+export type { FileSyncAction, FileTransfer } from "./chunk-transfer.js";
+export { chunkTransfer } from "./chunk-transfer.js";
+export { remoteStore } from "./remote-store.js";
+export { serveStore } from "./serve-store.js";
+export { transfer } from "./transfer.js";
+export type {
+  AssemblingStore,
+  Capabilities,
+  HashContent,
+  TransferCheckpoint,
+  TransferEvent,
+  TransferLimits,
+  TransferOptions,
+} from "./types.js";

package/src/protocol.ts

@@ -0,0 +1,89 @@
+/**
+ * Wire framing for the content-transfer RPC. One {@link Duplex} call carries one
+ * request→response. Every message is `[4-byte big-endian header length][header
+ * JSON][body bytes...]`: the JSON header names the op and its scalar args, and
+ * the remaining bytes are the streamed body (a chunk's bytes, or none).
+ *
+ * Small op surface, one per content-store transfer primitive:
+ *   - `capabilities`  → protocol handshake (BATCHED handshake, no body)
+ *   - `getManifest`   → object recipe (needed for the download direction)
+ *   - `hasChunks`     → BATCHED: many ids per request → the missing ids
+ *   - `getChunk`      → STREAMED: response body is the chunk bytes
+ *   - `putChunk`      → STREAMED: request body is the chunk bytes
+ *   - `putManifest`   → server-side assemble from already-present chunks
+ *
+ * (A Bloom prefilter for `hasChunks` is intentionally deferred — the plain
+ * batched id list is enough; add it only when a payload budget demands it.)
+ */
+
+import type { ChunkId, ObjectDescriptor, ObjectId } from "@statewalker/webrun-content-store";
+
+export type Request =
+  | { op: "capabilities" }
+  | { op: "getManifest"; id: ObjectId }
+  | { op: "hasChunks"; ids: ChunkId[] }
+  | { op: "getChunk"; id: ChunkId }
+  | { op: "putChunk"; id: ChunkId }
+  | { op: "putManifest"; manifest: ObjectDescriptor };
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+/** Encode `[len][header][body?]` as a byte stream. */
+export async function* frameMessage(
+  header: unknown,
+  body?: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+): AsyncGenerator<Uint8Array> {
+  const json = encoder.encode(JSON.stringify(header));
+  const prefix = new Uint8Array(4);
+  prefix[0] = (json.length >>> 24) & 0xff;
+  prefix[1] = (json.length >>> 16) & 0xff;
+  prefix[2] = (json.length >>> 8) & 0xff;
+  prefix[3] = json.length & 0xff;
+  yield prefix;
+  yield json;
+  if (body) for await (const b of body) if (b.byteLength) yield b;
+}
+
+/** Read one framed message: its parsed header + a body byte stream. Accepts
+ * either side of a {@link Duplex} (sync or async byte iterable). */
+export async function readFrame<H>(
+  input: AsyncIterable<Uint8Array> | Iterable<Uint8Array>,
+): Promise<{ header: H; body: AsyncGenerator<Uint8Array> }> {
+  const iter: AsyncIterator<Uint8Array> =
+    Symbol.asyncIterator in input
+      ? input[Symbol.asyncIterator]()
+      : (() => {
+          const sync = input[Symbol.iterator]();
+          return { next: async () => sync.next() };
+        })();
+  let buf = new Uint8Array(0);
+
+  async function fill(n: number): Promise<void> {
+    while (buf.length < n) {
+      const { value, done } = await iter.next();
+      if (done) throw new Error("content-transfer protocol: unexpected end of stream");
+      const next = new Uint8Array(buf.length + value.length);
+      next.set(buf, 0);
+      next.set(value, buf.length);
+      buf = next;
+    }
+  }
+
+  await fill(4);
+  const headerLen = ((buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]) >>> 0;
+  await fill(4 + headerLen);
+  const header = JSON.parse(decoder.decode(buf.subarray(4, 4 + headerLen))) as H;
+  const rest = buf.subarray(4 + headerLen);
+
+  async function* body(): AsyncGenerator<Uint8Array> {
+    if (rest.byteLength) yield rest;
+    while (true) {
+      const { value, done } = await iter.next();
+      if (done) return;
+      if (value.byteLength) yield value;
+    }
+  }
+
+  return { header, body: body() };
+}

package/src/remote-store.ts

@@ -0,0 +1,89 @@
+/**
+ * Client half of the wire protocol: wraps a {@link Duplex} as a
+ * {@link ContentStore}-shaped proxy. Each method opens one call, frames the
+ * request, and reads the framed response. The proxy exposes exactly the
+ * chunk-transfer surface a {@link transfer} needs from a remote peer —
+ * `capabilities` / `getManifest` / `hasChunks` / `getChunk` / `putChunk`, plus
+ * `putManifest` for the server-side assemble. Whole-object operations that would
+ * need the store's chunker (`put`) or have no transfer meaning (`has` / `read` /
+ * `remove` / `gc`) are not carried over the wire and throw.
+ */
+
+import type {
+  ByteStream,
+  ChunkId,
+  ObjectDescriptor,
+  ObjectId,
+} from "@statewalker/webrun-content-store";
+import type { Duplex } from "@statewalker/webrun-streams";
+import { frameMessage, readFrame } from "./protocol.js";
+import type { AssemblingStore, Capabilities } from "./types.js";
+
+interface Ok {
+  ok: true;
+  manifest?: ObjectDescriptor | null;
+  missing?: ChunkId[];
+  capabilities?: Capabilities;
+}
+interface Err {
+  ok: false;
+  error?: string;
+}
+type Response = Ok | Err;
+
+const unsupported = (name: string): never => {
+  throw new Error(`content-transfer: remote proxy does not support "${name}"`);
+};
+
+export function remoteStore(call: Duplex): AssemblingStore {
+  async function request(
+    header: unknown,
+    body?: ByteStream,
+  ): Promise<{ header: Ok; body: AsyncGenerator<Uint8Array> }> {
+    const response = call(frameMessage(header, body));
+    const { header: resp, body: respBody } = await readFrame<Response>(response);
+    if (!resp.ok) throw new Error(resp.error ?? "content-transfer: remote error");
+    return { header: resp, body: respBody };
+  }
+
+  const store: AssemblingStore = {
+    async capabilities(): Promise<Capabilities> {
+      const { header } = await request({ op: "capabilities" });
+      if (!header.capabilities) throw new Error("content-transfer: no capabilities in response");
+      return header.capabilities;
+    },
+
+    async getManifest(id: ObjectId): Promise<ObjectDescriptor | undefined> {
+      const { header } = await request({ op: "getManifest", id });
+      return header.manifest ?? undefined;
+    },
+
+    async hasChunks(ids: ChunkId[]): Promise<ChunkId[]> {
+      const { header } = await request({ op: "hasChunks", ids });
+      return header.missing ?? [];
+    },
+
+    getChunk(id: ChunkId): ByteStream {
+      return (async function* (): ByteStream {
+        const { body } = await request({ op: "getChunk", id });
+        yield* body;
+      })();
+    },
+
+    async putChunk(id: ChunkId, bytes: ByteStream): Promise<void> {
+      await request({ op: "putChunk", id }, bytes);
+    },
+
+    async putManifest(manifest: ObjectDescriptor): Promise<void> {
+      await request({ op: "putManifest", manifest });
+    },
+
+    put: () => unsupported("put"),
+    has: () => unsupported("has"),
+    read: () => unsupported("read"),
+    remove: () => unsupported("remove"),
+    gc: () => unsupported("gc"),
+  };
+
+  return store;
+}

package/src/serve-store.ts

@@ -0,0 +1,51 @@
+/**
+ * Server half of the wire protocol: turns a {@link ContentStore} into a
+ * {@link Duplex} handler. Each incoming call is one framed request; the handler
+ * dispatches it to the store and frames the response. Symmetric with
+ * {@link remoteStore} — an in-process test wires this Duplex straight into
+ * `remoteStore(...)` with no transport.
+ */
+
+import type { ContentStore } from "@statewalker/webrun-content-store";
+import type { Duplex } from "@statewalker/webrun-streams";
+import { frameMessage, type Request, readFrame } from "./protocol.js";
+import { registerManifest } from "./transfer.js";
+
+const PROTOCOL = "content-transfer/1";
+
+export function serveStore(store: ContentStore): Duplex {
+  return async function* handler(input): AsyncGenerator<Uint8Array> {
+    const { header, body } = await readFrame<Request>(input);
+    switch (header.op) {
+      case "capabilities":
+        yield* frameMessage({ ok: true, capabilities: { protocol: PROTOCOL } });
+        return;
+      case "getManifest": {
+        const manifest = (await store.getManifest(header.id)) ?? null;
+        yield* frameMessage({ ok: true, manifest });
+        return;
+      }
+      case "hasChunks": {
+        const missing = await store.hasChunks(header.ids);
+        yield* frameMessage({ ok: true, missing });
+        return;
+      }
+      case "getChunk":
+        yield* frameMessage({ ok: true }, store.getChunk(header.id));
+        return;
+      case "putChunk": {
+        await store.putChunk(header.id, body);
+        yield* frameMessage({ ok: true });
+        return;
+      }
+      case "putManifest": {
+        await registerManifest(store, header.manifest);
+        yield* frameMessage({ ok: true });
+        return;
+      }
+      default:
+        yield* frameMessage({ ok: false, error: `unknown op ${(header as { op: string }).op}` });
+        return;
+    }
+  };
+}

package/src/transfer.ts

@@ -0,0 +1,187 @@
+/**
+ * The transfer pipeline: negotiate → queue → bounded workers → verify → assemble.
+ *
+ * For the whole set of object ids it (1) fetches each manifest from the source,
+ * (2) negotiates the missing chunks against the destination in bounded batches,
+ * (3) streams each missing chunk through a concurrency- and byte-bounded pool,
+ * re-hashing it before it is written, and (4) registers each object's manifest
+ * at the destination once all its chunks are present.
+ */
+
+import type { ChunkRef, ContentStore, ObjectDescriptor } from "@statewalker/webrun-content-store";
+import type {
+  AssemblingStore,
+  ByteStream,
+  Capabilities,
+  TransferEvent,
+  TransferLimits,
+  TransferOptions,
+} from "./types.js";
+
+const DEFAULT_LIMITS: Required<TransferLimits> = {
+  concurrency: 4,
+  maxBufferedBytes: 8 * 1024 * 1024,
+  batchSize: 256,
+};
+
+/** A fresh single-shot stream over one buffer (streams are consumed once). */
+async function* one(bytes: Uint8Array): ByteStream {
+  yield bytes;
+}
+
+async function collect(stream: ByteStream): Promise<Uint8Array> {
+  const parts: Uint8Array[] = [];
+  for await (const chunk of stream) parts.push(chunk);
+  const total = parts.reduce((n, p) => n + p.length, 0);
+  const out = new Uint8Array(total);
+  let off = 0;
+  for (const p of parts) {
+    out.set(p, off);
+    off += p.length;
+  }
+  return out;
+}
+
+/** Re-stream a store's own chunks in manifest order — feeds `put` without buffering. */
+async function* reassembleFromStore(store: ContentStore, manifest: ObjectDescriptor): ByteStream {
+  for (const ref of manifest.chunks) yield* store.getChunk(ref.id);
+}
+
+/**
+ * Register `manifest` at `to`, so `to.read(id)` reconstructs the object. When
+ * `to` is a wire proxy it assembles server-side from chunks it already holds
+ * (only the manifest crosses the wire); otherwise it re-`put`s a local
+ * reassembly (cheap, same-process) and verifies the resulting id.
+ */
+export async function registerManifest(
+  to: ContentStore,
+  manifest: ObjectDescriptor,
+): Promise<void> {
+  const proxy = to as Partial<AssemblingStore>;
+  if (typeof proxy.putManifest === "function") {
+    await proxy.putManifest(manifest);
+    return;
+  }
+  const descriptor = await to.put(reassembleFromStore(to, manifest));
+  if (descriptor.id !== manifest.id) {
+    throw new Error(`content-transfer: assemble id mismatch ${descriptor.id} != ${manifest.id}`);
+  }
+}
+
+async function maybeCapabilities(to: ContentStore): Promise<Capabilities | undefined> {
+  const proxy = to as Partial<AssemblingStore>;
+  if (typeof proxy.capabilities === "function") return proxy.capabilities();
+  return undefined;
+}
+
+/**
+ * Run `work` over `refs` with at most `concurrency` in flight and never more
+ * than `maxBufferedBytes` of chunk bytes buffered (a single over-budget chunk is
+ * allowed alone, to guarantee progress). Yields each result as it settles.
+ */
+async function* pipeline<R>(
+  refs: ChunkRef[],
+  limits: Required<TransferLimits>,
+  work: (ref: ChunkRef) => Promise<R>,
+): AsyncGenerator<R> {
+  interface Entry {
+    ref: ChunkRef;
+    value: R;
+    promise: Promise<Entry>;
+  }
+  const inflight = new Set<Promise<Entry>>();
+  let buffered = 0;
+  let idx = 0;
+
+  const canLaunch = (): boolean =>
+    idx < refs.length &&
+    inflight.size < limits.concurrency &&
+    (inflight.size === 0 || buffered + refs[idx].size <= limits.maxBufferedBytes);
+
+  while (idx < refs.length || inflight.size > 0) {
+    while (canLaunch()) {
+      const ref = refs[idx++];
+      buffered += ref.size;
+      const entry = { ref } as Entry;
+      entry.promise = work(ref).then((value) => {
+        entry.value = value;
+        return entry;
+      });
+      inflight.add(entry.promise);
+    }
+    const settled = await Promise.race(inflight);
+    inflight.delete(settled.promise);
+    buffered -= settled.ref.size;
+    yield settled.value;
+  }
+}
+
+export async function* transfer(
+  objectIds: string[],
+  from: ContentStore,
+  to: ContentStore,
+  opts?: TransferOptions,
+): AsyncGenerator<TransferEvent> {
+  const limits: Required<TransferLimits> = { ...DEFAULT_LIMITS, ...opts?.limits };
+  const hashContent = opts?.hashContent;
+
+  const caps = await maybeCapabilities(to);
+
+  if (opts?.checkpoint) {
+    yield { type: "resumed", done: opts.checkpoint.done.length };
+  }
+
+  // 1. Gather manifests + the union of their chunk refs. This also re-verifies
+  //    the source still holds every requested object (a resume precondition).
+  const manifests = new Map<string, ObjectDescriptor>();
+  const refById = new Map<string, ChunkRef>();
+  for (const id of objectIds) {
+    const manifest = await from.getManifest(id);
+    if (!manifest) throw new Error(`content-transfer: source is missing object ${id}`);
+    manifests.set(id, manifest);
+    for (const ref of manifest.chunks) if (!refById.has(ref.id)) refById.set(ref.id, ref);
+  }
+
+  // 2. Batched negotiation: many ids per request, not one request per id.
+  const allIds = [...refById.keys()];
+  const missing: ChunkRef[] = [];
+  for (let i = 0; i < allIds.length; i += limits.batchSize) {
+    const batch = allIds.slice(i, i + limits.batchSize);
+    for (const cid of await to.hasChunks(batch)) {
+      const ref = refById.get(cid);
+      if (ref) missing.push(ref);
+    }
+  }
+  yield {
+    type: "negotiated",
+    objects: objectIds.length,
+    chunks: allIds.length,
+    missing: missing.length,
+    protocol: caps?.protocol,
+  };
+
+  // 3. Bounded pipeline: fetch from source, re-hash, write to destination.
+  for await (const sent of pipeline(missing, limits, async (ref) => {
+    const bytes = await collect(from.getChunk(ref.id));
+    if (hashContent) {
+      const actual = await hashContent(one(bytes));
+      if (actual !== ref.id) {
+        throw new Error(
+          `content-transfer: integrity check failed for chunk ${ref.id} (re-hash ${actual})`,
+        );
+      }
+    }
+    await to.putChunk(ref.id, one(bytes));
+    return { id: ref.id, size: bytes.length };
+  })) {
+    yield { type: "chunk-sent", chunkId: sent.id, size: sent.size };
+  }
+
+  // 4. Assemble: every object's chunks are present — register its manifest.
+  for (const id of objectIds) {
+    const manifest = manifests.get(id);
+    if (!manifest) continue;
+    await registerManifest(to, manifest);
+    yield { type: "object-done", objectId: id };
+  }
+}

package/src/types.ts

@@ -0,0 +1,75 @@
+/**
+ * Public contract for the resumable, chunk-aware content transfer.
+ *
+ * The mover copies the chunks a destination is MISSING from a source
+ * {@link ContentStore} to a destination store, over the domain-neutral
+ * `@statewalker/webrun-streams` {@link Duplex} substrate. It is symmetric:
+ * either endpoint may be a {@link remoteStore} proxy (upload = remote `to`,
+ * download = remote `from`). It knows nothing of git, LFS, Xet, or sync — it
+ * moves opaque chunks between two content stores and verifies them by re-hash.
+ */
+
+import type {
+  ByteStream,
+  ChunkId,
+  ContentStore,
+  ObjectDescriptor,
+  ObjectId,
+} from "@statewalker/webrun-content-store";
+
+export type { ByteStream, ChunkId, ContentStore, ObjectId };
+
+/** Bounds on the transfer pipeline. */
+export interface TransferLimits {
+  /** Max chunks fetched + verified + written concurrently. Default 4. */
+  concurrency?: number;
+  /** Ceiling on the bytes of in-flight (fetched-but-not-yet-written) chunks. Default 8 MiB. */
+  maxBufferedBytes?: number;
+  /** Max chunk ids per `hasChunks` negotiation request. Default 256. */
+  batchSize?: number;
+}
+
+/**
+ * Serializable resume point. `done` are chunk ids already delivered to the
+ * destination; `pending` is advisory (the still-missing set at checkpoint time).
+ * Resume re-verifies the source still holds every object before continuing.
+ */
+export interface TransferCheckpoint {
+  objectIds: ObjectId[];
+  done: ChunkId[];
+  pending: ChunkId[];
+}
+
+/** Injected identity — the SAME function the content stores use. Re-hash of a
+ * received chunk must equal its id, or the chunk is rejected and the transfer
+ * fails loudly. */
+export type HashContent = (bytes: ByteStream) => Promise<string>;
+
+/** Progress events streamed by {@link transfer}. */
+export type TransferEvent =
+  | { type: "resumed"; done: number }
+  | { type: "negotiated"; objects: number; chunks: number; missing: number; protocol?: string }
+  | { type: "chunk-sent"; chunkId: ChunkId; size: number }
+  | { type: "object-done"; objectId: ObjectId };
+
+export interface TransferOptions {
+  limits?: TransferLimits;
+  checkpoint?: TransferCheckpoint;
+  hashContent?: HashContent;
+}
+
+/** Peer capabilities exchanged before transfer over a {@link remoteStore}. */
+export interface Capabilities {
+  protocol: string;
+}
+
+/**
+ * A {@link ContentStore} that can register an object's manifest from chunks it
+ * already holds — the "assemble" step, without shipping the object's bytes.
+ * A local store lacks it (transfer falls back to {@link ContentStore.put}); the
+ * wire {@link remoteStore} proxy provides it so assemble costs only the manifest.
+ */
+export interface AssemblingStore extends ContentStore {
+  putManifest(manifest: ObjectDescriptor): Promise<void>;
+  capabilities(): Promise<Capabilities>;
+}