cursedbelt 2.7.0 → 3.0.0

package/src/server/sync/engine.spec.ts

@@ -1,496 +0,0 @@
-/**
- * The engine drives TWO REAL stores through the REAL receiver over Hono's
- * `app.request` — no HTTP mocks (the notes/vault discipline). The app-side state is
- * a deliberately tiny LWW key/value store; the assertions are about the KIT's
- * seams: cursor discipline, exactly-once, version-skew stop-before, StopBatch
- * resumability, conflict-context plumbing.
- */
-import { describe, expect, test } from "bun:test";
-import { Database } from "bun:sqlite";
-import { Hono } from "hono";
-import { createOpLog } from "./opLog";
-import { capPageBytes, createHttpRemote, createSyncReceiver } from "./http";
-import { chunkByBytes, createSyncApplier, pushToRemote, runSync } from "./engine";
-import { StopBatch, type ApplyOne, type RemoteApi, type SyncOp } from "./types";
-
-interface Entry {
-	value: string;
-	ts: number;
-	origin: string;
-}
-
-/** One instance: an op log + a tiny LWW map + concurrency bookkeeping. */
-function makeInstance(opKinds: readonly string[] = ["kv.set"]) {
-	const db = new Database(":memory:");
-	const log = createOpLog(db);
-	const state = new Map<string, Entry>();
-	const lastLocalSeqByKey = new Map<string, number>();
-	let conflictCopies = 0;
-	let stopOnKey: string | null = null;
-
-	const applyOne: ApplyOne = (op, ctx) => {
-		if (op.kind !== "kv.set") return "ignored";
-		const { key, value } = op.payload as { key: string; value: string };
-		if (stopOnKey === key) throw new StopBatch(`deferred: ${key}`);
-		const incoming: Entry = { value, ts: op.ts, origin: op.origin };
-		const local = state.get(key);
-		if (!local) {
-			state.set(key, incoming);
-			return "applied";
-		}
-		const incomingWins =
-			incoming.ts !== local.ts ? incoming.ts > local.ts : incoming.origin > local.origin;
-		const concurrent =
-			(lastLocalSeqByKey.get(key) ?? 0) > ctx.peerReceivedUpTo && local.value !== value;
-		if (incomingWins) state.set(key, incoming);
-		if (concurrent) {
-			conflictCopies++;
-			return "conflict";
-		}
-		return incomingWins ? "applied" : "ignored";
-	};
-
-	const set = (key: string, value: string, ts?: number): SyncOp => {
-		const op = log.recordLocal("kv.set", { key, value }, ts);
-		state.set(key, { value, ts: op.ts, origin: op.origin });
-		lastLocalSeqByKey.set(key, op.seq);
-		return op;
-	};
-
-	const applyBatch = createSyncApplier(log, applyOne);
-	return {
-		db,
-		log,
-		state,
-		set,
-		applyOne,
-		applyBatch,
-		opKinds,
-		conflicts: () => conflictCopies,
-		deferKey: (key: string | null) => {
-			stopOnKey = key;
-		},
-	};
-}
-
-/** Mount a receiver for an instance and hand back a RemoteApi that dials it. */
-function link(
-	dialer: ReturnType<typeof makeInstance>,
-	receiver: ReturnType<typeof makeInstance>,
-	token = "tok",
-) {
-	const app = new Hono();
-	app.route(
-		"/api/sync",
-		createSyncReceiver({
-			log: receiver.log,
-			opKinds: receiver.opKinds,
-			verifyToken: (t) => t === token,
-			applyOne: receiver.applyOne,
-		}),
-	);
-	const remote: RemoteApi = createHttpRemote({
-		basePath: "http://peer/api/sync",
-		token,
-		selfId: dialer.log.instanceId(),
-		fetchImpl: ((input: RequestInfo | URL, init?: RequestInit) =>
-			app.request(input as string, init)) as typeof fetch,
-	});
-	return { app, remote };
-}
-
-const engineOpts = (i: ReturnType<typeof makeInstance>) => ({ log: i.log, opKinds: i.opKinds });
-
-describe("sync engine over the real receiver", () => {
-	test("round trip: both sides converge, cursors advance, second run is a no-op", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote } = link(mac, prod);
-
-		mac.set("a", "1");
-		mac.set("b", "2");
-		const s1 = await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(s1.pushed).toBe(2);
-		expect(prod.state.get("a")?.value).toBe("1");
-
-		// Writes born on the receiver come back on the next pull.
-		prod.set("c", "3");
-		const s2 = await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(s2.pulled).toBe(1);
-		expect(mac.state.get("c")?.value).toBe("3");
-
-		// Fully converged — nothing moves, nothing re-applies.
-		const s3 = await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(s3.pushed).toBe(0);
-		expect(s3.pulled).toBe(0);
-	});
-
-	test("exactly-once: a re-push after a lost cursor is `seen`, never re-applied", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote } = link(mac, prod);
-		mac.set("a", "1");
-		await runSync(engineOpts(mac), remote, mac.applyBatch);
-		// Simulate a lost push cursor (crash before persist) — the ledger dedupes.
-		mac.log.setCursor(prod.log.instanceId(), "push", 0);
-		const report = await pushToRemote(engineOpts(mac), remote);
-		expect(report.pushed).toBe(0); // applied nothing new
-		expect(prod.state.get("a")?.value).toBe("1");
-	});
-
-	test("wrong token is a uniform 401", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote } = link(mac, prod, "right");
-		void remote;
-		const { remote: bad } = (() => {
-			const app = new Hono();
-			app.route(
-				"/api/sync",
-				createSyncReceiver({
-					log: prod.log,
-					opKinds: prod.opKinds,
-					verifyToken: (t) => t === "right",
-					applyOne: prod.applyOne,
-				}),
-			);
-			return {
-				remote: createHttpRemote({
-					basePath: "http://peer/api/sync",
-					token: "wrong",
-					selfId: mac.log.instanceId(),
-					fetchImpl: ((input: RequestInfo | URL, init?: RequestInit) =>
-						app.request(input as string, init)) as typeof fetch,
-				}),
-			};
-		})();
-		mac.set("a", "1");
-		await expect(runSync(engineOpts(mac), bad, mac.applyBatch)).rejects.toThrow("401");
-	});
-
-	test("version skew: push stops BEFORE the first unknown kind, resumes after upgrade", async () => {
-		const mac = makeInstance(["kv.set", "kv.fancy"]);
-		const oldProd = makeInstance(["kv.set"]); // an older build: no kv.fancy
-		const { remote: oldRemote } = link(mac, oldProd);
-
-		mac.set("a", "1");
-		mac.log.recordLocal("kv.fancy", { key: "f" });
-		mac.set("b", "2"); // born AFTER the blocked op — must NOT leapfrog it
-
-		const r1 = await pushToRemote(engineOpts(mac), oldRemote);
-		expect(r1.pushed).toBe(1); // only "a"
-		expect(r1.blockedBy).toBe("kv.fancy");
-		expect(oldProd.state.has("b")).toBe(false);
-
-		// A second run against the old peer stays paused at the same op, sends nothing.
-		const r2 = await pushToRemote(engineOpts(mac), oldRemote);
-		expect(r2.pushed).toBe(0);
-		expect(r2.blockedBy).toBe("kv.fancy");
-
-		// "Deploy" the peer: same store, now advertising the new kind.
-		const upgraded = { ...oldProd, opKinds: ["kv.set", "kv.fancy"] as const };
-		const { remote: newRemote } = link(mac, upgraded as unknown as ReturnType<typeof makeInstance>);
-		const r3 = await pushToRemote(engineOpts(mac), newRemote);
-		expect(r3.blockedBy).toBeUndefined();
-		expect(oldProd.state.get("b")?.value).toBe("2"); // nothing was lost or skipped
-	});
-
-	test("peerRefusal blocks FINER than opKinds, with the same stop-before-it contract", async () => {
-		// notes' metaKeys case in miniature: the kind is one the peer has always known,
-		// so `opKinds` cannot see the incompatibility — it lives in the payload.
-		const mac = makeInstance();
-		const oldProd = makeInstance();
-		const { remote } = link(mac, oldProd);
-		const acceptedKeys = new Set(["known"]);
-		const opts = {
-			log: mac.log,
-			opKinds: mac.opKinds,
-			peerRefusal: (op: SyncOp) => {
-				const { key } = op.payload as { key: string };
-				return acceptedKeys.has(key) ? null : `kv.set ${key}`;
-			},
-		};
-
-		// Explicit ts: the LWW tiebreak is by (ts, origin), and two same-origin writes
-		// in the same millisecond would tie — a test artifact, not the thing under test.
-		mac.set("known", "1", 1000);
-		mac.set("newish", "2", 2000);
-		mac.set("known", "3", 3000); // born AFTER the refused op — must NOT leapfrog it
-
-		const r1 = await pushToRemote(opts, remote);
-		expect(r1.pushed).toBe(1);
-		expect(r1.blockedBy).toBe("kv.set newish"); // the REASON, not just the kind
-		expect(oldProd.state.get("known")?.value).toBe("1");
-
-		// Paused, not dropped: a second run against the same peer sends nothing more.
-		expect((await pushToRemote(opts, remote)).pushed).toBe(0);
-
-		// "Deploy" the peer: it now accepts the key, and the push resumes from exactly
-		// the op it stopped in front of.
-		acceptedKeys.add("newish");
-		const r3 = await pushToRemote(opts, remote);
-		expect(r3.blockedBy).toBeUndefined();
-		expect(r3.pushed).toBe(2);
-		expect(oldProd.state.get("newish")?.value).toBe("2");
-		expect(oldProd.state.get("known")?.value).toBe("3");
-	});
-
-	test("StopBatch: pull stops at the deferred op, cursor holds, retry resumes there", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote } = link(mac, prod);
-		prod.set("x", "1");
-		prod.set("y", "2");
-		prod.set("z", "3");
-
-		mac.deferKey("y");
-		const s1 = await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(s1.pulled).toBe(1); // only x landed
-		expect(mac.state.has("y")).toBe(false);
-		expect(mac.state.has("z")).toBe(false);
-
-		mac.deferKey(null); // the world changed; the deferral clears
-		const s2 = await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(s2.pulled).toBe(2); // y then z — resumed exactly at the failure
-		expect(mac.state.get("z")?.value).toBe("3");
-	});
-
-	test("true concurrent edit surfaces via the peerReceivedUpTo bound", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote } = link(mac, prod);
-		mac.set("k", "base", 1000);
-		await runSync(engineOpts(mac), remote, mac.applyBatch);
-
-		// Both sides edit while apart. Prod's is newer, so it wins everywhere; the
-		// mac's applier sees its own unacked local edit and reports the conflict.
-		mac.set("k", "mac-edit", 2000);
-		prod.set("k", "prod-edit", 3000);
-		const s = await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(s.conflicts).toBe(1);
-		expect(mac.state.get("k")?.value).toBe("prod-edit");
-		// And after one more round both sides are byte-identical.
-		await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(prod.state.get("k")?.value).toBe("prod-edit");
-	});
-
-	test("export policy holds ops back and counts them", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote } = link(mac, prod);
-		mac.set("public", "yes");
-		mac.set("secret", "no");
-		const summary = await runSync(
-			{
-				...engineOpts(mac),
-				policy: (op) =>
-					((op.payload as { key: string }).key ?? "").startsWith("secret") ? null : op,
-			},
-			remote,
-			mac.applyBatch,
-		);
-		expect(summary.held).toBe(1);
-		expect(prod.state.has("public")).toBe(true);
-		expect(prod.state.has("secret")).toBe(false);
-		// Held is held FOREVER (cursor passed it) — the policy is a boundary, not a delay.
-		const again = await runSync(engineOpts(mac), remote, mac.applyBatch);
-		expect(prod.state.has("secret")).toBe(false);
-		expect(again.pushed).toBe(0);
-	});
-});
-
-describe("byte-capped push batches (the 413 wedge)", () => {
-	test("a backlog larger than the cap ships as several requests and converges", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const pushes: number[] = [];
-		const { remote } = (() => {
-			const linked = link(mac, prod);
-			const inner = linked.remote;
-			return {
-				remote: {
-					...inner,
-					push: (payload: { ops: SyncOp[]; peerHas: number }) => {
-						pushes.push(JSON.stringify(payload.ops).length);
-						return inner.push(payload);
-					},
-				} as RemoteApi,
-			};
-		})();
-		// ~40 KB per op, 12 ops ≈ 480 KB backlog against a 100 KB cap.
-		for (let i = 0; i < 12; i++) mac.set(`k${i}`, "x".repeat(40_000));
-		const summary = await runSync(
-			{ ...engineOpts(mac), maxPushBytes: 100_000 },
-			remote,
-			mac.applyBatch,
-		);
-		expect(summary.pushed).toBe(12);
-		expect(pushes.length).toBeGreaterThanOrEqual(5);
-		for (const bytes of pushes) expect(bytes).toBeLessThan(120_000);
-		expect(prod.state.get("k11")?.value.length).toBe(40_000);
-	});
-
-	test("chunkByBytes: a single oversized op still travels alone", () => {
-		const big = {
-			seq: 1,
-			origin: "o",
-			originSeq: 1,
-			ts: 1,
-			kind: "k",
-			payload: "y".repeat(50_000),
-		};
-		const small = { seq: 2, origin: "o", originSeq: 2, ts: 1, kind: "k", payload: "z" };
-		const chunks = chunkByBytes([big as SyncOp, small as SyncOp], 10_000);
-		expect(chunks.length).toBe(2);
-		expect(chunks[0]).toHaveLength(1);
-	});
-});
-
-describe("byte-capped PULL pages (the OOM the push cap's mirror was missing)", () => {
-	// 🔴 Measured against live prod on 2026-08-01, while proving station's sync
-	// link: `GET /pull?after=0` over a 709-op log served 37 MB at 309 ops and then
-	// OOM-killed the unit (memoryMax=300M, t3.micro). `SYNC_PAGE` is a COUNT cap
-	// and station's ops are not small — a 130 KB overview snapshot, a 68 KB fleet
-	// CSV — so 500 of them is tens of megabytes in one response buffer.
-	//
-	// It does not converge on its own, which is what makes it a wedge rather than
-	// a slow request: the unit dies, systemd restarts it, the dialer retries the
-	// identical oversized pull. Push learned this from a 413; the receiving half
-	// kept only the count cap because a caught-up peer never asks for enough rows.
-
-	test("a huge backlog is served as several bounded pages, and still converges", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		// 60 ops × ~100 KB ≈ 6 MB, well under SYNC_PAGE's 500 — the count cap alone
-		// would hand all of it over in one response.
-		for (let i = 0; i < 60; i++) prod.set(`big${i}`, "z".repeat(100_000));
-
-		const { remote } = link(mac, prod);
-		const pages: number[] = [];
-		const counted: RemoteApi = {
-			...remote,
-			pull: async (after: number, self: string) => {
-				const page = await remote.pull(after, self);
-				pages.push(JSON.stringify(page.ops).length);
-				return page;
-			},
-		};
-
-		const summary = await runSync(engineOpts(mac), counted, mac.applyBatch);
-
-		expect(summary.pulled).toBe(60);
-		expect(pages.length).toBeGreaterThan(1); // it was ONE page before the cap
-		for (const bytes of pages) expect(bytes).toBeLessThan(5_000_000);
-		// Convergence is the point: bounding the page must not lose an op.
-		expect(mac.state.get("big59")?.value.length).toBe(100_000);
-		expect(mac.state.get("big0")?.value.length).toBe(100_000);
-	});
-
-	test("capPageBytes keeps a PREFIX, so the cursor never skips an op", () => {
-		const ops = Array.from({ length: 5 }, (_, i) => ({
-			seq: i + 1,
-			origin: "o",
-			originSeq: i + 1,
-			ts: i,
-			kind: "doc.set",
-			payload: "q".repeat(1_000),
-		}));
-		const { ops: kept, truncated } = capPageBytes(ops, 2_500);
-		expect(truncated).toBe(true);
-		// A prefix — dropping from the middle would advance the peer past rows it
-		// never received, and no later pull offers them again.
-		expect(kept.map((o) => o.seq)).toEqual([1, 2]);
-	});
-
-	test("a single op over the budget still travels alone", () => {
-		// Refusing it would wedge the log on that op permanently, which is strictly
-		// worse than one oversized response. Same rule as chunkByBytes.
-		const ops = [
-			{ seq: 1, origin: "o", originSeq: 1, ts: 1, kind: "k", payload: "y".repeat(9_000) },
-		];
-		const { ops: kept, truncated } = capPageBytes(ops, 1_000);
-		expect(kept.length).toBe(1);
-		expect(truncated).toBe(false);
-	});
-});
-
-describe("receiver: per-batch state and post-commit work", () => {
-	/** Mount a receiver with the batch seams and dial it. */
-	function linkWithBatchSeams(
-		dialer: ReturnType<typeof makeInstance>,
-		receiver: ReturnType<typeof makeInstance>,
-	) {
-		const batches: number[] = [];
-		const finished: number[] = [];
-		let seenThisBatch = 0;
-		const app = new Hono();
-		app.route(
-			"/api/sync",
-			createSyncReceiver({
-				log: receiver.log,
-				opKinds: receiver.opKinds,
-				verifyToken: () => true,
-				beginBatch: () => {
-					seenThisBatch = 0;
-					batches.push(0);
-					return (op, ctx) => {
-						seenThisBatch++;
-						batches[batches.length - 1] = seenThisBatch;
-						return receiver.applyOne(op, ctx);
-					};
-				},
-				afterBatch: async (report) => {
-					// Async on purpose: the response must not go out before it resolves.
-					await Promise.resolve();
-					finished.push(report.applied);
-				},
-			}),
-		);
-		const remote: RemoteApi = createHttpRemote({
-			basePath: "http://peer/api/sync",
-			token: "tok",
-			selfId: dialer.log.instanceId(),
-			fetchImpl: ((input: RequestInfo | URL, init?: RequestInit) =>
-				app.request(input as string, init)) as typeof fetch,
-		});
-		return { remote, batches, finished };
-	}
-
-	test("beginBatch is called once per push, so two pushes never share a counter", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote, batches } = linkWithBatchSeams(mac, prod);
-
-		mac.set("a", "1", 1000);
-		await pushToRemote(engineOpts(mac), remote);
-		mac.set("b", "2", 2000);
-		await pushToRemote(engineOpts(mac), remote);
-
-		// Two batches of one — not one batch that saw two, and not a counter that
-		// carried the first push's op into the second's report.
-		expect(batches).toEqual([1, 1]);
-	});
-
-	test("afterBatch is AWAITED before the response — the cursor must not outrun the work", async () => {
-		const mac = makeInstance();
-		const prod = makeInstance();
-		const { remote, finished } = linkWithBatchSeams(mac, prod);
-
-		mac.set("a", "1", 1000);
-		const report = await pushToRemote(engineOpts(mac), remote);
-
-		// By the time the dialer has its answer (and has advanced its push cursor),
-		// the post-commit work has already run.
-		expect(report.pushed).toBe(1);
-		expect(finished).toEqual([1]);
-	});
-
-	test("a receiver given neither applier — or both — refuses to be built", () => {
-		const prod = makeInstance();
-		const base = { log: prod.log, opKinds: prod.opKinds, verifyToken: () => true };
-		expect(() => createSyncReceiver({ ...base })).toThrow(/exactly one/);
-		expect(() =>
-			createSyncReceiver({ ...base, applyOne: prod.applyOne, beginBatch: () => prod.applyOne }),
-		).toThrow(/exactly one/);
-	});
-});