cursedbelt 2.6.1 → 2.7.0

package/src/server/sync/engine.spec.ts

@@ -0,0 +1,496 @@
+/**
+ * The engine drives TWO REAL stores through the REAL receiver over Hono's
+ * `app.request` — no HTTP mocks (the notes/vault discipline). The app-side state is
+ * a deliberately tiny LWW key/value store; the assertions are about the KIT's
+ * seams: cursor discipline, exactly-once, version-skew stop-before, StopBatch
+ * resumability, conflict-context plumbing.
+ */
+import { describe, expect, test } from "bun:test";
+import { Database } from "bun:sqlite";
+import { Hono } from "hono";
+import { createOpLog } from "./opLog";
+import { capPageBytes, createHttpRemote, createSyncReceiver } from "./http";
+import { chunkByBytes, createSyncApplier, pushToRemote, runSync } from "./engine";
+import { StopBatch, type ApplyOne, type RemoteApi, type SyncOp } from "./types";
+
+interface Entry {
+	value: string;
+	ts: number;
+	origin: string;
+}
+
+/** One instance: an op log + a tiny LWW map + concurrency bookkeeping. */
+function makeInstance(opKinds: readonly string[] = ["kv.set"]) {
+	const db = new Database(":memory:");
+	const log = createOpLog(db);
+	const state = new Map<string, Entry>();
+	const lastLocalSeqByKey = new Map<string, number>();
+	let conflictCopies = 0;
+	let stopOnKey: string | null = null;
+
+	const applyOne: ApplyOne = (op, ctx) => {
+		if (op.kind !== "kv.set") return "ignored";
+		const { key, value } = op.payload as { key: string; value: string };
+		if (stopOnKey === key) throw new StopBatch(`deferred: ${key}`);
+		const incoming: Entry = { value, ts: op.ts, origin: op.origin };
+		const local = state.get(key);
+		if (!local) {
+			state.set(key, incoming);
+			return "applied";
+		}
+		const incomingWins =
+			incoming.ts !== local.ts ? incoming.ts > local.ts : incoming.origin > local.origin;
+		const concurrent =
+			(lastLocalSeqByKey.get(key) ?? 0) > ctx.peerReceivedUpTo && local.value !== value;
+		if (incomingWins) state.set(key, incoming);
+		if (concurrent) {
+			conflictCopies++;
+			return "conflict";
+		}
+		return incomingWins ? "applied" : "ignored";
+	};
+
+	const set = (key: string, value: string, ts?: number): SyncOp => {
+		const op = log.recordLocal("kv.set", { key, value }, ts);
+		state.set(key, { value, ts: op.ts, origin: op.origin });
+		lastLocalSeqByKey.set(key, op.seq);
+		return op;
+	};
+
+	const applyBatch = createSyncApplier(log, applyOne);
+	return {
+		db,
+		log,
+		state,
+		set,
+		applyOne,
+		applyBatch,
+		opKinds,
+		conflicts: () => conflictCopies,
+		deferKey: (key: string | null) => {
+			stopOnKey = key;
+		},
+	};
+}
+
+/** Mount a receiver for an instance and hand back a RemoteApi that dials it. */
+function link(
+	dialer: ReturnType<typeof makeInstance>,
+	receiver: ReturnType<typeof makeInstance>,
+	token = "tok",
+) {
+	const app = new Hono();
+	app.route(
+		"/api/sync",
+		createSyncReceiver({
+			log: receiver.log,
+			opKinds: receiver.opKinds,
+			verifyToken: (t) => t === token,
+			applyOne: receiver.applyOne,
+		}),
+	);
+	const remote: RemoteApi = createHttpRemote({
+		basePath: "http://peer/api/sync",
+		token,
+		selfId: dialer.log.instanceId(),
+		fetchImpl: ((input: RequestInfo | URL, init?: RequestInit) =>
+			app.request(input as string, init)) as typeof fetch,
+	});
+	return { app, remote };
+}
+
+const engineOpts = (i: ReturnType<typeof makeInstance>) => ({ log: i.log, opKinds: i.opKinds });
+
+describe("sync engine over the real receiver", () => {
+	test("round trip: both sides converge, cursors advance, second run is a no-op", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote } = link(mac, prod);
+
+		mac.set("a", "1");
+		mac.set("b", "2");
+		const s1 = await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(s1.pushed).toBe(2);
+		expect(prod.state.get("a")?.value).toBe("1");
+
+		// Writes born on the receiver come back on the next pull.
+		prod.set("c", "3");
+		const s2 = await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(s2.pulled).toBe(1);
+		expect(mac.state.get("c")?.value).toBe("3");
+
+		// Fully converged — nothing moves, nothing re-applies.
+		const s3 = await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(s3.pushed).toBe(0);
+		expect(s3.pulled).toBe(0);
+	});
+
+	test("exactly-once: a re-push after a lost cursor is `seen`, never re-applied", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote } = link(mac, prod);
+		mac.set("a", "1");
+		await runSync(engineOpts(mac), remote, mac.applyBatch);
+		// Simulate a lost push cursor (crash before persist) — the ledger dedupes.
+		mac.log.setCursor(prod.log.instanceId(), "push", 0);
+		const report = await pushToRemote(engineOpts(mac), remote);
+		expect(report.pushed).toBe(0); // applied nothing new
+		expect(prod.state.get("a")?.value).toBe("1");
+	});
+
+	test("wrong token is a uniform 401", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote } = link(mac, prod, "right");
+		void remote;
+		const { remote: bad } = (() => {
+			const app = new Hono();
+			app.route(
+				"/api/sync",
+				createSyncReceiver({
+					log: prod.log,
+					opKinds: prod.opKinds,
+					verifyToken: (t) => t === "right",
+					applyOne: prod.applyOne,
+				}),
+			);
+			return {
+				remote: createHttpRemote({
+					basePath: "http://peer/api/sync",
+					token: "wrong",
+					selfId: mac.log.instanceId(),
+					fetchImpl: ((input: RequestInfo | URL, init?: RequestInit) =>
+						app.request(input as string, init)) as typeof fetch,
+				}),
+			};
+		})();
+		mac.set("a", "1");
+		await expect(runSync(engineOpts(mac), bad, mac.applyBatch)).rejects.toThrow("401");
+	});
+
+	test("version skew: push stops BEFORE the first unknown kind, resumes after upgrade", async () => {
+		const mac = makeInstance(["kv.set", "kv.fancy"]);
+		const oldProd = makeInstance(["kv.set"]); // an older build: no kv.fancy
+		const { remote: oldRemote } = link(mac, oldProd);
+
+		mac.set("a", "1");
+		mac.log.recordLocal("kv.fancy", { key: "f" });
+		mac.set("b", "2"); // born AFTER the blocked op — must NOT leapfrog it
+
+		const r1 = await pushToRemote(engineOpts(mac), oldRemote);
+		expect(r1.pushed).toBe(1); // only "a"
+		expect(r1.blockedBy).toBe("kv.fancy");
+		expect(oldProd.state.has("b")).toBe(false);
+
+		// A second run against the old peer stays paused at the same op, sends nothing.
+		const r2 = await pushToRemote(engineOpts(mac), oldRemote);
+		expect(r2.pushed).toBe(0);
+		expect(r2.blockedBy).toBe("kv.fancy");
+
+		// "Deploy" the peer: same store, now advertising the new kind.
+		const upgraded = { ...oldProd, opKinds: ["kv.set", "kv.fancy"] as const };
+		const { remote: newRemote } = link(mac, upgraded as unknown as ReturnType<typeof makeInstance>);
+		const r3 = await pushToRemote(engineOpts(mac), newRemote);
+		expect(r3.blockedBy).toBeUndefined();
+		expect(oldProd.state.get("b")?.value).toBe("2"); // nothing was lost or skipped
+	});
+
+	test("peerRefusal blocks FINER than opKinds, with the same stop-before-it contract", async () => {
+		// notes' metaKeys case in miniature: the kind is one the peer has always known,
+		// so `opKinds` cannot see the incompatibility — it lives in the payload.
+		const mac = makeInstance();
+		const oldProd = makeInstance();
+		const { remote } = link(mac, oldProd);
+		const acceptedKeys = new Set(["known"]);
+		const opts = {
+			log: mac.log,
+			opKinds: mac.opKinds,
+			peerRefusal: (op: SyncOp) => {
+				const { key } = op.payload as { key: string };
+				return acceptedKeys.has(key) ? null : `kv.set ${key}`;
+			},
+		};
+
+		// Explicit ts: the LWW tiebreak is by (ts, origin), and two same-origin writes
+		// in the same millisecond would tie — a test artifact, not the thing under test.
+		mac.set("known", "1", 1000);
+		mac.set("newish", "2", 2000);
+		mac.set("known", "3", 3000); // born AFTER the refused op — must NOT leapfrog it
+
+		const r1 = await pushToRemote(opts, remote);
+		expect(r1.pushed).toBe(1);
+		expect(r1.blockedBy).toBe("kv.set newish"); // the REASON, not just the kind
+		expect(oldProd.state.get("known")?.value).toBe("1");
+
+		// Paused, not dropped: a second run against the same peer sends nothing more.
+		expect((await pushToRemote(opts, remote)).pushed).toBe(0);
+
+		// "Deploy" the peer: it now accepts the key, and the push resumes from exactly
+		// the op it stopped in front of.
+		acceptedKeys.add("newish");
+		const r3 = await pushToRemote(opts, remote);
+		expect(r3.blockedBy).toBeUndefined();
+		expect(r3.pushed).toBe(2);
+		expect(oldProd.state.get("newish")?.value).toBe("2");
+		expect(oldProd.state.get("known")?.value).toBe("3");
+	});
+
+	test("StopBatch: pull stops at the deferred op, cursor holds, retry resumes there", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote } = link(mac, prod);
+		prod.set("x", "1");
+		prod.set("y", "2");
+		prod.set("z", "3");
+
+		mac.deferKey("y");
+		const s1 = await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(s1.pulled).toBe(1); // only x landed
+		expect(mac.state.has("y")).toBe(false);
+		expect(mac.state.has("z")).toBe(false);
+
+		mac.deferKey(null); // the world changed; the deferral clears
+		const s2 = await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(s2.pulled).toBe(2); // y then z — resumed exactly at the failure
+		expect(mac.state.get("z")?.value).toBe("3");
+	});
+
+	test("true concurrent edit surfaces via the peerReceivedUpTo bound", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote } = link(mac, prod);
+		mac.set("k", "base", 1000);
+		await runSync(engineOpts(mac), remote, mac.applyBatch);
+
+		// Both sides edit while apart. Prod's is newer, so it wins everywhere; the
+		// mac's applier sees its own unacked local edit and reports the conflict.
+		mac.set("k", "mac-edit", 2000);
+		prod.set("k", "prod-edit", 3000);
+		const s = await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(s.conflicts).toBe(1);
+		expect(mac.state.get("k")?.value).toBe("prod-edit");
+		// And after one more round both sides are byte-identical.
+		await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(prod.state.get("k")?.value).toBe("prod-edit");
+	});
+
+	test("export policy holds ops back and counts them", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote } = link(mac, prod);
+		mac.set("public", "yes");
+		mac.set("secret", "no");
+		const summary = await runSync(
+			{
+				...engineOpts(mac),
+				policy: (op) =>
+					((op.payload as { key: string }).key ?? "").startsWith("secret") ? null : op,
+			},
+			remote,
+			mac.applyBatch,
+		);
+		expect(summary.held).toBe(1);
+		expect(prod.state.has("public")).toBe(true);
+		expect(prod.state.has("secret")).toBe(false);
+		// Held is held FOREVER (cursor passed it) — the policy is a boundary, not a delay.
+		const again = await runSync(engineOpts(mac), remote, mac.applyBatch);
+		expect(prod.state.has("secret")).toBe(false);
+		expect(again.pushed).toBe(0);
+	});
+});
+
+describe("byte-capped push batches (the 413 wedge)", () => {
+	test("a backlog larger than the cap ships as several requests and converges", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const pushes: number[] = [];
+		const { remote } = (() => {
+			const linked = link(mac, prod);
+			const inner = linked.remote;
+			return {
+				remote: {
+					...inner,
+					push: (payload: { ops: SyncOp[]; peerHas: number }) => {
+						pushes.push(JSON.stringify(payload.ops).length);
+						return inner.push(payload);
+					},
+				} as RemoteApi,
+			};
+		})();
+		// ~40 KB per op, 12 ops ≈ 480 KB backlog against a 100 KB cap.
+		for (let i = 0; i < 12; i++) mac.set(`k${i}`, "x".repeat(40_000));
+		const summary = await runSync(
+			{ ...engineOpts(mac), maxPushBytes: 100_000 },
+			remote,
+			mac.applyBatch,
+		);
+		expect(summary.pushed).toBe(12);
+		expect(pushes.length).toBeGreaterThanOrEqual(5);
+		for (const bytes of pushes) expect(bytes).toBeLessThan(120_000);
+		expect(prod.state.get("k11")?.value.length).toBe(40_000);
+	});
+
+	test("chunkByBytes: a single oversized op still travels alone", () => {
+		const big = {
+			seq: 1,
+			origin: "o",
+			originSeq: 1,
+			ts: 1,
+			kind: "k",
+			payload: "y".repeat(50_000),
+		};
+		const small = { seq: 2, origin: "o", originSeq: 2, ts: 1, kind: "k", payload: "z" };
+		const chunks = chunkByBytes([big as SyncOp, small as SyncOp], 10_000);
+		expect(chunks.length).toBe(2);
+		expect(chunks[0]).toHaveLength(1);
+	});
+});
+
+describe("byte-capped PULL pages (the OOM the push cap's mirror was missing)", () => {
+	// 🔴 Measured against live prod on 2026-08-01, while proving station's sync
+	// link: `GET /pull?after=0` over a 709-op log served 37 MB at 309 ops and then
+	// OOM-killed the unit (memoryMax=300M, t3.micro). `SYNC_PAGE` is a COUNT cap
+	// and station's ops are not small — a 130 KB overview snapshot, a 68 KB fleet
+	// CSV — so 500 of them is tens of megabytes in one response buffer.
+	//
+	// It does not converge on its own, which is what makes it a wedge rather than
+	// a slow request: the unit dies, systemd restarts it, the dialer retries the
+	// identical oversized pull. Push learned this from a 413; the receiving half
+	// kept only the count cap because a caught-up peer never asks for enough rows.
+
+	test("a huge backlog is served as several bounded pages, and still converges", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		// 60 ops × ~100 KB ≈ 6 MB, well under SYNC_PAGE's 500 — the count cap alone
+		// would hand all of it over in one response.
+		for (let i = 0; i < 60; i++) prod.set(`big${i}`, "z".repeat(100_000));
+
+		const { remote } = link(mac, prod);
+		const pages: number[] = [];
+		const counted: RemoteApi = {
+			...remote,
+			pull: async (after: number, self: string) => {
+				const page = await remote.pull(after, self);
+				pages.push(JSON.stringify(page.ops).length);
+				return page;
+			},
+		};
+
+		const summary = await runSync(engineOpts(mac), counted, mac.applyBatch);
+
+		expect(summary.pulled).toBe(60);
+		expect(pages.length).toBeGreaterThan(1); // it was ONE page before the cap
+		for (const bytes of pages) expect(bytes).toBeLessThan(5_000_000);
+		// Convergence is the point: bounding the page must not lose an op.
+		expect(mac.state.get("big59")?.value.length).toBe(100_000);
+		expect(mac.state.get("big0")?.value.length).toBe(100_000);
+	});
+
+	test("capPageBytes keeps a PREFIX, so the cursor never skips an op", () => {
+		const ops = Array.from({ length: 5 }, (_, i) => ({
+			seq: i + 1,
+			origin: "o",
+			originSeq: i + 1,
+			ts: i,
+			kind: "doc.set",
+			payload: "q".repeat(1_000),
+		}));
+		const { ops: kept, truncated } = capPageBytes(ops, 2_500);
+		expect(truncated).toBe(true);
+		// A prefix — dropping from the middle would advance the peer past rows it
+		// never received, and no later pull offers them again.
+		expect(kept.map((o) => o.seq)).toEqual([1, 2]);
+	});
+
+	test("a single op over the budget still travels alone", () => {
+		// Refusing it would wedge the log on that op permanently, which is strictly
+		// worse than one oversized response. Same rule as chunkByBytes.
+		const ops = [
+			{ seq: 1, origin: "o", originSeq: 1, ts: 1, kind: "k", payload: "y".repeat(9_000) },
+		];
+		const { ops: kept, truncated } = capPageBytes(ops, 1_000);
+		expect(kept.length).toBe(1);
+		expect(truncated).toBe(false);
+	});
+});
+
+describe("receiver: per-batch state and post-commit work", () => {
+	/** Mount a receiver with the batch seams and dial it. */
+	function linkWithBatchSeams(
+		dialer: ReturnType<typeof makeInstance>,
+		receiver: ReturnType<typeof makeInstance>,
+	) {
+		const batches: number[] = [];
+		const finished: number[] = [];
+		let seenThisBatch = 0;
+		const app = new Hono();
+		app.route(
+			"/api/sync",
+			createSyncReceiver({
+				log: receiver.log,
+				opKinds: receiver.opKinds,
+				verifyToken: () => true,
+				beginBatch: () => {
+					seenThisBatch = 0;
+					batches.push(0);
+					return (op, ctx) => {
+						seenThisBatch++;
+						batches[batches.length - 1] = seenThisBatch;
+						return receiver.applyOne(op, ctx);
+					};
+				},
+				afterBatch: async (report) => {
+					// Async on purpose: the response must not go out before it resolves.
+					await Promise.resolve();
+					finished.push(report.applied);
+				},
+			}),
+		);
+		const remote: RemoteApi = createHttpRemote({
+			basePath: "http://peer/api/sync",
+			token: "tok",
+			selfId: dialer.log.instanceId(),
+			fetchImpl: ((input: RequestInfo | URL, init?: RequestInit) =>
+				app.request(input as string, init)) as typeof fetch,
+		});
+		return { remote, batches, finished };
+	}
+
+	test("beginBatch is called once per push, so two pushes never share a counter", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote, batches } = linkWithBatchSeams(mac, prod);
+
+		mac.set("a", "1", 1000);
+		await pushToRemote(engineOpts(mac), remote);
+		mac.set("b", "2", 2000);
+		await pushToRemote(engineOpts(mac), remote);
+
+		// Two batches of one — not one batch that saw two, and not a counter that
+		// carried the first push's op into the second's report.
+		expect(batches).toEqual([1, 1]);
+	});
+
+	test("afterBatch is AWAITED before the response — the cursor must not outrun the work", async () => {
+		const mac = makeInstance();
+		const prod = makeInstance();
+		const { remote, finished } = linkWithBatchSeams(mac, prod);
+
+		mac.set("a", "1", 1000);
+		const report = await pushToRemote(engineOpts(mac), remote);
+
+		// By the time the dialer has its answer (and has advanced its push cursor),
+		// the post-commit work has already run.
+		expect(report.pushed).toBe(1);
+		expect(finished).toEqual([1]);
+	});
+
+	test("a receiver given neither applier — or both — refuses to be built", () => {
+		const prod = makeInstance();
+		const base = { log: prod.log, opKinds: prod.opKinds, verifyToken: () => true };
+		expect(() => createSyncReceiver({ ...base })).toThrow(/exactly one/);
+		expect(() =>
+			createSyncReceiver({ ...base, applyOne: prod.applyOne, beginBatch: () => prod.applyOne }),
+		).toThrow(/exactly one/);
+	});
+});