cursedbelt 2.6.0 → 2.7.0

package/src/server/sync/commands.spec.ts

@@ -0,0 +1,145 @@
+import { describe, expect, test } from "bun:test";
+import { Database } from "bun:sqlite";
+import { Hono } from "hono";
+import {
+	createCommandQueue,
+	createCommandReceiver,
+	createCommandRemote,
+	startCommandWorker,
+	type SyncCommand,
+} from "./commands";
+
+const queue = () => createCommandQueue(new Database(":memory:"));
+
+describe("command queue", () => {
+	test("enqueue → claim → complete lifecycle", () => {
+		const q = queue();
+		const cmd = q.enqueue("app.start", { key: "notes" }, "prod-ui");
+		expect(cmd.status).toBe("queued");
+		expect(q.pendingCount()).toBe(1);
+		const claimed = q.claimNext("mac");
+		expect(claimed?.uid).toBe(cmd.uid);
+		expect(claimed?.status).toBe("claimed");
+		// Nothing else to claim while it is held.
+		expect(q.claimNext("other")).toBeNull();
+		expect(q.complete(cmd.uid, { started: true })).toBe(true);
+		expect(q.get(cmd.uid)?.status).toBe("done");
+		expect(q.pendingCount()).toBe(0);
+		// Completing twice is refused — the row already finished.
+		expect(q.complete(cmd.uid, {})).toBe(false);
+	});
+
+	test("claims are ordered oldest-first and atomic across workers", () => {
+		const q = queue();
+		const first = q.enqueue("a", null, "ui");
+		const second = q.enqueue("b", null, "ui");
+		expect(q.claimNext("w1")?.uid).toBe(first.uid);
+		expect(q.claimNext("w2")?.uid).toBe(second.uid);
+		expect(q.claimNext("w3")).toBeNull();
+	});
+
+	test("a stale claim is reclaimable (a dead worker cannot strand the queue)", async () => {
+		const q = queue();
+		q.enqueue("a", null, "ui");
+		expect(q.claimNext("w1", 60_000)).not.toBeNull();
+		// Let the claim age past a 1ms stale window, then reclaim it.
+		await new Promise((resolve) => setTimeout(resolve, 5));
+		expect(q.claimNext("w2", 1)?.claimedBy).toBe("w2");
+	});
+
+	test("fail records the error", () => {
+		const q = queue();
+		const cmd = q.enqueue("a", null, "ui");
+		q.claimNext("w");
+		expect(q.fail(cmd.uid, "boom")).toBe(true);
+		const row = q.get(cmd.uid);
+		expect(row?.status).toBe("failed");
+		expect(row?.error).toBe("boom");
+	});
+});
+
+describe("command receiver + remote + worker", () => {
+	function mount(q: ReturnType<typeof createCommandQueue>) {
+		const app = new Hono();
+		app.route(
+			"/api/relay",
+			createCommandReceiver({
+				queue: q,
+				verifyToken: (t) => t === "tok",
+				maxHoldMs: 300,
+				pollStepMs: 20,
+			}),
+		);
+		const remote = createCommandRemote({
+			basePath: "http://peer/api/relay",
+			token: "tok",
+			fetchImpl: ((input: RequestInfo | URL, init?: RequestInit) =>
+				app.request(input as string, init)) as typeof fetch,
+		});
+		return { app, remote };
+	}
+
+	test("held poll returns a queued command immediately, null after the hold", async () => {
+		const q = queue();
+		const { remote } = mount(q);
+		const empty = await remote.poll("mac", 50);
+		expect(empty).toBeNull();
+		q.enqueue("repo.sync", null, "ui");
+		const got = await remote.poll("mac", 50);
+		expect(got?.kind).toBe("repo.sync");
+		await remote.report(got?.uid ?? "", { ok: true, result: { healed: 3 } });
+		expect(q.get(got?.uid ?? "")?.status).toBe("done");
+	});
+
+	test("a command enqueued DURING the hold is delivered within it", async () => {
+		const q = queue();
+		const { remote } = mount(q);
+		const pending = remote.poll("mac", 250);
+		setTimeout(() => q.enqueue("late", null, "ui"), 40);
+		const got = await pending;
+		expect(got?.kind).toBe("late");
+	});
+
+	test("the gate refuses a wrong token uniformly", async () => {
+		const q = queue();
+		const app = new Hono();
+		app.route("/api/relay", createCommandReceiver({ queue: q, verifyToken: (t) => t === "tok" }));
+		const res = await app.request("http://peer/api/relay/pending?wait=0", {
+			headers: { authorization: "Bearer wrong" },
+		});
+		expect(res.status).toBe(401);
+	});
+
+	test("the worker loop polls, executes, reports, and keeps cycling", async () => {
+		const q = queue();
+		const { remote } = mount(q);
+		const executed: string[] = [];
+		const worker = startCommandWorker({
+			resolveRemote: () => remote,
+			execute: async (command: SyncCommand) => {
+				executed.push(command.kind);
+				if (command.kind === "explode") throw new Error("kaboom");
+				return { ok: true };
+			},
+			holdMs: 60,
+			idleGapMs: 5,
+			log: () => {},
+			warn: () => {},
+		});
+		q.enqueue("good", null, "ui");
+		q.enqueue("explode", null, "ui");
+		const deadline = Date.now() + 3_000;
+		while (Date.now() < deadline) {
+			const rows = q.list();
+			if (rows.length === 2 && rows.every((r) => r.status === "done" || r.status === "failed"))
+				break;
+			await new Promise((resolve) => setTimeout(resolve, 20));
+		}
+		worker.stop();
+		expect(executed.sort()).toEqual(["explode", "good"]);
+		const byKind = new Map(q.list().map((r) => [r.kind, r]));
+		expect(byKind.get("good")?.status).toBe("done");
+		expect(byKind.get("explode")?.status).toBe("failed");
+		expect(byKind.get("explode")?.error).toContain("kaboom");
+	});
+});

package/src/server/sync/commands.ts

@@ -0,0 +1,361 @@
+/**
+ * The command relay — notes' ask-responder shape generalized: the internet-facing
+ * instance PARKS an action it cannot perform (it has no Mac powers), and the Mac's
+ * resident worker long-polls for it, executes locally, and posts the outcome back.
+ * The Mac dials OUT; the receiving instance never reaches into the house.
+ *
+ * Delivery is a held GET (default hold 25s) — remember the two fleet traps: the
+ * host server needs `idleTimeoutSeconds` ABOVE the hold (Bun's 10s default closed
+ * notes' 25s poll and nginx reported a bogus 502), and the routes must be mounted
+ * BEFORE any browser-session gate.
+ *
+ * A command left `claimed` past the stale window is reclaimable — a worker that
+ * died mid-execution must not strand the queue. Executors must therefore be
+ * idempotent or cheap to re-run.
+ */
+import type { Database } from "bun:sqlite";
+import { randomUUID } from "node:crypto";
+import { Hono } from "hono";
+
+export type CommandStatus = "queued" | "claimed" | "done" | "failed";
+
+export interface SyncCommand {
+	uid: string;
+	kind: string;
+	args: unknown;
+	status: CommandStatus;
+	requestedBy: string;
+	createdAt: number;
+	claimedAt: number | null;
+	claimedBy: string | null;
+	finishedAt: number | null;
+	result: unknown;
+	error: string | null;
+}
+
+const STALE_CLAIM_MS = 5 * 60_000;
+
+export interface CommandQueue {
+	enqueue(kind: string, args: unknown, requestedBy: string): SyncCommand;
+	/** Atomically claim the oldest queued (or stale-claimed) command. */
+	claimNext(claimedBy: string, staleMs?: number): SyncCommand | null;
+	complete(uid: string, result: unknown): boolean;
+	fail(uid: string, error: string): boolean;
+	get(uid: string): SyncCommand | null;
+	list(limit?: number): SyncCommand[];
+	pendingCount(): number;
+}
+
+interface Row {
+	uid: string;
+	kind: string;
+	args: string;
+	status: CommandStatus;
+	requested_by: string;
+	created_at: number;
+	claimed_at: number | null;
+	claimed_by: string | null;
+	finished_at: number | null;
+	result: string | null;
+	error: string | null;
+}
+
+const toCommand = (r: Row): SyncCommand => ({
+	uid: r.uid,
+	kind: r.kind,
+	args: JSON.parse(r.args) as unknown,
+	status: r.status,
+	requestedBy: r.requested_by,
+	createdAt: r.created_at,
+	claimedAt: r.claimed_at,
+	claimedBy: r.claimed_by,
+	finishedAt: r.finished_at,
+	result: r.result === null ? null : (JSON.parse(r.result) as unknown),
+	error: r.error,
+});
+
+export function createCommandQueue(db: Database): CommandQueue {
+	db.exec(`
+		CREATE TABLE IF NOT EXISTS sync_commands (
+			uid TEXT PRIMARY KEY,
+			kind TEXT NOT NULL,
+			args TEXT NOT NULL,
+			status TEXT NOT NULL DEFAULT 'queued',
+			requested_by TEXT NOT NULL,
+			created_at INTEGER NOT NULL,
+			claimed_at INTEGER,
+			claimed_by TEXT,
+			finished_at INTEGER,
+			result TEXT,
+			error TEXT
+		);
+		CREATE INDEX IF NOT EXISTS idx_sync_commands_status ON sync_commands(status, created_at);
+	`);
+	const insert = db.prepare(
+		"INSERT INTO sync_commands (uid, kind, args, status, requested_by, created_at) VALUES (?, ?, ?, 'queued', ?, ?)",
+	);
+	const getStmt = db.prepare("SELECT * FROM sync_commands WHERE uid = ?");
+	const listStmt = db.prepare("SELECT * FROM sync_commands ORDER BY created_at DESC LIMIT ?");
+	const pendingStmt = db.prepare(
+		"SELECT COUNT(*) AS n FROM sync_commands WHERE status IN ('queued','claimed')",
+	);
+	// The atomic claim: one UPDATE whose WHERE picks the oldest eligible row. sqlite
+	// serializes writers, so two workers can never claim the same row.
+	const claimStmt = db.prepare(`
+		UPDATE sync_commands SET status = 'claimed', claimed_at = ?, claimed_by = ?
+		WHERE uid = (
+			SELECT uid FROM sync_commands
+			WHERE status = 'queued' OR (status = 'claimed' AND claimed_at < ?)
+			ORDER BY created_at ASC LIMIT 1
+		)
+		RETURNING *
+	`);
+	const finishStmt = db.prepare(
+		"UPDATE sync_commands SET status = ?, finished_at = ?, result = ?, error = ? WHERE uid = ? AND status IN ('queued','claimed')",
+	);
+
+	return {
+		enqueue(kind, args, requestedBy) {
+			const uid = `cmd-${randomUUID().slice(0, 18)}`;
+			insert.run(uid, kind, JSON.stringify(args ?? null), requestedBy, Date.now());
+			return toCommand(getStmt.get(uid) as Row);
+		},
+		claimNext(claimedBy, staleMs = STALE_CLAIM_MS) {
+			const now = Date.now();
+			const row = claimStmt.get(now, claimedBy, now - staleMs) as Row | null;
+			return row ? toCommand(row) : null;
+		},
+		complete: (uid, result) =>
+			finishStmt.run("done", Date.now(), JSON.stringify(result ?? null), null, uid).changes > 0,
+		fail: (uid, error) => finishStmt.run("failed", Date.now(), null, error, uid).changes > 0,
+		get: (uid) => {
+			const row = getStmt.get(uid) as Row | null;
+			return row ? toCommand(row) : null;
+		},
+		list: (limit = 100) => (listStmt.all(limit) as Row[]).map(toCommand),
+		pendingCount: () => (pendingStmt.get() as { n: number }).n,
+	};
+}
+
+// ── the receiver routes (mount beside the sync receiver, same token gate) ────────
+
+const noStore = { "cache-control": "no-store" } as const;
+export const DEFAULT_HOLD_MS = 25_000;
+
+export interface CommandRoutesOptions {
+	queue: CommandQueue;
+	verifyToken: (presented: string) => boolean;
+	/** Cap on the held poll. The HOST SERVER's idleTimeout must exceed this. */
+	maxHoldMs?: number;
+	pollStepMs?: number;
+}
+
+/** GET /pending (held) + POST /pending/:uid — the worker's two verbs. */
+export function createCommandReceiver(options: CommandRoutesOptions): Hono {
+	const maxHold = options.maxHoldMs ?? DEFAULT_HOLD_MS;
+	const step = options.pollStepMs ?? 1_000;
+	const api = new Hono();
+	api.use("*", async (c, next) => {
+		const header = c.req.header("authorization") ?? "";
+		const presented = header.startsWith("Bearer ") ? header.slice(7) : "";
+		if (presented.length === 0 || !options.verifyToken(presented)) {
+			return c.json({ error: "unauthorized" }, 401, noStore);
+		}
+		await next();
+	});
+
+	api.get("/pending", async (c) => {
+		const worker = c.req.query("worker") ?? "worker";
+		const waitRaw = c.req.query("wait");
+		const wait = Math.min(
+			maxHold,
+			waitRaw !== undefined && /^\d+$/.test(waitRaw) ? Number(waitRaw) : maxHold,
+		);
+		const deadline = Date.now() + wait;
+		for (;;) {
+			const command = options.queue.claimNext(worker);
+			if (command) return c.json({ command }, 200, noStore);
+			if (Date.now() >= deadline) return c.json({ command: null }, 200, noStore);
+			await new Promise((resolve) =>
+				setTimeout(resolve, Math.min(step, Math.max(1, deadline - Date.now()))),
+			);
+		}
+	});
+
+	api.post("/pending/:uid", async (c) => {
+		const uid = c.req.param("uid");
+		let body: unknown;
+		try {
+			body = await c.req.json();
+		} catch {
+			return c.json({ error: "bad_request" }, 400, noStore);
+		}
+		const { ok, result, error } = (body ?? {}) as {
+			ok?: unknown;
+			result?: unknown;
+			error?: unknown;
+		};
+		if (typeof ok !== "boolean") return c.json({ error: "bad_request" }, 400, noStore);
+		const changed = ok
+			? options.queue.complete(uid, result ?? null)
+			: options.queue.fail(uid, typeof error === "string" ? error : "failed");
+		if (!changed) return c.json({ error: "not_found" }, 404, noStore);
+		return c.json({ ok: true }, 200, noStore);
+	});
+
+	return api;
+}
+
+// ── the dialing worker (runs on the Mac, inside the resident daemon) ─────────────
+
+export interface CommandRemote {
+	poll(worker: string, waitMs: number): Promise<SyncCommand | null>;
+	report(uid: string, outcome: { ok: boolean; result?: unknown; error?: string }): Promise<void>;
+}
+
+export function createCommandRemote(options: {
+	basePath: string;
+	token: string;
+	fetchImpl?: typeof fetch;
+	/** Host-scoped extras for this peer — see `createHttpRemote`'s note. */
+	extraHeaders?: Record<string, string>;
+}): CommandRemote {
+	const base = options.basePath.replace(/\/$/, "");
+	const doFetch = options.fetchImpl ?? fetch;
+	const headers = (extra?: Record<string, string>) => ({
+		authorization: `Bearer ${options.token}`,
+		...(options.extraHeaders ?? {}),
+		...(extra ?? {}),
+	});
+	/** 🔴 `redirect: "manual"`, for the reason in `http.ts`'s header: the relay
+	 *  travels the same road as the sync, so a moved peer hostname must fail here
+	 *  with the destination in the message rather than silently landing on a
+	 *  Cloudflare Access login page and reporting a JSON parse error. */
+	const refuseRedirect = (r: Response, path: string): void => {
+		if (r.status >= 300 && r.status < 400)
+			throw new Error(
+				`${path}: ${r.status} — the peer address redirects to ` +
+					`${r.headers.get("location") ?? "an unnamed destination"}; the relay URL points at a host that has moved.`,
+			);
+	};
+	return {
+		async poll(worker, waitMs) {
+			const r = await doFetch(
+				`${base}/pending?worker=${encodeURIComponent(worker)}&wait=${waitMs}`,
+				{
+					headers: headers(),
+					redirect: "manual",
+					// The abort budget must clear the hold plus network slack.
+					signal: AbortSignal.timeout(waitMs + 15_000),
+				},
+			);
+			refuseRedirect(r, "commands/pending");
+			if (!r.ok) throw new Error(`commands/pending: ${r.status}`);
+			const body = (await r.json()) as { command: SyncCommand | null };
+			return body.command;
+		},
+		async report(uid, outcome) {
+			const r = await doFetch(`${base}/pending/${encodeURIComponent(uid)}`, {
+				method: "POST",
+				headers: headers({ "content-type": "application/json" }),
+				redirect: "manual",
+				body: JSON.stringify(outcome),
+				signal: AbortSignal.timeout(30_000),
+			});
+			refuseRedirect(r, "commands/report");
+			if (!r.ok) throw new Error(`commands/report: ${r.status}`);
+		},
+	};
+}
+
+export interface CommandWorkerDeps {
+	/** Re-read every cycle, like the sync timer's peer — null = not linked. */
+	resolveRemote: () => CommandRemote | null;
+	execute: (command: SyncCommand) => Promise<unknown>;
+	worker?: string;
+	holdMs?: number;
+	/** Idle gap between an empty poll and the next (on top of the hold). */
+	idleGapMs?: number;
+	backoffMaxMs?: number;
+	log?: (msg: string) => void;
+	warn?: (msg: string) => void;
+	setTimer?: (fn: () => void, ms: number) => unknown;
+	clearTimer?: (handle: unknown) => void;
+}
+
+export interface CommandWorkerHandle {
+	stop(): void;
+}
+
+/** The resident long-poll → execute → report loop. Unreachable remote backs off to
+ *  `backoffMaxMs` and recovers silently — the nightly window rule. */
+export function startCommandWorker(deps: CommandWorkerDeps): CommandWorkerHandle {
+	const worker = deps.worker ?? "mac";
+	const holdMs = deps.holdMs ?? DEFAULT_HOLD_MS;
+	const idleGapMs = deps.idleGapMs ?? 1_000;
+	const backoffMax = deps.backoffMaxMs ?? 10 * 60_000;
+	const log = deps.log ?? ((m: string) => console.log(m));
+	const warn = deps.warn ?? ((m: string) => console.error(m));
+	const setTimer = deps.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
+	const clearTimer = deps.clearTimer ?? ((h) => clearTimeout(h as ReturnType<typeof setTimeout>));
+
+	let stopped = false;
+	let handle: unknown = null;
+	let failures = 0;
+	let reportedUnreachable = false;
+
+	const schedule = (ms: number): void => {
+		if (stopped) return;
+		if (handle !== null) clearTimer(handle);
+		handle = setTimer(cycle, ms);
+		(handle as { unref?: () => void })?.unref?.();
+	};
+
+	const cycle = (): void => {
+		if (stopped) return;
+		const remote = deps.resolveRemote();
+		if (remote === null) {
+			schedule(60_000);
+			return;
+		}
+		void remote
+			.poll(worker, holdMs)
+			.then(async (command) => {
+				failures = 0;
+				reportedUnreachable = false;
+				if (command === null) return;
+				log(`[commands] executing ${command.kind} (${command.uid})`);
+				try {
+					const result = await deps.execute(command);
+					await remote.report(command.uid, { ok: true, result });
+				} catch (err) {
+					const message = err instanceof Error ? err.message : String(err);
+					warn(`[commands] ${command.kind} failed: ${message}`);
+					// Best-effort: a failed report leaves the row claimed; the stale-claim
+					// window re-serves it, which is why executors must tolerate a re-run.
+					await remote.report(command.uid, { ok: false, error: message }).catch(() => {});
+				}
+			})
+			.then(() => {
+				if (!stopped) schedule(idleGapMs);
+			})
+			.catch((err) => {
+				failures += 1;
+				const wait = Math.min(backoffMax, 5_000 * 2 ** Math.min(failures - 1, 7));
+				if (!reportedUnreachable) {
+					const message = err instanceof Error ? err.message : String(err);
+					log(`[commands] remote unreachable (${message}) — backing off. This is normal offline.`);
+					reportedUnreachable = true;
+				}
+				schedule(wait);
+			});
+	};
+
+	schedule(1_000);
+	return {
+		stop: () => {
+			stopped = true;
+			if (handle !== null) clearTimer(handle);
+		},
+	};
+}