@rivet-dev/vercel-world 0.0.0-http-body-streaming.064e07d

package/src/runtime.ts

@@ -0,0 +1,29 @@
+import { registry } from "./actors.js";
+
+const endpoint = process.env.RIVET_ENDPOINT;
+const namespace = process.env.RIVET_NAMESPACE;
+const poolName = process.env.RIVET_POOL;
+
+if (!endpoint || !namespace || !poolName) {
+	throw new Error(
+		"RIVET_ENDPOINT, RIVET_NAMESPACE, and RIVET_POOL are required",
+	);
+}
+
+registry.config.test = {
+	...registry.config.test,
+	enabled: true,
+	sqliteBackend: "local",
+};
+registry.config.runtime = "native";
+registry.config.startEngine = false;
+registry.config.endpoint = endpoint;
+registry.config.token = process.env.RIVET_TOKEN;
+registry.config.namespace = namespace;
+registry.config.envoy = {
+	...registry.config.envoy,
+	poolName,
+};
+registry.config.noWelcome = true;
+
+await registry.startAndWait();

package/tests/conformance.test.ts

@@ -0,0 +1,8 @@
+import { createTestSuite } from "@workflow/world-testing";
+import { describe } from "vitest";
+
+if (process.env.RUN_WORKFLOW_CONFORMANCE === "1") {
+	createTestSuite("./dist/index.js");
+} else {
+	describe.skip("@workflow/world-testing conformance", () => {});
+}

package/tests/helpers/db.ts

@@ -0,0 +1,62 @@
+import { DatabaseSync } from "node:sqlite";
+import type { Ctx } from "../../src/actors/shared.ts";
+
+/**
+ * A node:sqlite-backed implementation of the actor `Ctx.db` surface used by the
+ * extracted SQL helpers (dispatcher queue ops, event materialization, hook and
+ * coordinator index transitions). Lets those helpers be unit-tested without an
+ * engine, against real SQLite semantics (RETURNING, partial unique indexes,
+ * BLOB round-trips).
+ */
+export type TestDb = Ctx["db"];
+
+export type MigrateFn = (database: {
+	execute(sql: string, ...args: unknown[]): Promise<unknown>;
+}) => Promise<void> | void;
+
+export type TestCtx = Ctx & { db: TestDb; raw: DatabaseSync; close(): void };
+
+function makeExecute(raw: DatabaseSync) {
+	return async (sql: string, ...args: unknown[]) => {
+		const stmt = raw.prepare(sql);
+		// node:sqlite `.all()` works for SELECT, RETURNING, and DML alike
+		// (returning [] when there are no result rows).
+		return stmt.all(...(args as never[])) as Record<string, unknown>[];
+	};
+}
+
+export async function makeCtx(onMigrate: MigrateFn): Promise<TestCtx> {
+	const raw = new DatabaseSync(":memory:");
+	const execute = makeExecute(raw);
+	await onMigrate({ execute });
+	let transactionTail = Promise.resolve();
+	const db: TestDb = {
+		execute,
+		transaction: async (callback) => {
+			const previous = transactionTail;
+			let release!: () => void;
+			transactionTail = new Promise<void>((resolve) => {
+				release = resolve;
+			});
+			await previous;
+			try {
+				raw.exec("BEGIN IMMEDIATE");
+				try {
+					const result = await callback(db);
+					raw.exec("COMMIT");
+					return result;
+				} catch (error) {
+					raw.exec("ROLLBACK");
+					throw error;
+				}
+			} finally {
+				release();
+			}
+		},
+	};
+	return {
+		db,
+		raw,
+		close: () => raw.close(),
+	};
+}

package/tests/helpers/dispatcher-driver.ts

@@ -0,0 +1,71 @@
+import {
+	createDispatcherVars,
+	startDispatchLoop,
+	type DispatchContext,
+} from "../../src/actors/dispatcher.ts";
+import { migrateWorkflowDb } from "../../src/actors/db.ts";
+import { makeCtx, type TestCtx } from "./db.ts";
+
+/**
+ * A deterministic stand-in for the dispatcher actor's runtime context. It wires
+ * the *real* drain loop (`startDispatchLoop` / `drainDispatchQueue` and the
+ * delivery/retry/dead-letter logic) to a node:sqlite-backed queue and a real
+ * `setTimeout`-driven schedule, so loop behavior is exercised faithfully without
+ * the `setupTest` runner's autonomous-loop provisioning flakiness.
+ *
+ * `schedule.after` mirrors the actor: it fires a `wake` that clears the
+ * coalescing flag and restarts the loop — exactly the production wiring.
+ */
+export type DispatcherDriver = {
+	ctx: DispatchContext;
+	/** Simulate an actor crash + restart: drop in-flight timers and rebuild the
+	 * context with fresh vars (latch cleared) over the SAME persisted db. */
+	restart(): DispatchContext;
+	close(): void;
+};
+
+export async function makeDispatcherDriver(
+	partition: string,
+): Promise<DispatcherDriver> {
+	const base: TestCtx = await makeCtx(migrateWorkflowDb);
+	let timers = new Set<ReturnType<typeof setTimeout>>();
+
+	const build = (): DispatchContext => {
+		const ctx = {
+			key: [partition],
+			db: base.db,
+			vars: createDispatcherVars(),
+			keepAwake: <T>(p: Promise<T>) => p,
+			schedule: {
+				after: (delayMs: number, _action: string, ...args: unknown[]) => {
+					const t = setTimeout(() => {
+						timers.delete(t);
+						ctx.vars.wakeArmedAt = null;
+						startDispatchLoop(ctx, String(args[0] ?? partition));
+					}, delayMs);
+					timers.add(t);
+					return Promise.resolve();
+				},
+				at: (timestampMs: number, action: string, ...args: unknown[]) =>
+					ctx.schedule.after(Math.max(0, timestampMs - Date.now()), action, ...args),
+			},
+		} as unknown as DispatchContext;
+		return ctx;
+	};
+
+	const driver: DispatcherDriver = {
+		ctx: build(),
+		restart() {
+			for (const t of timers) clearTimeout(t);
+			timers = new Set();
+			driver.ctx = build();
+			return driver.ctx;
+		},
+		close: () => {
+			for (const t of timers) clearTimeout(t);
+			timers.clear();
+			base.close();
+		},
+	};
+	return driver;
+}

package/tests/helpers/harness.ts

@@ -0,0 +1,161 @@
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+
+export type Delivery = {
+	route: string;
+	queueName: string | null;
+	messageId: string | null;
+	attempt: number;
+	body: any;
+};
+
+export type MockApp = {
+	url: string;
+	deliveries: Delivery[];
+	/** runId -> remaining forced 503 failures before success. */
+	failuresBeforeSuccess: Map<string, number>;
+	/** runIds that always respond 503. */
+	alwaysFail: Set<string>;
+	/** runId -> { timeoutSeconds } body to return on next success (long-step). */
+	timeoutResponse: Map<string, number>;
+	/** runIds whose success body is malformed/non-JSON. */
+	malformedSuccess: Set<string>;
+	/** Artificial delay (ms) before responding, to widen concurrency windows. */
+	responseDelayMs: number;
+	/** Peak number of simultaneously in-flight deliveries observed. */
+	maxConcurrent: number;
+	count(runId: string): number;
+	close(): Promise<void>;
+};
+
+function runIdOf(body: unknown): string | null {
+	if (body && typeof body === "object" && "runId" in body) {
+		return String((body as { runId: unknown }).runId);
+	}
+	return null;
+}
+
+/**
+ * Stands up a stand-in for the app's `.well-known/workflow/v1/{flow,step}`
+ * runtime endpoints with scriptable failure / long-step / malformed behavior so
+ * dispatcher-loop tests are deterministic. Mirrors the integration mock but is
+ * reusable from native (`setupTest`) actor tests.
+ */
+export async function startMockApp(): Promise<MockApp> {
+	const deliveries: Delivery[] = [];
+	const failuresBeforeSuccess = new Map<string, number>();
+	const alwaysFail = new Set<string>();
+	const timeoutResponse = new Map<string, number>();
+	const malformedSuccess = new Set<string>();
+	const state = { responseDelayMs: 0, maxConcurrent: 0, inFlight: 0 };
+
+	const server = createServer(async (req, res) => {
+		const path = req.url ?? "";
+		if (req.method === "GET" && path.includes("__health")) {
+			res.writeHead(200, { "content-type": "application/json" });
+			res.end(JSON.stringify({ healthy: true }));
+			return;
+		}
+		if (req.method !== "POST" || !path.startsWith("/.well-known/workflow/v1/")) {
+			res.writeHead(404);
+			res.end("not found");
+			return;
+		}
+		state.inFlight++;
+		state.maxConcurrent = Math.max(state.maxConcurrent, state.inFlight);
+		const finish = (status: number, payload: string) => {
+			const send = () => {
+				state.inFlight--;
+				res.writeHead(status, { "content-type": "application/json" });
+				res.end(payload);
+			};
+			if (state.responseDelayMs > 0) setTimeout(send, state.responseDelayMs);
+			else send();
+		};
+		const chunks: Buffer[] = [];
+		for await (const chunk of req) chunks.push(Buffer.from(chunk));
+		const body = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+		deliveries.push({
+			route: path.split("/").at(-1)?.split("?")[0] ?? "",
+			queueName: req.headers["x-vqs-queue-name"]
+				? String(req.headers["x-vqs-queue-name"])
+				: null,
+			messageId: req.headers["x-vqs-message-id"]
+				? String(req.headers["x-vqs-message-id"])
+				: null,
+			attempt: Number(req.headers["x-vqs-message-attempt"]),
+			body,
+		});
+		const runId = runIdOf(body);
+		if (runId && alwaysFail.has(runId)) {
+			finish(503, JSON.stringify({ error: "forced failure" }));
+			return;
+		}
+		if (runId) {
+			const remaining = failuresBeforeSuccess.get(runId) ?? 0;
+			if (remaining > 0) {
+				failuresBeforeSuccess.set(runId, remaining - 1);
+				finish(503, JSON.stringify({ error: "forced retry" }));
+				return;
+			}
+		}
+		if (runId && timeoutResponse.has(runId)) {
+			const seconds = timeoutResponse.get(runId)!;
+			timeoutResponse.delete(runId);
+			finish(200, JSON.stringify({ timeoutSeconds: seconds }));
+			return;
+		}
+		if (runId && malformedSuccess.has(runId)) {
+			finish(200, JSON.stringify({ timeoutSeconds: "not-a-number" }));
+			return;
+		}
+		finish(200, JSON.stringify({ ok: true }));
+	});
+
+	await new Promise<void>((resolve) =>
+		server.listen(0, "127.0.0.1", () => resolve()),
+	);
+	const port = (server.address() as AddressInfo).port;
+
+	return {
+		url: `http://127.0.0.1:${port}`,
+		deliveries,
+		failuresBeforeSuccess,
+		alwaysFail,
+		timeoutResponse,
+		malformedSuccess,
+		get responseDelayMs() {
+			return state.responseDelayMs;
+		},
+		set responseDelayMs(value: number) {
+			state.responseDelayMs = value;
+		},
+		get maxConcurrent() {
+			return state.maxConcurrent;
+		},
+		count: (runId: string) =>
+			deliveries.filter((d) => runIdOf(d.body) === runId).length,
+		close: () =>
+			new Promise<void>((resolve, reject) =>
+				(server as Server).close((err) => (err ? reject(err) : resolve())),
+			),
+	};
+}
+
+export function sleep(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function waitFor(
+	label: string,
+	predicate: () => boolean | Promise<boolean>,
+	timeoutMs = 5_000,
+	intervalMs = 25,
+) {
+	const deadline = Date.now() + timeoutMs;
+	while (Date.now() < deadline) {
+		if (await predicate()) return;
+		await sleep(intervalMs);
+	}
+	throw new Error(`Timed out waiting for ${label}`);
+}

package/tests/integration/crash-restart.test.ts

@@ -0,0 +1,145 @@
+import { randomUUID } from "node:crypto";
+import { afterEach, describe, expect, test } from "vitest";
+import {
+	startDispatchLoop,
+	type DispatchContext,
+} from "../../src/actors/dispatcher.ts";
+import {
+	makeDispatcherDriver,
+	type DispatcherDriver,
+} from "../helpers/dispatcher-driver.ts";
+import { startMockApp, waitFor, type MockApp } from "../helpers/harness.ts";
+
+const uid = () => randomUUID().slice(0, 8);
+
+async function insert(
+	ctx: DispatchContext,
+	runtimeUrl: string,
+	runId: string,
+	messageId: string,
+	status: string,
+	startedAtAgoMs: number | null,
+) {
+	const now = Date.now();
+	await ctx.db.execute(
+		`
+		INSERT INTO dispatcher_queue (
+			message_id, queue_name, route, runtime_url, body, headers, idem_key,
+			status, attempt, next_at, created_at, updated_at, started_at
+		)
+		VALUES (?, ?, 'flow', ?, ?, NULL, NULL, ?, ?, ?, ?, ?, ?)
+		`,
+		messageId,
+		`__wkf_workflow_${runId}`,
+		runtimeUrl,
+		JSON.stringify({ runId }),
+		status,
+		status === "inflight" ? 1 : 0,
+		now,
+		now,
+		now,
+		startedAtAgoMs == null ? null : now - startedAtAgoMs,
+	);
+}
+
+function snapshot(ctx: DispatchContext) {
+	return ctx.db.execute(
+		"SELECT message_id, status, attempt FROM dispatcher_queue ORDER BY message_id",
+	);
+}
+
+describe("crash/restart dispatch reconciliation", () => {
+	let driver: DispatcherDriver | undefined;
+	let app: MockApp | undefined;
+	afterEach(async () => {
+		driver?.close();
+		if (app) await app.close();
+		driver = undefined;
+		app = undefined;
+	});
+
+	test("restart reconciles stranded ready and stale-inflight rows to done, each delivered", async () => {
+		app = await startMockApp();
+		driver = await makeDispatcherDriver("wrun_crash");
+		const runId = `wrun_crash_${uid()}`;
+
+		// Crash signature left in the durable queue:
+		//  - msg_ready:  enqueued but the drain loop never started (crash before loop)
+		//  - msg_stale:  claimed inflight, then the process died mid-delivery
+		//  - msg_done2:  already completed before the crash (must NOT redeliver)
+		await insert(driver.ctx, app.url, runId, "msg_ready", "ready", null);
+		await insert(driver.ctx, app.url, runId, "msg_stale", "inflight", 60_000);
+		await insert(driver.ctx, app.url, runId, "msg_done2", "done", 60_000);
+
+		// Restart: fresh actor context (latch cleared) over the same db, then the
+		// onWake/self-heal equivalent kicks the drain loop.
+		const ctx = driver.restart();
+		startDispatchLoop(ctx, runId);
+
+		await waitFor(
+			"ready + stale reconciled to done",
+			async () => {
+				const rows = await snapshot(ctx);
+				const ready = rows.find((r) => r.message_id === "msg_ready");
+				const stale = rows.find((r) => r.message_id === "msg_stale");
+				return ready?.status === "done" && stale?.status === "done";
+			},
+			8_000,
+		);
+
+		const rows = await snapshot(ctx);
+		// No row left stranded in ready/inflight.
+		expect(rows.every((r) => r.status === "done")).toBe(true);
+		// The already-done row was never redelivered.
+		expect(app.count(runId)).toBe(2);
+		expect(app.deliveries.some((d) => d.messageId === "msg_done2")).toBe(false);
+	});
+
+	test("restart schedules a wake for inflight work that is not stale yet", async () => {
+		app = await startMockApp();
+		driver = await makeDispatcherDriver("wrun_fresh");
+		const runId = `wrun_fresh_${uid()}`;
+		await insert(driver.ctx, app.url, runId, "msg_fresh", "inflight", 0);
+
+		const ctx = driver.restart();
+		startDispatchLoop(ctx, runId);
+		expect((await snapshot(ctx))[0]?.status).toBe("inflight");
+
+		await waitFor(
+			"fresh inflight row to reach its stale deadline and redeliver",
+			async () => (await snapshot(ctx))[0]?.status === "done",
+			3_000,
+		);
+		expect(app.count(runId)).toBe(1);
+	});
+
+	test("repeated restarts mid-cycle never strand or duplicate a completed delivery", async () => {
+		app = await startMockApp();
+		driver = await makeDispatcherDriver("wrun_chaos");
+		const runId = `wrun_chaos_${uid()}`;
+		await insert(driver.ctx, app.url, runId, "msg_chaos", "ready", null);
+
+		// Bombard with restarts at varied points; each restart clears the latch and
+		// re-kicks the loop. Stale reclaim (>400ms) + the single-consumer latch must
+		// converge to exactly one completed delivery with the row marked done.
+		for (let i = 0; i < 5; i++) {
+			const ctx = driver.restart();
+			startDispatchLoop(ctx, runId);
+			await new Promise((r) => setTimeout(r, 120));
+		}
+		const finalCtx = driver.restart();
+		startDispatchLoop(finalCtx, runId);
+
+		await waitFor(
+			"converged to done",
+			async () =>
+				(await snapshot(finalCtx)).some((r) => r.status === "done"),
+			8_000,
+		);
+		// At-least-once is the contract; with a fast healthy app it must converge to
+		// exactly one successful delivery (no thrash, no duplicate completion).
+		expect(app.count(runId)).toBe(1);
+		const rows = await snapshot(finalCtx);
+		expect(rows.filter((r) => r.status === "done")).toHaveLength(1);
+	});
+});

package/tests/integration/dispatcher-loop.test.ts

@@ -0,0 +1,144 @@
+import { randomUUID } from "node:crypto";
+import { afterEach, describe, expect, test } from "vitest";
+import {
+	startDispatchLoop,
+	type DispatchContext,
+} from "../../src/actors/dispatcher.ts";
+import {
+	makeDispatcherDriver,
+	type DispatcherDriver,
+} from "../helpers/dispatcher-driver.ts";
+import { startMockApp, waitFor, type MockApp } from "../helpers/harness.ts";
+
+const uid = () => randomUUID().slice(0, 8);
+
+async function enqueue(
+	ctx: DispatchContext,
+	runtimeUrl: string,
+	runId: string,
+	messageId: string,
+	overrides: Partial<{ next_at: number; created_at: number }> = {},
+) {
+	const now = Date.now();
+	await ctx.db.execute(
+		`
+		INSERT INTO dispatcher_queue (
+			message_id, queue_name, route, runtime_url, body, headers, idem_key,
+			status, attempt, next_at, created_at, updated_at
+		)
+		VALUES (?, ?, 'flow', ?, ?, NULL, NULL, 'ready', 0, ?, ?, ?)
+		`,
+		messageId,
+		`__wkf_workflow_${runId}`,
+		runtimeUrl,
+		JSON.stringify({ runId }),
+		overrides.next_at ?? now,
+		overrides.created_at ?? now,
+		now,
+	);
+}
+
+function rows(ctx: DispatchContext) {
+	return ctx.db.execute(
+		"SELECT message_id, status, attempt FROM dispatcher_queue ORDER BY created_at ASC",
+	);
+}
+
+describe("dispatcher drain loop (deterministic driver)", () => {
+	let driver: DispatcherDriver | undefined;
+	let app: MockApp | undefined;
+	afterEach(async () => {
+		driver?.close();
+		if (app) await app.close();
+		driver = undefined;
+		app = undefined;
+	});
+
+	test("single drain loop claims each message once while deliveries overlap", async () => {
+		app = await startMockApp();
+		app.responseDelayMs = 60; // widen the window so parallel loops would overlap
+		driver = await makeDispatcherDriver("wrun_latch");
+		const runId = `wrun_latch_${uid()}`;
+		await enqueue(driver.ctx, app.url, runId, "msg_a", { created_at: 1 });
+		await enqueue(driver.ctx, app.url, runId, "msg_b", { created_at: 2 });
+
+		// Two concurrent start signals must still produce one claim loop and one
+		// delivery per message. HTTP deliveries intentionally overlap because a
+		// workflow turn can wait for a continuation enqueued behind it.
+		startDispatchLoop(driver.ctx, runId);
+		startDispatchLoop(driver.ctx, runId);
+
+		await waitFor("both delivered", () => app!.count(runId) === 2, 8_000);
+		expect(app.maxConcurrent).toBe(2);
+	});
+
+	test("delivery failures retry with backoff then succeed", async () => {
+		app = await startMockApp();
+		driver = await makeDispatcherDriver("wrun_retry");
+		const runId = `wrun_retry_${uid()}`;
+		app.failuresBeforeSuccess.set(runId, 2);
+		await enqueue(driver.ctx, app.url, runId, "msg_retry");
+		startDispatchLoop(driver.ctx, runId);
+		await waitFor("3 attempts", () => app!.count(runId) === 3, 8_000);
+		await waitFor(
+			"row done on attempt 3",
+			async () =>
+				(await rows(driver!.ctx)).some(
+					(r) => r.status === "done" && Number(r.attempt) === 3,
+				),
+			2_000,
+		);
+	});
+
+	test("permanently failing delivery dead-letters at the attempt cap", async () => {
+		app = await startMockApp();
+		driver = await makeDispatcherDriver("wrun_dead");
+		const runId = `wrun_dead_${uid()}`;
+		app.alwaysFail.add(runId);
+		await enqueue(driver.ctx, app.url, runId, "msg_dead");
+		startDispatchLoop(driver.ctx, runId);
+		await waitFor(
+			"failed at cap",
+			async () =>
+				(await rows(driver!.ctx)).some(
+					(r) => r.status === "failed" && Number(r.attempt) === 3,
+				),
+			8_000,
+		);
+		expect(app.count(runId)).toBe(3);
+	});
+
+	test("long-step {timeoutSeconds} response defers redelivery instead of acking", async () => {
+		app = await startMockApp();
+		driver = await makeDispatcherDriver("wrun_timeout");
+		const runId = `wrun_timeout_${uid()}`;
+		app.timeoutResponse.set(runId, 0); // first response carries timeoutSeconds
+		await enqueue(driver.ctx, app.url, runId, "msg_timeout");
+		startDispatchLoop(driver.ctx, runId);
+		// timeoutSeconds reschedules (not done); redelivered, then acks normally.
+		await waitFor("redelivered", () => app!.count(runId) === 2, 8_000);
+		await waitFor(
+			"eventually done",
+			async () => (await rows(driver!.ctx)).some((r) => r.status === "done"),
+			2_000,
+		);
+	});
+
+	test("malformed timeoutSeconds is retried, not silently acked", async () => {
+		app = await startMockApp();
+		driver = await makeDispatcherDriver("wrun_malformed");
+		const runId = `wrun_malformed_${uid()}`;
+		app.malformedSuccess.add(runId);
+		await enqueue(driver.ctx, app.url, runId, "msg_malformed");
+		startDispatchLoop(driver.ctx, runId);
+		await waitFor(
+			"retried to cap then failed",
+			async () =>
+				(await rows(driver!.ctx)).some(
+					(r) => r.status === "failed" && Number(r.attempt) === 3,
+				),
+			8_000,
+		);
+		expect(app.count(runId)).toBe(3);
+	});
+});