@ewhauser/world-celld 0.8.1 → 0.9.1

package/dist/testing.js

@@ -1,4 +1,4 @@
-import { $ as validateStreamReadRequest, B as MAX_STREAM_READ_BYTES, C as HOOK_CLAIM_LEASE_MS, I as MAX_STREAM_BATCH_BYTES, J as encodeStreamReadResult, K as decodeStreamWriteBatch, L as MAX_STREAM_CHUNK_BYTES, M as isPositiveSafeInteger, N as isRecord$1, O as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, Q as normalizeStreamError, U as STREAM_BATCH_CONTENT_TYPE, W as STREAM_CHUNK_LIMIT_HEADER, X as encodeStreamWriteResult, _ as rpcStringify, a as QUEUE_PUBLICATION_LEASE_MS, b as createWorkflowIndex, d as queuePayloadObjectKey, et as validateStreamWriteChunks, f as queuePayloadRegistryKey, g as rpcParse, h as compact, i as QUEUE_PAYLOAD_REGISTRY_PREFIX, j as isNonNegativeSafeInteger, l as queueClaimName, m as validateNativeQueueSendOptions, n as listRunsPage, o as QUEUE_RESERVATION_GRACE_MS, p as validateNativeQueueEnvelope, r as NATIVE_QUEUE_MAX_MESSAGE_BYTES, s as isInlineRunBody, t as MAX_RUN_LIST_LIMIT, u as queueOrphanName, v as parse, w as LIFECYCLE_COMPACTION_RETRY_MS, x as runIndexMetadata, z as MAX_STREAM_LONG_POLL_MS } from "./run-list-DGsdv6Jf.js";
+import { B as MAX_STREAM_BATCH_BYTES, C as parseDeploymentUrls, D as LIFECYCLE_COMPACTION_RETRY_MS, E as HOOK_CLAIM_LEASE_MS, F as isNonNegativeSafeInteger, I as isPositiveSafeInteger, J as STREAM_CHUNK_LIMIT_HEADER, L as isRecord$1, M as QUEUE_CLAIM_STALE_MS, Q as encodeStreamReadResult, R as queueDelayDeadline, S as configuredDeploymentUrl, U as MAX_STREAM_LONG_POLL_MS, V as MAX_STREAM_CHUNK_BYTES, W as MAX_STREAM_READ_BYTES, X as decodeStreamWriteBatch, _ as rpcStringify, a as QUEUE_PUBLICATION_LEASE_MS, b as createWorkflowIndex, c as nativeQueueDelaySeconds, d as queuePayloadObjectKey, et as encodeStreamWriteResult, f as queuePayloadRegistryKey, g as rpcParse, h as compact, i as QUEUE_PAYLOAD_REGISTRY_PREFIX, it as validateStreamWriteChunks, j as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, l as queueClaimName, m as validateNativeQueueSendOptions, n as listRunsPage, nt as normalizeStreamError, o as QUEUE_RESERVATION_GRACE_MS, p as validateNativeQueueEnvelope, q as STREAM_BATCH_CONTENT_TYPE, r as NATIVE_QUEUE_MAX_MESSAGE_BYTES, rt as validateStreamReadRequest, s as isInlineRunBody, t as MAX_RUN_LIST_LIMIT, u as queueOrphanName, v as parse, x as runIndexMetadata, y as stringify } from "./run-list-CHRaDRvU.js";
 import { CreateEventSchema, EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
 import http from "node:http";
 //#region src/testing/fake-cell.ts
@@ -1361,6 +1361,11 @@ async function authenticate(request, secret) {
 }
 //#endregion
 //#region src/worker/queue-claims.ts
+/**
+* Idempotency claims for Queue messages. A claim lives either in its own
+* claim cell (named by queue and idempotency key) or, for run-bearing
+* messages whose envelope says so, as an entry in the run's own cell.
+*/
 function cellQueueClaim(cell) {
 	return {
 		claim: (messageId, staleMs) => cell.claimInflight({
@@ -1413,6 +1418,12 @@ function runQueueClaim(run, claimName) {
 		})
 	};
 }
+/** The claim a delivered envelope was reserved under, if it has an idempotency key. */
+function deliveryQueueClaim(envelope, runStub) {
+	if (!envelope.idempotencyKey) return void 0;
+	const claimName = queueClaimName(envelope.queueName, envelope.idempotencyKey);
+	return envelope.claimScope === "run" && envelope.runId ? runQueueClaim(runStub(envelope.runId), claimName) : cellQueueClaim(runStub(claimName));
+}
 //#endregion
 //#region src/worker/index-validation.ts
 function invalid(message) {
@@ -2276,6 +2287,70 @@ async function resolveExpiredHookClaim(env, owner, claimId, token) {
 		token
 	});
 }
+/** Read-only owner check used by index repair; unlike the resolver it never fences. */
+async function getCommittedHook(env, owner, token) {
+	const namespace = env.WORKFLOW_DB;
+	if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
+	return await namespace.get(namespace.idFromName(owner.runId)).getCommittedHook({
+		hookId: owner.hookId,
+		token
+	});
+}
+function warnRepair(action, owner) {
+	console.warn(`world-celld ${action} hook index record for run ${owner.runId} hook ${owner.hookId}`);
+}
+function warnRepublished(owner) {
+	warnRepair("republished", owner);
+}
+/**
+* Publish the record for a claim whose hook committed in its run but whose
+* creating client stopped before finalizing. The write applies only while the
+* observed claim is unchanged, so a concurrent release or finalize wins.
+* Returns the record now stored for the key, if any.
+*/
+async function publishCommittedClaim(storage, env, keys, token) {
+	const observed = await storage.get(keys.claim);
+	if (!observed) return null;
+	const hook = await getCommittedHook(env, observed.owner, token);
+	if (!hook) return null;
+	const serialized = stringify(hook);
+	const published = await storage.transaction(async (txn) => {
+		const values = await txn.get([keys.record, keys.claim]);
+		const indexed = values.get(keys.record);
+		if (indexed !== void 0) return {
+			raw: indexed,
+			written: false
+		};
+		const claim = values.get(keys.claim);
+		if (claim?.claimId !== observed.claimId) return {
+			raw: null,
+			written: false
+		};
+		await txn.put(keys.record, serialized);
+		await txn.delete([keys.claim, claim.deadlineKey]);
+		return {
+			raw: serialized,
+			written: true
+		};
+	});
+	if (!published.written) return published.raw;
+	warnRepublished(observed.owner);
+	return await confirmRepublished(storage, env, keys.record, observed.owner, token) ? serialized : null;
+}
+/**
+* Re-ask the owner after a repair write. A hook disposed between the owner's
+* answer and the write would otherwise keep a record whose index release was
+* already spent, so withdraw it.
+*/
+async function confirmRepublished(storage, env, record, owner, token) {
+	if (await getCommittedHook(env, owner, token)) return true;
+	await storage.transaction(async (txn) => {
+		const indexed = await txn.get(record);
+		if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) await txn.delete(record);
+	});
+	warnRepair("withdrew", owner);
+	return false;
+}
 //#endregion
 //#region src/worker/durable-objects/HookIdDO.ts
 const MAX_RELEASE_BATCH$1 = 64;
@@ -2298,7 +2373,10 @@ var HookIdDO = class extends DurableObject {
 		return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
 	}
 	async get(hookId) {
-		const raw = await this.ctx.storage.get(recordKey$1(hookId));
+		const raw = await this.ctx.storage.get(recordKey$1(hookId)) ?? await publishCommittedClaim(this.ctx.storage, this.env, {
+			record: recordKey$1(hookId),
+			claim: claimKey$1(hookId)
+		});
 		if (!raw) return null;
 		const owner = ownerFromSerializedHook(raw);
 		return await runIsActive(this.env, owner.runId) ? raw : null;
@@ -2463,8 +2541,9 @@ var HookIdDO = class extends DurableObject {
 			const claim = observed.get(claimKey$1(item.hookId));
 			if (!claim || claim.claimId !== item.claimId || claim.expiresAt > now) return;
 			const hook = await resolveExpiredHookClaim(this.env, claim.owner, claim.claimId);
-			committed.set(claimKey$1(item.hookId), hook !== null);
+			committed.set(claimKey$1(item.hookId), hook);
 		}));
+		const published = [];
 		await this.ctx.storage.transaction(async (txn) => {
 			const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
 			const deletes = [];
@@ -2474,8 +2553,15 @@ var HookIdDO = class extends DurableObject {
 				const snapshot = observed.get(key);
 				if (claim?.claimId !== item.claimId || claim.expiresAt > now) deletes.push(deadline);
 				else if (claim.claimId === snapshot?.claimId && claim.expiresAt === snapshot.expiresAt) {
-					if (committed.get(key)) await this.renewClaim(txn, item.hookId, claim, now);
-					else deletes.push(deadline, key);
+					const hook = committed.get(key);
+					if (hook) {
+						await txn.put(recordKey$1(item.hookId), stringify(hook));
+						published.push({
+							hookId: item.hookId,
+							owner: claim.owner
+						});
+					}
+					deletes.push(deadline, key);
 				}
 			}
 			for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
@@ -2490,6 +2576,10 @@ var HookIdDO = class extends DurableObject {
 			if (nextAt === void 0) await txn.deleteAlarm();
 			else await txn.setAlarm(Math.max(now + 1, nextAt));
 		});
+		for (const { hookId, owner } of published) {
+			warnRepublished(owner);
+			await confirmRepublished(this.ctx.storage, this.env, recordKey$1(hookId), owner);
+		}
 	}
 };
 //#endregion
@@ -2514,7 +2604,10 @@ var HookTokenDO = class extends DurableObject {
 		return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
 	}
 	async get(token) {
-		const raw = await this.ctx.storage.get(recordKey(token));
+		const raw = await this.ctx.storage.get(recordKey(token)) ?? await publishCommittedClaim(this.ctx.storage, this.env, {
+			record: recordKey(token),
+			claim: claimKey(token)
+		}, token);
 		if (!raw) return null;
 		const owner = ownerFromSerializedHook(raw);
 		return await runIsActive(this.env, owner.runId) ? raw : null;
@@ -2678,8 +2771,9 @@ var HookTokenDO = class extends DurableObject {
 			const claim = observed.get(claimKey(item.token));
 			if (!claim || claim.claimId !== item.claimId || claim.expiresAt > now) return;
 			const hook = await resolveExpiredHookClaim(this.env, claim.owner, claim.claimId, item.token);
-			committed.set(claimKey(item.token), hook !== null);
+			committed.set(claimKey(item.token), hook);
 		}));
+		const published = [];
 		await this.ctx.storage.transaction(async (txn) => {
 			const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
 			const deletes = [];
@@ -2689,8 +2783,15 @@ var HookTokenDO = class extends DurableObject {
 				const snapshot = observed.get(key);
 				if (claim?.claimId !== item.claimId || claim.expiresAt > now) deletes.push(deadline);
 				else if (claim.claimId === snapshot?.claimId && claim.expiresAt === snapshot.expiresAt) {
-					if (committed.get(key)) await this.renewClaim(txn, item.token, claim, now);
-					else deletes.push(deadline, key);
+					const hook = committed.get(key);
+					if (hook) {
+						await txn.put(recordKey(item.token), stringify(hook));
+						published.push({
+							token: item.token,
+							owner: claim.owner
+						});
+					}
+					deletes.push(deadline, key);
 				}
 			}
 			for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
@@ -2705,6 +2806,10 @@ var HookTokenDO = class extends DurableObject {
 			if (nextAt === void 0) await txn.deleteAlarm();
 			else await txn.setAlarm(Math.max(now + 1, nextAt));
 		});
+		for (const { token, owner } of published) {
+			warnRepublished(owner);
+			await confirmRepublished(this.ctx.storage, this.env, recordKey(token), owner, token);
+		}
 	}
 };
 //#endregion
@@ -3810,6 +3915,23 @@ var WorkflowRunDO = class extends DurableObject {
 			return null;
 		});
 	}
+	/**
+	* Read-only twin of resolveExpiredHookClaim for index repair: the committed
+	* hook of a live run, or null. It fences nothing, so a shard may ask while
+	* the hook's creation is still in flight.
+	*/
+	async getCommittedHook(request) {
+		if (typeof request?.hookId !== "string" || request.hookId.length === 0 || request.token !== void 0 && typeof request.token !== "string") throw new TypeError("world-celld committed hook lookup is invalid");
+		return await this.ctx.storage.transaction(async (txn) => {
+			const { tombstone } = await this.retentionState(txn);
+			if (tombstone) return null;
+			const run = await txn.get("run");
+			if (!run || isTerminalWorkflowRunStatus(run.status)) return null;
+			const hook = await txn.get(`${HOOK_KEY_PREFIX}${request.hookId}`);
+			if (!hook || request.token !== void 0 && hook.token !== request.token) return null;
+			return hook;
+		});
+	}
 	/** Single authoritative lifecycle answer used after derivative compaction. */
 	async getLifecycleStatus() {
 		return await this.ctx.storage.transaction(async (txn) => {
@@ -4801,6 +4923,157 @@ var WorkflowRunDO = class extends DurableObject {
 	}
 };
 //#endregion
+//#region src/worker/queue-delivery.ts
+/** Internal delivery contract. No Request/Response objects cross the RPC boundary. */
+const MAX_QUEUE_DELIVERY_RESPONSE_BYTES = 65536;
+const QUEUE_ORPHAN_GRACE_MS = 432e6;
+const PERMANENT_QUEUE_STATUSES = /* @__PURE__ */ new Set([
+	404,
+	409,
+	410,
+	422
+]);
+async function readResponseText(response) {
+	if (!response.body) return "";
+	const reader = response.body.getReader();
+	const decoder = new TextDecoder();
+	let total = 0;
+	let text = "";
+	try {
+		for (;;) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			total += value.byteLength;
+			if (total > MAX_QUEUE_DELIVERY_RESPONSE_BYTES) {
+				await reader.cancel("queue delivery response is too large").catch(() => void 0);
+				throw new Error("workflow queue handler response exceeds configured limit");
+			}
+			text += decoder.decode(value, { stream: true });
+		}
+		return text + decoder.decode();
+	} finally {
+		reader.releaseLock();
+	}
+}
+/** Validate at the service boundary; callers cannot bypass auth via RPC. */
+async function deliverQueueMessage(env, secret, input, attempt) {
+	if (!env.WORLD_SECRET) throw new Error("WORLD_SECRET is not configured");
+	if (typeof secret !== "string" || !secret || !await timingSafeEqual(secret, env.WORLD_SECRET)) throw new Error("Unauthorized Queue delivery");
+	const envelope = validateNativeQueueEnvelope(input);
+	if (!Number.isSafeInteger(attempt) || attempt < 1) throw new TypeError("queue delivery attempt must be a positive safe integer");
+	if (!env.WORKFLOW_DB) throw new Error("missing binding: WORKFLOW_DB");
+	const namespace = env.WORKFLOW_DB;
+	const payloadStore = env.WORKFLOW_QUEUE_PAYLOADS;
+	const runStub = (name) => namespace.get(namespace.idFromName(name));
+	const claim = deliveryQueueClaim(envelope, runStub);
+	let claimed = false;
+	async function releaseClaim() {
+		if (claimed) await claim.release(envelope.messageId);
+	}
+	async function unregisterPayload() {
+		if (envelope.runId) {
+			await runStub(envelope.runId).unregisterQueuePayload(envelope.messageId);
+			await runStub(queueOrphanName(envelope.messageId)).cancelQueuePayloadOrphan(envelope.messageId);
+		}
+	}
+	try {
+		let admission;
+		if (envelope.runId && !envelope.payloadKey && envelope.claimScope !== "run") {
+			admission = runStub(envelope.runId).getQueueAdmission();
+			admission.catch(() => void 0);
+		}
+		if (claim) {
+			const result = await claim.claim(envelope.messageId, QUEUE_CLAIM_STALE_MS);
+			if (result.expired) return { kind: "complete" };
+			if (!result.claimed) {
+				const now = Date.now();
+				if (result.retryAt !== void 0 && result.retryAt > now) return {
+					kind: "suspend",
+					timeoutSeconds: Math.max(1, Math.ceil((result.retryAt - now) / 1e3))
+				};
+				return { kind: "complete" };
+			}
+			claimed = true;
+		}
+		if (admission && !(await admission).ok) {
+			await claim?.complete(envelope.messageId);
+			return { kind: "complete" };
+		}
+		let body = envelope.body;
+		if (envelope.payloadKey) {
+			if (!payloadStore) throw new Error("missing binding: WORKFLOW_QUEUE_PAYLOADS");
+			body = await payloadStore.read(envelope.payloadKey) ?? void 0;
+			if (body === void 0) {
+				await unregisterPayload();
+				await claim?.complete(envelope.messageId);
+				return { kind: "complete" };
+			}
+		}
+		const deploymentUrls = parseDeploymentUrls(env.WORKFLOW_DEPLOYMENT_URLS);
+		let targetBaseUrl = envelope.targetBaseUrl;
+		if (deploymentUrls && envelope.runId) {
+			const stored = await runStub(envelope.runId).getRun();
+			if (!stored.ok) throw new Error("world-celld: cannot route an expired run");
+			const initialId = parse(body).runInput?.deploymentId;
+			const pinnedId = stored.value?.deploymentId ?? initialId;
+			if (typeof pinnedId !== "string" || pinnedId.length === 0) throw new Error(`world-celld: no pinned deployment for run ${envelope.runId}`);
+			if (stored.value && initialId !== void 0 && initialId !== pinnedId) throw new Error(`world-celld: queue run deployment hints disagree for ${envelope.runId}`);
+			targetBaseUrl = configuredDeploymentUrl(deploymentUrls, pinnedId);
+		}
+		const headers = {
+			"content-type": "application/json",
+			"x-vqs-queue-name": envelope.queueName,
+			"x-vqs-message-id": envelope.messageId,
+			"x-vqs-message-attempt": String(attempt)
+		};
+		if (env.WORKFLOW_CALLBACK_SECRET) headers["x-workflow-callback-secret"] = env.WORKFLOW_CALLBACK_SECRET;
+		const callback = await fetch(`${targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/flow`, {
+			method: "POST",
+			headers,
+			body,
+			signal: AbortSignal.timeout(3e5)
+		});
+		if (callback.ok || !deploymentUrls && PERMANENT_QUEUE_STATUSES.has(callback.status) || !envelope.runId && PERMANENT_QUEUE_STATUSES.has(callback.status)) {
+			await callback.body?.cancel().catch(() => void 0);
+			if (envelope.payloadKey) {
+				await payloadStore.delete(envelope.payloadKey);
+				await unregisterPayload();
+			}
+			await claim?.complete(envelope.messageId);
+			return { kind: "complete" };
+		}
+		if (callback.status === 503) {
+			const responseBody = await readResponseText(callback);
+			const timeoutSeconds = JSON.parse(responseBody).timeoutSeconds;
+			const notBefore = queueDelayDeadline(Date.now(), timeoutSeconds, 1, QUEUE_CLAIM_STALE_MS);
+			if (notBefore === null) throw new Error("workflow queue suspension deadline is invalid");
+			if (envelope.payloadKey && envelope.runId) await runStub(queueOrphanName(envelope.messageId)).scheduleQueuePayloadOrphan({
+				messageId: envelope.messageId,
+				runId: envelope.runId,
+				key: envelope.payloadKey,
+				expiresAt: notBefore + QUEUE_ORPHAN_GRACE_MS
+			});
+			if (claim) await claim.hold({
+				messageId: envelope.messageId,
+				retryAt: notBefore,
+				expiresAt: notBefore + QUEUE_CLAIM_STALE_MS,
+				reservationExpiresAt: notBefore + QUEUE_ORPHAN_GRACE_MS
+			});
+			return {
+				kind: "suspend",
+				timeoutSeconds
+			};
+		}
+		await callback.body?.cancel().catch(() => void 0);
+		await releaseClaim();
+		claimed = false;
+		return { kind: "retry" };
+	} catch (error) {
+		await releaseClaim();
+		throw error;
+	}
+}
+//#endregion
 //#region src/testing/http-harness.ts
 /**
 * Boots the real worker router over real cell classes (on FakeFleet state)
@@ -4809,6 +5082,7 @@ var WorkflowRunDO = class extends DurableObject {
 async function startHarness(options = {}) {
 	const cellEnv = { ...options.cellEnv };
 	const queueMessages = [];
+	const queuePublications = [];
 	const queuePayloads = /* @__PURE__ */ new Map();
 	const queuePayloadBucket = {
 		async put(key, value) {
@@ -4822,8 +5096,12 @@ async function startHarness(options = {}) {
 			for (const key of Array.isArray(keys) ? keys : [keys]) queuePayloads.delete(key);
 		}
 	};
-	const nativeQueue = { async send(body) {
+	const nativeQueue = { async send(body, sendOptions) {
 		queueMessages.push(body);
+		queuePublications.push({
+			body,
+			delaySeconds: sendOptions?.delaySeconds ?? 0
+		});
 	} };
 	const fleet = new FakeFleet({
 		runs: WorkflowRunDO,
@@ -4842,7 +5120,7 @@ async function startHarness(options = {}) {
 		WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
 		WORKFLOW_QUEUE_PAYLOADS: queuePayloadBucket
 	});
-	const router = createRouter({
+	const env = {
 		WORKFLOW_DB: fleet.namespace("runs"),
 		WORKFLOW_STREAMS: fleet.namespace("streams"),
 		WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
@@ -4851,7 +5129,8 @@ async function startHarness(options = {}) {
 		WORKFLOW_QUEUE: nativeQueue,
 		WORKFLOW_QUEUE_PAYLOADS: createQueuePayloadStore(queuePayloadBucket),
 		WORLD_SECRET: options.secret
-	});
+	};
+	const router = createRouter(env);
 	const server = http.createServer(async (req, res) => {
 		const requestAbort = new AbortController();
 		const abortRequest = () => {
@@ -4894,7 +5173,9 @@ async function startHarness(options = {}) {
 		url: `http://127.0.0.1:${port}`,
 		fleet,
 		queueMessages,
+		queuePublications,
 		queuePayloads,
+		deliverQueueMessage: (secret, envelope, attempt) => deliverQueueMessage(env, secret, envelope, attempt),
 		close: () => new Promise((resolve, reject) => {
 			server.close((err) => err ? reject(err) : resolve());
 			server.closeAllConnections();
@@ -4902,11 +5183,148 @@ async function startHarness(options = {}) {
 	};
 }
 //#endregion
+//#region src/worker/queue-consumer.ts
+function backoffSeconds(attempt) {
+	return Math.min(60, 2 ** Math.min(attempt, 30));
+}
+async function republish(env, envelope, now = Date.now()) {
+	await env.WORKFLOW_QUEUE.send(JSON.stringify(envelope), {
+		contentType: "text",
+		delaySeconds: nativeQueueDelaySeconds(envelope.notBefore, now)
+	});
+}
+async function consume(message, env) {
+	let envelope;
+	try {
+		if (typeof message.body !== "string") throw new TypeError("native Queue message body must be text");
+		envelope = validateNativeQueueEnvelope(JSON.parse(message.body));
+	} catch (error) {
+		console.error("world-celld native Queue rejected malformed broker message", error);
+		message.retry({ delaySeconds: backoffSeconds(message.attempts) });
+		return;
+	}
+	const now = Date.now();
+	if (envelope.notBefore !== void 0 && envelope.notBefore > now) {
+		try {
+			await republish(env, envelope, now);
+			message.ack();
+		} catch (error) {
+			console.error("world-celld native Queue failed to continue a long delay", error);
+			message.retry({ delaySeconds: backoffSeconds(message.attempts) });
+		}
+		return;
+	}
+	const attempt = (envelope.deliveryFailures ?? 0) + message.attempts;
+	if (!env.WORLD_SECRET) {
+		console.error("world-celld Queue consumer missing WORLD_SECRET");
+		message.retry({ delaySeconds: backoffSeconds(attempt) });
+		return;
+	}
+	let response;
+	try {
+		response = await env.WORLD_SERVICE.deliver(env.WORLD_SECRET, envelope, attempt);
+	} catch (error) {
+		console.error("world-celld Queue consumer could not reach the world service", error);
+		message.retry({ delaySeconds: backoffSeconds(attempt) });
+		return;
+	}
+	if (response?.kind === "complete") {
+		message.ack();
+		return;
+	}
+	if (response?.kind === "suspend") try {
+		const notBefore = queueDelayDeadline(Date.now(), response.timeoutSeconds, 1, QUEUE_CLAIM_STALE_MS);
+		const suspensionCount = (envelope.suspensionCount ?? 0) + 1;
+		if (notBefore !== null && suspensionCount <= 256) {
+			await republish(env, {
+				...envelope,
+				notBefore,
+				deliveryFailures: attempt - 1,
+				suspensionCount
+			});
+			message.ack();
+			return;
+		}
+	} catch (error) {
+		console.error("world-celld Queue consumer received an invalid suspension response", error);
+	}
+	message.retry({ delaySeconds: backoffSeconds(attempt) });
+}
+var queue_consumer_default = { async queue(batch, env) {
+	await Promise.all(batch.messages.map((message) => consume(message, env)));
+} };
+//#endregion
 //#region src/testing/index.ts
 /** Alarm-pump interval for the dev fleet (real time). */
 const PUMP_INTERVAL_MS = 25;
 async function startDevFleet(options = {}) {
 	const harness = await startHarness(options);
+	const pending = [];
+	const deliveries = /* @__PURE__ */ new Set();
+	let publicationCursor = 0;
+	let stopped = false;
+	const queueEnv = {
+		WORKFLOW_QUEUE: { async send(body, sendOptions) {
+			harness.queueMessages.push(body);
+			harness.queuePublications.push({
+				body,
+				delaySeconds: sendOptions?.delaySeconds ?? 0
+			});
+		} },
+		WORLD_SERVICE: { deliver: (secret, envelope, attempt) => harness.deliverQueueMessage(secret, envelope, attempt) },
+		WORLD_SECRET: options.secret
+	};
+	function dispatch(message) {
+		let settled = false;
+		const delivery = queue_consumer_default.queue({ messages: [{
+			body: message.body,
+			attempts: message.attempts,
+			ack() {
+				settled = true;
+			},
+			retry(retryOptions) {
+				settled = true;
+				if (!stopped) pending.push({
+					body: message.body,
+					attempts: message.attempts + 1,
+					dueAt: Date.now() + (retryOptions?.delaySeconds ?? 0) * 1e3
+				});
+			}
+		}] }, queueEnv).then(() => {
+			if (!settled) {
+				console.error("[world-celld dev fleet] Queue delivery was not settled");
+				if (!stopped) pending.push({
+					body: message.body,
+					attempts: message.attempts + 1,
+					dueAt: Date.now() + 1e3
+				});
+			}
+		}).catch((error) => {
+			console.error("[world-celld dev fleet] Queue consumer failed", error);
+			if (!settled && !stopped) pending.push({
+				body: message.body,
+				attempts: message.attempts + 1,
+				dueAt: Date.now() + 1e3
+			});
+		});
+		deliveries.add(delivery);
+		delivery.finally(() => deliveries.delete(delivery));
+	}
+	const queuePump = setInterval(() => {
+		if (stopped) return;
+		while (publicationCursor < harness.queuePublications.length) {
+			const publication = harness.queuePublications[publicationCursor++];
+			pending.push({
+				body: publication.body,
+				attempts: 1,
+				dueAt: Date.now() + publication.delaySeconds * 1e3
+			});
+		}
+		const now = Date.now();
+		for (let index = 0; index < pending.length;) if (pending[index].dueAt <= now) dispatch(pending.splice(index, 1)[0]);
+		else index++;
+	}, PUMP_INTERVAL_MS);
+	queuePump.unref?.();
 	let pumping = false;
 	const pump = setInterval(async () => {
 		if (pumping) return;
@@ -4924,7 +5342,10 @@ async function startDevFleet(options = {}) {
 	return {
 		...harness,
 		async stop() {
+			stopped = true;
+			clearInterval(queuePump);
 			clearInterval(pump);
+			await Promise.allSettled(deliveries);
 			await harness.close();
 		}
 	};