@ewhauser/world-celld 0.8.1 → 0.9.0

package/dist/testing.js

@@ -1,4 +1,4 @@
-import { $ as validateStreamReadRequest, B as MAX_STREAM_READ_BYTES, C as HOOK_CLAIM_LEASE_MS, I as MAX_STREAM_BATCH_BYTES, J as encodeStreamReadResult, K as decodeStreamWriteBatch, L as MAX_STREAM_CHUNK_BYTES, M as isPositiveSafeInteger, N as isRecord$1, O as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, Q as normalizeStreamError, U as STREAM_BATCH_CONTENT_TYPE, W as STREAM_CHUNK_LIMIT_HEADER, X as encodeStreamWriteResult, _ as rpcStringify, a as QUEUE_PUBLICATION_LEASE_MS, b as createWorkflowIndex, d as queuePayloadObjectKey, et as validateStreamWriteChunks, f as queuePayloadRegistryKey, g as rpcParse, h as compact, i as QUEUE_PAYLOAD_REGISTRY_PREFIX, j as isNonNegativeSafeInteger, l as queueClaimName, m as validateNativeQueueSendOptions, n as listRunsPage, o as QUEUE_RESERVATION_GRACE_MS, p as validateNativeQueueEnvelope, r as NATIVE_QUEUE_MAX_MESSAGE_BYTES, s as isInlineRunBody, t as MAX_RUN_LIST_LIMIT, u as queueOrphanName, v as parse, w as LIFECYCLE_COMPACTION_RETRY_MS, x as runIndexMetadata, z as MAX_STREAM_LONG_POLL_MS } from "./run-list-DGsdv6Jf.js";
+import { $ as normalizeStreamError, B as MAX_STREAM_LONG_POLL_MS, C as HOOK_CLAIM_LEASE_MS, F as queueDelayDeadline, G as STREAM_CHUNK_LIMIT_HEADER, L as MAX_STREAM_BATCH_BYTES, M as isNonNegativeSafeInteger, N as isPositiveSafeInteger, O as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, P as isRecord$1, R as MAX_STREAM_CHUNK_BYTES, V as MAX_STREAM_READ_BYTES, W as STREAM_BATCH_CONTENT_TYPE, Y as encodeStreamReadResult, Z as encodeStreamWriteResult, _ as rpcStringify, a as QUEUE_PUBLICATION_LEASE_MS, b as createWorkflowIndex, c as nativeQueueDelaySeconds, d as queuePayloadObjectKey, et as validateStreamReadRequest, f as queuePayloadRegistryKey, g as rpcParse, h as compact, i as QUEUE_PAYLOAD_REGISTRY_PREFIX, k as QUEUE_CLAIM_STALE_MS, l as queueClaimName, m as validateNativeQueueSendOptions, n as listRunsPage, o as QUEUE_RESERVATION_GRACE_MS, p as validateNativeQueueEnvelope, q as decodeStreamWriteBatch, r as NATIVE_QUEUE_MAX_MESSAGE_BYTES, s as isInlineRunBody, t as MAX_RUN_LIST_LIMIT, tt as validateStreamWriteChunks, u as queueOrphanName, v as parse, w as LIFECYCLE_COMPACTION_RETRY_MS, x as runIndexMetadata } from "./run-list-C4B4SBYW.js";
 import { CreateEventSchema, EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
 import http from "node:http";
 //#region src/testing/fake-cell.ts
@@ -1361,6 +1361,11 @@ async function authenticate(request, secret) {
 }
 //#endregion
 //#region src/worker/queue-claims.ts
+/**
+* Idempotency claims for Queue messages. A claim lives either in its own
+* claim cell (named by queue and idempotency key) or, for run-bearing
+* messages whose envelope says so, as an entry in the run's own cell.
+*/
 function cellQueueClaim(cell) {
 	return {
 		claim: (messageId, staleMs) => cell.claimInflight({
@@ -1413,6 +1418,12 @@ function runQueueClaim(run, claimName) {
 		})
 	};
 }
+/** The claim a delivered envelope was reserved under, if it has an idempotency key. */
+function deliveryQueueClaim(envelope, runStub) {
+	if (!envelope.idempotencyKey) return void 0;
+	const claimName = queueClaimName(envelope.queueName, envelope.idempotencyKey);
+	return envelope.claimScope === "run" && envelope.runId ? runQueueClaim(runStub(envelope.runId), claimName) : cellQueueClaim(runStub(claimName));
+}
 //#endregion
 //#region src/worker/index-validation.ts
 function invalid(message) {
@@ -4801,6 +4812,146 @@ var WorkflowRunDO = class extends DurableObject {
 	}
 };
 //#endregion
+//#region src/worker/queue-delivery.ts
+/** Internal delivery contract. No Request/Response objects cross the RPC boundary. */
+const MAX_QUEUE_DELIVERY_RESPONSE_BYTES = 65536;
+const QUEUE_ORPHAN_GRACE_MS = 432e6;
+const PERMANENT_QUEUE_STATUSES = /* @__PURE__ */ new Set([
+	404,
+	409,
+	410,
+	422
+]);
+async function readResponseText(response) {
+	if (!response.body) return "";
+	const reader = response.body.getReader();
+	const decoder = new TextDecoder();
+	let total = 0;
+	let text = "";
+	try {
+		for (;;) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			total += value.byteLength;
+			if (total > MAX_QUEUE_DELIVERY_RESPONSE_BYTES) {
+				await reader.cancel("queue delivery response is too large").catch(() => void 0);
+				throw new Error("workflow queue handler response exceeds configured limit");
+			}
+			text += decoder.decode(value, { stream: true });
+		}
+		return text + decoder.decode();
+	} finally {
+		reader.releaseLock();
+	}
+}
+/** Validate at the service boundary; callers cannot bypass auth via RPC. */
+async function deliverQueueMessage(env, secret, input, attempt) {
+	if (!env.WORLD_SECRET) throw new Error("WORLD_SECRET is not configured");
+	if (typeof secret !== "string" || !secret || !await timingSafeEqual(secret, env.WORLD_SECRET)) throw new Error("Unauthorized Queue delivery");
+	const envelope = validateNativeQueueEnvelope(input);
+	if (!Number.isSafeInteger(attempt) || attempt < 1) throw new TypeError("queue delivery attempt must be a positive safe integer");
+	if (!env.WORKFLOW_DB) throw new Error("missing binding: WORKFLOW_DB");
+	const namespace = env.WORKFLOW_DB;
+	const payloadStore = env.WORKFLOW_QUEUE_PAYLOADS;
+	const runStub = (name) => namespace.get(namespace.idFromName(name));
+	const claim = deliveryQueueClaim(envelope, runStub);
+	let claimed = false;
+	async function releaseClaim() {
+		if (claimed) await claim.release(envelope.messageId);
+	}
+	async function unregisterPayload() {
+		if (envelope.runId) {
+			await runStub(envelope.runId).unregisterQueuePayload(envelope.messageId);
+			await runStub(queueOrphanName(envelope.messageId)).cancelQueuePayloadOrphan(envelope.messageId);
+		}
+	}
+	try {
+		let admission;
+		if (envelope.runId && !envelope.payloadKey && envelope.claimScope !== "run") {
+			admission = runStub(envelope.runId).getQueueAdmission();
+			admission.catch(() => void 0);
+		}
+		if (claim) {
+			const result = await claim.claim(envelope.messageId, QUEUE_CLAIM_STALE_MS);
+			if (result.expired) return { kind: "complete" };
+			if (!result.claimed) {
+				const now = Date.now();
+				if (result.retryAt !== void 0 && result.retryAt > now) return {
+					kind: "suspend",
+					timeoutSeconds: Math.max(1, Math.ceil((result.retryAt - now) / 1e3))
+				};
+				return { kind: "complete" };
+			}
+			claimed = true;
+		}
+		if (admission && !(await admission).ok) {
+			await claim?.complete(envelope.messageId);
+			return { kind: "complete" };
+		}
+		let body = envelope.body;
+		if (envelope.payloadKey) {
+			if (!payloadStore) throw new Error("missing binding: WORKFLOW_QUEUE_PAYLOADS");
+			body = await payloadStore.read(envelope.payloadKey) ?? void 0;
+			if (body === void 0) {
+				await unregisterPayload();
+				await claim?.complete(envelope.messageId);
+				return { kind: "complete" };
+			}
+		}
+		const headers = {
+			"content-type": "application/json",
+			"x-vqs-queue-name": envelope.queueName,
+			"x-vqs-message-id": envelope.messageId,
+			"x-vqs-message-attempt": String(attempt)
+		};
+		if (env.WORKFLOW_CALLBACK_SECRET) headers["x-workflow-callback-secret"] = env.WORKFLOW_CALLBACK_SECRET;
+		const callback = await fetch(`${envelope.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/flow`, {
+			method: "POST",
+			headers,
+			body,
+			signal: AbortSignal.timeout(3e5)
+		});
+		if (callback.ok || PERMANENT_QUEUE_STATUSES.has(callback.status)) {
+			await callback.body?.cancel().catch(() => void 0);
+			if (envelope.payloadKey) {
+				await payloadStore.delete(envelope.payloadKey);
+				await unregisterPayload();
+			}
+			await claim?.complete(envelope.messageId);
+			return { kind: "complete" };
+		}
+		if (callback.status === 503) {
+			const responseBody = await readResponseText(callback);
+			const timeoutSeconds = JSON.parse(responseBody).timeoutSeconds;
+			const notBefore = queueDelayDeadline(Date.now(), timeoutSeconds, 1, QUEUE_CLAIM_STALE_MS);
+			if (notBefore === null) throw new Error("workflow queue suspension deadline is invalid");
+			if (envelope.payloadKey && envelope.runId) await runStub(queueOrphanName(envelope.messageId)).scheduleQueuePayloadOrphan({
+				messageId: envelope.messageId,
+				runId: envelope.runId,
+				key: envelope.payloadKey,
+				expiresAt: notBefore + QUEUE_ORPHAN_GRACE_MS
+			});
+			if (claim) await claim.hold({
+				messageId: envelope.messageId,
+				retryAt: notBefore,
+				expiresAt: notBefore + QUEUE_CLAIM_STALE_MS,
+				reservationExpiresAt: notBefore + QUEUE_ORPHAN_GRACE_MS
+			});
+			return {
+				kind: "suspend",
+				timeoutSeconds
+			};
+		}
+		await callback.body?.cancel().catch(() => void 0);
+		await releaseClaim();
+		claimed = false;
+		return { kind: "retry" };
+	} catch (error) {
+		await releaseClaim();
+		throw error;
+	}
+}
+//#endregion
 //#region src/testing/http-harness.ts
 /**
 * Boots the real worker router over real cell classes (on FakeFleet state)
@@ -4809,6 +4960,7 @@ var WorkflowRunDO = class extends DurableObject {
 async function startHarness(options = {}) {
 	const cellEnv = { ...options.cellEnv };
 	const queueMessages = [];
+	const queuePublications = [];
 	const queuePayloads = /* @__PURE__ */ new Map();
 	const queuePayloadBucket = {
 		async put(key, value) {
@@ -4822,8 +4974,12 @@ async function startHarness(options = {}) {
 			for (const key of Array.isArray(keys) ? keys : [keys]) queuePayloads.delete(key);
 		}
 	};
-	const nativeQueue = { async send(body) {
+	const nativeQueue = { async send(body, sendOptions) {
 		queueMessages.push(body);
+		queuePublications.push({
+			body,
+			delaySeconds: sendOptions?.delaySeconds ?? 0
+		});
 	} };
 	const fleet = new FakeFleet({
 		runs: WorkflowRunDO,
@@ -4842,7 +4998,7 @@ async function startHarness(options = {}) {
 		WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
 		WORKFLOW_QUEUE_PAYLOADS: queuePayloadBucket
 	});
-	const router = createRouter({
+	const env = {
 		WORKFLOW_DB: fleet.namespace("runs"),
 		WORKFLOW_STREAMS: fleet.namespace("streams"),
 		WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
@@ -4851,7 +5007,8 @@ async function startHarness(options = {}) {
 		WORKFLOW_QUEUE: nativeQueue,
 		WORKFLOW_QUEUE_PAYLOADS: createQueuePayloadStore(queuePayloadBucket),
 		WORLD_SECRET: options.secret
-	});
+	};
+	const router = createRouter(env);
 	const server = http.createServer(async (req, res) => {
 		const requestAbort = new AbortController();
 		const abortRequest = () => {
@@ -4894,7 +5051,9 @@ async function startHarness(options = {}) {
 		url: `http://127.0.0.1:${port}`,
 		fleet,
 		queueMessages,
+		queuePublications,
 		queuePayloads,
+		deliverQueueMessage: (secret, envelope, attempt) => deliverQueueMessage(env, secret, envelope, attempt),
 		close: () => new Promise((resolve, reject) => {
 			server.close((err) => err ? reject(err) : resolve());
 			server.closeAllConnections();
@@ -4902,11 +5061,148 @@ async function startHarness(options = {}) {
 	};
 }
 //#endregion
+//#region src/worker/queue-consumer.ts
+function backoffSeconds(attempt) {
+	return Math.min(60, 2 ** Math.min(attempt, 30));
+}
+async function republish(env, envelope, now = Date.now()) {
+	await env.WORKFLOW_QUEUE.send(JSON.stringify(envelope), {
+		contentType: "text",
+		delaySeconds: nativeQueueDelaySeconds(envelope.notBefore, now)
+	});
+}
+async function consume(message, env) {
+	let envelope;
+	try {
+		if (typeof message.body !== "string") throw new TypeError("native Queue message body must be text");
+		envelope = validateNativeQueueEnvelope(JSON.parse(message.body));
+	} catch (error) {
+		console.error("world-celld native Queue rejected malformed broker message", error);
+		message.retry({ delaySeconds: backoffSeconds(message.attempts) });
+		return;
+	}
+	const now = Date.now();
+	if (envelope.notBefore !== void 0 && envelope.notBefore > now) {
+		try {
+			await republish(env, envelope, now);
+			message.ack();
+		} catch (error) {
+			console.error("world-celld native Queue failed to continue a long delay", error);
+			message.retry({ delaySeconds: backoffSeconds(message.attempts) });
+		}
+		return;
+	}
+	const attempt = (envelope.deliveryFailures ?? 0) + message.attempts;
+	if (!env.WORLD_SECRET) {
+		console.error("world-celld Queue consumer missing WORLD_SECRET");
+		message.retry({ delaySeconds: backoffSeconds(attempt) });
+		return;
+	}
+	let response;
+	try {
+		response = await env.WORLD_SERVICE.deliver(env.WORLD_SECRET, envelope, attempt);
+	} catch (error) {
+		console.error("world-celld Queue consumer could not reach the world service", error);
+		message.retry({ delaySeconds: backoffSeconds(attempt) });
+		return;
+	}
+	if (response?.kind === "complete") {
+		message.ack();
+		return;
+	}
+	if (response?.kind === "suspend") try {
+		const notBefore = queueDelayDeadline(Date.now(), response.timeoutSeconds, 1, QUEUE_CLAIM_STALE_MS);
+		const suspensionCount = (envelope.suspensionCount ?? 0) + 1;
+		if (notBefore !== null && suspensionCount <= 256) {
+			await republish(env, {
+				...envelope,
+				notBefore,
+				deliveryFailures: attempt - 1,
+				suspensionCount
+			});
+			message.ack();
+			return;
+		}
+	} catch (error) {
+		console.error("world-celld Queue consumer received an invalid suspension response", error);
+	}
+	message.retry({ delaySeconds: backoffSeconds(attempt) });
+}
+var queue_consumer_default = { async queue(batch, env) {
+	await Promise.all(batch.messages.map((message) => consume(message, env)));
+} };
+//#endregion
 //#region src/testing/index.ts
 /** Alarm-pump interval for the dev fleet (real time). */
 const PUMP_INTERVAL_MS = 25;
 async function startDevFleet(options = {}) {
 	const harness = await startHarness(options);
+	const pending = [];
+	const deliveries = /* @__PURE__ */ new Set();
+	let publicationCursor = 0;
+	let stopped = false;
+	const queueEnv = {
+		WORKFLOW_QUEUE: { async send(body, sendOptions) {
+			harness.queueMessages.push(body);
+			harness.queuePublications.push({
+				body,
+				delaySeconds: sendOptions?.delaySeconds ?? 0
+			});
+		} },
+		WORLD_SERVICE: { deliver: (secret, envelope, attempt) => harness.deliverQueueMessage(secret, envelope, attempt) },
+		WORLD_SECRET: options.secret
+	};
+	function dispatch(message) {
+		let settled = false;
+		const delivery = queue_consumer_default.queue({ messages: [{
+			body: message.body,
+			attempts: message.attempts,
+			ack() {
+				settled = true;
+			},
+			retry(retryOptions) {
+				settled = true;
+				if (!stopped) pending.push({
+					body: message.body,
+					attempts: message.attempts + 1,
+					dueAt: Date.now() + (retryOptions?.delaySeconds ?? 0) * 1e3
+				});
+			}
+		}] }, queueEnv).then(() => {
+			if (!settled) {
+				console.error("[world-celld dev fleet] Queue delivery was not settled");
+				if (!stopped) pending.push({
+					body: message.body,
+					attempts: message.attempts + 1,
+					dueAt: Date.now() + 1e3
+				});
+			}
+		}).catch((error) => {
+			console.error("[world-celld dev fleet] Queue consumer failed", error);
+			if (!settled && !stopped) pending.push({
+				body: message.body,
+				attempts: message.attempts + 1,
+				dueAt: Date.now() + 1e3
+			});
+		});
+		deliveries.add(delivery);
+		delivery.finally(() => deliveries.delete(delivery));
+	}
+	const queuePump = setInterval(() => {
+		if (stopped) return;
+		while (publicationCursor < harness.queuePublications.length) {
+			const publication = harness.queuePublications[publicationCursor++];
+			pending.push({
+				body: publication.body,
+				attempts: 1,
+				dueAt: Date.now() + publication.delaySeconds * 1e3
+			});
+		}
+		const now = Date.now();
+		for (let index = 0; index < pending.length;) if (pending[index].dueAt <= now) dispatch(pending.splice(index, 1)[0]);
+		else index++;
+	}, PUMP_INTERVAL_MS);
+	queuePump.unref?.();
 	let pumping = false;
 	const pump = setInterval(async () => {
 		if (pumping) return;
@@ -4924,7 +5220,10 @@ async function startDevFleet(options = {}) {
 	return {
 		...harness,
 		async stop() {
+			stopped = true;
+			clearInterval(queuePump);
 			clearInterval(pump);
+			await Promise.allSettled(deliveries);
 			await harness.close();
 		}
 	};