@ewhauser/world-celld 0.8.0 → 0.8.1

package/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
+## 0.8.1 (2026-10-02)
+
+## What's Changed
+* fix: harden workflow fault recovery by @ewhauser in https://github.com/ewhauser/world-celld/pull/87
+* chore(deps): bump hono from 4.13.2 to 4.13.7 by @dependabot[bot] in https://github.com/ewhauser/world-celld/pull/88
+
+
+**Full Changelog**: https://github.com/ewhauser/world-celld/compare/v0.8.0...v0.8.1
+
 ## 0.8.0 (2026-09-27)
 
 ## What's Changed

package/README.md

@@ -316,6 +316,11 @@ fleet-wide maximum age are both enabled, the earlier deadline wins.
 
 - Delivery is at least once. Workflow steps and other external side effects
   must be idempotent.
+- A concurrent enqueue with the same idempotency key returns a retryable `503`
+  while broker publication is pending. Publication confirmation starts the
+  normal deduplication window. If a producer stops before publication completes,
+  its pending reservation can be replaced after five minutes. Broker delivery
+  also confirms publication when the producer's confirmation was interrupted.
 - Native Queue operations are administered with celld's `queue info`, `peek`,
   `pause`, `resume`, `purge`, and `redrive` commands.
 - Run cells expose retention status, scheduling, immediate cleanup, and alarm

package/dist/index.d.ts

@@ -175,6 +175,11 @@ interface WorkflowRunDOStub {
   }): Promise<{
     committed: boolean;
   }>;
+  resolveExpiredHookClaim(request: {
+    hookId: string;
+    claimId: string;
+    token?: string;
+  }): Promise<Hook | null>;
   getLifecycleStatus(): Promise<RunLifecycleStatus>;
   getRun(): Promise<RunReadOutcome<WorkflowRun | null>>;
   getStep(stepId: string): Promise<RunReadOutcome<Step | null>>;

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../src/retention.ts","../src/apply-event.ts","../src/storage.ts","../src/stream-protocol.ts","../src/streamer.ts","../src/queue-protocol.ts","../src/queue.ts","../src/run-list.ts","../src/indexes.ts","../src/config.ts","../src/index.ts"],"mappings":";;KAOY;UAEK;EACf;EACA;EACA;EACA,WAAW;;EAEX,SAAS;;EAET,cAAc;;EAEd,iBAAiB;EACjB;EACA,OAAO;EACP,OAAO;;EAEP;EACA;EACA;EACA,gBAAgB;EAChB,eAAe;EACf;EACA;;EAEA;;UAGe;EACf;EACA;;EAEA,SAAS;EACT,cAAc;EACd,iBAAiB;EACjB,WAAW;EACX,cAAc;;EAEd,SAAS;;KAGC;KAEA,eAAe;EACrB;EAAU,OAAO;;EACjB;EAAW;EAAqB;EAAiB,WAAW;;UAEjD;EACf;EACA;;UAae;EACf;EACA,OAAO;;UAGQ;EACf;;UAGe;EACf;EACA,OAAO;EACP;;UAGe;EACf;;UAGe;EACf;;UAGe;;EAEf;EACA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;;UAGe;EACf;;;;UC0Ce;;EAEf;EACA,MAAM,qBAAqB;;EAE3B,SAAS;;;;;EAKT;IAAgB;IAAe;;;EAE/B;;EAEA,UAAU;;KAiFA;UAWK;EACf;EACA,MAAM;EACN;;EAEA;;EAEA;;UAGe;EACf;;EAEA,QAAQ;EACR,MAAM;EACN,OAAO;EACP,OAAO;EACP,OAAO;;EAEP;;EAEA,eAAe;IAAQ;IAAgB;;;EAEvC;IAAe;IAAsB,WAAW;;;EAEhD,cAAc;;EAEd,SAAS;;EAET;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;EAKA,YAAY;;KAGF,oBAAoB,oBAAoB;;;;;;;UClPnC;EACf,WAAW,SAAS,oBAAoB,QAAQ;EAChD,sBAAsB;IACpB;IACA;IACA;MACE;IAAU;;EACd,sBAAsB,QADlB;EAEJ,UAAU,QAAQ,eAAe;EACjC,QAAQ,iBAAiB,QAAQ,eAAe;EAChD,SAAS,kBAAkB,QAAQ,eAAe;EAClD,WAAW;IACT;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAS;IAAuB;;EACnE,UAAU;IACR;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAQ;IAAuB;;EAClE,UAAU;IACR;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAQ;IAAuB;;EAClE,oBAAoB,QADhB;EAEJ,gBACE,SAFkB,yBAGjB,QADiC;EAEpC,WACE,SAFC,yBAGA,QADiC;EAEpC,gBAAgB,QADb;;UAIY;EACf,WAAW,eAAe;EAC1B,IAAI,IAAI,sBAAsB;;UAGf;EACf;;;;;;;;cC3FW;;cAEA;;cAEA;;cAEA;cAKA;cACA;;cAEA;KAED;UAEK;EACf;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,QAAQ;EACR,OAAO;EACP;EACA,QAAQ;;;;UClBO,sBAAsB,KAAK;EAC1C,SAAS;IACP,KAAK,eAAe,cAAc,OAAO,2BAA2B;;EAEtE,cACE,cACA,gBAAgB,iBAChB,gBAAgB,aACf;EACH,oBACE,cACA,gBAAgB,iBAChB,2BAA2B,gBAC1B;EACH,YAAY,cAAc,gBAAgB,kBAAkB;EAC5D,YACE,cACA,gBAAgB,iBAChB,OAAO,2BACN;EACH,eACE,cACA,eACA,sBACC,QAAQ,eAAe;EAC1B,mBAAmB,gBAAgB;EACnC,gBACE,cACA,eACA,UAAU,mBACT,QAAQ;EACX,cAAc,cAAc,gBAAgB,QAAQ;;;UAIrC;;;;;EAKf;;EAEA;EACA,YAAY,eAAe,QAAQ,eAAe,QAAQ;EAC1D,WAAW,SAAS,mBAAmB,SAAS,cAAc,QAAQ;EACtE,YAAY,gBAAgB;EAC5B,WAAW,eAAe,OAAO,2BAA2B;EAC5D,eAAe,eAAe,eAAe;EAC7C,eAAe;EACf,eACE,eACA,mBACA;IAAY;MACX,QAAQ;EACX,iBAAiB,eAAe,oBAAoB,QAAQ;EAC5D,aACE,eACA,mBACA;IAAY;IAAgB;MAC3B,QAAQ;;UAGI;EACf,WAAW,eAAe;EAC1B,IAAI,IAAI,aAAa;;UAGN;EACf;;;;;;;;;;;UCpDe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;;;;EAMA;;EAEA;;;;;EAKA;;EAEA;EACA;;UAae;EACf;;UAGe;EACf;;;;UCGe;EACf,KACE,UAAU,qBACV,UAAU,yBACT,QAAQ;;;;KC7FD,YAAY;;UAGP;EACf;EACA,WAAW;EACX;EACA;EACA;;EAEA;;UAGe;EACf,MAAM;EACN;EACA;;;;UCPe;EACf;EACA;;EAEA;EACA;EACA;;UAGe;;EAEf,MAAM;IAAQ;IAAc;IAAe;;EAC3C;EACA;;UAuBe;EACf;EACA;EACA;;KAGU;EACN;EAAgB,aAAa;;EAC7B;EAAiB,QAAQ;;;UAgDd;EACf,UACE,KAAK,aACL,4BACA,+BACC;IAAU;;EACb,SAAS,UAAU,mBAAmB,QAAQ;;;;;EAK9C,kBAAkB,SAAS,iBAAiB,QAAQ;EACpD,eAAe,gBAAgB;EAC/B,YAAY,iBAAiB;EAC7B,YAAY,eAAe,OAAO,iBAAiB,QAAQ;EAC3D,oBACE,eACA,gBACA,wBACA,OAAO,gBACP,cAAc,kBACb;EACH,uBACE,eACA,OAAO,gBACP,aAAa,kBACZ;EACH,mBAAmB,SAAS,4BAA4B,QAAQ;EAChE,UAAU,SAAS,0BAA0B,QAAQ;;;;;KChI3C,iBAAiB;UAEZ;EACf;EACA;;UAGe;EACf,aAAa;EACb,gBAAgB;EAChB,gBAAgB;EAChB,kBAAkB;;UAGH;;;;;;;EAOf;;;;;EAKA;;;;;EAKA,MAAM;;EAEN;;;;;;EAMA;;;;;;EAMA;;EAEA;;EAEA;;EAEA;;;;UCpBe;EACf,UAAU,gBAAgB,QAAQ;EAClC,SAAS,gBAAgB,QAAQ;EACjC,WAAW,gBAAgB,QAAQ;EACnC,MAAM,gBAAgB,QAAQ;;KAGpB,aAAa,QAAQ;EAAkB,WAAW;;iBAS9C,iBAAiB,SAAS,mBAAmB"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../src/retention.ts","../src/apply-event.ts","../src/storage.ts","../src/stream-protocol.ts","../src/streamer.ts","../src/queue-protocol.ts","../src/queue.ts","../src/run-list.ts","../src/indexes.ts","../src/config.ts","../src/index.ts"],"mappings":";;KAOY;UAEK;EACf;EACA;EACA;EACA,WAAW;;EAEX,SAAS;;EAET,cAAc;;EAEd,iBAAiB;EACjB;EACA,OAAO;EACP,OAAO;;EAEP;EACA;EACA;EACA,gBAAgB;EAChB,eAAe;EACf;EACA;;EAEA;;UAGe;EACf;EACA;;EAEA,SAAS;EACT,cAAc;EACd,iBAAiB;EACjB,WAAW;EACX,cAAc;;EAEd,SAAS;;KAGC;KAEA,eAAe;EACrB;EAAU,OAAO;;EACjB;EAAW;EAAqB;EAAiB,WAAW;;UAEjD;EACf;EACA;;UAae;EACf;EACA,OAAO;;UAGQ;EACf;;UAGe;EACf;EACA,OAAO;EACP;;UAGe;EACf;;UAGe;EACf;;UAGe;;EAEf;EACA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;;UAGe;EACf;;;;UC0Ce;;EAEf;EACA,MAAM,qBAAqB;;EAE3B,SAAS;;;;;EAKT;IAAgB;IAAe;;;EAE/B;;EAEA,UAAU;;KAiFA;UAWK;EACf;EACA,MAAM;EACN;;EAEA;;EAEA;;UAGe;EACf;;EAEA,QAAQ;EACR,MAAM;EACN,OAAO;EACP,OAAO;EACP,OAAO;;EAEP;;EAEA,eAAe;IAAQ;IAAgB;;;EAEvC;IAAe;IAAsB,WAAW;;;EAEhD,cAAc;;EAEd,SAAS;;EAET;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;EAKA,YAAY;;KAGF,oBAAoB,oBAAoB;;;;;;;UClPnC;EACf,WAAW,SAAS,oBAAoB,QAAQ;EAChD,sBAAsB;IACpB;IACA;IACA;MACE;IAAU;;EACd,wBAAwB;IACtB;IACA;IACA;MACE,QAAQ;EACZ,sBAAsB,QADlB;EAEJ,UAAU,QAAQ,eAAe;EACjC,QAAQ,iBAAiB,QAAQ,eAAe;EAChD,SAAS,kBAAkB,QAAQ,eAAe;EAClD,WAAW;IACT;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAS;IAAuB;;EACnE,UAAU;IACR;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAQ;IAAuB;;EAClE,UAAU;IACR;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAQ;IAAuB;;EAClE,oBAAoB,QADhB;EAEJ,gBACE,SAFkB,yBAGjB,QADiC;EAEpC,WACE,SAFC,yBAGA,QADiC;EAEpC,gBAAgB,QADb;;UAIY;EACf,WAAW,eAAe;EAC1B,IAAI,IAAI,sBAAsB;;UAGf;EACf;;;;;;;;cChGW;;cAEA;;cAEA;;cAEA;cAKA;cACA;;cAEA;KAED;UAEK;EACf;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,QAAQ;EACR,OAAO;EACP;EACA,QAAQ;;;;UClBO,sBAAsB,KAAK;EAC1C,SAAS;IACP,KAAK,eAAe,cAAc,OAAO,2BAA2B;;EAEtE,cACE,cACA,gBAAgB,iBAChB,gBAAgB,aACf;EACH,oBACE,cACA,gBAAgB,iBAChB,2BAA2B,gBAC1B;EACH,YAAY,cAAc,gBAAgB,kBAAkB;EAC5D,YACE,cACA,gBAAgB,iBAChB,OAAO,2BACN;EACH,eACE,cACA,eACA,sBACC,QAAQ,eAAe;EAC1B,mBAAmB,gBAAgB;EACnC,gBACE,cACA,eACA,UAAU,mBACT,QAAQ;EACX,cAAc,cAAc,gBAAgB,QAAQ;;;UAIrC;;;;;EAKf;;EAEA;EACA,YAAY,eAAe,QAAQ,eAAe,QAAQ;EAC1D,WAAW,SAAS,mBAAmB,SAAS,cAAc,QAAQ;EACtE,YAAY,gBAAgB;EAC5B,WAAW,eAAe,OAAO,2BAA2B;EAC5D,eAAe,eAAe,eAAe;EAC7C,eAAe;EACf,eACE,eACA,mBACA;IAAY;MACX,QAAQ;EACX,iBAAiB,eAAe,oBAAoB,QAAQ;EAC5D,aACE,eACA,mBACA;IAAY;IAAgB;MAC3B,QAAQ;;UAGI;EACf,WAAW,eAAe;EAC1B,IAAI,IAAI,aAAa;;UAGN;EACf;;;;;;;;;;;UCjDe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;;;;;EAMA;;EAEA;;;;;EAKA;;EAEA;EACA;;UAae;EACf;;UAGe;EACf;;;;UCCe;EACf,KACE,UAAU,qBACV,UAAU,yBACT,QAAQ;;;;KC9FD,YAAY;;UAGP;EACf;EACA,WAAW;EACX;EACA;EACA;;EAEA;;UAGe;EACf,MAAM;EACN;EACA;;;;UCPe;EACf;EACA;;EAEA;EACA;EACA;;UAGe;;EAEf,MAAM;IAAQ;IAAc;IAAe;;EAC3C;EACA;;UAuBe;EACf;EACA;EACA;;KAGU;EACN;EAAgB,aAAa;;EAC7B;EAAiB,QAAQ;;;UAgDd;EACf,UACE,KAAK,aACL,4BACA,+BACC;IAAU;;EACb,SAAS,UAAU,mBAAmB,QAAQ;;;;;EAK9C,kBAAkB,SAAS,iBAAiB,QAAQ;EACpD,eAAe,gBAAgB;EAC/B,YAAY,iBAAiB;EAC7B,YAAY,eAAe,OAAO,iBAAiB,QAAQ;EAC3D,oBACE,eACA,gBACA,wBACA,OAAO,gBACP,cAAc,kBACb;EACH,uBACE,eACA,OAAO,gBACP,aAAa,kBACZ;EACH,mBAAmB,SAAS,4BAA4B,QAAQ;EAChE,UAAU,SAAS,0BAA0B,QAAQ;;;;;KChI3C,iBAAiB;UAEZ;EACf;EACA;;UAGe;EACf,aAAa;EACb,gBAAgB;EAChB,gBAAgB;EAChB,kBAAkB;;UAGH;;;;;;;EAOf;;;;;EAKA;;;;;EAKA,MAAM;;EAEN;;;;;;EAMA;;;;;;EAMA;;EAEA;;EAEA;;EAEA;;;;UCpBe;EACf,UAAU,gBAAgB,QAAQ;EAClC,SAAS,gBAAgB,QAAQ;EACjC,WAAW,gBAAgB,QAAQ;EACnC,MAAM,gBAAgB,QAAQ;;KAGpB,aAAa,QAAQ;EAAkB,WAAW;;iBAS9C,iBAAiB,SAAS,mBAAmB"}

package/dist/index.js

@@ -1,4 +1,4 @@
-import { B as MAX_STREAM_WRITE_CHUNKS, C as MAX_FLEET_RPC_TIMEOUT_MS, D as boundedIntegerOption, F as MAX_STREAM_CHUNK_BYTES, G as decodeStreamWriteResult, H as STREAM_CHUNK_LIMIT_HEADER, I as MAX_STREAM_ERROR_BYTES, L as MAX_STREAM_LONG_POLL_MS, M as queueDelayDeadline, N as strictIntegerSetting, O as checkedQueueTimestampAdd, P as MAX_STREAM_BATCH_BYTES, R as MAX_STREAM_READ_BYTES, T as MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS, U as decodeStreamReadResult, V as STREAM_BATCH_CONTENT_TYPE, X as normalizeStreamError, Y as isChunkLimitRejection, _ as stringify, g as parse, h as rpcStringify, m as rpcParse, n as listRunsPage, o as nativeQueueDelaySeconds, p as compact, q as encodeStreamWriteBatch, v as createWorkflowIndex, w as MAX_QUEUE_DELIVERY_TIMEOUT_MS, y as runIndexMetadata, z as MAX_STREAM_READ_CHUNKS } from "./run-list-C987PkJ2.js";
+import { A as checkedQueueTimestampAdd, B as MAX_STREAM_READ_BYTES, D as MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS, E as MAX_QUEUE_DELIVERY_TIMEOUT_MS, F as strictIntegerSetting, G as decodeStreamReadResult, H as MAX_STREAM_WRITE_CHUNKS, I as MAX_STREAM_BATCH_BYTES, L as MAX_STREAM_CHUNK_BYTES, P as queueDelayDeadline, Q as normalizeStreamError, R as MAX_STREAM_ERROR_BYTES, T as MAX_FLEET_RPC_TIMEOUT_MS, U as STREAM_BATCH_CONTENT_TYPE, V as MAX_STREAM_READ_CHUNKS, W as STREAM_CHUNK_LIMIT_HEADER, Y as encodeStreamWriteBatch, Z as isChunkLimitRejection, _ as rpcStringify, b as createWorkflowIndex, c as nativeQueueDelaySeconds, g as rpcParse, h as compact, k as boundedIntegerOption, l as queueClaimName, n as listRunsPage, q as decodeStreamWriteResult, v as parse, x as runIndexMetadata, y as stringify, z as MAX_STREAM_LONG_POLL_MS } from "./run-list-DGsdv6Jf.js";
 import { CreateEventSchema, EventSchema, HookSchema, MessageId, QueuePayloadSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, parseQueueName } from "@workflow/world";
 import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
 import { monotonicFactory } from "ulid";
@@ -106,7 +106,13 @@ async function callFleetRoute(transport, path, args, opts) {
 			if (attempt < attempts) continue;
 			throw lastError;
 		}
-		const text = await response.text();
+		let text;
+		try {
+			text = await response.text();
+		} catch (error) {
+			lastError = new FleetTransportError(`world-celld: interrupted error response from ${url}`, error);
+			continue;
+		}
 		let wire;
 		try {
 			wire = JSON.parse(text).error;
@@ -240,7 +246,9 @@ async function writeStreamChunks(transport, name, runId, chunks) {
 	if (!response.ok) throw await errorFromResponse(response);
 	recordStreamChunkLimit(transport, response);
 	try {
-		return decodeStreamWriteResult(await readBoundedResponse(response, MAX_WRITE_RESPONSE_BYTES));
+		const result = decodeStreamWriteResult(await readBoundedResponse(response, MAX_WRITE_RESPONSE_BYTES));
+		if (result.count !== chunks.length) throw new Error("Stream write acknowledgement does not match the requested chunk count");
+		return result;
 	} catch (error) {
 		if (error instanceof FleetTransportError) throw error;
 		throw new FleetTransportError(`world-celld: malformed stream response from ${url.href}`, error);
@@ -256,6 +264,7 @@ async function readStreamChunks(transport, name, request, signal) {
 	url.searchParams.set("waitMs", String(request.waitMs));
 	const doFetch = transport.fetchImpl ?? fetch;
 	let lastError;
+	let maxChunks = request.maxChunks;
 	for (let attempt = 1; attempt <= READ_ATTEMPTS; attempt++) {
 		if (signal?.aborted) throw aborted(signal);
 		if (attempt > 1) await delayMs(100 * attempt + Math.random() * 200, signal);
@@ -278,7 +287,9 @@ async function readStreamChunks(transport, name, request, signal) {
 		if (response.ok) {
 			recordStreamChunkLimit(transport, response);
 			try {
-				return decodeStreamReadResult(await readBoundedResponse(response, MAX_READ_RESPONSE_BYTES));
+				const result = decodeStreamReadResult(await readBoundedResponse(response, MAX_READ_RESPONSE_BYTES));
+				if (result.startIndex !== request.startIndex || result.chunks.length > maxChunks || result.chunks.reduce((bytes, chunk) => bytes + chunk.byteLength, 0) > request.maxBytes || result.chunks.length > 0 && result.startIndex + result.chunks.length - 1 > result.tailIndex || maxChunks > 0 && result.chunks.length === 0 && result.startIndex <= result.tailIndex && result.state !== "expired") throw new Error("Stream read response does not match the requested range and bounds");
+				return result;
 			} catch (error) {
 				if (signal?.aborted) throw aborted(signal);
 				lastError = error instanceof FleetTransportError ? error : new FleetTransportError(`world-celld: malformed stream response from ${url.href}`, error);
@@ -299,6 +310,7 @@ async function readStreamChunks(transport, name, request, signal) {
 		}
 		if (request.maxChunks > 32 && isChunkLimitRejection(error) && attempt < READ_ATTEMPTS) {
 			resetStreamChunkLimit(transport);
+			maxChunks = 32;
 			url.searchParams.set("maxChunks", String(32));
 			lastError = error;
 			continue;
@@ -323,14 +335,16 @@ const RUNS = {
 		"scheduleCleanup",
 		"cleanupNow",
 		"rearmCleanup",
-		"resolveHookTokenClaim"
+		"resolveHookTokenClaim",
+		"resolveExpiredHookClaim"
 	],
 	mutating: /* @__PURE__ */ new Set([
 		"applyEvent",
 		"scheduleCleanup",
 		"cleanupNow",
 		"rearmCleanup",
-		"resolveHookTokenClaim"
+		"resolveHookTokenClaim",
+		"resolveExpiredHookClaim"
 	])
 };
 const STREAMS = {
@@ -581,7 +595,7 @@ function createTestPump(config) {
 	const inflightMessages = /* @__PURE__ */ new Map();
 	let running = false;
 	function release(envelope) {
-		if (envelope.idempotencyKey) inflightMessages.delete(envelope.idempotencyKey);
+		if (envelope.idempotencyKey) inflightMessages.delete(queueClaimName(envelope.queueName, envelope.idempotencyKey));
 	}
 	function enqueue(pathname, envelope) {
 		queues[pathname].push(envelope);
@@ -594,6 +608,24 @@ function createTestPump(config) {
 			wakers[pathname].push(() => resolve(queues[pathname].shift() ?? null));
 		});
 	}
+	function retry(envelope, pathname, reason) {
+		if (envelope.attempt >= maxAttempts) {
+			release(envelope);
+			console.error(`[world-celld test pump] dropping ${envelope.messageId} after ${envelope.attempt} attempts: ${reason}`);
+			return;
+		}
+		const next = {
+			...envelope,
+			attempt: envelope.attempt + 1
+		};
+		const backoff = Math.min(MAX_TEST_PUMP_BACKOFF_MS, baseBackoffMs * 2 ** Math.min(next.attempt - 1, 30));
+		if (checkedQueueTimestampAdd(Date.now(), backoff, 900001) === null) {
+			release(envelope);
+			console.error(`[world-celld test pump] dropping ${envelope.messageId}: retry deadline exceeds the queue timestamp horizon`);
+			return;
+		}
+		delayFor(backoff).then(() => enqueue(pathname, next));
+	}
 	async function dispatch(envelope, pathname) {
 		const url = `${resolveBaseUrl(config)}/.well-known/workflow/v1/${pathname}`;
 		const response = await fetch(url, {
@@ -607,11 +639,12 @@ function createTestPump(config) {
 			body: stringify(envelope.message),
 			signal: AbortSignal.timeout(httpTimeoutMs)
 		});
-		if (response.ok) {
+		if (response.ok || PERMANENT_ERROR_STATUSES.has(response.status)) {
 			try {
 				await response.body?.cancel();
 			} catch {}
 			release(envelope);
+			if (!response.ok) debug(`[world-celld test pump] dropping ${envelope.messageId}: permanent HTTP ${response.status}`);
 			return;
 		}
 		const text = await response.text();
@@ -630,27 +663,7 @@ function createTestPump(config) {
 				return;
 			}
 		}
-		if (PERMANENT_ERROR_STATUSES.has(response.status)) {
-			release(envelope);
-			debug(`[world-celld test pump] dropping ${envelope.messageId}: permanent HTTP ${response.status}: ${text}`);
-			return;
-		}
-		if (envelope.attempt < maxAttempts) {
-			const next = {
-				...envelope,
-				attempt: envelope.attempt + 1
-			};
-			const backoff = Math.min(MAX_TEST_PUMP_BACKOFF_MS, baseBackoffMs * 2 ** Math.min(next.attempt - 1, 30));
-			if (checkedQueueTimestampAdd(Date.now(), backoff, 900001) === null) {
-				release(envelope);
-				console.error(`[world-celld test pump] dropping ${envelope.messageId}: retry deadline exceeds the queue timestamp horizon`);
-				return;
-			}
-			delayFor(backoff).then(() => enqueue(pathname, next));
-		} else {
-			release(envelope);
-			console.error(`[world-celld test pump] dropping ${envelope.messageId} after ${envelope.attempt} attempts: HTTP ${response.status}: ${text}`);
-		}
+		retry(envelope, pathname, `HTTP ${response.status}: ${text}`);
 	}
 	async function loop(pathname) {
 		while (true) {
@@ -660,19 +673,18 @@ function createTestPump(config) {
 			try {
 				await dispatch(envelope, pathname);
 			} catch (err) {
-				release(envelope);
-				console.error(`[world-celld test pump] dispatch error on ${pathname}:`, err);
+				retry(envelope, pathname, err instanceof Error ? err.message : String(err));
 			}
 		}
 	}
 	return {
 		/** Returns the inflight messageId when the key is already claimed. */
-		inflight(idempotencyKey) {
-			return idempotencyKey ? inflightMessages.get(idempotencyKey) : void 0;
+		inflight(queueName, idempotencyKey) {
+			return idempotencyKey ? inflightMessages.get(queueClaimName(queueName, idempotencyKey)) : void 0;
 		},
 		push(pathname, envelope, delaySeconds) {
 			if (queueDelayDeadline(Date.now(), delaySeconds ?? 0, 0, 900001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
-			if (envelope.idempotencyKey) inflightMessages.set(envelope.idempotencyKey, MessageId.parse(envelope.messageId));
+			if (envelope.idempotencyKey) inflightMessages.set(queueClaimName(envelope.queueName, envelope.idempotencyKey), MessageId.parse(envelope.messageId));
 			if (delaySeconds && delaySeconds > 0) delayFor(delaySeconds * 1e3).then(() => enqueue(pathname, envelope));
 			else enqueue(pathname, envelope);
 		},
@@ -700,7 +712,7 @@ function createQueue(config) {
 			if (queueDelayDeadline(Date.now(), opts?.delaySeconds ?? 0, 0, 900001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
 			const runId = "runId" in parsedMessage.data && typeof parsedMessage.data.runId === "string" ? parsedMessage.data.runId : void 0;
 			if (isTestMode()) {
-				const existing = testPump.inflight(opts?.idempotencyKey);
+				const existing = testPump.inflight(queueName, opts?.idempotencyKey);
 				if (existing) return { messageId: existing };
 				const messageId = MessageId.parse(`msg_${generateMessageId()}`);
 				testPump.push(QUEUE_PATHNAME, {
@@ -836,7 +848,9 @@ function parseSuccessEntity(outcome, key, schema) {
 	const value = outcome[key];
 	if (value === void 0) return void 0;
 	if (!isRecord(value)) malformedApplyEventOutcome(`${key} is invalid`);
-	const result = schema.safeParse(compact(value));
+	const entity = compact(value);
+	if (key === "step") entity.error = value.error;
+	const result = schema.safeParse(entity);
 	if (!result.success) malformedApplyEventOutcome(`${key} is invalid`);
 	return result.data;
 }
@@ -991,7 +1005,10 @@ function throwOutcomeError(outcome, runId, data) {
 	}
 }
 const parseRun = (run) => WorkflowRunSchema.parse(compact(run));
-const parseStep = (step) => StepSchema.parse(compact(step));
+const parseStep = (step) => StepSchema.parse({
+	...compact(step),
+	error: step.error
+});
 const parseHook = (hook) => HookSchema.parse(compact(hook));
 const parseEvent = (event) => EventSchema.parse(compact(event));
 function unwrapRead(outcome) {