@ewhauser/world-celld 0.1.1 → 0.2.0

package/dist/testing.js

@@ -1,4 +1,4 @@
-import { i as parse, n as rpcParse, r as rpcStringify, t as compact } from "./util-BvWlkqgV.js";
+import { a as cleanupTombstone, c as globalRunIndexKey, d as workflowRunIndexKey, f as compact, h as parse, i as TOMBSTONE_KEY, l as hookMarkerKey, m as rpcStringify, n as HOOK_MARKER_PREFIX, o as correlationIndexKey, p as rpcParse, r as INDEX_MARKER_PREFIX, s as expiredRead, t as CLEANUP_RECORD_KEY, u as indexMarkerKey } from "./retention-bLY1vcOu.js";
 import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema, isLegacySpecVersion, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld } from "@workflow/world";
 import http from "node:http";
 //#region src/testing/fake-cell.ts
@@ -309,7 +309,11 @@ const BINDINGS = {
 			"getEvent",
 			"listEvents",
 			"listSteps",
-			"listHooks"
+			"listHooks",
+			"getCleanupStatus",
+			"scheduleCleanup",
+			"cleanupNow",
+			"rearmCleanup"
 		])
 	},
 	streams: {
@@ -320,7 +324,10 @@ const BINDINGS = {
 			"getChunks",
 			"getInfo",
 			"registerStream",
-			"listStreams"
+			"listStreams",
+			"expireRegistry",
+			"finalizeRegistry",
+			"expireStream"
 		])
 	},
 	index: {
@@ -329,6 +336,8 @@ const BINDINGS = {
 			"get",
 			"put",
 			"delete",
+			"putOwned",
+			"expireRun",
 			"list",
 			"reserveHookToken",
 			"finalizeHookIndexes",
@@ -344,7 +353,8 @@ const BINDINGS = {
 			"listDeadLetters",
 			"redriveDeadLetter",
 			"purgeDeadLetters",
-			"rearmAlarm"
+			"rearmAlarm",
+			"expireRun"
 		])
 	}
 };
@@ -478,6 +488,14 @@ var IndexDO = class extends DurableObject {
 	async put(key, value) {
 		await this.ctx.storage.put(key, value);
 	}
+	/** Publish a derived index only while its owning run is not expired. */
+	async putOwned(runId, key, value) {
+		return await this.ctx.storage.transaction(async (txn) => {
+			if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
+			await txn.put(key, value);
+			return { stored: true };
+		});
+	}
 	async delete(key) {
 		await this.ctx.storage.delete(key);
 	}
@@ -523,6 +541,11 @@ var IndexDO = class extends DurableObject {
 	}
 	async finalizeHookIndexes(token, hookId, serializedHook, owner) {
 		await this.ctx.storage.transaction(async (txn) => {
+			if (await txn.get(`expired:${owner.runId}`) !== void 0) {
+				const claim = await txn.get(`hookclaim:${token}`);
+				if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(`hookclaim:${token}`);
+				return;
+			}
 			const indexedHook = await txn.get(`hook:${token}`);
 			if (indexedHook !== void 0 && !sameOwner(ownerFromHook(indexedHook), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
 			const claim = await txn.get(`hookclaim:${token}`);
@@ -554,6 +577,42 @@ var IndexDO = class extends DurableObject {
 			if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
 		});
 	}
+	/**
+	* Fence a run and remove all of its known derived indexes atomically.
+	* Once the fence exists, putOwned() and hook finalization cannot resurrect
+	* entries from a delayed request.
+	*/
+	async expireRun(request) {
+		return await this.ctx.storage.transaction(async (txn) => {
+			await txn.put(`expired:${request.runId}`, request.expiredAt);
+			let deleted = 0;
+			for (const key of new Set(request.keys)) if (await txn.delete(key)) deleted++;
+			for (const hook of request.hooks) {
+				const owner = {
+					runId: request.runId,
+					hookId: hook.hookId
+				};
+				const tokenKey = `hook:${hook.token}`;
+				const idKey = `hookid:${hook.hookId}`;
+				const claimKey = `hookclaim:${hook.token}`;
+				const [byToken, byId, claim] = await Promise.all([
+					txn.get(tokenKey),
+					txn.get(idKey),
+					txn.get(claimKey)
+				]);
+				if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
+					if (await txn.delete(tokenKey)) deleted++;
+				}
+				if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
+					if (await txn.delete(idKey)) deleted++;
+				}
+				if (claim !== void 0 && sameOwner(claim.owner, owner)) {
+					if (await txn.delete(claimKey)) deleted++;
+				}
+			}
+			return { deleted };
+		});
+	}
 };
 //#endregion
 //#region src/worker/durable-objects/QueueDO.ts
@@ -576,6 +635,12 @@ function pad(ms) {
 function dueKey(atMs, messageId) {
 	return `due:${pad(atMs)}:${messageId}`;
 }
+function runReferenceKey(runId, messageId) {
+	return `run:${runId}:${messageId}`;
+}
+function expiredRunKey(runId) {
+	return `expired-run:${runId}`;
+}
 function backoffSeconds(attempt) {
 	return Math.min(60, 2 ** attempt);
 }
@@ -597,6 +662,11 @@ var QueueDO = class extends DurableObject {
 		const storage = this.ctx.storage;
 		const now = this.#now();
 		return await storage.transaction(async (txn) => {
+			if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
+				ok: false,
+				code: "RUN_EXPIRED",
+				message: `Workflow run "${request.runId}" has expired`
+			};
 			const pinned = await txn.get("cfg");
 			if (!pinned) await txn.put("cfg", {
 				queueShards: request.config.queueShards,
@@ -627,14 +697,21 @@ var QueueDO = class extends DurableObject {
 				queueName: request.queueName,
 				pathname: request.pathname,
 				body: request.body,
+				runId: request.runId,
 				targetBaseUrl: request.config.targetBaseUrl,
 				idempotencyKey: request.idempotencyKey,
 				attempt: 0,
 				enqueuedAt: now
 			};
 			const dueAt = now + Math.max(0, request.delaySeconds ?? 0) * 1e3;
+			const scheduleKey = dueKey(dueAt, row.messageId);
 			await txn.put(`msg:${row.messageId}`, row);
-			await txn.put(dueKey(dueAt, row.messageId), row.messageId);
+			await txn.put(scheduleKey, row.messageId);
+			if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+				messageId: row.messageId,
+				dueKey: scheduleKey,
+				idempotencyKey: row.idempotencyKey
+			});
 			await this.#armAlarmAtMost(txn, dueAt, now);
 			return {
 				ok: true,
@@ -672,7 +749,16 @@ var QueueDO = class extends DurableObject {
 			for (const [key, deadline] of inflight) if (deadline <= now) {
 				const messageId = key.slice(9);
 				await txn.delete(key);
-				if (await txn.get(`msg:${messageId}`)) await txn.put(dueKey(now, messageId), messageId);
+				const row = await txn.get(`msg:${messageId}`);
+				if (row) {
+					const scheduleKey = dueKey(now, messageId);
+					await txn.put(scheduleKey, messageId);
+					if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+						messageId: row.messageId,
+						dueKey: scheduleKey,
+						idempotencyKey: row.idempotencyKey
+					});
+				}
 			} else inflightCount++;
 			const rows = [];
 			if (inflightCount < maxInflight) {
@@ -727,8 +813,25 @@ var QueueDO = class extends DurableObject {
 				timeoutSeconds = void 0;
 			}
 			if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
+				if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
+					await txn.delete(`msg:${row.messageId}`);
+					await txn.delete(`inflight:${row.messageId}`);
+					await txn.delete(runReferenceKey(row.runId, row.messageId));
+					if (row.idempotencyKey) {
+						const dedupKey = `key:${row.idempotencyKey}`;
+						if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
+					}
+					await this.#scheduleNextAlarm(txn, now);
+					return;
+				}
 				await txn.delete(`inflight:${row.messageId}`);
-				await txn.put(dueKey(now + timeoutSeconds * 1e3, row.messageId), row.messageId);
+				const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
+				await txn.put(scheduleKey, row.messageId);
+				if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+					messageId: row.messageId,
+					dueKey: scheduleKey,
+					idempotencyKey: row.idempotencyKey
+				});
 				await this.#scheduleNextAlarm(txn, now);
 			});
 			else await this.#retry(row, "HTTP 503");
@@ -745,6 +848,7 @@ var QueueDO = class extends DurableObject {
 			if (row.idempotencyKey) {
 				if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
 			}
+			if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
 			await this.#scheduleNextAlarm(txn, this.#now());
 		});
 	}
@@ -754,6 +858,17 @@ var QueueDO = class extends DurableObject {
 		const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
 		const attempt = row.attempt + 1;
 		await storage.transaction(async (txn) => {
+			if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
+				await txn.delete(`msg:${row.messageId}`);
+				await txn.delete(`inflight:${row.messageId}`);
+				await txn.delete(runReferenceKey(row.runId, row.messageId));
+				if (row.idempotencyKey) {
+					const dedupKey = `key:${row.idempotencyKey}`;
+					if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
+				}
+				await this.#scheduleNextAlarm(txn, now);
+				return;
+			}
 			await txn.delete(`inflight:${row.messageId}`);
 			if (attempt >= maxAttempts) {
 				const dead = {
@@ -762,11 +877,17 @@ var QueueDO = class extends DurableObject {
 					lastError: reason,
 					failedAt: now
 				};
-				await txn.put(`dlq:${pad(now)}:${row.messageId}`, dead);
+				const deadKey = `dlq:${pad(now)}:${row.messageId}`;
+				await txn.put(deadKey, dead);
 				await txn.delete(`msg:${row.messageId}`);
 				if (row.idempotencyKey) {
 					if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
 				}
+				if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+					messageId: row.messageId,
+					dlqKey: deadKey,
+					idempotencyKey: row.idempotencyKey
+				});
 				await this.#scheduleNextAlarm(txn, now);
 				return;
 			}
@@ -775,8 +896,14 @@ var QueueDO = class extends DurableObject {
 				attempt,
 				lastError: reason
 			};
+			const scheduleKey = dueKey(now + backoffSeconds(attempt) * 1e3, row.messageId);
 			await txn.put(`msg:${row.messageId}`, updated);
-			await txn.put(dueKey(now + backoffSeconds(attempt) * 1e3, row.messageId), row.messageId);
+			await txn.put(scheduleKey, row.messageId);
+			if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+				messageId: row.messageId,
+				dueKey: scheduleKey,
+				idempotencyKey: row.idempotencyKey
+			});
 			await this.#scheduleNextAlarm(txn, now);
 		});
 	}
@@ -860,8 +987,14 @@ var QueueDO = class extends DurableObject {
 					await txn.put(dedupKey, row.messageId);
 				}
 				await txn.put(`msg:${row.messageId}`, row);
-				await txn.put(dueKey(now, row.messageId), row.messageId);
+				const scheduleKey = dueKey(now, row.messageId);
+				await txn.put(scheduleKey, row.messageId);
 				await txn.delete(key);
+				if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+					messageId: row.messageId,
+					dueKey: scheduleKey,
+					idempotencyKey: row.idempotencyKey
+				});
 				await this.#armAlarmAtMost(txn, now, now);
 				return { ok: true };
 			}
@@ -871,7 +1004,11 @@ var QueueDO = class extends DurableObject {
 	async purgeDeadLetters() {
 		return await this.ctx.storage.transaction(async (txn) => {
 			const dlq = await txn.list({ prefix: "dlq:" });
-			for (const key of dlq.keys()) await txn.delete(key);
+			for (const [key, value] of dlq) {
+				await txn.delete(key);
+				const dead = value;
+				if (dead.runId) await txn.delete(runReferenceKey(dead.runId, dead.messageId));
+			}
 			return { purged: dlq.size };
 		});
 	}
@@ -883,6 +1020,32 @@ var QueueDO = class extends DurableObject {
 		await this.#rearm();
 		return { alarmAt: await this.ctx.storage.getAlarm() };
 	}
+	/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
+	async expireRun(runId, expiredAt) {
+		return { deleted: await this.ctx.storage.transaction(async (txn) => {
+			const existing = await txn.get(expiredRunKey(runId));
+			const references = await txn.list({ prefix: `run:${runId}:` });
+			let count = existing?.deleted ?? 0;
+			for (const [key, reference] of references) {
+				await txn.delete(`msg:${reference.messageId}`);
+				await txn.delete(`inflight:${reference.messageId}`);
+				if (reference.dueKey) await txn.delete(reference.dueKey);
+				if (reference.dlqKey) await txn.delete(reference.dlqKey);
+				if (reference.idempotencyKey) {
+					const dedupKey = `key:${reference.idempotencyKey}`;
+					if (await txn.get(dedupKey) === reference.messageId) await txn.delete(dedupKey);
+				}
+				await txn.delete(key);
+				count++;
+			}
+			await txn.put(expiredRunKey(runId), {
+				expiredAt,
+				deleted: count
+			});
+			await this.#scheduleNextAlarm(txn, this.#now());
+			return count;
+		}) };
+	}
 };
 //#endregion
 //#region src/worker/durable-objects/StreamDO.ts
@@ -890,6 +1053,8 @@ const META_KEY = "meta";
 const CHUNK_KEY_PREFIX = "chunk:";
 /** Registry keys used when this DO instance acts as a per-run stream index. */
 const STREAM_REGISTRY_PREFIX = "stream:";
+const REGISTRY_OWNER_KEY = "registry:owner";
+const REGISTRY_EXPIRED_KEY = "registry:expired";
 /**
 * Zero-pad chunk indexes so `storage.list({ prefix })` returns chunks in
 * write order. 12 digits comfortably exceeds any realistic chunk count.
@@ -914,22 +1079,28 @@ var StreamDO = class extends DurableObject {
 			closed: false
 		};
 	}
+	assertOwner(meta, runId) {
+		if (meta.expiredAt !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
+		if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
+	}
 	/**
 	* Append a chunk. The index is allocated inside the transaction, so
 	* concurrent writers can never collide or skip.
 	*/
-	async writeChunk(data) {
+	async writeChunk(runId, data) {
 		return await this.ctx.storage.transaction(async (txn) => {
 			const meta = await txn.get(META_KEY) ?? {
 				count: 0,
 				closed: false
 			};
+			this.assertOwner(meta, runId);
 			if (meta.closed) throw new Error("Cannot write to a closed stream");
 			const index = meta.count;
 			await txn.put(chunkKey(index), data);
 			await txn.put(META_KEY, {
 				count: index + 1,
-				closed: false
+				closed: false,
+				ownerRunId: runId
 			});
 			return index;
 		});
@@ -938,14 +1109,16 @@ var StreamDO = class extends DurableObject {
 	* Close the stream (idempotent). Readers observe `done: true` once all
 	* previously written chunks have been consumed.
 	*/
-	async closeStream() {
+	async closeStream(runId) {
 		await this.ctx.storage.transaction(async (txn) => {
 			const meta = await txn.get(META_KEY) ?? {
 				count: 0,
 				closed: false
 			};
+			this.assertOwner(meta, runId);
 			await txn.put(META_KEY, {
 				...meta,
+				ownerRunId: runId,
 				closed: true
 			});
 		});
@@ -955,6 +1128,7 @@ var StreamDO = class extends DurableObject {
 	*/
 	async getChunks(params) {
 		const meta = await this.getMeta();
+		if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
 		const start = Math.max(0, params.startIndex);
 		const entries = await this.ctx.storage.list({
 			prefix: CHUNK_KEY_PREFIX,
@@ -972,6 +1146,7 @@ var StreamDO = class extends DurableObject {
 	*/
 	async getInfo() {
 		const meta = await this.getMeta();
+		if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
 		return {
 			tailIndex: meta.count - 1,
 			done: meta.closed
@@ -980,16 +1155,66 @@ var StreamDO = class extends DurableObject {
 	/**
 	* Register a stream name against this per-run registry instance.
 	*/
-	async registerStream(name) {
-		await this.ctx.storage.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
+	async registerStream(runId, name) {
+		await this.ctx.storage.transaction(async (txn) => {
+			if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
+			const owner = await txn.get(REGISTRY_OWNER_KEY);
+			if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
+			await txn.put(REGISTRY_OWNER_KEY, runId);
+			await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
+		});
 	}
 	/**
 	* List stream names registered against this per-run registry instance.
 	*/
 	async listStreams() {
+		if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
 		const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
 		return Array.from(entries.keys()).map((key) => key.slice(7));
 	}
+	/** Fence a run's registry before any stream cells are removed. */
+	async expireRegistry(runId, expiredAt) {
+		return await this.ctx.storage.transaction(async (txn) => {
+			const owner = await txn.get(REGISTRY_OWNER_KEY);
+			if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
+			await txn.put(REGISTRY_OWNER_KEY, runId);
+			await txn.put(REGISTRY_EXPIRED_KEY, expiredAt);
+			const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
+			return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
+		});
+	}
+	/** Remove the registry payload after every referenced stream was fenced. */
+	async finalizeRegistry(runId) {
+		await this.ctx.storage.transaction(async (txn) => {
+			const owner = await txn.get(REGISTRY_OWNER_KEY);
+			const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
+			if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
+			const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
+			for (const key of entries.keys()) await txn.delete(key);
+		});
+	}
+	/** Delete stream payload while leaving a small ownership tombstone. */
+	async expireStream(runId, expiredAt) {
+		return await this.ctx.storage.transaction(async (txn) => {
+			const meta = await txn.get(META_KEY) ?? {
+				count: 0,
+				closed: false
+			};
+			if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
+			const chunks = await txn.list({ prefix: CHUNK_KEY_PREFIX });
+			for (const key of chunks.keys()) await txn.delete(key);
+			await txn.put(META_KEY, {
+				count: 0,
+				closed: true,
+				ownerRunId: runId,
+				expiredAt
+			});
+			return {
+				deleted: true,
+				chunks: chunks.size
+			};
+		});
+	}
 };
 //#endregion
 //#region src/apply-event.ts
@@ -1526,6 +1751,9 @@ async function applyEvent(store, ctx) {
 }
 //#endregion
 //#region src/worker/durable-objects/WorkflowRunDO.ts
+const PAYLOAD_DELETE_BATCH = 256;
+const STREAM_DELETE_CONCURRENCY = 16;
+const CLEANUP_RETRY_MAX_MS = 36e5;
 /**
 * Current schema version for DO storage.
 * Increment when making breaking changes to the storage layout.
@@ -1610,104 +1838,347 @@ function storeFrom(storage) {
 */
 var WorkflowRunDO = class extends DurableObject {
 	schemaReady = null;
-	/**
-	* Lazily ensure schema migrations have run.
-	* Memoized so it only runs once per DO instantiation.
-	*/
+	now() {
+		const clock = this.env?.clock;
+		return typeof clock === "function" ? clock() : Date.now();
+	}
 	ensureSchema() {
 		if (!this.schemaReady) this.schemaReady = ensureSchemaUpToDate(this.ctx.storage);
 		return this.schemaReady;
 	}
-	/**
-	* Apply an event: validate, guard, append the event and mutate the entity
-	* in ONE storage transaction. Guard failures are returned as structured
-	* outcomes (not thrown) because typed errors do not survive the RPC
-	* boundary with their `name` intact.
-	*/
+	async expiredTombstone(storage, now = this.now()) {
+		const tombstone = await storage.get(TOMBSTONE_KEY);
+		if (tombstone) return tombstone;
+		const cleanup = await storage.get(CLEANUP_RECORD_KEY);
+		if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
+		return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
+	}
+	async read(reader) {
+		await this.ensureSchema();
+		const tombstone = await this.expiredTombstone(this.ctx.storage);
+		if (tombstone) return expiredRead(tombstone);
+		return {
+			ok: true,
+			value: await reader()
+		};
+	}
+	/** Apply an event and capture all retention metadata in the same transaction. */
 	async applyEvent(request) {
 		await this.ensureSchema();
 		return await this.ctx.storage.transaction(async (txn) => {
+			const now = new Date(this.now());
+			const tombstone = await this.expiredTombstone(txn, now.getTime());
+			if (tombstone) return {
+				ok: false,
+				code: "RUN_EXPIRED",
+				message: `Workflow run "${request.runId}" expired at ${tombstone.expiredAt.toISOString()}`
+			};
 			let eventSequence = await txn.get("event_sequence") ?? 0;
 			const outcome = await applyEvent(storeFrom(txn), {
 				...request,
 				nextEventId: () => `wevt_z${String(++eventSequence).padStart(20, "0")}`,
-				now: /* @__PURE__ */ new Date()
+				now
 			});
-			if (outcome.ok) await txn.put("event_sequence", eventSequence);
+			if (!outcome.ok) return outcome;
+			await txn.put("event_sequence", eventSequence);
+			if (outcome.event?.correlationId) {
+				const key = correlationIndexKey(outcome.event.correlationId, outcome.event.createdAt, outcome.event.eventId, request.runId);
+				await txn.put(indexMarkerKey(key), key);
+			}
+			if (outcome.hookToIndex) {
+				const reference = {
+					hookId: outcome.hookToIndex.hookId,
+					token: outcome.hookToIndex.token
+				};
+				await txn.put(hookMarkerKey(reference), reference);
+			}
+			for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
+			const retentionMs = request.cleanup?.retentionMs ?? 0;
+			if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !await txn.get("retention:cleanup")) {
+				const terminalRun = outcome.run;
+				const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
+				const run = WorkflowRunSchema.parse({
+					...terminalRun,
+					expiredAt: dueAt
+				});
+				const cleanup = {
+					version: 1,
+					runId: run.runId,
+					workflowName: run.workflowName,
+					createdAt: run.createdAt,
+					completedAt: run.completedAt,
+					terminalStatus: run.status,
+					dueAt,
+					queueShards: request.cleanup?.queueShards ?? 1,
+					phase: "retained",
+					attempts: 0,
+					deletedPayloadKeys: 0,
+					deletedStreams: 0,
+					deletedQueueMessages: 0
+				};
+				await txn.put("run", run);
+				await txn.put(CLEANUP_RECORD_KEY, cleanup);
+				await txn.setAlarm(dueAt);
+				outcome.run = run;
+			}
 			return outcome;
 		});
 	}
-	/**
-	* Get the workflow run
-	*/
 	async getRun() {
-		await this.ensureSchema();
-		return await this.ctx.storage.get("run") ?? null;
+		return this.read(async () => await this.ctx.storage.get("run") ?? null);
 	}
-	/**
-	* Get a specific step
-	*/
 	async getStep(stepId) {
-		await this.ensureSchema();
-		return await this.ctx.storage.get(`step:${stepId}`) ?? null;
+		return this.read(async () => await this.ctx.storage.get(`step:${stepId}`) ?? null);
 	}
-	/**
-	* List steps with real limit/cursor pagination (cursor = last stepId).
-	*/
 	async listSteps(params) {
-		await this.ensureSchema();
-		return listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
+		return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
 			limit: params?.limit ?? 20,
 			cursor: params?.cursor,
 			sortOrder: params?.sortOrder ?? "asc"
-		});
+		}));
 	}
-	/**
-	* Get a specific event
-	*/
 	async getEvent(eventId) {
-		await this.ensureSchema();
-		return await this.ctx.storage.get(`event:${eventId}`) ?? null;
+		return this.read(async () => await this.ctx.storage.get(`event:${eventId}`) ?? null);
 	}
-	/**
-	* List events with real limit/cursor pagination. Events are keyed by their
-	* monotonic ULID eventId, so storage order == event order.
-	*/
 	async listEvents(params) {
-		await this.ensureSchema();
-		return listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
+		return this.read(() => listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
 			limit: params?.limit ?? 100,
 			cursor: params?.cursor,
 			sortOrder: params?.sortOrder ?? "asc"
-		}, (event) => event.eventId);
+		}, (event) => event.eventId));
 	}
-	/**
-	* List hooks with real limit/cursor pagination (cursor = last hookId).
-	*/
 	async listHooks(params) {
-		await this.ensureSchema();
-		return listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
+		return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
 			limit: params?.limit ?? 100,
 			cursor: params?.cursor,
 			sortOrder: params?.sortOrder ?? "asc"
+		}));
+	}
+	async getCleanupStatus() {
+		await this.ensureSchema();
+		return await this.ctx.storage.get("retention:cleanup") ?? null;
+	}
+	/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
+	async scheduleCleanup(request) {
+		await this.ensureSchema();
+		if (request.retentionMs <= 0) return this.getCleanupStatus();
+		return await this.ctx.storage.transaction(async (txn) => {
+			const existing = await txn.get(CLEANUP_RECORD_KEY);
+			if (existing) return existing;
+			if (await txn.get("retention:tombstone")) return null;
+			const current = await txn.get("run");
+			if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
+			const terminalRun = current;
+			const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
+			const run = WorkflowRunSchema.parse({
+				...terminalRun,
+				expiredAt: dueAt
+			});
+			const cleanup = {
+				version: 1,
+				runId: run.runId,
+				workflowName: run.workflowName,
+				createdAt: run.createdAt,
+				completedAt: run.completedAt,
+				terminalStatus: run.status,
+				dueAt,
+				queueShards: request.queueShards,
+				phase: "retained",
+				attempts: 0,
+				deletedPayloadKeys: 0,
+				deletedStreams: 0,
+				deletedQueueMessages: 0
+			};
+			await txn.put("run", run);
+			await txn.put(CLEANUP_RECORD_KEY, cleanup);
+			await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
+			return cleanup;
+		});
+	}
+	async cleanupNow(request) {
+		await this.ensureSchema();
+		if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
+			...request,
+			retentionMs: Math.max(1, request.retentionMs)
+		});
+		await this.ctx.storage.transaction(async (txn) => {
+			const cleanup = await txn.get(CLEANUP_RECORD_KEY);
+			if (!cleanup || cleanup.phase === "tombstoned") return;
+			const dueAt = new Date(this.now());
+			const run = await txn.get("run");
+			if (run) await txn.put("run", WorkflowRunSchema.parse({
+				...run,
+				expiredAt: dueAt
+			}));
+			await txn.put(CLEANUP_RECORD_KEY, {
+				...cleanup,
+				dueAt
+			});
+			await txn.setAlarm(this.now() + 1);
+		});
+		try {
+			await this.executeCleanup();
+		} catch (error) {
+			await this.recordCleanupFailure(error);
+		}
+		return this.getCleanupStatus();
+	}
+	async rearmCleanup() {
+		await this.ensureSchema();
+		const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
+		if (!cleanup) return null;
+		if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
+		else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
+		return cleanup;
+	}
+	async alarm() {
+		await this.ensureSchema();
+		try {
+			await this.executeCleanup();
+		} catch (error) {
+			await this.recordCleanupFailure(error);
+		}
+	}
+	namespace(name) {
+		const namespace = this.env?.[name];
+		if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
+		return namespace;
+	}
+	async setCleanupPhase(expected, next, updates = {}) {
+		await this.ctx.storage.transaction(async (txn) => {
+			const cleanup = await txn.get(CLEANUP_RECORD_KEY);
+			if (!cleanup || cleanup.phase !== expected) return;
+			await txn.put(CLEANUP_RECORD_KEY, {
+				...cleanup,
+				...updates,
+				phase: next,
+				attempts: 0,
+				lastError: void 0
+			});
+		});
+	}
+	async executeCleanup() {
+		for (;;) {
+			const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
+			if (!cleanup) {
+				await this.ctx.storage.deleteAlarm();
+				return;
+			}
+			if (cleanup.phase === "tombstoned") {
+				await this.ctx.storage.deleteAlarm();
+				return;
+			}
+			if (cleanup.phase === "retained") {
+				if (cleanup.dueAt.getTime() > this.now()) {
+					await this.ctx.storage.setAlarm(cleanup.dueAt);
+					return;
+				}
+				await this.setCleanupPhase("retained", "index");
+				continue;
+			}
+			if (cleanup.phase === "index") {
+				const [indexMarkers, hookMarkers] = await Promise.all([this.ctx.storage.list({ prefix: INDEX_MARKER_PREFIX }), this.ctx.storage.list({ prefix: HOOK_MARKER_PREFIX })]);
+				const indexNamespace = this.namespace("WORKFLOW_INDEX");
+				await indexNamespace.get(indexNamespace.idFromName("index")).expireRun({
+					runId: cleanup.runId,
+					keys: [
+						workflowRunIndexKey(cleanup),
+						globalRunIndexKey(cleanup),
+						...indexMarkers.values()
+					],
+					hooks: Array.from(hookMarkers.values()),
+					expiredAt: cleanup.dueAt.getTime()
+				});
+				await this.setCleanupPhase("index", "streams");
+				continue;
+			}
+			if (cleanup.phase === "streams") {
+				const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
+				const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
+				const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime());
+				let deletedStreams = streams.length === 0 ? cleanup.deletedStreams : 0;
+				for (let offset = 0; offset < streams.length; offset += STREAM_DELETE_CONCURRENCY) {
+					const batch = streams.slice(offset, offset + STREAM_DELETE_CONCURRENCY);
+					const results = await Promise.all(batch.map((name) => {
+						return streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime());
+					}));
+					deletedStreams += results.filter((result) => result.deleted).length;
+				}
+				await this.ctx.storage.transaction(async (txn) => {
+					const current = await txn.get(CLEANUP_RECORD_KEY);
+					if (current?.phase === "streams") await txn.put(CLEANUP_RECORD_KEY, {
+						...current,
+						deletedStreams
+					});
+				});
+				await registry.finalizeRegistry(cleanup.runId);
+				await this.setCleanupPhase("streams", "queues");
+				continue;
+			}
+			if (cleanup.phase === "queues") {
+				const queues = this.namespace("WORKFLOW_QUEUE");
+				let deletedQueueMessages = 0;
+				for (let shard = 0; shard < cleanup.queueShards; shard++) {
+					const result = await queues.get(queues.idFromName(`q:${shard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime());
+					deletedQueueMessages += result.deleted;
+				}
+				await this.setCleanupPhase("queues", "payload", { deletedQueueMessages });
+				continue;
+			}
+			if (cleanup.phase === "payload") {
+				if (!await this.deletePayloadBatch()) return;
+			}
+		}
+	}
+	async deletePayloadBatch() {
+		return await this.ctx.storage.transaction(async (txn) => {
+			const cleanup = await txn.get(CLEANUP_RECORD_KEY);
+			if (!cleanup || cleanup.phase !== "payload") return true;
+			const entries = await txn.list({ limit: 259 });
+			const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone" && key !== "schema_version").slice(0, PAYLOAD_DELETE_BATCH);
+			const now = new Date(this.now());
+			await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
+			for (const key of keys) await txn.delete(key);
+			if (keys.length > 0) {
+				await txn.put(CLEANUP_RECORD_KEY, {
+					...cleanup,
+					deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
+					lastError: void 0
+				});
+				await txn.setAlarm(this.now() + 1);
+				return false;
+			}
+			await txn.put(CLEANUP_RECORD_KEY, {
+				...cleanup,
+				phase: "tombstoned",
+				attempts: 0,
+				lastError: void 0,
+				tombstonedAt: now
+			});
+			await txn.deleteAlarm();
+			return true;
+		});
+	}
+	async recordCleanupFailure(error) {
+		await this.ctx.storage.transaction(async (txn) => {
+			const cleanup = await txn.get(CLEANUP_RECORD_KEY);
+			if (!cleanup || cleanup.phase === "tombstoned") return;
+			const attempts = cleanup.attempts + 1;
+			const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
+			await txn.put(CLEANUP_RECORD_KEY, {
+				...cleanup,
+				attempts,
+				lastError: error instanceof Error ? error.message : String(error),
+				lastAttemptAt: new Date(this.now())
+			});
+			await txn.setAlarm(this.now() + delay);
 		});
 	}
-	/**
-	* Claim a queue-message dedup slot. Used on DO instances dedicated to a
-	* single idempotency key (`claim:<queueName>:<idempotencyKey>`).
-	*
-	* - Unclaimed -> claim and proceed.
-	* - Claimed by the SAME messageId -> redelivery of the same message
-	*   (previous attempt failed transiently) -> proceed.
-	* - Claimed by a DIFFERENT messageId -> concurrent duplicate -> reject,
-	*   unless the claim is stale (holder crashed / went to the DLQ without
-	*   releasing), in which case the claim is stolen.
-	*/
 	async claimInflight(params) {
 		await this.ensureSchema();
 		return await this.ctx.storage.transaction(async (txn) => {
 			const existing = await txn.get("claim");
-			const now = Date.now();
+			const now = this.now();
 			if (existing && existing.messageId !== params.messageId && now - existing.claimedAt < params.staleMs) return { claimed: false };
 			await txn.put("claim", {
 				messageId: params.messageId,
@@ -1716,9 +2187,6 @@ var WorkflowRunDO = class extends DurableObject {
 			return { claimed: true };
 		});
 	}
-	/**
-	* Release a queue-message dedup claim after the message was fully handled.
-	*/
 	async releaseInflight() {
 		await this.ensureSchema();
 		await this.ctx.storage.delete("claim");
@@ -1731,13 +2199,21 @@ var WorkflowRunDO = class extends DurableObject {
 * behind a node:http server — the full wire protocol without a celld binary.
 */
 async function startHarness(options = {}) {
+	const cellEnv = { ...options.cellEnv };
 	const fleet = new FakeFleet({
 		runs: WorkflowRunDO,
 		streams: StreamDO,
 		index: IndexDO,
 		queue: QueueDO,
 		...options.extraClasses
-	}, options.cellEnv ?? {});
+	}, cellEnv);
+	cellEnv.clock ??= options.virtualClock ? () => fleet.now : () => Date.now();
+	Object.assign(cellEnv, {
+		WORKFLOW_DB: fleet.namespace("runs"),
+		WORKFLOW_STREAMS: fleet.namespace("streams"),
+		WORKFLOW_INDEX: fleet.namespace("index"),
+		WORKFLOW_QUEUE: fleet.namespace("queue")
+	});
 	const router = createRouter({
 		WORKFLOW_DB: fleet.namespace("runs"),
 		WORKFLOW_STREAMS: fleet.namespace("streams"),
@@ -1771,7 +2247,10 @@ async function startHarness(options = {}) {
 	return {
 		url: `http://127.0.0.1:${port}`,
 		fleet,
-		close: () => new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve()))
+		close: () => new Promise((resolve, reject) => {
+			server.close((err) => err ? reject(err) : resolve());
+			server.closeAllConnections();
+		})
 	};
 }
 //#endregion