@ewhauser/world-celld 0.3.1 → 0.3.2

package/dist/testing.js

@@ -1,4 +1,4 @@
-import { C as validateStreamReadRequest, S as normalizeStreamError, _ as decodeStreamWriteBatch, c as MAX_STREAM_BATCH_BYTES, d as MAX_STREAM_LONG_POLL_MS, f as MAX_STREAM_READ_BYTES, h as STREAM_BATCH_CONTENT_TYPE, i as parse, l as MAX_STREAM_CHUNK_BYTES, n as rpcParse, o as createWorkflowIndex, r as rpcStringify, s as runFenceCellName, t as compact, w as validateStreamWriteChunks, x as encodeStreamWriteResult, y as encodeStreamReadResult } from "./util-DmO0XkFw.js";
+import { A as validateStreamReadRequest, E as encodeStreamReadResult, O as encodeStreamWriteResult, S as STREAM_BATCH_CONTENT_TYPE, c as HOOK_CLAIM_LEASE_MS, d as MAX_QUEUE_DELIVERY_TIMEOUT_MS, f as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, g as MAX_STREAM_CHUNK_BYTES, h as MAX_STREAM_BATCH_BYTES, i as parse, j as validateStreamWriteChunks, k as normalizeStreamError, l as LIFECYCLE_COMPACTION_RETRY_MS, m as QUEUE_INFLIGHT_GRACE_MS, n as rpcParse, o as createWorkflowIndex, p as QUEUE_FENCE_GRACE_MS, r as rpcStringify, t as compact, v as MAX_STREAM_LONG_POLL_MS, w as decodeStreamWriteBatch, y as MAX_STREAM_READ_BYTES } from "./util-ya3V_FTA.js";
 import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
 import http from "node:http";
 //#region src/testing/fake-cell.ts
@@ -453,7 +453,6 @@ const WORLD_VERSION = "0.1.0";
 function workflowIndex(env) {
 	return createWorkflowIndex({
 		runCatalog: env.WORKFLOW_RUN_CATALOG,
-		runFences: env.WORKFLOW_RUN_FENCES,
 		hookTokens: env.WORKFLOW_HOOK_TOKENS,
 		hookIds: env.WORKFLOW_HOOK_IDS
 	});
@@ -463,6 +462,7 @@ const BINDINGS = {
 		env: "WORKFLOW_DB",
 		methods: /* @__PURE__ */ new Set([
 			"applyEvent",
+			"getLifecycleStatus",
 			"getRun",
 			"getStep",
 			"getEvent",
@@ -505,7 +505,8 @@ const BINDINGS = {
 			"redriveDeadLetter",
 			"purgeDeadLetters",
 			"rearmAlarm",
-			"expireRun"
+			"expireRun",
+			"acknowledgeExpireRun"
 		])
 	}
 };
@@ -614,7 +615,7 @@ function createRouter(env) {
 				let result;
 				const operation = `${parts[2]}.${parts[3]}`;
 				if (operation === "runs.list") result = await index.listRuns(args[0]);
-				else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
+				else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
 				else if (operation === "runs.expire") result = await index.expireRun(args[0]);
 				else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
 				else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
@@ -740,26 +741,32 @@ function ownerFromSerializedHook(raw) {
 function indexKey(prefix, value) {
 	return `${prefix}:${encodeURIComponent(value)}`;
 }
-/** Compact shard-local fence for every hook owned by one terminal or expired run. */
-function shardRunFenceKey(runId) {
-	return `runfence:${encodeURIComponent(runId)}`;
-}
 async function runIsActive(env, runId) {
-	const namespace = env.WORKFLOW_RUN_FENCES;
-	if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
-	return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
+	const namespace = env.WORKFLOW_DB;
+	if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
+	return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
 }
 //#endregion
 //#region src/worker/durable-objects/HookIdDO.ts
 const MAX_RELEASE_BATCH$1 = 64;
+const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
+function pad$3(ms) {
+	return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
+}
 function recordKey$1(hookId) {
 	return indexKey("hookid", hookId);
 }
 function claimKey$1(hookId) {
 	return indexKey("claim", hookId);
 }
+function claimDeadlineKey$1(hookId, expiresAt) {
+	return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
+}
 /** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
 var HookIdDO = class extends DurableObject {
+	now() {
+		return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
+	}
 	async get(hookId) {
 		const raw = await this.ctx.storage.get(recordKey$1(hookId));
 		if (!raw) return null;
@@ -776,17 +783,33 @@ var HookIdDO = class extends DurableObject {
 				claimed: false,
 				holder: ownerFromSerializedHook(indexed)
 			};
-			const existing = values.get(claim);
+			let existing = values.get(claim);
+			const now = this.now();
+			if (existing && existing.expiresAt <= now) {
+				await txn.delete([claim, existing.deadlineKey]);
+				existing = void 0;
+			}
 			if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
 				claimed: false,
 				holder: existing.owner
 			};
 			if (existing === void 0 || existing.claimId !== proposedClaimId) {
+				if (existing) await txn.delete(existing.deadlineKey);
+				const expiresAt = now + HOOK_CLAIM_LEASE_MS;
+				const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
 				await txn.put(claim, {
 					owner,
 					claimId: proposedClaimId,
-					reservedAt: Date.now()
+					reservedAt: now,
+					expiresAt,
+					deadlineKey
+				});
+				await txn.put(deadlineKey, {
+					hookId,
+					claimId: proposedClaimId,
+					expiresAt
 				});
+				await this.armAtMost(txn, expiresAt);
 				return {
 					claimed: true,
 					claimId: proposedClaimId
@@ -803,30 +826,24 @@ var HookIdDO = class extends DurableObject {
 			await this.releaseBatch([{
 				hookId,
 				owner
-			}], Date.now(), true);
+			}]);
 			return { stored: false };
 		}
 		return await this.ctx.storage.transaction(async (txn) => {
 			const record = recordKey$1(hookId);
 			const claim = claimKey$1(hookId);
-			const runFence = shardRunFenceKey(owner.runId);
-			const values = await txn.get([
-				record,
-				claim,
-				runFence
-			]);
-			if (values.has(runFence)) {
-				const existingClaim = values.get(claim);
-				if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
-				return { stored: false };
-			}
+			const values = await txn.get([record, claim]);
 			const indexed = values.get(record);
 			if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
 			const existingClaim = values.get(claim);
+			if (existingClaim && existingClaim.expiresAt <= this.now()) {
+				await txn.delete([claim, existingClaim.deadlineKey]);
+				return { stored: false };
+			}
 			if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
 			if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
 			await txn.put(record, serializedHook);
-			if (existingClaim !== void 0) await txn.delete(claim);
+			if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
 			return { stored: true };
 		});
 	}
@@ -834,40 +851,94 @@ var HookIdDO = class extends DurableObject {
 		await this.ctx.storage.transaction(async (txn) => {
 			const key = claimKey$1(hookId);
 			const claim = await txn.get(key);
-			if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
+			if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
 		});
 	}
-	async releaseBatch(hooks, fencedAt, runFenced) {
+	async releaseBatch(hooks) {
 		if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
 		if (hooks.length === 0) return { deleted: 0 };
 		return await this.ctx.storage.transaction(async (txn) => {
 			const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
 			const values = await txn.get(keys);
 			const deletes = [];
+			const deadlineDeletes = [];
 			for (const { hookId, owner } of hooks) {
 				const record = recordKey$1(hookId);
 				const claim = claimKey$1(hookId);
 				const indexed = values.get(record);
 				const existingClaim = values.get(claim);
 				if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
-				if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
+				if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
+					deletes.push(claim);
+					deadlineDeletes.push(existingClaim.deadlineKey);
+				}
+			}
+			const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
+			if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
+			return { deleted };
+		});
+	}
+	async alarm() {
+		try {
+			await this.compactClaims();
+		} catch {
+			await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
+		}
+	}
+	async armAtMost(storage, at) {
+		const current = await storage.getAlarm();
+		if (current === null || at < current) await storage.setAlarm(at);
+	}
+	async compactClaims() {
+		const now = this.now();
+		await this.ctx.storage.transaction(async (txn) => {
+			const due = await txn.list({
+				prefix: CLAIM_DEADLINE_PREFIX$1,
+				end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
+				limit: 129
+			});
+			const page = Array.from(due).slice(0, 128);
+			const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
+			const deletes = page.flatMap(([deadline, item]) => {
+				const key = claimKey$1(item.hookId);
+				const claim = claims.get(key);
+				return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
+			});
+			for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
+			if (due.size > 128) {
+				await txn.setAlarm(now + 1);
+				return;
 			}
-			if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
-			return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
+			const nextAt = (await txn.list({
+				prefix: CLAIM_DEADLINE_PREFIX$1,
+				limit: 1
+			})).values().next().value?.expiresAt;
+			if (nextAt === void 0) await txn.deleteAlarm();
+			else await txn.setAlarm(Math.max(now + 1, nextAt));
 		});
 	}
 };
 //#endregion
 //#region src/worker/durable-objects/HookTokenDO.ts
 const MAX_RELEASE_BATCH = 64;
+const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
+function pad$2(ms) {
+	return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
+}
 function recordKey(token) {
 	return indexKey("hook", token);
 }
 function claimKey(token) {
 	return indexKey("claim", token);
 }
+function claimDeadlineKey(token, expiresAt) {
+	return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
+}
 /** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
 var HookTokenDO = class extends DurableObject {
+	now() {
+		return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
+	}
 	async get(token) {
 		const raw = await this.ctx.storage.get(recordKey(token));
 		if (!raw) return null;
@@ -884,17 +955,32 @@ var HookTokenDO = class extends DurableObject {
 				claimed: false,
 				holder: ownerFromSerializedHook(indexed)
 			};
-			const existing = values.get(claim);
+			let existing = values.get(claim);
+			const now = this.now();
+			if (existing && existing.expiresAt <= now) {
+				await txn.delete([claim, existing.deadlineKey]);
+				existing = void 0;
+			}
 			if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
 				claimed: false,
 				holder: existing.owner
 			};
 			if (existing === void 0) {
+				const expiresAt = now + HOOK_CLAIM_LEASE_MS;
+				const deadlineKey = claimDeadlineKey(token, expiresAt);
 				await txn.put(claim, {
 					owner,
 					claimId: proposedClaimId,
-					reservedAt: Date.now()
+					reservedAt: now,
+					expiresAt,
+					deadlineKey
+				});
+				await txn.put(deadlineKey, {
+					token,
+					claimId: proposedClaimId,
+					expiresAt
 				});
+				await this.armAtMost(txn, expiresAt);
 				return {
 					claimed: true,
 					claimId: proposedClaimId
@@ -911,30 +997,24 @@ var HookTokenDO = class extends DurableObject {
 			await this.releaseBatch([{
 				token,
 				owner
-			}], Date.now(), true);
+			}]);
 			return { stored: false };
 		}
 		return await this.ctx.storage.transaction(async (txn) => {
 			const record = recordKey(token);
 			const claim = claimKey(token);
-			const runFence = shardRunFenceKey(owner.runId);
-			const values = await txn.get([
-				record,
-				claim,
-				runFence
-			]);
-			if (values.has(runFence)) {
-				const existingClaim = values.get(claim);
-				if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
-				return { stored: false };
-			}
+			const values = await txn.get([record, claim]);
 			const indexed = values.get(record);
 			if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
 			const existingClaim = values.get(claim);
+			if (existingClaim && existingClaim.expiresAt <= this.now()) {
+				await txn.delete([claim, existingClaim.deadlineKey]);
+				return { stored: false };
+			}
 			if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
 			if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
 			await txn.put(record, serializedHook);
-			if (existingClaim !== void 0) await txn.delete(claim);
+			if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
 			return { stored: true };
 		});
 	}
@@ -942,26 +1022,70 @@ var HookTokenDO = class extends DurableObject {
 		await this.ctx.storage.transaction(async (txn) => {
 			const key = claimKey(token);
 			const claim = await txn.get(key);
-			if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
+			if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
 		});
 	}
-	async releaseBatch(hooks, fencedAt, runFenced) {
+	async releaseBatch(hooks) {
 		if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
 		if (hooks.length === 0) return { deleted: 0 };
 		return await this.ctx.storage.transaction(async (txn) => {
 			const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
 			const values = await txn.get(keys);
 			const deletes = [];
+			const deadlineDeletes = [];
 			for (const { token, owner } of hooks) {
 				const record = recordKey(token);
 				const claim = claimKey(token);
 				const indexed = values.get(record);
 				const existingClaim = values.get(claim);
 				if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
-				if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
+				if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
+					deletes.push(claim);
+					deadlineDeletes.push(existingClaim.deadlineKey);
+				}
 			}
-			if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
-			return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
+			const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
+			if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
+			return { deleted };
+		});
+	}
+	async alarm() {
+		try {
+			await this.compactClaims();
+		} catch {
+			await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
+		}
+	}
+	async armAtMost(storage, at) {
+		const current = await storage.getAlarm();
+		if (current === null || at < current) await storage.setAlarm(at);
+	}
+	async compactClaims() {
+		const now = this.now();
+		await this.ctx.storage.transaction(async (txn) => {
+			const due = await txn.list({
+				prefix: CLAIM_DEADLINE_PREFIX,
+				end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
+				limit: 129
+			});
+			const page = Array.from(due).slice(0, 128);
+			const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
+			const deletes = page.flatMap(([deadline, item]) => {
+				const key = claimKey(item.token);
+				const claim = claims.get(key);
+				return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
+			});
+			for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
+			if (due.size > 128) {
+				await txn.setAlarm(now + 1);
+				return;
+			}
+			const nextAt = (await txn.list({
+				prefix: CLAIM_DEADLINE_PREFIX,
+				limit: 1
+			})).values().next().value?.expiresAt;
+			if (nextAt === void 0) await txn.deleteAlarm();
+			else await txn.setAlarm(Math.max(now + 1, nextAt));
 		});
 	}
 };
@@ -976,8 +1100,6 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
 const DEFAULT_MAX_ATTEMPTS = 5;
 const DEFAULT_MAX_INFLIGHT = 5;
 const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
-/** Grace period past the delivery timeout before a claim counts as lost. */
-const INFLIGHT_GRACE_MS = 3e4;
 /** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
 const MIN_ALARM_DELAY_MS = 1;
 /** Bound expiry recovery so one alarm cannot monopolize the cell. */
@@ -986,14 +1108,15 @@ const STORAGE_BATCH_SIZE$2 = 128;
 const EXPIRE_RUN_REFERENCE_BATCH = 64;
 const PURGE_DEAD_LETTER_BATCH = 128;
 const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
-function pad(ms) {
+const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
+function pad$1(ms) {
 	return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
 }
 function dueKey(atMs, messageId) {
-	return `due:${pad(atMs)}:${messageId}`;
+	return `due:${pad$1(atMs)}:${messageId}`;
 }
 function inflightKey(deadlineMs, messageId) {
-	return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
+	return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
 }
 function deadlineFromInflightKey(key) {
 	return Number.parseInt(key.slice(18, 31), 10);
@@ -1004,6 +1127,9 @@ function runReferenceKey(runId, messageId) {
 function expiredRunKey(runId) {
 	return `expired-run:${runId}`;
 }
+function expiredRunGcKey(runId, compactAt) {
+	return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
+}
 function boundedLimit$1(value, fallback, maximum) {
 	if (value === void 0 || !Number.isFinite(value)) return fallback;
 	return Math.max(1, Math.min(maximum, Math.floor(value)));
@@ -1054,16 +1180,29 @@ var QueueDO = class extends DurableObject {
 		if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
 		return configured;
 	}
+	#deliveryTimeoutMs() {
+		const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
+		if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
+		return configured;
+	}
+	async #authoritativeRunExpired(runId) {
+		const namespace = this.env?.WORKFLOW_DB;
+		if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
+		return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
+	}
+	async #withRunMutation(runId, onExpired, onAllowed) {
+		const expired = await this.#authoritativeRunExpired(runId);
+		return await this.ctx.storage.transaction(async (txn) => {
+			if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
+			return await onAllowed(txn);
+		});
+	}
 	async enqueue(request) {
 		this.#maxInflight();
+		this.#deliveryTimeoutMs();
 		const storage = this.ctx.storage;
 		const now = this.#now();
-		return await storage.transaction(async (txn) => {
-			if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
-				ok: false,
-				code: "RUN_EXPIRED",
-				message: `Workflow run "${request.runId}" has expired`
-			};
+		const persist = async (txn) => {
 			const pinned = await txn.get("cfg");
 			if (!pinned) await txn.put("cfg", {
 				queueShards: request.config.queueShards,
@@ -1115,7 +1254,12 @@ var QueueDO = class extends DurableObject {
 				messageId: row.messageId,
 				deduped: false
 			};
-		});
+		};
+		return request.runId ? await this.#withRunMutation(request.runId, async () => ({
+			ok: false,
+			code: "RUN_EXPIRED",
+			message: `Workflow run "${request.runId}" has expired`
+		}), persist) : await storage.transaction(persist);
 	}
 	async alarm() {
 		let claimed = [];
@@ -1129,7 +1273,10 @@ var QueueDO = class extends DurableObject {
 		};
 		if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
 		else await attempt();
-		if (failure !== null) throw failure;
+		if (failure !== null) {
+			await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
+			throw failure;
+		}
 		for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
 	}
 	/**
@@ -1141,9 +1288,13 @@ var QueueDO = class extends DurableObject {
 		const now = this.#now();
 		const maxInflight = this.#maxInflight();
 		return await storage.transaction(async (txn) => {
+			if (await this.#compactExpiredRunFences(txn, now)) {
+				await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
+				return [];
+			}
 			const expiredPage = await txn.list({
 				prefix: INFLIGHT_DEADLINE_PREFIX,
-				end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
+				end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
 				limit: 129
 			});
 			const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
@@ -1175,17 +1326,17 @@ var QueueDO = class extends DurableObject {
 			if (inflightCount < maxInflight) {
 				const due = await txn.list({
 					prefix: "due:",
-					end: `due:${pad(now + 1)}`,
+					end: `due:${pad$1(now + 1)}`,
 					limit: maxInflight - inflightCount
 				});
-				const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
+				const timeoutMs = this.#deliveryTimeoutMs();
 				const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
 				await deleteMany$1(txn, Array.from(due.keys()));
 				const claimWrites = [];
 				for (const [, messageId] of due) {
 					const row = dueRows.get(`msg:${messageId}`);
 					if (!row) continue;
-					const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
+					const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
 					claimWrites.push([claimKey, messageId]);
 					if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
 						messageId: row.messageId,
@@ -1205,9 +1356,13 @@ var QueueDO = class extends DurableObject {
 	}
 	async #deliver({ row, inflightKey: claimKey }) {
 		const storage = this.ctx.storage;
-		const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
+		const timeoutMs = this.#deliveryTimeoutMs();
 		const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
 		const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
+		if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
+			await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
+			return;
+		}
 		let response = null;
 		let transportError = null;
 		try {
@@ -1237,28 +1392,21 @@ var QueueDO = class extends DurableObject {
 			} catch {
 				timeoutSeconds = void 0;
 			}
-			if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
-				if (!await txn.delete(claimKey)) return;
-				if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
-					await txn.delete(`msg:${row.messageId}`);
-					await txn.delete(runReferenceKey(row.runId, row.messageId));
-					if (row.idempotencyKey) {
-						const dedupKey = `key:${row.idempotencyKey}`;
-						if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
-					}
+			if (typeof timeoutSeconds === "number") {
+				const reschedule = async (txn) => {
+					if (!await txn.delete(claimKey)) return;
+					const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
+					await txn.put(scheduleKey, row.messageId);
+					if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+						messageId: row.messageId,
+						dueKey: scheduleKey,
+						idempotencyKey: row.idempotencyKey
+					});
 					await this.#scheduleNextAlarm(txn, now);
-					return;
-				}
-				const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
-				await txn.put(scheduleKey, row.messageId);
-				if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
-					messageId: row.messageId,
-					dueKey: scheduleKey,
-					idempotencyKey: row.idempotencyKey
-				});
-				await this.#scheduleNextAlarm(txn, now);
-			});
-			else await this.#retry(row, claimKey, "HTTP 503");
+				};
+				if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
+				else await storage.transaction(reschedule);
+			} else await this.#retry(row, claimKey, "HTTP 503");
 		} else if (response && PERMANENT_STATUSES.has(response.status)) {
 			await cancelUnusedResponseBody(response);
 			await this.#ack(row, claimKey);
@@ -1279,23 +1427,23 @@ var QueueDO = class extends DurableObject {
 			await this.#scheduleNextAlarm(txn, this.#now());
 		});
 	}
+	async #dropExpiredMessage(txn, row, claimKey, now) {
+		if (!await txn.delete(claimKey)) return;
+		await txn.delete(`msg:${row.messageId}`);
+		if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
+		if (row.idempotencyKey) {
+			const dedupKey = `key:${row.idempotencyKey}`;
+			if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
+		}
+		await this.#scheduleNextAlarm(txn, now);
+	}
 	async #retry(row, claimKey, reason) {
 		const storage = this.ctx.storage;
 		const now = this.#now();
 		const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
 		const attempt = row.attempt + 1;
-		await storage.transaction(async (txn) => {
+		const persistRetry = async (txn) => {
 			if (!await txn.delete(claimKey)) return;
-			if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
-				await txn.delete(`msg:${row.messageId}`);
-				await txn.delete(runReferenceKey(row.runId, row.messageId));
-				if (row.idempotencyKey) {
-					const dedupKey = `key:${row.idempotencyKey}`;
-					if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
-				}
-				await this.#scheduleNextAlarm(txn, now);
-				return;
-			}
 			if (attempt >= maxAttempts) {
 				const dead = {
 					...row,
@@ -1303,7 +1451,7 @@ var QueueDO = class extends DurableObject {
 					lastError: reason,
 					failedAt: now
 				};
-				const deadKey = `dlq:${pad(now)}:${row.messageId}`;
+				const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
 				await txn.put(deadKey, dead);
 				await txn.delete(`msg:${row.messageId}`);
 				if (row.idempotencyKey) {
@@ -1331,7 +1479,32 @@ var QueueDO = class extends DurableObject {
 				idempotencyKey: row.idempotencyKey
 			});
 			await this.#scheduleNextAlarm(txn, now);
-		});
+		};
+		if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
+		else await storage.transaction(persistRetry);
+	}
+	/** Return true when another due compaction page must run immediately. */
+	async #compactExpiredRunFences(txn, now) {
+		const due = await txn.list({
+			prefix: EXPIRED_RUN_GC_PREFIX,
+			end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
+			limit: 129
+		});
+		const page = Array.from(due).slice(0, 128);
+		if (page.length === 0) return false;
+		const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
+		const markers = await getMany$1(txn, markerKeys);
+		const deletes = [];
+		for (let index = 0; index < page.length; index++) {
+			const [gcKey, item] = page[index];
+			const markerKey = markerKeys[index];
+			const marker = markers.get(markerKey);
+			deletes.push(gcKey);
+			if (marker?.compactAt !== item.compactAt) continue;
+			deletes.push(markerKey);
+		}
+		await deleteMany$1(txn, deletes);
+		return due.size > 128;
 	}
 	/** Arm the alarm no later than `atMs` (durable backoff never throws). */
 	async #armAlarmAtMost(storage, atMs, now) {
@@ -1363,6 +1536,11 @@ var QueueDO = class extends DurableObject {
 			const deadline = deadlineFromInflightKey(key);
 			if (next === null || deadline < next) next = deadline;
 		}
+		const expiryGc = await storage.list({
+			prefix: EXPIRED_RUN_GC_PREFIX,
+			limit: 1
+		});
+		for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
 		if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
 		else await storage.deleteAlarm();
 	}
@@ -1401,37 +1579,42 @@ var QueueDO = class extends DurableObject {
 	async redriveDeadLetter(messageId) {
 		const storage = this.ctx.storage;
 		const now = this.#now();
-		return await storage.transaction(async (txn) => {
-			const dlq = await txn.list({ prefix: "dlq:" });
-			for (const [key, dead] of dlq) {
-				if (dead.messageId !== messageId) continue;
-				const { failedAt: _failedAt, ...rest } = dead;
-				const row = {
-					...rest,
-					attempt: 0,
-					lastError: void 0
-				};
-				if (row.idempotencyKey) {
-					const dedupKey = `key:${row.idempotencyKey}`;
-					const holder = await txn.get(dedupKey);
-					if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
-					if (holder && holder !== row.messageId) await txn.delete(dedupKey);
-					await txn.put(dedupKey, row.messageId);
-				}
-				await txn.put(`msg:${row.messageId}`, row);
-				const scheduleKey = dueKey(now, row.messageId);
-				await txn.put(scheduleKey, row.messageId);
-				await txn.delete(key);
-				if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
-					messageId: row.messageId,
-					dueKey: scheduleKey,
-					idempotencyKey: row.idempotencyKey
-				});
-				await this.#armAlarmAtMost(txn, now, now);
-				return { ok: true };
+		const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
+		if (!found) return { ok: false };
+		const [deadKey, candidate] = found;
+		const redrive = async (txn) => {
+			const dead = await txn.get(deadKey);
+			if (!dead || dead.messageId !== messageId) return { ok: false };
+			const { failedAt: _failedAt, ...rest } = dead;
+			const row = {
+				...rest,
+				attempt: 0,
+				lastError: void 0
+			};
+			if (row.idempotencyKey) {
+				const dedupKey = `key:${row.idempotencyKey}`;
+				const holder = await txn.get(dedupKey);
+				if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
+				if (holder && holder !== row.messageId) await txn.delete(dedupKey);
+				await txn.put(dedupKey, row.messageId);
 			}
+			await txn.put(`msg:${row.messageId}`, row);
+			const scheduleKey = dueKey(now, row.messageId);
+			await txn.put(scheduleKey, row.messageId);
+			await txn.delete(deadKey);
+			if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
+				messageId: row.messageId,
+				dueKey: scheduleKey,
+				idempotencyKey: row.idempotencyKey
+			});
+			await this.#armAlarmAtMost(txn, now, now);
+			return { ok: true };
+		};
+		const rejectExpired = async (txn) => {
+			if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
 			return { ok: false };
-		});
+		};
+		return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
 	}
 	async purgeDeadLetters() {
 		return await this.ctx.storage.transaction(async (txn) => {
@@ -1481,27 +1664,75 @@ var QueueDO = class extends DurableObject {
 				count++;
 			}
 			await deleteMany$1(txn, keys);
+			const done = references.size <= limit;
+			const markerExpiredAt = existing?.expiredAt ?? expiredAt;
 			await txn.put(expiredRunKey(runId), {
-				expiredAt,
-				deleted: count
+				expiredAt: markerExpiredAt,
+				deleted: count,
+				compactAt: existing?.compactAt ?? null
 			});
 			await this.#scheduleNextAlarm(txn, this.#now());
-			return {
+			return done ? {
+				deleted: count,
+				done: true,
+				receipt: {
+					expiredAt: markerExpiredAt,
+					deleted: count
+				}
+			} : {
 				deleted: count,
-				done: references.size <= limit
+				done: false
 			};
 		});
 	}
+	/** Acknowledge a final cleanup receipt before making its marker compactable. */
+	async acknowledgeExpireRun(runId, receipt) {
+		return await this.ctx.storage.transaction(async (txn) => {
+			const key = expiredRunKey(runId);
+			const marker = await txn.get(key);
+			if (!marker) return { acknowledged: true };
+			if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
+			if (marker.compactAt === null) {
+				const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
+				await txn.put(key, {
+					...marker,
+					compactAt
+				});
+				await txn.put(expiredRunGcKey(runId, compactAt), {
+					runId,
+					compactAt
+				});
+			}
+			await this.#scheduleNextAlarm(txn, this.#now());
+			return { acknowledged: true };
+		});
+	}
 };
 //#endregion
 //#region src/worker/durable-objects/RunCatalogDO.ts
 const MAX_INDEX_LIST_SIZE = 1001;
+const EXPIRY_GC_PREFIX = "expiry-gc:";
+function pad(ms) {
+	return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
+}
+function expiredKey(runId) {
+	return `expired:${runId}`;
+}
+function expiryGcKey(runId, compactAt) {
+	return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
+}
 /** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
 var RunCatalogDO = class extends DurableObject {
-	async upsertRun(runId, keys, serializedMetadata) {
+	now() {
+		const clock = this.env?.clock;
+		return typeof clock === "function" ? clock() : Date.now();
+	}
+	async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
 		if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
 		return await this.ctx.storage.transaction(async (txn) => {
-			if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
+			if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
+			const now = this.now();
+			if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
 			await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
 			return { stored: true };
 		});
@@ -1531,38 +1762,56 @@ var RunCatalogDO = class extends DurableObject {
 	async expireRun(runId, keys, expiredAt) {
 		if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
 		return await this.ctx.storage.transaction(async (txn) => {
-			await txn.put(`expired:${runId}`, expiredAt);
+			const marker = expiredKey(runId);
+			const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
+			await txn.put(marker, {
+				expiredAt,
+				compactAt
+			});
+			await txn.put(expiryGcKey(runId, compactAt), {
+				runId,
+				compactAt
+			});
+			await this.armAtMost(txn, compactAt);
 			return { deleted: await txn.delete(keys) };
 		});
 	}
-};
-//#endregion
-//#region src/worker/durable-objects/RunFenceDO.ts
-/** Per-run lifecycle fence used by hook shards without a global lookup cell. */
-var RunFenceDO = class extends DurableObject {
-	async getStatus() {
-		return (await this.ctx.storage.get("fence"))?.status ?? "active";
+	async alarm() {
+		try {
+			await this.compactExpiryFences();
+		} catch {
+			await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
+		}
 	}
-	async fenceTerminal(at) {
-		return await this.ctx.storage.transaction(async (txn) => {
-			const current = await txn.get("fence");
-			if (current) return current.status;
-			await txn.put("fence", {
-				status: "terminal",
-				at
-			});
-			return "terminal";
-		});
+	async armAtMost(storage, at) {
+		const current = await storage.getAlarm();
+		if (current === null || at < current) await storage.setAlarm(at);
 	}
-	async fenceExpired(at) {
-		return await this.ctx.storage.transaction(async (txn) => {
-			const current = await txn.get("fence");
-			if (current?.status === "expired") return current.status;
-			await txn.put("fence", {
-				status: "expired",
-				at
+	async compactExpiryFences() {
+		const now = this.now();
+		await this.ctx.storage.transaction(async (txn) => {
+			const due = await txn.list({
+				prefix: EXPIRY_GC_PREFIX,
+				end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
+				limit: 129
 			});
-			return "expired";
+			const page = Array.from(due).slice(0, 128);
+			const markerKeys = page.map(([, item]) => expiredKey(item.runId));
+			const markers = await txn.get(markerKeys);
+			const deletes = page.flatMap(([key, item], index) => {
+				return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
+			});
+			for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
+			if (due.size > 128) {
+				await txn.setAlarm(now + 1);
+				return;
+			}
+			const nextAt = (await txn.list({
+				prefix: EXPIRY_GC_PREFIX,
+				limit: 1
+			})).values().next().value?.compactAt;
+			if (nextAt === void 0) await txn.deleteAlarm();
+			else await txn.setAlarm(Math.max(now + 1, nextAt));
 		});
 	}
 };
@@ -2786,15 +3035,24 @@ function hookMarkerKey(reference) {
 	return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
 }
 function cleanupTombstone(record, now) {
+	const cleanup = {
+		...record,
+		phase: "tombstoned",
+		tombstonedAt: now
+	};
 	return {
 		version: 1,
 		runId: record.runId,
 		terminalStatus: record.terminalStatus,
 		completedAt: record.completedAt,
 		expiredAt: record.dueAt,
-		tombstonedAt: now
+		tombstonedAt: now,
+		cleanup
 	};
 }
+function cleanupFromTombstone(tombstone) {
+	return tombstone.cleanup;
+}
 function expiredRead(tombstone) {
 	return {
 		ok: false,
@@ -2814,7 +3072,6 @@ const STREAMS_PER_CLEANUP_PAGE = 16;
 const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
 const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
 const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
-const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
 const CLEANUP_RETRY_MAX_MS = 36e5;
 const CLEANUP_PROGRESS_KEY = "retention:progress";
 /**
@@ -2956,6 +3213,7 @@ var WorkflowRunDO = class extends DurableObject {
 				});
 				await txn.setAlarm(this.now() + 1);
 			}
+			if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
 			return finalizeEventPage(storeFrom(txn), outcome, request.params);
 		});
 	}
@@ -2966,6 +3224,8 @@ var WorkflowRunDO = class extends DurableObject {
 	*/
 	async resolveHookTokenClaim(request) {
 		return await this.ctx.storage.transaction(async (txn) => {
+			const { tombstone } = await this.retentionState(txn);
+			if (tombstone) return { committed: false };
 			if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
 			await txn.put(hookClaimCancellationKey(request.claimId), {
 				hookId: request.hookId,
@@ -2978,6 +3238,20 @@ var WorkflowRunDO = class extends DurableObject {
 	async getRun() {
 		return this.readKey("run");
 	}
+	/** Single authoritative lifecycle answer used after derivative compaction. */
+	async getLifecycleStatus() {
+		return await this.ctx.storage.transaction(async (txn) => {
+			const values = await txn.get([
+				TOMBSTONE_KEY,
+				CLEANUP_RECORD_KEY,
+				"run"
+			]);
+			if (this.tombstoneFrom(values)) return "expired";
+			const run = values.get("run");
+			if (!run) return "missing";
+			return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
+		});
+	}
 	async getStep(stepId) {
 		return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
 	}
@@ -3006,7 +3280,11 @@ var WorkflowRunDO = class extends DurableObject {
 		}));
 	}
 	async getCleanupStatus() {
-		return await this.ctx.storage.get("retention:cleanup") ?? null;
+		const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
+		const cleanup = values.get(CLEANUP_RECORD_KEY);
+		if (cleanup) return cleanup;
+		const tombstone = values.get(TOMBSTONE_KEY);
+		return tombstone ? cleanupFromTombstone(tombstone) : null;
 	}
 	/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
 	async scheduleCleanup(request) {
@@ -3094,7 +3372,6 @@ var WorkflowRunDO = class extends DurableObject {
 	workflowIndex() {
 		return createWorkflowIndex({
 			runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
-			runFences: this.namespace("WORKFLOW_RUN_FENCES"),
 			hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
 			hookIds: this.namespace("WORKFLOW_HOOK_IDS")
 		});
@@ -3174,8 +3451,7 @@ var WorkflowRunDO = class extends DurableObject {
 			hooks: page.map((hook) => ({
 				hookId: hook.hookId,
 				token: hook.token
-			})),
-			terminal: true
+			}))
 		});
 		await this.ctx.storage.transaction(async (txn) => {
 			const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3200,8 +3476,7 @@ var WorkflowRunDO = class extends DurableObject {
 		const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
 		await this.workflowIndex().releaseHookIndexes({
 			runId: cleanup.runId,
-			hooks: page.map(([, reference]) => reference),
-			terminal: true
+			hooks: page.map(([, reference]) => reference)
 		});
 		await this.ctx.storage.transaction(async (txn) => {
 			const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3311,33 +3586,56 @@ var WorkflowRunDO = class extends DurableObject {
 	}
 	async deleteQueuePage(cleanup) {
 		const queues = this.namespace("WORKFLOW_QUEUE");
-		let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
+		const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
 			queueShard: 0,
 			queueShardDeleted: 0
 		};
-		let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
-		let expectedGeneration = cleanup.generation;
-		while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
+		if (progress.pendingAck) {
+			const { queueShard, receipt } = progress.pendingAck;
+			if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
+			await this.ctx.storage.transaction(async (txn) => {
+				const current = await txn.get(CLEANUP_RECORD_KEY);
+				if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
+				const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
+				if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
+				await txn.put(CLEANUP_RECORD_KEY, {
+					...current,
+					generation: current.generation + 1,
+					attempts: 0,
+					lastError: void 0
+				});
+				await txn.put(CLEANUP_PROGRESS_KEY, {
+					queueShard: queueShard + 1,
+					queueShardDeleted: 0
+				});
+				await txn.setAlarm(this.now() + 1);
+			});
+			return;
+		}
+		if (progress.queueShard < cleanup.queueShards) {
 			const requestedShard = progress.queueShard;
 			const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
-			const reconciled = await this.ctx.storage.transaction(async (txn) => {
+			await this.ctx.storage.transaction(async (txn) => {
 				const current = await txn.get(CLEANUP_RECORD_KEY);
-				if (current?.phase !== "queues") return null;
-				if (current.generation !== expectedGeneration) return { stale: true };
+				if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
 				const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
 					queueShard: 0,
 					queueShardDeleted: 0
 				};
-				if (persisted.queueShard !== requestedShard) return { stale: true };
-				if (result.deleted < persisted.queueShardDeleted) return { stale: true };
+				if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
+				if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
+				const delta = result.deleted - persisted.queueShardDeleted;
 				const next = result.done ? {
-					queueShard: persisted.queueShard + 1,
-					queueShardDeleted: 0
+					queueShard: requestedShard,
+					queueShardDeleted: result.deleted,
+					pendingAck: {
+						queueShard: requestedShard,
+						receipt: result.receipt
+					}
 				} : {
-					...persisted,
+					queueShard: requestedShard,
 					queueShardDeleted: result.deleted
 				};
-				const delta = result.deleted - persisted.queueShardDeleted;
 				await txn.put(CLEANUP_RECORD_KEY, {
 					...current,
 					deletedQueueMessages: current.deletedQueueMessages + delta,
@@ -3347,27 +3645,17 @@ var WorkflowRunDO = class extends DurableObject {
 				});
 				await txn.put(CLEANUP_PROGRESS_KEY, next);
 				await txn.setAlarm(this.now() + 1);
-				return {
-					stale: false,
-					progress: next,
-					generation: current.generation + 1,
-					applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
-				};
 			});
-			if (!reconciled) return;
-			if (reconciled.stale) return;
-			progress = reconciled.progress;
-			expectedGeneration = reconciled.generation;
-			if (!reconciled.applied && progress.queueShard === requestedShard) break;
-			if (!result.done && progress.queueShard === requestedShard) break;
+			return;
 		}
 		await this.ctx.storage.transaction(async (txn) => {
 			const current = await txn.get(CLEANUP_RECORD_KEY);
-			if (current?.phase !== "queues") return;
-			if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
+			if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
+			const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
 				queueShard: 0,
 				queueShardDeleted: 0
-			}).queueShard >= current.queueShards) {
+			};
+			if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
 				await txn.put(CLEANUP_RECORD_KEY, {
 					...current,
 					phase: "payload",
@@ -3400,14 +3688,16 @@ var WorkflowRunDO = class extends DurableObject {
 				await txn.setAlarm(this.now() + 1);
 				return false;
 			}
-			await txn.put(CLEANUP_RECORD_KEY, {
+			const completed = {
 				...cleanup,
 				phase: "tombstoned",
 				generation: cleanup.generation + 1,
 				attempts: 0,
 				lastError: void 0,
 				tombstonedAt: now
-			});
+			};
+			await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
+			await txn.delete(CLEANUP_RECORD_KEY);
 			await txn.deleteAlarm();
 			return true;
 		});
@@ -3472,7 +3762,6 @@ async function startHarness(options = {}) {
 		runs: WorkflowRunDO,
 		streams: StreamDO,
 		"run-catalog": RunCatalogDO,
-		"run-fences": RunFenceDO,
 		"hook-tokens": HookTokenDO,
 		"hook-ids": HookIdDO,
 		queue: QueueDO,
@@ -3483,7 +3772,6 @@ async function startHarness(options = {}) {
 		WORKFLOW_DB: fleet.namespace("runs"),
 		WORKFLOW_STREAMS: fleet.namespace("streams"),
 		WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
-		WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
 		WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
 		WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
 		WORKFLOW_QUEUE: fleet.namespace("queue")
@@ -3492,7 +3780,6 @@ async function startHarness(options = {}) {
 		WORKFLOW_DB: fleet.namespace("runs"),
 		WORKFLOW_STREAMS: fleet.namespace("streams"),
 		WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
-		WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
 		WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
 		WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
 		WORKFLOW_QUEUE: fleet.namespace("queue"),