@ewhauser/world-celld 0.3.2 → 0.4.0

package/dist/worker.js

@@ -736,6 +736,120 @@ function createRouter(env) {
 	};
 }
 //#endregion
+//#region src/retention.ts
+const CLEANUP_RECORD_KEY = "retention:cleanup";
+const TOMBSTONE_KEY = "retention:tombstone";
+const HOOK_MARKER_PREFIX = "retention:hook:";
+const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
+function sortableTimestamp(date) {
+	return date.getTime().toString().padStart(13, "0");
+}
+function workflowRunIndexKey(run) {
+	return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
+}
+function globalRunIndexKey(run) {
+	return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
+}
+function hookMarkerKey(reference) {
+	return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
+}
+function cleanupTombstone(record, now) {
+	const cleanup = {
+		...record,
+		phase: "tombstoned",
+		tombstonedAt: now
+	};
+	return {
+		version: 1,
+		runId: record.runId,
+		status: record.status ?? record.terminalStatus,
+		terminalStatus: record.terminalStatus,
+		completedAt: record.completedAt,
+		expiredAt: record.dueAt,
+		tombstonedAt: now,
+		cleanup
+	};
+}
+function cleanupFromTombstone(tombstone) {
+	return tombstone.cleanup;
+}
+function expiredRead(tombstone) {
+	return {
+		ok: false,
+		code: "RUN_EXPIRED",
+		message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
+		tombstone
+	};
+}
+const MAX_RETENTION_SWEEP_BATCH_SIZE = 1e3;
+const RETENTION_SWEEP_CONCURRENCY = 8;
+function integerSetting(name, raw, fallback, minimum, maximum = Number.MAX_SAFE_INTEGER) {
+	const value = raw === void 0 || raw === "" ? fallback : Number(raw);
+	if (!Number.isSafeInteger(value) || value < minimum || value > maximum) throw new Error(`world-celld: ${name} must be an integer between ${minimum} and ${maximum}`);
+	return value;
+}
+function runIdFromCatalogValue(value) {
+	const parsed = JSON.parse(value);
+	if (typeof parsed.runId !== "string" || parsed.runId.length === 0) throw new Error("world-celld: run catalog entry has no runId");
+	return parsed.runId;
+}
+/**
+* Discover and fence one bounded page of workflows older than the configured
+* maximum age. RunDO alarms finish the persisted, idempotent deletion phases.
+*/
+async function runRetentionSweep(scheduledTime, env) {
+	if (!Number.isSafeInteger(scheduledTime) || scheduledTime < 0) throw new Error("world-celld: scheduledTime must be a non-negative safe integer");
+	const retentionMs = integerSetting("WORKFLOW_RETENTION_MS", env.WORKFLOW_RETENTION_MS, 0, 0);
+	if (retentionMs === 0 || scheduledTime < retentionMs) return {
+		disabled: retentionMs === 0,
+		cutoff: retentionMs === 0 ? null : scheduledTime - retentionMs,
+		scanned: 0,
+		scheduled: 0,
+		expired: 0,
+		missing: 0,
+		notDue: 0
+	};
+	const batchSize = integerSetting("WORKFLOW_RETENTION_BATCH_SIZE", env.WORKFLOW_RETENTION_BATCH_SIZE, 128, 1, MAX_RETENTION_SWEEP_BATCH_SIZE);
+	const queueShards = integerSetting("WORKFLOW_RETENTION_QUEUE_SHARDS", env.WORKFLOW_RETENTION_QUEUE_SHARDS, 1, 1);
+	const cutoff = scheduledTime - retentionMs;
+	const page = await createWorkflowIndex({
+		runCatalog: env.WORKFLOW_RUN_CATALOG,
+		hookTokens: env.WORKFLOW_HOOK_TOKENS,
+		hookIds: env.WORKFLOW_HOOK_IDS
+	}).listRuns({
+		prefix: "runall:",
+		end: `runall:${sortableTimestamp(new Date(cutoff + 1))}:`,
+		limit: batchSize
+	});
+	const result = {
+		disabled: false,
+		cutoff,
+		scanned: page.keys.length,
+		scheduled: 0,
+		expired: 0,
+		missing: 0,
+		notDue: 0
+	};
+	const failures = [];
+	for (let offset = 0; offset < page.keys.length; offset += RETENTION_SWEEP_CONCURRENCY) {
+		const candidates = page.keys.slice(offset, offset + RETENTION_SWEEP_CONCURRENCY);
+		const settled = await Promise.allSettled(candidates.map(async (entry) => {
+			const runId = runIdFromCatalogValue(entry.value);
+			const outcome = await env.WORKFLOW_DB.get(env.WORKFLOW_DB.idFromName(runId)).enforceRetention({
+				retentionMs,
+				queueShards,
+				scheduledTime
+			});
+			if (outcome.state === "missing" || outcome.state === "not-due") await env.WORKFLOW_RUN_CATALOG.get(env.WORKFLOW_RUN_CATALOG.idFromName(runCatalogShardName(runId))).deleteStaleGlobalRun(runId, entry.name, entry.value);
+			return outcome;
+		}));
+		for (const outcome of settled) if (outcome.status === "rejected") failures.push(outcome.reason);
+		else result[outcome.value.state === "not-due" ? "notDue" : outcome.value.state]++;
+	}
+	if (failures.length > 0) throw new AggregateError(failures, `world-celld retention sweep failed for ${failures.length} run(s)`);
+	return result;
+}
+//#endregion
 //#region src/vendor/shared/util.ts
 function compact(obj) {
 	const value = {};
@@ -1781,8 +1895,11 @@ var RunCatalogDO = class extends DurableObject {
 			prefix: options.prefix ?? "",
 			...options.reverse ? {
 				reverse: true,
-				end: options.cursor
-			} : { startAfter: options.cursor },
+				end: options.cursor ?? options.end
+			} : {
+				startAfter: options.cursor,
+				end: options.end
+			},
 			limit: limit + 1
 		});
 		const page = Array.from(entries).slice(0, limit);
@@ -1796,6 +1913,15 @@ var RunCatalogDO = class extends DurableObject {
 			cursor: listComplete ? void 0 : page.at(-1)?.[0]
 		};
 	}
+	/** Compare-and-delete one rejected retention discovery candidate. */
+	async deleteStaleGlobalRun(runId, key, expectedValue) {
+		if (!key.startsWith("runall:") || !key.endsWith(`:${runId}`)) throw new Error("Stale run catalog deletion requires the exact global run key");
+		if (JSON.parse(expectedValue).runId !== runId) throw new Error("Stale run catalog deletion requires matching run metadata");
+		return await this.ctx.storage.transaction(async (txn) => {
+			if (await txn.get(key) !== expectedValue) return { deleted: false };
+			return { deleted: await txn.delete(key) };
+		});
+	}
 	async expireRun(runId, keys, expiredAt) {
 		if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
 		return await this.ctx.storage.transaction(async (txn) => {
@@ -3054,51 +3180,6 @@ async function finalizeEventPage(store, outcome, params) {
 	};
 }
 //#endregion
-//#region src/retention.ts
-const CLEANUP_RECORD_KEY = "retention:cleanup";
-const TOMBSTONE_KEY = "retention:tombstone";
-const HOOK_MARKER_PREFIX = "retention:hook:";
-const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
-function sortableTimestamp(date) {
-	return date.getTime().toString().padStart(13, "0");
-}
-function workflowRunIndexKey(run) {
-	return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
-}
-function globalRunIndexKey(run) {
-	return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
-}
-function hookMarkerKey(reference) {
-	return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
-}
-function cleanupTombstone(record, now) {
-	const cleanup = {
-		...record,
-		phase: "tombstoned",
-		tombstonedAt: now
-	};
-	return {
-		version: 1,
-		runId: record.runId,
-		terminalStatus: record.terminalStatus,
-		completedAt: record.completedAt,
-		expiredAt: record.dueAt,
-		tombstonedAt: now,
-		cleanup
-	};
-}
-function cleanupFromTombstone(tombstone) {
-	return tombstone.cleanup;
-}
-function expiredRead(tombstone) {
-	return {
-		ok: false,
-		code: "RUN_EXPIRED",
-		message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
-		tombstone
-	};
-}
-//#endregion
 //#region src/worker/durable-objects/WorkflowRunDO.ts
 const STORAGE_BATCH_SIZE = 128;
 const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
@@ -3111,6 +3192,28 @@ const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
 const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
 const CLEANUP_RETRY_MAX_MS = 36e5;
 const CLEANUP_PROGRESS_KEY = "retention:progress";
+const RUN_QUEUE_SHARDS_KEY = "retention:queue-shards";
+function cleanupRecord(run, dueAt, queueShards, reason) {
+	const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
+	return {
+		version: 1,
+		runId: run.runId,
+		workflowName: run.workflowName,
+		createdAt: run.createdAt,
+		status: run.status,
+		completedAt: terminal?.completedAt,
+		terminalStatus: terminal?.status,
+		reason,
+		dueAt,
+		queueShards,
+		phase: "retained",
+		generation: 0,
+		attempts: 0,
+		deletedPayloadKeys: 0,
+		deletedStreams: 0,
+		deletedQueueMessages: 0
+	};
+}
 /**
 * Adapt DO storage (or a transaction) to the {@link EventStore} interface
 * shared with the test mocks. No casts: the methods line up structurally,
@@ -3206,6 +3309,7 @@ var WorkflowRunDO = class extends DurableObject {
 				now
 			});
 			if (!outcome.ok) return outcome;
+			if (outcome.runCreated && request.cleanup) await txn.put(RUN_QUEUE_SHARDS_KEY, request.cleanup.queueShards);
 			await txn.put("event_sequence", eventSequence);
 			await putStorageEntries(txn, (outcome.hookToIndex ? [{
 				hookId: outcome.hookToIndex.hookId,
@@ -3219,22 +3323,7 @@ var WorkflowRunDO = class extends DurableObject {
 					...terminalRun,
 					expiredAt: dueAt
 				});
-				const cleanup = {
-					version: 1,
-					runId: run.runId,
-					workflowName: run.workflowName,
-					createdAt: run.createdAt,
-					completedAt: run.completedAt,
-					terminalStatus: run.status,
-					dueAt,
-					queueShards: request.cleanup?.queueShards ?? 1,
-					phase: "retained",
-					generation: 0,
-					attempts: 0,
-					deletedPayloadKeys: 0,
-					deletedStreams: 0,
-					deletedQueueMessages: 0
-				};
+				const cleanup = cleanupRecord(run, dueAt, request.cleanup?.queueShards ?? 1, "terminal-retention");
 				await txn.put("run", run);
 				await txn.put(CLEANUP_RECORD_KEY, cleanup);
 				await txn.setAlarm(dueAt);
@@ -3344,28 +3433,95 @@ var WorkflowRunDO = class extends DurableObject {
 				...terminalRun,
 				expiredAt: dueAt
 			});
-			const cleanup = {
-				version: 1,
-				runId: run.runId,
-				workflowName: run.workflowName,
-				createdAt: run.createdAt,
-				completedAt: run.completedAt,
-				terminalStatus: run.status,
-				dueAt,
-				queueShards: request.queueShards,
-				phase: "retained",
-				generation: 0,
-				attempts: 0,
-				deletedPayloadKeys: 0,
-				deletedStreams: 0,
-				deletedQueueMessages: 0
-			};
+			const cleanup = cleanupRecord(run, dueAt, request.queueShards, "manual");
 			await txn.put("run", run);
 			await txn.put(CLEANUP_RECORD_KEY, cleanup);
 			await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
 			return cleanup;
 		});
 	}
+	/**
+	* Apply the fleet-wide maximum-age policy to one authoritative run.
+	*
+	* The catalog is only a discovery index: eligibility is rechecked from the
+	* RunDO's own createdAt. Once eligible, the cleanup deadline immediately
+	* becomes an authoritative write/read fence, including for active runs.
+	*/
+	async enforceRetention(request) {
+		if (!Number.isSafeInteger(request.retentionMs) || request.retentionMs <= 0) throw new Error("world-celld retentionMs must be a positive safe integer");
+		if (!Number.isSafeInteger(request.queueShards) || request.queueShards <= 0) throw new Error("world-celld retention queueShards must be a positive safe integer");
+		if (!Number.isSafeInteger(request.scheduledTime) || request.scheduledTime < 0) throw new Error("world-celld retention scheduledTime must be a non-negative safe integer");
+		const result = await this.ctx.storage.transaction(async (txn) => {
+			const values = await txn.get([
+				CLEANUP_RECORD_KEY,
+				TOMBSTONE_KEY,
+				RUN_QUEUE_SHARDS_KEY,
+				"run"
+			]);
+			const tombstone = values.get(TOMBSTONE_KEY);
+			if (tombstone) return {
+				state: "expired",
+				cleanup: cleanupFromTombstone(tombstone)
+			};
+			const run = values.get("run");
+			if (!run) return {
+				state: "missing",
+				cleanup: null
+			};
+			const dueAtMs = run.createdAt.getTime() + request.retentionMs;
+			if (!Number.isSafeInteger(dueAtMs) || dueAtMs > 864e13) throw new Error(`world-celld retention deadline is out of range for ${run.runId}`);
+			if (dueAtMs > request.scheduledTime) return {
+				state: "not-due",
+				cleanup: null
+			};
+			const dueAt = new Date(dueAtMs);
+			const existing = values.get(CLEANUP_RECORD_KEY);
+			const storedQueueShards = values.get(RUN_QUEUE_SHARDS_KEY);
+			const queueShards = Number.isSafeInteger(storedQueueShards) && storedQueueShards > 0 ? storedQueueShards : request.queueShards;
+			let cleanup = existing;
+			if (!cleanup) {
+				cleanup = cleanupRecord(run, dueAt, queueShards, "maximum-age");
+				await txn.put("run", WorkflowRunSchema.parse({
+					...run,
+					expiredAt: dueAt
+				}));
+				await txn.put(CLEANUP_RECORD_KEY, cleanup);
+			} else if (cleanup.phase === "retained" && dueAtMs < cleanup.dueAt.getTime()) {
+				const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
+				cleanup = {
+					...cleanup,
+					status: run.status,
+					completedAt: terminal?.completedAt,
+					terminalStatus: terminal?.status,
+					reason: "maximum-age",
+					dueAt,
+					generation: cleanup.generation + 1
+				};
+				await txn.put("run", WorkflowRunSchema.parse({
+					...run,
+					expiredAt: dueAt
+				}));
+				await txn.put(CLEANUP_RECORD_KEY, cleanup);
+			}
+			await txn.setAlarm(this.now() + 1);
+			return {
+				state: cleanup.dueAt.getTime() <= this.now() ? "expired" : "scheduled",
+				cleanup
+			};
+		});
+		if (result.state === "scheduled" || result.state === "expired") {
+			for (let page = 0; page < 2; page++) {
+				const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
+				if (cleanup?.phase !== "retained" && cleanup?.phase !== "index") break;
+				await this.executeScheduledCleanupPage();
+			}
+			return {
+				...result,
+				cleanup: await this.getCleanupStatus() ?? result.cleanup
+			};
+		}
+		return result;
+	}
 	async cleanupNow(request) {
 		if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
 			...request,
@@ -3429,7 +3585,17 @@ var WorkflowRunDO = class extends DurableObject {
 		});
 	}
 	async executeScheduledCleanupPage() {
-		const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
+		const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
+		const cleanup = values.get(CLEANUP_RECORD_KEY);
+		const terminalCleanup = values.get(TERMINAL_CLEANUP_KEY);
+		if (cleanup && cleanup.phase !== "tombstoned" && cleanup.dueAt.getTime() <= this.now()) {
+			try {
+				await this.executeCleanup(cleanup);
+			} catch (error) {
+				await this.recordCleanupFailure(error, cleanup);
+			}
+			return;
+		}
 		if (terminalCleanup) {
 			try {
 				await this.executeTerminalCleanup(terminalCleanup);
@@ -3438,7 +3604,6 @@ var WorkflowRunDO = class extends DurableObject {
 			}
 			return;
 		}
-		const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
 		if (!cleanup || cleanup.phase === "tombstoned") {
 			await this.ctx.storage.deleteAlarm();
 			return;
@@ -3794,9 +3959,19 @@ var WorkflowRunDO = class extends DurableObject {
 * default fetch router. This module (and everything it imports) must stay
 * free of Node built-ins — it runs inside celld's workerd runtime.
 */
-var worker_default = { async fetch(request, env) {
-	return createRouter(env)(request);
-} };
+async function scheduled(controller, env) {
+	const result = await runRetentionSweep(controller.scheduledTime, env);
+	if (result.scanned > 0) console.info("world-celld retention sweep", {
+		cron: controller.cron,
+		...result
+	});
+}
+var worker_default = {
+	async fetch(request, env) {
+		return createRouter(env)(request);
+	},
+	scheduled
+};
 //#endregion
 export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };