@ewhauser/world-celld 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/worker.js CHANGED
@@ -121,9 +121,6 @@ function hookTokenShardName(token) {
121
121
  function hookIdShardName(hookId) {
122
122
  return shardName("hook-id", hookId, 32);
123
123
  }
124
- function runFenceCellName(runId) {
125
- return `run-fence:${INDEX_PROTOCOL_VERSION}:${runId}`;
126
- }
127
124
  function allRunCatalogShardNames() {
128
125
  return Array.from({ length: 16 }, (_, shard) => `run-catalog:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(2, "0")}`);
129
126
  }
@@ -161,8 +158,7 @@ async function inBatches(values, operation) {
161
158
  return deleted;
162
159
  }
163
160
  function createWorkflowIndex(bindings) {
164
- const fence = (runId) => stub(bindings.runFences, runFenceCellName(runId));
165
- const releaseHooks = async (request, fencedAt) => {
161
+ const releaseHooks = async (request) => {
166
162
  const tokenGroups = groupByShard(request.hooks, (hook) => hookTokenShardName(hook.token));
167
163
  const idGroups = groupByShard(request.hooks, (hook) => hookIdShardName(hook.hookId));
168
164
  const releases = [];
@@ -174,7 +170,7 @@ function createWorkflowIndex(bindings) {
174
170
  runId: request.runId,
175
171
  hookId: hook.hookId
176
172
  }
177
- })), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
173
+ })), (batch) => target.releaseBatch(batch)));
178
174
  }
179
175
  for (const [name, hooks] of idGroups) {
180
176
  const target = stub(bindings.hookIds, name);
@@ -184,14 +180,13 @@ function createWorkflowIndex(bindings) {
184
180
  runId: request.runId,
185
181
  hookId: hook.hookId
186
182
  }
187
- })), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
183
+ })), (batch) => target.releaseBatch(batch)));
188
184
  }
189
185
  return (await Promise.all(releases)).reduce((total, count) => total + count, 0);
190
186
  };
191
187
  return {
192
- async commitRun(run, serializedMetadata) {
193
- if (isTerminalWorkflowRunStatus(run.status)) await fence(run.runId).fenceTerminal(Date.now());
194
- return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata);
188
+ async commitRun(run, serializedMetadata, publicationExpiresAt) {
189
+ return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata, publicationExpiresAt);
195
190
  },
196
191
  async listRuns(options) {
197
192
  const suppliedLimit = options?.limit;
@@ -267,16 +262,13 @@ function createWorkflowIndex(bindings) {
267
262
  await Promise.all(releases);
268
263
  },
269
264
  async releaseHookIndexes(request) {
270
- if (request.terminal) await fence(request.runId).fenceTerminal(Date.now());
271
- return { deleted: await releaseHooks(request, Date.now()) };
265
+ return { deleted: await releaseHooks(request) };
272
266
  },
273
267
  async expireRun(request) {
274
- await fence(request.runId).fenceExpired(request.expiredAt);
275
268
  const [catalog, hookDeletes] = await Promise.all([stub(bindings.runCatalog, runCatalogShardName(request.runId)).expireRun(request.runId, request.keys, request.expiredAt), releaseHooks({
276
269
  runId: request.runId,
277
- hooks: request.hooks,
278
- terminal: true
279
- }, request.expiredAt)]);
270
+ hooks: request.hooks
271
+ })]);
280
272
  return { deleted: catalog.deleted + hookDeletes };
281
273
  }
282
274
  };
@@ -493,7 +485,6 @@ const WORLD_VERSION = "0.1.0";
493
485
  function workflowIndex(env) {
494
486
  return createWorkflowIndex({
495
487
  runCatalog: env.WORKFLOW_RUN_CATALOG,
496
- runFences: env.WORKFLOW_RUN_FENCES,
497
488
  hookTokens: env.WORKFLOW_HOOK_TOKENS,
498
489
  hookIds: env.WORKFLOW_HOOK_IDS
499
490
  });
@@ -503,6 +494,7 @@ const BINDINGS = {
503
494
  env: "WORKFLOW_DB",
504
495
  methods: /* @__PURE__ */ new Set([
505
496
  "applyEvent",
497
+ "getLifecycleStatus",
506
498
  "getRun",
507
499
  "getStep",
508
500
  "getEvent",
@@ -545,7 +537,8 @@ const BINDINGS = {
545
537
  "redriveDeadLetter",
546
538
  "purgeDeadLetters",
547
539
  "rearmAlarm",
548
- "expireRun"
540
+ "expireRun",
541
+ "acknowledgeExpireRun"
549
542
  ])
550
543
  }
551
544
  };
@@ -654,7 +647,7 @@ function createRouter(env) {
654
647
  let result;
655
648
  const operation = `${parts[2]}.${parts[3]}`;
656
649
  if (operation === "runs.list") result = await index.listRuns(args[0]);
657
- else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
650
+ else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
658
651
  else if (operation === "runs.expire") result = await index.expireRun(args[0]);
659
652
  else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
660
653
  else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
@@ -743,6 +736,120 @@ function createRouter(env) {
743
736
  };
744
737
  }
745
738
  //#endregion
739
+ //#region src/retention.ts
740
+ const CLEANUP_RECORD_KEY = "retention:cleanup";
741
+ const TOMBSTONE_KEY = "retention:tombstone";
742
+ const HOOK_MARKER_PREFIX = "retention:hook:";
743
+ const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
744
+ function sortableTimestamp(date) {
745
+ return date.getTime().toString().padStart(13, "0");
746
+ }
747
+ function workflowRunIndexKey(run) {
748
+ return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
749
+ }
750
+ function globalRunIndexKey(run) {
751
+ return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
752
+ }
753
+ function hookMarkerKey(reference) {
754
+ return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
755
+ }
756
+ function cleanupTombstone(record, now) {
757
+ const cleanup = {
758
+ ...record,
759
+ phase: "tombstoned",
760
+ tombstonedAt: now
761
+ };
762
+ return {
763
+ version: 1,
764
+ runId: record.runId,
765
+ status: record.status ?? record.terminalStatus,
766
+ terminalStatus: record.terminalStatus,
767
+ completedAt: record.completedAt,
768
+ expiredAt: record.dueAt,
769
+ tombstonedAt: now,
770
+ cleanup
771
+ };
772
+ }
773
+ function cleanupFromTombstone(tombstone) {
774
+ return tombstone.cleanup;
775
+ }
776
+ function expiredRead(tombstone) {
777
+ return {
778
+ ok: false,
779
+ code: "RUN_EXPIRED",
780
+ message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
781
+ tombstone
782
+ };
783
+ }
784
+ const MAX_RETENTION_SWEEP_BATCH_SIZE = 1e3;
785
+ const RETENTION_SWEEP_CONCURRENCY = 8;
786
+ function integerSetting(name, raw, fallback, minimum, maximum = Number.MAX_SAFE_INTEGER) {
787
+ const value = raw === void 0 || raw === "" ? fallback : Number(raw);
788
+ if (!Number.isSafeInteger(value) || value < minimum || value > maximum) throw new Error(`world-celld: ${name} must be an integer between ${minimum} and ${maximum}`);
789
+ return value;
790
+ }
791
+ function runIdFromCatalogValue(value) {
792
+ const parsed = JSON.parse(value);
793
+ if (typeof parsed.runId !== "string" || parsed.runId.length === 0) throw new Error("world-celld: run catalog entry has no runId");
794
+ return parsed.runId;
795
+ }
796
+ /**
797
+ * Discover and fence one bounded page of workflows older than the configured
798
+ * maximum age. RunDO alarms finish the persisted, idempotent deletion phases.
799
+ */
800
+ async function runRetentionSweep(scheduledTime, env) {
801
+ if (!Number.isSafeInteger(scheduledTime) || scheduledTime < 0) throw new Error("world-celld: scheduledTime must be a non-negative safe integer");
802
+ const retentionMs = integerSetting("WORKFLOW_RETENTION_MS", env.WORKFLOW_RETENTION_MS, 0, 0);
803
+ if (retentionMs === 0 || scheduledTime < retentionMs) return {
804
+ disabled: retentionMs === 0,
805
+ cutoff: retentionMs === 0 ? null : scheduledTime - retentionMs,
806
+ scanned: 0,
807
+ scheduled: 0,
808
+ expired: 0,
809
+ missing: 0,
810
+ notDue: 0
811
+ };
812
+ const batchSize = integerSetting("WORKFLOW_RETENTION_BATCH_SIZE", env.WORKFLOW_RETENTION_BATCH_SIZE, 128, 1, MAX_RETENTION_SWEEP_BATCH_SIZE);
813
+ const queueShards = integerSetting("WORKFLOW_RETENTION_QUEUE_SHARDS", env.WORKFLOW_RETENTION_QUEUE_SHARDS, 1, 1);
814
+ const cutoff = scheduledTime - retentionMs;
815
+ const page = await createWorkflowIndex({
816
+ runCatalog: env.WORKFLOW_RUN_CATALOG,
817
+ hookTokens: env.WORKFLOW_HOOK_TOKENS,
818
+ hookIds: env.WORKFLOW_HOOK_IDS
819
+ }).listRuns({
820
+ prefix: "runall:",
821
+ end: `runall:${sortableTimestamp(new Date(cutoff + 1))}:`,
822
+ limit: batchSize
823
+ });
824
+ const result = {
825
+ disabled: false,
826
+ cutoff,
827
+ scanned: page.keys.length,
828
+ scheduled: 0,
829
+ expired: 0,
830
+ missing: 0,
831
+ notDue: 0
832
+ };
833
+ const failures = [];
834
+ for (let offset = 0; offset < page.keys.length; offset += RETENTION_SWEEP_CONCURRENCY) {
835
+ const candidates = page.keys.slice(offset, offset + RETENTION_SWEEP_CONCURRENCY);
836
+ const settled = await Promise.allSettled(candidates.map(async (entry) => {
837
+ const runId = runIdFromCatalogValue(entry.value);
838
+ const outcome = await env.WORKFLOW_DB.get(env.WORKFLOW_DB.idFromName(runId)).enforceRetention({
839
+ retentionMs,
840
+ queueShards,
841
+ scheduledTime
842
+ });
843
+ if (outcome.state === "missing" || outcome.state === "not-due") await env.WORKFLOW_RUN_CATALOG.get(env.WORKFLOW_RUN_CATALOG.idFromName(runCatalogShardName(runId))).deleteStaleGlobalRun(runId, entry.name, entry.value);
844
+ return outcome;
845
+ }));
846
+ for (const outcome of settled) if (outcome.status === "rejected") failures.push(outcome.reason);
847
+ else result[outcome.value.state === "not-due" ? "notDue" : outcome.value.state]++;
848
+ }
849
+ if (failures.length > 0) throw new AggregateError(failures, `world-celld retention sweep failed for ${failures.length} run(s)`);
850
+ return result;
851
+ }
852
+ //#endregion
746
853
  //#region src/vendor/shared/util.ts
747
854
  function compact(obj) {
748
855
  const value = {};
@@ -765,26 +872,52 @@ function ownerFromSerializedHook(raw) {
765
872
  function indexKey(prefix, value) {
766
873
  return `${prefix}:${encodeURIComponent(value)}`;
767
874
  }
768
- /** Compact shard-local fence for every hook owned by one terminal or expired run. */
769
- function shardRunFenceKey(runId) {
770
- return `runfence:${encodeURIComponent(runId)}`;
771
- }
772
875
  async function runIsActive(env, runId) {
773
- const namespace = env.WORKFLOW_RUN_FENCES;
774
- if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
775
- return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
876
+ const namespace = env.WORKFLOW_DB;
877
+ if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
878
+ return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
776
879
  }
777
880
  //#endregion
881
+ //#region src/lifecycle.ts
882
+ /**
883
+ * Hard protocol bounds used by lifecycle leases and derivative-fence cleanup.
884
+ *
885
+ * These are not retention TTLs. They bound how long a request which was
886
+ * admitted before expiry may still mutate a derivative Durable Object:
887
+ *
888
+ * - fleet calls have at most three 5-minute attempts plus bounded retry delay;
889
+ * - one queue delivery owns its lease for at most 5 minutes plus 30 seconds;
890
+ * - hook admission spans reserve retries, one authoritative apply, and
891
+ * finalize retries before the exact claim must be reacquired.
892
+ */
893
+ const MAX_FLEET_RPC_TIMEOUT_MS = 3e5;
894
+ const MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS = 1200900;
895
+ const MAX_QUEUE_DELIVERY_TIMEOUT_MS = 3e5;
896
+ const QUEUE_INFLIGHT_GRACE_MS = 3e4;
897
+ const QUEUE_FENCE_GRACE_MS = Math.max(MAX_FLEET_RPC_TIMEOUT_MS, 33e4);
898
+ const HOOK_CLAIM_LEASE_MS = 2101800;
899
+ const LIFECYCLE_COMPACTION_RETRY_MS = 1e3;
900
+ //#endregion
778
901
  //#region src/worker/durable-objects/HookIdDO.ts
779
902
  const MAX_RELEASE_BATCH$1 = 64;
903
+ const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
904
+ function pad$3(ms) {
905
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
906
+ }
780
907
  function recordKey$1(hookId) {
781
908
  return indexKey("hookid", hookId);
782
909
  }
783
910
  function claimKey$1(hookId) {
784
911
  return indexKey("claim", hookId);
785
912
  }
913
+ function claimDeadlineKey$1(hookId, expiresAt) {
914
+ return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
915
+ }
786
916
  /** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
787
917
  var HookIdDO = class extends DurableObject {
918
+ now() {
919
+ return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
920
+ }
788
921
  async get(hookId) {
789
922
  const raw = await this.ctx.storage.get(recordKey$1(hookId));
790
923
  if (!raw) return null;
@@ -801,17 +934,33 @@ var HookIdDO = class extends DurableObject {
801
934
  claimed: false,
802
935
  holder: ownerFromSerializedHook(indexed)
803
936
  };
804
- const existing = values.get(claim);
937
+ let existing = values.get(claim);
938
+ const now = this.now();
939
+ if (existing && existing.expiresAt <= now) {
940
+ await txn.delete([claim, existing.deadlineKey]);
941
+ existing = void 0;
942
+ }
805
943
  if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
806
944
  claimed: false,
807
945
  holder: existing.owner
808
946
  };
809
947
  if (existing === void 0 || existing.claimId !== proposedClaimId) {
948
+ if (existing) await txn.delete(existing.deadlineKey);
949
+ const expiresAt = now + HOOK_CLAIM_LEASE_MS;
950
+ const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
810
951
  await txn.put(claim, {
811
952
  owner,
812
953
  claimId: proposedClaimId,
813
- reservedAt: Date.now()
954
+ reservedAt: now,
955
+ expiresAt,
956
+ deadlineKey
957
+ });
958
+ await txn.put(deadlineKey, {
959
+ hookId,
960
+ claimId: proposedClaimId,
961
+ expiresAt
814
962
  });
963
+ await this.armAtMost(txn, expiresAt);
815
964
  return {
816
965
  claimed: true,
817
966
  claimId: proposedClaimId
@@ -828,30 +977,24 @@ var HookIdDO = class extends DurableObject {
828
977
  await this.releaseBatch([{
829
978
  hookId,
830
979
  owner
831
- }], Date.now(), true);
980
+ }]);
832
981
  return { stored: false };
833
982
  }
834
983
  return await this.ctx.storage.transaction(async (txn) => {
835
984
  const record = recordKey$1(hookId);
836
985
  const claim = claimKey$1(hookId);
837
- const runFence = shardRunFenceKey(owner.runId);
838
- const values = await txn.get([
839
- record,
840
- claim,
841
- runFence
842
- ]);
843
- if (values.has(runFence)) {
844
- const existingClaim = values.get(claim);
845
- if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
846
- return { stored: false };
847
- }
986
+ const values = await txn.get([record, claim]);
848
987
  const indexed = values.get(record);
849
988
  if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
850
989
  const existingClaim = values.get(claim);
990
+ if (existingClaim && existingClaim.expiresAt <= this.now()) {
991
+ await txn.delete([claim, existingClaim.deadlineKey]);
992
+ return { stored: false };
993
+ }
851
994
  if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
852
995
  if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
853
996
  await txn.put(record, serializedHook);
854
- if (existingClaim !== void 0) await txn.delete(claim);
997
+ if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
855
998
  return { stored: true };
856
999
  });
857
1000
  }
@@ -859,40 +1002,94 @@ var HookIdDO = class extends DurableObject {
859
1002
  await this.ctx.storage.transaction(async (txn) => {
860
1003
  const key = claimKey$1(hookId);
861
1004
  const claim = await txn.get(key);
862
- if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
1005
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
863
1006
  });
864
1007
  }
865
- async releaseBatch(hooks, fencedAt, runFenced) {
1008
+ async releaseBatch(hooks) {
866
1009
  if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
867
1010
  if (hooks.length === 0) return { deleted: 0 };
868
1011
  return await this.ctx.storage.transaction(async (txn) => {
869
1012
  const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
870
1013
  const values = await txn.get(keys);
871
1014
  const deletes = [];
1015
+ const deadlineDeletes = [];
872
1016
  for (const { hookId, owner } of hooks) {
873
1017
  const record = recordKey$1(hookId);
874
1018
  const claim = claimKey$1(hookId);
875
1019
  const indexed = values.get(record);
876
1020
  const existingClaim = values.get(claim);
877
1021
  if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
878
- if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
1022
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
1023
+ deletes.push(claim);
1024
+ deadlineDeletes.push(existingClaim.deadlineKey);
1025
+ }
1026
+ }
1027
+ const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
1028
+ if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
1029
+ return { deleted };
1030
+ });
1031
+ }
1032
+ async alarm() {
1033
+ try {
1034
+ await this.compactClaims();
1035
+ } catch {
1036
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
1037
+ }
1038
+ }
1039
+ async armAtMost(storage, at) {
1040
+ const current = await storage.getAlarm();
1041
+ if (current === null || at < current) await storage.setAlarm(at);
1042
+ }
1043
+ async compactClaims() {
1044
+ const now = this.now();
1045
+ await this.ctx.storage.transaction(async (txn) => {
1046
+ const due = await txn.list({
1047
+ prefix: CLAIM_DEADLINE_PREFIX$1,
1048
+ end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
1049
+ limit: 129
1050
+ });
1051
+ const page = Array.from(due).slice(0, 128);
1052
+ const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
1053
+ const deletes = page.flatMap(([deadline, item]) => {
1054
+ const key = claimKey$1(item.hookId);
1055
+ const claim = claims.get(key);
1056
+ return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
1057
+ });
1058
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
1059
+ if (due.size > 128) {
1060
+ await txn.setAlarm(now + 1);
1061
+ return;
879
1062
  }
880
- if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
881
- return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
1063
+ const nextAt = (await txn.list({
1064
+ prefix: CLAIM_DEADLINE_PREFIX$1,
1065
+ limit: 1
1066
+ })).values().next().value?.expiresAt;
1067
+ if (nextAt === void 0) await txn.deleteAlarm();
1068
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
882
1069
  });
883
1070
  }
884
1071
  };
885
1072
  //#endregion
886
1073
  //#region src/worker/durable-objects/HookTokenDO.ts
887
1074
  const MAX_RELEASE_BATCH = 64;
1075
+ const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
1076
+ function pad$2(ms) {
1077
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
1078
+ }
888
1079
  function recordKey(token) {
889
1080
  return indexKey("hook", token);
890
1081
  }
891
1082
  function claimKey(token) {
892
1083
  return indexKey("claim", token);
893
1084
  }
1085
+ function claimDeadlineKey(token, expiresAt) {
1086
+ return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
1087
+ }
894
1088
  /** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
895
1089
  var HookTokenDO = class extends DurableObject {
1090
+ now() {
1091
+ return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
1092
+ }
896
1093
  async get(token) {
897
1094
  const raw = await this.ctx.storage.get(recordKey(token));
898
1095
  if (!raw) return null;
@@ -909,17 +1106,32 @@ var HookTokenDO = class extends DurableObject {
909
1106
  claimed: false,
910
1107
  holder: ownerFromSerializedHook(indexed)
911
1108
  };
912
- const existing = values.get(claim);
1109
+ let existing = values.get(claim);
1110
+ const now = this.now();
1111
+ if (existing && existing.expiresAt <= now) {
1112
+ await txn.delete([claim, existing.deadlineKey]);
1113
+ existing = void 0;
1114
+ }
913
1115
  if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
914
1116
  claimed: false,
915
1117
  holder: existing.owner
916
1118
  };
917
1119
  if (existing === void 0) {
1120
+ const expiresAt = now + HOOK_CLAIM_LEASE_MS;
1121
+ const deadlineKey = claimDeadlineKey(token, expiresAt);
918
1122
  await txn.put(claim, {
919
1123
  owner,
920
1124
  claimId: proposedClaimId,
921
- reservedAt: Date.now()
1125
+ reservedAt: now,
1126
+ expiresAt,
1127
+ deadlineKey
1128
+ });
1129
+ await txn.put(deadlineKey, {
1130
+ token,
1131
+ claimId: proposedClaimId,
1132
+ expiresAt
922
1133
  });
1134
+ await this.armAtMost(txn, expiresAt);
923
1135
  return {
924
1136
  claimed: true,
925
1137
  claimId: proposedClaimId
@@ -936,30 +1148,24 @@ var HookTokenDO = class extends DurableObject {
936
1148
  await this.releaseBatch([{
937
1149
  token,
938
1150
  owner
939
- }], Date.now(), true);
1151
+ }]);
940
1152
  return { stored: false };
941
1153
  }
942
1154
  return await this.ctx.storage.transaction(async (txn) => {
943
1155
  const record = recordKey(token);
944
1156
  const claim = claimKey(token);
945
- const runFence = shardRunFenceKey(owner.runId);
946
- const values = await txn.get([
947
- record,
948
- claim,
949
- runFence
950
- ]);
951
- if (values.has(runFence)) {
952
- const existingClaim = values.get(claim);
953
- if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
954
- return { stored: false };
955
- }
1157
+ const values = await txn.get([record, claim]);
956
1158
  const indexed = values.get(record);
957
1159
  if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
958
1160
  const existingClaim = values.get(claim);
1161
+ if (existingClaim && existingClaim.expiresAt <= this.now()) {
1162
+ await txn.delete([claim, existingClaim.deadlineKey]);
1163
+ return { stored: false };
1164
+ }
959
1165
  if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
960
1166
  if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
961
1167
  await txn.put(record, serializedHook);
962
- if (existingClaim !== void 0) await txn.delete(claim);
1168
+ if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
963
1169
  return { stored: true };
964
1170
  });
965
1171
  }
@@ -967,26 +1173,70 @@ var HookTokenDO = class extends DurableObject {
967
1173
  await this.ctx.storage.transaction(async (txn) => {
968
1174
  const key = claimKey(token);
969
1175
  const claim = await txn.get(key);
970
- if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
1176
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
971
1177
  });
972
1178
  }
973
- async releaseBatch(hooks, fencedAt, runFenced) {
1179
+ async releaseBatch(hooks) {
974
1180
  if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
975
1181
  if (hooks.length === 0) return { deleted: 0 };
976
1182
  return await this.ctx.storage.transaction(async (txn) => {
977
1183
  const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
978
1184
  const values = await txn.get(keys);
979
1185
  const deletes = [];
1186
+ const deadlineDeletes = [];
980
1187
  for (const { token, owner } of hooks) {
981
1188
  const record = recordKey(token);
982
1189
  const claim = claimKey(token);
983
1190
  const indexed = values.get(record);
984
1191
  const existingClaim = values.get(claim);
985
1192
  if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
986
- if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
1193
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
1194
+ deletes.push(claim);
1195
+ deadlineDeletes.push(existingClaim.deadlineKey);
1196
+ }
987
1197
  }
988
- if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
989
- return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
1198
+ const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
1199
+ if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
1200
+ return { deleted };
1201
+ });
1202
+ }
1203
+ async alarm() {
1204
+ try {
1205
+ await this.compactClaims();
1206
+ } catch {
1207
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
1208
+ }
1209
+ }
1210
+ async armAtMost(storage, at) {
1211
+ const current = await storage.getAlarm();
1212
+ if (current === null || at < current) await storage.setAlarm(at);
1213
+ }
1214
+ async compactClaims() {
1215
+ const now = this.now();
1216
+ await this.ctx.storage.transaction(async (txn) => {
1217
+ const due = await txn.list({
1218
+ prefix: CLAIM_DEADLINE_PREFIX,
1219
+ end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
1220
+ limit: 129
1221
+ });
1222
+ const page = Array.from(due).slice(0, 128);
1223
+ const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
1224
+ const deletes = page.flatMap(([deadline, item]) => {
1225
+ const key = claimKey(item.token);
1226
+ const claim = claims.get(key);
1227
+ return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
1228
+ });
1229
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
1230
+ if (due.size > 128) {
1231
+ await txn.setAlarm(now + 1);
1232
+ return;
1233
+ }
1234
+ const nextAt = (await txn.list({
1235
+ prefix: CLAIM_DEADLINE_PREFIX,
1236
+ limit: 1
1237
+ })).values().next().value?.expiresAt;
1238
+ if (nextAt === void 0) await txn.deleteAlarm();
1239
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
990
1240
  });
991
1241
  }
992
1242
  };
@@ -1001,8 +1251,6 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
1001
1251
  const DEFAULT_MAX_ATTEMPTS = 5;
1002
1252
  const DEFAULT_MAX_INFLIGHT = 5;
1003
1253
  const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
1004
- /** Grace period past the delivery timeout before a claim counts as lost. */
1005
- const INFLIGHT_GRACE_MS = 3e4;
1006
1254
  /** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
1007
1255
  const MIN_ALARM_DELAY_MS = 1;
1008
1256
  /** Bound expiry recovery so one alarm cannot monopolize the cell. */
@@ -1011,14 +1259,15 @@ const STORAGE_BATCH_SIZE$2 = 128;
1011
1259
  const EXPIRE_RUN_REFERENCE_BATCH = 64;
1012
1260
  const PURGE_DEAD_LETTER_BATCH = 128;
1013
1261
  const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
1014
- function pad(ms) {
1262
+ const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
1263
+ function pad$1(ms) {
1015
1264
  return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
1016
1265
  }
1017
1266
  function dueKey(atMs, messageId) {
1018
- return `due:${pad(atMs)}:${messageId}`;
1267
+ return `due:${pad$1(atMs)}:${messageId}`;
1019
1268
  }
1020
1269
  function inflightKey(deadlineMs, messageId) {
1021
- return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
1270
+ return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
1022
1271
  }
1023
1272
  function deadlineFromInflightKey(key) {
1024
1273
  return Number.parseInt(key.slice(18, 31), 10);
@@ -1029,6 +1278,9 @@ function runReferenceKey(runId, messageId) {
1029
1278
  function expiredRunKey(runId) {
1030
1279
  return `expired-run:${runId}`;
1031
1280
  }
1281
+ function expiredRunGcKey(runId, compactAt) {
1282
+ return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
1283
+ }
1032
1284
  function boundedLimit$1(value, fallback, maximum) {
1033
1285
  if (value === void 0 || !Number.isFinite(value)) return fallback;
1034
1286
  return Math.max(1, Math.min(maximum, Math.floor(value)));
@@ -1079,16 +1331,29 @@ var QueueDO = class extends DurableObject {
1079
1331
  if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
1080
1332
  return configured;
1081
1333
  }
1334
+ #deliveryTimeoutMs() {
1335
+ const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1336
+ if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
1337
+ return configured;
1338
+ }
1339
+ async #authoritativeRunExpired(runId) {
1340
+ const namespace = this.env?.WORKFLOW_DB;
1341
+ if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
1342
+ return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
1343
+ }
1344
+ async #withRunMutation(runId, onExpired, onAllowed) {
1345
+ const expired = await this.#authoritativeRunExpired(runId);
1346
+ return await this.ctx.storage.transaction(async (txn) => {
1347
+ if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
1348
+ return await onAllowed(txn);
1349
+ });
1350
+ }
1082
1351
  async enqueue(request) {
1083
1352
  this.#maxInflight();
1353
+ this.#deliveryTimeoutMs();
1084
1354
  const storage = this.ctx.storage;
1085
1355
  const now = this.#now();
1086
- return await storage.transaction(async (txn) => {
1087
- if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
1088
- ok: false,
1089
- code: "RUN_EXPIRED",
1090
- message: `Workflow run "${request.runId}" has expired`
1091
- };
1356
+ const persist = async (txn) => {
1092
1357
  const pinned = await txn.get("cfg");
1093
1358
  if (!pinned) await txn.put("cfg", {
1094
1359
  queueShards: request.config.queueShards,
@@ -1140,7 +1405,12 @@ var QueueDO = class extends DurableObject {
1140
1405
  messageId: row.messageId,
1141
1406
  deduped: false
1142
1407
  };
1143
- });
1408
+ };
1409
+ return request.runId ? await this.#withRunMutation(request.runId, async () => ({
1410
+ ok: false,
1411
+ code: "RUN_EXPIRED",
1412
+ message: `Workflow run "${request.runId}" has expired`
1413
+ }), persist) : await storage.transaction(persist);
1144
1414
  }
1145
1415
  async alarm() {
1146
1416
  let claimed = [];
@@ -1154,7 +1424,10 @@ var QueueDO = class extends DurableObject {
1154
1424
  };
1155
1425
  if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
1156
1426
  else await attempt();
1157
- if (failure !== null) throw failure;
1427
+ if (failure !== null) {
1428
+ await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
1429
+ throw failure;
1430
+ }
1158
1431
  for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
1159
1432
  }
1160
1433
  /**
@@ -1166,9 +1439,13 @@ var QueueDO = class extends DurableObject {
1166
1439
  const now = this.#now();
1167
1440
  const maxInflight = this.#maxInflight();
1168
1441
  return await storage.transaction(async (txn) => {
1442
+ if (await this.#compactExpiredRunFences(txn, now)) {
1443
+ await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
1444
+ return [];
1445
+ }
1169
1446
  const expiredPage = await txn.list({
1170
1447
  prefix: INFLIGHT_DEADLINE_PREFIX,
1171
- end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
1448
+ end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
1172
1449
  limit: 129
1173
1450
  });
1174
1451
  const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
@@ -1200,17 +1477,17 @@ var QueueDO = class extends DurableObject {
1200
1477
  if (inflightCount < maxInflight) {
1201
1478
  const due = await txn.list({
1202
1479
  prefix: "due:",
1203
- end: `due:${pad(now + 1)}`,
1480
+ end: `due:${pad$1(now + 1)}`,
1204
1481
  limit: maxInflight - inflightCount
1205
1482
  });
1206
- const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1483
+ const timeoutMs = this.#deliveryTimeoutMs();
1207
1484
  const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
1208
1485
  await deleteMany$1(txn, Array.from(due.keys()));
1209
1486
  const claimWrites = [];
1210
1487
  for (const [, messageId] of due) {
1211
1488
  const row = dueRows.get(`msg:${messageId}`);
1212
1489
  if (!row) continue;
1213
- const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
1490
+ const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
1214
1491
  claimWrites.push([claimKey, messageId]);
1215
1492
  if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
1216
1493
  messageId: row.messageId,
@@ -1230,9 +1507,13 @@ var QueueDO = class extends DurableObject {
1230
1507
  }
1231
1508
  async #deliver({ row, inflightKey: claimKey }) {
1232
1509
  const storage = this.ctx.storage;
1233
- const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1510
+ const timeoutMs = this.#deliveryTimeoutMs();
1234
1511
  const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
1235
1512
  const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
1513
+ if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
1514
+ await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
1515
+ return;
1516
+ }
1236
1517
  let response = null;
1237
1518
  let transportError = null;
1238
1519
  try {
@@ -1262,28 +1543,21 @@ var QueueDO = class extends DurableObject {
1262
1543
  } catch {
1263
1544
  timeoutSeconds = void 0;
1264
1545
  }
1265
- if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
1266
- if (!await txn.delete(claimKey)) return;
1267
- if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
1268
- await txn.delete(`msg:${row.messageId}`);
1269
- await txn.delete(runReferenceKey(row.runId, row.messageId));
1270
- if (row.idempotencyKey) {
1271
- const dedupKey = `key:${row.idempotencyKey}`;
1272
- if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1273
- }
1546
+ if (typeof timeoutSeconds === "number") {
1547
+ const reschedule = async (txn) => {
1548
+ if (!await txn.delete(claimKey)) return;
1549
+ const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
1550
+ await txn.put(scheduleKey, row.messageId);
1551
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1552
+ messageId: row.messageId,
1553
+ dueKey: scheduleKey,
1554
+ idempotencyKey: row.idempotencyKey
1555
+ });
1274
1556
  await this.#scheduleNextAlarm(txn, now);
1275
- return;
1276
- }
1277
- const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
1278
- await txn.put(scheduleKey, row.messageId);
1279
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1280
- messageId: row.messageId,
1281
- dueKey: scheduleKey,
1282
- idempotencyKey: row.idempotencyKey
1283
- });
1284
- await this.#scheduleNextAlarm(txn, now);
1285
- });
1286
- else await this.#retry(row, claimKey, "HTTP 503");
1557
+ };
1558
+ if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
1559
+ else await storage.transaction(reschedule);
1560
+ } else await this.#retry(row, claimKey, "HTTP 503");
1287
1561
  } else if (response && PERMANENT_STATUSES.has(response.status)) {
1288
1562
  await cancelUnusedResponseBody(response);
1289
1563
  await this.#ack(row, claimKey);
@@ -1304,23 +1578,23 @@ var QueueDO = class extends DurableObject {
1304
1578
  await this.#scheduleNextAlarm(txn, this.#now());
1305
1579
  });
1306
1580
  }
1581
+ async #dropExpiredMessage(txn, row, claimKey, now) {
1582
+ if (!await txn.delete(claimKey)) return;
1583
+ await txn.delete(`msg:${row.messageId}`);
1584
+ if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
1585
+ if (row.idempotencyKey) {
1586
+ const dedupKey = `key:${row.idempotencyKey}`;
1587
+ if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1588
+ }
1589
+ await this.#scheduleNextAlarm(txn, now);
1590
+ }
1307
1591
  async #retry(row, claimKey, reason) {
1308
1592
  const storage = this.ctx.storage;
1309
1593
  const now = this.#now();
1310
1594
  const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
1311
1595
  const attempt = row.attempt + 1;
1312
- await storage.transaction(async (txn) => {
1596
+ const persistRetry = async (txn) => {
1313
1597
  if (!await txn.delete(claimKey)) return;
1314
- if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
1315
- await txn.delete(`msg:${row.messageId}`);
1316
- await txn.delete(runReferenceKey(row.runId, row.messageId));
1317
- if (row.idempotencyKey) {
1318
- const dedupKey = `key:${row.idempotencyKey}`;
1319
- if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1320
- }
1321
- await this.#scheduleNextAlarm(txn, now);
1322
- return;
1323
- }
1324
1598
  if (attempt >= maxAttempts) {
1325
1599
  const dead = {
1326
1600
  ...row,
@@ -1328,7 +1602,7 @@ var QueueDO = class extends DurableObject {
1328
1602
  lastError: reason,
1329
1603
  failedAt: now
1330
1604
  };
1331
- const deadKey = `dlq:${pad(now)}:${row.messageId}`;
1605
+ const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
1332
1606
  await txn.put(deadKey, dead);
1333
1607
  await txn.delete(`msg:${row.messageId}`);
1334
1608
  if (row.idempotencyKey) {
@@ -1356,7 +1630,32 @@ var QueueDO = class extends DurableObject {
1356
1630
  idempotencyKey: row.idempotencyKey
1357
1631
  });
1358
1632
  await this.#scheduleNextAlarm(txn, now);
1633
+ };
1634
+ if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
1635
+ else await storage.transaction(persistRetry);
1636
+ }
1637
+ /** Return true when another due compaction page must run immediately. */
1638
+ async #compactExpiredRunFences(txn, now) {
1639
+ const due = await txn.list({
1640
+ prefix: EXPIRED_RUN_GC_PREFIX,
1641
+ end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
1642
+ limit: 129
1359
1643
  });
1644
+ const page = Array.from(due).slice(0, 128);
1645
+ if (page.length === 0) return false;
1646
+ const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
1647
+ const markers = await getMany$1(txn, markerKeys);
1648
+ const deletes = [];
1649
+ for (let index = 0; index < page.length; index++) {
1650
+ const [gcKey, item] = page[index];
1651
+ const markerKey = markerKeys[index];
1652
+ const marker = markers.get(markerKey);
1653
+ deletes.push(gcKey);
1654
+ if (marker?.compactAt !== item.compactAt) continue;
1655
+ deletes.push(markerKey);
1656
+ }
1657
+ await deleteMany$1(txn, deletes);
1658
+ return due.size > 128;
1360
1659
  }
1361
1660
  /** Arm the alarm no later than `atMs` (durable backoff never throws). */
1362
1661
  async #armAlarmAtMost(storage, atMs, now) {
@@ -1388,6 +1687,11 @@ var QueueDO = class extends DurableObject {
1388
1687
  const deadline = deadlineFromInflightKey(key);
1389
1688
  if (next === null || deadline < next) next = deadline;
1390
1689
  }
1690
+ const expiryGc = await storage.list({
1691
+ prefix: EXPIRED_RUN_GC_PREFIX,
1692
+ limit: 1
1693
+ });
1694
+ for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
1391
1695
  if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
1392
1696
  else await storage.deleteAlarm();
1393
1697
  }
@@ -1426,37 +1730,42 @@ var QueueDO = class extends DurableObject {
1426
1730
  async redriveDeadLetter(messageId) {
1427
1731
  const storage = this.ctx.storage;
1428
1732
  const now = this.#now();
1429
- return await storage.transaction(async (txn) => {
1430
- const dlq = await txn.list({ prefix: "dlq:" });
1431
- for (const [key, dead] of dlq) {
1432
- if (dead.messageId !== messageId) continue;
1433
- const { failedAt: _failedAt, ...rest } = dead;
1434
- const row = {
1435
- ...rest,
1436
- attempt: 0,
1437
- lastError: void 0
1438
- };
1439
- if (row.idempotencyKey) {
1440
- const dedupKey = `key:${row.idempotencyKey}`;
1441
- const holder = await txn.get(dedupKey);
1442
- if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
1443
- if (holder && holder !== row.messageId) await txn.delete(dedupKey);
1444
- await txn.put(dedupKey, row.messageId);
1445
- }
1446
- await txn.put(`msg:${row.messageId}`, row);
1447
- const scheduleKey = dueKey(now, row.messageId);
1448
- await txn.put(scheduleKey, row.messageId);
1449
- await txn.delete(key);
1450
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1451
- messageId: row.messageId,
1452
- dueKey: scheduleKey,
1453
- idempotencyKey: row.idempotencyKey
1454
- });
1455
- await this.#armAlarmAtMost(txn, now, now);
1456
- return { ok: true };
1733
+ const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
1734
+ if (!found) return { ok: false };
1735
+ const [deadKey, candidate] = found;
1736
+ const redrive = async (txn) => {
1737
+ const dead = await txn.get(deadKey);
1738
+ if (!dead || dead.messageId !== messageId) return { ok: false };
1739
+ const { failedAt: _failedAt, ...rest } = dead;
1740
+ const row = {
1741
+ ...rest,
1742
+ attempt: 0,
1743
+ lastError: void 0
1744
+ };
1745
+ if (row.idempotencyKey) {
1746
+ const dedupKey = `key:${row.idempotencyKey}`;
1747
+ const holder = await txn.get(dedupKey);
1748
+ if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
1749
+ if (holder && holder !== row.messageId) await txn.delete(dedupKey);
1750
+ await txn.put(dedupKey, row.messageId);
1457
1751
  }
1752
+ await txn.put(`msg:${row.messageId}`, row);
1753
+ const scheduleKey = dueKey(now, row.messageId);
1754
+ await txn.put(scheduleKey, row.messageId);
1755
+ await txn.delete(deadKey);
1756
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1757
+ messageId: row.messageId,
1758
+ dueKey: scheduleKey,
1759
+ idempotencyKey: row.idempotencyKey
1760
+ });
1761
+ await this.#armAlarmAtMost(txn, now, now);
1762
+ return { ok: true };
1763
+ };
1764
+ const rejectExpired = async (txn) => {
1765
+ if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
1458
1766
  return { ok: false };
1459
- });
1767
+ };
1768
+ return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
1460
1769
  }
1461
1770
  async purgeDeadLetters() {
1462
1771
  return await this.ctx.storage.transaction(async (txn) => {
@@ -1506,27 +1815,75 @@ var QueueDO = class extends DurableObject {
1506
1815
  count++;
1507
1816
  }
1508
1817
  await deleteMany$1(txn, keys);
1818
+ const done = references.size <= limit;
1819
+ const markerExpiredAt = existing?.expiredAt ?? expiredAt;
1509
1820
  await txn.put(expiredRunKey(runId), {
1510
- expiredAt,
1511
- deleted: count
1821
+ expiredAt: markerExpiredAt,
1822
+ deleted: count,
1823
+ compactAt: existing?.compactAt ?? null
1512
1824
  });
1513
1825
  await this.#scheduleNextAlarm(txn, this.#now());
1514
- return {
1826
+ return done ? {
1827
+ deleted: count,
1828
+ done: true,
1829
+ receipt: {
1830
+ expiredAt: markerExpiredAt,
1831
+ deleted: count
1832
+ }
1833
+ } : {
1515
1834
  deleted: count,
1516
- done: references.size <= limit
1835
+ done: false
1517
1836
  };
1518
1837
  });
1519
1838
  }
1839
+ /** Acknowledge a final cleanup receipt before making its marker compactable. */
1840
+ async acknowledgeExpireRun(runId, receipt) {
1841
+ return await this.ctx.storage.transaction(async (txn) => {
1842
+ const key = expiredRunKey(runId);
1843
+ const marker = await txn.get(key);
1844
+ if (!marker) return { acknowledged: true };
1845
+ if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
1846
+ if (marker.compactAt === null) {
1847
+ const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
1848
+ await txn.put(key, {
1849
+ ...marker,
1850
+ compactAt
1851
+ });
1852
+ await txn.put(expiredRunGcKey(runId, compactAt), {
1853
+ runId,
1854
+ compactAt
1855
+ });
1856
+ }
1857
+ await this.#scheduleNextAlarm(txn, this.#now());
1858
+ return { acknowledged: true };
1859
+ });
1860
+ }
1520
1861
  };
1521
1862
  //#endregion
1522
1863
  //#region src/worker/durable-objects/RunCatalogDO.ts
1523
1864
  const MAX_INDEX_LIST_SIZE = 1001;
1865
+ const EXPIRY_GC_PREFIX = "expiry-gc:";
1866
+ function pad(ms) {
1867
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
1868
+ }
1869
+ function expiredKey(runId) {
1870
+ return `expired:${runId}`;
1871
+ }
1872
+ function expiryGcKey(runId, compactAt) {
1873
+ return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
1874
+ }
1524
1875
  /** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
1525
1876
  var RunCatalogDO = class extends DurableObject {
1526
- async upsertRun(runId, keys, serializedMetadata) {
1877
+ now() {
1878
+ const clock = this.env?.clock;
1879
+ return typeof clock === "function" ? clock() : Date.now();
1880
+ }
1881
+ async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
1527
1882
  if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
1528
1883
  return await this.ctx.storage.transaction(async (txn) => {
1529
- if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
1884
+ if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
1885
+ const now = this.now();
1886
+ if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
1530
1887
  await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
1531
1888
  return { stored: true };
1532
1889
  });
@@ -1538,8 +1895,11 @@ var RunCatalogDO = class extends DurableObject {
1538
1895
  prefix: options.prefix ?? "",
1539
1896
  ...options.reverse ? {
1540
1897
  reverse: true,
1541
- end: options.cursor
1542
- } : { startAfter: options.cursor },
1898
+ end: options.cursor ?? options.end
1899
+ } : {
1900
+ startAfter: options.cursor,
1901
+ end: options.end
1902
+ },
1543
1903
  limit: limit + 1
1544
1904
  });
1545
1905
  const page = Array.from(entries).slice(0, limit);
@@ -1553,41 +1913,68 @@ var RunCatalogDO = class extends DurableObject {
1553
1913
  cursor: listComplete ? void 0 : page.at(-1)?.[0]
1554
1914
  };
1555
1915
  }
1916
+ /** Compare-and-delete one rejected retention discovery candidate. */
1917
+ async deleteStaleGlobalRun(runId, key, expectedValue) {
1918
+ if (!key.startsWith("runall:") || !key.endsWith(`:${runId}`)) throw new Error("Stale run catalog deletion requires the exact global run key");
1919
+ if (JSON.parse(expectedValue).runId !== runId) throw new Error("Stale run catalog deletion requires matching run metadata");
1920
+ return await this.ctx.storage.transaction(async (txn) => {
1921
+ if (await txn.get(key) !== expectedValue) return { deleted: false };
1922
+ return { deleted: await txn.delete(key) };
1923
+ });
1924
+ }
1556
1925
  async expireRun(runId, keys, expiredAt) {
1557
1926
  if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
1558
1927
  return await this.ctx.storage.transaction(async (txn) => {
1559
- await txn.put(`expired:${runId}`, expiredAt);
1928
+ const marker = expiredKey(runId);
1929
+ const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
1930
+ await txn.put(marker, {
1931
+ expiredAt,
1932
+ compactAt
1933
+ });
1934
+ await txn.put(expiryGcKey(runId, compactAt), {
1935
+ runId,
1936
+ compactAt
1937
+ });
1938
+ await this.armAtMost(txn, compactAt);
1560
1939
  return { deleted: await txn.delete(keys) };
1561
1940
  });
1562
1941
  }
1563
- };
1564
- //#endregion
1565
- //#region src/worker/durable-objects/RunFenceDO.ts
1566
- /** Per-run lifecycle fence used by hook shards without a global lookup cell. */
1567
- var RunFenceDO = class extends DurableObject {
1568
- async getStatus() {
1569
- return (await this.ctx.storage.get("fence"))?.status ?? "active";
1942
+ async alarm() {
1943
+ try {
1944
+ await this.compactExpiryFences();
1945
+ } catch {
1946
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
1947
+ }
1570
1948
  }
1571
- async fenceTerminal(at) {
1572
- return await this.ctx.storage.transaction(async (txn) => {
1573
- const current = await txn.get("fence");
1574
- if (current) return current.status;
1575
- await txn.put("fence", {
1576
- status: "terminal",
1577
- at
1578
- });
1579
- return "terminal";
1580
- });
1949
+ async armAtMost(storage, at) {
1950
+ const current = await storage.getAlarm();
1951
+ if (current === null || at < current) await storage.setAlarm(at);
1581
1952
  }
1582
- async fenceExpired(at) {
1583
- return await this.ctx.storage.transaction(async (txn) => {
1584
- const current = await txn.get("fence");
1585
- if (current?.status === "expired") return current.status;
1586
- await txn.put("fence", {
1587
- status: "expired",
1588
- at
1953
+ async compactExpiryFences() {
1954
+ const now = this.now();
1955
+ await this.ctx.storage.transaction(async (txn) => {
1956
+ const due = await txn.list({
1957
+ prefix: EXPIRY_GC_PREFIX,
1958
+ end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
1959
+ limit: 129
1960
+ });
1961
+ const page = Array.from(due).slice(0, 128);
1962
+ const markerKeys = page.map(([, item]) => expiredKey(item.runId));
1963
+ const markers = await txn.get(markerKeys);
1964
+ const deletes = page.flatMap(([key, item], index) => {
1965
+ return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
1589
1966
  });
1590
- return "expired";
1967
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
1968
+ if (due.size > 128) {
1969
+ await txn.setAlarm(now + 1);
1970
+ return;
1971
+ }
1972
+ const nextAt = (await txn.list({
1973
+ prefix: EXPIRY_GC_PREFIX,
1974
+ limit: 1
1975
+ })).values().next().value?.compactAt;
1976
+ if (nextAt === void 0) await txn.deleteAlarm();
1977
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
1591
1978
  });
1592
1979
  }
1593
1980
  };
@@ -2793,42 +3180,6 @@ async function finalizeEventPage(store, outcome, params) {
2793
3180
  };
2794
3181
  }
2795
3182
  //#endregion
2796
- //#region src/retention.ts
2797
- const CLEANUP_RECORD_KEY = "retention:cleanup";
2798
- const TOMBSTONE_KEY = "retention:tombstone";
2799
- const HOOK_MARKER_PREFIX = "retention:hook:";
2800
- const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
2801
- function sortableTimestamp(date) {
2802
- return date.getTime().toString().padStart(13, "0");
2803
- }
2804
- function workflowRunIndexKey(run) {
2805
- return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
2806
- }
2807
- function globalRunIndexKey(run) {
2808
- return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
2809
- }
2810
- function hookMarkerKey(reference) {
2811
- return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
2812
- }
2813
- function cleanupTombstone(record, now) {
2814
- return {
2815
- version: 1,
2816
- runId: record.runId,
2817
- terminalStatus: record.terminalStatus,
2818
- completedAt: record.completedAt,
2819
- expiredAt: record.dueAt,
2820
- tombstonedAt: now
2821
- };
2822
- }
2823
- function expiredRead(tombstone) {
2824
- return {
2825
- ok: false,
2826
- code: "RUN_EXPIRED",
2827
- message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
2828
- tombstone
2829
- };
2830
- }
2831
- //#endregion
2832
3183
  //#region src/worker/durable-objects/WorkflowRunDO.ts
2833
3184
  const STORAGE_BATCH_SIZE = 128;
2834
3185
  const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
@@ -2839,9 +3190,30 @@ const STREAMS_PER_CLEANUP_PAGE = 16;
2839
3190
  const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
2840
3191
  const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
2841
3192
  const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
2842
- const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
2843
3193
  const CLEANUP_RETRY_MAX_MS = 36e5;
2844
3194
  const CLEANUP_PROGRESS_KEY = "retention:progress";
3195
+ const RUN_QUEUE_SHARDS_KEY = "retention:queue-shards";
3196
+ function cleanupRecord(run, dueAt, queueShards, reason) {
3197
+ const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
3198
+ return {
3199
+ version: 1,
3200
+ runId: run.runId,
3201
+ workflowName: run.workflowName,
3202
+ createdAt: run.createdAt,
3203
+ status: run.status,
3204
+ completedAt: terminal?.completedAt,
3205
+ terminalStatus: terminal?.status,
3206
+ reason,
3207
+ dueAt,
3208
+ queueShards,
3209
+ phase: "retained",
3210
+ generation: 0,
3211
+ attempts: 0,
3212
+ deletedPayloadKeys: 0,
3213
+ deletedStreams: 0,
3214
+ deletedQueueMessages: 0
3215
+ };
3216
+ }
2845
3217
  /**
2846
3218
  * Adapt DO storage (or a transaction) to the {@link EventStore} interface
2847
3219
  * shared with the test mocks. No casts: the methods line up structurally,
@@ -2937,6 +3309,7 @@ var WorkflowRunDO = class extends DurableObject {
2937
3309
  now
2938
3310
  });
2939
3311
  if (!outcome.ok) return outcome;
3312
+ if (outcome.runCreated && request.cleanup) await txn.put(RUN_QUEUE_SHARDS_KEY, request.cleanup.queueShards);
2940
3313
  await txn.put("event_sequence", eventSequence);
2941
3314
  await putStorageEntries(txn, (outcome.hookToIndex ? [{
2942
3315
  hookId: outcome.hookToIndex.hookId,
@@ -2950,22 +3323,7 @@ var WorkflowRunDO = class extends DurableObject {
2950
3323
  ...terminalRun,
2951
3324
  expiredAt: dueAt
2952
3325
  });
2953
- const cleanup = {
2954
- version: 1,
2955
- runId: run.runId,
2956
- workflowName: run.workflowName,
2957
- createdAt: run.createdAt,
2958
- completedAt: run.completedAt,
2959
- terminalStatus: run.status,
2960
- dueAt,
2961
- queueShards: request.cleanup?.queueShards ?? 1,
2962
- phase: "retained",
2963
- generation: 0,
2964
- attempts: 0,
2965
- deletedPayloadKeys: 0,
2966
- deletedStreams: 0,
2967
- deletedQueueMessages: 0
2968
- };
3326
+ const cleanup = cleanupRecord(run, dueAt, request.cleanup?.queueShards ?? 1, "terminal-retention");
2969
3327
  await txn.put("run", run);
2970
3328
  await txn.put(CLEANUP_RECORD_KEY, cleanup);
2971
3329
  await txn.setAlarm(dueAt);
@@ -2981,6 +3339,7 @@ var WorkflowRunDO = class extends DurableObject {
2981
3339
  });
2982
3340
  await txn.setAlarm(this.now() + 1);
2983
3341
  }
3342
+ if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
2984
3343
  return finalizeEventPage(storeFrom(txn), outcome, request.params);
2985
3344
  });
2986
3345
  }
@@ -2991,6 +3350,8 @@ var WorkflowRunDO = class extends DurableObject {
2991
3350
  */
2992
3351
  async resolveHookTokenClaim(request) {
2993
3352
  return await this.ctx.storage.transaction(async (txn) => {
3353
+ const { tombstone } = await this.retentionState(txn);
3354
+ if (tombstone) return { committed: false };
2994
3355
  if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
2995
3356
  await txn.put(hookClaimCancellationKey(request.claimId), {
2996
3357
  hookId: request.hookId,
@@ -3003,6 +3364,20 @@ var WorkflowRunDO = class extends DurableObject {
3003
3364
  async getRun() {
3004
3365
  return this.readKey("run");
3005
3366
  }
3367
+ /** Single authoritative lifecycle answer used after derivative compaction. */
3368
+ async getLifecycleStatus() {
3369
+ return await this.ctx.storage.transaction(async (txn) => {
3370
+ const values = await txn.get([
3371
+ TOMBSTONE_KEY,
3372
+ CLEANUP_RECORD_KEY,
3373
+ "run"
3374
+ ]);
3375
+ if (this.tombstoneFrom(values)) return "expired";
3376
+ const run = values.get("run");
3377
+ if (!run) return "missing";
3378
+ return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
3379
+ });
3380
+ }
3006
3381
  async getStep(stepId) {
3007
3382
  return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
3008
3383
  }
@@ -3031,7 +3406,11 @@ var WorkflowRunDO = class extends DurableObject {
3031
3406
  }));
3032
3407
  }
3033
3408
  async getCleanupStatus() {
3034
- return await this.ctx.storage.get("retention:cleanup") ?? null;
3409
+ const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
3410
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
3411
+ if (cleanup) return cleanup;
3412
+ const tombstone = values.get(TOMBSTONE_KEY);
3413
+ return tombstone ? cleanupFromTombstone(tombstone) : null;
3035
3414
  }
3036
3415
  /** Explicitly schedule an existing terminal run, used for opt-in backfills. */
3037
3416
  async scheduleCleanup(request) {
@@ -3054,28 +3433,95 @@ var WorkflowRunDO = class extends DurableObject {
3054
3433
  ...terminalRun,
3055
3434
  expiredAt: dueAt
3056
3435
  });
3057
- const cleanup = {
3058
- version: 1,
3059
- runId: run.runId,
3060
- workflowName: run.workflowName,
3061
- createdAt: run.createdAt,
3062
- completedAt: run.completedAt,
3063
- terminalStatus: run.status,
3064
- dueAt,
3065
- queueShards: request.queueShards,
3066
- phase: "retained",
3067
- generation: 0,
3068
- attempts: 0,
3069
- deletedPayloadKeys: 0,
3070
- deletedStreams: 0,
3071
- deletedQueueMessages: 0
3072
- };
3436
+ const cleanup = cleanupRecord(run, dueAt, request.queueShards, "manual");
3073
3437
  await txn.put("run", run);
3074
3438
  await txn.put(CLEANUP_RECORD_KEY, cleanup);
3075
3439
  await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
3076
3440
  return cleanup;
3077
3441
  });
3078
3442
  }
3443
+ /**
3444
+ * Apply the fleet-wide maximum-age policy to one authoritative run.
3445
+ *
3446
+ * The catalog is only a discovery index: eligibility is rechecked from the
3447
+ * RunDO's own createdAt. Once eligible, the cleanup deadline immediately
3448
+ * becomes an authoritative write/read fence, including for active runs.
3449
+ */
3450
+ async enforceRetention(request) {
3451
+ if (!Number.isSafeInteger(request.retentionMs) || request.retentionMs <= 0) throw new Error("world-celld retentionMs must be a positive safe integer");
3452
+ if (!Number.isSafeInteger(request.queueShards) || request.queueShards <= 0) throw new Error("world-celld retention queueShards must be a positive safe integer");
3453
+ if (!Number.isSafeInteger(request.scheduledTime) || request.scheduledTime < 0) throw new Error("world-celld retention scheduledTime must be a non-negative safe integer");
3454
+ const result = await this.ctx.storage.transaction(async (txn) => {
3455
+ const values = await txn.get([
3456
+ CLEANUP_RECORD_KEY,
3457
+ TOMBSTONE_KEY,
3458
+ RUN_QUEUE_SHARDS_KEY,
3459
+ "run"
3460
+ ]);
3461
+ const tombstone = values.get(TOMBSTONE_KEY);
3462
+ if (tombstone) return {
3463
+ state: "expired",
3464
+ cleanup: cleanupFromTombstone(tombstone)
3465
+ };
3466
+ const run = values.get("run");
3467
+ if (!run) return {
3468
+ state: "missing",
3469
+ cleanup: null
3470
+ };
3471
+ const dueAtMs = run.createdAt.getTime() + request.retentionMs;
3472
+ if (!Number.isSafeInteger(dueAtMs) || dueAtMs > 864e13) throw new Error(`world-celld retention deadline is out of range for ${run.runId}`);
3473
+ if (dueAtMs > request.scheduledTime) return {
3474
+ state: "not-due",
3475
+ cleanup: null
3476
+ };
3477
+ const dueAt = new Date(dueAtMs);
3478
+ const existing = values.get(CLEANUP_RECORD_KEY);
3479
+ const storedQueueShards = values.get(RUN_QUEUE_SHARDS_KEY);
3480
+ const queueShards = Number.isSafeInteger(storedQueueShards) && storedQueueShards > 0 ? storedQueueShards : request.queueShards;
3481
+ let cleanup = existing;
3482
+ if (!cleanup) {
3483
+ cleanup = cleanupRecord(run, dueAt, queueShards, "maximum-age");
3484
+ await txn.put("run", WorkflowRunSchema.parse({
3485
+ ...run,
3486
+ expiredAt: dueAt
3487
+ }));
3488
+ await txn.put(CLEANUP_RECORD_KEY, cleanup);
3489
+ } else if (cleanup.phase === "retained" && dueAtMs < cleanup.dueAt.getTime()) {
3490
+ const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
3491
+ cleanup = {
3492
+ ...cleanup,
3493
+ status: run.status,
3494
+ completedAt: terminal?.completedAt,
3495
+ terminalStatus: terminal?.status,
3496
+ reason: "maximum-age",
3497
+ dueAt,
3498
+ generation: cleanup.generation + 1
3499
+ };
3500
+ await txn.put("run", WorkflowRunSchema.parse({
3501
+ ...run,
3502
+ expiredAt: dueAt
3503
+ }));
3504
+ await txn.put(CLEANUP_RECORD_KEY, cleanup);
3505
+ }
3506
+ await txn.setAlarm(this.now() + 1);
3507
+ return {
3508
+ state: cleanup.dueAt.getTime() <= this.now() ? "expired" : "scheduled",
3509
+ cleanup
3510
+ };
3511
+ });
3512
+ if (result.state === "scheduled" || result.state === "expired") {
3513
+ for (let page = 0; page < 2; page++) {
3514
+ const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
3515
+ if (cleanup?.phase !== "retained" && cleanup?.phase !== "index") break;
3516
+ await this.executeScheduledCleanupPage();
3517
+ }
3518
+ return {
3519
+ ...result,
3520
+ cleanup: await this.getCleanupStatus() ?? result.cleanup
3521
+ };
3522
+ }
3523
+ return result;
3524
+ }
3079
3525
  async cleanupNow(request) {
3080
3526
  if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
3081
3527
  ...request,
@@ -3119,7 +3565,6 @@ var WorkflowRunDO = class extends DurableObject {
3119
3565
  workflowIndex() {
3120
3566
  return createWorkflowIndex({
3121
3567
  runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
3122
- runFences: this.namespace("WORKFLOW_RUN_FENCES"),
3123
3568
  hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
3124
3569
  hookIds: this.namespace("WORKFLOW_HOOK_IDS")
3125
3570
  });
@@ -3140,7 +3585,17 @@ var WorkflowRunDO = class extends DurableObject {
3140
3585
  });
3141
3586
  }
3142
3587
  async executeScheduledCleanupPage() {
3143
- const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
3588
+ const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
3589
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
3590
+ const terminalCleanup = values.get(TERMINAL_CLEANUP_KEY);
3591
+ if (cleanup && cleanup.phase !== "tombstoned" && cleanup.dueAt.getTime() <= this.now()) {
3592
+ try {
3593
+ await this.executeCleanup(cleanup);
3594
+ } catch (error) {
3595
+ await this.recordCleanupFailure(error, cleanup);
3596
+ }
3597
+ return;
3598
+ }
3144
3599
  if (terminalCleanup) {
3145
3600
  try {
3146
3601
  await this.executeTerminalCleanup(terminalCleanup);
@@ -3149,7 +3604,6 @@ var WorkflowRunDO = class extends DurableObject {
3149
3604
  }
3150
3605
  return;
3151
3606
  }
3152
- const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
3153
3607
  if (!cleanup || cleanup.phase === "tombstoned") {
3154
3608
  await this.ctx.storage.deleteAlarm();
3155
3609
  return;
@@ -3199,8 +3653,7 @@ var WorkflowRunDO = class extends DurableObject {
3199
3653
  hooks: page.map((hook) => ({
3200
3654
  hookId: hook.hookId,
3201
3655
  token: hook.token
3202
- })),
3203
- terminal: true
3656
+ }))
3204
3657
  });
3205
3658
  await this.ctx.storage.transaction(async (txn) => {
3206
3659
  const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3225,8 +3678,7 @@ var WorkflowRunDO = class extends DurableObject {
3225
3678
  const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
3226
3679
  await this.workflowIndex().releaseHookIndexes({
3227
3680
  runId: cleanup.runId,
3228
- hooks: page.map(([, reference]) => reference),
3229
- terminal: true
3681
+ hooks: page.map(([, reference]) => reference)
3230
3682
  });
3231
3683
  await this.ctx.storage.transaction(async (txn) => {
3232
3684
  const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3336,33 +3788,56 @@ var WorkflowRunDO = class extends DurableObject {
3336
3788
  }
3337
3789
  async deleteQueuePage(cleanup) {
3338
3790
  const queues = this.namespace("WORKFLOW_QUEUE");
3339
- let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3791
+ const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3340
3792
  queueShard: 0,
3341
3793
  queueShardDeleted: 0
3342
3794
  };
3343
- let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
3344
- let expectedGeneration = cleanup.generation;
3345
- while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
3795
+ if (progress.pendingAck) {
3796
+ const { queueShard, receipt } = progress.pendingAck;
3797
+ if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
3798
+ await this.ctx.storage.transaction(async (txn) => {
3799
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3800
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3801
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
3802
+ if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
3803
+ await txn.put(CLEANUP_RECORD_KEY, {
3804
+ ...current,
3805
+ generation: current.generation + 1,
3806
+ attempts: 0,
3807
+ lastError: void 0
3808
+ });
3809
+ await txn.put(CLEANUP_PROGRESS_KEY, {
3810
+ queueShard: queueShard + 1,
3811
+ queueShardDeleted: 0
3812
+ });
3813
+ await txn.setAlarm(this.now() + 1);
3814
+ });
3815
+ return;
3816
+ }
3817
+ if (progress.queueShard < cleanup.queueShards) {
3346
3818
  const requestedShard = progress.queueShard;
3347
3819
  const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
3348
- const reconciled = await this.ctx.storage.transaction(async (txn) => {
3820
+ await this.ctx.storage.transaction(async (txn) => {
3349
3821
  const current = await txn.get(CLEANUP_RECORD_KEY);
3350
- if (current?.phase !== "queues") return null;
3351
- if (current.generation !== expectedGeneration) return { stale: true };
3822
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3352
3823
  const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3353
3824
  queueShard: 0,
3354
3825
  queueShardDeleted: 0
3355
3826
  };
3356
- if (persisted.queueShard !== requestedShard) return { stale: true };
3357
- if (result.deleted < persisted.queueShardDeleted) return { stale: true };
3827
+ if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
3828
+ if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
3829
+ const delta = result.deleted - persisted.queueShardDeleted;
3358
3830
  const next = result.done ? {
3359
- queueShard: persisted.queueShard + 1,
3360
- queueShardDeleted: 0
3831
+ queueShard: requestedShard,
3832
+ queueShardDeleted: result.deleted,
3833
+ pendingAck: {
3834
+ queueShard: requestedShard,
3835
+ receipt: result.receipt
3836
+ }
3361
3837
  } : {
3362
- ...persisted,
3838
+ queueShard: requestedShard,
3363
3839
  queueShardDeleted: result.deleted
3364
3840
  };
3365
- const delta = result.deleted - persisted.queueShardDeleted;
3366
3841
  await txn.put(CLEANUP_RECORD_KEY, {
3367
3842
  ...current,
3368
3843
  deletedQueueMessages: current.deletedQueueMessages + delta,
@@ -3372,27 +3847,17 @@ var WorkflowRunDO = class extends DurableObject {
3372
3847
  });
3373
3848
  await txn.put(CLEANUP_PROGRESS_KEY, next);
3374
3849
  await txn.setAlarm(this.now() + 1);
3375
- return {
3376
- stale: false,
3377
- progress: next,
3378
- generation: current.generation + 1,
3379
- applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
3380
- };
3381
3850
  });
3382
- if (!reconciled) return;
3383
- if (reconciled.stale) return;
3384
- progress = reconciled.progress;
3385
- expectedGeneration = reconciled.generation;
3386
- if (!reconciled.applied && progress.queueShard === requestedShard) break;
3387
- if (!result.done && progress.queueShard === requestedShard) break;
3851
+ return;
3388
3852
  }
3389
3853
  await this.ctx.storage.transaction(async (txn) => {
3390
3854
  const current = await txn.get(CLEANUP_RECORD_KEY);
3391
- if (current?.phase !== "queues") return;
3392
- if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3855
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3856
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3393
3857
  queueShard: 0,
3394
3858
  queueShardDeleted: 0
3395
- }).queueShard >= current.queueShards) {
3859
+ };
3860
+ if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
3396
3861
  await txn.put(CLEANUP_RECORD_KEY, {
3397
3862
  ...current,
3398
3863
  phase: "payload",
@@ -3425,14 +3890,16 @@ var WorkflowRunDO = class extends DurableObject {
3425
3890
  await txn.setAlarm(this.now() + 1);
3426
3891
  return false;
3427
3892
  }
3428
- await txn.put(CLEANUP_RECORD_KEY, {
3893
+ const completed = {
3429
3894
  ...cleanup,
3430
3895
  phase: "tombstoned",
3431
3896
  generation: cleanup.generation + 1,
3432
3897
  attempts: 0,
3433
3898
  lastError: void 0,
3434
3899
  tombstonedAt: now
3435
- });
3900
+ };
3901
+ await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
3902
+ await txn.delete(CLEANUP_RECORD_KEY);
3436
3903
  await txn.deleteAlarm();
3437
3904
  return true;
3438
3905
  });
@@ -3492,10 +3959,20 @@ var WorkflowRunDO = class extends DurableObject {
3492
3959
  * default fetch router. This module (and everything it imports) must stay
3493
3960
  * free of Node built-ins — it runs inside celld's workerd runtime.
3494
3961
  */
3495
- var worker_default = { async fetch(request, env) {
3496
- return createRouter(env)(request);
3497
- } };
3962
+ async function scheduled(controller, env) {
3963
+ const result = await runRetentionSweep(controller.scheduledTime, env);
3964
+ if (result.scanned > 0) console.info("world-celld retention sweep", {
3965
+ cron: controller.cron,
3966
+ ...result
3967
+ });
3968
+ }
3969
+ var worker_default = {
3970
+ async fetch(request, env) {
3971
+ return createRouter(env)(request);
3972
+ },
3973
+ scheduled
3974
+ };
3498
3975
  //#endregion
3499
- export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, RunFenceDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
3976
+ export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
3500
3977
 
3501
3978
  //# sourceMappingURL=worker.js.map