@ewhauser/world-celld 0.3.1 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/worker.js CHANGED
@@ -121,9 +121,6 @@ function hookTokenShardName(token) {
121
121
  function hookIdShardName(hookId) {
122
122
  return shardName("hook-id", hookId, 32);
123
123
  }
124
- function runFenceCellName(runId) {
125
- return `run-fence:${INDEX_PROTOCOL_VERSION}:${runId}`;
126
- }
127
124
  function allRunCatalogShardNames() {
128
125
  return Array.from({ length: 16 }, (_, shard) => `run-catalog:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(2, "0")}`);
129
126
  }
@@ -161,8 +158,7 @@ async function inBatches(values, operation) {
161
158
  return deleted;
162
159
  }
163
160
  function createWorkflowIndex(bindings) {
164
- const fence = (runId) => stub(bindings.runFences, runFenceCellName(runId));
165
- const releaseHooks = async (request, fencedAt) => {
161
+ const releaseHooks = async (request) => {
166
162
  const tokenGroups = groupByShard(request.hooks, (hook) => hookTokenShardName(hook.token));
167
163
  const idGroups = groupByShard(request.hooks, (hook) => hookIdShardName(hook.hookId));
168
164
  const releases = [];
@@ -174,7 +170,7 @@ function createWorkflowIndex(bindings) {
174
170
  runId: request.runId,
175
171
  hookId: hook.hookId
176
172
  }
177
- })), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
173
+ })), (batch) => target.releaseBatch(batch)));
178
174
  }
179
175
  for (const [name, hooks] of idGroups) {
180
176
  const target = stub(bindings.hookIds, name);
@@ -184,14 +180,13 @@ function createWorkflowIndex(bindings) {
184
180
  runId: request.runId,
185
181
  hookId: hook.hookId
186
182
  }
187
- })), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
183
+ })), (batch) => target.releaseBatch(batch)));
188
184
  }
189
185
  return (await Promise.all(releases)).reduce((total, count) => total + count, 0);
190
186
  };
191
187
  return {
192
- async commitRun(run, serializedMetadata) {
193
- if (isTerminalWorkflowRunStatus(run.status)) await fence(run.runId).fenceTerminal(Date.now());
194
- return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata);
188
+ async commitRun(run, serializedMetadata, publicationExpiresAt) {
189
+ return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata, publicationExpiresAt);
195
190
  },
196
191
  async listRuns(options) {
197
192
  const suppliedLimit = options?.limit;
@@ -267,16 +262,13 @@ function createWorkflowIndex(bindings) {
267
262
  await Promise.all(releases);
268
263
  },
269
264
  async releaseHookIndexes(request) {
270
- if (request.terminal) await fence(request.runId).fenceTerminal(Date.now());
271
- return { deleted: await releaseHooks(request, Date.now()) };
265
+ return { deleted: await releaseHooks(request) };
272
266
  },
273
267
  async expireRun(request) {
274
- await fence(request.runId).fenceExpired(request.expiredAt);
275
268
  const [catalog, hookDeletes] = await Promise.all([stub(bindings.runCatalog, runCatalogShardName(request.runId)).expireRun(request.runId, request.keys, request.expiredAt), releaseHooks({
276
269
  runId: request.runId,
277
- hooks: request.hooks,
278
- terminal: true
279
- }, request.expiredAt)]);
270
+ hooks: request.hooks
271
+ })]);
280
272
  return { deleted: catalog.deleted + hookDeletes };
281
273
  }
282
274
  };
@@ -493,7 +485,6 @@ const WORLD_VERSION = "0.1.0";
493
485
  function workflowIndex(env) {
494
486
  return createWorkflowIndex({
495
487
  runCatalog: env.WORKFLOW_RUN_CATALOG,
496
- runFences: env.WORKFLOW_RUN_FENCES,
497
488
  hookTokens: env.WORKFLOW_HOOK_TOKENS,
498
489
  hookIds: env.WORKFLOW_HOOK_IDS
499
490
  });
@@ -503,6 +494,7 @@ const BINDINGS = {
503
494
  env: "WORKFLOW_DB",
504
495
  methods: /* @__PURE__ */ new Set([
505
496
  "applyEvent",
497
+ "getLifecycleStatus",
506
498
  "getRun",
507
499
  "getStep",
508
500
  "getEvent",
@@ -545,7 +537,8 @@ const BINDINGS = {
545
537
  "redriveDeadLetter",
546
538
  "purgeDeadLetters",
547
539
  "rearmAlarm",
548
- "expireRun"
540
+ "expireRun",
541
+ "acknowledgeExpireRun"
549
542
  ])
550
543
  }
551
544
  };
@@ -654,7 +647,7 @@ function createRouter(env) {
654
647
  let result;
655
648
  const operation = `${parts[2]}.${parts[3]}`;
656
649
  if (operation === "runs.list") result = await index.listRuns(args[0]);
657
- else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
650
+ else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
658
651
  else if (operation === "runs.expire") result = await index.expireRun(args[0]);
659
652
  else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
660
653
  else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
@@ -765,26 +758,52 @@ function ownerFromSerializedHook(raw) {
765
758
  function indexKey(prefix, value) {
766
759
  return `${prefix}:${encodeURIComponent(value)}`;
767
760
  }
768
- /** Compact shard-local fence for every hook owned by one terminal or expired run. */
769
- function shardRunFenceKey(runId) {
770
- return `runfence:${encodeURIComponent(runId)}`;
771
- }
772
761
  async function runIsActive(env, runId) {
773
- const namespace = env.WORKFLOW_RUN_FENCES;
774
- if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
775
- return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
762
+ const namespace = env.WORKFLOW_DB;
763
+ if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
764
+ return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
776
765
  }
777
766
  //#endregion
767
+ //#region src/lifecycle.ts
768
+ /**
769
+ * Hard protocol bounds used by lifecycle leases and derivative-fence cleanup.
770
+ *
771
+ * These are not retention TTLs. They bound how long a request which was
772
+ * admitted before expiry may still mutate a derivative Durable Object:
773
+ *
774
+ * - fleet calls have at most three 5-minute attempts plus bounded retry delay;
775
+ * - one queue delivery owns its lease for at most 5 minutes plus 30 seconds;
776
+ * - hook admission spans reserve retries, one authoritative apply, and
777
+ * finalize retries before the exact claim must be reacquired.
778
+ */
779
+ const MAX_FLEET_RPC_TIMEOUT_MS = 3e5;
780
+ const MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS = 1200900;
781
+ const MAX_QUEUE_DELIVERY_TIMEOUT_MS = 3e5;
782
+ const QUEUE_INFLIGHT_GRACE_MS = 3e4;
783
+ const QUEUE_FENCE_GRACE_MS = Math.max(MAX_FLEET_RPC_TIMEOUT_MS, 33e4);
784
+ const HOOK_CLAIM_LEASE_MS = 2101800;
785
+ const LIFECYCLE_COMPACTION_RETRY_MS = 1e3;
786
+ //#endregion
778
787
  //#region src/worker/durable-objects/HookIdDO.ts
779
788
  const MAX_RELEASE_BATCH$1 = 64;
789
+ const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
790
+ function pad$3(ms) {
791
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
792
+ }
780
793
  function recordKey$1(hookId) {
781
794
  return indexKey("hookid", hookId);
782
795
  }
783
796
  function claimKey$1(hookId) {
784
797
  return indexKey("claim", hookId);
785
798
  }
799
+ function claimDeadlineKey$1(hookId, expiresAt) {
800
+ return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
801
+ }
786
802
  /** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
787
803
  var HookIdDO = class extends DurableObject {
804
+ now() {
805
+ return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
806
+ }
788
807
  async get(hookId) {
789
808
  const raw = await this.ctx.storage.get(recordKey$1(hookId));
790
809
  if (!raw) return null;
@@ -801,17 +820,33 @@ var HookIdDO = class extends DurableObject {
801
820
  claimed: false,
802
821
  holder: ownerFromSerializedHook(indexed)
803
822
  };
804
- const existing = values.get(claim);
823
+ let existing = values.get(claim);
824
+ const now = this.now();
825
+ if (existing && existing.expiresAt <= now) {
826
+ await txn.delete([claim, existing.deadlineKey]);
827
+ existing = void 0;
828
+ }
805
829
  if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
806
830
  claimed: false,
807
831
  holder: existing.owner
808
832
  };
809
833
  if (existing === void 0 || existing.claimId !== proposedClaimId) {
834
+ if (existing) await txn.delete(existing.deadlineKey);
835
+ const expiresAt = now + HOOK_CLAIM_LEASE_MS;
836
+ const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
810
837
  await txn.put(claim, {
811
838
  owner,
812
839
  claimId: proposedClaimId,
813
- reservedAt: Date.now()
840
+ reservedAt: now,
841
+ expiresAt,
842
+ deadlineKey
814
843
  });
844
+ await txn.put(deadlineKey, {
845
+ hookId,
846
+ claimId: proposedClaimId,
847
+ expiresAt
848
+ });
849
+ await this.armAtMost(txn, expiresAt);
815
850
  return {
816
851
  claimed: true,
817
852
  claimId: proposedClaimId
@@ -828,30 +863,24 @@ var HookIdDO = class extends DurableObject {
828
863
  await this.releaseBatch([{
829
864
  hookId,
830
865
  owner
831
- }], Date.now(), true);
866
+ }]);
832
867
  return { stored: false };
833
868
  }
834
869
  return await this.ctx.storage.transaction(async (txn) => {
835
870
  const record = recordKey$1(hookId);
836
871
  const claim = claimKey$1(hookId);
837
- const runFence = shardRunFenceKey(owner.runId);
838
- const values = await txn.get([
839
- record,
840
- claim,
841
- runFence
842
- ]);
843
- if (values.has(runFence)) {
844
- const existingClaim = values.get(claim);
845
- if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
846
- return { stored: false };
847
- }
872
+ const values = await txn.get([record, claim]);
848
873
  const indexed = values.get(record);
849
874
  if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
850
875
  const existingClaim = values.get(claim);
876
+ if (existingClaim && existingClaim.expiresAt <= this.now()) {
877
+ await txn.delete([claim, existingClaim.deadlineKey]);
878
+ return { stored: false };
879
+ }
851
880
  if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
852
881
  if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
853
882
  await txn.put(record, serializedHook);
854
- if (existingClaim !== void 0) await txn.delete(claim);
883
+ if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
855
884
  return { stored: true };
856
885
  });
857
886
  }
@@ -859,40 +888,94 @@ var HookIdDO = class extends DurableObject {
859
888
  await this.ctx.storage.transaction(async (txn) => {
860
889
  const key = claimKey$1(hookId);
861
890
  const claim = await txn.get(key);
862
- if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
891
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
863
892
  });
864
893
  }
865
- async releaseBatch(hooks, fencedAt, runFenced) {
894
+ async releaseBatch(hooks) {
866
895
  if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
867
896
  if (hooks.length === 0) return { deleted: 0 };
868
897
  return await this.ctx.storage.transaction(async (txn) => {
869
898
  const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
870
899
  const values = await txn.get(keys);
871
900
  const deletes = [];
901
+ const deadlineDeletes = [];
872
902
  for (const { hookId, owner } of hooks) {
873
903
  const record = recordKey$1(hookId);
874
904
  const claim = claimKey$1(hookId);
875
905
  const indexed = values.get(record);
876
906
  const existingClaim = values.get(claim);
877
907
  if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
878
- if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
908
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
909
+ deletes.push(claim);
910
+ deadlineDeletes.push(existingClaim.deadlineKey);
911
+ }
879
912
  }
880
- if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
881
- return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
913
+ const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
914
+ if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
915
+ return { deleted };
916
+ });
917
+ }
918
+ async alarm() {
919
+ try {
920
+ await this.compactClaims();
921
+ } catch {
922
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
923
+ }
924
+ }
925
+ async armAtMost(storage, at) {
926
+ const current = await storage.getAlarm();
927
+ if (current === null || at < current) await storage.setAlarm(at);
928
+ }
929
+ async compactClaims() {
930
+ const now = this.now();
931
+ await this.ctx.storage.transaction(async (txn) => {
932
+ const due = await txn.list({
933
+ prefix: CLAIM_DEADLINE_PREFIX$1,
934
+ end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
935
+ limit: 129
936
+ });
937
+ const page = Array.from(due).slice(0, 128);
938
+ const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
939
+ const deletes = page.flatMap(([deadline, item]) => {
940
+ const key = claimKey$1(item.hookId);
941
+ const claim = claims.get(key);
942
+ return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
943
+ });
944
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
945
+ if (due.size > 128) {
946
+ await txn.setAlarm(now + 1);
947
+ return;
948
+ }
949
+ const nextAt = (await txn.list({
950
+ prefix: CLAIM_DEADLINE_PREFIX$1,
951
+ limit: 1
952
+ })).values().next().value?.expiresAt;
953
+ if (nextAt === void 0) await txn.deleteAlarm();
954
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
882
955
  });
883
956
  }
884
957
  };
885
958
  //#endregion
886
959
  //#region src/worker/durable-objects/HookTokenDO.ts
887
960
  const MAX_RELEASE_BATCH = 64;
961
+ const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
962
+ function pad$2(ms) {
963
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
964
+ }
888
965
  function recordKey(token) {
889
966
  return indexKey("hook", token);
890
967
  }
891
968
  function claimKey(token) {
892
969
  return indexKey("claim", token);
893
970
  }
971
+ function claimDeadlineKey(token, expiresAt) {
972
+ return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
973
+ }
894
974
  /** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
895
975
  var HookTokenDO = class extends DurableObject {
976
+ now() {
977
+ return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
978
+ }
896
979
  async get(token) {
897
980
  const raw = await this.ctx.storage.get(recordKey(token));
898
981
  if (!raw) return null;
@@ -909,17 +992,32 @@ var HookTokenDO = class extends DurableObject {
909
992
  claimed: false,
910
993
  holder: ownerFromSerializedHook(indexed)
911
994
  };
912
- const existing = values.get(claim);
995
+ let existing = values.get(claim);
996
+ const now = this.now();
997
+ if (existing && existing.expiresAt <= now) {
998
+ await txn.delete([claim, existing.deadlineKey]);
999
+ existing = void 0;
1000
+ }
913
1001
  if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
914
1002
  claimed: false,
915
1003
  holder: existing.owner
916
1004
  };
917
1005
  if (existing === void 0) {
1006
+ const expiresAt = now + HOOK_CLAIM_LEASE_MS;
1007
+ const deadlineKey = claimDeadlineKey(token, expiresAt);
918
1008
  await txn.put(claim, {
919
1009
  owner,
920
1010
  claimId: proposedClaimId,
921
- reservedAt: Date.now()
1011
+ reservedAt: now,
1012
+ expiresAt,
1013
+ deadlineKey
922
1014
  });
1015
+ await txn.put(deadlineKey, {
1016
+ token,
1017
+ claimId: proposedClaimId,
1018
+ expiresAt
1019
+ });
1020
+ await this.armAtMost(txn, expiresAt);
923
1021
  return {
924
1022
  claimed: true,
925
1023
  claimId: proposedClaimId
@@ -936,30 +1034,24 @@ var HookTokenDO = class extends DurableObject {
936
1034
  await this.releaseBatch([{
937
1035
  token,
938
1036
  owner
939
- }], Date.now(), true);
1037
+ }]);
940
1038
  return { stored: false };
941
1039
  }
942
1040
  return await this.ctx.storage.transaction(async (txn) => {
943
1041
  const record = recordKey(token);
944
1042
  const claim = claimKey(token);
945
- const runFence = shardRunFenceKey(owner.runId);
946
- const values = await txn.get([
947
- record,
948
- claim,
949
- runFence
950
- ]);
951
- if (values.has(runFence)) {
952
- const existingClaim = values.get(claim);
953
- if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
954
- return { stored: false };
955
- }
1043
+ const values = await txn.get([record, claim]);
956
1044
  const indexed = values.get(record);
957
1045
  if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
958
1046
  const existingClaim = values.get(claim);
1047
+ if (existingClaim && existingClaim.expiresAt <= this.now()) {
1048
+ await txn.delete([claim, existingClaim.deadlineKey]);
1049
+ return { stored: false };
1050
+ }
959
1051
  if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
960
1052
  if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
961
1053
  await txn.put(record, serializedHook);
962
- if (existingClaim !== void 0) await txn.delete(claim);
1054
+ if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
963
1055
  return { stored: true };
964
1056
  });
965
1057
  }
@@ -967,26 +1059,70 @@ var HookTokenDO = class extends DurableObject {
967
1059
  await this.ctx.storage.transaction(async (txn) => {
968
1060
  const key = claimKey(token);
969
1061
  const claim = await txn.get(key);
970
- if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
1062
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
971
1063
  });
972
1064
  }
973
- async releaseBatch(hooks, fencedAt, runFenced) {
1065
+ async releaseBatch(hooks) {
974
1066
  if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
975
1067
  if (hooks.length === 0) return { deleted: 0 };
976
1068
  return await this.ctx.storage.transaction(async (txn) => {
977
1069
  const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
978
1070
  const values = await txn.get(keys);
979
1071
  const deletes = [];
1072
+ const deadlineDeletes = [];
980
1073
  for (const { token, owner } of hooks) {
981
1074
  const record = recordKey(token);
982
1075
  const claim = claimKey(token);
983
1076
  const indexed = values.get(record);
984
1077
  const existingClaim = values.get(claim);
985
1078
  if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
986
- if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
1079
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
1080
+ deletes.push(claim);
1081
+ deadlineDeletes.push(existingClaim.deadlineKey);
1082
+ }
987
1083
  }
988
- if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
989
- return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
1084
+ const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
1085
+ if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
1086
+ return { deleted };
1087
+ });
1088
+ }
1089
+ async alarm() {
1090
+ try {
1091
+ await this.compactClaims();
1092
+ } catch {
1093
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
1094
+ }
1095
+ }
1096
+ async armAtMost(storage, at) {
1097
+ const current = await storage.getAlarm();
1098
+ if (current === null || at < current) await storage.setAlarm(at);
1099
+ }
1100
+ async compactClaims() {
1101
+ const now = this.now();
1102
+ await this.ctx.storage.transaction(async (txn) => {
1103
+ const due = await txn.list({
1104
+ prefix: CLAIM_DEADLINE_PREFIX,
1105
+ end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
1106
+ limit: 129
1107
+ });
1108
+ const page = Array.from(due).slice(0, 128);
1109
+ const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
1110
+ const deletes = page.flatMap(([deadline, item]) => {
1111
+ const key = claimKey(item.token);
1112
+ const claim = claims.get(key);
1113
+ return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
1114
+ });
1115
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
1116
+ if (due.size > 128) {
1117
+ await txn.setAlarm(now + 1);
1118
+ return;
1119
+ }
1120
+ const nextAt = (await txn.list({
1121
+ prefix: CLAIM_DEADLINE_PREFIX,
1122
+ limit: 1
1123
+ })).values().next().value?.expiresAt;
1124
+ if (nextAt === void 0) await txn.deleteAlarm();
1125
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
990
1126
  });
991
1127
  }
992
1128
  };
@@ -1001,8 +1137,6 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
1001
1137
  const DEFAULT_MAX_ATTEMPTS = 5;
1002
1138
  const DEFAULT_MAX_INFLIGHT = 5;
1003
1139
  const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
1004
- /** Grace period past the delivery timeout before a claim counts as lost. */
1005
- const INFLIGHT_GRACE_MS = 3e4;
1006
1140
  /** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
1007
1141
  const MIN_ALARM_DELAY_MS = 1;
1008
1142
  /** Bound expiry recovery so one alarm cannot monopolize the cell. */
@@ -1011,14 +1145,15 @@ const STORAGE_BATCH_SIZE$2 = 128;
1011
1145
  const EXPIRE_RUN_REFERENCE_BATCH = 64;
1012
1146
  const PURGE_DEAD_LETTER_BATCH = 128;
1013
1147
  const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
1014
- function pad(ms) {
1148
+ const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
1149
+ function pad$1(ms) {
1015
1150
  return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
1016
1151
  }
1017
1152
  function dueKey(atMs, messageId) {
1018
- return `due:${pad(atMs)}:${messageId}`;
1153
+ return `due:${pad$1(atMs)}:${messageId}`;
1019
1154
  }
1020
1155
  function inflightKey(deadlineMs, messageId) {
1021
- return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
1156
+ return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
1022
1157
  }
1023
1158
  function deadlineFromInflightKey(key) {
1024
1159
  return Number.parseInt(key.slice(18, 31), 10);
@@ -1029,6 +1164,9 @@ function runReferenceKey(runId, messageId) {
1029
1164
  function expiredRunKey(runId) {
1030
1165
  return `expired-run:${runId}`;
1031
1166
  }
1167
+ function expiredRunGcKey(runId, compactAt) {
1168
+ return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
1169
+ }
1032
1170
  function boundedLimit$1(value, fallback, maximum) {
1033
1171
  if (value === void 0 || !Number.isFinite(value)) return fallback;
1034
1172
  return Math.max(1, Math.min(maximum, Math.floor(value)));
@@ -1079,16 +1217,29 @@ var QueueDO = class extends DurableObject {
1079
1217
  if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
1080
1218
  return configured;
1081
1219
  }
1220
+ #deliveryTimeoutMs() {
1221
+ const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1222
+ if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
1223
+ return configured;
1224
+ }
1225
+ async #authoritativeRunExpired(runId) {
1226
+ const namespace = this.env?.WORKFLOW_DB;
1227
+ if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
1228
+ return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
1229
+ }
1230
+ async #withRunMutation(runId, onExpired, onAllowed) {
1231
+ const expired = await this.#authoritativeRunExpired(runId);
1232
+ return await this.ctx.storage.transaction(async (txn) => {
1233
+ if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
1234
+ return await onAllowed(txn);
1235
+ });
1236
+ }
1082
1237
  async enqueue(request) {
1083
1238
  this.#maxInflight();
1239
+ this.#deliveryTimeoutMs();
1084
1240
  const storage = this.ctx.storage;
1085
1241
  const now = this.#now();
1086
- return await storage.transaction(async (txn) => {
1087
- if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
1088
- ok: false,
1089
- code: "RUN_EXPIRED",
1090
- message: `Workflow run "${request.runId}" has expired`
1091
- };
1242
+ const persist = async (txn) => {
1092
1243
  const pinned = await txn.get("cfg");
1093
1244
  if (!pinned) await txn.put("cfg", {
1094
1245
  queueShards: request.config.queueShards,
@@ -1140,7 +1291,12 @@ var QueueDO = class extends DurableObject {
1140
1291
  messageId: row.messageId,
1141
1292
  deduped: false
1142
1293
  };
1143
- });
1294
+ };
1295
+ return request.runId ? await this.#withRunMutation(request.runId, async () => ({
1296
+ ok: false,
1297
+ code: "RUN_EXPIRED",
1298
+ message: `Workflow run "${request.runId}" has expired`
1299
+ }), persist) : await storage.transaction(persist);
1144
1300
  }
1145
1301
  async alarm() {
1146
1302
  let claimed = [];
@@ -1154,7 +1310,10 @@ var QueueDO = class extends DurableObject {
1154
1310
  };
1155
1311
  if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
1156
1312
  else await attempt();
1157
- if (failure !== null) throw failure;
1313
+ if (failure !== null) {
1314
+ await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
1315
+ throw failure;
1316
+ }
1158
1317
  for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
1159
1318
  }
1160
1319
  /**
@@ -1166,9 +1325,13 @@ var QueueDO = class extends DurableObject {
1166
1325
  const now = this.#now();
1167
1326
  const maxInflight = this.#maxInflight();
1168
1327
  return await storage.transaction(async (txn) => {
1328
+ if (await this.#compactExpiredRunFences(txn, now)) {
1329
+ await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
1330
+ return [];
1331
+ }
1169
1332
  const expiredPage = await txn.list({
1170
1333
  prefix: INFLIGHT_DEADLINE_PREFIX,
1171
- end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
1334
+ end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
1172
1335
  limit: 129
1173
1336
  });
1174
1337
  const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
@@ -1200,17 +1363,17 @@ var QueueDO = class extends DurableObject {
1200
1363
  if (inflightCount < maxInflight) {
1201
1364
  const due = await txn.list({
1202
1365
  prefix: "due:",
1203
- end: `due:${pad(now + 1)}`,
1366
+ end: `due:${pad$1(now + 1)}`,
1204
1367
  limit: maxInflight - inflightCount
1205
1368
  });
1206
- const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1369
+ const timeoutMs = this.#deliveryTimeoutMs();
1207
1370
  const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
1208
1371
  await deleteMany$1(txn, Array.from(due.keys()));
1209
1372
  const claimWrites = [];
1210
1373
  for (const [, messageId] of due) {
1211
1374
  const row = dueRows.get(`msg:${messageId}`);
1212
1375
  if (!row) continue;
1213
- const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
1376
+ const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
1214
1377
  claimWrites.push([claimKey, messageId]);
1215
1378
  if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
1216
1379
  messageId: row.messageId,
@@ -1230,9 +1393,13 @@ var QueueDO = class extends DurableObject {
1230
1393
  }
1231
1394
  async #deliver({ row, inflightKey: claimKey }) {
1232
1395
  const storage = this.ctx.storage;
1233
- const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1396
+ const timeoutMs = this.#deliveryTimeoutMs();
1234
1397
  const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
1235
1398
  const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
1399
+ if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
1400
+ await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
1401
+ return;
1402
+ }
1236
1403
  let response = null;
1237
1404
  let transportError = null;
1238
1405
  try {
@@ -1262,28 +1429,21 @@ var QueueDO = class extends DurableObject {
1262
1429
  } catch {
1263
1430
  timeoutSeconds = void 0;
1264
1431
  }
1265
- if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
1266
- if (!await txn.delete(claimKey)) return;
1267
- if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
1268
- await txn.delete(`msg:${row.messageId}`);
1269
- await txn.delete(runReferenceKey(row.runId, row.messageId));
1270
- if (row.idempotencyKey) {
1271
- const dedupKey = `key:${row.idempotencyKey}`;
1272
- if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1273
- }
1432
+ if (typeof timeoutSeconds === "number") {
1433
+ const reschedule = async (txn) => {
1434
+ if (!await txn.delete(claimKey)) return;
1435
+ const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
1436
+ await txn.put(scheduleKey, row.messageId);
1437
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1438
+ messageId: row.messageId,
1439
+ dueKey: scheduleKey,
1440
+ idempotencyKey: row.idempotencyKey
1441
+ });
1274
1442
  await this.#scheduleNextAlarm(txn, now);
1275
- return;
1276
- }
1277
- const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
1278
- await txn.put(scheduleKey, row.messageId);
1279
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1280
- messageId: row.messageId,
1281
- dueKey: scheduleKey,
1282
- idempotencyKey: row.idempotencyKey
1283
- });
1284
- await this.#scheduleNextAlarm(txn, now);
1285
- });
1286
- else await this.#retry(row, claimKey, "HTTP 503");
1443
+ };
1444
+ if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
1445
+ else await storage.transaction(reschedule);
1446
+ } else await this.#retry(row, claimKey, "HTTP 503");
1287
1447
  } else if (response && PERMANENT_STATUSES.has(response.status)) {
1288
1448
  await cancelUnusedResponseBody(response);
1289
1449
  await this.#ack(row, claimKey);
@@ -1304,23 +1464,23 @@ var QueueDO = class extends DurableObject {
1304
1464
  await this.#scheduleNextAlarm(txn, this.#now());
1305
1465
  });
1306
1466
  }
1467
+ async #dropExpiredMessage(txn, row, claimKey, now) {
1468
+ if (!await txn.delete(claimKey)) return;
1469
+ await txn.delete(`msg:${row.messageId}`);
1470
+ if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
1471
+ if (row.idempotencyKey) {
1472
+ const dedupKey = `key:${row.idempotencyKey}`;
1473
+ if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1474
+ }
1475
+ await this.#scheduleNextAlarm(txn, now);
1476
+ }
1307
1477
  async #retry(row, claimKey, reason) {
1308
1478
  const storage = this.ctx.storage;
1309
1479
  const now = this.#now();
1310
1480
  const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
1311
1481
  const attempt = row.attempt + 1;
1312
- await storage.transaction(async (txn) => {
1482
+ const persistRetry = async (txn) => {
1313
1483
  if (!await txn.delete(claimKey)) return;
1314
- if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
1315
- await txn.delete(`msg:${row.messageId}`);
1316
- await txn.delete(runReferenceKey(row.runId, row.messageId));
1317
- if (row.idempotencyKey) {
1318
- const dedupKey = `key:${row.idempotencyKey}`;
1319
- if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1320
- }
1321
- await this.#scheduleNextAlarm(txn, now);
1322
- return;
1323
- }
1324
1484
  if (attempt >= maxAttempts) {
1325
1485
  const dead = {
1326
1486
  ...row,
@@ -1328,7 +1488,7 @@ var QueueDO = class extends DurableObject {
1328
1488
  lastError: reason,
1329
1489
  failedAt: now
1330
1490
  };
1331
- const deadKey = `dlq:${pad(now)}:${row.messageId}`;
1491
+ const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
1332
1492
  await txn.put(deadKey, dead);
1333
1493
  await txn.delete(`msg:${row.messageId}`);
1334
1494
  if (row.idempotencyKey) {
@@ -1356,7 +1516,32 @@ var QueueDO = class extends DurableObject {
1356
1516
  idempotencyKey: row.idempotencyKey
1357
1517
  });
1358
1518
  await this.#scheduleNextAlarm(txn, now);
1519
+ };
1520
+ if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
1521
+ else await storage.transaction(persistRetry);
1522
+ }
1523
+ /** Return true when another due compaction page must run immediately. */
1524
+ async #compactExpiredRunFences(txn, now) {
1525
+ const due = await txn.list({
1526
+ prefix: EXPIRED_RUN_GC_PREFIX,
1527
+ end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
1528
+ limit: 129
1359
1529
  });
1530
+ const page = Array.from(due).slice(0, 128);
1531
+ if (page.length === 0) return false;
1532
+ const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
1533
+ const markers = await getMany$1(txn, markerKeys);
1534
+ const deletes = [];
1535
+ for (let index = 0; index < page.length; index++) {
1536
+ const [gcKey, item] = page[index];
1537
+ const markerKey = markerKeys[index];
1538
+ const marker = markers.get(markerKey);
1539
+ deletes.push(gcKey);
1540
+ if (marker?.compactAt !== item.compactAt) continue;
1541
+ deletes.push(markerKey);
1542
+ }
1543
+ await deleteMany$1(txn, deletes);
1544
+ return due.size > 128;
1360
1545
  }
1361
1546
  /** Arm the alarm no later than `atMs` (durable backoff never throws). */
1362
1547
  async #armAlarmAtMost(storage, atMs, now) {
@@ -1388,6 +1573,11 @@ var QueueDO = class extends DurableObject {
1388
1573
  const deadline = deadlineFromInflightKey(key);
1389
1574
  if (next === null || deadline < next) next = deadline;
1390
1575
  }
1576
+ const expiryGc = await storage.list({
1577
+ prefix: EXPIRED_RUN_GC_PREFIX,
1578
+ limit: 1
1579
+ });
1580
+ for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
1391
1581
  if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
1392
1582
  else await storage.deleteAlarm();
1393
1583
  }
@@ -1426,37 +1616,42 @@ var QueueDO = class extends DurableObject {
1426
1616
  async redriveDeadLetter(messageId) {
1427
1617
  const storage = this.ctx.storage;
1428
1618
  const now = this.#now();
1429
- return await storage.transaction(async (txn) => {
1430
- const dlq = await txn.list({ prefix: "dlq:" });
1431
- for (const [key, dead] of dlq) {
1432
- if (dead.messageId !== messageId) continue;
1433
- const { failedAt: _failedAt, ...rest } = dead;
1434
- const row = {
1435
- ...rest,
1436
- attempt: 0,
1437
- lastError: void 0
1438
- };
1439
- if (row.idempotencyKey) {
1440
- const dedupKey = `key:${row.idempotencyKey}`;
1441
- const holder = await txn.get(dedupKey);
1442
- if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
1443
- if (holder && holder !== row.messageId) await txn.delete(dedupKey);
1444
- await txn.put(dedupKey, row.messageId);
1445
- }
1446
- await txn.put(`msg:${row.messageId}`, row);
1447
- const scheduleKey = dueKey(now, row.messageId);
1448
- await txn.put(scheduleKey, row.messageId);
1449
- await txn.delete(key);
1450
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1451
- messageId: row.messageId,
1452
- dueKey: scheduleKey,
1453
- idempotencyKey: row.idempotencyKey
1454
- });
1455
- await this.#armAlarmAtMost(txn, now, now);
1456
- return { ok: true };
1619
+ const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
1620
+ if (!found) return { ok: false };
1621
+ const [deadKey, candidate] = found;
1622
+ const redrive = async (txn) => {
1623
+ const dead = await txn.get(deadKey);
1624
+ if (!dead || dead.messageId !== messageId) return { ok: false };
1625
+ const { failedAt: _failedAt, ...rest } = dead;
1626
+ const row = {
1627
+ ...rest,
1628
+ attempt: 0,
1629
+ lastError: void 0
1630
+ };
1631
+ if (row.idempotencyKey) {
1632
+ const dedupKey = `key:${row.idempotencyKey}`;
1633
+ const holder = await txn.get(dedupKey);
1634
+ if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
1635
+ if (holder && holder !== row.messageId) await txn.delete(dedupKey);
1636
+ await txn.put(dedupKey, row.messageId);
1457
1637
  }
1638
+ await txn.put(`msg:${row.messageId}`, row);
1639
+ const scheduleKey = dueKey(now, row.messageId);
1640
+ await txn.put(scheduleKey, row.messageId);
1641
+ await txn.delete(deadKey);
1642
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1643
+ messageId: row.messageId,
1644
+ dueKey: scheduleKey,
1645
+ idempotencyKey: row.idempotencyKey
1646
+ });
1647
+ await this.#armAlarmAtMost(txn, now, now);
1648
+ return { ok: true };
1649
+ };
1650
+ const rejectExpired = async (txn) => {
1651
+ if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
1458
1652
  return { ok: false };
1459
- });
1653
+ };
1654
+ return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
1460
1655
  }
1461
1656
  async purgeDeadLetters() {
1462
1657
  return await this.ctx.storage.transaction(async (txn) => {
@@ -1506,27 +1701,75 @@ var QueueDO = class extends DurableObject {
1506
1701
  count++;
1507
1702
  }
1508
1703
  await deleteMany$1(txn, keys);
1704
+ const done = references.size <= limit;
1705
+ const markerExpiredAt = existing?.expiredAt ?? expiredAt;
1509
1706
  await txn.put(expiredRunKey(runId), {
1510
- expiredAt,
1511
- deleted: count
1707
+ expiredAt: markerExpiredAt,
1708
+ deleted: count,
1709
+ compactAt: existing?.compactAt ?? null
1512
1710
  });
1513
1711
  await this.#scheduleNextAlarm(txn, this.#now());
1514
- return {
1712
+ return done ? {
1515
1713
  deleted: count,
1516
- done: references.size <= limit
1714
+ done: true,
1715
+ receipt: {
1716
+ expiredAt: markerExpiredAt,
1717
+ deleted: count
1718
+ }
1719
+ } : {
1720
+ deleted: count,
1721
+ done: false
1517
1722
  };
1518
1723
  });
1519
1724
  }
1725
+ /** Acknowledge a final cleanup receipt before making its marker compactable. */
1726
+ async acknowledgeExpireRun(runId, receipt) {
1727
+ return await this.ctx.storage.transaction(async (txn) => {
1728
+ const key = expiredRunKey(runId);
1729
+ const marker = await txn.get(key);
1730
+ if (!marker) return { acknowledged: true };
1731
+ if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
1732
+ if (marker.compactAt === null) {
1733
+ const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
1734
+ await txn.put(key, {
1735
+ ...marker,
1736
+ compactAt
1737
+ });
1738
+ await txn.put(expiredRunGcKey(runId, compactAt), {
1739
+ runId,
1740
+ compactAt
1741
+ });
1742
+ }
1743
+ await this.#scheduleNextAlarm(txn, this.#now());
1744
+ return { acknowledged: true };
1745
+ });
1746
+ }
1520
1747
  };
1521
1748
  //#endregion
1522
1749
  //#region src/worker/durable-objects/RunCatalogDO.ts
1523
1750
  const MAX_INDEX_LIST_SIZE = 1001;
1751
+ const EXPIRY_GC_PREFIX = "expiry-gc:";
1752
+ function pad(ms) {
1753
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
1754
+ }
1755
+ function expiredKey(runId) {
1756
+ return `expired:${runId}`;
1757
+ }
1758
+ function expiryGcKey(runId, compactAt) {
1759
+ return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
1760
+ }
1524
1761
  /** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
1525
1762
  var RunCatalogDO = class extends DurableObject {
1526
- async upsertRun(runId, keys, serializedMetadata) {
1763
+ now() {
1764
+ const clock = this.env?.clock;
1765
+ return typeof clock === "function" ? clock() : Date.now();
1766
+ }
1767
+ async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
1527
1768
  if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
1528
1769
  return await this.ctx.storage.transaction(async (txn) => {
1529
- if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
1770
+ if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
1771
+ const now = this.now();
1772
+ if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
1530
1773
  await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
1531
1774
  return { stored: true };
1532
1775
  });
@@ -1556,38 +1799,56 @@ var RunCatalogDO = class extends DurableObject {
1556
1799
  async expireRun(runId, keys, expiredAt) {
1557
1800
  if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
1558
1801
  return await this.ctx.storage.transaction(async (txn) => {
1559
- await txn.put(`expired:${runId}`, expiredAt);
1802
+ const marker = expiredKey(runId);
1803
+ const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
1804
+ await txn.put(marker, {
1805
+ expiredAt,
1806
+ compactAt
1807
+ });
1808
+ await txn.put(expiryGcKey(runId, compactAt), {
1809
+ runId,
1810
+ compactAt
1811
+ });
1812
+ await this.armAtMost(txn, compactAt);
1560
1813
  return { deleted: await txn.delete(keys) };
1561
1814
  });
1562
1815
  }
1563
- };
1564
- //#endregion
1565
- //#region src/worker/durable-objects/RunFenceDO.ts
1566
- /** Per-run lifecycle fence used by hook shards without a global lookup cell. */
1567
- var RunFenceDO = class extends DurableObject {
1568
- async getStatus() {
1569
- return (await this.ctx.storage.get("fence"))?.status ?? "active";
1816
+ async alarm() {
1817
+ try {
1818
+ await this.compactExpiryFences();
1819
+ } catch {
1820
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
1821
+ }
1570
1822
  }
1571
- async fenceTerminal(at) {
1572
- return await this.ctx.storage.transaction(async (txn) => {
1573
- const current = await txn.get("fence");
1574
- if (current) return current.status;
1575
- await txn.put("fence", {
1576
- status: "terminal",
1577
- at
1578
- });
1579
- return "terminal";
1580
- });
1823
+ async armAtMost(storage, at) {
1824
+ const current = await storage.getAlarm();
1825
+ if (current === null || at < current) await storage.setAlarm(at);
1581
1826
  }
1582
- async fenceExpired(at) {
1583
- return await this.ctx.storage.transaction(async (txn) => {
1584
- const current = await txn.get("fence");
1585
- if (current?.status === "expired") return current.status;
1586
- await txn.put("fence", {
1587
- status: "expired",
1588
- at
1827
+ async compactExpiryFences() {
1828
+ const now = this.now();
1829
+ await this.ctx.storage.transaction(async (txn) => {
1830
+ const due = await txn.list({
1831
+ prefix: EXPIRY_GC_PREFIX,
1832
+ end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
1833
+ limit: 129
1834
+ });
1835
+ const page = Array.from(due).slice(0, 128);
1836
+ const markerKeys = page.map(([, item]) => expiredKey(item.runId));
1837
+ const markers = await txn.get(markerKeys);
1838
+ const deletes = page.flatMap(([key, item], index) => {
1839
+ return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
1589
1840
  });
1590
- return "expired";
1841
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
1842
+ if (due.size > 128) {
1843
+ await txn.setAlarm(now + 1);
1844
+ return;
1845
+ }
1846
+ const nextAt = (await txn.list({
1847
+ prefix: EXPIRY_GC_PREFIX,
1848
+ limit: 1
1849
+ })).values().next().value?.compactAt;
1850
+ if (nextAt === void 0) await txn.deleteAlarm();
1851
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
1591
1852
  });
1592
1853
  }
1593
1854
  };
@@ -2811,15 +3072,24 @@ function hookMarkerKey(reference) {
2811
3072
  return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
2812
3073
  }
2813
3074
  function cleanupTombstone(record, now) {
3075
+ const cleanup = {
3076
+ ...record,
3077
+ phase: "tombstoned",
3078
+ tombstonedAt: now
3079
+ };
2814
3080
  return {
2815
3081
  version: 1,
2816
3082
  runId: record.runId,
2817
3083
  terminalStatus: record.terminalStatus,
2818
3084
  completedAt: record.completedAt,
2819
3085
  expiredAt: record.dueAt,
2820
- tombstonedAt: now
3086
+ tombstonedAt: now,
3087
+ cleanup
2821
3088
  };
2822
3089
  }
3090
+ function cleanupFromTombstone(tombstone) {
3091
+ return tombstone.cleanup;
3092
+ }
2823
3093
  function expiredRead(tombstone) {
2824
3094
  return {
2825
3095
  ok: false,
@@ -2839,7 +3109,6 @@ const STREAMS_PER_CLEANUP_PAGE = 16;
2839
3109
  const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
2840
3110
  const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
2841
3111
  const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
2842
- const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
2843
3112
  const CLEANUP_RETRY_MAX_MS = 36e5;
2844
3113
  const CLEANUP_PROGRESS_KEY = "retention:progress";
2845
3114
  /**
@@ -2981,6 +3250,7 @@ var WorkflowRunDO = class extends DurableObject {
2981
3250
  });
2982
3251
  await txn.setAlarm(this.now() + 1);
2983
3252
  }
3253
+ if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
2984
3254
  return finalizeEventPage(storeFrom(txn), outcome, request.params);
2985
3255
  });
2986
3256
  }
@@ -2991,6 +3261,8 @@ var WorkflowRunDO = class extends DurableObject {
2991
3261
  */
2992
3262
  async resolveHookTokenClaim(request) {
2993
3263
  return await this.ctx.storage.transaction(async (txn) => {
3264
+ const { tombstone } = await this.retentionState(txn);
3265
+ if (tombstone) return { committed: false };
2994
3266
  if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
2995
3267
  await txn.put(hookClaimCancellationKey(request.claimId), {
2996
3268
  hookId: request.hookId,
@@ -3003,6 +3275,20 @@ var WorkflowRunDO = class extends DurableObject {
3003
3275
  async getRun() {
3004
3276
  return this.readKey("run");
3005
3277
  }
3278
+ /** Single authoritative lifecycle answer used after derivative compaction. */
3279
+ async getLifecycleStatus() {
3280
+ return await this.ctx.storage.transaction(async (txn) => {
3281
+ const values = await txn.get([
3282
+ TOMBSTONE_KEY,
3283
+ CLEANUP_RECORD_KEY,
3284
+ "run"
3285
+ ]);
3286
+ if (this.tombstoneFrom(values)) return "expired";
3287
+ const run = values.get("run");
3288
+ if (!run) return "missing";
3289
+ return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
3290
+ });
3291
+ }
3006
3292
  async getStep(stepId) {
3007
3293
  return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
3008
3294
  }
@@ -3031,7 +3317,11 @@ var WorkflowRunDO = class extends DurableObject {
3031
3317
  }));
3032
3318
  }
3033
3319
  async getCleanupStatus() {
3034
- return await this.ctx.storage.get("retention:cleanup") ?? null;
3320
+ const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
3321
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
3322
+ if (cleanup) return cleanup;
3323
+ const tombstone = values.get(TOMBSTONE_KEY);
3324
+ return tombstone ? cleanupFromTombstone(tombstone) : null;
3035
3325
  }
3036
3326
  /** Explicitly schedule an existing terminal run, used for opt-in backfills. */
3037
3327
  async scheduleCleanup(request) {
@@ -3119,7 +3409,6 @@ var WorkflowRunDO = class extends DurableObject {
3119
3409
  workflowIndex() {
3120
3410
  return createWorkflowIndex({
3121
3411
  runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
3122
- runFences: this.namespace("WORKFLOW_RUN_FENCES"),
3123
3412
  hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
3124
3413
  hookIds: this.namespace("WORKFLOW_HOOK_IDS")
3125
3414
  });
@@ -3199,8 +3488,7 @@ var WorkflowRunDO = class extends DurableObject {
3199
3488
  hooks: page.map((hook) => ({
3200
3489
  hookId: hook.hookId,
3201
3490
  token: hook.token
3202
- })),
3203
- terminal: true
3491
+ }))
3204
3492
  });
3205
3493
  await this.ctx.storage.transaction(async (txn) => {
3206
3494
  const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3225,8 +3513,7 @@ var WorkflowRunDO = class extends DurableObject {
3225
3513
  const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
3226
3514
  await this.workflowIndex().releaseHookIndexes({
3227
3515
  runId: cleanup.runId,
3228
- hooks: page.map(([, reference]) => reference),
3229
- terminal: true
3516
+ hooks: page.map(([, reference]) => reference)
3230
3517
  });
3231
3518
  await this.ctx.storage.transaction(async (txn) => {
3232
3519
  const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3336,33 +3623,56 @@ var WorkflowRunDO = class extends DurableObject {
3336
3623
  }
3337
3624
  async deleteQueuePage(cleanup) {
3338
3625
  const queues = this.namespace("WORKFLOW_QUEUE");
3339
- let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3626
+ const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3340
3627
  queueShard: 0,
3341
3628
  queueShardDeleted: 0
3342
3629
  };
3343
- let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
3344
- let expectedGeneration = cleanup.generation;
3345
- while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
3630
+ if (progress.pendingAck) {
3631
+ const { queueShard, receipt } = progress.pendingAck;
3632
+ if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
3633
+ await this.ctx.storage.transaction(async (txn) => {
3634
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3635
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3636
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
3637
+ if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
3638
+ await txn.put(CLEANUP_RECORD_KEY, {
3639
+ ...current,
3640
+ generation: current.generation + 1,
3641
+ attempts: 0,
3642
+ lastError: void 0
3643
+ });
3644
+ await txn.put(CLEANUP_PROGRESS_KEY, {
3645
+ queueShard: queueShard + 1,
3646
+ queueShardDeleted: 0
3647
+ });
3648
+ await txn.setAlarm(this.now() + 1);
3649
+ });
3650
+ return;
3651
+ }
3652
+ if (progress.queueShard < cleanup.queueShards) {
3346
3653
  const requestedShard = progress.queueShard;
3347
3654
  const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
3348
- const reconciled = await this.ctx.storage.transaction(async (txn) => {
3655
+ await this.ctx.storage.transaction(async (txn) => {
3349
3656
  const current = await txn.get(CLEANUP_RECORD_KEY);
3350
- if (current?.phase !== "queues") return null;
3351
- if (current.generation !== expectedGeneration) return { stale: true };
3657
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3352
3658
  const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3353
3659
  queueShard: 0,
3354
3660
  queueShardDeleted: 0
3355
3661
  };
3356
- if (persisted.queueShard !== requestedShard) return { stale: true };
3357
- if (result.deleted < persisted.queueShardDeleted) return { stale: true };
3662
+ if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
3663
+ if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
3664
+ const delta = result.deleted - persisted.queueShardDeleted;
3358
3665
  const next = result.done ? {
3359
- queueShard: persisted.queueShard + 1,
3360
- queueShardDeleted: 0
3666
+ queueShard: requestedShard,
3667
+ queueShardDeleted: result.deleted,
3668
+ pendingAck: {
3669
+ queueShard: requestedShard,
3670
+ receipt: result.receipt
3671
+ }
3361
3672
  } : {
3362
- ...persisted,
3673
+ queueShard: requestedShard,
3363
3674
  queueShardDeleted: result.deleted
3364
3675
  };
3365
- const delta = result.deleted - persisted.queueShardDeleted;
3366
3676
  await txn.put(CLEANUP_RECORD_KEY, {
3367
3677
  ...current,
3368
3678
  deletedQueueMessages: current.deletedQueueMessages + delta,
@@ -3372,27 +3682,17 @@ var WorkflowRunDO = class extends DurableObject {
3372
3682
  });
3373
3683
  await txn.put(CLEANUP_PROGRESS_KEY, next);
3374
3684
  await txn.setAlarm(this.now() + 1);
3375
- return {
3376
- stale: false,
3377
- progress: next,
3378
- generation: current.generation + 1,
3379
- applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
3380
- };
3381
3685
  });
3382
- if (!reconciled) return;
3383
- if (reconciled.stale) return;
3384
- progress = reconciled.progress;
3385
- expectedGeneration = reconciled.generation;
3386
- if (!reconciled.applied && progress.queueShard === requestedShard) break;
3387
- if (!result.done && progress.queueShard === requestedShard) break;
3686
+ return;
3388
3687
  }
3389
3688
  await this.ctx.storage.transaction(async (txn) => {
3390
3689
  const current = await txn.get(CLEANUP_RECORD_KEY);
3391
- if (current?.phase !== "queues") return;
3392
- if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3690
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3691
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3393
3692
  queueShard: 0,
3394
3693
  queueShardDeleted: 0
3395
- }).queueShard >= current.queueShards) {
3694
+ };
3695
+ if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
3396
3696
  await txn.put(CLEANUP_RECORD_KEY, {
3397
3697
  ...current,
3398
3698
  phase: "payload",
@@ -3425,14 +3725,16 @@ var WorkflowRunDO = class extends DurableObject {
3425
3725
  await txn.setAlarm(this.now() + 1);
3426
3726
  return false;
3427
3727
  }
3428
- await txn.put(CLEANUP_RECORD_KEY, {
3728
+ const completed = {
3429
3729
  ...cleanup,
3430
3730
  phase: "tombstoned",
3431
3731
  generation: cleanup.generation + 1,
3432
3732
  attempts: 0,
3433
3733
  lastError: void 0,
3434
3734
  tombstonedAt: now
3435
- });
3735
+ };
3736
+ await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
3737
+ await txn.delete(CLEANUP_RECORD_KEY);
3436
3738
  await txn.deleteAlarm();
3437
3739
  return true;
3438
3740
  });
@@ -3496,6 +3798,6 @@ var worker_default = { async fetch(request, env) {
3496
3798
  return createRouter(env)(request);
3497
3799
  } };
3498
3800
  //#endregion
3499
- export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, RunFenceDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
3801
+ export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
3500
3802
 
3501
3803
  //# sourceMappingURL=worker.js.map