@ewhauser/world-celld 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/testing.js CHANGED
@@ -1,4 +1,4 @@
1
- import { C as validateStreamReadRequest, S as normalizeStreamError, _ as decodeStreamWriteBatch, c as MAX_STREAM_BATCH_BYTES, d as MAX_STREAM_LONG_POLL_MS, f as MAX_STREAM_READ_BYTES, h as STREAM_BATCH_CONTENT_TYPE, i as parse, l as MAX_STREAM_CHUNK_BYTES, n as rpcParse, o as createWorkflowIndex, r as rpcStringify, s as runFenceCellName, t as compact, w as validateStreamWriteChunks, x as encodeStreamWriteResult, y as encodeStreamReadResult } from "./util-DmO0XkFw.js";
1
+ import { A as validateStreamReadRequest, E as encodeStreamReadResult, O as encodeStreamWriteResult, S as STREAM_BATCH_CONTENT_TYPE, c as HOOK_CLAIM_LEASE_MS, d as MAX_QUEUE_DELIVERY_TIMEOUT_MS, f as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, g as MAX_STREAM_CHUNK_BYTES, h as MAX_STREAM_BATCH_BYTES, i as parse, j as validateStreamWriteChunks, k as normalizeStreamError, l as LIFECYCLE_COMPACTION_RETRY_MS, m as QUEUE_INFLIGHT_GRACE_MS, n as rpcParse, o as createWorkflowIndex, p as QUEUE_FENCE_GRACE_MS, r as rpcStringify, t as compact, v as MAX_STREAM_LONG_POLL_MS, w as decodeStreamWriteBatch, y as MAX_STREAM_READ_BYTES } from "./util-ya3V_FTA.js";
2
2
  import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
3
3
  import http from "node:http";
4
4
  //#region src/testing/fake-cell.ts
@@ -453,7 +453,6 @@ const WORLD_VERSION = "0.1.0";
453
453
  function workflowIndex(env) {
454
454
  return createWorkflowIndex({
455
455
  runCatalog: env.WORKFLOW_RUN_CATALOG,
456
- runFences: env.WORKFLOW_RUN_FENCES,
457
456
  hookTokens: env.WORKFLOW_HOOK_TOKENS,
458
457
  hookIds: env.WORKFLOW_HOOK_IDS
459
458
  });
@@ -463,6 +462,7 @@ const BINDINGS = {
463
462
  env: "WORKFLOW_DB",
464
463
  methods: /* @__PURE__ */ new Set([
465
464
  "applyEvent",
465
+ "getLifecycleStatus",
466
466
  "getRun",
467
467
  "getStep",
468
468
  "getEvent",
@@ -505,7 +505,8 @@ const BINDINGS = {
505
505
  "redriveDeadLetter",
506
506
  "purgeDeadLetters",
507
507
  "rearmAlarm",
508
- "expireRun"
508
+ "expireRun",
509
+ "acknowledgeExpireRun"
509
510
  ])
510
511
  }
511
512
  };
@@ -614,7 +615,7 @@ function createRouter(env) {
614
615
  let result;
615
616
  const operation = `${parts[2]}.${parts[3]}`;
616
617
  if (operation === "runs.list") result = await index.listRuns(args[0]);
617
- else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
618
+ else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
618
619
  else if (operation === "runs.expire") result = await index.expireRun(args[0]);
619
620
  else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
620
621
  else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
@@ -740,26 +741,32 @@ function ownerFromSerializedHook(raw) {
740
741
  function indexKey(prefix, value) {
741
742
  return `${prefix}:${encodeURIComponent(value)}`;
742
743
  }
743
- /** Compact shard-local fence for every hook owned by one terminal or expired run. */
744
- function shardRunFenceKey(runId) {
745
- return `runfence:${encodeURIComponent(runId)}`;
746
- }
747
744
  async function runIsActive(env, runId) {
748
- const namespace = env.WORKFLOW_RUN_FENCES;
749
- if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
750
- return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
745
+ const namespace = env.WORKFLOW_DB;
746
+ if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
747
+ return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
751
748
  }
752
749
  //#endregion
753
750
  //#region src/worker/durable-objects/HookIdDO.ts
754
751
  const MAX_RELEASE_BATCH$1 = 64;
752
+ const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
753
+ function pad$3(ms) {
754
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
755
+ }
755
756
  function recordKey$1(hookId) {
756
757
  return indexKey("hookid", hookId);
757
758
  }
758
759
  function claimKey$1(hookId) {
759
760
  return indexKey("claim", hookId);
760
761
  }
762
+ function claimDeadlineKey$1(hookId, expiresAt) {
763
+ return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
764
+ }
761
765
  /** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
762
766
  var HookIdDO = class extends DurableObject {
767
+ now() {
768
+ return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
769
+ }
763
770
  async get(hookId) {
764
771
  const raw = await this.ctx.storage.get(recordKey$1(hookId));
765
772
  if (!raw) return null;
@@ -776,17 +783,33 @@ var HookIdDO = class extends DurableObject {
776
783
  claimed: false,
777
784
  holder: ownerFromSerializedHook(indexed)
778
785
  };
779
- const existing = values.get(claim);
786
+ let existing = values.get(claim);
787
+ const now = this.now();
788
+ if (existing && existing.expiresAt <= now) {
789
+ await txn.delete([claim, existing.deadlineKey]);
790
+ existing = void 0;
791
+ }
780
792
  if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
781
793
  claimed: false,
782
794
  holder: existing.owner
783
795
  };
784
796
  if (existing === void 0 || existing.claimId !== proposedClaimId) {
797
+ if (existing) await txn.delete(existing.deadlineKey);
798
+ const expiresAt = now + HOOK_CLAIM_LEASE_MS;
799
+ const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
785
800
  await txn.put(claim, {
786
801
  owner,
787
802
  claimId: proposedClaimId,
788
- reservedAt: Date.now()
803
+ reservedAt: now,
804
+ expiresAt,
805
+ deadlineKey
806
+ });
807
+ await txn.put(deadlineKey, {
808
+ hookId,
809
+ claimId: proposedClaimId,
810
+ expiresAt
789
811
  });
812
+ await this.armAtMost(txn, expiresAt);
790
813
  return {
791
814
  claimed: true,
792
815
  claimId: proposedClaimId
@@ -803,30 +826,24 @@ var HookIdDO = class extends DurableObject {
803
826
  await this.releaseBatch([{
804
827
  hookId,
805
828
  owner
806
- }], Date.now(), true);
829
+ }]);
807
830
  return { stored: false };
808
831
  }
809
832
  return await this.ctx.storage.transaction(async (txn) => {
810
833
  const record = recordKey$1(hookId);
811
834
  const claim = claimKey$1(hookId);
812
- const runFence = shardRunFenceKey(owner.runId);
813
- const values = await txn.get([
814
- record,
815
- claim,
816
- runFence
817
- ]);
818
- if (values.has(runFence)) {
819
- const existingClaim = values.get(claim);
820
- if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
821
- return { stored: false };
822
- }
835
+ const values = await txn.get([record, claim]);
823
836
  const indexed = values.get(record);
824
837
  if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
825
838
  const existingClaim = values.get(claim);
839
+ if (existingClaim && existingClaim.expiresAt <= this.now()) {
840
+ await txn.delete([claim, existingClaim.deadlineKey]);
841
+ return { stored: false };
842
+ }
826
843
  if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
827
844
  if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
828
845
  await txn.put(record, serializedHook);
829
- if (existingClaim !== void 0) await txn.delete(claim);
846
+ if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
830
847
  return { stored: true };
831
848
  });
832
849
  }
@@ -834,40 +851,94 @@ var HookIdDO = class extends DurableObject {
834
851
  await this.ctx.storage.transaction(async (txn) => {
835
852
  const key = claimKey$1(hookId);
836
853
  const claim = await txn.get(key);
837
- if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
854
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
838
855
  });
839
856
  }
840
- async releaseBatch(hooks, fencedAt, runFenced) {
857
+ async releaseBatch(hooks) {
841
858
  if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
842
859
  if (hooks.length === 0) return { deleted: 0 };
843
860
  return await this.ctx.storage.transaction(async (txn) => {
844
861
  const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
845
862
  const values = await txn.get(keys);
846
863
  const deletes = [];
864
+ const deadlineDeletes = [];
847
865
  for (const { hookId, owner } of hooks) {
848
866
  const record = recordKey$1(hookId);
849
867
  const claim = claimKey$1(hookId);
850
868
  const indexed = values.get(record);
851
869
  const existingClaim = values.get(claim);
852
870
  if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
853
- if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
871
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
872
+ deletes.push(claim);
873
+ deadlineDeletes.push(existingClaim.deadlineKey);
874
+ }
854
875
  }
855
- if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
856
- return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
876
+ const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
877
+ if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
878
+ return { deleted };
879
+ });
880
+ }
881
+ async alarm() {
882
+ try {
883
+ await this.compactClaims();
884
+ } catch {
885
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
886
+ }
887
+ }
888
+ async armAtMost(storage, at) {
889
+ const current = await storage.getAlarm();
890
+ if (current === null || at < current) await storage.setAlarm(at);
891
+ }
892
+ async compactClaims() {
893
+ const now = this.now();
894
+ await this.ctx.storage.transaction(async (txn) => {
895
+ const due = await txn.list({
896
+ prefix: CLAIM_DEADLINE_PREFIX$1,
897
+ end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
898
+ limit: 129
899
+ });
900
+ const page = Array.from(due).slice(0, 128);
901
+ const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
902
+ const deletes = page.flatMap(([deadline, item]) => {
903
+ const key = claimKey$1(item.hookId);
904
+ const claim = claims.get(key);
905
+ return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
906
+ });
907
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
908
+ if (due.size > 128) {
909
+ await txn.setAlarm(now + 1);
910
+ return;
911
+ }
912
+ const nextAt = (await txn.list({
913
+ prefix: CLAIM_DEADLINE_PREFIX$1,
914
+ limit: 1
915
+ })).values().next().value?.expiresAt;
916
+ if (nextAt === void 0) await txn.deleteAlarm();
917
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
857
918
  });
858
919
  }
859
920
  };
860
921
  //#endregion
861
922
  //#region src/worker/durable-objects/HookTokenDO.ts
862
923
  const MAX_RELEASE_BATCH = 64;
924
+ const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
925
+ function pad$2(ms) {
926
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
927
+ }
863
928
  function recordKey(token) {
864
929
  return indexKey("hook", token);
865
930
  }
866
931
  function claimKey(token) {
867
932
  return indexKey("claim", token);
868
933
  }
934
+ function claimDeadlineKey(token, expiresAt) {
935
+ return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
936
+ }
869
937
  /** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
870
938
  var HookTokenDO = class extends DurableObject {
939
+ now() {
940
+ return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
941
+ }
871
942
  async get(token) {
872
943
  const raw = await this.ctx.storage.get(recordKey(token));
873
944
  if (!raw) return null;
@@ -884,17 +955,32 @@ var HookTokenDO = class extends DurableObject {
884
955
  claimed: false,
885
956
  holder: ownerFromSerializedHook(indexed)
886
957
  };
887
- const existing = values.get(claim);
958
+ let existing = values.get(claim);
959
+ const now = this.now();
960
+ if (existing && existing.expiresAt <= now) {
961
+ await txn.delete([claim, existing.deadlineKey]);
962
+ existing = void 0;
963
+ }
888
964
  if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
889
965
  claimed: false,
890
966
  holder: existing.owner
891
967
  };
892
968
  if (existing === void 0) {
969
+ const expiresAt = now + HOOK_CLAIM_LEASE_MS;
970
+ const deadlineKey = claimDeadlineKey(token, expiresAt);
893
971
  await txn.put(claim, {
894
972
  owner,
895
973
  claimId: proposedClaimId,
896
- reservedAt: Date.now()
974
+ reservedAt: now,
975
+ expiresAt,
976
+ deadlineKey
977
+ });
978
+ await txn.put(deadlineKey, {
979
+ token,
980
+ claimId: proposedClaimId,
981
+ expiresAt
897
982
  });
983
+ await this.armAtMost(txn, expiresAt);
898
984
  return {
899
985
  claimed: true,
900
986
  claimId: proposedClaimId
@@ -911,30 +997,24 @@ var HookTokenDO = class extends DurableObject {
911
997
  await this.releaseBatch([{
912
998
  token,
913
999
  owner
914
- }], Date.now(), true);
1000
+ }]);
915
1001
  return { stored: false };
916
1002
  }
917
1003
  return await this.ctx.storage.transaction(async (txn) => {
918
1004
  const record = recordKey(token);
919
1005
  const claim = claimKey(token);
920
- const runFence = shardRunFenceKey(owner.runId);
921
- const values = await txn.get([
922
- record,
923
- claim,
924
- runFence
925
- ]);
926
- if (values.has(runFence)) {
927
- const existingClaim = values.get(claim);
928
- if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
929
- return { stored: false };
930
- }
1006
+ const values = await txn.get([record, claim]);
931
1007
  const indexed = values.get(record);
932
1008
  if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
933
1009
  const existingClaim = values.get(claim);
1010
+ if (existingClaim && existingClaim.expiresAt <= this.now()) {
1011
+ await txn.delete([claim, existingClaim.deadlineKey]);
1012
+ return { stored: false };
1013
+ }
934
1014
  if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
935
1015
  if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
936
1016
  await txn.put(record, serializedHook);
937
- if (existingClaim !== void 0) await txn.delete(claim);
1017
+ if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
938
1018
  return { stored: true };
939
1019
  });
940
1020
  }
@@ -942,26 +1022,70 @@ var HookTokenDO = class extends DurableObject {
942
1022
  await this.ctx.storage.transaction(async (txn) => {
943
1023
  const key = claimKey(token);
944
1024
  const claim = await txn.get(key);
945
- if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
1025
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
946
1026
  });
947
1027
  }
948
- async releaseBatch(hooks, fencedAt, runFenced) {
1028
+ async releaseBatch(hooks) {
949
1029
  if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
950
1030
  if (hooks.length === 0) return { deleted: 0 };
951
1031
  return await this.ctx.storage.transaction(async (txn) => {
952
1032
  const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
953
1033
  const values = await txn.get(keys);
954
1034
  const deletes = [];
1035
+ const deadlineDeletes = [];
955
1036
  for (const { token, owner } of hooks) {
956
1037
  const record = recordKey(token);
957
1038
  const claim = claimKey(token);
958
1039
  const indexed = values.get(record);
959
1040
  const existingClaim = values.get(claim);
960
1041
  if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
961
- if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
1042
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
1043
+ deletes.push(claim);
1044
+ deadlineDeletes.push(existingClaim.deadlineKey);
1045
+ }
962
1046
  }
963
- if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
964
- return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
1047
+ const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
1048
+ if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
1049
+ return { deleted };
1050
+ });
1051
+ }
1052
+ async alarm() {
1053
+ try {
1054
+ await this.compactClaims();
1055
+ } catch {
1056
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
1057
+ }
1058
+ }
1059
+ async armAtMost(storage, at) {
1060
+ const current = await storage.getAlarm();
1061
+ if (current === null || at < current) await storage.setAlarm(at);
1062
+ }
1063
+ async compactClaims() {
1064
+ const now = this.now();
1065
+ await this.ctx.storage.transaction(async (txn) => {
1066
+ const due = await txn.list({
1067
+ prefix: CLAIM_DEADLINE_PREFIX,
1068
+ end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
1069
+ limit: 129
1070
+ });
1071
+ const page = Array.from(due).slice(0, 128);
1072
+ const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
1073
+ const deletes = page.flatMap(([deadline, item]) => {
1074
+ const key = claimKey(item.token);
1075
+ const claim = claims.get(key);
1076
+ return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
1077
+ });
1078
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
1079
+ if (due.size > 128) {
1080
+ await txn.setAlarm(now + 1);
1081
+ return;
1082
+ }
1083
+ const nextAt = (await txn.list({
1084
+ prefix: CLAIM_DEADLINE_PREFIX,
1085
+ limit: 1
1086
+ })).values().next().value?.expiresAt;
1087
+ if (nextAt === void 0) await txn.deleteAlarm();
1088
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
965
1089
  });
966
1090
  }
967
1091
  };
@@ -976,8 +1100,6 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
976
1100
  const DEFAULT_MAX_ATTEMPTS = 5;
977
1101
  const DEFAULT_MAX_INFLIGHT = 5;
978
1102
  const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
979
- /** Grace period past the delivery timeout before a claim counts as lost. */
980
- const INFLIGHT_GRACE_MS = 3e4;
981
1103
  /** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
982
1104
  const MIN_ALARM_DELAY_MS = 1;
983
1105
  /** Bound expiry recovery so one alarm cannot monopolize the cell. */
@@ -986,14 +1108,15 @@ const STORAGE_BATCH_SIZE$2 = 128;
986
1108
  const EXPIRE_RUN_REFERENCE_BATCH = 64;
987
1109
  const PURGE_DEAD_LETTER_BATCH = 128;
988
1110
  const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
989
- function pad(ms) {
1111
+ const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
1112
+ function pad$1(ms) {
990
1113
  return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
991
1114
  }
992
1115
  function dueKey(atMs, messageId) {
993
- return `due:${pad(atMs)}:${messageId}`;
1116
+ return `due:${pad$1(atMs)}:${messageId}`;
994
1117
  }
995
1118
  function inflightKey(deadlineMs, messageId) {
996
- return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
1119
+ return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
997
1120
  }
998
1121
  function deadlineFromInflightKey(key) {
999
1122
  return Number.parseInt(key.slice(18, 31), 10);
@@ -1004,6 +1127,9 @@ function runReferenceKey(runId, messageId) {
1004
1127
  function expiredRunKey(runId) {
1005
1128
  return `expired-run:${runId}`;
1006
1129
  }
1130
+ function expiredRunGcKey(runId, compactAt) {
1131
+ return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
1132
+ }
1007
1133
  function boundedLimit$1(value, fallback, maximum) {
1008
1134
  if (value === void 0 || !Number.isFinite(value)) return fallback;
1009
1135
  return Math.max(1, Math.min(maximum, Math.floor(value)));
@@ -1054,16 +1180,29 @@ var QueueDO = class extends DurableObject {
1054
1180
  if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
1055
1181
  return configured;
1056
1182
  }
1183
+ #deliveryTimeoutMs() {
1184
+ const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1185
+ if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
1186
+ return configured;
1187
+ }
1188
+ async #authoritativeRunExpired(runId) {
1189
+ const namespace = this.env?.WORKFLOW_DB;
1190
+ if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
1191
+ return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
1192
+ }
1193
+ async #withRunMutation(runId, onExpired, onAllowed) {
1194
+ const expired = await this.#authoritativeRunExpired(runId);
1195
+ return await this.ctx.storage.transaction(async (txn) => {
1196
+ if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
1197
+ return await onAllowed(txn);
1198
+ });
1199
+ }
1057
1200
  async enqueue(request) {
1058
1201
  this.#maxInflight();
1202
+ this.#deliveryTimeoutMs();
1059
1203
  const storage = this.ctx.storage;
1060
1204
  const now = this.#now();
1061
- return await storage.transaction(async (txn) => {
1062
- if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
1063
- ok: false,
1064
- code: "RUN_EXPIRED",
1065
- message: `Workflow run "${request.runId}" has expired`
1066
- };
1205
+ const persist = async (txn) => {
1067
1206
  const pinned = await txn.get("cfg");
1068
1207
  if (!pinned) await txn.put("cfg", {
1069
1208
  queueShards: request.config.queueShards,
@@ -1115,7 +1254,12 @@ var QueueDO = class extends DurableObject {
1115
1254
  messageId: row.messageId,
1116
1255
  deduped: false
1117
1256
  };
1118
- });
1257
+ };
1258
+ return request.runId ? await this.#withRunMutation(request.runId, async () => ({
1259
+ ok: false,
1260
+ code: "RUN_EXPIRED",
1261
+ message: `Workflow run "${request.runId}" has expired`
1262
+ }), persist) : await storage.transaction(persist);
1119
1263
  }
1120
1264
  async alarm() {
1121
1265
  let claimed = [];
@@ -1129,7 +1273,10 @@ var QueueDO = class extends DurableObject {
1129
1273
  };
1130
1274
  if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
1131
1275
  else await attempt();
1132
- if (failure !== null) throw failure;
1276
+ if (failure !== null) {
1277
+ await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
1278
+ throw failure;
1279
+ }
1133
1280
  for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
1134
1281
  }
1135
1282
  /**
@@ -1141,9 +1288,13 @@ var QueueDO = class extends DurableObject {
1141
1288
  const now = this.#now();
1142
1289
  const maxInflight = this.#maxInflight();
1143
1290
  return await storage.transaction(async (txn) => {
1291
+ if (await this.#compactExpiredRunFences(txn, now)) {
1292
+ await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
1293
+ return [];
1294
+ }
1144
1295
  const expiredPage = await txn.list({
1145
1296
  prefix: INFLIGHT_DEADLINE_PREFIX,
1146
- end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
1297
+ end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
1147
1298
  limit: 129
1148
1299
  });
1149
1300
  const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
@@ -1175,17 +1326,17 @@ var QueueDO = class extends DurableObject {
1175
1326
  if (inflightCount < maxInflight) {
1176
1327
  const due = await txn.list({
1177
1328
  prefix: "due:",
1178
- end: `due:${pad(now + 1)}`,
1329
+ end: `due:${pad$1(now + 1)}`,
1179
1330
  limit: maxInflight - inflightCount
1180
1331
  });
1181
- const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1332
+ const timeoutMs = this.#deliveryTimeoutMs();
1182
1333
  const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
1183
1334
  await deleteMany$1(txn, Array.from(due.keys()));
1184
1335
  const claimWrites = [];
1185
1336
  for (const [, messageId] of due) {
1186
1337
  const row = dueRows.get(`msg:${messageId}`);
1187
1338
  if (!row) continue;
1188
- const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
1339
+ const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
1189
1340
  claimWrites.push([claimKey, messageId]);
1190
1341
  if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
1191
1342
  messageId: row.messageId,
@@ -1205,9 +1356,13 @@ var QueueDO = class extends DurableObject {
1205
1356
  }
1206
1357
  async #deliver({ row, inflightKey: claimKey }) {
1207
1358
  const storage = this.ctx.storage;
1208
- const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
1359
+ const timeoutMs = this.#deliveryTimeoutMs();
1209
1360
  const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
1210
1361
  const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
1362
+ if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
1363
+ await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
1364
+ return;
1365
+ }
1211
1366
  let response = null;
1212
1367
  let transportError = null;
1213
1368
  try {
@@ -1237,28 +1392,21 @@ var QueueDO = class extends DurableObject {
1237
1392
  } catch {
1238
1393
  timeoutSeconds = void 0;
1239
1394
  }
1240
- if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
1241
- if (!await txn.delete(claimKey)) return;
1242
- if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
1243
- await txn.delete(`msg:${row.messageId}`);
1244
- await txn.delete(runReferenceKey(row.runId, row.messageId));
1245
- if (row.idempotencyKey) {
1246
- const dedupKey = `key:${row.idempotencyKey}`;
1247
- if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1248
- }
1395
+ if (typeof timeoutSeconds === "number") {
1396
+ const reschedule = async (txn) => {
1397
+ if (!await txn.delete(claimKey)) return;
1398
+ const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
1399
+ await txn.put(scheduleKey, row.messageId);
1400
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1401
+ messageId: row.messageId,
1402
+ dueKey: scheduleKey,
1403
+ idempotencyKey: row.idempotencyKey
1404
+ });
1249
1405
  await this.#scheduleNextAlarm(txn, now);
1250
- return;
1251
- }
1252
- const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
1253
- await txn.put(scheduleKey, row.messageId);
1254
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1255
- messageId: row.messageId,
1256
- dueKey: scheduleKey,
1257
- idempotencyKey: row.idempotencyKey
1258
- });
1259
- await this.#scheduleNextAlarm(txn, now);
1260
- });
1261
- else await this.#retry(row, claimKey, "HTTP 503");
1406
+ };
1407
+ if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
1408
+ else await storage.transaction(reschedule);
1409
+ } else await this.#retry(row, claimKey, "HTTP 503");
1262
1410
  } else if (response && PERMANENT_STATUSES.has(response.status)) {
1263
1411
  await cancelUnusedResponseBody(response);
1264
1412
  await this.#ack(row, claimKey);
@@ -1279,23 +1427,23 @@ var QueueDO = class extends DurableObject {
1279
1427
  await this.#scheduleNextAlarm(txn, this.#now());
1280
1428
  });
1281
1429
  }
1430
+ async #dropExpiredMessage(txn, row, claimKey, now) {
1431
+ if (!await txn.delete(claimKey)) return;
1432
+ await txn.delete(`msg:${row.messageId}`);
1433
+ if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
1434
+ if (row.idempotencyKey) {
1435
+ const dedupKey = `key:${row.idempotencyKey}`;
1436
+ if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1437
+ }
1438
+ await this.#scheduleNextAlarm(txn, now);
1439
+ }
1282
1440
  async #retry(row, claimKey, reason) {
1283
1441
  const storage = this.ctx.storage;
1284
1442
  const now = this.#now();
1285
1443
  const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
1286
1444
  const attempt = row.attempt + 1;
1287
- await storage.transaction(async (txn) => {
1445
+ const persistRetry = async (txn) => {
1288
1446
  if (!await txn.delete(claimKey)) return;
1289
- if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
1290
- await txn.delete(`msg:${row.messageId}`);
1291
- await txn.delete(runReferenceKey(row.runId, row.messageId));
1292
- if (row.idempotencyKey) {
1293
- const dedupKey = `key:${row.idempotencyKey}`;
1294
- if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
1295
- }
1296
- await this.#scheduleNextAlarm(txn, now);
1297
- return;
1298
- }
1299
1447
  if (attempt >= maxAttempts) {
1300
1448
  const dead = {
1301
1449
  ...row,
@@ -1303,7 +1451,7 @@ var QueueDO = class extends DurableObject {
1303
1451
  lastError: reason,
1304
1452
  failedAt: now
1305
1453
  };
1306
- const deadKey = `dlq:${pad(now)}:${row.messageId}`;
1454
+ const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
1307
1455
  await txn.put(deadKey, dead);
1308
1456
  await txn.delete(`msg:${row.messageId}`);
1309
1457
  if (row.idempotencyKey) {
@@ -1331,7 +1479,32 @@ var QueueDO = class extends DurableObject {
1331
1479
  idempotencyKey: row.idempotencyKey
1332
1480
  });
1333
1481
  await this.#scheduleNextAlarm(txn, now);
1334
- });
1482
+ };
1483
+ if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
1484
+ else await storage.transaction(persistRetry);
1485
+ }
1486
+ /** Return true when another due compaction page must run immediately. */
1487
+ async #compactExpiredRunFences(txn, now) {
1488
+ const due = await txn.list({
1489
+ prefix: EXPIRED_RUN_GC_PREFIX,
1490
+ end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
1491
+ limit: 129
1492
+ });
1493
+ const page = Array.from(due).slice(0, 128);
1494
+ if (page.length === 0) return false;
1495
+ const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
1496
+ const markers = await getMany$1(txn, markerKeys);
1497
+ const deletes = [];
1498
+ for (let index = 0; index < page.length; index++) {
1499
+ const [gcKey, item] = page[index];
1500
+ const markerKey = markerKeys[index];
1501
+ const marker = markers.get(markerKey);
1502
+ deletes.push(gcKey);
1503
+ if (marker?.compactAt !== item.compactAt) continue;
1504
+ deletes.push(markerKey);
1505
+ }
1506
+ await deleteMany$1(txn, deletes);
1507
+ return due.size > 128;
1335
1508
  }
1336
1509
  /** Arm the alarm no later than `atMs` (durable backoff never throws). */
1337
1510
  async #armAlarmAtMost(storage, atMs, now) {
@@ -1363,6 +1536,11 @@ var QueueDO = class extends DurableObject {
1363
1536
  const deadline = deadlineFromInflightKey(key);
1364
1537
  if (next === null || deadline < next) next = deadline;
1365
1538
  }
1539
+ const expiryGc = await storage.list({
1540
+ prefix: EXPIRED_RUN_GC_PREFIX,
1541
+ limit: 1
1542
+ });
1543
+ for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
1366
1544
  if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
1367
1545
  else await storage.deleteAlarm();
1368
1546
  }
@@ -1401,37 +1579,42 @@ var QueueDO = class extends DurableObject {
1401
1579
  async redriveDeadLetter(messageId) {
1402
1580
  const storage = this.ctx.storage;
1403
1581
  const now = this.#now();
1404
- return await storage.transaction(async (txn) => {
1405
- const dlq = await txn.list({ prefix: "dlq:" });
1406
- for (const [key, dead] of dlq) {
1407
- if (dead.messageId !== messageId) continue;
1408
- const { failedAt: _failedAt, ...rest } = dead;
1409
- const row = {
1410
- ...rest,
1411
- attempt: 0,
1412
- lastError: void 0
1413
- };
1414
- if (row.idempotencyKey) {
1415
- const dedupKey = `key:${row.idempotencyKey}`;
1416
- const holder = await txn.get(dedupKey);
1417
- if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
1418
- if (holder && holder !== row.messageId) await txn.delete(dedupKey);
1419
- await txn.put(dedupKey, row.messageId);
1420
- }
1421
- await txn.put(`msg:${row.messageId}`, row);
1422
- const scheduleKey = dueKey(now, row.messageId);
1423
- await txn.put(scheduleKey, row.messageId);
1424
- await txn.delete(key);
1425
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1426
- messageId: row.messageId,
1427
- dueKey: scheduleKey,
1428
- idempotencyKey: row.idempotencyKey
1429
- });
1430
- await this.#armAlarmAtMost(txn, now, now);
1431
- return { ok: true };
1582
+ const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
1583
+ if (!found) return { ok: false };
1584
+ const [deadKey, candidate] = found;
1585
+ const redrive = async (txn) => {
1586
+ const dead = await txn.get(deadKey);
1587
+ if (!dead || dead.messageId !== messageId) return { ok: false };
1588
+ const { failedAt: _failedAt, ...rest } = dead;
1589
+ const row = {
1590
+ ...rest,
1591
+ attempt: 0,
1592
+ lastError: void 0
1593
+ };
1594
+ if (row.idempotencyKey) {
1595
+ const dedupKey = `key:${row.idempotencyKey}`;
1596
+ const holder = await txn.get(dedupKey);
1597
+ if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
1598
+ if (holder && holder !== row.messageId) await txn.delete(dedupKey);
1599
+ await txn.put(dedupKey, row.messageId);
1432
1600
  }
1601
+ await txn.put(`msg:${row.messageId}`, row);
1602
+ const scheduleKey = dueKey(now, row.messageId);
1603
+ await txn.put(scheduleKey, row.messageId);
1604
+ await txn.delete(deadKey);
1605
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1606
+ messageId: row.messageId,
1607
+ dueKey: scheduleKey,
1608
+ idempotencyKey: row.idempotencyKey
1609
+ });
1610
+ await this.#armAlarmAtMost(txn, now, now);
1611
+ return { ok: true };
1612
+ };
1613
+ const rejectExpired = async (txn) => {
1614
+ if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
1433
1615
  return { ok: false };
1434
- });
1616
+ };
1617
+ return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
1435
1618
  }
1436
1619
  async purgeDeadLetters() {
1437
1620
  return await this.ctx.storage.transaction(async (txn) => {
@@ -1481,27 +1664,75 @@ var QueueDO = class extends DurableObject {
1481
1664
  count++;
1482
1665
  }
1483
1666
  await deleteMany$1(txn, keys);
1667
+ const done = references.size <= limit;
1668
+ const markerExpiredAt = existing?.expiredAt ?? expiredAt;
1484
1669
  await txn.put(expiredRunKey(runId), {
1485
- expiredAt,
1486
- deleted: count
1670
+ expiredAt: markerExpiredAt,
1671
+ deleted: count,
1672
+ compactAt: existing?.compactAt ?? null
1487
1673
  });
1488
1674
  await this.#scheduleNextAlarm(txn, this.#now());
1489
- return {
1675
+ return done ? {
1490
1676
  deleted: count,
1491
- done: references.size <= limit
1677
+ done: true,
1678
+ receipt: {
1679
+ expiredAt: markerExpiredAt,
1680
+ deleted: count
1681
+ }
1682
+ } : {
1683
+ deleted: count,
1684
+ done: false
1492
1685
  };
1493
1686
  });
1494
1687
  }
1688
+ /** Acknowledge a final cleanup receipt before making its marker compactable. */
1689
+ async acknowledgeExpireRun(runId, receipt) {
1690
+ return await this.ctx.storage.transaction(async (txn) => {
1691
+ const key = expiredRunKey(runId);
1692
+ const marker = await txn.get(key);
1693
+ if (!marker) return { acknowledged: true };
1694
+ if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
1695
+ if (marker.compactAt === null) {
1696
+ const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
1697
+ await txn.put(key, {
1698
+ ...marker,
1699
+ compactAt
1700
+ });
1701
+ await txn.put(expiredRunGcKey(runId, compactAt), {
1702
+ runId,
1703
+ compactAt
1704
+ });
1705
+ }
1706
+ await this.#scheduleNextAlarm(txn, this.#now());
1707
+ return { acknowledged: true };
1708
+ });
1709
+ }
1495
1710
  };
1496
1711
  //#endregion
1497
1712
  //#region src/worker/durable-objects/RunCatalogDO.ts
1498
1713
  const MAX_INDEX_LIST_SIZE = 1001;
1714
+ const EXPIRY_GC_PREFIX = "expiry-gc:";
1715
+ function pad(ms) {
1716
+ return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
1717
+ }
1718
+ function expiredKey(runId) {
1719
+ return `expired:${runId}`;
1720
+ }
1721
+ function expiryGcKey(runId, compactAt) {
1722
+ return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
1723
+ }
1499
1724
  /** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
1500
1725
  var RunCatalogDO = class extends DurableObject {
1501
- async upsertRun(runId, keys, serializedMetadata) {
1726
+ now() {
1727
+ const clock = this.env?.clock;
1728
+ return typeof clock === "function" ? clock() : Date.now();
1729
+ }
1730
+ async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
1502
1731
  if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
1503
1732
  return await this.ctx.storage.transaction(async (txn) => {
1504
- if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
1733
+ if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
1734
+ const now = this.now();
1735
+ if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
1505
1736
  await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
1506
1737
  return { stored: true };
1507
1738
  });
@@ -1513,8 +1744,11 @@ var RunCatalogDO = class extends DurableObject {
1513
1744
  prefix: options.prefix ?? "",
1514
1745
  ...options.reverse ? {
1515
1746
  reverse: true,
1516
- end: options.cursor
1517
- } : { startAfter: options.cursor },
1747
+ end: options.cursor ?? options.end
1748
+ } : {
1749
+ startAfter: options.cursor,
1750
+ end: options.end
1751
+ },
1518
1752
  limit: limit + 1
1519
1753
  });
1520
1754
  const page = Array.from(entries).slice(0, limit);
@@ -1528,41 +1762,68 @@ var RunCatalogDO = class extends DurableObject {
1528
1762
  cursor: listComplete ? void 0 : page.at(-1)?.[0]
1529
1763
  };
1530
1764
  }
1765
+ /** Compare-and-delete one rejected retention discovery candidate. */
1766
+ async deleteStaleGlobalRun(runId, key, expectedValue) {
1767
+ if (!key.startsWith("runall:") || !key.endsWith(`:${runId}`)) throw new Error("Stale run catalog deletion requires the exact global run key");
1768
+ if (JSON.parse(expectedValue).runId !== runId) throw new Error("Stale run catalog deletion requires matching run metadata");
1769
+ return await this.ctx.storage.transaction(async (txn) => {
1770
+ if (await txn.get(key) !== expectedValue) return { deleted: false };
1771
+ return { deleted: await txn.delete(key) };
1772
+ });
1773
+ }
1531
1774
  async expireRun(runId, keys, expiredAt) {
1532
1775
  if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
1533
1776
  return await this.ctx.storage.transaction(async (txn) => {
1534
- await txn.put(`expired:${runId}`, expiredAt);
1777
+ const marker = expiredKey(runId);
1778
+ const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
1779
+ await txn.put(marker, {
1780
+ expiredAt,
1781
+ compactAt
1782
+ });
1783
+ await txn.put(expiryGcKey(runId, compactAt), {
1784
+ runId,
1785
+ compactAt
1786
+ });
1787
+ await this.armAtMost(txn, compactAt);
1535
1788
  return { deleted: await txn.delete(keys) };
1536
1789
  });
1537
1790
  }
1538
- };
1539
- //#endregion
1540
- //#region src/worker/durable-objects/RunFenceDO.ts
1541
- /** Per-run lifecycle fence used by hook shards without a global lookup cell. */
1542
- var RunFenceDO = class extends DurableObject {
1543
- async getStatus() {
1544
- return (await this.ctx.storage.get("fence"))?.status ?? "active";
1791
+ async alarm() {
1792
+ try {
1793
+ await this.compactExpiryFences();
1794
+ } catch {
1795
+ await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
1796
+ }
1545
1797
  }
1546
- async fenceTerminal(at) {
1547
- return await this.ctx.storage.transaction(async (txn) => {
1548
- const current = await txn.get("fence");
1549
- if (current) return current.status;
1550
- await txn.put("fence", {
1551
- status: "terminal",
1552
- at
1553
- });
1554
- return "terminal";
1555
- });
1798
+ async armAtMost(storage, at) {
1799
+ const current = await storage.getAlarm();
1800
+ if (current === null || at < current) await storage.setAlarm(at);
1556
1801
  }
1557
- async fenceExpired(at) {
1558
- return await this.ctx.storage.transaction(async (txn) => {
1559
- const current = await txn.get("fence");
1560
- if (current?.status === "expired") return current.status;
1561
- await txn.put("fence", {
1562
- status: "expired",
1563
- at
1802
+ async compactExpiryFences() {
1803
+ const now = this.now();
1804
+ await this.ctx.storage.transaction(async (txn) => {
1805
+ const due = await txn.list({
1806
+ prefix: EXPIRY_GC_PREFIX,
1807
+ end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
1808
+ limit: 129
1809
+ });
1810
+ const page = Array.from(due).slice(0, 128);
1811
+ const markerKeys = page.map(([, item]) => expiredKey(item.runId));
1812
+ const markers = await txn.get(markerKeys);
1813
+ const deletes = page.flatMap(([key, item], index) => {
1814
+ return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
1564
1815
  });
1565
- return "expired";
1816
+ for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
1817
+ if (due.size > 128) {
1818
+ await txn.setAlarm(now + 1);
1819
+ return;
1820
+ }
1821
+ const nextAt = (await txn.list({
1822
+ prefix: EXPIRY_GC_PREFIX,
1823
+ limit: 1
1824
+ })).values().next().value?.compactAt;
1825
+ if (nextAt === void 0) await txn.deleteAlarm();
1826
+ else await txn.setAlarm(Math.max(now + 1, nextAt));
1566
1827
  });
1567
1828
  }
1568
1829
  };
@@ -2786,15 +3047,25 @@ function hookMarkerKey(reference) {
2786
3047
  return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
2787
3048
  }
2788
3049
  function cleanupTombstone(record, now) {
3050
+ const cleanup = {
3051
+ ...record,
3052
+ phase: "tombstoned",
3053
+ tombstonedAt: now
3054
+ };
2789
3055
  return {
2790
3056
  version: 1,
2791
3057
  runId: record.runId,
3058
+ status: record.status ?? record.terminalStatus,
2792
3059
  terminalStatus: record.terminalStatus,
2793
3060
  completedAt: record.completedAt,
2794
3061
  expiredAt: record.dueAt,
2795
- tombstonedAt: now
3062
+ tombstonedAt: now,
3063
+ cleanup
2796
3064
  };
2797
3065
  }
3066
+ function cleanupFromTombstone(tombstone) {
3067
+ return tombstone.cleanup;
3068
+ }
2798
3069
  function expiredRead(tombstone) {
2799
3070
  return {
2800
3071
  ok: false,
@@ -2814,9 +3085,30 @@ const STREAMS_PER_CLEANUP_PAGE = 16;
2814
3085
  const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
2815
3086
  const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
2816
3087
  const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
2817
- const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
2818
3088
  const CLEANUP_RETRY_MAX_MS = 36e5;
2819
3089
  const CLEANUP_PROGRESS_KEY = "retention:progress";
3090
+ const RUN_QUEUE_SHARDS_KEY = "retention:queue-shards";
3091
+ function cleanupRecord(run, dueAt, queueShards, reason) {
3092
+ const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
3093
+ return {
3094
+ version: 1,
3095
+ runId: run.runId,
3096
+ workflowName: run.workflowName,
3097
+ createdAt: run.createdAt,
3098
+ status: run.status,
3099
+ completedAt: terminal?.completedAt,
3100
+ terminalStatus: terminal?.status,
3101
+ reason,
3102
+ dueAt,
3103
+ queueShards,
3104
+ phase: "retained",
3105
+ generation: 0,
3106
+ attempts: 0,
3107
+ deletedPayloadKeys: 0,
3108
+ deletedStreams: 0,
3109
+ deletedQueueMessages: 0
3110
+ };
3111
+ }
2820
3112
  /**
2821
3113
  * Adapt DO storage (or a transaction) to the {@link EventStore} interface
2822
3114
  * shared with the test mocks. No casts: the methods line up structurally,
@@ -2912,6 +3204,7 @@ var WorkflowRunDO = class extends DurableObject {
2912
3204
  now
2913
3205
  });
2914
3206
  if (!outcome.ok) return outcome;
3207
+ if (outcome.runCreated && request.cleanup) await txn.put(RUN_QUEUE_SHARDS_KEY, request.cleanup.queueShards);
2915
3208
  await txn.put("event_sequence", eventSequence);
2916
3209
  await putStorageEntries(txn, (outcome.hookToIndex ? [{
2917
3210
  hookId: outcome.hookToIndex.hookId,
@@ -2925,22 +3218,7 @@ var WorkflowRunDO = class extends DurableObject {
2925
3218
  ...terminalRun,
2926
3219
  expiredAt: dueAt
2927
3220
  });
2928
- const cleanup = {
2929
- version: 1,
2930
- runId: run.runId,
2931
- workflowName: run.workflowName,
2932
- createdAt: run.createdAt,
2933
- completedAt: run.completedAt,
2934
- terminalStatus: run.status,
2935
- dueAt,
2936
- queueShards: request.cleanup?.queueShards ?? 1,
2937
- phase: "retained",
2938
- generation: 0,
2939
- attempts: 0,
2940
- deletedPayloadKeys: 0,
2941
- deletedStreams: 0,
2942
- deletedQueueMessages: 0
2943
- };
3221
+ const cleanup = cleanupRecord(run, dueAt, request.cleanup?.queueShards ?? 1, "terminal-retention");
2944
3222
  await txn.put("run", run);
2945
3223
  await txn.put(CLEANUP_RECORD_KEY, cleanup);
2946
3224
  await txn.setAlarm(dueAt);
@@ -2956,6 +3234,7 @@ var WorkflowRunDO = class extends DurableObject {
2956
3234
  });
2957
3235
  await txn.setAlarm(this.now() + 1);
2958
3236
  }
3237
+ if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
2959
3238
  return finalizeEventPage(storeFrom(txn), outcome, request.params);
2960
3239
  });
2961
3240
  }
@@ -2966,6 +3245,8 @@ var WorkflowRunDO = class extends DurableObject {
2966
3245
  */
2967
3246
  async resolveHookTokenClaim(request) {
2968
3247
  return await this.ctx.storage.transaction(async (txn) => {
3248
+ const { tombstone } = await this.retentionState(txn);
3249
+ if (tombstone) return { committed: false };
2969
3250
  if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
2970
3251
  await txn.put(hookClaimCancellationKey(request.claimId), {
2971
3252
  hookId: request.hookId,
@@ -2978,6 +3259,20 @@ var WorkflowRunDO = class extends DurableObject {
2978
3259
  async getRun() {
2979
3260
  return this.readKey("run");
2980
3261
  }
3262
+ /** Single authoritative lifecycle answer used after derivative compaction. */
3263
+ async getLifecycleStatus() {
3264
+ return await this.ctx.storage.transaction(async (txn) => {
3265
+ const values = await txn.get([
3266
+ TOMBSTONE_KEY,
3267
+ CLEANUP_RECORD_KEY,
3268
+ "run"
3269
+ ]);
3270
+ if (this.tombstoneFrom(values)) return "expired";
3271
+ const run = values.get("run");
3272
+ if (!run) return "missing";
3273
+ return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
3274
+ });
3275
+ }
2981
3276
  async getStep(stepId) {
2982
3277
  return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
2983
3278
  }
@@ -3006,7 +3301,11 @@ var WorkflowRunDO = class extends DurableObject {
3006
3301
  }));
3007
3302
  }
3008
3303
  async getCleanupStatus() {
3009
- return await this.ctx.storage.get("retention:cleanup") ?? null;
3304
+ const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
3305
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
3306
+ if (cleanup) return cleanup;
3307
+ const tombstone = values.get(TOMBSTONE_KEY);
3308
+ return tombstone ? cleanupFromTombstone(tombstone) : null;
3010
3309
  }
3011
3310
  /** Explicitly schedule an existing terminal run, used for opt-in backfills. */
3012
3311
  async scheduleCleanup(request) {
@@ -3029,28 +3328,95 @@ var WorkflowRunDO = class extends DurableObject {
3029
3328
  ...terminalRun,
3030
3329
  expiredAt: dueAt
3031
3330
  });
3032
- const cleanup = {
3033
- version: 1,
3034
- runId: run.runId,
3035
- workflowName: run.workflowName,
3036
- createdAt: run.createdAt,
3037
- completedAt: run.completedAt,
3038
- terminalStatus: run.status,
3039
- dueAt,
3040
- queueShards: request.queueShards,
3041
- phase: "retained",
3042
- generation: 0,
3043
- attempts: 0,
3044
- deletedPayloadKeys: 0,
3045
- deletedStreams: 0,
3046
- deletedQueueMessages: 0
3047
- };
3331
+ const cleanup = cleanupRecord(run, dueAt, request.queueShards, "manual");
3048
3332
  await txn.put("run", run);
3049
3333
  await txn.put(CLEANUP_RECORD_KEY, cleanup);
3050
3334
  await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
3051
3335
  return cleanup;
3052
3336
  });
3053
3337
  }
3338
+ /**
3339
+ * Apply the fleet-wide maximum-age policy to one authoritative run.
3340
+ *
3341
+ * The catalog is only a discovery index: eligibility is rechecked from the
3342
+ * RunDO's own createdAt. Once eligible, the cleanup deadline immediately
3343
+ * becomes an authoritative write/read fence, including for active runs.
3344
+ */
3345
+ async enforceRetention(request) {
3346
+ if (!Number.isSafeInteger(request.retentionMs) || request.retentionMs <= 0) throw new Error("world-celld retentionMs must be a positive safe integer");
3347
+ if (!Number.isSafeInteger(request.queueShards) || request.queueShards <= 0) throw new Error("world-celld retention queueShards must be a positive safe integer");
3348
+ if (!Number.isSafeInteger(request.scheduledTime) || request.scheduledTime < 0) throw new Error("world-celld retention scheduledTime must be a non-negative safe integer");
3349
+ const result = await this.ctx.storage.transaction(async (txn) => {
3350
+ const values = await txn.get([
3351
+ CLEANUP_RECORD_KEY,
3352
+ TOMBSTONE_KEY,
3353
+ RUN_QUEUE_SHARDS_KEY,
3354
+ "run"
3355
+ ]);
3356
+ const tombstone = values.get(TOMBSTONE_KEY);
3357
+ if (tombstone) return {
3358
+ state: "expired",
3359
+ cleanup: cleanupFromTombstone(tombstone)
3360
+ };
3361
+ const run = values.get("run");
3362
+ if (!run) return {
3363
+ state: "missing",
3364
+ cleanup: null
3365
+ };
3366
+ const dueAtMs = run.createdAt.getTime() + request.retentionMs;
3367
+ if (!Number.isSafeInteger(dueAtMs) || dueAtMs > 864e13) throw new Error(`world-celld retention deadline is out of range for ${run.runId}`);
3368
+ if (dueAtMs > request.scheduledTime) return {
3369
+ state: "not-due",
3370
+ cleanup: null
3371
+ };
3372
+ const dueAt = new Date(dueAtMs);
3373
+ const existing = values.get(CLEANUP_RECORD_KEY);
3374
+ const storedQueueShards = values.get(RUN_QUEUE_SHARDS_KEY);
3375
+ const queueShards = Number.isSafeInteger(storedQueueShards) && storedQueueShards > 0 ? storedQueueShards : request.queueShards;
3376
+ let cleanup = existing;
3377
+ if (!cleanup) {
3378
+ cleanup = cleanupRecord(run, dueAt, queueShards, "maximum-age");
3379
+ await txn.put("run", WorkflowRunSchema.parse({
3380
+ ...run,
3381
+ expiredAt: dueAt
3382
+ }));
3383
+ await txn.put(CLEANUP_RECORD_KEY, cleanup);
3384
+ } else if (cleanup.phase === "retained" && dueAtMs < cleanup.dueAt.getTime()) {
3385
+ const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
3386
+ cleanup = {
3387
+ ...cleanup,
3388
+ status: run.status,
3389
+ completedAt: terminal?.completedAt,
3390
+ terminalStatus: terminal?.status,
3391
+ reason: "maximum-age",
3392
+ dueAt,
3393
+ generation: cleanup.generation + 1
3394
+ };
3395
+ await txn.put("run", WorkflowRunSchema.parse({
3396
+ ...run,
3397
+ expiredAt: dueAt
3398
+ }));
3399
+ await txn.put(CLEANUP_RECORD_KEY, cleanup);
3400
+ }
3401
+ await txn.setAlarm(this.now() + 1);
3402
+ return {
3403
+ state: cleanup.dueAt.getTime() <= this.now() ? "expired" : "scheduled",
3404
+ cleanup
3405
+ };
3406
+ });
3407
+ if (result.state === "scheduled" || result.state === "expired") {
3408
+ for (let page = 0; page < 2; page++) {
3409
+ const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
3410
+ if (cleanup?.phase !== "retained" && cleanup?.phase !== "index") break;
3411
+ await this.executeScheduledCleanupPage();
3412
+ }
3413
+ return {
3414
+ ...result,
3415
+ cleanup: await this.getCleanupStatus() ?? result.cleanup
3416
+ };
3417
+ }
3418
+ return result;
3419
+ }
3054
3420
  async cleanupNow(request) {
3055
3421
  if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
3056
3422
  ...request,
@@ -3094,7 +3460,6 @@ var WorkflowRunDO = class extends DurableObject {
3094
3460
  workflowIndex() {
3095
3461
  return createWorkflowIndex({
3096
3462
  runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
3097
- runFences: this.namespace("WORKFLOW_RUN_FENCES"),
3098
3463
  hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
3099
3464
  hookIds: this.namespace("WORKFLOW_HOOK_IDS")
3100
3465
  });
@@ -3115,7 +3480,17 @@ var WorkflowRunDO = class extends DurableObject {
3115
3480
  });
3116
3481
  }
3117
3482
  async executeScheduledCleanupPage() {
3118
- const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
3483
+ const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
3484
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
3485
+ const terminalCleanup = values.get(TERMINAL_CLEANUP_KEY);
3486
+ if (cleanup && cleanup.phase !== "tombstoned" && cleanup.dueAt.getTime() <= this.now()) {
3487
+ try {
3488
+ await this.executeCleanup(cleanup);
3489
+ } catch (error) {
3490
+ await this.recordCleanupFailure(error, cleanup);
3491
+ }
3492
+ return;
3493
+ }
3119
3494
  if (terminalCleanup) {
3120
3495
  try {
3121
3496
  await this.executeTerminalCleanup(terminalCleanup);
@@ -3124,7 +3499,6 @@ var WorkflowRunDO = class extends DurableObject {
3124
3499
  }
3125
3500
  return;
3126
3501
  }
3127
- const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
3128
3502
  if (!cleanup || cleanup.phase === "tombstoned") {
3129
3503
  await this.ctx.storage.deleteAlarm();
3130
3504
  return;
@@ -3174,8 +3548,7 @@ var WorkflowRunDO = class extends DurableObject {
3174
3548
  hooks: page.map((hook) => ({
3175
3549
  hookId: hook.hookId,
3176
3550
  token: hook.token
3177
- })),
3178
- terminal: true
3551
+ }))
3179
3552
  });
3180
3553
  await this.ctx.storage.transaction(async (txn) => {
3181
3554
  const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3200,8 +3573,7 @@ var WorkflowRunDO = class extends DurableObject {
3200
3573
  const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
3201
3574
  await this.workflowIndex().releaseHookIndexes({
3202
3575
  runId: cleanup.runId,
3203
- hooks: page.map(([, reference]) => reference),
3204
- terminal: true
3576
+ hooks: page.map(([, reference]) => reference)
3205
3577
  });
3206
3578
  await this.ctx.storage.transaction(async (txn) => {
3207
3579
  const current = await txn.get(TERMINAL_CLEANUP_KEY);
@@ -3311,33 +3683,56 @@ var WorkflowRunDO = class extends DurableObject {
3311
3683
  }
3312
3684
  async deleteQueuePage(cleanup) {
3313
3685
  const queues = this.namespace("WORKFLOW_QUEUE");
3314
- let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3686
+ const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3315
3687
  queueShard: 0,
3316
3688
  queueShardDeleted: 0
3317
3689
  };
3318
- let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
3319
- let expectedGeneration = cleanup.generation;
3320
- while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
3690
+ if (progress.pendingAck) {
3691
+ const { queueShard, receipt } = progress.pendingAck;
3692
+ if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
3693
+ await this.ctx.storage.transaction(async (txn) => {
3694
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3695
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3696
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
3697
+ if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
3698
+ await txn.put(CLEANUP_RECORD_KEY, {
3699
+ ...current,
3700
+ generation: current.generation + 1,
3701
+ attempts: 0,
3702
+ lastError: void 0
3703
+ });
3704
+ await txn.put(CLEANUP_PROGRESS_KEY, {
3705
+ queueShard: queueShard + 1,
3706
+ queueShardDeleted: 0
3707
+ });
3708
+ await txn.setAlarm(this.now() + 1);
3709
+ });
3710
+ return;
3711
+ }
3712
+ if (progress.queueShard < cleanup.queueShards) {
3321
3713
  const requestedShard = progress.queueShard;
3322
3714
  const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
3323
- const reconciled = await this.ctx.storage.transaction(async (txn) => {
3715
+ await this.ctx.storage.transaction(async (txn) => {
3324
3716
  const current = await txn.get(CLEANUP_RECORD_KEY);
3325
- if (current?.phase !== "queues") return null;
3326
- if (current.generation !== expectedGeneration) return { stale: true };
3717
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3327
3718
  const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3328
3719
  queueShard: 0,
3329
3720
  queueShardDeleted: 0
3330
3721
  };
3331
- if (persisted.queueShard !== requestedShard) return { stale: true };
3332
- if (result.deleted < persisted.queueShardDeleted) return { stale: true };
3722
+ if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
3723
+ if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
3724
+ const delta = result.deleted - persisted.queueShardDeleted;
3333
3725
  const next = result.done ? {
3334
- queueShard: persisted.queueShard + 1,
3335
- queueShardDeleted: 0
3726
+ queueShard: requestedShard,
3727
+ queueShardDeleted: result.deleted,
3728
+ pendingAck: {
3729
+ queueShard: requestedShard,
3730
+ receipt: result.receipt
3731
+ }
3336
3732
  } : {
3337
- ...persisted,
3733
+ queueShard: requestedShard,
3338
3734
  queueShardDeleted: result.deleted
3339
3735
  };
3340
- const delta = result.deleted - persisted.queueShardDeleted;
3341
3736
  await txn.put(CLEANUP_RECORD_KEY, {
3342
3737
  ...current,
3343
3738
  deletedQueueMessages: current.deletedQueueMessages + delta,
@@ -3347,27 +3742,17 @@ var WorkflowRunDO = class extends DurableObject {
3347
3742
  });
3348
3743
  await txn.put(CLEANUP_PROGRESS_KEY, next);
3349
3744
  await txn.setAlarm(this.now() + 1);
3350
- return {
3351
- stale: false,
3352
- progress: next,
3353
- generation: current.generation + 1,
3354
- applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
3355
- };
3356
3745
  });
3357
- if (!reconciled) return;
3358
- if (reconciled.stale) return;
3359
- progress = reconciled.progress;
3360
- expectedGeneration = reconciled.generation;
3361
- if (!reconciled.applied && progress.queueShard === requestedShard) break;
3362
- if (!result.done && progress.queueShard === requestedShard) break;
3746
+ return;
3363
3747
  }
3364
3748
  await this.ctx.storage.transaction(async (txn) => {
3365
3749
  const current = await txn.get(CLEANUP_RECORD_KEY);
3366
- if (current?.phase !== "queues") return;
3367
- if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3750
+ if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
3751
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3368
3752
  queueShard: 0,
3369
3753
  queueShardDeleted: 0
3370
- }).queueShard >= current.queueShards) {
3754
+ };
3755
+ if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
3371
3756
  await txn.put(CLEANUP_RECORD_KEY, {
3372
3757
  ...current,
3373
3758
  phase: "payload",
@@ -3400,14 +3785,16 @@ var WorkflowRunDO = class extends DurableObject {
3400
3785
  await txn.setAlarm(this.now() + 1);
3401
3786
  return false;
3402
3787
  }
3403
- await txn.put(CLEANUP_RECORD_KEY, {
3788
+ const completed = {
3404
3789
  ...cleanup,
3405
3790
  phase: "tombstoned",
3406
3791
  generation: cleanup.generation + 1,
3407
3792
  attempts: 0,
3408
3793
  lastError: void 0,
3409
3794
  tombstonedAt: now
3410
- });
3795
+ };
3796
+ await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
3797
+ await txn.delete(CLEANUP_RECORD_KEY);
3411
3798
  await txn.deleteAlarm();
3412
3799
  return true;
3413
3800
  });
@@ -3472,7 +3859,6 @@ async function startHarness(options = {}) {
3472
3859
  runs: WorkflowRunDO,
3473
3860
  streams: StreamDO,
3474
3861
  "run-catalog": RunCatalogDO,
3475
- "run-fences": RunFenceDO,
3476
3862
  "hook-tokens": HookTokenDO,
3477
3863
  "hook-ids": HookIdDO,
3478
3864
  queue: QueueDO,
@@ -3483,7 +3869,6 @@ async function startHarness(options = {}) {
3483
3869
  WORKFLOW_DB: fleet.namespace("runs"),
3484
3870
  WORKFLOW_STREAMS: fleet.namespace("streams"),
3485
3871
  WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
3486
- WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
3487
3872
  WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
3488
3873
  WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
3489
3874
  WORKFLOW_QUEUE: fleet.namespace("queue")
@@ -3492,7 +3877,6 @@ async function startHarness(options = {}) {
3492
3877
  WORKFLOW_DB: fleet.namespace("runs"),
3493
3878
  WORKFLOW_STREAMS: fleet.namespace("streams"),
3494
3879
  WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
3495
- WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
3496
3880
  WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
3497
3881
  WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
3498
3882
  WORKFLOW_QUEUE: fleet.namespace("queue"),