@ewhauser/world-celld 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +70 -28
- package/celld-worker/README.md +25 -3
- package/celld-worker/worker.ts +0 -1
- package/celld-worker/wrangler.jsonc +12 -2
- package/dist/index.d.ts +36 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +21 -21
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +625 -241
- package/dist/testing.js.map +1 -1
- package/dist/{util-DmO0XkFw.js → util-ya3V_FTA.js} +32 -19
- package/dist/util-ya3V_FTA.js.map +1 -0
- package/dist/worker.d.ts +95 -28
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +768 -291
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
- package/dist/util-DmO0XkFw.js.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { A as validateStreamReadRequest, E as encodeStreamReadResult, O as encodeStreamWriteResult, S as STREAM_BATCH_CONTENT_TYPE, c as HOOK_CLAIM_LEASE_MS, d as MAX_QUEUE_DELIVERY_TIMEOUT_MS, f as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, g as MAX_STREAM_CHUNK_BYTES, h as MAX_STREAM_BATCH_BYTES, i as parse, j as validateStreamWriteChunks, k as normalizeStreamError, l as LIFECYCLE_COMPACTION_RETRY_MS, m as QUEUE_INFLIGHT_GRACE_MS, n as rpcParse, o as createWorkflowIndex, p as QUEUE_FENCE_GRACE_MS, r as rpcStringify, t as compact, v as MAX_STREAM_LONG_POLL_MS, w as decodeStreamWriteBatch, y as MAX_STREAM_READ_BYTES } from "./util-ya3V_FTA.js";
|
|
2
2
|
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
//#region src/testing/fake-cell.ts
|
|
@@ -453,7 +453,6 @@ const WORLD_VERSION = "0.1.0";
|
|
|
453
453
|
function workflowIndex(env) {
|
|
454
454
|
return createWorkflowIndex({
|
|
455
455
|
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
456
|
-
runFences: env.WORKFLOW_RUN_FENCES,
|
|
457
456
|
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
458
457
|
hookIds: env.WORKFLOW_HOOK_IDS
|
|
459
458
|
});
|
|
@@ -463,6 +462,7 @@ const BINDINGS = {
|
|
|
463
462
|
env: "WORKFLOW_DB",
|
|
464
463
|
methods: /* @__PURE__ */ new Set([
|
|
465
464
|
"applyEvent",
|
|
465
|
+
"getLifecycleStatus",
|
|
466
466
|
"getRun",
|
|
467
467
|
"getStep",
|
|
468
468
|
"getEvent",
|
|
@@ -505,7 +505,8 @@ const BINDINGS = {
|
|
|
505
505
|
"redriveDeadLetter",
|
|
506
506
|
"purgeDeadLetters",
|
|
507
507
|
"rearmAlarm",
|
|
508
|
-
"expireRun"
|
|
508
|
+
"expireRun",
|
|
509
|
+
"acknowledgeExpireRun"
|
|
509
510
|
])
|
|
510
511
|
}
|
|
511
512
|
};
|
|
@@ -614,7 +615,7 @@ function createRouter(env) {
|
|
|
614
615
|
let result;
|
|
615
616
|
const operation = `${parts[2]}.${parts[3]}`;
|
|
616
617
|
if (operation === "runs.list") result = await index.listRuns(args[0]);
|
|
617
|
-
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
|
|
618
|
+
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
|
|
618
619
|
else if (operation === "runs.expire") result = await index.expireRun(args[0]);
|
|
619
620
|
else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
|
|
620
621
|
else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
|
|
@@ -740,26 +741,32 @@ function ownerFromSerializedHook(raw) {
|
|
|
740
741
|
function indexKey(prefix, value) {
|
|
741
742
|
return `${prefix}:${encodeURIComponent(value)}`;
|
|
742
743
|
}
|
|
743
|
-
/** Compact shard-local fence for every hook owned by one terminal or expired run. */
|
|
744
|
-
function shardRunFenceKey(runId) {
|
|
745
|
-
return `runfence:${encodeURIComponent(runId)}`;
|
|
746
|
-
}
|
|
747
744
|
async function runIsActive(env, runId) {
|
|
748
|
-
const namespace = env.
|
|
749
|
-
if (!namespace) throw new Error("world-celld hook index missing
|
|
750
|
-
return await namespace.get(namespace.idFromName(
|
|
745
|
+
const namespace = env.WORKFLOW_DB;
|
|
746
|
+
if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
|
|
747
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
|
|
751
748
|
}
|
|
752
749
|
//#endregion
|
|
753
750
|
//#region src/worker/durable-objects/HookIdDO.ts
|
|
754
751
|
const MAX_RELEASE_BATCH$1 = 64;
|
|
752
|
+
const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
|
|
753
|
+
function pad$3(ms) {
|
|
754
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
755
|
+
}
|
|
755
756
|
function recordKey$1(hookId) {
|
|
756
757
|
return indexKey("hookid", hookId);
|
|
757
758
|
}
|
|
758
759
|
function claimKey$1(hookId) {
|
|
759
760
|
return indexKey("claim", hookId);
|
|
760
761
|
}
|
|
762
|
+
function claimDeadlineKey$1(hookId, expiresAt) {
|
|
763
|
+
return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
|
|
764
|
+
}
|
|
761
765
|
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
762
766
|
var HookIdDO = class extends DurableObject {
|
|
767
|
+
now() {
|
|
768
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
769
|
+
}
|
|
763
770
|
async get(hookId) {
|
|
764
771
|
const raw = await this.ctx.storage.get(recordKey$1(hookId));
|
|
765
772
|
if (!raw) return null;
|
|
@@ -776,17 +783,33 @@ var HookIdDO = class extends DurableObject {
|
|
|
776
783
|
claimed: false,
|
|
777
784
|
holder: ownerFromSerializedHook(indexed)
|
|
778
785
|
};
|
|
779
|
-
|
|
786
|
+
let existing = values.get(claim);
|
|
787
|
+
const now = this.now();
|
|
788
|
+
if (existing && existing.expiresAt <= now) {
|
|
789
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
790
|
+
existing = void 0;
|
|
791
|
+
}
|
|
780
792
|
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
781
793
|
claimed: false,
|
|
782
794
|
holder: existing.owner
|
|
783
795
|
};
|
|
784
796
|
if (existing === void 0 || existing.claimId !== proposedClaimId) {
|
|
797
|
+
if (existing) await txn.delete(existing.deadlineKey);
|
|
798
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
799
|
+
const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
|
|
785
800
|
await txn.put(claim, {
|
|
786
801
|
owner,
|
|
787
802
|
claimId: proposedClaimId,
|
|
788
|
-
reservedAt:
|
|
803
|
+
reservedAt: now,
|
|
804
|
+
expiresAt,
|
|
805
|
+
deadlineKey
|
|
806
|
+
});
|
|
807
|
+
await txn.put(deadlineKey, {
|
|
808
|
+
hookId,
|
|
809
|
+
claimId: proposedClaimId,
|
|
810
|
+
expiresAt
|
|
789
811
|
});
|
|
812
|
+
await this.armAtMost(txn, expiresAt);
|
|
790
813
|
return {
|
|
791
814
|
claimed: true,
|
|
792
815
|
claimId: proposedClaimId
|
|
@@ -803,30 +826,24 @@ var HookIdDO = class extends DurableObject {
|
|
|
803
826
|
await this.releaseBatch([{
|
|
804
827
|
hookId,
|
|
805
828
|
owner
|
|
806
|
-
}]
|
|
829
|
+
}]);
|
|
807
830
|
return { stored: false };
|
|
808
831
|
}
|
|
809
832
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
810
833
|
const record = recordKey$1(hookId);
|
|
811
834
|
const claim = claimKey$1(hookId);
|
|
812
|
-
const
|
|
813
|
-
const values = await txn.get([
|
|
814
|
-
record,
|
|
815
|
-
claim,
|
|
816
|
-
runFence
|
|
817
|
-
]);
|
|
818
|
-
if (values.has(runFence)) {
|
|
819
|
-
const existingClaim = values.get(claim);
|
|
820
|
-
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
821
|
-
return { stored: false };
|
|
822
|
-
}
|
|
835
|
+
const values = await txn.get([record, claim]);
|
|
823
836
|
const indexed = values.get(record);
|
|
824
837
|
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
|
|
825
838
|
const existingClaim = values.get(claim);
|
|
839
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
840
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
841
|
+
return { stored: false };
|
|
842
|
+
}
|
|
826
843
|
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
|
|
827
844
|
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
|
|
828
845
|
await txn.put(record, serializedHook);
|
|
829
|
-
if (existingClaim !== void 0) await txn.delete(claim);
|
|
846
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
830
847
|
return { stored: true };
|
|
831
848
|
});
|
|
832
849
|
}
|
|
@@ -834,40 +851,94 @@ var HookIdDO = class extends DurableObject {
|
|
|
834
851
|
await this.ctx.storage.transaction(async (txn) => {
|
|
835
852
|
const key = claimKey$1(hookId);
|
|
836
853
|
const claim = await txn.get(key);
|
|
837
|
-
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
854
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
838
855
|
});
|
|
839
856
|
}
|
|
840
|
-
async releaseBatch(hooks
|
|
857
|
+
async releaseBatch(hooks) {
|
|
841
858
|
if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
|
|
842
859
|
if (hooks.length === 0) return { deleted: 0 };
|
|
843
860
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
844
861
|
const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
|
|
845
862
|
const values = await txn.get(keys);
|
|
846
863
|
const deletes = [];
|
|
864
|
+
const deadlineDeletes = [];
|
|
847
865
|
for (const { hookId, owner } of hooks) {
|
|
848
866
|
const record = recordKey$1(hookId);
|
|
849
867
|
const claim = claimKey$1(hookId);
|
|
850
868
|
const indexed = values.get(record);
|
|
851
869
|
const existingClaim = values.get(claim);
|
|
852
870
|
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
853
|
-
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner))
|
|
871
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
872
|
+
deletes.push(claim);
|
|
873
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
874
|
+
}
|
|
854
875
|
}
|
|
855
|
-
|
|
856
|
-
|
|
876
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
877
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
878
|
+
return { deleted };
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
async alarm() {
|
|
882
|
+
try {
|
|
883
|
+
await this.compactClaims();
|
|
884
|
+
} catch {
|
|
885
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
async armAtMost(storage, at) {
|
|
889
|
+
const current = await storage.getAlarm();
|
|
890
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
891
|
+
}
|
|
892
|
+
async compactClaims() {
|
|
893
|
+
const now = this.now();
|
|
894
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
895
|
+
const due = await txn.list({
|
|
896
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
897
|
+
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
|
|
898
|
+
limit: 129
|
|
899
|
+
});
|
|
900
|
+
const page = Array.from(due).slice(0, 128);
|
|
901
|
+
const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
|
|
902
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
903
|
+
const key = claimKey$1(item.hookId);
|
|
904
|
+
const claim = claims.get(key);
|
|
905
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
906
|
+
});
|
|
907
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
908
|
+
if (due.size > 128) {
|
|
909
|
+
await txn.setAlarm(now + 1);
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
912
|
+
const nextAt = (await txn.list({
|
|
913
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
914
|
+
limit: 1
|
|
915
|
+
})).values().next().value?.expiresAt;
|
|
916
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
917
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
857
918
|
});
|
|
858
919
|
}
|
|
859
920
|
};
|
|
860
921
|
//#endregion
|
|
861
922
|
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
862
923
|
const MAX_RELEASE_BATCH = 64;
|
|
924
|
+
const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
|
|
925
|
+
function pad$2(ms) {
|
|
926
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
927
|
+
}
|
|
863
928
|
function recordKey(token) {
|
|
864
929
|
return indexKey("hook", token);
|
|
865
930
|
}
|
|
866
931
|
function claimKey(token) {
|
|
867
932
|
return indexKey("claim", token);
|
|
868
933
|
}
|
|
934
|
+
function claimDeadlineKey(token, expiresAt) {
|
|
935
|
+
return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
|
|
936
|
+
}
|
|
869
937
|
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
870
938
|
var HookTokenDO = class extends DurableObject {
|
|
939
|
+
now() {
|
|
940
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
941
|
+
}
|
|
871
942
|
async get(token) {
|
|
872
943
|
const raw = await this.ctx.storage.get(recordKey(token));
|
|
873
944
|
if (!raw) return null;
|
|
@@ -884,17 +955,32 @@ var HookTokenDO = class extends DurableObject {
|
|
|
884
955
|
claimed: false,
|
|
885
956
|
holder: ownerFromSerializedHook(indexed)
|
|
886
957
|
};
|
|
887
|
-
|
|
958
|
+
let existing = values.get(claim);
|
|
959
|
+
const now = this.now();
|
|
960
|
+
if (existing && existing.expiresAt <= now) {
|
|
961
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
962
|
+
existing = void 0;
|
|
963
|
+
}
|
|
888
964
|
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
889
965
|
claimed: false,
|
|
890
966
|
holder: existing.owner
|
|
891
967
|
};
|
|
892
968
|
if (existing === void 0) {
|
|
969
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
970
|
+
const deadlineKey = claimDeadlineKey(token, expiresAt);
|
|
893
971
|
await txn.put(claim, {
|
|
894
972
|
owner,
|
|
895
973
|
claimId: proposedClaimId,
|
|
896
|
-
reservedAt:
|
|
974
|
+
reservedAt: now,
|
|
975
|
+
expiresAt,
|
|
976
|
+
deadlineKey
|
|
977
|
+
});
|
|
978
|
+
await txn.put(deadlineKey, {
|
|
979
|
+
token,
|
|
980
|
+
claimId: proposedClaimId,
|
|
981
|
+
expiresAt
|
|
897
982
|
});
|
|
983
|
+
await this.armAtMost(txn, expiresAt);
|
|
898
984
|
return {
|
|
899
985
|
claimed: true,
|
|
900
986
|
claimId: proposedClaimId
|
|
@@ -911,30 +997,24 @@ var HookTokenDO = class extends DurableObject {
|
|
|
911
997
|
await this.releaseBatch([{
|
|
912
998
|
token,
|
|
913
999
|
owner
|
|
914
|
-
}]
|
|
1000
|
+
}]);
|
|
915
1001
|
return { stored: false };
|
|
916
1002
|
}
|
|
917
1003
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
918
1004
|
const record = recordKey(token);
|
|
919
1005
|
const claim = claimKey(token);
|
|
920
|
-
const
|
|
921
|
-
const values = await txn.get([
|
|
922
|
-
record,
|
|
923
|
-
claim,
|
|
924
|
-
runFence
|
|
925
|
-
]);
|
|
926
|
-
if (values.has(runFence)) {
|
|
927
|
-
const existingClaim = values.get(claim);
|
|
928
|
-
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
929
|
-
return { stored: false };
|
|
930
|
-
}
|
|
1006
|
+
const values = await txn.get([record, claim]);
|
|
931
1007
|
const indexed = values.get(record);
|
|
932
1008
|
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
933
1009
|
const existingClaim = values.get(claim);
|
|
1010
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
1011
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
1012
|
+
return { stored: false };
|
|
1013
|
+
}
|
|
934
1014
|
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
935
1015
|
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
|
|
936
1016
|
await txn.put(record, serializedHook);
|
|
937
|
-
if (existingClaim !== void 0) await txn.delete(claim);
|
|
1017
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
938
1018
|
return { stored: true };
|
|
939
1019
|
});
|
|
940
1020
|
}
|
|
@@ -942,26 +1022,70 @@ var HookTokenDO = class extends DurableObject {
|
|
|
942
1022
|
await this.ctx.storage.transaction(async (txn) => {
|
|
943
1023
|
const key = claimKey(token);
|
|
944
1024
|
const claim = await txn.get(key);
|
|
945
|
-
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
1025
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
946
1026
|
});
|
|
947
1027
|
}
|
|
948
|
-
async releaseBatch(hooks
|
|
1028
|
+
async releaseBatch(hooks) {
|
|
949
1029
|
if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
|
|
950
1030
|
if (hooks.length === 0) return { deleted: 0 };
|
|
951
1031
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
952
1032
|
const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
|
|
953
1033
|
const values = await txn.get(keys);
|
|
954
1034
|
const deletes = [];
|
|
1035
|
+
const deadlineDeletes = [];
|
|
955
1036
|
for (const { token, owner } of hooks) {
|
|
956
1037
|
const record = recordKey(token);
|
|
957
1038
|
const claim = claimKey(token);
|
|
958
1039
|
const indexed = values.get(record);
|
|
959
1040
|
const existingClaim = values.get(claim);
|
|
960
1041
|
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
961
|
-
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner))
|
|
1042
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
1043
|
+
deletes.push(claim);
|
|
1044
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
1045
|
+
}
|
|
962
1046
|
}
|
|
963
|
-
|
|
964
|
-
|
|
1047
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
1048
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
1049
|
+
return { deleted };
|
|
1050
|
+
});
|
|
1051
|
+
}
|
|
1052
|
+
async alarm() {
|
|
1053
|
+
try {
|
|
1054
|
+
await this.compactClaims();
|
|
1055
|
+
} catch {
|
|
1056
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
async armAtMost(storage, at) {
|
|
1060
|
+
const current = await storage.getAlarm();
|
|
1061
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1062
|
+
}
|
|
1063
|
+
async compactClaims() {
|
|
1064
|
+
const now = this.now();
|
|
1065
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1066
|
+
const due = await txn.list({
|
|
1067
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1068
|
+
end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
|
|
1069
|
+
limit: 129
|
|
1070
|
+
});
|
|
1071
|
+
const page = Array.from(due).slice(0, 128);
|
|
1072
|
+
const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
|
|
1073
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
1074
|
+
const key = claimKey(item.token);
|
|
1075
|
+
const claim = claims.get(key);
|
|
1076
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
1077
|
+
});
|
|
1078
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1079
|
+
if (due.size > 128) {
|
|
1080
|
+
await txn.setAlarm(now + 1);
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
const nextAt = (await txn.list({
|
|
1084
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1085
|
+
limit: 1
|
|
1086
|
+
})).values().next().value?.expiresAt;
|
|
1087
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1088
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
965
1089
|
});
|
|
966
1090
|
}
|
|
967
1091
|
};
|
|
@@ -976,8 +1100,6 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
|
976
1100
|
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
977
1101
|
const DEFAULT_MAX_INFLIGHT = 5;
|
|
978
1102
|
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
979
|
-
/** Grace period past the delivery timeout before a claim counts as lost. */
|
|
980
|
-
const INFLIGHT_GRACE_MS = 3e4;
|
|
981
1103
|
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
982
1104
|
const MIN_ALARM_DELAY_MS = 1;
|
|
983
1105
|
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
@@ -986,14 +1108,15 @@ const STORAGE_BATCH_SIZE$2 = 128;
|
|
|
986
1108
|
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
987
1109
|
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
988
1110
|
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
989
|
-
|
|
1111
|
+
const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
|
|
1112
|
+
function pad$1(ms) {
|
|
990
1113
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
991
1114
|
}
|
|
992
1115
|
function dueKey(atMs, messageId) {
|
|
993
|
-
return `due:${pad(atMs)}:${messageId}`;
|
|
1116
|
+
return `due:${pad$1(atMs)}:${messageId}`;
|
|
994
1117
|
}
|
|
995
1118
|
function inflightKey(deadlineMs, messageId) {
|
|
996
|
-
return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
|
|
1119
|
+
return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
|
|
997
1120
|
}
|
|
998
1121
|
function deadlineFromInflightKey(key) {
|
|
999
1122
|
return Number.parseInt(key.slice(18, 31), 10);
|
|
@@ -1004,6 +1127,9 @@ function runReferenceKey(runId, messageId) {
|
|
|
1004
1127
|
function expiredRunKey(runId) {
|
|
1005
1128
|
return `expired-run:${runId}`;
|
|
1006
1129
|
}
|
|
1130
|
+
function expiredRunGcKey(runId, compactAt) {
|
|
1131
|
+
return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1132
|
+
}
|
|
1007
1133
|
function boundedLimit$1(value, fallback, maximum) {
|
|
1008
1134
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1009
1135
|
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
@@ -1054,16 +1180,29 @@ var QueueDO = class extends DurableObject {
|
|
|
1054
1180
|
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
1055
1181
|
return configured;
|
|
1056
1182
|
}
|
|
1183
|
+
#deliveryTimeoutMs() {
|
|
1184
|
+
const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
1185
|
+
if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
|
|
1186
|
+
return configured;
|
|
1187
|
+
}
|
|
1188
|
+
async #authoritativeRunExpired(runId) {
|
|
1189
|
+
const namespace = this.env?.WORKFLOW_DB;
|
|
1190
|
+
if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
|
|
1191
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
|
|
1192
|
+
}
|
|
1193
|
+
async #withRunMutation(runId, onExpired, onAllowed) {
|
|
1194
|
+
const expired = await this.#authoritativeRunExpired(runId);
|
|
1195
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1196
|
+
if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
|
|
1197
|
+
return await onAllowed(txn);
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
1057
1200
|
async enqueue(request) {
|
|
1058
1201
|
this.#maxInflight();
|
|
1202
|
+
this.#deliveryTimeoutMs();
|
|
1059
1203
|
const storage = this.ctx.storage;
|
|
1060
1204
|
const now = this.#now();
|
|
1061
|
-
|
|
1062
|
-
if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
|
|
1063
|
-
ok: false,
|
|
1064
|
-
code: "RUN_EXPIRED",
|
|
1065
|
-
message: `Workflow run "${request.runId}" has expired`
|
|
1066
|
-
};
|
|
1205
|
+
const persist = async (txn) => {
|
|
1067
1206
|
const pinned = await txn.get("cfg");
|
|
1068
1207
|
if (!pinned) await txn.put("cfg", {
|
|
1069
1208
|
queueShards: request.config.queueShards,
|
|
@@ -1115,7 +1254,12 @@ var QueueDO = class extends DurableObject {
|
|
|
1115
1254
|
messageId: row.messageId,
|
|
1116
1255
|
deduped: false
|
|
1117
1256
|
};
|
|
1118
|
-
}
|
|
1257
|
+
};
|
|
1258
|
+
return request.runId ? await this.#withRunMutation(request.runId, async () => ({
|
|
1259
|
+
ok: false,
|
|
1260
|
+
code: "RUN_EXPIRED",
|
|
1261
|
+
message: `Workflow run "${request.runId}" has expired`
|
|
1262
|
+
}), persist) : await storage.transaction(persist);
|
|
1119
1263
|
}
|
|
1120
1264
|
async alarm() {
|
|
1121
1265
|
let claimed = [];
|
|
@@ -1129,7 +1273,10 @@ var QueueDO = class extends DurableObject {
|
|
|
1129
1273
|
};
|
|
1130
1274
|
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
1131
1275
|
else await attempt();
|
|
1132
|
-
if (failure !== null)
|
|
1276
|
+
if (failure !== null) {
|
|
1277
|
+
await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1278
|
+
throw failure;
|
|
1279
|
+
}
|
|
1133
1280
|
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
1134
1281
|
}
|
|
1135
1282
|
/**
|
|
@@ -1141,9 +1288,13 @@ var QueueDO = class extends DurableObject {
|
|
|
1141
1288
|
const now = this.#now();
|
|
1142
1289
|
const maxInflight = this.#maxInflight();
|
|
1143
1290
|
return await storage.transaction(async (txn) => {
|
|
1291
|
+
if (await this.#compactExpiredRunFences(txn, now)) {
|
|
1292
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1293
|
+
return [];
|
|
1294
|
+
}
|
|
1144
1295
|
const expiredPage = await txn.list({
|
|
1145
1296
|
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1146
|
-
end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
|
|
1297
|
+
end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
|
|
1147
1298
|
limit: 129
|
|
1148
1299
|
});
|
|
1149
1300
|
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
@@ -1175,17 +1326,17 @@ var QueueDO = class extends DurableObject {
|
|
|
1175
1326
|
if (inflightCount < maxInflight) {
|
|
1176
1327
|
const due = await txn.list({
|
|
1177
1328
|
prefix: "due:",
|
|
1178
|
-
end: `due:${pad(now + 1)}`,
|
|
1329
|
+
end: `due:${pad$1(now + 1)}`,
|
|
1179
1330
|
limit: maxInflight - inflightCount
|
|
1180
1331
|
});
|
|
1181
|
-
const timeoutMs = this.#
|
|
1332
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1182
1333
|
const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
1183
1334
|
await deleteMany$1(txn, Array.from(due.keys()));
|
|
1184
1335
|
const claimWrites = [];
|
|
1185
1336
|
for (const [, messageId] of due) {
|
|
1186
1337
|
const row = dueRows.get(`msg:${messageId}`);
|
|
1187
1338
|
if (!row) continue;
|
|
1188
|
-
const claimKey = inflightKey(now + timeoutMs +
|
|
1339
|
+
const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
|
|
1189
1340
|
claimWrites.push([claimKey, messageId]);
|
|
1190
1341
|
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
1191
1342
|
messageId: row.messageId,
|
|
@@ -1205,9 +1356,13 @@ var QueueDO = class extends DurableObject {
|
|
|
1205
1356
|
}
|
|
1206
1357
|
async #deliver({ row, inflightKey: claimKey }) {
|
|
1207
1358
|
const storage = this.ctx.storage;
|
|
1208
|
-
const timeoutMs = this.#
|
|
1359
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1209
1360
|
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
1210
1361
|
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
1362
|
+
if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
|
|
1363
|
+
await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1211
1366
|
let response = null;
|
|
1212
1367
|
let transportError = null;
|
|
1213
1368
|
try {
|
|
@@ -1237,28 +1392,21 @@ var QueueDO = class extends DurableObject {
|
|
|
1237
1392
|
} catch {
|
|
1238
1393
|
timeoutSeconds = void 0;
|
|
1239
1394
|
}
|
|
1240
|
-
if (typeof timeoutSeconds === "number")
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
await txn.
|
|
1245
|
-
if (row.
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1395
|
+
if (typeof timeoutSeconds === "number") {
|
|
1396
|
+
const reschedule = async (txn) => {
|
|
1397
|
+
if (!await txn.delete(claimKey)) return;
|
|
1398
|
+
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
1399
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1400
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1401
|
+
messageId: row.messageId,
|
|
1402
|
+
dueKey: scheduleKey,
|
|
1403
|
+
idempotencyKey: row.idempotencyKey
|
|
1404
|
+
});
|
|
1249
1405
|
await this.#scheduleNextAlarm(txn, now);
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1255
|
-
messageId: row.messageId,
|
|
1256
|
-
dueKey: scheduleKey,
|
|
1257
|
-
idempotencyKey: row.idempotencyKey
|
|
1258
|
-
});
|
|
1259
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
1260
|
-
});
|
|
1261
|
-
else await this.#retry(row, claimKey, "HTTP 503");
|
|
1406
|
+
};
|
|
1407
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
|
|
1408
|
+
else await storage.transaction(reschedule);
|
|
1409
|
+
} else await this.#retry(row, claimKey, "HTTP 503");
|
|
1262
1410
|
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
1263
1411
|
await cancelUnusedResponseBody(response);
|
|
1264
1412
|
await this.#ack(row, claimKey);
|
|
@@ -1279,23 +1427,23 @@ var QueueDO = class extends DurableObject {
|
|
|
1279
1427
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1280
1428
|
});
|
|
1281
1429
|
}
|
|
1430
|
+
async #dropExpiredMessage(txn, row, claimKey, now) {
|
|
1431
|
+
if (!await txn.delete(claimKey)) return;
|
|
1432
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
1433
|
+
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1434
|
+
if (row.idempotencyKey) {
|
|
1435
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1436
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1437
|
+
}
|
|
1438
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
1439
|
+
}
|
|
1282
1440
|
async #retry(row, claimKey, reason) {
|
|
1283
1441
|
const storage = this.ctx.storage;
|
|
1284
1442
|
const now = this.#now();
|
|
1285
1443
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
1286
1444
|
const attempt = row.attempt + 1;
|
|
1287
|
-
|
|
1445
|
+
const persistRetry = async (txn) => {
|
|
1288
1446
|
if (!await txn.delete(claimKey)) return;
|
|
1289
|
-
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
1290
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
1291
|
-
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1292
|
-
if (row.idempotencyKey) {
|
|
1293
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1294
|
-
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1295
|
-
}
|
|
1296
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
1297
|
-
return;
|
|
1298
|
-
}
|
|
1299
1447
|
if (attempt >= maxAttempts) {
|
|
1300
1448
|
const dead = {
|
|
1301
1449
|
...row,
|
|
@@ -1303,7 +1451,7 @@ var QueueDO = class extends DurableObject {
|
|
|
1303
1451
|
lastError: reason,
|
|
1304
1452
|
failedAt: now
|
|
1305
1453
|
};
|
|
1306
|
-
const deadKey = `dlq:${pad(now)}:${row.messageId}`;
|
|
1454
|
+
const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
|
|
1307
1455
|
await txn.put(deadKey, dead);
|
|
1308
1456
|
await txn.delete(`msg:${row.messageId}`);
|
|
1309
1457
|
if (row.idempotencyKey) {
|
|
@@ -1331,7 +1479,32 @@ var QueueDO = class extends DurableObject {
|
|
|
1331
1479
|
idempotencyKey: row.idempotencyKey
|
|
1332
1480
|
});
|
|
1333
1481
|
await this.#scheduleNextAlarm(txn, now);
|
|
1334
|
-
}
|
|
1482
|
+
};
|
|
1483
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
|
|
1484
|
+
else await storage.transaction(persistRetry);
|
|
1485
|
+
}
|
|
1486
|
+
/** Return true when another due compaction page must run immediately. */
|
|
1487
|
+
async #compactExpiredRunFences(txn, now) {
|
|
1488
|
+
const due = await txn.list({
|
|
1489
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1490
|
+
end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
|
|
1491
|
+
limit: 129
|
|
1492
|
+
});
|
|
1493
|
+
const page = Array.from(due).slice(0, 128);
|
|
1494
|
+
if (page.length === 0) return false;
|
|
1495
|
+
const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
|
|
1496
|
+
const markers = await getMany$1(txn, markerKeys);
|
|
1497
|
+
const deletes = [];
|
|
1498
|
+
for (let index = 0; index < page.length; index++) {
|
|
1499
|
+
const [gcKey, item] = page[index];
|
|
1500
|
+
const markerKey = markerKeys[index];
|
|
1501
|
+
const marker = markers.get(markerKey);
|
|
1502
|
+
deletes.push(gcKey);
|
|
1503
|
+
if (marker?.compactAt !== item.compactAt) continue;
|
|
1504
|
+
deletes.push(markerKey);
|
|
1505
|
+
}
|
|
1506
|
+
await deleteMany$1(txn, deletes);
|
|
1507
|
+
return due.size > 128;
|
|
1335
1508
|
}
|
|
1336
1509
|
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
1337
1510
|
async #armAlarmAtMost(storage, atMs, now) {
|
|
@@ -1363,6 +1536,11 @@ var QueueDO = class extends DurableObject {
|
|
|
1363
1536
|
const deadline = deadlineFromInflightKey(key);
|
|
1364
1537
|
if (next === null || deadline < next) next = deadline;
|
|
1365
1538
|
}
|
|
1539
|
+
const expiryGc = await storage.list({
|
|
1540
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1541
|
+
limit: 1
|
|
1542
|
+
});
|
|
1543
|
+
for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
|
|
1366
1544
|
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
1367
1545
|
else await storage.deleteAlarm();
|
|
1368
1546
|
}
|
|
@@ -1401,37 +1579,42 @@ var QueueDO = class extends DurableObject {
|
|
|
1401
1579
|
async redriveDeadLetter(messageId) {
|
|
1402
1580
|
const storage = this.ctx.storage;
|
|
1403
1581
|
const now = this.#now();
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
await txn.put(
|
|
1422
|
-
const scheduleKey = dueKey(now, row.messageId);
|
|
1423
|
-
await txn.put(scheduleKey, row.messageId);
|
|
1424
|
-
await txn.delete(key);
|
|
1425
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1426
|
-
messageId: row.messageId,
|
|
1427
|
-
dueKey: scheduleKey,
|
|
1428
|
-
idempotencyKey: row.idempotencyKey
|
|
1429
|
-
});
|
|
1430
|
-
await this.#armAlarmAtMost(txn, now, now);
|
|
1431
|
-
return { ok: true };
|
|
1582
|
+
const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
|
|
1583
|
+
if (!found) return { ok: false };
|
|
1584
|
+
const [deadKey, candidate] = found;
|
|
1585
|
+
const redrive = async (txn) => {
|
|
1586
|
+
const dead = await txn.get(deadKey);
|
|
1587
|
+
if (!dead || dead.messageId !== messageId) return { ok: false };
|
|
1588
|
+
const { failedAt: _failedAt, ...rest } = dead;
|
|
1589
|
+
const row = {
|
|
1590
|
+
...rest,
|
|
1591
|
+
attempt: 0,
|
|
1592
|
+
lastError: void 0
|
|
1593
|
+
};
|
|
1594
|
+
if (row.idempotencyKey) {
|
|
1595
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1596
|
+
const holder = await txn.get(dedupKey);
|
|
1597
|
+
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
1598
|
+
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
1599
|
+
await txn.put(dedupKey, row.messageId);
|
|
1432
1600
|
}
|
|
1601
|
+
await txn.put(`msg:${row.messageId}`, row);
|
|
1602
|
+
const scheduleKey = dueKey(now, row.messageId);
|
|
1603
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1604
|
+
await txn.delete(deadKey);
|
|
1605
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1606
|
+
messageId: row.messageId,
|
|
1607
|
+
dueKey: scheduleKey,
|
|
1608
|
+
idempotencyKey: row.idempotencyKey
|
|
1609
|
+
});
|
|
1610
|
+
await this.#armAlarmAtMost(txn, now, now);
|
|
1611
|
+
return { ok: true };
|
|
1612
|
+
};
|
|
1613
|
+
const rejectExpired = async (txn) => {
|
|
1614
|
+
if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
|
|
1433
1615
|
return { ok: false };
|
|
1434
|
-
}
|
|
1616
|
+
};
|
|
1617
|
+
return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
|
|
1435
1618
|
}
|
|
1436
1619
|
async purgeDeadLetters() {
|
|
1437
1620
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
@@ -1481,27 +1664,75 @@ var QueueDO = class extends DurableObject {
|
|
|
1481
1664
|
count++;
|
|
1482
1665
|
}
|
|
1483
1666
|
await deleteMany$1(txn, keys);
|
|
1667
|
+
const done = references.size <= limit;
|
|
1668
|
+
const markerExpiredAt = existing?.expiredAt ?? expiredAt;
|
|
1484
1669
|
await txn.put(expiredRunKey(runId), {
|
|
1485
|
-
expiredAt,
|
|
1486
|
-
deleted: count
|
|
1670
|
+
expiredAt: markerExpiredAt,
|
|
1671
|
+
deleted: count,
|
|
1672
|
+
compactAt: existing?.compactAt ?? null
|
|
1487
1673
|
});
|
|
1488
1674
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1489
|
-
return {
|
|
1675
|
+
return done ? {
|
|
1490
1676
|
deleted: count,
|
|
1491
|
-
done:
|
|
1677
|
+
done: true,
|
|
1678
|
+
receipt: {
|
|
1679
|
+
expiredAt: markerExpiredAt,
|
|
1680
|
+
deleted: count
|
|
1681
|
+
}
|
|
1682
|
+
} : {
|
|
1683
|
+
deleted: count,
|
|
1684
|
+
done: false
|
|
1492
1685
|
};
|
|
1493
1686
|
});
|
|
1494
1687
|
}
|
|
1688
|
+
/** Acknowledge a final cleanup receipt before making its marker compactable. */
|
|
1689
|
+
async acknowledgeExpireRun(runId, receipt) {
|
|
1690
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1691
|
+
const key = expiredRunKey(runId);
|
|
1692
|
+
const marker = await txn.get(key);
|
|
1693
|
+
if (!marker) return { acknowledged: true };
|
|
1694
|
+
if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
|
|
1695
|
+
if (marker.compactAt === null) {
|
|
1696
|
+
const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
|
|
1697
|
+
await txn.put(key, {
|
|
1698
|
+
...marker,
|
|
1699
|
+
compactAt
|
|
1700
|
+
});
|
|
1701
|
+
await txn.put(expiredRunGcKey(runId, compactAt), {
|
|
1702
|
+
runId,
|
|
1703
|
+
compactAt
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1707
|
+
return { acknowledged: true };
|
|
1708
|
+
});
|
|
1709
|
+
}
|
|
1495
1710
|
};
|
|
1496
1711
|
//#endregion
|
|
1497
1712
|
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
1498
1713
|
const MAX_INDEX_LIST_SIZE = 1001;
|
|
1714
|
+
const EXPIRY_GC_PREFIX = "expiry-gc:";
|
|
1715
|
+
function pad(ms) {
|
|
1716
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1717
|
+
}
|
|
1718
|
+
function expiredKey(runId) {
|
|
1719
|
+
return `expired:${runId}`;
|
|
1720
|
+
}
|
|
1721
|
+
function expiryGcKey(runId, compactAt) {
|
|
1722
|
+
return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1723
|
+
}
|
|
1499
1724
|
/** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
|
|
1500
1725
|
var RunCatalogDO = class extends DurableObject {
|
|
1501
|
-
|
|
1726
|
+
now() {
|
|
1727
|
+
const clock = this.env?.clock;
|
|
1728
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
1729
|
+
}
|
|
1730
|
+
async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
|
|
1502
1731
|
if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
|
|
1503
1732
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1504
|
-
if (await txn.get(
|
|
1733
|
+
if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
|
|
1734
|
+
const now = this.now();
|
|
1735
|
+
if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
|
|
1505
1736
|
await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
|
|
1506
1737
|
return { stored: true };
|
|
1507
1738
|
});
|
|
@@ -1513,8 +1744,11 @@ var RunCatalogDO = class extends DurableObject {
|
|
|
1513
1744
|
prefix: options.prefix ?? "",
|
|
1514
1745
|
...options.reverse ? {
|
|
1515
1746
|
reverse: true,
|
|
1516
|
-
end: options.cursor
|
|
1517
|
-
} : {
|
|
1747
|
+
end: options.cursor ?? options.end
|
|
1748
|
+
} : {
|
|
1749
|
+
startAfter: options.cursor,
|
|
1750
|
+
end: options.end
|
|
1751
|
+
},
|
|
1518
1752
|
limit: limit + 1
|
|
1519
1753
|
});
|
|
1520
1754
|
const page = Array.from(entries).slice(0, limit);
|
|
@@ -1528,41 +1762,68 @@ var RunCatalogDO = class extends DurableObject {
|
|
|
1528
1762
|
cursor: listComplete ? void 0 : page.at(-1)?.[0]
|
|
1529
1763
|
};
|
|
1530
1764
|
}
|
|
1765
|
+
/** Compare-and-delete one rejected retention discovery candidate. */
|
|
1766
|
+
async deleteStaleGlobalRun(runId, key, expectedValue) {
|
|
1767
|
+
if (!key.startsWith("runall:") || !key.endsWith(`:${runId}`)) throw new Error("Stale run catalog deletion requires the exact global run key");
|
|
1768
|
+
if (JSON.parse(expectedValue).runId !== runId) throw new Error("Stale run catalog deletion requires matching run metadata");
|
|
1769
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1770
|
+
if (await txn.get(key) !== expectedValue) return { deleted: false };
|
|
1771
|
+
return { deleted: await txn.delete(key) };
|
|
1772
|
+
});
|
|
1773
|
+
}
|
|
1531
1774
|
async expireRun(runId, keys, expiredAt) {
|
|
1532
1775
|
if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
|
|
1533
1776
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1534
|
-
|
|
1777
|
+
const marker = expiredKey(runId);
|
|
1778
|
+
const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
|
|
1779
|
+
await txn.put(marker, {
|
|
1780
|
+
expiredAt,
|
|
1781
|
+
compactAt
|
|
1782
|
+
});
|
|
1783
|
+
await txn.put(expiryGcKey(runId, compactAt), {
|
|
1784
|
+
runId,
|
|
1785
|
+
compactAt
|
|
1786
|
+
});
|
|
1787
|
+
await this.armAtMost(txn, compactAt);
|
|
1535
1788
|
return { deleted: await txn.delete(keys) };
|
|
1536
1789
|
});
|
|
1537
1790
|
}
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
return (await this.ctx.storage.get("fence"))?.status ?? "active";
|
|
1791
|
+
async alarm() {
|
|
1792
|
+
try {
|
|
1793
|
+
await this.compactExpiryFences();
|
|
1794
|
+
} catch {
|
|
1795
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1796
|
+
}
|
|
1545
1797
|
}
|
|
1546
|
-
async
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
if (current) return current.status;
|
|
1550
|
-
await txn.put("fence", {
|
|
1551
|
-
status: "terminal",
|
|
1552
|
-
at
|
|
1553
|
-
});
|
|
1554
|
-
return "terminal";
|
|
1555
|
-
});
|
|
1798
|
+
async armAtMost(storage, at) {
|
|
1799
|
+
const current = await storage.getAlarm();
|
|
1800
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1556
1801
|
}
|
|
1557
|
-
async
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1802
|
+
async compactExpiryFences() {
|
|
1803
|
+
const now = this.now();
|
|
1804
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1805
|
+
const due = await txn.list({
|
|
1806
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1807
|
+
end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
|
|
1808
|
+
limit: 129
|
|
1809
|
+
});
|
|
1810
|
+
const page = Array.from(due).slice(0, 128);
|
|
1811
|
+
const markerKeys = page.map(([, item]) => expiredKey(item.runId));
|
|
1812
|
+
const markers = await txn.get(markerKeys);
|
|
1813
|
+
const deletes = page.flatMap(([key, item], index) => {
|
|
1814
|
+
return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
|
|
1564
1815
|
});
|
|
1565
|
-
|
|
1816
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1817
|
+
if (due.size > 128) {
|
|
1818
|
+
await txn.setAlarm(now + 1);
|
|
1819
|
+
return;
|
|
1820
|
+
}
|
|
1821
|
+
const nextAt = (await txn.list({
|
|
1822
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1823
|
+
limit: 1
|
|
1824
|
+
})).values().next().value?.compactAt;
|
|
1825
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1826
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
1566
1827
|
});
|
|
1567
1828
|
}
|
|
1568
1829
|
};
|
|
@@ -2786,15 +3047,25 @@ function hookMarkerKey(reference) {
|
|
|
2786
3047
|
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
2787
3048
|
}
|
|
2788
3049
|
function cleanupTombstone(record, now) {
|
|
3050
|
+
const cleanup = {
|
|
3051
|
+
...record,
|
|
3052
|
+
phase: "tombstoned",
|
|
3053
|
+
tombstonedAt: now
|
|
3054
|
+
};
|
|
2789
3055
|
return {
|
|
2790
3056
|
version: 1,
|
|
2791
3057
|
runId: record.runId,
|
|
3058
|
+
status: record.status ?? record.terminalStatus,
|
|
2792
3059
|
terminalStatus: record.terminalStatus,
|
|
2793
3060
|
completedAt: record.completedAt,
|
|
2794
3061
|
expiredAt: record.dueAt,
|
|
2795
|
-
tombstonedAt: now
|
|
3062
|
+
tombstonedAt: now,
|
|
3063
|
+
cleanup
|
|
2796
3064
|
};
|
|
2797
3065
|
}
|
|
3066
|
+
function cleanupFromTombstone(tombstone) {
|
|
3067
|
+
return tombstone.cleanup;
|
|
3068
|
+
}
|
|
2798
3069
|
function expiredRead(tombstone) {
|
|
2799
3070
|
return {
|
|
2800
3071
|
ok: false,
|
|
@@ -2814,9 +3085,30 @@ const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
|
2814
3085
|
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
2815
3086
|
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
2816
3087
|
const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
|
|
2817
|
-
const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
|
|
2818
3088
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
2819
3089
|
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
3090
|
+
const RUN_QUEUE_SHARDS_KEY = "retention:queue-shards";
|
|
3091
|
+
function cleanupRecord(run, dueAt, queueShards, reason) {
|
|
3092
|
+
const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
|
|
3093
|
+
return {
|
|
3094
|
+
version: 1,
|
|
3095
|
+
runId: run.runId,
|
|
3096
|
+
workflowName: run.workflowName,
|
|
3097
|
+
createdAt: run.createdAt,
|
|
3098
|
+
status: run.status,
|
|
3099
|
+
completedAt: terminal?.completedAt,
|
|
3100
|
+
terminalStatus: terminal?.status,
|
|
3101
|
+
reason,
|
|
3102
|
+
dueAt,
|
|
3103
|
+
queueShards,
|
|
3104
|
+
phase: "retained",
|
|
3105
|
+
generation: 0,
|
|
3106
|
+
attempts: 0,
|
|
3107
|
+
deletedPayloadKeys: 0,
|
|
3108
|
+
deletedStreams: 0,
|
|
3109
|
+
deletedQueueMessages: 0
|
|
3110
|
+
};
|
|
3111
|
+
}
|
|
2820
3112
|
/**
|
|
2821
3113
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
2822
3114
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -2912,6 +3204,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2912
3204
|
now
|
|
2913
3205
|
});
|
|
2914
3206
|
if (!outcome.ok) return outcome;
|
|
3207
|
+
if (outcome.runCreated && request.cleanup) await txn.put(RUN_QUEUE_SHARDS_KEY, request.cleanup.queueShards);
|
|
2915
3208
|
await txn.put("event_sequence", eventSequence);
|
|
2916
3209
|
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
2917
3210
|
hookId: outcome.hookToIndex.hookId,
|
|
@@ -2925,22 +3218,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2925
3218
|
...terminalRun,
|
|
2926
3219
|
expiredAt: dueAt
|
|
2927
3220
|
});
|
|
2928
|
-
const cleanup =
|
|
2929
|
-
version: 1,
|
|
2930
|
-
runId: run.runId,
|
|
2931
|
-
workflowName: run.workflowName,
|
|
2932
|
-
createdAt: run.createdAt,
|
|
2933
|
-
completedAt: run.completedAt,
|
|
2934
|
-
terminalStatus: run.status,
|
|
2935
|
-
dueAt,
|
|
2936
|
-
queueShards: request.cleanup?.queueShards ?? 1,
|
|
2937
|
-
phase: "retained",
|
|
2938
|
-
generation: 0,
|
|
2939
|
-
attempts: 0,
|
|
2940
|
-
deletedPayloadKeys: 0,
|
|
2941
|
-
deletedStreams: 0,
|
|
2942
|
-
deletedQueueMessages: 0
|
|
2943
|
-
};
|
|
3221
|
+
const cleanup = cleanupRecord(run, dueAt, request.cleanup?.queueShards ?? 1, "terminal-retention");
|
|
2944
3222
|
await txn.put("run", run);
|
|
2945
3223
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
2946
3224
|
await txn.setAlarm(dueAt);
|
|
@@ -2956,6 +3234,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2956
3234
|
});
|
|
2957
3235
|
await txn.setAlarm(this.now() + 1);
|
|
2958
3236
|
}
|
|
3237
|
+
if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
|
|
2959
3238
|
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
2960
3239
|
});
|
|
2961
3240
|
}
|
|
@@ -2966,6 +3245,8 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2966
3245
|
*/
|
|
2967
3246
|
async resolveHookTokenClaim(request) {
|
|
2968
3247
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
3248
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3249
|
+
if (tombstone) return { committed: false };
|
|
2969
3250
|
if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
|
|
2970
3251
|
await txn.put(hookClaimCancellationKey(request.claimId), {
|
|
2971
3252
|
hookId: request.hookId,
|
|
@@ -2978,6 +3259,20 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2978
3259
|
async getRun() {
|
|
2979
3260
|
return this.readKey("run");
|
|
2980
3261
|
}
|
|
3262
|
+
/** Single authoritative lifecycle answer used after derivative compaction. */
|
|
3263
|
+
async getLifecycleStatus() {
|
|
3264
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3265
|
+
const values = await txn.get([
|
|
3266
|
+
TOMBSTONE_KEY,
|
|
3267
|
+
CLEANUP_RECORD_KEY,
|
|
3268
|
+
"run"
|
|
3269
|
+
]);
|
|
3270
|
+
if (this.tombstoneFrom(values)) return "expired";
|
|
3271
|
+
const run = values.get("run");
|
|
3272
|
+
if (!run) return "missing";
|
|
3273
|
+
return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
|
|
3274
|
+
});
|
|
3275
|
+
}
|
|
2981
3276
|
async getStep(stepId) {
|
|
2982
3277
|
return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
|
|
2983
3278
|
}
|
|
@@ -3006,7 +3301,11 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3006
3301
|
}));
|
|
3007
3302
|
}
|
|
3008
3303
|
async getCleanupStatus() {
|
|
3009
|
-
|
|
3304
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
|
|
3305
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3306
|
+
if (cleanup) return cleanup;
|
|
3307
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
3308
|
+
return tombstone ? cleanupFromTombstone(tombstone) : null;
|
|
3010
3309
|
}
|
|
3011
3310
|
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
3012
3311
|
async scheduleCleanup(request) {
|
|
@@ -3029,28 +3328,95 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3029
3328
|
...terminalRun,
|
|
3030
3329
|
expiredAt: dueAt
|
|
3031
3330
|
});
|
|
3032
|
-
const cleanup =
|
|
3033
|
-
version: 1,
|
|
3034
|
-
runId: run.runId,
|
|
3035
|
-
workflowName: run.workflowName,
|
|
3036
|
-
createdAt: run.createdAt,
|
|
3037
|
-
completedAt: run.completedAt,
|
|
3038
|
-
terminalStatus: run.status,
|
|
3039
|
-
dueAt,
|
|
3040
|
-
queueShards: request.queueShards,
|
|
3041
|
-
phase: "retained",
|
|
3042
|
-
generation: 0,
|
|
3043
|
-
attempts: 0,
|
|
3044
|
-
deletedPayloadKeys: 0,
|
|
3045
|
-
deletedStreams: 0,
|
|
3046
|
-
deletedQueueMessages: 0
|
|
3047
|
-
};
|
|
3331
|
+
const cleanup = cleanupRecord(run, dueAt, request.queueShards, "manual");
|
|
3048
3332
|
await txn.put("run", run);
|
|
3049
3333
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3050
3334
|
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
3051
3335
|
return cleanup;
|
|
3052
3336
|
});
|
|
3053
3337
|
}
|
|
3338
|
+
/**
|
|
3339
|
+
* Apply the fleet-wide maximum-age policy to one authoritative run.
|
|
3340
|
+
*
|
|
3341
|
+
* The catalog is only a discovery index: eligibility is rechecked from the
|
|
3342
|
+
* RunDO's own createdAt. Once eligible, the cleanup deadline immediately
|
|
3343
|
+
* becomes an authoritative write/read fence, including for active runs.
|
|
3344
|
+
*/
|
|
3345
|
+
async enforceRetention(request) {
|
|
3346
|
+
if (!Number.isSafeInteger(request.retentionMs) || request.retentionMs <= 0) throw new Error("world-celld retentionMs must be a positive safe integer");
|
|
3347
|
+
if (!Number.isSafeInteger(request.queueShards) || request.queueShards <= 0) throw new Error("world-celld retention queueShards must be a positive safe integer");
|
|
3348
|
+
if (!Number.isSafeInteger(request.scheduledTime) || request.scheduledTime < 0) throw new Error("world-celld retention scheduledTime must be a non-negative safe integer");
|
|
3349
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3350
|
+
const values = await txn.get([
|
|
3351
|
+
CLEANUP_RECORD_KEY,
|
|
3352
|
+
TOMBSTONE_KEY,
|
|
3353
|
+
RUN_QUEUE_SHARDS_KEY,
|
|
3354
|
+
"run"
|
|
3355
|
+
]);
|
|
3356
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
3357
|
+
if (tombstone) return {
|
|
3358
|
+
state: "expired",
|
|
3359
|
+
cleanup: cleanupFromTombstone(tombstone)
|
|
3360
|
+
};
|
|
3361
|
+
const run = values.get("run");
|
|
3362
|
+
if (!run) return {
|
|
3363
|
+
state: "missing",
|
|
3364
|
+
cleanup: null
|
|
3365
|
+
};
|
|
3366
|
+
const dueAtMs = run.createdAt.getTime() + request.retentionMs;
|
|
3367
|
+
if (!Number.isSafeInteger(dueAtMs) || dueAtMs > 864e13) throw new Error(`world-celld retention deadline is out of range for ${run.runId}`);
|
|
3368
|
+
if (dueAtMs > request.scheduledTime) return {
|
|
3369
|
+
state: "not-due",
|
|
3370
|
+
cleanup: null
|
|
3371
|
+
};
|
|
3372
|
+
const dueAt = new Date(dueAtMs);
|
|
3373
|
+
const existing = values.get(CLEANUP_RECORD_KEY);
|
|
3374
|
+
const storedQueueShards = values.get(RUN_QUEUE_SHARDS_KEY);
|
|
3375
|
+
const queueShards = Number.isSafeInteger(storedQueueShards) && storedQueueShards > 0 ? storedQueueShards : request.queueShards;
|
|
3376
|
+
let cleanup = existing;
|
|
3377
|
+
if (!cleanup) {
|
|
3378
|
+
cleanup = cleanupRecord(run, dueAt, queueShards, "maximum-age");
|
|
3379
|
+
await txn.put("run", WorkflowRunSchema.parse({
|
|
3380
|
+
...run,
|
|
3381
|
+
expiredAt: dueAt
|
|
3382
|
+
}));
|
|
3383
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3384
|
+
} else if (cleanup.phase === "retained" && dueAtMs < cleanup.dueAt.getTime()) {
|
|
3385
|
+
const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
|
|
3386
|
+
cleanup = {
|
|
3387
|
+
...cleanup,
|
|
3388
|
+
status: run.status,
|
|
3389
|
+
completedAt: terminal?.completedAt,
|
|
3390
|
+
terminalStatus: terminal?.status,
|
|
3391
|
+
reason: "maximum-age",
|
|
3392
|
+
dueAt,
|
|
3393
|
+
generation: cleanup.generation + 1
|
|
3394
|
+
};
|
|
3395
|
+
await txn.put("run", WorkflowRunSchema.parse({
|
|
3396
|
+
...run,
|
|
3397
|
+
expiredAt: dueAt
|
|
3398
|
+
}));
|
|
3399
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3400
|
+
}
|
|
3401
|
+
await txn.setAlarm(this.now() + 1);
|
|
3402
|
+
return {
|
|
3403
|
+
state: cleanup.dueAt.getTime() <= this.now() ? "expired" : "scheduled",
|
|
3404
|
+
cleanup
|
|
3405
|
+
};
|
|
3406
|
+
});
|
|
3407
|
+
if (result.state === "scheduled" || result.state === "expired") {
|
|
3408
|
+
for (let page = 0; page < 2; page++) {
|
|
3409
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3410
|
+
if (cleanup?.phase !== "retained" && cleanup?.phase !== "index") break;
|
|
3411
|
+
await this.executeScheduledCleanupPage();
|
|
3412
|
+
}
|
|
3413
|
+
return {
|
|
3414
|
+
...result,
|
|
3415
|
+
cleanup: await this.getCleanupStatus() ?? result.cleanup
|
|
3416
|
+
};
|
|
3417
|
+
}
|
|
3418
|
+
return result;
|
|
3419
|
+
}
|
|
3054
3420
|
async cleanupNow(request) {
|
|
3055
3421
|
if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
|
|
3056
3422
|
...request,
|
|
@@ -3094,7 +3460,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3094
3460
|
workflowIndex() {
|
|
3095
3461
|
return createWorkflowIndex({
|
|
3096
3462
|
runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
|
|
3097
|
-
runFences: this.namespace("WORKFLOW_RUN_FENCES"),
|
|
3098
3463
|
hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
|
|
3099
3464
|
hookIds: this.namespace("WORKFLOW_HOOK_IDS")
|
|
3100
3465
|
});
|
|
@@ -3115,7 +3480,17 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3115
3480
|
});
|
|
3116
3481
|
}
|
|
3117
3482
|
async executeScheduledCleanupPage() {
|
|
3118
|
-
const
|
|
3483
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
|
|
3484
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3485
|
+
const terminalCleanup = values.get(TERMINAL_CLEANUP_KEY);
|
|
3486
|
+
if (cleanup && cleanup.phase !== "tombstoned" && cleanup.dueAt.getTime() <= this.now()) {
|
|
3487
|
+
try {
|
|
3488
|
+
await this.executeCleanup(cleanup);
|
|
3489
|
+
} catch (error) {
|
|
3490
|
+
await this.recordCleanupFailure(error, cleanup);
|
|
3491
|
+
}
|
|
3492
|
+
return;
|
|
3493
|
+
}
|
|
3119
3494
|
if (terminalCleanup) {
|
|
3120
3495
|
try {
|
|
3121
3496
|
await this.executeTerminalCleanup(terminalCleanup);
|
|
@@ -3124,7 +3499,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3124
3499
|
}
|
|
3125
3500
|
return;
|
|
3126
3501
|
}
|
|
3127
|
-
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3128
3502
|
if (!cleanup || cleanup.phase === "tombstoned") {
|
|
3129
3503
|
await this.ctx.storage.deleteAlarm();
|
|
3130
3504
|
return;
|
|
@@ -3174,8 +3548,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3174
3548
|
hooks: page.map((hook) => ({
|
|
3175
3549
|
hookId: hook.hookId,
|
|
3176
3550
|
token: hook.token
|
|
3177
|
-
}))
|
|
3178
|
-
terminal: true
|
|
3551
|
+
}))
|
|
3179
3552
|
});
|
|
3180
3553
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3181
3554
|
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
@@ -3200,8 +3573,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3200
3573
|
const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3201
3574
|
await this.workflowIndex().releaseHookIndexes({
|
|
3202
3575
|
runId: cleanup.runId,
|
|
3203
|
-
hooks: page.map(([, reference]) => reference)
|
|
3204
|
-
terminal: true
|
|
3576
|
+
hooks: page.map(([, reference]) => reference)
|
|
3205
3577
|
});
|
|
3206
3578
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3207
3579
|
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
@@ -3311,33 +3683,56 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3311
3683
|
}
|
|
3312
3684
|
async deleteQueuePage(cleanup) {
|
|
3313
3685
|
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
3314
|
-
|
|
3686
|
+
const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3315
3687
|
queueShard: 0,
|
|
3316
3688
|
queueShardDeleted: 0
|
|
3317
3689
|
};
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3690
|
+
if (progress.pendingAck) {
|
|
3691
|
+
const { queueShard, receipt } = progress.pendingAck;
|
|
3692
|
+
if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
|
|
3693
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3694
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3695
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3696
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
|
|
3697
|
+
if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
|
|
3698
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3699
|
+
...current,
|
|
3700
|
+
generation: current.generation + 1,
|
|
3701
|
+
attempts: 0,
|
|
3702
|
+
lastError: void 0
|
|
3703
|
+
});
|
|
3704
|
+
await txn.put(CLEANUP_PROGRESS_KEY, {
|
|
3705
|
+
queueShard: queueShard + 1,
|
|
3706
|
+
queueShardDeleted: 0
|
|
3707
|
+
});
|
|
3708
|
+
await txn.setAlarm(this.now() + 1);
|
|
3709
|
+
});
|
|
3710
|
+
return;
|
|
3711
|
+
}
|
|
3712
|
+
if (progress.queueShard < cleanup.queueShards) {
|
|
3321
3713
|
const requestedShard = progress.queueShard;
|
|
3322
3714
|
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
3323
|
-
|
|
3715
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3324
3716
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3325
|
-
if (current?.phase !== "queues") return
|
|
3326
|
-
if (current.generation !== expectedGeneration) return { stale: true };
|
|
3717
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3327
3718
|
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3328
3719
|
queueShard: 0,
|
|
3329
3720
|
queueShardDeleted: 0
|
|
3330
3721
|
};
|
|
3331
|
-
if (persisted.queueShard !== requestedShard) return
|
|
3332
|
-
if (result.deleted < persisted.queueShardDeleted)
|
|
3722
|
+
if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
|
|
3723
|
+
if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
|
|
3724
|
+
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3333
3725
|
const next = result.done ? {
|
|
3334
|
-
queueShard:
|
|
3335
|
-
queueShardDeleted:
|
|
3726
|
+
queueShard: requestedShard,
|
|
3727
|
+
queueShardDeleted: result.deleted,
|
|
3728
|
+
pendingAck: {
|
|
3729
|
+
queueShard: requestedShard,
|
|
3730
|
+
receipt: result.receipt
|
|
3731
|
+
}
|
|
3336
3732
|
} : {
|
|
3337
|
-
|
|
3733
|
+
queueShard: requestedShard,
|
|
3338
3734
|
queueShardDeleted: result.deleted
|
|
3339
3735
|
};
|
|
3340
|
-
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3341
3736
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3342
3737
|
...current,
|
|
3343
3738
|
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
@@ -3347,27 +3742,17 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3347
3742
|
});
|
|
3348
3743
|
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
3349
3744
|
await txn.setAlarm(this.now() + 1);
|
|
3350
|
-
return {
|
|
3351
|
-
stale: false,
|
|
3352
|
-
progress: next,
|
|
3353
|
-
generation: current.generation + 1,
|
|
3354
|
-
applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
|
|
3355
|
-
};
|
|
3356
3745
|
});
|
|
3357
|
-
|
|
3358
|
-
if (reconciled.stale) return;
|
|
3359
|
-
progress = reconciled.progress;
|
|
3360
|
-
expectedGeneration = reconciled.generation;
|
|
3361
|
-
if (!reconciled.applied && progress.queueShard === requestedShard) break;
|
|
3362
|
-
if (!result.done && progress.queueShard === requestedShard) break;
|
|
3746
|
+
return;
|
|
3363
3747
|
}
|
|
3364
3748
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3365
3749
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3366
|
-
if (current?.phase !== "queues") return;
|
|
3367
|
-
|
|
3750
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3751
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3368
3752
|
queueShard: 0,
|
|
3369
3753
|
queueShardDeleted: 0
|
|
3370
|
-
}
|
|
3754
|
+
};
|
|
3755
|
+
if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
|
|
3371
3756
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3372
3757
|
...current,
|
|
3373
3758
|
phase: "payload",
|
|
@@ -3400,14 +3785,16 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3400
3785
|
await txn.setAlarm(this.now() + 1);
|
|
3401
3786
|
return false;
|
|
3402
3787
|
}
|
|
3403
|
-
|
|
3788
|
+
const completed = {
|
|
3404
3789
|
...cleanup,
|
|
3405
3790
|
phase: "tombstoned",
|
|
3406
3791
|
generation: cleanup.generation + 1,
|
|
3407
3792
|
attempts: 0,
|
|
3408
3793
|
lastError: void 0,
|
|
3409
3794
|
tombstonedAt: now
|
|
3410
|
-
}
|
|
3795
|
+
};
|
|
3796
|
+
await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
|
|
3797
|
+
await txn.delete(CLEANUP_RECORD_KEY);
|
|
3411
3798
|
await txn.deleteAlarm();
|
|
3412
3799
|
return true;
|
|
3413
3800
|
});
|
|
@@ -3472,7 +3859,6 @@ async function startHarness(options = {}) {
|
|
|
3472
3859
|
runs: WorkflowRunDO,
|
|
3473
3860
|
streams: StreamDO,
|
|
3474
3861
|
"run-catalog": RunCatalogDO,
|
|
3475
|
-
"run-fences": RunFenceDO,
|
|
3476
3862
|
"hook-tokens": HookTokenDO,
|
|
3477
3863
|
"hook-ids": HookIdDO,
|
|
3478
3864
|
queue: QueueDO,
|
|
@@ -3483,7 +3869,6 @@ async function startHarness(options = {}) {
|
|
|
3483
3869
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
3484
3870
|
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
3485
3871
|
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
3486
|
-
WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
|
|
3487
3872
|
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
3488
3873
|
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
3489
3874
|
WORKFLOW_QUEUE: fleet.namespace("queue")
|
|
@@ -3492,7 +3877,6 @@ async function startHarness(options = {}) {
|
|
|
3492
3877
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
3493
3878
|
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
3494
3879
|
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
3495
|
-
WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
|
|
3496
3880
|
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
3497
3881
|
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
3498
3882
|
WORKFLOW_QUEUE: fleet.namespace("queue"),
|