@ewhauser/world-celld 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +11 -9
- package/celld-worker/README.md +3 -3
- package/celld-worker/worker.ts +0 -1
- package/celld-worker/wrangler.jsonc +0 -2
- package/dist/index.d.ts +22 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -20
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +492 -205
- package/dist/testing.js.map +1 -1
- package/dist/{util-DmO0XkFw.js → util-ya3V_FTA.js} +32 -19
- package/dist/util-ya3V_FTA.js.map +1 -0
- package/dist/worker.d.ts +49 -23
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +519 -217
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
- package/dist/util-DmO0XkFw.js.map +0 -1
package/dist/worker.js
CHANGED
|
@@ -121,9 +121,6 @@ function hookTokenShardName(token) {
|
|
|
121
121
|
function hookIdShardName(hookId) {
|
|
122
122
|
return shardName("hook-id", hookId, 32);
|
|
123
123
|
}
|
|
124
|
-
function runFenceCellName(runId) {
|
|
125
|
-
return `run-fence:${INDEX_PROTOCOL_VERSION}:${runId}`;
|
|
126
|
-
}
|
|
127
124
|
function allRunCatalogShardNames() {
|
|
128
125
|
return Array.from({ length: 16 }, (_, shard) => `run-catalog:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(2, "0")}`);
|
|
129
126
|
}
|
|
@@ -161,8 +158,7 @@ async function inBatches(values, operation) {
|
|
|
161
158
|
return deleted;
|
|
162
159
|
}
|
|
163
160
|
function createWorkflowIndex(bindings) {
|
|
164
|
-
const
|
|
165
|
-
const releaseHooks = async (request, fencedAt) => {
|
|
161
|
+
const releaseHooks = async (request) => {
|
|
166
162
|
const tokenGroups = groupByShard(request.hooks, (hook) => hookTokenShardName(hook.token));
|
|
167
163
|
const idGroups = groupByShard(request.hooks, (hook) => hookIdShardName(hook.hookId));
|
|
168
164
|
const releases = [];
|
|
@@ -174,7 +170,7 @@ function createWorkflowIndex(bindings) {
|
|
|
174
170
|
runId: request.runId,
|
|
175
171
|
hookId: hook.hookId
|
|
176
172
|
}
|
|
177
|
-
})), (batch) => target.releaseBatch(batch
|
|
173
|
+
})), (batch) => target.releaseBatch(batch)));
|
|
178
174
|
}
|
|
179
175
|
for (const [name, hooks] of idGroups) {
|
|
180
176
|
const target = stub(bindings.hookIds, name);
|
|
@@ -184,14 +180,13 @@ function createWorkflowIndex(bindings) {
|
|
|
184
180
|
runId: request.runId,
|
|
185
181
|
hookId: hook.hookId
|
|
186
182
|
}
|
|
187
|
-
})), (batch) => target.releaseBatch(batch
|
|
183
|
+
})), (batch) => target.releaseBatch(batch)));
|
|
188
184
|
}
|
|
189
185
|
return (await Promise.all(releases)).reduce((total, count) => total + count, 0);
|
|
190
186
|
};
|
|
191
187
|
return {
|
|
192
|
-
async commitRun(run, serializedMetadata) {
|
|
193
|
-
|
|
194
|
-
return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata);
|
|
188
|
+
async commitRun(run, serializedMetadata, publicationExpiresAt) {
|
|
189
|
+
return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata, publicationExpiresAt);
|
|
195
190
|
},
|
|
196
191
|
async listRuns(options) {
|
|
197
192
|
const suppliedLimit = options?.limit;
|
|
@@ -267,16 +262,13 @@ function createWorkflowIndex(bindings) {
|
|
|
267
262
|
await Promise.all(releases);
|
|
268
263
|
},
|
|
269
264
|
async releaseHookIndexes(request) {
|
|
270
|
-
|
|
271
|
-
return { deleted: await releaseHooks(request, Date.now()) };
|
|
265
|
+
return { deleted: await releaseHooks(request) };
|
|
272
266
|
},
|
|
273
267
|
async expireRun(request) {
|
|
274
|
-
await fence(request.runId).fenceExpired(request.expiredAt);
|
|
275
268
|
const [catalog, hookDeletes] = await Promise.all([stub(bindings.runCatalog, runCatalogShardName(request.runId)).expireRun(request.runId, request.keys, request.expiredAt), releaseHooks({
|
|
276
269
|
runId: request.runId,
|
|
277
|
-
hooks: request.hooks
|
|
278
|
-
|
|
279
|
-
}, request.expiredAt)]);
|
|
270
|
+
hooks: request.hooks
|
|
271
|
+
})]);
|
|
280
272
|
return { deleted: catalog.deleted + hookDeletes };
|
|
281
273
|
}
|
|
282
274
|
};
|
|
@@ -493,7 +485,6 @@ const WORLD_VERSION = "0.1.0";
|
|
|
493
485
|
function workflowIndex(env) {
|
|
494
486
|
return createWorkflowIndex({
|
|
495
487
|
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
496
|
-
runFences: env.WORKFLOW_RUN_FENCES,
|
|
497
488
|
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
498
489
|
hookIds: env.WORKFLOW_HOOK_IDS
|
|
499
490
|
});
|
|
@@ -503,6 +494,7 @@ const BINDINGS = {
|
|
|
503
494
|
env: "WORKFLOW_DB",
|
|
504
495
|
methods: /* @__PURE__ */ new Set([
|
|
505
496
|
"applyEvent",
|
|
497
|
+
"getLifecycleStatus",
|
|
506
498
|
"getRun",
|
|
507
499
|
"getStep",
|
|
508
500
|
"getEvent",
|
|
@@ -545,7 +537,8 @@ const BINDINGS = {
|
|
|
545
537
|
"redriveDeadLetter",
|
|
546
538
|
"purgeDeadLetters",
|
|
547
539
|
"rearmAlarm",
|
|
548
|
-
"expireRun"
|
|
540
|
+
"expireRun",
|
|
541
|
+
"acknowledgeExpireRun"
|
|
549
542
|
])
|
|
550
543
|
}
|
|
551
544
|
};
|
|
@@ -654,7 +647,7 @@ function createRouter(env) {
|
|
|
654
647
|
let result;
|
|
655
648
|
const operation = `${parts[2]}.${parts[3]}`;
|
|
656
649
|
if (operation === "runs.list") result = await index.listRuns(args[0]);
|
|
657
|
-
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
|
|
650
|
+
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
|
|
658
651
|
else if (operation === "runs.expire") result = await index.expireRun(args[0]);
|
|
659
652
|
else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
|
|
660
653
|
else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
|
|
@@ -765,26 +758,52 @@ function ownerFromSerializedHook(raw) {
|
|
|
765
758
|
function indexKey(prefix, value) {
|
|
766
759
|
return `${prefix}:${encodeURIComponent(value)}`;
|
|
767
760
|
}
|
|
768
|
-
/** Compact shard-local fence for every hook owned by one terminal or expired run. */
|
|
769
|
-
function shardRunFenceKey(runId) {
|
|
770
|
-
return `runfence:${encodeURIComponent(runId)}`;
|
|
771
|
-
}
|
|
772
761
|
async function runIsActive(env, runId) {
|
|
773
|
-
const namespace = env.
|
|
774
|
-
if (!namespace) throw new Error("world-celld hook index missing
|
|
775
|
-
return await namespace.get(namespace.idFromName(
|
|
762
|
+
const namespace = env.WORKFLOW_DB;
|
|
763
|
+
if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
|
|
764
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
|
|
776
765
|
}
|
|
777
766
|
//#endregion
|
|
767
|
+
//#region src/lifecycle.ts
|
|
768
|
+
/**
|
|
769
|
+
* Hard protocol bounds used by lifecycle leases and derivative-fence cleanup.
|
|
770
|
+
*
|
|
771
|
+
* These are not retention TTLs. They bound how long a request which was
|
|
772
|
+
* admitted before expiry may still mutate a derivative Durable Object:
|
|
773
|
+
*
|
|
774
|
+
* - fleet calls have at most three 5-minute attempts plus bounded retry delay;
|
|
775
|
+
* - one queue delivery owns its lease for at most 5 minutes plus 30 seconds;
|
|
776
|
+
* - hook admission spans reserve retries, one authoritative apply, and
|
|
777
|
+
* finalize retries before the exact claim must be reacquired.
|
|
778
|
+
*/
|
|
779
|
+
const MAX_FLEET_RPC_TIMEOUT_MS = 3e5;
|
|
780
|
+
const MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS = 1200900;
|
|
781
|
+
const MAX_QUEUE_DELIVERY_TIMEOUT_MS = 3e5;
|
|
782
|
+
const QUEUE_INFLIGHT_GRACE_MS = 3e4;
|
|
783
|
+
const QUEUE_FENCE_GRACE_MS = Math.max(MAX_FLEET_RPC_TIMEOUT_MS, 33e4);
|
|
784
|
+
const HOOK_CLAIM_LEASE_MS = 2101800;
|
|
785
|
+
const LIFECYCLE_COMPACTION_RETRY_MS = 1e3;
|
|
786
|
+
//#endregion
|
|
778
787
|
//#region src/worker/durable-objects/HookIdDO.ts
|
|
779
788
|
const MAX_RELEASE_BATCH$1 = 64;
|
|
789
|
+
const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
|
|
790
|
+
function pad$3(ms) {
|
|
791
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
792
|
+
}
|
|
780
793
|
function recordKey$1(hookId) {
|
|
781
794
|
return indexKey("hookid", hookId);
|
|
782
795
|
}
|
|
783
796
|
function claimKey$1(hookId) {
|
|
784
797
|
return indexKey("claim", hookId);
|
|
785
798
|
}
|
|
799
|
+
function claimDeadlineKey$1(hookId, expiresAt) {
|
|
800
|
+
return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
|
|
801
|
+
}
|
|
786
802
|
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
787
803
|
var HookIdDO = class extends DurableObject {
|
|
804
|
+
now() {
|
|
805
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
806
|
+
}
|
|
788
807
|
async get(hookId) {
|
|
789
808
|
const raw = await this.ctx.storage.get(recordKey$1(hookId));
|
|
790
809
|
if (!raw) return null;
|
|
@@ -801,17 +820,33 @@ var HookIdDO = class extends DurableObject {
|
|
|
801
820
|
claimed: false,
|
|
802
821
|
holder: ownerFromSerializedHook(indexed)
|
|
803
822
|
};
|
|
804
|
-
|
|
823
|
+
let existing = values.get(claim);
|
|
824
|
+
const now = this.now();
|
|
825
|
+
if (existing && existing.expiresAt <= now) {
|
|
826
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
827
|
+
existing = void 0;
|
|
828
|
+
}
|
|
805
829
|
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
806
830
|
claimed: false,
|
|
807
831
|
holder: existing.owner
|
|
808
832
|
};
|
|
809
833
|
if (existing === void 0 || existing.claimId !== proposedClaimId) {
|
|
834
|
+
if (existing) await txn.delete(existing.deadlineKey);
|
|
835
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
836
|
+
const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
|
|
810
837
|
await txn.put(claim, {
|
|
811
838
|
owner,
|
|
812
839
|
claimId: proposedClaimId,
|
|
813
|
-
reservedAt:
|
|
840
|
+
reservedAt: now,
|
|
841
|
+
expiresAt,
|
|
842
|
+
deadlineKey
|
|
814
843
|
});
|
|
844
|
+
await txn.put(deadlineKey, {
|
|
845
|
+
hookId,
|
|
846
|
+
claimId: proposedClaimId,
|
|
847
|
+
expiresAt
|
|
848
|
+
});
|
|
849
|
+
await this.armAtMost(txn, expiresAt);
|
|
815
850
|
return {
|
|
816
851
|
claimed: true,
|
|
817
852
|
claimId: proposedClaimId
|
|
@@ -828,30 +863,24 @@ var HookIdDO = class extends DurableObject {
|
|
|
828
863
|
await this.releaseBatch([{
|
|
829
864
|
hookId,
|
|
830
865
|
owner
|
|
831
|
-
}]
|
|
866
|
+
}]);
|
|
832
867
|
return { stored: false };
|
|
833
868
|
}
|
|
834
869
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
835
870
|
const record = recordKey$1(hookId);
|
|
836
871
|
const claim = claimKey$1(hookId);
|
|
837
|
-
const
|
|
838
|
-
const values = await txn.get([
|
|
839
|
-
record,
|
|
840
|
-
claim,
|
|
841
|
-
runFence
|
|
842
|
-
]);
|
|
843
|
-
if (values.has(runFence)) {
|
|
844
|
-
const existingClaim = values.get(claim);
|
|
845
|
-
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
846
|
-
return { stored: false };
|
|
847
|
-
}
|
|
872
|
+
const values = await txn.get([record, claim]);
|
|
848
873
|
const indexed = values.get(record);
|
|
849
874
|
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
|
|
850
875
|
const existingClaim = values.get(claim);
|
|
876
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
877
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
878
|
+
return { stored: false };
|
|
879
|
+
}
|
|
851
880
|
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
|
|
852
881
|
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
|
|
853
882
|
await txn.put(record, serializedHook);
|
|
854
|
-
if (existingClaim !== void 0) await txn.delete(claim);
|
|
883
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
855
884
|
return { stored: true };
|
|
856
885
|
});
|
|
857
886
|
}
|
|
@@ -859,40 +888,94 @@ var HookIdDO = class extends DurableObject {
|
|
|
859
888
|
await this.ctx.storage.transaction(async (txn) => {
|
|
860
889
|
const key = claimKey$1(hookId);
|
|
861
890
|
const claim = await txn.get(key);
|
|
862
|
-
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
891
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
863
892
|
});
|
|
864
893
|
}
|
|
865
|
-
async releaseBatch(hooks
|
|
894
|
+
async releaseBatch(hooks) {
|
|
866
895
|
if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
|
|
867
896
|
if (hooks.length === 0) return { deleted: 0 };
|
|
868
897
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
869
898
|
const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
|
|
870
899
|
const values = await txn.get(keys);
|
|
871
900
|
const deletes = [];
|
|
901
|
+
const deadlineDeletes = [];
|
|
872
902
|
for (const { hookId, owner } of hooks) {
|
|
873
903
|
const record = recordKey$1(hookId);
|
|
874
904
|
const claim = claimKey$1(hookId);
|
|
875
905
|
const indexed = values.get(record);
|
|
876
906
|
const existingClaim = values.get(claim);
|
|
877
907
|
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
878
|
-
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner))
|
|
908
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
909
|
+
deletes.push(claim);
|
|
910
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
911
|
+
}
|
|
879
912
|
}
|
|
880
|
-
|
|
881
|
-
|
|
913
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
914
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
915
|
+
return { deleted };
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
async alarm() {
|
|
919
|
+
try {
|
|
920
|
+
await this.compactClaims();
|
|
921
|
+
} catch {
|
|
922
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
async armAtMost(storage, at) {
|
|
926
|
+
const current = await storage.getAlarm();
|
|
927
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
928
|
+
}
|
|
929
|
+
async compactClaims() {
|
|
930
|
+
const now = this.now();
|
|
931
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
932
|
+
const due = await txn.list({
|
|
933
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
934
|
+
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
|
|
935
|
+
limit: 129
|
|
936
|
+
});
|
|
937
|
+
const page = Array.from(due).slice(0, 128);
|
|
938
|
+
const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
|
|
939
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
940
|
+
const key = claimKey$1(item.hookId);
|
|
941
|
+
const claim = claims.get(key);
|
|
942
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
943
|
+
});
|
|
944
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
945
|
+
if (due.size > 128) {
|
|
946
|
+
await txn.setAlarm(now + 1);
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
const nextAt = (await txn.list({
|
|
950
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
951
|
+
limit: 1
|
|
952
|
+
})).values().next().value?.expiresAt;
|
|
953
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
954
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
882
955
|
});
|
|
883
956
|
}
|
|
884
957
|
};
|
|
885
958
|
//#endregion
|
|
886
959
|
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
887
960
|
const MAX_RELEASE_BATCH = 64;
|
|
961
|
+
const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
|
|
962
|
+
function pad$2(ms) {
|
|
963
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
964
|
+
}
|
|
888
965
|
function recordKey(token) {
|
|
889
966
|
return indexKey("hook", token);
|
|
890
967
|
}
|
|
891
968
|
function claimKey(token) {
|
|
892
969
|
return indexKey("claim", token);
|
|
893
970
|
}
|
|
971
|
+
function claimDeadlineKey(token, expiresAt) {
|
|
972
|
+
return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
|
|
973
|
+
}
|
|
894
974
|
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
895
975
|
var HookTokenDO = class extends DurableObject {
|
|
976
|
+
now() {
|
|
977
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
978
|
+
}
|
|
896
979
|
async get(token) {
|
|
897
980
|
const raw = await this.ctx.storage.get(recordKey(token));
|
|
898
981
|
if (!raw) return null;
|
|
@@ -909,17 +992,32 @@ var HookTokenDO = class extends DurableObject {
|
|
|
909
992
|
claimed: false,
|
|
910
993
|
holder: ownerFromSerializedHook(indexed)
|
|
911
994
|
};
|
|
912
|
-
|
|
995
|
+
let existing = values.get(claim);
|
|
996
|
+
const now = this.now();
|
|
997
|
+
if (existing && existing.expiresAt <= now) {
|
|
998
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
999
|
+
existing = void 0;
|
|
1000
|
+
}
|
|
913
1001
|
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
914
1002
|
claimed: false,
|
|
915
1003
|
holder: existing.owner
|
|
916
1004
|
};
|
|
917
1005
|
if (existing === void 0) {
|
|
1006
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
1007
|
+
const deadlineKey = claimDeadlineKey(token, expiresAt);
|
|
918
1008
|
await txn.put(claim, {
|
|
919
1009
|
owner,
|
|
920
1010
|
claimId: proposedClaimId,
|
|
921
|
-
reservedAt:
|
|
1011
|
+
reservedAt: now,
|
|
1012
|
+
expiresAt,
|
|
1013
|
+
deadlineKey
|
|
922
1014
|
});
|
|
1015
|
+
await txn.put(deadlineKey, {
|
|
1016
|
+
token,
|
|
1017
|
+
claimId: proposedClaimId,
|
|
1018
|
+
expiresAt
|
|
1019
|
+
});
|
|
1020
|
+
await this.armAtMost(txn, expiresAt);
|
|
923
1021
|
return {
|
|
924
1022
|
claimed: true,
|
|
925
1023
|
claimId: proposedClaimId
|
|
@@ -936,30 +1034,24 @@ var HookTokenDO = class extends DurableObject {
|
|
|
936
1034
|
await this.releaseBatch([{
|
|
937
1035
|
token,
|
|
938
1036
|
owner
|
|
939
|
-
}]
|
|
1037
|
+
}]);
|
|
940
1038
|
return { stored: false };
|
|
941
1039
|
}
|
|
942
1040
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
943
1041
|
const record = recordKey(token);
|
|
944
1042
|
const claim = claimKey(token);
|
|
945
|
-
const
|
|
946
|
-
const values = await txn.get([
|
|
947
|
-
record,
|
|
948
|
-
claim,
|
|
949
|
-
runFence
|
|
950
|
-
]);
|
|
951
|
-
if (values.has(runFence)) {
|
|
952
|
-
const existingClaim = values.get(claim);
|
|
953
|
-
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
954
|
-
return { stored: false };
|
|
955
|
-
}
|
|
1043
|
+
const values = await txn.get([record, claim]);
|
|
956
1044
|
const indexed = values.get(record);
|
|
957
1045
|
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
958
1046
|
const existingClaim = values.get(claim);
|
|
1047
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
1048
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
1049
|
+
return { stored: false };
|
|
1050
|
+
}
|
|
959
1051
|
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
960
1052
|
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
|
|
961
1053
|
await txn.put(record, serializedHook);
|
|
962
|
-
if (existingClaim !== void 0) await txn.delete(claim);
|
|
1054
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
963
1055
|
return { stored: true };
|
|
964
1056
|
});
|
|
965
1057
|
}
|
|
@@ -967,26 +1059,70 @@ var HookTokenDO = class extends DurableObject {
|
|
|
967
1059
|
await this.ctx.storage.transaction(async (txn) => {
|
|
968
1060
|
const key = claimKey(token);
|
|
969
1061
|
const claim = await txn.get(key);
|
|
970
|
-
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
1062
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
971
1063
|
});
|
|
972
1064
|
}
|
|
973
|
-
async releaseBatch(hooks
|
|
1065
|
+
async releaseBatch(hooks) {
|
|
974
1066
|
if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
|
|
975
1067
|
if (hooks.length === 0) return { deleted: 0 };
|
|
976
1068
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
977
1069
|
const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
|
|
978
1070
|
const values = await txn.get(keys);
|
|
979
1071
|
const deletes = [];
|
|
1072
|
+
const deadlineDeletes = [];
|
|
980
1073
|
for (const { token, owner } of hooks) {
|
|
981
1074
|
const record = recordKey(token);
|
|
982
1075
|
const claim = claimKey(token);
|
|
983
1076
|
const indexed = values.get(record);
|
|
984
1077
|
const existingClaim = values.get(claim);
|
|
985
1078
|
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
986
|
-
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner))
|
|
1079
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
1080
|
+
deletes.push(claim);
|
|
1081
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
1082
|
+
}
|
|
987
1083
|
}
|
|
988
|
-
|
|
989
|
-
|
|
1084
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
1085
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
1086
|
+
return { deleted };
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
async alarm() {
|
|
1090
|
+
try {
|
|
1091
|
+
await this.compactClaims();
|
|
1092
|
+
} catch {
|
|
1093
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
async armAtMost(storage, at) {
|
|
1097
|
+
const current = await storage.getAlarm();
|
|
1098
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1099
|
+
}
|
|
1100
|
+
async compactClaims() {
|
|
1101
|
+
const now = this.now();
|
|
1102
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1103
|
+
const due = await txn.list({
|
|
1104
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1105
|
+
end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
|
|
1106
|
+
limit: 129
|
|
1107
|
+
});
|
|
1108
|
+
const page = Array.from(due).slice(0, 128);
|
|
1109
|
+
const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
|
|
1110
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
1111
|
+
const key = claimKey(item.token);
|
|
1112
|
+
const claim = claims.get(key);
|
|
1113
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
1114
|
+
});
|
|
1115
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1116
|
+
if (due.size > 128) {
|
|
1117
|
+
await txn.setAlarm(now + 1);
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
const nextAt = (await txn.list({
|
|
1121
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1122
|
+
limit: 1
|
|
1123
|
+
})).values().next().value?.expiresAt;
|
|
1124
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1125
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
990
1126
|
});
|
|
991
1127
|
}
|
|
992
1128
|
};
|
|
@@ -1001,8 +1137,6 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
|
1001
1137
|
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
1002
1138
|
const DEFAULT_MAX_INFLIGHT = 5;
|
|
1003
1139
|
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
1004
|
-
/** Grace period past the delivery timeout before a claim counts as lost. */
|
|
1005
|
-
const INFLIGHT_GRACE_MS = 3e4;
|
|
1006
1140
|
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
1007
1141
|
const MIN_ALARM_DELAY_MS = 1;
|
|
1008
1142
|
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
@@ -1011,14 +1145,15 @@ const STORAGE_BATCH_SIZE$2 = 128;
|
|
|
1011
1145
|
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
1012
1146
|
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
1013
1147
|
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
1014
|
-
|
|
1148
|
+
const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
|
|
1149
|
+
function pad$1(ms) {
|
|
1015
1150
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1016
1151
|
}
|
|
1017
1152
|
function dueKey(atMs, messageId) {
|
|
1018
|
-
return `due:${pad(atMs)}:${messageId}`;
|
|
1153
|
+
return `due:${pad$1(atMs)}:${messageId}`;
|
|
1019
1154
|
}
|
|
1020
1155
|
function inflightKey(deadlineMs, messageId) {
|
|
1021
|
-
return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
|
|
1156
|
+
return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
|
|
1022
1157
|
}
|
|
1023
1158
|
function deadlineFromInflightKey(key) {
|
|
1024
1159
|
return Number.parseInt(key.slice(18, 31), 10);
|
|
@@ -1029,6 +1164,9 @@ function runReferenceKey(runId, messageId) {
|
|
|
1029
1164
|
function expiredRunKey(runId) {
|
|
1030
1165
|
return `expired-run:${runId}`;
|
|
1031
1166
|
}
|
|
1167
|
+
function expiredRunGcKey(runId, compactAt) {
|
|
1168
|
+
return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1169
|
+
}
|
|
1032
1170
|
function boundedLimit$1(value, fallback, maximum) {
|
|
1033
1171
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1034
1172
|
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
@@ -1079,16 +1217,29 @@ var QueueDO = class extends DurableObject {
|
|
|
1079
1217
|
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
1080
1218
|
return configured;
|
|
1081
1219
|
}
|
|
1220
|
+
#deliveryTimeoutMs() {
|
|
1221
|
+
const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
1222
|
+
if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
|
|
1223
|
+
return configured;
|
|
1224
|
+
}
|
|
1225
|
+
async #authoritativeRunExpired(runId) {
|
|
1226
|
+
const namespace = this.env?.WORKFLOW_DB;
|
|
1227
|
+
if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
|
|
1228
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
|
|
1229
|
+
}
|
|
1230
|
+
async #withRunMutation(runId, onExpired, onAllowed) {
|
|
1231
|
+
const expired = await this.#authoritativeRunExpired(runId);
|
|
1232
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1233
|
+
if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
|
|
1234
|
+
return await onAllowed(txn);
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
1082
1237
|
async enqueue(request) {
|
|
1083
1238
|
this.#maxInflight();
|
|
1239
|
+
this.#deliveryTimeoutMs();
|
|
1084
1240
|
const storage = this.ctx.storage;
|
|
1085
1241
|
const now = this.#now();
|
|
1086
|
-
|
|
1087
|
-
if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
|
|
1088
|
-
ok: false,
|
|
1089
|
-
code: "RUN_EXPIRED",
|
|
1090
|
-
message: `Workflow run "${request.runId}" has expired`
|
|
1091
|
-
};
|
|
1242
|
+
const persist = async (txn) => {
|
|
1092
1243
|
const pinned = await txn.get("cfg");
|
|
1093
1244
|
if (!pinned) await txn.put("cfg", {
|
|
1094
1245
|
queueShards: request.config.queueShards,
|
|
@@ -1140,7 +1291,12 @@ var QueueDO = class extends DurableObject {
|
|
|
1140
1291
|
messageId: row.messageId,
|
|
1141
1292
|
deduped: false
|
|
1142
1293
|
};
|
|
1143
|
-
}
|
|
1294
|
+
};
|
|
1295
|
+
return request.runId ? await this.#withRunMutation(request.runId, async () => ({
|
|
1296
|
+
ok: false,
|
|
1297
|
+
code: "RUN_EXPIRED",
|
|
1298
|
+
message: `Workflow run "${request.runId}" has expired`
|
|
1299
|
+
}), persist) : await storage.transaction(persist);
|
|
1144
1300
|
}
|
|
1145
1301
|
async alarm() {
|
|
1146
1302
|
let claimed = [];
|
|
@@ -1154,7 +1310,10 @@ var QueueDO = class extends DurableObject {
|
|
|
1154
1310
|
};
|
|
1155
1311
|
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
1156
1312
|
else await attempt();
|
|
1157
|
-
if (failure !== null)
|
|
1313
|
+
if (failure !== null) {
|
|
1314
|
+
await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1315
|
+
throw failure;
|
|
1316
|
+
}
|
|
1158
1317
|
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
1159
1318
|
}
|
|
1160
1319
|
/**
|
|
@@ -1166,9 +1325,13 @@ var QueueDO = class extends DurableObject {
|
|
|
1166
1325
|
const now = this.#now();
|
|
1167
1326
|
const maxInflight = this.#maxInflight();
|
|
1168
1327
|
return await storage.transaction(async (txn) => {
|
|
1328
|
+
if (await this.#compactExpiredRunFences(txn, now)) {
|
|
1329
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1330
|
+
return [];
|
|
1331
|
+
}
|
|
1169
1332
|
const expiredPage = await txn.list({
|
|
1170
1333
|
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1171
|
-
end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
|
|
1334
|
+
end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
|
|
1172
1335
|
limit: 129
|
|
1173
1336
|
});
|
|
1174
1337
|
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
@@ -1200,17 +1363,17 @@ var QueueDO = class extends DurableObject {
|
|
|
1200
1363
|
if (inflightCount < maxInflight) {
|
|
1201
1364
|
const due = await txn.list({
|
|
1202
1365
|
prefix: "due:",
|
|
1203
|
-
end: `due:${pad(now + 1)}`,
|
|
1366
|
+
end: `due:${pad$1(now + 1)}`,
|
|
1204
1367
|
limit: maxInflight - inflightCount
|
|
1205
1368
|
});
|
|
1206
|
-
const timeoutMs = this.#
|
|
1369
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1207
1370
|
const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
1208
1371
|
await deleteMany$1(txn, Array.from(due.keys()));
|
|
1209
1372
|
const claimWrites = [];
|
|
1210
1373
|
for (const [, messageId] of due) {
|
|
1211
1374
|
const row = dueRows.get(`msg:${messageId}`);
|
|
1212
1375
|
if (!row) continue;
|
|
1213
|
-
const claimKey = inflightKey(now + timeoutMs +
|
|
1376
|
+
const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
|
|
1214
1377
|
claimWrites.push([claimKey, messageId]);
|
|
1215
1378
|
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
1216
1379
|
messageId: row.messageId,
|
|
@@ -1230,9 +1393,13 @@ var QueueDO = class extends DurableObject {
|
|
|
1230
1393
|
}
|
|
1231
1394
|
async #deliver({ row, inflightKey: claimKey }) {
|
|
1232
1395
|
const storage = this.ctx.storage;
|
|
1233
|
-
const timeoutMs = this.#
|
|
1396
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1234
1397
|
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
1235
1398
|
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
1399
|
+
if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
|
|
1400
|
+
await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
|
|
1401
|
+
return;
|
|
1402
|
+
}
|
|
1236
1403
|
let response = null;
|
|
1237
1404
|
let transportError = null;
|
|
1238
1405
|
try {
|
|
@@ -1262,28 +1429,21 @@ var QueueDO = class extends DurableObject {
|
|
|
1262
1429
|
} catch {
|
|
1263
1430
|
timeoutSeconds = void 0;
|
|
1264
1431
|
}
|
|
1265
|
-
if (typeof timeoutSeconds === "number")
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
await txn.
|
|
1270
|
-
if (row.
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1432
|
+
if (typeof timeoutSeconds === "number") {
|
|
1433
|
+
const reschedule = async (txn) => {
|
|
1434
|
+
if (!await txn.delete(claimKey)) return;
|
|
1435
|
+
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
1436
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1437
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1438
|
+
messageId: row.messageId,
|
|
1439
|
+
dueKey: scheduleKey,
|
|
1440
|
+
idempotencyKey: row.idempotencyKey
|
|
1441
|
+
});
|
|
1274
1442
|
await this.#scheduleNextAlarm(txn, now);
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1280
|
-
messageId: row.messageId,
|
|
1281
|
-
dueKey: scheduleKey,
|
|
1282
|
-
idempotencyKey: row.idempotencyKey
|
|
1283
|
-
});
|
|
1284
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
1285
|
-
});
|
|
1286
|
-
else await this.#retry(row, claimKey, "HTTP 503");
|
|
1443
|
+
};
|
|
1444
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
|
|
1445
|
+
else await storage.transaction(reschedule);
|
|
1446
|
+
} else await this.#retry(row, claimKey, "HTTP 503");
|
|
1287
1447
|
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
1288
1448
|
await cancelUnusedResponseBody(response);
|
|
1289
1449
|
await this.#ack(row, claimKey);
|
|
@@ -1304,23 +1464,23 @@ var QueueDO = class extends DurableObject {
|
|
|
1304
1464
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1305
1465
|
});
|
|
1306
1466
|
}
|
|
1467
|
+
async #dropExpiredMessage(txn, row, claimKey, now) {
|
|
1468
|
+
if (!await txn.delete(claimKey)) return;
|
|
1469
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
1470
|
+
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1471
|
+
if (row.idempotencyKey) {
|
|
1472
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1473
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1474
|
+
}
|
|
1475
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
1476
|
+
}
|
|
1307
1477
|
async #retry(row, claimKey, reason) {
|
|
1308
1478
|
const storage = this.ctx.storage;
|
|
1309
1479
|
const now = this.#now();
|
|
1310
1480
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
1311
1481
|
const attempt = row.attempt + 1;
|
|
1312
|
-
|
|
1482
|
+
const persistRetry = async (txn) => {
|
|
1313
1483
|
if (!await txn.delete(claimKey)) return;
|
|
1314
|
-
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
1315
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
1316
|
-
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1317
|
-
if (row.idempotencyKey) {
|
|
1318
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1319
|
-
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1320
|
-
}
|
|
1321
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
1322
|
-
return;
|
|
1323
|
-
}
|
|
1324
1484
|
if (attempt >= maxAttempts) {
|
|
1325
1485
|
const dead = {
|
|
1326
1486
|
...row,
|
|
@@ -1328,7 +1488,7 @@ var QueueDO = class extends DurableObject {
|
|
|
1328
1488
|
lastError: reason,
|
|
1329
1489
|
failedAt: now
|
|
1330
1490
|
};
|
|
1331
|
-
const deadKey = `dlq:${pad(now)}:${row.messageId}`;
|
|
1491
|
+
const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
|
|
1332
1492
|
await txn.put(deadKey, dead);
|
|
1333
1493
|
await txn.delete(`msg:${row.messageId}`);
|
|
1334
1494
|
if (row.idempotencyKey) {
|
|
@@ -1356,7 +1516,32 @@ var QueueDO = class extends DurableObject {
|
|
|
1356
1516
|
idempotencyKey: row.idempotencyKey
|
|
1357
1517
|
});
|
|
1358
1518
|
await this.#scheduleNextAlarm(txn, now);
|
|
1519
|
+
};
|
|
1520
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
|
|
1521
|
+
else await storage.transaction(persistRetry);
|
|
1522
|
+
}
|
|
1523
|
+
/** Return true when another due compaction page must run immediately. */
|
|
1524
|
+
async #compactExpiredRunFences(txn, now) {
|
|
1525
|
+
const due = await txn.list({
|
|
1526
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1527
|
+
end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
|
|
1528
|
+
limit: 129
|
|
1359
1529
|
});
|
|
1530
|
+
const page = Array.from(due).slice(0, 128);
|
|
1531
|
+
if (page.length === 0) return false;
|
|
1532
|
+
const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
|
|
1533
|
+
const markers = await getMany$1(txn, markerKeys);
|
|
1534
|
+
const deletes = [];
|
|
1535
|
+
for (let index = 0; index < page.length; index++) {
|
|
1536
|
+
const [gcKey, item] = page[index];
|
|
1537
|
+
const markerKey = markerKeys[index];
|
|
1538
|
+
const marker = markers.get(markerKey);
|
|
1539
|
+
deletes.push(gcKey);
|
|
1540
|
+
if (marker?.compactAt !== item.compactAt) continue;
|
|
1541
|
+
deletes.push(markerKey);
|
|
1542
|
+
}
|
|
1543
|
+
await deleteMany$1(txn, deletes);
|
|
1544
|
+
return due.size > 128;
|
|
1360
1545
|
}
|
|
1361
1546
|
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
1362
1547
|
async #armAlarmAtMost(storage, atMs, now) {
|
|
@@ -1388,6 +1573,11 @@ var QueueDO = class extends DurableObject {
|
|
|
1388
1573
|
const deadline = deadlineFromInflightKey(key);
|
|
1389
1574
|
if (next === null || deadline < next) next = deadline;
|
|
1390
1575
|
}
|
|
1576
|
+
const expiryGc = await storage.list({
|
|
1577
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1578
|
+
limit: 1
|
|
1579
|
+
});
|
|
1580
|
+
for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
|
|
1391
1581
|
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
1392
1582
|
else await storage.deleteAlarm();
|
|
1393
1583
|
}
|
|
@@ -1426,37 +1616,42 @@ var QueueDO = class extends DurableObject {
|
|
|
1426
1616
|
async redriveDeadLetter(messageId) {
|
|
1427
1617
|
const storage = this.ctx.storage;
|
|
1428
1618
|
const now = this.#now();
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
await txn.put(
|
|
1447
|
-
const scheduleKey = dueKey(now, row.messageId);
|
|
1448
|
-
await txn.put(scheduleKey, row.messageId);
|
|
1449
|
-
await txn.delete(key);
|
|
1450
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1451
|
-
messageId: row.messageId,
|
|
1452
|
-
dueKey: scheduleKey,
|
|
1453
|
-
idempotencyKey: row.idempotencyKey
|
|
1454
|
-
});
|
|
1455
|
-
await this.#armAlarmAtMost(txn, now, now);
|
|
1456
|
-
return { ok: true };
|
|
1619
|
+
const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
|
|
1620
|
+
if (!found) return { ok: false };
|
|
1621
|
+
const [deadKey, candidate] = found;
|
|
1622
|
+
const redrive = async (txn) => {
|
|
1623
|
+
const dead = await txn.get(deadKey);
|
|
1624
|
+
if (!dead || dead.messageId !== messageId) return { ok: false };
|
|
1625
|
+
const { failedAt: _failedAt, ...rest } = dead;
|
|
1626
|
+
const row = {
|
|
1627
|
+
...rest,
|
|
1628
|
+
attempt: 0,
|
|
1629
|
+
lastError: void 0
|
|
1630
|
+
};
|
|
1631
|
+
if (row.idempotencyKey) {
|
|
1632
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1633
|
+
const holder = await txn.get(dedupKey);
|
|
1634
|
+
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
1635
|
+
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
1636
|
+
await txn.put(dedupKey, row.messageId);
|
|
1457
1637
|
}
|
|
1638
|
+
await txn.put(`msg:${row.messageId}`, row);
|
|
1639
|
+
const scheduleKey = dueKey(now, row.messageId);
|
|
1640
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1641
|
+
await txn.delete(deadKey);
|
|
1642
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1643
|
+
messageId: row.messageId,
|
|
1644
|
+
dueKey: scheduleKey,
|
|
1645
|
+
idempotencyKey: row.idempotencyKey
|
|
1646
|
+
});
|
|
1647
|
+
await this.#armAlarmAtMost(txn, now, now);
|
|
1648
|
+
return { ok: true };
|
|
1649
|
+
};
|
|
1650
|
+
const rejectExpired = async (txn) => {
|
|
1651
|
+
if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
|
|
1458
1652
|
return { ok: false };
|
|
1459
|
-
}
|
|
1653
|
+
};
|
|
1654
|
+
return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
|
|
1460
1655
|
}
|
|
1461
1656
|
async purgeDeadLetters() {
|
|
1462
1657
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
@@ -1506,27 +1701,75 @@ var QueueDO = class extends DurableObject {
|
|
|
1506
1701
|
count++;
|
|
1507
1702
|
}
|
|
1508
1703
|
await deleteMany$1(txn, keys);
|
|
1704
|
+
const done = references.size <= limit;
|
|
1705
|
+
const markerExpiredAt = existing?.expiredAt ?? expiredAt;
|
|
1509
1706
|
await txn.put(expiredRunKey(runId), {
|
|
1510
|
-
expiredAt,
|
|
1511
|
-
deleted: count
|
|
1707
|
+
expiredAt: markerExpiredAt,
|
|
1708
|
+
deleted: count,
|
|
1709
|
+
compactAt: existing?.compactAt ?? null
|
|
1512
1710
|
});
|
|
1513
1711
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1514
|
-
return {
|
|
1712
|
+
return done ? {
|
|
1515
1713
|
deleted: count,
|
|
1516
|
-
done:
|
|
1714
|
+
done: true,
|
|
1715
|
+
receipt: {
|
|
1716
|
+
expiredAt: markerExpiredAt,
|
|
1717
|
+
deleted: count
|
|
1718
|
+
}
|
|
1719
|
+
} : {
|
|
1720
|
+
deleted: count,
|
|
1721
|
+
done: false
|
|
1517
1722
|
};
|
|
1518
1723
|
});
|
|
1519
1724
|
}
|
|
1725
|
+
/** Acknowledge a final cleanup receipt before making its marker compactable. */
|
|
1726
|
+
async acknowledgeExpireRun(runId, receipt) {
|
|
1727
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1728
|
+
const key = expiredRunKey(runId);
|
|
1729
|
+
const marker = await txn.get(key);
|
|
1730
|
+
if (!marker) return { acknowledged: true };
|
|
1731
|
+
if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
|
|
1732
|
+
if (marker.compactAt === null) {
|
|
1733
|
+
const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
|
|
1734
|
+
await txn.put(key, {
|
|
1735
|
+
...marker,
|
|
1736
|
+
compactAt
|
|
1737
|
+
});
|
|
1738
|
+
await txn.put(expiredRunGcKey(runId, compactAt), {
|
|
1739
|
+
runId,
|
|
1740
|
+
compactAt
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1744
|
+
return { acknowledged: true };
|
|
1745
|
+
});
|
|
1746
|
+
}
|
|
1520
1747
|
};
|
|
1521
1748
|
//#endregion
|
|
1522
1749
|
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
1523
1750
|
const MAX_INDEX_LIST_SIZE = 1001;
|
|
1751
|
+
const EXPIRY_GC_PREFIX = "expiry-gc:";
|
|
1752
|
+
function pad(ms) {
|
|
1753
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1754
|
+
}
|
|
1755
|
+
function expiredKey(runId) {
|
|
1756
|
+
return `expired:${runId}`;
|
|
1757
|
+
}
|
|
1758
|
+
function expiryGcKey(runId, compactAt) {
|
|
1759
|
+
return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1760
|
+
}
|
|
1524
1761
|
/** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
|
|
1525
1762
|
var RunCatalogDO = class extends DurableObject {
|
|
1526
|
-
|
|
1763
|
+
now() {
|
|
1764
|
+
const clock = this.env?.clock;
|
|
1765
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
1766
|
+
}
|
|
1767
|
+
async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
|
|
1527
1768
|
if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
|
|
1528
1769
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1529
|
-
if (await txn.get(
|
|
1770
|
+
if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
|
|
1771
|
+
const now = this.now();
|
|
1772
|
+
if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
|
|
1530
1773
|
await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
|
|
1531
1774
|
return { stored: true };
|
|
1532
1775
|
});
|
|
@@ -1556,38 +1799,56 @@ var RunCatalogDO = class extends DurableObject {
|
|
|
1556
1799
|
async expireRun(runId, keys, expiredAt) {
|
|
1557
1800
|
if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
|
|
1558
1801
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1559
|
-
|
|
1802
|
+
const marker = expiredKey(runId);
|
|
1803
|
+
const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
|
|
1804
|
+
await txn.put(marker, {
|
|
1805
|
+
expiredAt,
|
|
1806
|
+
compactAt
|
|
1807
|
+
});
|
|
1808
|
+
await txn.put(expiryGcKey(runId, compactAt), {
|
|
1809
|
+
runId,
|
|
1810
|
+
compactAt
|
|
1811
|
+
});
|
|
1812
|
+
await this.armAtMost(txn, compactAt);
|
|
1560
1813
|
return { deleted: await txn.delete(keys) };
|
|
1561
1814
|
});
|
|
1562
1815
|
}
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
return (await this.ctx.storage.get("fence"))?.status ?? "active";
|
|
1816
|
+
async alarm() {
|
|
1817
|
+
try {
|
|
1818
|
+
await this.compactExpiryFences();
|
|
1819
|
+
} catch {
|
|
1820
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1821
|
+
}
|
|
1570
1822
|
}
|
|
1571
|
-
async
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
if (current) return current.status;
|
|
1575
|
-
await txn.put("fence", {
|
|
1576
|
-
status: "terminal",
|
|
1577
|
-
at
|
|
1578
|
-
});
|
|
1579
|
-
return "terminal";
|
|
1580
|
-
});
|
|
1823
|
+
async armAtMost(storage, at) {
|
|
1824
|
+
const current = await storage.getAlarm();
|
|
1825
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1581
1826
|
}
|
|
1582
|
-
async
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1827
|
+
async compactExpiryFences() {
|
|
1828
|
+
const now = this.now();
|
|
1829
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1830
|
+
const due = await txn.list({
|
|
1831
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1832
|
+
end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
|
|
1833
|
+
limit: 129
|
|
1834
|
+
});
|
|
1835
|
+
const page = Array.from(due).slice(0, 128);
|
|
1836
|
+
const markerKeys = page.map(([, item]) => expiredKey(item.runId));
|
|
1837
|
+
const markers = await txn.get(markerKeys);
|
|
1838
|
+
const deletes = page.flatMap(([key, item], index) => {
|
|
1839
|
+
return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
|
|
1589
1840
|
});
|
|
1590
|
-
|
|
1841
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1842
|
+
if (due.size > 128) {
|
|
1843
|
+
await txn.setAlarm(now + 1);
|
|
1844
|
+
return;
|
|
1845
|
+
}
|
|
1846
|
+
const nextAt = (await txn.list({
|
|
1847
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1848
|
+
limit: 1
|
|
1849
|
+
})).values().next().value?.compactAt;
|
|
1850
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1851
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
1591
1852
|
});
|
|
1592
1853
|
}
|
|
1593
1854
|
};
|
|
@@ -2811,15 +3072,24 @@ function hookMarkerKey(reference) {
|
|
|
2811
3072
|
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
2812
3073
|
}
|
|
2813
3074
|
function cleanupTombstone(record, now) {
|
|
3075
|
+
const cleanup = {
|
|
3076
|
+
...record,
|
|
3077
|
+
phase: "tombstoned",
|
|
3078
|
+
tombstonedAt: now
|
|
3079
|
+
};
|
|
2814
3080
|
return {
|
|
2815
3081
|
version: 1,
|
|
2816
3082
|
runId: record.runId,
|
|
2817
3083
|
terminalStatus: record.terminalStatus,
|
|
2818
3084
|
completedAt: record.completedAt,
|
|
2819
3085
|
expiredAt: record.dueAt,
|
|
2820
|
-
tombstonedAt: now
|
|
3086
|
+
tombstonedAt: now,
|
|
3087
|
+
cleanup
|
|
2821
3088
|
};
|
|
2822
3089
|
}
|
|
3090
|
+
function cleanupFromTombstone(tombstone) {
|
|
3091
|
+
return tombstone.cleanup;
|
|
3092
|
+
}
|
|
2823
3093
|
function expiredRead(tombstone) {
|
|
2824
3094
|
return {
|
|
2825
3095
|
ok: false,
|
|
@@ -2839,7 +3109,6 @@ const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
|
2839
3109
|
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
2840
3110
|
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
2841
3111
|
const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
|
|
2842
|
-
const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
|
|
2843
3112
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
2844
3113
|
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
2845
3114
|
/**
|
|
@@ -2981,6 +3250,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2981
3250
|
});
|
|
2982
3251
|
await txn.setAlarm(this.now() + 1);
|
|
2983
3252
|
}
|
|
3253
|
+
if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
|
|
2984
3254
|
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
2985
3255
|
});
|
|
2986
3256
|
}
|
|
@@ -2991,6 +3261,8 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2991
3261
|
*/
|
|
2992
3262
|
async resolveHookTokenClaim(request) {
|
|
2993
3263
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
3264
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3265
|
+
if (tombstone) return { committed: false };
|
|
2994
3266
|
if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
|
|
2995
3267
|
await txn.put(hookClaimCancellationKey(request.claimId), {
|
|
2996
3268
|
hookId: request.hookId,
|
|
@@ -3003,6 +3275,20 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3003
3275
|
async getRun() {
|
|
3004
3276
|
return this.readKey("run");
|
|
3005
3277
|
}
|
|
3278
|
+
/** Single authoritative lifecycle answer used after derivative compaction. */
|
|
3279
|
+
async getLifecycleStatus() {
|
|
3280
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3281
|
+
const values = await txn.get([
|
|
3282
|
+
TOMBSTONE_KEY,
|
|
3283
|
+
CLEANUP_RECORD_KEY,
|
|
3284
|
+
"run"
|
|
3285
|
+
]);
|
|
3286
|
+
if (this.tombstoneFrom(values)) return "expired";
|
|
3287
|
+
const run = values.get("run");
|
|
3288
|
+
if (!run) return "missing";
|
|
3289
|
+
return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
|
|
3290
|
+
});
|
|
3291
|
+
}
|
|
3006
3292
|
async getStep(stepId) {
|
|
3007
3293
|
return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
|
|
3008
3294
|
}
|
|
@@ -3031,7 +3317,11 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3031
3317
|
}));
|
|
3032
3318
|
}
|
|
3033
3319
|
async getCleanupStatus() {
|
|
3034
|
-
|
|
3320
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
|
|
3321
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3322
|
+
if (cleanup) return cleanup;
|
|
3323
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
3324
|
+
return tombstone ? cleanupFromTombstone(tombstone) : null;
|
|
3035
3325
|
}
|
|
3036
3326
|
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
3037
3327
|
async scheduleCleanup(request) {
|
|
@@ -3119,7 +3409,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3119
3409
|
workflowIndex() {
|
|
3120
3410
|
return createWorkflowIndex({
|
|
3121
3411
|
runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
|
|
3122
|
-
runFences: this.namespace("WORKFLOW_RUN_FENCES"),
|
|
3123
3412
|
hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
|
|
3124
3413
|
hookIds: this.namespace("WORKFLOW_HOOK_IDS")
|
|
3125
3414
|
});
|
|
@@ -3199,8 +3488,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3199
3488
|
hooks: page.map((hook) => ({
|
|
3200
3489
|
hookId: hook.hookId,
|
|
3201
3490
|
token: hook.token
|
|
3202
|
-
}))
|
|
3203
|
-
terminal: true
|
|
3491
|
+
}))
|
|
3204
3492
|
});
|
|
3205
3493
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3206
3494
|
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
@@ -3225,8 +3513,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3225
3513
|
const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3226
3514
|
await this.workflowIndex().releaseHookIndexes({
|
|
3227
3515
|
runId: cleanup.runId,
|
|
3228
|
-
hooks: page.map(([, reference]) => reference)
|
|
3229
|
-
terminal: true
|
|
3516
|
+
hooks: page.map(([, reference]) => reference)
|
|
3230
3517
|
});
|
|
3231
3518
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3232
3519
|
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
@@ -3336,33 +3623,56 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3336
3623
|
}
|
|
3337
3624
|
async deleteQueuePage(cleanup) {
|
|
3338
3625
|
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
3339
|
-
|
|
3626
|
+
const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3340
3627
|
queueShard: 0,
|
|
3341
3628
|
queueShardDeleted: 0
|
|
3342
3629
|
};
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3630
|
+
if (progress.pendingAck) {
|
|
3631
|
+
const { queueShard, receipt } = progress.pendingAck;
|
|
3632
|
+
if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
|
|
3633
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3634
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3635
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3636
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
|
|
3637
|
+
if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
|
|
3638
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3639
|
+
...current,
|
|
3640
|
+
generation: current.generation + 1,
|
|
3641
|
+
attempts: 0,
|
|
3642
|
+
lastError: void 0
|
|
3643
|
+
});
|
|
3644
|
+
await txn.put(CLEANUP_PROGRESS_KEY, {
|
|
3645
|
+
queueShard: queueShard + 1,
|
|
3646
|
+
queueShardDeleted: 0
|
|
3647
|
+
});
|
|
3648
|
+
await txn.setAlarm(this.now() + 1);
|
|
3649
|
+
});
|
|
3650
|
+
return;
|
|
3651
|
+
}
|
|
3652
|
+
if (progress.queueShard < cleanup.queueShards) {
|
|
3346
3653
|
const requestedShard = progress.queueShard;
|
|
3347
3654
|
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
3348
|
-
|
|
3655
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3349
3656
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3350
|
-
if (current?.phase !== "queues") return
|
|
3351
|
-
if (current.generation !== expectedGeneration) return { stale: true };
|
|
3657
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3352
3658
|
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3353
3659
|
queueShard: 0,
|
|
3354
3660
|
queueShardDeleted: 0
|
|
3355
3661
|
};
|
|
3356
|
-
if (persisted.queueShard !== requestedShard) return
|
|
3357
|
-
if (result.deleted < persisted.queueShardDeleted)
|
|
3662
|
+
if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
|
|
3663
|
+
if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
|
|
3664
|
+
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3358
3665
|
const next = result.done ? {
|
|
3359
|
-
queueShard:
|
|
3360
|
-
queueShardDeleted:
|
|
3666
|
+
queueShard: requestedShard,
|
|
3667
|
+
queueShardDeleted: result.deleted,
|
|
3668
|
+
pendingAck: {
|
|
3669
|
+
queueShard: requestedShard,
|
|
3670
|
+
receipt: result.receipt
|
|
3671
|
+
}
|
|
3361
3672
|
} : {
|
|
3362
|
-
|
|
3673
|
+
queueShard: requestedShard,
|
|
3363
3674
|
queueShardDeleted: result.deleted
|
|
3364
3675
|
};
|
|
3365
|
-
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3366
3676
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3367
3677
|
...current,
|
|
3368
3678
|
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
@@ -3372,27 +3682,17 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3372
3682
|
});
|
|
3373
3683
|
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
3374
3684
|
await txn.setAlarm(this.now() + 1);
|
|
3375
|
-
return {
|
|
3376
|
-
stale: false,
|
|
3377
|
-
progress: next,
|
|
3378
|
-
generation: current.generation + 1,
|
|
3379
|
-
applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
|
|
3380
|
-
};
|
|
3381
3685
|
});
|
|
3382
|
-
|
|
3383
|
-
if (reconciled.stale) return;
|
|
3384
|
-
progress = reconciled.progress;
|
|
3385
|
-
expectedGeneration = reconciled.generation;
|
|
3386
|
-
if (!reconciled.applied && progress.queueShard === requestedShard) break;
|
|
3387
|
-
if (!result.done && progress.queueShard === requestedShard) break;
|
|
3686
|
+
return;
|
|
3388
3687
|
}
|
|
3389
3688
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3390
3689
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3391
|
-
if (current?.phase !== "queues") return;
|
|
3392
|
-
|
|
3690
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3691
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3393
3692
|
queueShard: 0,
|
|
3394
3693
|
queueShardDeleted: 0
|
|
3395
|
-
}
|
|
3694
|
+
};
|
|
3695
|
+
if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
|
|
3396
3696
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3397
3697
|
...current,
|
|
3398
3698
|
phase: "payload",
|
|
@@ -3425,14 +3725,16 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3425
3725
|
await txn.setAlarm(this.now() + 1);
|
|
3426
3726
|
return false;
|
|
3427
3727
|
}
|
|
3428
|
-
|
|
3728
|
+
const completed = {
|
|
3429
3729
|
...cleanup,
|
|
3430
3730
|
phase: "tombstoned",
|
|
3431
3731
|
generation: cleanup.generation + 1,
|
|
3432
3732
|
attempts: 0,
|
|
3433
3733
|
lastError: void 0,
|
|
3434
3734
|
tombstonedAt: now
|
|
3435
|
-
}
|
|
3735
|
+
};
|
|
3736
|
+
await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
|
|
3737
|
+
await txn.delete(CLEANUP_RECORD_KEY);
|
|
3436
3738
|
await txn.deleteAlarm();
|
|
3437
3739
|
return true;
|
|
3438
3740
|
});
|
|
@@ -3496,6 +3798,6 @@ var worker_default = { async fetch(request, env) {
|
|
|
3496
3798
|
return createRouter(env)(request);
|
|
3497
3799
|
} };
|
|
3498
3800
|
//#endregion
|
|
3499
|
-
export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO,
|
|
3801
|
+
export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
|
|
3500
3802
|
|
|
3501
3803
|
//# sourceMappingURL=worker.js.map
|