@ewhauser/world-celld 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +70 -28
- package/celld-worker/README.md +25 -3
- package/celld-worker/worker.ts +0 -1
- package/celld-worker/wrangler.jsonc +12 -2
- package/dist/index.d.ts +36 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +21 -21
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +625 -241
- package/dist/testing.js.map +1 -1
- package/dist/{util-DmO0XkFw.js → util-ya3V_FTA.js} +32 -19
- package/dist/util-ya3V_FTA.js.map +1 -0
- package/dist/worker.d.ts +95 -28
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +768 -291
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
- package/dist/util-DmO0XkFw.js.map +0 -1
package/dist/worker.js
CHANGED
|
@@ -121,9 +121,6 @@ function hookTokenShardName(token) {
|
|
|
121
121
|
function hookIdShardName(hookId) {
|
|
122
122
|
return shardName("hook-id", hookId, 32);
|
|
123
123
|
}
|
|
124
|
-
function runFenceCellName(runId) {
|
|
125
|
-
return `run-fence:${INDEX_PROTOCOL_VERSION}:${runId}`;
|
|
126
|
-
}
|
|
127
124
|
function allRunCatalogShardNames() {
|
|
128
125
|
return Array.from({ length: 16 }, (_, shard) => `run-catalog:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(2, "0")}`);
|
|
129
126
|
}
|
|
@@ -161,8 +158,7 @@ async function inBatches(values, operation) {
|
|
|
161
158
|
return deleted;
|
|
162
159
|
}
|
|
163
160
|
function createWorkflowIndex(bindings) {
|
|
164
|
-
const
|
|
165
|
-
const releaseHooks = async (request, fencedAt) => {
|
|
161
|
+
const releaseHooks = async (request) => {
|
|
166
162
|
const tokenGroups = groupByShard(request.hooks, (hook) => hookTokenShardName(hook.token));
|
|
167
163
|
const idGroups = groupByShard(request.hooks, (hook) => hookIdShardName(hook.hookId));
|
|
168
164
|
const releases = [];
|
|
@@ -174,7 +170,7 @@ function createWorkflowIndex(bindings) {
|
|
|
174
170
|
runId: request.runId,
|
|
175
171
|
hookId: hook.hookId
|
|
176
172
|
}
|
|
177
|
-
})), (batch) => target.releaseBatch(batch
|
|
173
|
+
})), (batch) => target.releaseBatch(batch)));
|
|
178
174
|
}
|
|
179
175
|
for (const [name, hooks] of idGroups) {
|
|
180
176
|
const target = stub(bindings.hookIds, name);
|
|
@@ -184,14 +180,13 @@ function createWorkflowIndex(bindings) {
|
|
|
184
180
|
runId: request.runId,
|
|
185
181
|
hookId: hook.hookId
|
|
186
182
|
}
|
|
187
|
-
})), (batch) => target.releaseBatch(batch
|
|
183
|
+
})), (batch) => target.releaseBatch(batch)));
|
|
188
184
|
}
|
|
189
185
|
return (await Promise.all(releases)).reduce((total, count) => total + count, 0);
|
|
190
186
|
};
|
|
191
187
|
return {
|
|
192
|
-
async commitRun(run, serializedMetadata) {
|
|
193
|
-
|
|
194
|
-
return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata);
|
|
188
|
+
async commitRun(run, serializedMetadata, publicationExpiresAt) {
|
|
189
|
+
return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata, publicationExpiresAt);
|
|
195
190
|
},
|
|
196
191
|
async listRuns(options) {
|
|
197
192
|
const suppliedLimit = options?.limit;
|
|
@@ -267,16 +262,13 @@ function createWorkflowIndex(bindings) {
|
|
|
267
262
|
await Promise.all(releases);
|
|
268
263
|
},
|
|
269
264
|
async releaseHookIndexes(request) {
|
|
270
|
-
|
|
271
|
-
return { deleted: await releaseHooks(request, Date.now()) };
|
|
265
|
+
return { deleted: await releaseHooks(request) };
|
|
272
266
|
},
|
|
273
267
|
async expireRun(request) {
|
|
274
|
-
await fence(request.runId).fenceExpired(request.expiredAt);
|
|
275
268
|
const [catalog, hookDeletes] = await Promise.all([stub(bindings.runCatalog, runCatalogShardName(request.runId)).expireRun(request.runId, request.keys, request.expiredAt), releaseHooks({
|
|
276
269
|
runId: request.runId,
|
|
277
|
-
hooks: request.hooks
|
|
278
|
-
|
|
279
|
-
}, request.expiredAt)]);
|
|
270
|
+
hooks: request.hooks
|
|
271
|
+
})]);
|
|
280
272
|
return { deleted: catalog.deleted + hookDeletes };
|
|
281
273
|
}
|
|
282
274
|
};
|
|
@@ -493,7 +485,6 @@ const WORLD_VERSION = "0.1.0";
|
|
|
493
485
|
function workflowIndex(env) {
|
|
494
486
|
return createWorkflowIndex({
|
|
495
487
|
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
496
|
-
runFences: env.WORKFLOW_RUN_FENCES,
|
|
497
488
|
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
498
489
|
hookIds: env.WORKFLOW_HOOK_IDS
|
|
499
490
|
});
|
|
@@ -503,6 +494,7 @@ const BINDINGS = {
|
|
|
503
494
|
env: "WORKFLOW_DB",
|
|
504
495
|
methods: /* @__PURE__ */ new Set([
|
|
505
496
|
"applyEvent",
|
|
497
|
+
"getLifecycleStatus",
|
|
506
498
|
"getRun",
|
|
507
499
|
"getStep",
|
|
508
500
|
"getEvent",
|
|
@@ -545,7 +537,8 @@ const BINDINGS = {
|
|
|
545
537
|
"redriveDeadLetter",
|
|
546
538
|
"purgeDeadLetters",
|
|
547
539
|
"rearmAlarm",
|
|
548
|
-
"expireRun"
|
|
540
|
+
"expireRun",
|
|
541
|
+
"acknowledgeExpireRun"
|
|
549
542
|
])
|
|
550
543
|
}
|
|
551
544
|
};
|
|
@@ -654,7 +647,7 @@ function createRouter(env) {
|
|
|
654
647
|
let result;
|
|
655
648
|
const operation = `${parts[2]}.${parts[3]}`;
|
|
656
649
|
if (operation === "runs.list") result = await index.listRuns(args[0]);
|
|
657
|
-
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
|
|
650
|
+
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
|
|
658
651
|
else if (operation === "runs.expire") result = await index.expireRun(args[0]);
|
|
659
652
|
else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
|
|
660
653
|
else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
|
|
@@ -743,6 +736,120 @@ function createRouter(env) {
|
|
|
743
736
|
};
|
|
744
737
|
}
|
|
745
738
|
//#endregion
|
|
739
|
+
//#region src/retention.ts
|
|
740
|
+
const CLEANUP_RECORD_KEY = "retention:cleanup";
|
|
741
|
+
const TOMBSTONE_KEY = "retention:tombstone";
|
|
742
|
+
const HOOK_MARKER_PREFIX = "retention:hook:";
|
|
743
|
+
const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
|
|
744
|
+
function sortableTimestamp(date) {
|
|
745
|
+
return date.getTime().toString().padStart(13, "0");
|
|
746
|
+
}
|
|
747
|
+
function workflowRunIndexKey(run) {
|
|
748
|
+
return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
749
|
+
}
|
|
750
|
+
function globalRunIndexKey(run) {
|
|
751
|
+
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
752
|
+
}
|
|
753
|
+
function hookMarkerKey(reference) {
|
|
754
|
+
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
755
|
+
}
|
|
756
|
+
function cleanupTombstone(record, now) {
|
|
757
|
+
const cleanup = {
|
|
758
|
+
...record,
|
|
759
|
+
phase: "tombstoned",
|
|
760
|
+
tombstonedAt: now
|
|
761
|
+
};
|
|
762
|
+
return {
|
|
763
|
+
version: 1,
|
|
764
|
+
runId: record.runId,
|
|
765
|
+
status: record.status ?? record.terminalStatus,
|
|
766
|
+
terminalStatus: record.terminalStatus,
|
|
767
|
+
completedAt: record.completedAt,
|
|
768
|
+
expiredAt: record.dueAt,
|
|
769
|
+
tombstonedAt: now,
|
|
770
|
+
cleanup
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
function cleanupFromTombstone(tombstone) {
|
|
774
|
+
return tombstone.cleanup;
|
|
775
|
+
}
|
|
776
|
+
function expiredRead(tombstone) {
|
|
777
|
+
return {
|
|
778
|
+
ok: false,
|
|
779
|
+
code: "RUN_EXPIRED",
|
|
780
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
|
|
781
|
+
tombstone
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
const MAX_RETENTION_SWEEP_BATCH_SIZE = 1e3;
|
|
785
|
+
const RETENTION_SWEEP_CONCURRENCY = 8;
|
|
786
|
+
function integerSetting(name, raw, fallback, minimum, maximum = Number.MAX_SAFE_INTEGER) {
|
|
787
|
+
const value = raw === void 0 || raw === "" ? fallback : Number(raw);
|
|
788
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) throw new Error(`world-celld: ${name} must be an integer between ${minimum} and ${maximum}`);
|
|
789
|
+
return value;
|
|
790
|
+
}
|
|
791
|
+
function runIdFromCatalogValue(value) {
|
|
792
|
+
const parsed = JSON.parse(value);
|
|
793
|
+
if (typeof parsed.runId !== "string" || parsed.runId.length === 0) throw new Error("world-celld: run catalog entry has no runId");
|
|
794
|
+
return parsed.runId;
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* Discover and fence one bounded page of workflows older than the configured
|
|
798
|
+
* maximum age. RunDO alarms finish the persisted, idempotent deletion phases.
|
|
799
|
+
*/
|
|
800
|
+
async function runRetentionSweep(scheduledTime, env) {
|
|
801
|
+
if (!Number.isSafeInteger(scheduledTime) || scheduledTime < 0) throw new Error("world-celld: scheduledTime must be a non-negative safe integer");
|
|
802
|
+
const retentionMs = integerSetting("WORKFLOW_RETENTION_MS", env.WORKFLOW_RETENTION_MS, 0, 0);
|
|
803
|
+
if (retentionMs === 0 || scheduledTime < retentionMs) return {
|
|
804
|
+
disabled: retentionMs === 0,
|
|
805
|
+
cutoff: retentionMs === 0 ? null : scheduledTime - retentionMs,
|
|
806
|
+
scanned: 0,
|
|
807
|
+
scheduled: 0,
|
|
808
|
+
expired: 0,
|
|
809
|
+
missing: 0,
|
|
810
|
+
notDue: 0
|
|
811
|
+
};
|
|
812
|
+
const batchSize = integerSetting("WORKFLOW_RETENTION_BATCH_SIZE", env.WORKFLOW_RETENTION_BATCH_SIZE, 128, 1, MAX_RETENTION_SWEEP_BATCH_SIZE);
|
|
813
|
+
const queueShards = integerSetting("WORKFLOW_RETENTION_QUEUE_SHARDS", env.WORKFLOW_RETENTION_QUEUE_SHARDS, 1, 1);
|
|
814
|
+
const cutoff = scheduledTime - retentionMs;
|
|
815
|
+
const page = await createWorkflowIndex({
|
|
816
|
+
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
817
|
+
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
818
|
+
hookIds: env.WORKFLOW_HOOK_IDS
|
|
819
|
+
}).listRuns({
|
|
820
|
+
prefix: "runall:",
|
|
821
|
+
end: `runall:${sortableTimestamp(new Date(cutoff + 1))}:`,
|
|
822
|
+
limit: batchSize
|
|
823
|
+
});
|
|
824
|
+
const result = {
|
|
825
|
+
disabled: false,
|
|
826
|
+
cutoff,
|
|
827
|
+
scanned: page.keys.length,
|
|
828
|
+
scheduled: 0,
|
|
829
|
+
expired: 0,
|
|
830
|
+
missing: 0,
|
|
831
|
+
notDue: 0
|
|
832
|
+
};
|
|
833
|
+
const failures = [];
|
|
834
|
+
for (let offset = 0; offset < page.keys.length; offset += RETENTION_SWEEP_CONCURRENCY) {
|
|
835
|
+
const candidates = page.keys.slice(offset, offset + RETENTION_SWEEP_CONCURRENCY);
|
|
836
|
+
const settled = await Promise.allSettled(candidates.map(async (entry) => {
|
|
837
|
+
const runId = runIdFromCatalogValue(entry.value);
|
|
838
|
+
const outcome = await env.WORKFLOW_DB.get(env.WORKFLOW_DB.idFromName(runId)).enforceRetention({
|
|
839
|
+
retentionMs,
|
|
840
|
+
queueShards,
|
|
841
|
+
scheduledTime
|
|
842
|
+
});
|
|
843
|
+
if (outcome.state === "missing" || outcome.state === "not-due") await env.WORKFLOW_RUN_CATALOG.get(env.WORKFLOW_RUN_CATALOG.idFromName(runCatalogShardName(runId))).deleteStaleGlobalRun(runId, entry.name, entry.value);
|
|
844
|
+
return outcome;
|
|
845
|
+
}));
|
|
846
|
+
for (const outcome of settled) if (outcome.status === "rejected") failures.push(outcome.reason);
|
|
847
|
+
else result[outcome.value.state === "not-due" ? "notDue" : outcome.value.state]++;
|
|
848
|
+
}
|
|
849
|
+
if (failures.length > 0) throw new AggregateError(failures, `world-celld retention sweep failed for ${failures.length} run(s)`);
|
|
850
|
+
return result;
|
|
851
|
+
}
|
|
852
|
+
//#endregion
|
|
746
853
|
//#region src/vendor/shared/util.ts
|
|
747
854
|
function compact(obj) {
|
|
748
855
|
const value = {};
|
|
@@ -765,26 +872,52 @@ function ownerFromSerializedHook(raw) {
|
|
|
765
872
|
function indexKey(prefix, value) {
|
|
766
873
|
return `${prefix}:${encodeURIComponent(value)}`;
|
|
767
874
|
}
|
|
768
|
-
/** Compact shard-local fence for every hook owned by one terminal or expired run. */
|
|
769
|
-
function shardRunFenceKey(runId) {
|
|
770
|
-
return `runfence:${encodeURIComponent(runId)}`;
|
|
771
|
-
}
|
|
772
875
|
async function runIsActive(env, runId) {
|
|
773
|
-
const namespace = env.
|
|
774
|
-
if (!namespace) throw new Error("world-celld hook index missing
|
|
775
|
-
return await namespace.get(namespace.idFromName(
|
|
876
|
+
const namespace = env.WORKFLOW_DB;
|
|
877
|
+
if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
|
|
878
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
|
|
776
879
|
}
|
|
777
880
|
//#endregion
|
|
881
|
+
//#region src/lifecycle.ts
|
|
882
|
+
/**
|
|
883
|
+
* Hard protocol bounds used by lifecycle leases and derivative-fence cleanup.
|
|
884
|
+
*
|
|
885
|
+
* These are not retention TTLs. They bound how long a request which was
|
|
886
|
+
* admitted before expiry may still mutate a derivative Durable Object:
|
|
887
|
+
*
|
|
888
|
+
* - fleet calls have at most three 5-minute attempts plus bounded retry delay;
|
|
889
|
+
* - one queue delivery owns its lease for at most 5 minutes plus 30 seconds;
|
|
890
|
+
* - hook admission spans reserve retries, one authoritative apply, and
|
|
891
|
+
* finalize retries before the exact claim must be reacquired.
|
|
892
|
+
*/
|
|
893
|
+
const MAX_FLEET_RPC_TIMEOUT_MS = 3e5;
|
|
894
|
+
const MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS = 1200900;
|
|
895
|
+
const MAX_QUEUE_DELIVERY_TIMEOUT_MS = 3e5;
|
|
896
|
+
const QUEUE_INFLIGHT_GRACE_MS = 3e4;
|
|
897
|
+
const QUEUE_FENCE_GRACE_MS = Math.max(MAX_FLEET_RPC_TIMEOUT_MS, 33e4);
|
|
898
|
+
const HOOK_CLAIM_LEASE_MS = 2101800;
|
|
899
|
+
const LIFECYCLE_COMPACTION_RETRY_MS = 1e3;
|
|
900
|
+
//#endregion
|
|
778
901
|
//#region src/worker/durable-objects/HookIdDO.ts
|
|
779
902
|
const MAX_RELEASE_BATCH$1 = 64;
|
|
903
|
+
const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
|
|
904
|
+
function pad$3(ms) {
|
|
905
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
906
|
+
}
|
|
780
907
|
function recordKey$1(hookId) {
|
|
781
908
|
return indexKey("hookid", hookId);
|
|
782
909
|
}
|
|
783
910
|
function claimKey$1(hookId) {
|
|
784
911
|
return indexKey("claim", hookId);
|
|
785
912
|
}
|
|
913
|
+
function claimDeadlineKey$1(hookId, expiresAt) {
|
|
914
|
+
return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
|
|
915
|
+
}
|
|
786
916
|
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
787
917
|
var HookIdDO = class extends DurableObject {
|
|
918
|
+
now() {
|
|
919
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
920
|
+
}
|
|
788
921
|
async get(hookId) {
|
|
789
922
|
const raw = await this.ctx.storage.get(recordKey$1(hookId));
|
|
790
923
|
if (!raw) return null;
|
|
@@ -801,17 +934,33 @@ var HookIdDO = class extends DurableObject {
|
|
|
801
934
|
claimed: false,
|
|
802
935
|
holder: ownerFromSerializedHook(indexed)
|
|
803
936
|
};
|
|
804
|
-
|
|
937
|
+
let existing = values.get(claim);
|
|
938
|
+
const now = this.now();
|
|
939
|
+
if (existing && existing.expiresAt <= now) {
|
|
940
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
941
|
+
existing = void 0;
|
|
942
|
+
}
|
|
805
943
|
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
806
944
|
claimed: false,
|
|
807
945
|
holder: existing.owner
|
|
808
946
|
};
|
|
809
947
|
if (existing === void 0 || existing.claimId !== proposedClaimId) {
|
|
948
|
+
if (existing) await txn.delete(existing.deadlineKey);
|
|
949
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
950
|
+
const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
|
|
810
951
|
await txn.put(claim, {
|
|
811
952
|
owner,
|
|
812
953
|
claimId: proposedClaimId,
|
|
813
|
-
reservedAt:
|
|
954
|
+
reservedAt: now,
|
|
955
|
+
expiresAt,
|
|
956
|
+
deadlineKey
|
|
957
|
+
});
|
|
958
|
+
await txn.put(deadlineKey, {
|
|
959
|
+
hookId,
|
|
960
|
+
claimId: proposedClaimId,
|
|
961
|
+
expiresAt
|
|
814
962
|
});
|
|
963
|
+
await this.armAtMost(txn, expiresAt);
|
|
815
964
|
return {
|
|
816
965
|
claimed: true,
|
|
817
966
|
claimId: proposedClaimId
|
|
@@ -828,30 +977,24 @@ var HookIdDO = class extends DurableObject {
|
|
|
828
977
|
await this.releaseBatch([{
|
|
829
978
|
hookId,
|
|
830
979
|
owner
|
|
831
|
-
}]
|
|
980
|
+
}]);
|
|
832
981
|
return { stored: false };
|
|
833
982
|
}
|
|
834
983
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
835
984
|
const record = recordKey$1(hookId);
|
|
836
985
|
const claim = claimKey$1(hookId);
|
|
837
|
-
const
|
|
838
|
-
const values = await txn.get([
|
|
839
|
-
record,
|
|
840
|
-
claim,
|
|
841
|
-
runFence
|
|
842
|
-
]);
|
|
843
|
-
if (values.has(runFence)) {
|
|
844
|
-
const existingClaim = values.get(claim);
|
|
845
|
-
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
846
|
-
return { stored: false };
|
|
847
|
-
}
|
|
986
|
+
const values = await txn.get([record, claim]);
|
|
848
987
|
const indexed = values.get(record);
|
|
849
988
|
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
|
|
850
989
|
const existingClaim = values.get(claim);
|
|
990
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
991
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
992
|
+
return { stored: false };
|
|
993
|
+
}
|
|
851
994
|
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
|
|
852
995
|
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
|
|
853
996
|
await txn.put(record, serializedHook);
|
|
854
|
-
if (existingClaim !== void 0) await txn.delete(claim);
|
|
997
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
855
998
|
return { stored: true };
|
|
856
999
|
});
|
|
857
1000
|
}
|
|
@@ -859,40 +1002,94 @@ var HookIdDO = class extends DurableObject {
|
|
|
859
1002
|
await this.ctx.storage.transaction(async (txn) => {
|
|
860
1003
|
const key = claimKey$1(hookId);
|
|
861
1004
|
const claim = await txn.get(key);
|
|
862
|
-
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
1005
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
863
1006
|
});
|
|
864
1007
|
}
|
|
865
|
-
async releaseBatch(hooks
|
|
1008
|
+
async releaseBatch(hooks) {
|
|
866
1009
|
if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
|
|
867
1010
|
if (hooks.length === 0) return { deleted: 0 };
|
|
868
1011
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
869
1012
|
const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
|
|
870
1013
|
const values = await txn.get(keys);
|
|
871
1014
|
const deletes = [];
|
|
1015
|
+
const deadlineDeletes = [];
|
|
872
1016
|
for (const { hookId, owner } of hooks) {
|
|
873
1017
|
const record = recordKey$1(hookId);
|
|
874
1018
|
const claim = claimKey$1(hookId);
|
|
875
1019
|
const indexed = values.get(record);
|
|
876
1020
|
const existingClaim = values.get(claim);
|
|
877
1021
|
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
878
|
-
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner))
|
|
1022
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
1023
|
+
deletes.push(claim);
|
|
1024
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
1028
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
1029
|
+
return { deleted };
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
async alarm() {
|
|
1033
|
+
try {
|
|
1034
|
+
await this.compactClaims();
|
|
1035
|
+
} catch {
|
|
1036
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
async armAtMost(storage, at) {
|
|
1040
|
+
const current = await storage.getAlarm();
|
|
1041
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1042
|
+
}
|
|
1043
|
+
async compactClaims() {
|
|
1044
|
+
const now = this.now();
|
|
1045
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1046
|
+
const due = await txn.list({
|
|
1047
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
1048
|
+
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
|
|
1049
|
+
limit: 129
|
|
1050
|
+
});
|
|
1051
|
+
const page = Array.from(due).slice(0, 128);
|
|
1052
|
+
const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
|
|
1053
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
1054
|
+
const key = claimKey$1(item.hookId);
|
|
1055
|
+
const claim = claims.get(key);
|
|
1056
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
1057
|
+
});
|
|
1058
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1059
|
+
if (due.size > 128) {
|
|
1060
|
+
await txn.setAlarm(now + 1);
|
|
1061
|
+
return;
|
|
879
1062
|
}
|
|
880
|
-
|
|
881
|
-
|
|
1063
|
+
const nextAt = (await txn.list({
|
|
1064
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
1065
|
+
limit: 1
|
|
1066
|
+
})).values().next().value?.expiresAt;
|
|
1067
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1068
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
882
1069
|
});
|
|
883
1070
|
}
|
|
884
1071
|
};
|
|
885
1072
|
//#endregion
|
|
886
1073
|
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
887
1074
|
const MAX_RELEASE_BATCH = 64;
|
|
1075
|
+
const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
|
|
1076
|
+
function pad$2(ms) {
|
|
1077
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1078
|
+
}
|
|
888
1079
|
function recordKey(token) {
|
|
889
1080
|
return indexKey("hook", token);
|
|
890
1081
|
}
|
|
891
1082
|
function claimKey(token) {
|
|
892
1083
|
return indexKey("claim", token);
|
|
893
1084
|
}
|
|
1085
|
+
function claimDeadlineKey(token, expiresAt) {
|
|
1086
|
+
return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
|
|
1087
|
+
}
|
|
894
1088
|
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
895
1089
|
var HookTokenDO = class extends DurableObject {
|
|
1090
|
+
now() {
|
|
1091
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
1092
|
+
}
|
|
896
1093
|
async get(token) {
|
|
897
1094
|
const raw = await this.ctx.storage.get(recordKey(token));
|
|
898
1095
|
if (!raw) return null;
|
|
@@ -909,17 +1106,32 @@ var HookTokenDO = class extends DurableObject {
|
|
|
909
1106
|
claimed: false,
|
|
910
1107
|
holder: ownerFromSerializedHook(indexed)
|
|
911
1108
|
};
|
|
912
|
-
|
|
1109
|
+
let existing = values.get(claim);
|
|
1110
|
+
const now = this.now();
|
|
1111
|
+
if (existing && existing.expiresAt <= now) {
|
|
1112
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
1113
|
+
existing = void 0;
|
|
1114
|
+
}
|
|
913
1115
|
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
914
1116
|
claimed: false,
|
|
915
1117
|
holder: existing.owner
|
|
916
1118
|
};
|
|
917
1119
|
if (existing === void 0) {
|
|
1120
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
1121
|
+
const deadlineKey = claimDeadlineKey(token, expiresAt);
|
|
918
1122
|
await txn.put(claim, {
|
|
919
1123
|
owner,
|
|
920
1124
|
claimId: proposedClaimId,
|
|
921
|
-
reservedAt:
|
|
1125
|
+
reservedAt: now,
|
|
1126
|
+
expiresAt,
|
|
1127
|
+
deadlineKey
|
|
1128
|
+
});
|
|
1129
|
+
await txn.put(deadlineKey, {
|
|
1130
|
+
token,
|
|
1131
|
+
claimId: proposedClaimId,
|
|
1132
|
+
expiresAt
|
|
922
1133
|
});
|
|
1134
|
+
await this.armAtMost(txn, expiresAt);
|
|
923
1135
|
return {
|
|
924
1136
|
claimed: true,
|
|
925
1137
|
claimId: proposedClaimId
|
|
@@ -936,30 +1148,24 @@ var HookTokenDO = class extends DurableObject {
|
|
|
936
1148
|
await this.releaseBatch([{
|
|
937
1149
|
token,
|
|
938
1150
|
owner
|
|
939
|
-
}]
|
|
1151
|
+
}]);
|
|
940
1152
|
return { stored: false };
|
|
941
1153
|
}
|
|
942
1154
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
943
1155
|
const record = recordKey(token);
|
|
944
1156
|
const claim = claimKey(token);
|
|
945
|
-
const
|
|
946
|
-
const values = await txn.get([
|
|
947
|
-
record,
|
|
948
|
-
claim,
|
|
949
|
-
runFence
|
|
950
|
-
]);
|
|
951
|
-
if (values.has(runFence)) {
|
|
952
|
-
const existingClaim = values.get(claim);
|
|
953
|
-
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
954
|
-
return { stored: false };
|
|
955
|
-
}
|
|
1157
|
+
const values = await txn.get([record, claim]);
|
|
956
1158
|
const indexed = values.get(record);
|
|
957
1159
|
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
958
1160
|
const existingClaim = values.get(claim);
|
|
1161
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
1162
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
1163
|
+
return { stored: false };
|
|
1164
|
+
}
|
|
959
1165
|
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
960
1166
|
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
|
|
961
1167
|
await txn.put(record, serializedHook);
|
|
962
|
-
if (existingClaim !== void 0) await txn.delete(claim);
|
|
1168
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
963
1169
|
return { stored: true };
|
|
964
1170
|
});
|
|
965
1171
|
}
|
|
@@ -967,26 +1173,70 @@ var HookTokenDO = class extends DurableObject {
|
|
|
967
1173
|
await this.ctx.storage.transaction(async (txn) => {
|
|
968
1174
|
const key = claimKey(token);
|
|
969
1175
|
const claim = await txn.get(key);
|
|
970
|
-
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
1176
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
971
1177
|
});
|
|
972
1178
|
}
|
|
973
|
-
async releaseBatch(hooks
|
|
1179
|
+
async releaseBatch(hooks) {
|
|
974
1180
|
if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
|
|
975
1181
|
if (hooks.length === 0) return { deleted: 0 };
|
|
976
1182
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
977
1183
|
const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
|
|
978
1184
|
const values = await txn.get(keys);
|
|
979
1185
|
const deletes = [];
|
|
1186
|
+
const deadlineDeletes = [];
|
|
980
1187
|
for (const { token, owner } of hooks) {
|
|
981
1188
|
const record = recordKey(token);
|
|
982
1189
|
const claim = claimKey(token);
|
|
983
1190
|
const indexed = values.get(record);
|
|
984
1191
|
const existingClaim = values.get(claim);
|
|
985
1192
|
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
986
|
-
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner))
|
|
1193
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
1194
|
+
deletes.push(claim);
|
|
1195
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
1196
|
+
}
|
|
987
1197
|
}
|
|
988
|
-
|
|
989
|
-
|
|
1198
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
1199
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
1200
|
+
return { deleted };
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
async alarm() {
|
|
1204
|
+
try {
|
|
1205
|
+
await this.compactClaims();
|
|
1206
|
+
} catch {
|
|
1207
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
async armAtMost(storage, at) {
|
|
1211
|
+
const current = await storage.getAlarm();
|
|
1212
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1213
|
+
}
|
|
1214
|
+
async compactClaims() {
|
|
1215
|
+
const now = this.now();
|
|
1216
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1217
|
+
const due = await txn.list({
|
|
1218
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1219
|
+
end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
|
|
1220
|
+
limit: 129
|
|
1221
|
+
});
|
|
1222
|
+
const page = Array.from(due).slice(0, 128);
|
|
1223
|
+
const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
|
|
1224
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
1225
|
+
const key = claimKey(item.token);
|
|
1226
|
+
const claim = claims.get(key);
|
|
1227
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
1228
|
+
});
|
|
1229
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1230
|
+
if (due.size > 128) {
|
|
1231
|
+
await txn.setAlarm(now + 1);
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
const nextAt = (await txn.list({
|
|
1235
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1236
|
+
limit: 1
|
|
1237
|
+
})).values().next().value?.expiresAt;
|
|
1238
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1239
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
990
1240
|
});
|
|
991
1241
|
}
|
|
992
1242
|
};
|
|
@@ -1001,8 +1251,6 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
|
1001
1251
|
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
1002
1252
|
const DEFAULT_MAX_INFLIGHT = 5;
|
|
1003
1253
|
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
1004
|
-
/** Grace period past the delivery timeout before a claim counts as lost. */
|
|
1005
|
-
const INFLIGHT_GRACE_MS = 3e4;
|
|
1006
1254
|
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
1007
1255
|
const MIN_ALARM_DELAY_MS = 1;
|
|
1008
1256
|
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
@@ -1011,14 +1259,15 @@ const STORAGE_BATCH_SIZE$2 = 128;
|
|
|
1011
1259
|
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
1012
1260
|
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
1013
1261
|
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
1014
|
-
|
|
1262
|
+
const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
|
|
1263
|
+
function pad$1(ms) {
|
|
1015
1264
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1016
1265
|
}
|
|
1017
1266
|
function dueKey(atMs, messageId) {
|
|
1018
|
-
return `due:${pad(atMs)}:${messageId}`;
|
|
1267
|
+
return `due:${pad$1(atMs)}:${messageId}`;
|
|
1019
1268
|
}
|
|
1020
1269
|
function inflightKey(deadlineMs, messageId) {
|
|
1021
|
-
return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
|
|
1270
|
+
return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
|
|
1022
1271
|
}
|
|
1023
1272
|
function deadlineFromInflightKey(key) {
|
|
1024
1273
|
return Number.parseInt(key.slice(18, 31), 10);
|
|
@@ -1029,6 +1278,9 @@ function runReferenceKey(runId, messageId) {
|
|
|
1029
1278
|
function expiredRunKey(runId) {
|
|
1030
1279
|
return `expired-run:${runId}`;
|
|
1031
1280
|
}
|
|
1281
|
+
function expiredRunGcKey(runId, compactAt) {
|
|
1282
|
+
return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1283
|
+
}
|
|
1032
1284
|
function boundedLimit$1(value, fallback, maximum) {
|
|
1033
1285
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1034
1286
|
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
@@ -1079,16 +1331,29 @@ var QueueDO = class extends DurableObject {
|
|
|
1079
1331
|
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
1080
1332
|
return configured;
|
|
1081
1333
|
}
|
|
1334
|
+
#deliveryTimeoutMs() {
|
|
1335
|
+
const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
1336
|
+
if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
|
|
1337
|
+
return configured;
|
|
1338
|
+
}
|
|
1339
|
+
async #authoritativeRunExpired(runId) {
|
|
1340
|
+
const namespace = this.env?.WORKFLOW_DB;
|
|
1341
|
+
if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
|
|
1342
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
|
|
1343
|
+
}
|
|
1344
|
+
async #withRunMutation(runId, onExpired, onAllowed) {
|
|
1345
|
+
const expired = await this.#authoritativeRunExpired(runId);
|
|
1346
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1347
|
+
if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
|
|
1348
|
+
return await onAllowed(txn);
|
|
1349
|
+
});
|
|
1350
|
+
}
|
|
1082
1351
|
async enqueue(request) {
|
|
1083
1352
|
this.#maxInflight();
|
|
1353
|
+
this.#deliveryTimeoutMs();
|
|
1084
1354
|
const storage = this.ctx.storage;
|
|
1085
1355
|
const now = this.#now();
|
|
1086
|
-
|
|
1087
|
-
if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
|
|
1088
|
-
ok: false,
|
|
1089
|
-
code: "RUN_EXPIRED",
|
|
1090
|
-
message: `Workflow run "${request.runId}" has expired`
|
|
1091
|
-
};
|
|
1356
|
+
const persist = async (txn) => {
|
|
1092
1357
|
const pinned = await txn.get("cfg");
|
|
1093
1358
|
if (!pinned) await txn.put("cfg", {
|
|
1094
1359
|
queueShards: request.config.queueShards,
|
|
@@ -1140,7 +1405,12 @@ var QueueDO = class extends DurableObject {
|
|
|
1140
1405
|
messageId: row.messageId,
|
|
1141
1406
|
deduped: false
|
|
1142
1407
|
};
|
|
1143
|
-
}
|
|
1408
|
+
};
|
|
1409
|
+
return request.runId ? await this.#withRunMutation(request.runId, async () => ({
|
|
1410
|
+
ok: false,
|
|
1411
|
+
code: "RUN_EXPIRED",
|
|
1412
|
+
message: `Workflow run "${request.runId}" has expired`
|
|
1413
|
+
}), persist) : await storage.transaction(persist);
|
|
1144
1414
|
}
|
|
1145
1415
|
async alarm() {
|
|
1146
1416
|
let claimed = [];
|
|
@@ -1154,7 +1424,10 @@ var QueueDO = class extends DurableObject {
|
|
|
1154
1424
|
};
|
|
1155
1425
|
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
1156
1426
|
else await attempt();
|
|
1157
|
-
if (failure !== null)
|
|
1427
|
+
if (failure !== null) {
|
|
1428
|
+
await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1429
|
+
throw failure;
|
|
1430
|
+
}
|
|
1158
1431
|
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
1159
1432
|
}
|
|
1160
1433
|
/**
|
|
@@ -1166,9 +1439,13 @@ var QueueDO = class extends DurableObject {
|
|
|
1166
1439
|
const now = this.#now();
|
|
1167
1440
|
const maxInflight = this.#maxInflight();
|
|
1168
1441
|
return await storage.transaction(async (txn) => {
|
|
1442
|
+
if (await this.#compactExpiredRunFences(txn, now)) {
|
|
1443
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1444
|
+
return [];
|
|
1445
|
+
}
|
|
1169
1446
|
const expiredPage = await txn.list({
|
|
1170
1447
|
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1171
|
-
end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
|
|
1448
|
+
end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
|
|
1172
1449
|
limit: 129
|
|
1173
1450
|
});
|
|
1174
1451
|
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
@@ -1200,17 +1477,17 @@ var QueueDO = class extends DurableObject {
|
|
|
1200
1477
|
if (inflightCount < maxInflight) {
|
|
1201
1478
|
const due = await txn.list({
|
|
1202
1479
|
prefix: "due:",
|
|
1203
|
-
end: `due:${pad(now + 1)}`,
|
|
1480
|
+
end: `due:${pad$1(now + 1)}`,
|
|
1204
1481
|
limit: maxInflight - inflightCount
|
|
1205
1482
|
});
|
|
1206
|
-
const timeoutMs = this.#
|
|
1483
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1207
1484
|
const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
1208
1485
|
await deleteMany$1(txn, Array.from(due.keys()));
|
|
1209
1486
|
const claimWrites = [];
|
|
1210
1487
|
for (const [, messageId] of due) {
|
|
1211
1488
|
const row = dueRows.get(`msg:${messageId}`);
|
|
1212
1489
|
if (!row) continue;
|
|
1213
|
-
const claimKey = inflightKey(now + timeoutMs +
|
|
1490
|
+
const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
|
|
1214
1491
|
claimWrites.push([claimKey, messageId]);
|
|
1215
1492
|
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
1216
1493
|
messageId: row.messageId,
|
|
@@ -1230,9 +1507,13 @@ var QueueDO = class extends DurableObject {
|
|
|
1230
1507
|
}
|
|
1231
1508
|
async #deliver({ row, inflightKey: claimKey }) {
|
|
1232
1509
|
const storage = this.ctx.storage;
|
|
1233
|
-
const timeoutMs = this.#
|
|
1510
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1234
1511
|
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
1235
1512
|
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
1513
|
+
if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
|
|
1514
|
+
await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
|
|
1515
|
+
return;
|
|
1516
|
+
}
|
|
1236
1517
|
let response = null;
|
|
1237
1518
|
let transportError = null;
|
|
1238
1519
|
try {
|
|
@@ -1262,28 +1543,21 @@ var QueueDO = class extends DurableObject {
|
|
|
1262
1543
|
} catch {
|
|
1263
1544
|
timeoutSeconds = void 0;
|
|
1264
1545
|
}
|
|
1265
|
-
if (typeof timeoutSeconds === "number")
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
await txn.
|
|
1270
|
-
if (row.
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1546
|
+
if (typeof timeoutSeconds === "number") {
|
|
1547
|
+
const reschedule = async (txn) => {
|
|
1548
|
+
if (!await txn.delete(claimKey)) return;
|
|
1549
|
+
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
1550
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1551
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1552
|
+
messageId: row.messageId,
|
|
1553
|
+
dueKey: scheduleKey,
|
|
1554
|
+
idempotencyKey: row.idempotencyKey
|
|
1555
|
+
});
|
|
1274
1556
|
await this.#scheduleNextAlarm(txn, now);
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1280
|
-
messageId: row.messageId,
|
|
1281
|
-
dueKey: scheduleKey,
|
|
1282
|
-
idempotencyKey: row.idempotencyKey
|
|
1283
|
-
});
|
|
1284
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
1285
|
-
});
|
|
1286
|
-
else await this.#retry(row, claimKey, "HTTP 503");
|
|
1557
|
+
};
|
|
1558
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
|
|
1559
|
+
else await storage.transaction(reschedule);
|
|
1560
|
+
} else await this.#retry(row, claimKey, "HTTP 503");
|
|
1287
1561
|
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
1288
1562
|
await cancelUnusedResponseBody(response);
|
|
1289
1563
|
await this.#ack(row, claimKey);
|
|
@@ -1304,23 +1578,23 @@ var QueueDO = class extends DurableObject {
|
|
|
1304
1578
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1305
1579
|
});
|
|
1306
1580
|
}
|
|
1581
|
+
async #dropExpiredMessage(txn, row, claimKey, now) {
|
|
1582
|
+
if (!await txn.delete(claimKey)) return;
|
|
1583
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
1584
|
+
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1585
|
+
if (row.idempotencyKey) {
|
|
1586
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1587
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1588
|
+
}
|
|
1589
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
1590
|
+
}
|
|
1307
1591
|
async #retry(row, claimKey, reason) {
|
|
1308
1592
|
const storage = this.ctx.storage;
|
|
1309
1593
|
const now = this.#now();
|
|
1310
1594
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
1311
1595
|
const attempt = row.attempt + 1;
|
|
1312
|
-
|
|
1596
|
+
const persistRetry = async (txn) => {
|
|
1313
1597
|
if (!await txn.delete(claimKey)) return;
|
|
1314
|
-
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
1315
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
1316
|
-
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1317
|
-
if (row.idempotencyKey) {
|
|
1318
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1319
|
-
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1320
|
-
}
|
|
1321
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
1322
|
-
return;
|
|
1323
|
-
}
|
|
1324
1598
|
if (attempt >= maxAttempts) {
|
|
1325
1599
|
const dead = {
|
|
1326
1600
|
...row,
|
|
@@ -1328,7 +1602,7 @@ var QueueDO = class extends DurableObject {
|
|
|
1328
1602
|
lastError: reason,
|
|
1329
1603
|
failedAt: now
|
|
1330
1604
|
};
|
|
1331
|
-
const deadKey = `dlq:${pad(now)}:${row.messageId}`;
|
|
1605
|
+
const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
|
|
1332
1606
|
await txn.put(deadKey, dead);
|
|
1333
1607
|
await txn.delete(`msg:${row.messageId}`);
|
|
1334
1608
|
if (row.idempotencyKey) {
|
|
@@ -1356,7 +1630,32 @@ var QueueDO = class extends DurableObject {
|
|
|
1356
1630
|
idempotencyKey: row.idempotencyKey
|
|
1357
1631
|
});
|
|
1358
1632
|
await this.#scheduleNextAlarm(txn, now);
|
|
1633
|
+
};
|
|
1634
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
|
|
1635
|
+
else await storage.transaction(persistRetry);
|
|
1636
|
+
}
|
|
1637
|
+
/** Return true when another due compaction page must run immediately. */
|
|
1638
|
+
async #compactExpiredRunFences(txn, now) {
|
|
1639
|
+
const due = await txn.list({
|
|
1640
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1641
|
+
end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
|
|
1642
|
+
limit: 129
|
|
1359
1643
|
});
|
|
1644
|
+
const page = Array.from(due).slice(0, 128);
|
|
1645
|
+
if (page.length === 0) return false;
|
|
1646
|
+
const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
|
|
1647
|
+
const markers = await getMany$1(txn, markerKeys);
|
|
1648
|
+
const deletes = [];
|
|
1649
|
+
for (let index = 0; index < page.length; index++) {
|
|
1650
|
+
const [gcKey, item] = page[index];
|
|
1651
|
+
const markerKey = markerKeys[index];
|
|
1652
|
+
const marker = markers.get(markerKey);
|
|
1653
|
+
deletes.push(gcKey);
|
|
1654
|
+
if (marker?.compactAt !== item.compactAt) continue;
|
|
1655
|
+
deletes.push(markerKey);
|
|
1656
|
+
}
|
|
1657
|
+
await deleteMany$1(txn, deletes);
|
|
1658
|
+
return due.size > 128;
|
|
1360
1659
|
}
|
|
1361
1660
|
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
1362
1661
|
async #armAlarmAtMost(storage, atMs, now) {
|
|
@@ -1388,6 +1687,11 @@ var QueueDO = class extends DurableObject {
|
|
|
1388
1687
|
const deadline = deadlineFromInflightKey(key);
|
|
1389
1688
|
if (next === null || deadline < next) next = deadline;
|
|
1390
1689
|
}
|
|
1690
|
+
const expiryGc = await storage.list({
|
|
1691
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1692
|
+
limit: 1
|
|
1693
|
+
});
|
|
1694
|
+
for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
|
|
1391
1695
|
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
1392
1696
|
else await storage.deleteAlarm();
|
|
1393
1697
|
}
|
|
@@ -1426,37 +1730,42 @@ var QueueDO = class extends DurableObject {
|
|
|
1426
1730
|
async redriveDeadLetter(messageId) {
|
|
1427
1731
|
const storage = this.ctx.storage;
|
|
1428
1732
|
const now = this.#now();
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
await txn.put(
|
|
1447
|
-
const scheduleKey = dueKey(now, row.messageId);
|
|
1448
|
-
await txn.put(scheduleKey, row.messageId);
|
|
1449
|
-
await txn.delete(key);
|
|
1450
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1451
|
-
messageId: row.messageId,
|
|
1452
|
-
dueKey: scheduleKey,
|
|
1453
|
-
idempotencyKey: row.idempotencyKey
|
|
1454
|
-
});
|
|
1455
|
-
await this.#armAlarmAtMost(txn, now, now);
|
|
1456
|
-
return { ok: true };
|
|
1733
|
+
const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
|
|
1734
|
+
if (!found) return { ok: false };
|
|
1735
|
+
const [deadKey, candidate] = found;
|
|
1736
|
+
const redrive = async (txn) => {
|
|
1737
|
+
const dead = await txn.get(deadKey);
|
|
1738
|
+
if (!dead || dead.messageId !== messageId) return { ok: false };
|
|
1739
|
+
const { failedAt: _failedAt, ...rest } = dead;
|
|
1740
|
+
const row = {
|
|
1741
|
+
...rest,
|
|
1742
|
+
attempt: 0,
|
|
1743
|
+
lastError: void 0
|
|
1744
|
+
};
|
|
1745
|
+
if (row.idempotencyKey) {
|
|
1746
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1747
|
+
const holder = await txn.get(dedupKey);
|
|
1748
|
+
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
1749
|
+
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
1750
|
+
await txn.put(dedupKey, row.messageId);
|
|
1457
1751
|
}
|
|
1752
|
+
await txn.put(`msg:${row.messageId}`, row);
|
|
1753
|
+
const scheduleKey = dueKey(now, row.messageId);
|
|
1754
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1755
|
+
await txn.delete(deadKey);
|
|
1756
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1757
|
+
messageId: row.messageId,
|
|
1758
|
+
dueKey: scheduleKey,
|
|
1759
|
+
idempotencyKey: row.idempotencyKey
|
|
1760
|
+
});
|
|
1761
|
+
await this.#armAlarmAtMost(txn, now, now);
|
|
1762
|
+
return { ok: true };
|
|
1763
|
+
};
|
|
1764
|
+
const rejectExpired = async (txn) => {
|
|
1765
|
+
if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
|
|
1458
1766
|
return { ok: false };
|
|
1459
|
-
}
|
|
1767
|
+
};
|
|
1768
|
+
return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
|
|
1460
1769
|
}
|
|
1461
1770
|
async purgeDeadLetters() {
|
|
1462
1771
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
@@ -1506,27 +1815,75 @@ var QueueDO = class extends DurableObject {
|
|
|
1506
1815
|
count++;
|
|
1507
1816
|
}
|
|
1508
1817
|
await deleteMany$1(txn, keys);
|
|
1818
|
+
const done = references.size <= limit;
|
|
1819
|
+
const markerExpiredAt = existing?.expiredAt ?? expiredAt;
|
|
1509
1820
|
await txn.put(expiredRunKey(runId), {
|
|
1510
|
-
expiredAt,
|
|
1511
|
-
deleted: count
|
|
1821
|
+
expiredAt: markerExpiredAt,
|
|
1822
|
+
deleted: count,
|
|
1823
|
+
compactAt: existing?.compactAt ?? null
|
|
1512
1824
|
});
|
|
1513
1825
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1514
|
-
return {
|
|
1826
|
+
return done ? {
|
|
1827
|
+
deleted: count,
|
|
1828
|
+
done: true,
|
|
1829
|
+
receipt: {
|
|
1830
|
+
expiredAt: markerExpiredAt,
|
|
1831
|
+
deleted: count
|
|
1832
|
+
}
|
|
1833
|
+
} : {
|
|
1515
1834
|
deleted: count,
|
|
1516
|
-
done:
|
|
1835
|
+
done: false
|
|
1517
1836
|
};
|
|
1518
1837
|
});
|
|
1519
1838
|
}
|
|
1839
|
+
/** Acknowledge a final cleanup receipt before making its marker compactable. */
|
|
1840
|
+
async acknowledgeExpireRun(runId, receipt) {
|
|
1841
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1842
|
+
const key = expiredRunKey(runId);
|
|
1843
|
+
const marker = await txn.get(key);
|
|
1844
|
+
if (!marker) return { acknowledged: true };
|
|
1845
|
+
if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
|
|
1846
|
+
if (marker.compactAt === null) {
|
|
1847
|
+
const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
|
|
1848
|
+
await txn.put(key, {
|
|
1849
|
+
...marker,
|
|
1850
|
+
compactAt
|
|
1851
|
+
});
|
|
1852
|
+
await txn.put(expiredRunGcKey(runId, compactAt), {
|
|
1853
|
+
runId,
|
|
1854
|
+
compactAt
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1858
|
+
return { acknowledged: true };
|
|
1859
|
+
});
|
|
1860
|
+
}
|
|
1520
1861
|
};
|
|
1521
1862
|
//#endregion
|
|
1522
1863
|
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
1523
1864
|
const MAX_INDEX_LIST_SIZE = 1001;
|
|
1865
|
+
const EXPIRY_GC_PREFIX = "expiry-gc:";
|
|
1866
|
+
function pad(ms) {
|
|
1867
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1868
|
+
}
|
|
1869
|
+
function expiredKey(runId) {
|
|
1870
|
+
return `expired:${runId}`;
|
|
1871
|
+
}
|
|
1872
|
+
function expiryGcKey(runId, compactAt) {
|
|
1873
|
+
return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1874
|
+
}
|
|
1524
1875
|
/** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
|
|
1525
1876
|
var RunCatalogDO = class extends DurableObject {
|
|
1526
|
-
|
|
1877
|
+
now() {
|
|
1878
|
+
const clock = this.env?.clock;
|
|
1879
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
1880
|
+
}
|
|
1881
|
+
async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
|
|
1527
1882
|
if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
|
|
1528
1883
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1529
|
-
if (await txn.get(
|
|
1884
|
+
if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
|
|
1885
|
+
const now = this.now();
|
|
1886
|
+
if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
|
|
1530
1887
|
await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
|
|
1531
1888
|
return { stored: true };
|
|
1532
1889
|
});
|
|
@@ -1538,8 +1895,11 @@ var RunCatalogDO = class extends DurableObject {
|
|
|
1538
1895
|
prefix: options.prefix ?? "",
|
|
1539
1896
|
...options.reverse ? {
|
|
1540
1897
|
reverse: true,
|
|
1541
|
-
end: options.cursor
|
|
1542
|
-
} : {
|
|
1898
|
+
end: options.cursor ?? options.end
|
|
1899
|
+
} : {
|
|
1900
|
+
startAfter: options.cursor,
|
|
1901
|
+
end: options.end
|
|
1902
|
+
},
|
|
1543
1903
|
limit: limit + 1
|
|
1544
1904
|
});
|
|
1545
1905
|
const page = Array.from(entries).slice(0, limit);
|
|
@@ -1553,41 +1913,68 @@ var RunCatalogDO = class extends DurableObject {
|
|
|
1553
1913
|
cursor: listComplete ? void 0 : page.at(-1)?.[0]
|
|
1554
1914
|
};
|
|
1555
1915
|
}
|
|
1916
|
+
/** Compare-and-delete one rejected retention discovery candidate. */
|
|
1917
|
+
async deleteStaleGlobalRun(runId, key, expectedValue) {
|
|
1918
|
+
if (!key.startsWith("runall:") || !key.endsWith(`:${runId}`)) throw new Error("Stale run catalog deletion requires the exact global run key");
|
|
1919
|
+
if (JSON.parse(expectedValue).runId !== runId) throw new Error("Stale run catalog deletion requires matching run metadata");
|
|
1920
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1921
|
+
if (await txn.get(key) !== expectedValue) return { deleted: false };
|
|
1922
|
+
return { deleted: await txn.delete(key) };
|
|
1923
|
+
});
|
|
1924
|
+
}
|
|
1556
1925
|
async expireRun(runId, keys, expiredAt) {
|
|
1557
1926
|
if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
|
|
1558
1927
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1559
|
-
|
|
1928
|
+
const marker = expiredKey(runId);
|
|
1929
|
+
const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
|
|
1930
|
+
await txn.put(marker, {
|
|
1931
|
+
expiredAt,
|
|
1932
|
+
compactAt
|
|
1933
|
+
});
|
|
1934
|
+
await txn.put(expiryGcKey(runId, compactAt), {
|
|
1935
|
+
runId,
|
|
1936
|
+
compactAt
|
|
1937
|
+
});
|
|
1938
|
+
await this.armAtMost(txn, compactAt);
|
|
1560
1939
|
return { deleted: await txn.delete(keys) };
|
|
1561
1940
|
});
|
|
1562
1941
|
}
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
return (await this.ctx.storage.get("fence"))?.status ?? "active";
|
|
1942
|
+
async alarm() {
|
|
1943
|
+
try {
|
|
1944
|
+
await this.compactExpiryFences();
|
|
1945
|
+
} catch {
|
|
1946
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1947
|
+
}
|
|
1570
1948
|
}
|
|
1571
|
-
async
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
if (current) return current.status;
|
|
1575
|
-
await txn.put("fence", {
|
|
1576
|
-
status: "terminal",
|
|
1577
|
-
at
|
|
1578
|
-
});
|
|
1579
|
-
return "terminal";
|
|
1580
|
-
});
|
|
1949
|
+
async armAtMost(storage, at) {
|
|
1950
|
+
const current = await storage.getAlarm();
|
|
1951
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1581
1952
|
}
|
|
1582
|
-
async
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1953
|
+
async compactExpiryFences() {
|
|
1954
|
+
const now = this.now();
|
|
1955
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1956
|
+
const due = await txn.list({
|
|
1957
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1958
|
+
end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
|
|
1959
|
+
limit: 129
|
|
1960
|
+
});
|
|
1961
|
+
const page = Array.from(due).slice(0, 128);
|
|
1962
|
+
const markerKeys = page.map(([, item]) => expiredKey(item.runId));
|
|
1963
|
+
const markers = await txn.get(markerKeys);
|
|
1964
|
+
const deletes = page.flatMap(([key, item], index) => {
|
|
1965
|
+
return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
|
|
1589
1966
|
});
|
|
1590
|
-
|
|
1967
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1968
|
+
if (due.size > 128) {
|
|
1969
|
+
await txn.setAlarm(now + 1);
|
|
1970
|
+
return;
|
|
1971
|
+
}
|
|
1972
|
+
const nextAt = (await txn.list({
|
|
1973
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1974
|
+
limit: 1
|
|
1975
|
+
})).values().next().value?.compactAt;
|
|
1976
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1977
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
1591
1978
|
});
|
|
1592
1979
|
}
|
|
1593
1980
|
};
|
|
@@ -2793,42 +3180,6 @@ async function finalizeEventPage(store, outcome, params) {
|
|
|
2793
3180
|
};
|
|
2794
3181
|
}
|
|
2795
3182
|
//#endregion
|
|
2796
|
-
//#region src/retention.ts
|
|
2797
|
-
const CLEANUP_RECORD_KEY = "retention:cleanup";
|
|
2798
|
-
const TOMBSTONE_KEY = "retention:tombstone";
|
|
2799
|
-
const HOOK_MARKER_PREFIX = "retention:hook:";
|
|
2800
|
-
const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
|
|
2801
|
-
function sortableTimestamp(date) {
|
|
2802
|
-
return date.getTime().toString().padStart(13, "0");
|
|
2803
|
-
}
|
|
2804
|
-
function workflowRunIndexKey(run) {
|
|
2805
|
-
return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
2806
|
-
}
|
|
2807
|
-
function globalRunIndexKey(run) {
|
|
2808
|
-
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
2809
|
-
}
|
|
2810
|
-
function hookMarkerKey(reference) {
|
|
2811
|
-
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
2812
|
-
}
|
|
2813
|
-
function cleanupTombstone(record, now) {
|
|
2814
|
-
return {
|
|
2815
|
-
version: 1,
|
|
2816
|
-
runId: record.runId,
|
|
2817
|
-
terminalStatus: record.terminalStatus,
|
|
2818
|
-
completedAt: record.completedAt,
|
|
2819
|
-
expiredAt: record.dueAt,
|
|
2820
|
-
tombstonedAt: now
|
|
2821
|
-
};
|
|
2822
|
-
}
|
|
2823
|
-
function expiredRead(tombstone) {
|
|
2824
|
-
return {
|
|
2825
|
-
ok: false,
|
|
2826
|
-
code: "RUN_EXPIRED",
|
|
2827
|
-
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
|
|
2828
|
-
tombstone
|
|
2829
|
-
};
|
|
2830
|
-
}
|
|
2831
|
-
//#endregion
|
|
2832
3183
|
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
2833
3184
|
const STORAGE_BATCH_SIZE = 128;
|
|
2834
3185
|
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
@@ -2839,9 +3190,30 @@ const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
|
2839
3190
|
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
2840
3191
|
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
2841
3192
|
const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
|
|
2842
|
-
const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
|
|
2843
3193
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
2844
3194
|
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
3195
|
+
const RUN_QUEUE_SHARDS_KEY = "retention:queue-shards";
|
|
3196
|
+
function cleanupRecord(run, dueAt, queueShards, reason) {
|
|
3197
|
+
const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
|
|
3198
|
+
return {
|
|
3199
|
+
version: 1,
|
|
3200
|
+
runId: run.runId,
|
|
3201
|
+
workflowName: run.workflowName,
|
|
3202
|
+
createdAt: run.createdAt,
|
|
3203
|
+
status: run.status,
|
|
3204
|
+
completedAt: terminal?.completedAt,
|
|
3205
|
+
terminalStatus: terminal?.status,
|
|
3206
|
+
reason,
|
|
3207
|
+
dueAt,
|
|
3208
|
+
queueShards,
|
|
3209
|
+
phase: "retained",
|
|
3210
|
+
generation: 0,
|
|
3211
|
+
attempts: 0,
|
|
3212
|
+
deletedPayloadKeys: 0,
|
|
3213
|
+
deletedStreams: 0,
|
|
3214
|
+
deletedQueueMessages: 0
|
|
3215
|
+
};
|
|
3216
|
+
}
|
|
2845
3217
|
/**
|
|
2846
3218
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
2847
3219
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -2937,6 +3309,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2937
3309
|
now
|
|
2938
3310
|
});
|
|
2939
3311
|
if (!outcome.ok) return outcome;
|
|
3312
|
+
if (outcome.runCreated && request.cleanup) await txn.put(RUN_QUEUE_SHARDS_KEY, request.cleanup.queueShards);
|
|
2940
3313
|
await txn.put("event_sequence", eventSequence);
|
|
2941
3314
|
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
2942
3315
|
hookId: outcome.hookToIndex.hookId,
|
|
@@ -2950,22 +3323,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2950
3323
|
...terminalRun,
|
|
2951
3324
|
expiredAt: dueAt
|
|
2952
3325
|
});
|
|
2953
|
-
const cleanup =
|
|
2954
|
-
version: 1,
|
|
2955
|
-
runId: run.runId,
|
|
2956
|
-
workflowName: run.workflowName,
|
|
2957
|
-
createdAt: run.createdAt,
|
|
2958
|
-
completedAt: run.completedAt,
|
|
2959
|
-
terminalStatus: run.status,
|
|
2960
|
-
dueAt,
|
|
2961
|
-
queueShards: request.cleanup?.queueShards ?? 1,
|
|
2962
|
-
phase: "retained",
|
|
2963
|
-
generation: 0,
|
|
2964
|
-
attempts: 0,
|
|
2965
|
-
deletedPayloadKeys: 0,
|
|
2966
|
-
deletedStreams: 0,
|
|
2967
|
-
deletedQueueMessages: 0
|
|
2968
|
-
};
|
|
3326
|
+
const cleanup = cleanupRecord(run, dueAt, request.cleanup?.queueShards ?? 1, "terminal-retention");
|
|
2969
3327
|
await txn.put("run", run);
|
|
2970
3328
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
2971
3329
|
await txn.setAlarm(dueAt);
|
|
@@ -2981,6 +3339,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2981
3339
|
});
|
|
2982
3340
|
await txn.setAlarm(this.now() + 1);
|
|
2983
3341
|
}
|
|
3342
|
+
if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
|
|
2984
3343
|
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
2985
3344
|
});
|
|
2986
3345
|
}
|
|
@@ -2991,6 +3350,8 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2991
3350
|
*/
|
|
2992
3351
|
async resolveHookTokenClaim(request) {
|
|
2993
3352
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
3353
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3354
|
+
if (tombstone) return { committed: false };
|
|
2994
3355
|
if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
|
|
2995
3356
|
await txn.put(hookClaimCancellationKey(request.claimId), {
|
|
2996
3357
|
hookId: request.hookId,
|
|
@@ -3003,6 +3364,20 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3003
3364
|
async getRun() {
|
|
3004
3365
|
return this.readKey("run");
|
|
3005
3366
|
}
|
|
3367
|
+
/** Single authoritative lifecycle answer used after derivative compaction. */
|
|
3368
|
+
async getLifecycleStatus() {
|
|
3369
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3370
|
+
const values = await txn.get([
|
|
3371
|
+
TOMBSTONE_KEY,
|
|
3372
|
+
CLEANUP_RECORD_KEY,
|
|
3373
|
+
"run"
|
|
3374
|
+
]);
|
|
3375
|
+
if (this.tombstoneFrom(values)) return "expired";
|
|
3376
|
+
const run = values.get("run");
|
|
3377
|
+
if (!run) return "missing";
|
|
3378
|
+
return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
|
|
3379
|
+
});
|
|
3380
|
+
}
|
|
3006
3381
|
async getStep(stepId) {
|
|
3007
3382
|
return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
|
|
3008
3383
|
}
|
|
@@ -3031,7 +3406,11 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3031
3406
|
}));
|
|
3032
3407
|
}
|
|
3033
3408
|
async getCleanupStatus() {
|
|
3034
|
-
|
|
3409
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
|
|
3410
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3411
|
+
if (cleanup) return cleanup;
|
|
3412
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
3413
|
+
return tombstone ? cleanupFromTombstone(tombstone) : null;
|
|
3035
3414
|
}
|
|
3036
3415
|
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
3037
3416
|
async scheduleCleanup(request) {
|
|
@@ -3054,28 +3433,95 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3054
3433
|
...terminalRun,
|
|
3055
3434
|
expiredAt: dueAt
|
|
3056
3435
|
});
|
|
3057
|
-
const cleanup =
|
|
3058
|
-
version: 1,
|
|
3059
|
-
runId: run.runId,
|
|
3060
|
-
workflowName: run.workflowName,
|
|
3061
|
-
createdAt: run.createdAt,
|
|
3062
|
-
completedAt: run.completedAt,
|
|
3063
|
-
terminalStatus: run.status,
|
|
3064
|
-
dueAt,
|
|
3065
|
-
queueShards: request.queueShards,
|
|
3066
|
-
phase: "retained",
|
|
3067
|
-
generation: 0,
|
|
3068
|
-
attempts: 0,
|
|
3069
|
-
deletedPayloadKeys: 0,
|
|
3070
|
-
deletedStreams: 0,
|
|
3071
|
-
deletedQueueMessages: 0
|
|
3072
|
-
};
|
|
3436
|
+
const cleanup = cleanupRecord(run, dueAt, request.queueShards, "manual");
|
|
3073
3437
|
await txn.put("run", run);
|
|
3074
3438
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3075
3439
|
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
3076
3440
|
return cleanup;
|
|
3077
3441
|
});
|
|
3078
3442
|
}
|
|
3443
|
+
/**
|
|
3444
|
+
* Apply the fleet-wide maximum-age policy to one authoritative run.
|
|
3445
|
+
*
|
|
3446
|
+
* The catalog is only a discovery index: eligibility is rechecked from the
|
|
3447
|
+
* RunDO's own createdAt. Once eligible, the cleanup deadline immediately
|
|
3448
|
+
* becomes an authoritative write/read fence, including for active runs.
|
|
3449
|
+
*/
|
|
3450
|
+
async enforceRetention(request) {
|
|
3451
|
+
if (!Number.isSafeInteger(request.retentionMs) || request.retentionMs <= 0) throw new Error("world-celld retentionMs must be a positive safe integer");
|
|
3452
|
+
if (!Number.isSafeInteger(request.queueShards) || request.queueShards <= 0) throw new Error("world-celld retention queueShards must be a positive safe integer");
|
|
3453
|
+
if (!Number.isSafeInteger(request.scheduledTime) || request.scheduledTime < 0) throw new Error("world-celld retention scheduledTime must be a non-negative safe integer");
|
|
3454
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
3455
|
+
const values = await txn.get([
|
|
3456
|
+
CLEANUP_RECORD_KEY,
|
|
3457
|
+
TOMBSTONE_KEY,
|
|
3458
|
+
RUN_QUEUE_SHARDS_KEY,
|
|
3459
|
+
"run"
|
|
3460
|
+
]);
|
|
3461
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
3462
|
+
if (tombstone) return {
|
|
3463
|
+
state: "expired",
|
|
3464
|
+
cleanup: cleanupFromTombstone(tombstone)
|
|
3465
|
+
};
|
|
3466
|
+
const run = values.get("run");
|
|
3467
|
+
if (!run) return {
|
|
3468
|
+
state: "missing",
|
|
3469
|
+
cleanup: null
|
|
3470
|
+
};
|
|
3471
|
+
const dueAtMs = run.createdAt.getTime() + request.retentionMs;
|
|
3472
|
+
if (!Number.isSafeInteger(dueAtMs) || dueAtMs > 864e13) throw new Error(`world-celld retention deadline is out of range for ${run.runId}`);
|
|
3473
|
+
if (dueAtMs > request.scheduledTime) return {
|
|
3474
|
+
state: "not-due",
|
|
3475
|
+
cleanup: null
|
|
3476
|
+
};
|
|
3477
|
+
const dueAt = new Date(dueAtMs);
|
|
3478
|
+
const existing = values.get(CLEANUP_RECORD_KEY);
|
|
3479
|
+
const storedQueueShards = values.get(RUN_QUEUE_SHARDS_KEY);
|
|
3480
|
+
const queueShards = Number.isSafeInteger(storedQueueShards) && storedQueueShards > 0 ? storedQueueShards : request.queueShards;
|
|
3481
|
+
let cleanup = existing;
|
|
3482
|
+
if (!cleanup) {
|
|
3483
|
+
cleanup = cleanupRecord(run, dueAt, queueShards, "maximum-age");
|
|
3484
|
+
await txn.put("run", WorkflowRunSchema.parse({
|
|
3485
|
+
...run,
|
|
3486
|
+
expiredAt: dueAt
|
|
3487
|
+
}));
|
|
3488
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3489
|
+
} else if (cleanup.phase === "retained" && dueAtMs < cleanup.dueAt.getTime()) {
|
|
3490
|
+
const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
|
|
3491
|
+
cleanup = {
|
|
3492
|
+
...cleanup,
|
|
3493
|
+
status: run.status,
|
|
3494
|
+
completedAt: terminal?.completedAt,
|
|
3495
|
+
terminalStatus: terminal?.status,
|
|
3496
|
+
reason: "maximum-age",
|
|
3497
|
+
dueAt,
|
|
3498
|
+
generation: cleanup.generation + 1
|
|
3499
|
+
};
|
|
3500
|
+
await txn.put("run", WorkflowRunSchema.parse({
|
|
3501
|
+
...run,
|
|
3502
|
+
expiredAt: dueAt
|
|
3503
|
+
}));
|
|
3504
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3505
|
+
}
|
|
3506
|
+
await txn.setAlarm(this.now() + 1);
|
|
3507
|
+
return {
|
|
3508
|
+
state: cleanup.dueAt.getTime() <= this.now() ? "expired" : "scheduled",
|
|
3509
|
+
cleanup
|
|
3510
|
+
};
|
|
3511
|
+
});
|
|
3512
|
+
if (result.state === "scheduled" || result.state === "expired") {
|
|
3513
|
+
for (let page = 0; page < 2; page++) {
|
|
3514
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3515
|
+
if (cleanup?.phase !== "retained" && cleanup?.phase !== "index") break;
|
|
3516
|
+
await this.executeScheduledCleanupPage();
|
|
3517
|
+
}
|
|
3518
|
+
return {
|
|
3519
|
+
...result,
|
|
3520
|
+
cleanup: await this.getCleanupStatus() ?? result.cleanup
|
|
3521
|
+
};
|
|
3522
|
+
}
|
|
3523
|
+
return result;
|
|
3524
|
+
}
|
|
3079
3525
|
async cleanupNow(request) {
|
|
3080
3526
|
if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
|
|
3081
3527
|
...request,
|
|
@@ -3119,7 +3565,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3119
3565
|
workflowIndex() {
|
|
3120
3566
|
return createWorkflowIndex({
|
|
3121
3567
|
runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
|
|
3122
|
-
runFences: this.namespace("WORKFLOW_RUN_FENCES"),
|
|
3123
3568
|
hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
|
|
3124
3569
|
hookIds: this.namespace("WORKFLOW_HOOK_IDS")
|
|
3125
3570
|
});
|
|
@@ -3140,7 +3585,17 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3140
3585
|
});
|
|
3141
3586
|
}
|
|
3142
3587
|
async executeScheduledCleanupPage() {
|
|
3143
|
-
const
|
|
3588
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
|
|
3589
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3590
|
+
const terminalCleanup = values.get(TERMINAL_CLEANUP_KEY);
|
|
3591
|
+
if (cleanup && cleanup.phase !== "tombstoned" && cleanup.dueAt.getTime() <= this.now()) {
|
|
3592
|
+
try {
|
|
3593
|
+
await this.executeCleanup(cleanup);
|
|
3594
|
+
} catch (error) {
|
|
3595
|
+
await this.recordCleanupFailure(error, cleanup);
|
|
3596
|
+
}
|
|
3597
|
+
return;
|
|
3598
|
+
}
|
|
3144
3599
|
if (terminalCleanup) {
|
|
3145
3600
|
try {
|
|
3146
3601
|
await this.executeTerminalCleanup(terminalCleanup);
|
|
@@ -3149,7 +3604,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3149
3604
|
}
|
|
3150
3605
|
return;
|
|
3151
3606
|
}
|
|
3152
|
-
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3153
3607
|
if (!cleanup || cleanup.phase === "tombstoned") {
|
|
3154
3608
|
await this.ctx.storage.deleteAlarm();
|
|
3155
3609
|
return;
|
|
@@ -3199,8 +3653,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3199
3653
|
hooks: page.map((hook) => ({
|
|
3200
3654
|
hookId: hook.hookId,
|
|
3201
3655
|
token: hook.token
|
|
3202
|
-
}))
|
|
3203
|
-
terminal: true
|
|
3656
|
+
}))
|
|
3204
3657
|
});
|
|
3205
3658
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3206
3659
|
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
@@ -3225,8 +3678,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3225
3678
|
const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3226
3679
|
await this.workflowIndex().releaseHookIndexes({
|
|
3227
3680
|
runId: cleanup.runId,
|
|
3228
|
-
hooks: page.map(([, reference]) => reference)
|
|
3229
|
-
terminal: true
|
|
3681
|
+
hooks: page.map(([, reference]) => reference)
|
|
3230
3682
|
});
|
|
3231
3683
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3232
3684
|
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
@@ -3336,33 +3788,56 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3336
3788
|
}
|
|
3337
3789
|
async deleteQueuePage(cleanup) {
|
|
3338
3790
|
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
3339
|
-
|
|
3791
|
+
const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3340
3792
|
queueShard: 0,
|
|
3341
3793
|
queueShardDeleted: 0
|
|
3342
3794
|
};
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3795
|
+
if (progress.pendingAck) {
|
|
3796
|
+
const { queueShard, receipt } = progress.pendingAck;
|
|
3797
|
+
if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
|
|
3798
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3799
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3800
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3801
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
|
|
3802
|
+
if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
|
|
3803
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3804
|
+
...current,
|
|
3805
|
+
generation: current.generation + 1,
|
|
3806
|
+
attempts: 0,
|
|
3807
|
+
lastError: void 0
|
|
3808
|
+
});
|
|
3809
|
+
await txn.put(CLEANUP_PROGRESS_KEY, {
|
|
3810
|
+
queueShard: queueShard + 1,
|
|
3811
|
+
queueShardDeleted: 0
|
|
3812
|
+
});
|
|
3813
|
+
await txn.setAlarm(this.now() + 1);
|
|
3814
|
+
});
|
|
3815
|
+
return;
|
|
3816
|
+
}
|
|
3817
|
+
if (progress.queueShard < cleanup.queueShards) {
|
|
3346
3818
|
const requestedShard = progress.queueShard;
|
|
3347
3819
|
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
3348
|
-
|
|
3820
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3349
3821
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3350
|
-
if (current?.phase !== "queues") return
|
|
3351
|
-
if (current.generation !== expectedGeneration) return { stale: true };
|
|
3822
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3352
3823
|
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3353
3824
|
queueShard: 0,
|
|
3354
3825
|
queueShardDeleted: 0
|
|
3355
3826
|
};
|
|
3356
|
-
if (persisted.queueShard !== requestedShard) return
|
|
3357
|
-
if (result.deleted < persisted.queueShardDeleted)
|
|
3827
|
+
if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
|
|
3828
|
+
if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
|
|
3829
|
+
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3358
3830
|
const next = result.done ? {
|
|
3359
|
-
queueShard:
|
|
3360
|
-
queueShardDeleted:
|
|
3831
|
+
queueShard: requestedShard,
|
|
3832
|
+
queueShardDeleted: result.deleted,
|
|
3833
|
+
pendingAck: {
|
|
3834
|
+
queueShard: requestedShard,
|
|
3835
|
+
receipt: result.receipt
|
|
3836
|
+
}
|
|
3361
3837
|
} : {
|
|
3362
|
-
|
|
3838
|
+
queueShard: requestedShard,
|
|
3363
3839
|
queueShardDeleted: result.deleted
|
|
3364
3840
|
};
|
|
3365
|
-
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3366
3841
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3367
3842
|
...current,
|
|
3368
3843
|
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
@@ -3372,27 +3847,17 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3372
3847
|
});
|
|
3373
3848
|
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
3374
3849
|
await txn.setAlarm(this.now() + 1);
|
|
3375
|
-
return {
|
|
3376
|
-
stale: false,
|
|
3377
|
-
progress: next,
|
|
3378
|
-
generation: current.generation + 1,
|
|
3379
|
-
applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
|
|
3380
|
-
};
|
|
3381
3850
|
});
|
|
3382
|
-
|
|
3383
|
-
if (reconciled.stale) return;
|
|
3384
|
-
progress = reconciled.progress;
|
|
3385
|
-
expectedGeneration = reconciled.generation;
|
|
3386
|
-
if (!reconciled.applied && progress.queueShard === requestedShard) break;
|
|
3387
|
-
if (!result.done && progress.queueShard === requestedShard) break;
|
|
3851
|
+
return;
|
|
3388
3852
|
}
|
|
3389
3853
|
await this.ctx.storage.transaction(async (txn) => {
|
|
3390
3854
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3391
|
-
if (current?.phase !== "queues") return;
|
|
3392
|
-
|
|
3855
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3856
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3393
3857
|
queueShard: 0,
|
|
3394
3858
|
queueShardDeleted: 0
|
|
3395
|
-
}
|
|
3859
|
+
};
|
|
3860
|
+
if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
|
|
3396
3861
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3397
3862
|
...current,
|
|
3398
3863
|
phase: "payload",
|
|
@@ -3425,14 +3890,16 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3425
3890
|
await txn.setAlarm(this.now() + 1);
|
|
3426
3891
|
return false;
|
|
3427
3892
|
}
|
|
3428
|
-
|
|
3893
|
+
const completed = {
|
|
3429
3894
|
...cleanup,
|
|
3430
3895
|
phase: "tombstoned",
|
|
3431
3896
|
generation: cleanup.generation + 1,
|
|
3432
3897
|
attempts: 0,
|
|
3433
3898
|
lastError: void 0,
|
|
3434
3899
|
tombstonedAt: now
|
|
3435
|
-
}
|
|
3900
|
+
};
|
|
3901
|
+
await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
|
|
3902
|
+
await txn.delete(CLEANUP_RECORD_KEY);
|
|
3436
3903
|
await txn.deleteAlarm();
|
|
3437
3904
|
return true;
|
|
3438
3905
|
});
|
|
@@ -3492,10 +3959,20 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3492
3959
|
* default fetch router. This module (and everything it imports) must stay
|
|
3493
3960
|
* free of Node built-ins — it runs inside celld's workerd runtime.
|
|
3494
3961
|
*/
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
|
|
3962
|
+
async function scheduled(controller, env) {
|
|
3963
|
+
const result = await runRetentionSweep(controller.scheduledTime, env);
|
|
3964
|
+
if (result.scanned > 0) console.info("world-celld retention sweep", {
|
|
3965
|
+
cron: controller.cron,
|
|
3966
|
+
...result
|
|
3967
|
+
});
|
|
3968
|
+
}
|
|
3969
|
+
var worker_default = {
|
|
3970
|
+
async fetch(request, env) {
|
|
3971
|
+
return createRouter(env)(request);
|
|
3972
|
+
},
|
|
3973
|
+
scheduled
|
|
3974
|
+
};
|
|
3498
3975
|
//#endregion
|
|
3499
|
-
export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO,
|
|
3976
|
+
export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
|
|
3500
3977
|
|
|
3501
3978
|
//# sourceMappingURL=worker.js.map
|