@ewhauser/world-celld 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +61 -23
- package/dist/index.d.ts +126 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +206 -135
- package/dist/index.js.map +1 -1
- package/dist/{util-BvWlkqgV.js → retention-bLY1vcOu.js} +44 -2
- package/dist/retention-bLY1vcOu.js.map +1 -0
- package/dist/testing.d.ts +2 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +802 -174
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +133 -62
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +829 -170
- package/dist/worker.js.map +1 -1
- package/package.json +7 -7
- package/dist/util-BvWlkqgV.js.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema,
|
|
1
|
+
import { a as cleanupTombstone, c as globalRunIndexKey, d as workflowRunIndexKey, f as compact, h as parse, i as TOMBSTONE_KEY, l as hookMarkerKey, m as rpcStringify, n as HOOK_MARKER_PREFIX, o as correlationIndexKey, p as rpcParse, r as INDEX_MARKER_PREFIX, s as expiredRead, t as CLEANUP_RECORD_KEY, u as indexMarkerKey } from "./retention-bLY1vcOu.js";
|
|
2
|
+
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
//#region src/testing/fake-cell.ts
|
|
5
5
|
function applyListOptions(keys, options) {
|
|
@@ -309,7 +309,11 @@ const BINDINGS = {
|
|
|
309
309
|
"getEvent",
|
|
310
310
|
"listEvents",
|
|
311
311
|
"listSteps",
|
|
312
|
-
"listHooks"
|
|
312
|
+
"listHooks",
|
|
313
|
+
"getCleanupStatus",
|
|
314
|
+
"scheduleCleanup",
|
|
315
|
+
"cleanupNow",
|
|
316
|
+
"rearmCleanup"
|
|
313
317
|
])
|
|
314
318
|
},
|
|
315
319
|
streams: {
|
|
@@ -320,7 +324,10 @@ const BINDINGS = {
|
|
|
320
324
|
"getChunks",
|
|
321
325
|
"getInfo",
|
|
322
326
|
"registerStream",
|
|
323
|
-
"listStreams"
|
|
327
|
+
"listStreams",
|
|
328
|
+
"expireRegistry",
|
|
329
|
+
"finalizeRegistry",
|
|
330
|
+
"expireStream"
|
|
324
331
|
])
|
|
325
332
|
},
|
|
326
333
|
index: {
|
|
@@ -329,6 +336,8 @@ const BINDINGS = {
|
|
|
329
336
|
"get",
|
|
330
337
|
"put",
|
|
331
338
|
"delete",
|
|
339
|
+
"putOwned",
|
|
340
|
+
"expireRun",
|
|
332
341
|
"list",
|
|
333
342
|
"reserveHookToken",
|
|
334
343
|
"finalizeHookIndexes",
|
|
@@ -344,7 +353,8 @@ const BINDINGS = {
|
|
|
344
353
|
"listDeadLetters",
|
|
345
354
|
"redriveDeadLetter",
|
|
346
355
|
"purgeDeadLetters",
|
|
347
|
-
"rearmAlarm"
|
|
356
|
+
"rearmAlarm",
|
|
357
|
+
"expireRun"
|
|
348
358
|
])
|
|
349
359
|
}
|
|
350
360
|
};
|
|
@@ -478,6 +488,14 @@ var IndexDO = class extends DurableObject {
|
|
|
478
488
|
async put(key, value) {
|
|
479
489
|
await this.ctx.storage.put(key, value);
|
|
480
490
|
}
|
|
491
|
+
/** Publish a derived index only while its owning run is not expired. */
|
|
492
|
+
async putOwned(runId, key, value) {
|
|
493
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
494
|
+
if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
|
|
495
|
+
await txn.put(key, value);
|
|
496
|
+
return { stored: true };
|
|
497
|
+
});
|
|
498
|
+
}
|
|
481
499
|
async delete(key) {
|
|
482
500
|
await this.ctx.storage.delete(key);
|
|
483
501
|
}
|
|
@@ -523,6 +541,11 @@ var IndexDO = class extends DurableObject {
|
|
|
523
541
|
}
|
|
524
542
|
async finalizeHookIndexes(token, hookId, serializedHook, owner) {
|
|
525
543
|
await this.ctx.storage.transaction(async (txn) => {
|
|
544
|
+
if (await txn.get(`expired:${owner.runId}`) !== void 0) {
|
|
545
|
+
const claim = await txn.get(`hookclaim:${token}`);
|
|
546
|
+
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(`hookclaim:${token}`);
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
526
549
|
const indexedHook = await txn.get(`hook:${token}`);
|
|
527
550
|
if (indexedHook !== void 0 && !sameOwner(ownerFromHook(indexedHook), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
528
551
|
const claim = await txn.get(`hookclaim:${token}`);
|
|
@@ -554,6 +577,42 @@ var IndexDO = class extends DurableObject {
|
|
|
554
577
|
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
|
|
555
578
|
});
|
|
556
579
|
}
|
|
580
|
+
/**
|
|
581
|
+
* Fence a run and remove all of its known derived indexes atomically.
|
|
582
|
+
* Once the fence exists, putOwned() and hook finalization cannot resurrect
|
|
583
|
+
* entries from a delayed request.
|
|
584
|
+
*/
|
|
585
|
+
async expireRun(request) {
|
|
586
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
587
|
+
await txn.put(`expired:${request.runId}`, request.expiredAt);
|
|
588
|
+
let deleted = 0;
|
|
589
|
+
for (const key of new Set(request.keys)) if (await txn.delete(key)) deleted++;
|
|
590
|
+
for (const hook of request.hooks) {
|
|
591
|
+
const owner = {
|
|
592
|
+
runId: request.runId,
|
|
593
|
+
hookId: hook.hookId
|
|
594
|
+
};
|
|
595
|
+
const tokenKey = `hook:${hook.token}`;
|
|
596
|
+
const idKey = `hookid:${hook.hookId}`;
|
|
597
|
+
const claimKey = `hookclaim:${hook.token}`;
|
|
598
|
+
const [byToken, byId, claim] = await Promise.all([
|
|
599
|
+
txn.get(tokenKey),
|
|
600
|
+
txn.get(idKey),
|
|
601
|
+
txn.get(claimKey)
|
|
602
|
+
]);
|
|
603
|
+
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
|
|
604
|
+
if (await txn.delete(tokenKey)) deleted++;
|
|
605
|
+
}
|
|
606
|
+
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
|
|
607
|
+
if (await txn.delete(idKey)) deleted++;
|
|
608
|
+
}
|
|
609
|
+
if (claim !== void 0 && sameOwner(claim.owner, owner)) {
|
|
610
|
+
if (await txn.delete(claimKey)) deleted++;
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
return { deleted };
|
|
614
|
+
});
|
|
615
|
+
}
|
|
557
616
|
};
|
|
558
617
|
//#endregion
|
|
559
618
|
//#region src/worker/durable-objects/QueueDO.ts
|
|
@@ -576,6 +635,12 @@ function pad(ms) {
|
|
|
576
635
|
function dueKey(atMs, messageId) {
|
|
577
636
|
return `due:${pad(atMs)}:${messageId}`;
|
|
578
637
|
}
|
|
638
|
+
function runReferenceKey(runId, messageId) {
|
|
639
|
+
return `run:${runId}:${messageId}`;
|
|
640
|
+
}
|
|
641
|
+
function expiredRunKey(runId) {
|
|
642
|
+
return `expired-run:${runId}`;
|
|
643
|
+
}
|
|
579
644
|
function backoffSeconds(attempt) {
|
|
580
645
|
return Math.min(60, 2 ** attempt);
|
|
581
646
|
}
|
|
@@ -597,6 +662,11 @@ var QueueDO = class extends DurableObject {
|
|
|
597
662
|
const storage = this.ctx.storage;
|
|
598
663
|
const now = this.#now();
|
|
599
664
|
return await storage.transaction(async (txn) => {
|
|
665
|
+
if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
|
|
666
|
+
ok: false,
|
|
667
|
+
code: "RUN_EXPIRED",
|
|
668
|
+
message: `Workflow run "${request.runId}" has expired`
|
|
669
|
+
};
|
|
600
670
|
const pinned = await txn.get("cfg");
|
|
601
671
|
if (!pinned) await txn.put("cfg", {
|
|
602
672
|
queueShards: request.config.queueShards,
|
|
@@ -627,14 +697,21 @@ var QueueDO = class extends DurableObject {
|
|
|
627
697
|
queueName: request.queueName,
|
|
628
698
|
pathname: request.pathname,
|
|
629
699
|
body: request.body,
|
|
700
|
+
runId: request.runId,
|
|
630
701
|
targetBaseUrl: request.config.targetBaseUrl,
|
|
631
702
|
idempotencyKey: request.idempotencyKey,
|
|
632
703
|
attempt: 0,
|
|
633
704
|
enqueuedAt: now
|
|
634
705
|
};
|
|
635
706
|
const dueAt = now + Math.max(0, request.delaySeconds ?? 0) * 1e3;
|
|
707
|
+
const scheduleKey = dueKey(dueAt, row.messageId);
|
|
636
708
|
await txn.put(`msg:${row.messageId}`, row);
|
|
637
|
-
await txn.put(
|
|
709
|
+
await txn.put(scheduleKey, row.messageId);
|
|
710
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
711
|
+
messageId: row.messageId,
|
|
712
|
+
dueKey: scheduleKey,
|
|
713
|
+
idempotencyKey: row.idempotencyKey
|
|
714
|
+
});
|
|
638
715
|
await this.#armAlarmAtMost(txn, dueAt, now);
|
|
639
716
|
return {
|
|
640
717
|
ok: true,
|
|
@@ -672,7 +749,16 @@ var QueueDO = class extends DurableObject {
|
|
|
672
749
|
for (const [key, deadline] of inflight) if (deadline <= now) {
|
|
673
750
|
const messageId = key.slice(9);
|
|
674
751
|
await txn.delete(key);
|
|
675
|
-
|
|
752
|
+
const row = await txn.get(`msg:${messageId}`);
|
|
753
|
+
if (row) {
|
|
754
|
+
const scheduleKey = dueKey(now, messageId);
|
|
755
|
+
await txn.put(scheduleKey, messageId);
|
|
756
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
757
|
+
messageId: row.messageId,
|
|
758
|
+
dueKey: scheduleKey,
|
|
759
|
+
idempotencyKey: row.idempotencyKey
|
|
760
|
+
});
|
|
761
|
+
}
|
|
676
762
|
} else inflightCount++;
|
|
677
763
|
const rows = [];
|
|
678
764
|
if (inflightCount < maxInflight) {
|
|
@@ -727,8 +813,25 @@ var QueueDO = class extends DurableObject {
|
|
|
727
813
|
timeoutSeconds = void 0;
|
|
728
814
|
}
|
|
729
815
|
if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
|
|
816
|
+
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
817
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
818
|
+
await txn.delete(`inflight:${row.messageId}`);
|
|
819
|
+
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
820
|
+
if (row.idempotencyKey) {
|
|
821
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
822
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
823
|
+
}
|
|
824
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
730
827
|
await txn.delete(`inflight:${row.messageId}`);
|
|
731
|
-
|
|
828
|
+
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
829
|
+
await txn.put(scheduleKey, row.messageId);
|
|
830
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
831
|
+
messageId: row.messageId,
|
|
832
|
+
dueKey: scheduleKey,
|
|
833
|
+
idempotencyKey: row.idempotencyKey
|
|
834
|
+
});
|
|
732
835
|
await this.#scheduleNextAlarm(txn, now);
|
|
733
836
|
});
|
|
734
837
|
else await this.#retry(row, "HTTP 503");
|
|
@@ -745,6 +848,7 @@ var QueueDO = class extends DurableObject {
|
|
|
745
848
|
if (row.idempotencyKey) {
|
|
746
849
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
747
850
|
}
|
|
851
|
+
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
748
852
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
749
853
|
});
|
|
750
854
|
}
|
|
@@ -754,6 +858,17 @@ var QueueDO = class extends DurableObject {
|
|
|
754
858
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
755
859
|
const attempt = row.attempt + 1;
|
|
756
860
|
await storage.transaction(async (txn) => {
|
|
861
|
+
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
862
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
863
|
+
await txn.delete(`inflight:${row.messageId}`);
|
|
864
|
+
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
865
|
+
if (row.idempotencyKey) {
|
|
866
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
867
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
868
|
+
}
|
|
869
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
870
|
+
return;
|
|
871
|
+
}
|
|
757
872
|
await txn.delete(`inflight:${row.messageId}`);
|
|
758
873
|
if (attempt >= maxAttempts) {
|
|
759
874
|
const dead = {
|
|
@@ -762,11 +877,17 @@ var QueueDO = class extends DurableObject {
|
|
|
762
877
|
lastError: reason,
|
|
763
878
|
failedAt: now
|
|
764
879
|
};
|
|
765
|
-
|
|
880
|
+
const deadKey = `dlq:${pad(now)}:${row.messageId}`;
|
|
881
|
+
await txn.put(deadKey, dead);
|
|
766
882
|
await txn.delete(`msg:${row.messageId}`);
|
|
767
883
|
if (row.idempotencyKey) {
|
|
768
884
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
769
885
|
}
|
|
886
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
887
|
+
messageId: row.messageId,
|
|
888
|
+
dlqKey: deadKey,
|
|
889
|
+
idempotencyKey: row.idempotencyKey
|
|
890
|
+
});
|
|
770
891
|
await this.#scheduleNextAlarm(txn, now);
|
|
771
892
|
return;
|
|
772
893
|
}
|
|
@@ -775,8 +896,14 @@ var QueueDO = class extends DurableObject {
|
|
|
775
896
|
attempt,
|
|
776
897
|
lastError: reason
|
|
777
898
|
};
|
|
899
|
+
const scheduleKey = dueKey(now + backoffSeconds(attempt) * 1e3, row.messageId);
|
|
778
900
|
await txn.put(`msg:${row.messageId}`, updated);
|
|
779
|
-
await txn.put(
|
|
901
|
+
await txn.put(scheduleKey, row.messageId);
|
|
902
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
903
|
+
messageId: row.messageId,
|
|
904
|
+
dueKey: scheduleKey,
|
|
905
|
+
idempotencyKey: row.idempotencyKey
|
|
906
|
+
});
|
|
780
907
|
await this.#scheduleNextAlarm(txn, now);
|
|
781
908
|
});
|
|
782
909
|
}
|
|
@@ -860,8 +987,14 @@ var QueueDO = class extends DurableObject {
|
|
|
860
987
|
await txn.put(dedupKey, row.messageId);
|
|
861
988
|
}
|
|
862
989
|
await txn.put(`msg:${row.messageId}`, row);
|
|
863
|
-
|
|
990
|
+
const scheduleKey = dueKey(now, row.messageId);
|
|
991
|
+
await txn.put(scheduleKey, row.messageId);
|
|
864
992
|
await txn.delete(key);
|
|
993
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
994
|
+
messageId: row.messageId,
|
|
995
|
+
dueKey: scheduleKey,
|
|
996
|
+
idempotencyKey: row.idempotencyKey
|
|
997
|
+
});
|
|
865
998
|
await this.#armAlarmAtMost(txn, now, now);
|
|
866
999
|
return { ok: true };
|
|
867
1000
|
}
|
|
@@ -871,7 +1004,11 @@ var QueueDO = class extends DurableObject {
|
|
|
871
1004
|
async purgeDeadLetters() {
|
|
872
1005
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
873
1006
|
const dlq = await txn.list({ prefix: "dlq:" });
|
|
874
|
-
for (const key of dlq
|
|
1007
|
+
for (const [key, value] of dlq) {
|
|
1008
|
+
await txn.delete(key);
|
|
1009
|
+
const dead = value;
|
|
1010
|
+
if (dead.runId) await txn.delete(runReferenceKey(dead.runId, dead.messageId));
|
|
1011
|
+
}
|
|
875
1012
|
return { purged: dlq.size };
|
|
876
1013
|
});
|
|
877
1014
|
}
|
|
@@ -883,6 +1020,32 @@ var QueueDO = class extends DurableObject {
|
|
|
883
1020
|
await this.#rearm();
|
|
884
1021
|
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
885
1022
|
}
|
|
1023
|
+
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
1024
|
+
async expireRun(runId, expiredAt) {
|
|
1025
|
+
return { deleted: await this.ctx.storage.transaction(async (txn) => {
|
|
1026
|
+
const existing = await txn.get(expiredRunKey(runId));
|
|
1027
|
+
const references = await txn.list({ prefix: `run:${runId}:` });
|
|
1028
|
+
let count = existing?.deleted ?? 0;
|
|
1029
|
+
for (const [key, reference] of references) {
|
|
1030
|
+
await txn.delete(`msg:${reference.messageId}`);
|
|
1031
|
+
await txn.delete(`inflight:${reference.messageId}`);
|
|
1032
|
+
if (reference.dueKey) await txn.delete(reference.dueKey);
|
|
1033
|
+
if (reference.dlqKey) await txn.delete(reference.dlqKey);
|
|
1034
|
+
if (reference.idempotencyKey) {
|
|
1035
|
+
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
1036
|
+
if (await txn.get(dedupKey) === reference.messageId) await txn.delete(dedupKey);
|
|
1037
|
+
}
|
|
1038
|
+
await txn.delete(key);
|
|
1039
|
+
count++;
|
|
1040
|
+
}
|
|
1041
|
+
await txn.put(expiredRunKey(runId), {
|
|
1042
|
+
expiredAt,
|
|
1043
|
+
deleted: count
|
|
1044
|
+
});
|
|
1045
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1046
|
+
return count;
|
|
1047
|
+
}) };
|
|
1048
|
+
}
|
|
886
1049
|
};
|
|
887
1050
|
//#endregion
|
|
888
1051
|
//#region src/worker/durable-objects/StreamDO.ts
|
|
@@ -890,6 +1053,8 @@ const META_KEY = "meta";
|
|
|
890
1053
|
const CHUNK_KEY_PREFIX = "chunk:";
|
|
891
1054
|
/** Registry keys used when this DO instance acts as a per-run stream index. */
|
|
892
1055
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
1056
|
+
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
1057
|
+
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
893
1058
|
/**
|
|
894
1059
|
* Zero-pad chunk indexes so `storage.list({ prefix })` returns chunks in
|
|
895
1060
|
* write order. 12 digits comfortably exceeds any realistic chunk count.
|
|
@@ -914,22 +1079,28 @@ var StreamDO = class extends DurableObject {
|
|
|
914
1079
|
closed: false
|
|
915
1080
|
};
|
|
916
1081
|
}
|
|
1082
|
+
assertOwner(meta, runId) {
|
|
1083
|
+
if (meta.expiredAt !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
1084
|
+
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
1085
|
+
}
|
|
917
1086
|
/**
|
|
918
1087
|
* Append a chunk. The index is allocated inside the transaction, so
|
|
919
1088
|
* concurrent writers can never collide or skip.
|
|
920
1089
|
*/
|
|
921
|
-
async writeChunk(data) {
|
|
1090
|
+
async writeChunk(runId, data) {
|
|
922
1091
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
923
1092
|
const meta = await txn.get(META_KEY) ?? {
|
|
924
1093
|
count: 0,
|
|
925
1094
|
closed: false
|
|
926
1095
|
};
|
|
1096
|
+
this.assertOwner(meta, runId);
|
|
927
1097
|
if (meta.closed) throw new Error("Cannot write to a closed stream");
|
|
928
1098
|
const index = meta.count;
|
|
929
1099
|
await txn.put(chunkKey(index), data);
|
|
930
1100
|
await txn.put(META_KEY, {
|
|
931
1101
|
count: index + 1,
|
|
932
|
-
closed: false
|
|
1102
|
+
closed: false,
|
|
1103
|
+
ownerRunId: runId
|
|
933
1104
|
});
|
|
934
1105
|
return index;
|
|
935
1106
|
});
|
|
@@ -938,14 +1109,16 @@ var StreamDO = class extends DurableObject {
|
|
|
938
1109
|
* Close the stream (idempotent). Readers observe `done: true` once all
|
|
939
1110
|
* previously written chunks have been consumed.
|
|
940
1111
|
*/
|
|
941
|
-
async closeStream() {
|
|
1112
|
+
async closeStream(runId) {
|
|
942
1113
|
await this.ctx.storage.transaction(async (txn) => {
|
|
943
1114
|
const meta = await txn.get(META_KEY) ?? {
|
|
944
1115
|
count: 0,
|
|
945
1116
|
closed: false
|
|
946
1117
|
};
|
|
1118
|
+
this.assertOwner(meta, runId);
|
|
947
1119
|
await txn.put(META_KEY, {
|
|
948
1120
|
...meta,
|
|
1121
|
+
ownerRunId: runId,
|
|
949
1122
|
closed: true
|
|
950
1123
|
});
|
|
951
1124
|
});
|
|
@@ -955,6 +1128,7 @@ var StreamDO = class extends DurableObject {
|
|
|
955
1128
|
*/
|
|
956
1129
|
async getChunks(params) {
|
|
957
1130
|
const meta = await this.getMeta();
|
|
1131
|
+
if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
|
|
958
1132
|
const start = Math.max(0, params.startIndex);
|
|
959
1133
|
const entries = await this.ctx.storage.list({
|
|
960
1134
|
prefix: CHUNK_KEY_PREFIX,
|
|
@@ -972,6 +1146,7 @@ var StreamDO = class extends DurableObject {
|
|
|
972
1146
|
*/
|
|
973
1147
|
async getInfo() {
|
|
974
1148
|
const meta = await this.getMeta();
|
|
1149
|
+
if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
|
|
975
1150
|
return {
|
|
976
1151
|
tailIndex: meta.count - 1,
|
|
977
1152
|
done: meta.closed
|
|
@@ -980,16 +1155,66 @@ var StreamDO = class extends DurableObject {
|
|
|
980
1155
|
/**
|
|
981
1156
|
* Register a stream name against this per-run registry instance.
|
|
982
1157
|
*/
|
|
983
|
-
async registerStream(name) {
|
|
984
|
-
await this.ctx.storage.
|
|
1158
|
+
async registerStream(runId, name) {
|
|
1159
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1160
|
+
if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
1161
|
+
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1162
|
+
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1163
|
+
await txn.put(REGISTRY_OWNER_KEY, runId);
|
|
1164
|
+
await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
|
|
1165
|
+
});
|
|
985
1166
|
}
|
|
986
1167
|
/**
|
|
987
1168
|
* List stream names registered against this per-run registry instance.
|
|
988
1169
|
*/
|
|
989
1170
|
async listStreams() {
|
|
1171
|
+
if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
|
|
990
1172
|
const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
991
1173
|
return Array.from(entries.keys()).map((key) => key.slice(7));
|
|
992
1174
|
}
|
|
1175
|
+
/** Fence a run's registry before any stream cells are removed. */
|
|
1176
|
+
async expireRegistry(runId, expiredAt) {
|
|
1177
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1178
|
+
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1179
|
+
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1180
|
+
await txn.put(REGISTRY_OWNER_KEY, runId);
|
|
1181
|
+
await txn.put(REGISTRY_EXPIRED_KEY, expiredAt);
|
|
1182
|
+
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1183
|
+
return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
/** Remove the registry payload after every referenced stream was fenced. */
|
|
1187
|
+
async finalizeRegistry(runId) {
|
|
1188
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1189
|
+
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1190
|
+
const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
|
|
1191
|
+
if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1192
|
+
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1193
|
+
for (const key of entries.keys()) await txn.delete(key);
|
|
1194
|
+
});
|
|
1195
|
+
}
|
|
1196
|
+
/** Delete stream payload while leaving a small ownership tombstone. */
|
|
1197
|
+
async expireStream(runId, expiredAt) {
|
|
1198
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1199
|
+
const meta = await txn.get(META_KEY) ?? {
|
|
1200
|
+
count: 0,
|
|
1201
|
+
closed: false
|
|
1202
|
+
};
|
|
1203
|
+
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
1204
|
+
const chunks = await txn.list({ prefix: CHUNK_KEY_PREFIX });
|
|
1205
|
+
for (const key of chunks.keys()) await txn.delete(key);
|
|
1206
|
+
await txn.put(META_KEY, {
|
|
1207
|
+
count: 0,
|
|
1208
|
+
closed: true,
|
|
1209
|
+
ownerRunId: runId,
|
|
1210
|
+
expiredAt
|
|
1211
|
+
});
|
|
1212
|
+
return {
|
|
1213
|
+
deleted: true,
|
|
1214
|
+
chunks: chunks.size
|
|
1215
|
+
};
|
|
1216
|
+
});
|
|
1217
|
+
}
|
|
993
1218
|
};
|
|
994
1219
|
//#endregion
|
|
995
1220
|
//#region src/apply-event.ts
|
|
@@ -999,6 +1224,7 @@ const RUN_KEY = "run";
|
|
|
999
1224
|
const EVENT_KEY_PREFIX = "event:";
|
|
1000
1225
|
const STEP_KEY_PREFIX = "step:";
|
|
1001
1226
|
const HOOK_KEY_PREFIX = "hook:";
|
|
1227
|
+
const WAIT_KEY_PREFIX = "wait:";
|
|
1002
1228
|
const STEP_CREATED_KEY_PREFIX = "stepcreated:";
|
|
1003
1229
|
const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
1004
1230
|
/**
|
|
@@ -1008,6 +1234,10 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
|
1008
1234
|
* rejected as a duplicate instead of resurrecting the hook.
|
|
1009
1235
|
*/
|
|
1010
1236
|
const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
|
|
1237
|
+
/** Workflow-authored attr_set correlation claims. */
|
|
1238
|
+
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
1239
|
+
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
1240
|
+
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
1011
1241
|
function failure(code, message, extra) {
|
|
1012
1242
|
return {
|
|
1013
1243
|
ok: false,
|
|
@@ -1110,9 +1340,15 @@ async function releaseAllHooks(store) {
|
|
|
1110
1340
|
}
|
|
1111
1341
|
return released;
|
|
1112
1342
|
}
|
|
1343
|
+
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
1344
|
+
async function releaseAllWaits(store) {
|
|
1345
|
+
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
1346
|
+
for (const key of waits.keys()) await store.delete(key);
|
|
1347
|
+
}
|
|
1113
1348
|
async function applyEvent(store, ctx) {
|
|
1114
1349
|
const { runId, data, now } = ctx;
|
|
1115
1350
|
const specVersion = data.specVersion ?? SPEC_VERSION_CURRENT;
|
|
1351
|
+
if (specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: specVersion });
|
|
1116
1352
|
/** Build and validate the stored event BEFORE anything is persisted. */
|
|
1117
1353
|
const buildEvent = (record) => {
|
|
1118
1354
|
const withMeta = {
|
|
@@ -1120,9 +1356,16 @@ async function applyEvent(store, ctx) {
|
|
|
1120
1356
|
runId,
|
|
1121
1357
|
eventId: ctx.nextEventId(),
|
|
1122
1358
|
createdAt: now,
|
|
1359
|
+
occurredAt: ctx.params?.occurredAt,
|
|
1360
|
+
resumeId: ctx.params?.resumeId,
|
|
1123
1361
|
specVersion
|
|
1124
1362
|
};
|
|
1363
|
+
if (record.eventType !== "hook_received") delete withMeta.resumeId;
|
|
1125
1364
|
if (record.eventType === "run_started") delete withMeta.eventData;
|
|
1365
|
+
if (record.eventType === "step_started" && withMeta.eventData !== null && typeof withMeta.eventData === "object") {
|
|
1366
|
+
const { input: _input, ...eventData } = withMeta.eventData;
|
|
1367
|
+
withMeta.eventData = eventData;
|
|
1368
|
+
}
|
|
1126
1369
|
return EventSchema.parse(compact(withMeta));
|
|
1127
1370
|
};
|
|
1128
1371
|
const putEvent = async (event) => {
|
|
@@ -1133,14 +1376,21 @@ async function applyEvent(store, ctx) {
|
|
|
1133
1376
|
if (!skipRunValidation) currentRun = await store.get(RUN_KEY);
|
|
1134
1377
|
let bootstrapped;
|
|
1135
1378
|
if (data.eventType === "run_started" && !currentRun && data.eventData) {
|
|
1136
|
-
const { deploymentId, workflowName, input, executionContext } = data.eventData;
|
|
1379
|
+
const { deploymentId, workflowName, input, executionContext, attributes, allowReservedAttributes, encryptionPublicKey } = data.eventData;
|
|
1137
1380
|
if (deploymentId && workflowName && input !== void 0) {
|
|
1381
|
+
const attributeChanges = Object.entries(attributes ?? {}).map(([key, value]) => ({
|
|
1382
|
+
key,
|
|
1383
|
+
value
|
|
1384
|
+
}));
|
|
1385
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes });
|
|
1138
1386
|
const createdRun = WorkflowRunSchema.parse(compact({
|
|
1139
1387
|
runId,
|
|
1140
1388
|
deploymentId,
|
|
1141
1389
|
workflowName,
|
|
1142
1390
|
specVersion,
|
|
1143
1391
|
executionContext,
|
|
1392
|
+
attributes,
|
|
1393
|
+
encryptionPublicKey,
|
|
1144
1394
|
input,
|
|
1145
1395
|
status: "pending",
|
|
1146
1396
|
output: void 0,
|
|
@@ -1156,7 +1406,10 @@ async function applyEvent(store, ctx) {
|
|
|
1156
1406
|
deploymentId,
|
|
1157
1407
|
workflowName,
|
|
1158
1408
|
input,
|
|
1159
|
-
executionContext
|
|
1409
|
+
executionContext,
|
|
1410
|
+
attributes,
|
|
1411
|
+
allowReservedAttributes,
|
|
1412
|
+
encryptionPublicKey
|
|
1160
1413
|
}
|
|
1161
1414
|
});
|
|
1162
1415
|
await store.put(RUN_KEY, createdRun);
|
|
@@ -1168,10 +1421,7 @@ async function applyEvent(store, ctx) {
|
|
|
1168
1421
|
};
|
|
1169
1422
|
}
|
|
1170
1423
|
}
|
|
1171
|
-
if (currentRun) {
|
|
1172
|
-
if (requiresNewerWorld(currentRun.specVersion)) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" requires a newer world (run specVersion ${currentRun.specVersion}, world ${SPEC_VERSION_CURRENT})`, { runSpecVersion: currentRun.specVersion });
|
|
1173
|
-
if (isLegacySpecVersion(currentRun.specVersion)) return failure("LEGACY_RUN_NOT_SUPPORTED", `Run "${runId}" was created with a legacy spec version (${currentRun.specVersion ?? "undefined"}) not supported by this world`);
|
|
1174
|
-
}
|
|
1424
|
+
if (currentRun && currentRun.specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${currentRun.specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: currentRun.specVersion });
|
|
1175
1425
|
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) {
|
|
1176
1426
|
if (data.eventType === "run_cancelled" && currentRun.status === "cancelled") {
|
|
1177
1427
|
const event = buildEvent({ ...data });
|
|
@@ -1185,14 +1435,32 @@ async function applyEvent(store, ctx) {
|
|
|
1185
1435
|
}
|
|
1186
1436
|
if (data.eventType === "run_started") return failure("RUN_EXPIRED", `Workflow run "${runId}" is already in terminal state "${currentRun.status}"`);
|
|
1187
1437
|
if (data.eventType === "run_completed" || data.eventType === "run_failed" || data.eventType === "run_cancelled") return failure("ENTITY_CONFLICT", `Cannot transition run from terminal state "${currentRun.status}"`);
|
|
1188
|
-
if (data
|
|
1438
|
+
if (isChildEntityCreationEvent(data)) return failure("ENTITY_CONFLICT", `Cannot create new entities on run in terminal state "${currentRun.status}"`);
|
|
1439
|
+
if (data.eventType === "attr_set") return failure("ENTITY_CONFLICT", `Cannot set attributes on run in terminal state "${currentRun.status}"`);
|
|
1189
1440
|
}
|
|
1190
1441
|
let validatedStep;
|
|
1191
1442
|
if (data.eventType === "step_started" || data.eventType === "step_completed" || data.eventType === "step_failed" || data.eventType === "step_retrying") {
|
|
1443
|
+
const lazyStepStart = data.eventType === "step_started" && isChildEntityCreationEvent(data);
|
|
1192
1444
|
validatedStep = await store.get(`${STEP_KEY_PREFIX}${data.correlationId}`);
|
|
1193
|
-
if (!validatedStep)
|
|
1194
|
-
|
|
1195
|
-
|
|
1445
|
+
if (!validatedStep) {
|
|
1446
|
+
if (!lazyStepStart) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1447
|
+
} else if (lazyStepStart) return failure("ENTITY_CONFLICT", `Step "${data.correlationId}" already exists`);
|
|
1448
|
+
if (validatedStep && isTerminalStepStatus(validatedStep.status)) return failure("ENTITY_CONFLICT", `Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
1449
|
+
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status) && validatedStep && validatedStep.status !== "running") return failure("RUN_EXPIRED", `Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
|
|
1450
|
+
}
|
|
1451
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) {
|
|
1452
|
+
const claim = await store.get(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`);
|
|
1453
|
+
if (claim) {
|
|
1454
|
+
if (claim.hookId !== data.correlationId) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded for a different hook`);
|
|
1455
|
+
if (claim.payloadDigest && ctx.params.resumePayloadDigest && claim.payloadDigest !== ctx.params.resumePayloadDigest) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded with a different payload`);
|
|
1456
|
+
const committed = await store.get(`${EVENT_KEY_PREFIX}${claim.eventId}`);
|
|
1457
|
+
if (committed?.eventType === "hook_received" && committed.correlationId === claim.hookId && committed.resumeId === ctx.params.resumeId) return {
|
|
1458
|
+
ok: true,
|
|
1459
|
+
event: committed,
|
|
1460
|
+
releasedHooks: []
|
|
1461
|
+
};
|
|
1462
|
+
if (committed) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" points to a different event`);
|
|
1463
|
+
}
|
|
1196
1464
|
}
|
|
1197
1465
|
if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
|
|
1198
1466
|
if (!await store.get(`hook:${data.correlationId}`)) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
@@ -1206,12 +1474,19 @@ async function applyEvent(store, ctx) {
|
|
|
1206
1474
|
run: existing,
|
|
1207
1475
|
releasedHooks: []
|
|
1208
1476
|
};
|
|
1477
|
+
const attributeChanges = Object.entries(data.eventData.attributes ?? {}).map(([key, value]) => ({
|
|
1478
|
+
key,
|
|
1479
|
+
value
|
|
1480
|
+
}));
|
|
1481
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes: data.eventData.allowReservedAttributes });
|
|
1209
1482
|
const run = WorkflowRunSchema.parse(compact({
|
|
1210
1483
|
runId,
|
|
1211
1484
|
deploymentId: data.eventData.deploymentId,
|
|
1212
1485
|
workflowName: data.eventData.workflowName,
|
|
1213
1486
|
specVersion,
|
|
1214
1487
|
executionContext: data.eventData.executionContext,
|
|
1488
|
+
attributes: data.eventData.attributes,
|
|
1489
|
+
encryptionPublicKey: data.eventData.encryptionPublicKey,
|
|
1215
1490
|
input: data.eventData.input,
|
|
1216
1491
|
status: "pending",
|
|
1217
1492
|
output: void 0,
|
|
@@ -1237,11 +1512,25 @@ async function applyEvent(store, ctx) {
|
|
|
1237
1512
|
}
|
|
1238
1513
|
case "run_started": {
|
|
1239
1514
|
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1240
|
-
if (currentRun.status === "running")
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1515
|
+
if (currentRun.status === "running") {
|
|
1516
|
+
const result = {
|
|
1517
|
+
ok: true,
|
|
1518
|
+
run: currentRun,
|
|
1519
|
+
releasedHooks: [],
|
|
1520
|
+
maxEvents: getMaxEventsPerRun()
|
|
1521
|
+
};
|
|
1522
|
+
if (ctx.params?.skipPreload) return result;
|
|
1523
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1524
|
+
limit: getMaxEventsPerRun(),
|
|
1525
|
+
sortOrder: "asc"
|
|
1526
|
+
}, (event) => event.eventId);
|
|
1527
|
+
return {
|
|
1528
|
+
...result,
|
|
1529
|
+
events: allEvents.data,
|
|
1530
|
+
cursor: allEvents.cursor,
|
|
1531
|
+
hasMore: allEvents.hasMore
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1245
1534
|
const run = WorkflowRunSchema.parse(compact({
|
|
1246
1535
|
...currentRun,
|
|
1247
1536
|
status: "running",
|
|
@@ -1254,17 +1543,24 @@ async function applyEvent(store, ctx) {
|
|
|
1254
1543
|
const event = buildEvent({ ...data });
|
|
1255
1544
|
await store.put(RUN_KEY, run);
|
|
1256
1545
|
await putEvent(event);
|
|
1257
|
-
const
|
|
1258
|
-
limit: 1e3,
|
|
1259
|
-
sortOrder: "asc"
|
|
1260
|
-
}, (e) => e.eventId);
|
|
1261
|
-
return {
|
|
1546
|
+
const result = {
|
|
1262
1547
|
ok: true,
|
|
1263
1548
|
event,
|
|
1264
1549
|
run,
|
|
1265
1550
|
releasedHooks: [],
|
|
1266
1551
|
runCreated: bootstrapped,
|
|
1267
|
-
|
|
1552
|
+
maxEvents: getMaxEventsPerRun()
|
|
1553
|
+
};
|
|
1554
|
+
if (ctx.params?.skipPreload) return result;
|
|
1555
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1556
|
+
limit: getMaxEventsPerRun(),
|
|
1557
|
+
sortOrder: "asc"
|
|
1558
|
+
}, (e) => e.eventId);
|
|
1559
|
+
return {
|
|
1560
|
+
...result,
|
|
1561
|
+
events: allEvents.data,
|
|
1562
|
+
cursor: allEvents.cursor,
|
|
1563
|
+
hasMore: allEvents.hasMore
|
|
1268
1564
|
};
|
|
1269
1565
|
}
|
|
1270
1566
|
case "run_completed": {
|
|
@@ -1280,11 +1576,13 @@ async function applyEvent(store, ctx) {
|
|
|
1280
1576
|
const event = buildEvent({ ...data });
|
|
1281
1577
|
await store.put(RUN_KEY, run);
|
|
1282
1578
|
await putEvent(event);
|
|
1579
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
1580
|
+
await releaseAllWaits(store);
|
|
1283
1581
|
return {
|
|
1284
1582
|
ok: true,
|
|
1285
1583
|
event,
|
|
1286
1584
|
run,
|
|
1287
|
-
releasedHooks
|
|
1585
|
+
releasedHooks
|
|
1288
1586
|
};
|
|
1289
1587
|
}
|
|
1290
1588
|
case "run_failed": {
|
|
@@ -1300,11 +1598,13 @@ async function applyEvent(store, ctx) {
|
|
|
1300
1598
|
const event = buildEvent({ ...data });
|
|
1301
1599
|
await store.put(RUN_KEY, run);
|
|
1302
1600
|
await putEvent(event);
|
|
1601
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
1602
|
+
await releaseAllWaits(store);
|
|
1303
1603
|
return {
|
|
1304
1604
|
ok: true,
|
|
1305
1605
|
event,
|
|
1306
1606
|
run,
|
|
1307
|
-
releasedHooks
|
|
1607
|
+
releasedHooks
|
|
1308
1608
|
};
|
|
1309
1609
|
}
|
|
1310
1610
|
case "run_cancelled": {
|
|
@@ -1320,11 +1620,37 @@ async function applyEvent(store, ctx) {
|
|
|
1320
1620
|
const event = buildEvent({ ...data });
|
|
1321
1621
|
await store.put(RUN_KEY, run);
|
|
1322
1622
|
await putEvent(event);
|
|
1623
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
1624
|
+
await releaseAllWaits(store);
|
|
1323
1625
|
return {
|
|
1324
1626
|
ok: true,
|
|
1325
1627
|
event,
|
|
1326
1628
|
run,
|
|
1327
|
-
releasedHooks
|
|
1629
|
+
releasedHooks
|
|
1630
|
+
};
|
|
1631
|
+
}
|
|
1632
|
+
case "attr_set": {
|
|
1633
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1634
|
+
validateAttributeChanges(data.eventData.changes, {
|
|
1635
|
+
existingKeys: Object.keys(currentRun.attributes),
|
|
1636
|
+
allowReservedAttributes: data.eventData.allowReservedAttributes
|
|
1637
|
+
});
|
|
1638
|
+
const markerKey = data.correlationId && data.eventData.writer.type === "workflow" ? `${ATTRIBUTE_EVENT_MARKER_PREFIX}${data.correlationId}` : void 0;
|
|
1639
|
+
if (markerKey && await store.get(markerKey) !== void 0) return failure("ENTITY_CONFLICT", `Attribute event "${data.correlationId}" already exists`);
|
|
1640
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
1641
|
+
...currentRun,
|
|
1642
|
+
attributes: applyAttributeChanges(currentRun.attributes, data.eventData.changes),
|
|
1643
|
+
updatedAt: now
|
|
1644
|
+
}));
|
|
1645
|
+
const event = buildEvent({ ...data });
|
|
1646
|
+
await store.put(RUN_KEY, run);
|
|
1647
|
+
if (markerKey) await store.put(markerKey, event.eventId);
|
|
1648
|
+
await putEvent(event);
|
|
1649
|
+
return {
|
|
1650
|
+
ok: true,
|
|
1651
|
+
event,
|
|
1652
|
+
run,
|
|
1653
|
+
releasedHooks: []
|
|
1328
1654
|
};
|
|
1329
1655
|
}
|
|
1330
1656
|
case "step_created": {
|
|
@@ -1358,7 +1684,34 @@ async function applyEvent(store, ctx) {
|
|
|
1358
1684
|
};
|
|
1359
1685
|
}
|
|
1360
1686
|
case "step_started": {
|
|
1361
|
-
|
|
1687
|
+
let step = validatedStep;
|
|
1688
|
+
let stepCreated = false;
|
|
1689
|
+
if (!step && data.eventData?.input !== void 0 && data.eventData.stepName) {
|
|
1690
|
+
step = StepSchema.parse(compact({
|
|
1691
|
+
runId,
|
|
1692
|
+
stepId: data.correlationId,
|
|
1693
|
+
stepName: data.eventData.stepName,
|
|
1694
|
+
status: "pending",
|
|
1695
|
+
input: data.eventData.input,
|
|
1696
|
+
attempt: 0,
|
|
1697
|
+
createdAt: now,
|
|
1698
|
+
updatedAt: now,
|
|
1699
|
+
specVersion
|
|
1700
|
+
}));
|
|
1701
|
+
const createdEvent = buildEvent({
|
|
1702
|
+
eventType: "step_created",
|
|
1703
|
+
correlationId: data.correlationId,
|
|
1704
|
+
eventData: {
|
|
1705
|
+
stepName: data.eventData.stepName,
|
|
1706
|
+
workflowName: data.eventData.workflowName,
|
|
1707
|
+
input: data.eventData.input
|
|
1708
|
+
}
|
|
1709
|
+
});
|
|
1710
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, step);
|
|
1711
|
+
await store.put(creationIndexKey(STEP_CREATED_KEY_PREFIX, step.createdAt, step.stepId), step.stepId);
|
|
1712
|
+
await putEvent(createdEvent);
|
|
1713
|
+
stepCreated = true;
|
|
1714
|
+
}
|
|
1362
1715
|
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1363
1716
|
if (step.retryAfter && step.retryAfter.getTime() > now.getTime()) return failure("TOO_EARLY", `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, { retryAfterSeconds: Math.ceil((step.retryAfter.getTime() - now.getTime()) / 1e3) });
|
|
1364
1717
|
const updated = StepSchema.parse(compact({
|
|
@@ -1376,7 +1729,8 @@ async function applyEvent(store, ctx) {
|
|
|
1376
1729
|
ok: true,
|
|
1377
1730
|
event,
|
|
1378
1731
|
step: updated,
|
|
1379
|
-
releasedHooks: []
|
|
1732
|
+
releasedHooks: [],
|
|
1733
|
+
...stepCreated ? { stepCreated: true } : {}
|
|
1380
1734
|
};
|
|
1381
1735
|
}
|
|
1382
1736
|
case "step_completed": {
|
|
@@ -1405,7 +1759,7 @@ async function applyEvent(store, ctx) {
|
|
|
1405
1759
|
const updated = StepSchema.parse(compact({
|
|
1406
1760
|
...step,
|
|
1407
1761
|
status: "failed",
|
|
1408
|
-
error: toStructuredError(data.eventData.error
|
|
1762
|
+
error: toStructuredError(data.eventData.error),
|
|
1409
1763
|
completedAt: now,
|
|
1410
1764
|
updatedAt: now
|
|
1411
1765
|
}));
|
|
@@ -1425,7 +1779,7 @@ async function applyEvent(store, ctx) {
|
|
|
1425
1779
|
const updated = StepSchema.parse(compact({
|
|
1426
1780
|
...step,
|
|
1427
1781
|
status: "pending",
|
|
1428
|
-
error: toStructuredError(data.eventData.error
|
|
1782
|
+
error: toStructuredError(data.eventData.error),
|
|
1429
1783
|
retryAfter: data.eventData.retryAfter ? new Date(data.eventData.retryAfter) : void 0,
|
|
1430
1784
|
updatedAt: now
|
|
1431
1785
|
}));
|
|
@@ -1481,7 +1835,9 @@ async function applyEvent(store, ctx) {
|
|
|
1481
1835
|
environment: "",
|
|
1482
1836
|
createdAt: now,
|
|
1483
1837
|
specVersion,
|
|
1484
|
-
isWebhook: data.eventData.isWebhook ?? false
|
|
1838
|
+
isWebhook: data.eventData.isWebhook ?? false,
|
|
1839
|
+
isSystem: data.eventData.isSystem,
|
|
1840
|
+
tokenRetentionUntil: data.eventData.tokenRetentionUntil
|
|
1485
1841
|
}));
|
|
1486
1842
|
const event = buildEvent({ ...data });
|
|
1487
1843
|
await store.put(hookKey, hook);
|
|
@@ -1513,78 +1869,113 @@ async function applyEvent(store, ctx) {
|
|
|
1513
1869
|
}]
|
|
1514
1870
|
};
|
|
1515
1871
|
}
|
|
1516
|
-
|
|
1872
|
+
case "wait_created": {
|
|
1873
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
1874
|
+
if (await store.get(waitKey) !== void 0) return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already exists`);
|
|
1875
|
+
const wait = WaitSchema.parse({
|
|
1876
|
+
waitId: `${runId}-${data.correlationId}`,
|
|
1877
|
+
runId,
|
|
1878
|
+
status: "waiting",
|
|
1879
|
+
resumeAt: data.eventData.resumeAt,
|
|
1880
|
+
createdAt: now,
|
|
1881
|
+
updatedAt: now,
|
|
1882
|
+
specVersion
|
|
1883
|
+
});
|
|
1517
1884
|
const event = buildEvent({ ...data });
|
|
1885
|
+
await store.put(waitKey, wait);
|
|
1518
1886
|
await putEvent(event);
|
|
1519
1887
|
return {
|
|
1520
1888
|
ok: true,
|
|
1521
1889
|
event,
|
|
1890
|
+
wait,
|
|
1522
1891
|
releasedHooks: []
|
|
1523
1892
|
};
|
|
1524
1893
|
}
|
|
1525
|
-
|
|
1526
|
-
}
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
await
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
const entities = await storage.list({
|
|
1561
|
-
prefix: entityPrefix,
|
|
1562
|
-
startAfter,
|
|
1563
|
-
limit: 100
|
|
1564
|
-
});
|
|
1565
|
-
if (entities.size === 0) return;
|
|
1566
|
-
for (const [key, entity] of entities) {
|
|
1567
|
-
const id = getId(entity);
|
|
1568
|
-
await storage.put(`${indexPrefix}${entity.createdAt.toISOString()}:${id}`, id);
|
|
1569
|
-
startAfter = key;
|
|
1894
|
+
case "wait_completed": {
|
|
1895
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
1896
|
+
const existing = await store.get(waitKey);
|
|
1897
|
+
if (!existing) return failure("WAIT_NOT_FOUND", `Wait "${data.correlationId}" not found`);
|
|
1898
|
+
if (existing.status === "completed") return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already completed`);
|
|
1899
|
+
const event = buildEvent({ ...data });
|
|
1900
|
+
const wait = WaitSchema.parse({
|
|
1901
|
+
...existing,
|
|
1902
|
+
status: "completed",
|
|
1903
|
+
resumeAt: data.eventData?.resumeAt ?? existing.resumeAt,
|
|
1904
|
+
completedAt: now,
|
|
1905
|
+
updatedAt: now
|
|
1906
|
+
});
|
|
1907
|
+
await store.put(waitKey, wait);
|
|
1908
|
+
await putEvent(event);
|
|
1909
|
+
return {
|
|
1910
|
+
ok: true,
|
|
1911
|
+
event,
|
|
1912
|
+
wait,
|
|
1913
|
+
releasedHooks: []
|
|
1914
|
+
};
|
|
1915
|
+
}
|
|
1916
|
+
default: {
|
|
1917
|
+
const event = buildEvent({ ...data });
|
|
1918
|
+
await putEvent(event);
|
|
1919
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) await store.put(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`, {
|
|
1920
|
+
hookId: data.correlationId,
|
|
1921
|
+
eventId: event.eventId,
|
|
1922
|
+
...ctx.params.resumePayloadDigest ? { payloadDigest: ctx.params.resumePayloadDigest } : {}
|
|
1923
|
+
});
|
|
1924
|
+
return {
|
|
1925
|
+
ok: true,
|
|
1926
|
+
event,
|
|
1927
|
+
releasedHooks: []
|
|
1928
|
+
};
|
|
1570
1929
|
}
|
|
1571
|
-
if (entities.size < 100) return;
|
|
1572
1930
|
}
|
|
1573
1931
|
}
|
|
1574
1932
|
/**
|
|
1575
|
-
*
|
|
1576
|
-
*
|
|
1577
|
-
*
|
|
1578
|
-
*
|
|
1933
|
+
* Complete Workflow 5's event-page response after an atomic apply. A celld
|
|
1934
|
+
* run is a single serialized writer, so the committed sequence is dense; a
|
|
1935
|
+
* stale eventCount therefore needs only the slots between the requested
|
|
1936
|
+
* position and the event this call committed.
|
|
1579
1937
|
*/
|
|
1580
|
-
async function
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1938
|
+
async function finalizeEventPage(store, outcome, params) {
|
|
1939
|
+
if (!outcome.ok) return outcome;
|
|
1940
|
+
const result = outcome.run && outcome.maxEvents === void 0 ? {
|
|
1941
|
+
...outcome,
|
|
1942
|
+
maxEvents: getMaxEventsPerRun()
|
|
1943
|
+
} : outcome;
|
|
1944
|
+
if (!result.event) return result;
|
|
1945
|
+
if (typeof params?.sinceCursor === "string") {
|
|
1946
|
+
const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1947
|
+
limit: 100,
|
|
1948
|
+
cursor: params.sinceCursor,
|
|
1949
|
+
sortOrder: "asc"
|
|
1950
|
+
}, (event) => event.eventId);
|
|
1951
|
+
return {
|
|
1952
|
+
...result,
|
|
1953
|
+
events: page.data,
|
|
1954
|
+
cursor: page.cursor,
|
|
1955
|
+
hasMore: page.hasMore
|
|
1956
|
+
};
|
|
1586
1957
|
}
|
|
1958
|
+
if (params?.eventCount === void 0) return result;
|
|
1959
|
+
const committedSlot = eventIdToSlot(result.event.eventId);
|
|
1960
|
+
if (committedSlot === null || committedSlot <= params.eventCount + 1) return result;
|
|
1961
|
+
const skippedCount = committedSlot - params.eventCount - 1;
|
|
1962
|
+
const events = (await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1963
|
+
limit: skippedCount,
|
|
1964
|
+
cursor: params.eventCount > 0 ? slotToEventId(params.eventCount) : void 0,
|
|
1965
|
+
sortOrder: "asc"
|
|
1966
|
+
}, (event) => event.eventId)).data.filter((event) => event.eventId < result.event.eventId);
|
|
1967
|
+
return {
|
|
1968
|
+
...result,
|
|
1969
|
+
events,
|
|
1970
|
+
cursor: null,
|
|
1971
|
+
hasMore: events.length < skippedCount
|
|
1972
|
+
};
|
|
1587
1973
|
}
|
|
1974
|
+
//#endregion
|
|
1975
|
+
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1976
|
+
const PAYLOAD_DELETE_BATCH = 256;
|
|
1977
|
+
const STREAM_DELETE_CONCURRENCY = 16;
|
|
1978
|
+
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
1588
1979
|
/**
|
|
1589
1980
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
1590
1981
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -1609,105 +2000,335 @@ function storeFrom(storage) {
|
|
|
1609
2000
|
* queue-message dedup claims (see claimInflight/releaseInflight).
|
|
1610
2001
|
*/
|
|
1611
2002
|
var WorkflowRunDO = class extends DurableObject {
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
* Memoized so it only runs once per DO instantiation.
|
|
1616
|
-
*/
|
|
1617
|
-
ensureSchema() {
|
|
1618
|
-
if (!this.schemaReady) this.schemaReady = ensureSchemaUpToDate(this.ctx.storage);
|
|
1619
|
-
return this.schemaReady;
|
|
2003
|
+
now() {
|
|
2004
|
+
const clock = this.env?.clock;
|
|
2005
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
1620
2006
|
}
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
2007
|
+
async expiredTombstone(storage, now = this.now()) {
|
|
2008
|
+
const tombstone = await storage.get(TOMBSTONE_KEY);
|
|
2009
|
+
if (tombstone) return tombstone;
|
|
2010
|
+
const cleanup = await storage.get(CLEANUP_RECORD_KEY);
|
|
2011
|
+
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
2012
|
+
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
2013
|
+
}
|
|
2014
|
+
async read(reader) {
|
|
2015
|
+
const tombstone = await this.expiredTombstone(this.ctx.storage);
|
|
2016
|
+
if (tombstone) return expiredRead(tombstone);
|
|
2017
|
+
return {
|
|
2018
|
+
ok: true,
|
|
2019
|
+
value: await reader()
|
|
2020
|
+
};
|
|
2021
|
+
}
|
|
2022
|
+
/** Apply an event and capture all retention metadata in the same transaction. */
|
|
1627
2023
|
async applyEvent(request) {
|
|
1628
|
-
await this.ensureSchema();
|
|
1629
2024
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2025
|
+
const now = new Date(this.now());
|
|
2026
|
+
const tombstone = await this.expiredTombstone(txn, now.getTime());
|
|
2027
|
+
if (tombstone) return {
|
|
2028
|
+
ok: false,
|
|
2029
|
+
code: "RUN_EXPIRED",
|
|
2030
|
+
message: `Workflow run "${request.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
2031
|
+
};
|
|
1630
2032
|
let eventSequence = await txn.get("event_sequence") ?? 0;
|
|
1631
2033
|
const outcome = await applyEvent(storeFrom(txn), {
|
|
1632
2034
|
...request,
|
|
1633
|
-
nextEventId: () =>
|
|
1634
|
-
now
|
|
2035
|
+
nextEventId: () => slotToEventId(++eventSequence),
|
|
2036
|
+
now
|
|
1635
2037
|
});
|
|
1636
|
-
if (outcome.ok)
|
|
1637
|
-
|
|
2038
|
+
if (!outcome.ok) return outcome;
|
|
2039
|
+
await txn.put("event_sequence", eventSequence);
|
|
2040
|
+
if (outcome.event?.correlationId) {
|
|
2041
|
+
const key = correlationIndexKey(outcome.event.correlationId, outcome.event.createdAt, outcome.event.eventId, request.runId);
|
|
2042
|
+
await txn.put(indexMarkerKey(key), key);
|
|
2043
|
+
}
|
|
2044
|
+
if (outcome.hookToIndex) {
|
|
2045
|
+
const reference = {
|
|
2046
|
+
hookId: outcome.hookToIndex.hookId,
|
|
2047
|
+
token: outcome.hookToIndex.token
|
|
2048
|
+
};
|
|
2049
|
+
await txn.put(hookMarkerKey(reference), reference);
|
|
2050
|
+
}
|
|
2051
|
+
for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
|
|
2052
|
+
const retentionMs = request.cleanup?.retentionMs ?? 0;
|
|
2053
|
+
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !await txn.get("retention:cleanup")) {
|
|
2054
|
+
const terminalRun = outcome.run;
|
|
2055
|
+
const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
|
|
2056
|
+
const run = WorkflowRunSchema.parse({
|
|
2057
|
+
...terminalRun,
|
|
2058
|
+
expiredAt: dueAt
|
|
2059
|
+
});
|
|
2060
|
+
const cleanup = {
|
|
2061
|
+
version: 1,
|
|
2062
|
+
runId: run.runId,
|
|
2063
|
+
workflowName: run.workflowName,
|
|
2064
|
+
createdAt: run.createdAt,
|
|
2065
|
+
completedAt: run.completedAt,
|
|
2066
|
+
terminalStatus: run.status,
|
|
2067
|
+
dueAt,
|
|
2068
|
+
queueShards: request.cleanup?.queueShards ?? 1,
|
|
2069
|
+
phase: "retained",
|
|
2070
|
+
attempts: 0,
|
|
2071
|
+
deletedPayloadKeys: 0,
|
|
2072
|
+
deletedStreams: 0,
|
|
2073
|
+
deletedQueueMessages: 0
|
|
2074
|
+
};
|
|
2075
|
+
await txn.put("run", run);
|
|
2076
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
2077
|
+
await txn.setAlarm(dueAt);
|
|
2078
|
+
outcome.run = run;
|
|
2079
|
+
}
|
|
2080
|
+
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
1638
2081
|
});
|
|
1639
2082
|
}
|
|
1640
|
-
/**
|
|
1641
|
-
* Get the workflow run
|
|
1642
|
-
*/
|
|
1643
2083
|
async getRun() {
|
|
1644
|
-
await this.
|
|
1645
|
-
return await this.ctx.storage.get("run") ?? null;
|
|
2084
|
+
return this.read(async () => await this.ctx.storage.get("run") ?? null);
|
|
1646
2085
|
}
|
|
1647
|
-
/**
|
|
1648
|
-
* Get a specific step
|
|
1649
|
-
*/
|
|
1650
2086
|
async getStep(stepId) {
|
|
1651
|
-
await this.
|
|
1652
|
-
return await this.ctx.storage.get(`step:${stepId}`) ?? null;
|
|
2087
|
+
return this.read(async () => await this.ctx.storage.get(`step:${stepId}`) ?? null);
|
|
1653
2088
|
}
|
|
1654
|
-
/**
|
|
1655
|
-
* List steps with real limit/cursor pagination (cursor = last stepId).
|
|
1656
|
-
*/
|
|
1657
2089
|
async listSteps(params) {
|
|
1658
|
-
|
|
1659
|
-
return listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
2090
|
+
return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
1660
2091
|
limit: params?.limit ?? 20,
|
|
1661
2092
|
cursor: params?.cursor,
|
|
1662
2093
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1663
|
-
});
|
|
2094
|
+
}));
|
|
1664
2095
|
}
|
|
1665
|
-
/**
|
|
1666
|
-
* Get a specific event
|
|
1667
|
-
*/
|
|
1668
2096
|
async getEvent(eventId) {
|
|
1669
|
-
await this.
|
|
1670
|
-
return await this.ctx.storage.get(`event:${eventId}`) ?? null;
|
|
2097
|
+
return this.read(async () => await this.ctx.storage.get(`event:${eventId}`) ?? null);
|
|
1671
2098
|
}
|
|
1672
|
-
/**
|
|
1673
|
-
* List events with real limit/cursor pagination. Events are keyed by their
|
|
1674
|
-
* monotonic ULID eventId, so storage order == event order.
|
|
1675
|
-
*/
|
|
1676
2099
|
async listEvents(params) {
|
|
1677
|
-
|
|
1678
|
-
return listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
|
|
2100
|
+
return this.read(() => listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
|
|
1679
2101
|
limit: params?.limit ?? 100,
|
|
1680
2102
|
cursor: params?.cursor,
|
|
1681
2103
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1682
|
-
}, (event) => event.eventId);
|
|
2104
|
+
}, (event) => event.eventId));
|
|
1683
2105
|
}
|
|
1684
|
-
/**
|
|
1685
|
-
* List hooks with real limit/cursor pagination (cursor = last hookId).
|
|
1686
|
-
*/
|
|
1687
2106
|
async listHooks(params) {
|
|
1688
|
-
|
|
1689
|
-
return listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
2107
|
+
return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
1690
2108
|
limit: params?.limit ?? 100,
|
|
1691
2109
|
cursor: params?.cursor,
|
|
1692
2110
|
sortOrder: params?.sortOrder ?? "asc"
|
|
2111
|
+
}));
|
|
2112
|
+
}
|
|
2113
|
+
async getCleanupStatus() {
|
|
2114
|
+
return await this.ctx.storage.get("retention:cleanup") ?? null;
|
|
2115
|
+
}
|
|
2116
|
+
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
2117
|
+
async scheduleCleanup(request) {
|
|
2118
|
+
if (request.retentionMs <= 0) return this.getCleanupStatus();
|
|
2119
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2120
|
+
const existing = await txn.get(CLEANUP_RECORD_KEY);
|
|
2121
|
+
if (existing) return existing;
|
|
2122
|
+
if (await txn.get("retention:tombstone")) return null;
|
|
2123
|
+
const current = await txn.get("run");
|
|
2124
|
+
if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
|
|
2125
|
+
const terminalRun = current;
|
|
2126
|
+
const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
|
|
2127
|
+
const run = WorkflowRunSchema.parse({
|
|
2128
|
+
...terminalRun,
|
|
2129
|
+
expiredAt: dueAt
|
|
2130
|
+
});
|
|
2131
|
+
const cleanup = {
|
|
2132
|
+
version: 1,
|
|
2133
|
+
runId: run.runId,
|
|
2134
|
+
workflowName: run.workflowName,
|
|
2135
|
+
createdAt: run.createdAt,
|
|
2136
|
+
completedAt: run.completedAt,
|
|
2137
|
+
terminalStatus: run.status,
|
|
2138
|
+
dueAt,
|
|
2139
|
+
queueShards: request.queueShards,
|
|
2140
|
+
phase: "retained",
|
|
2141
|
+
attempts: 0,
|
|
2142
|
+
deletedPayloadKeys: 0,
|
|
2143
|
+
deletedStreams: 0,
|
|
2144
|
+
deletedQueueMessages: 0
|
|
2145
|
+
};
|
|
2146
|
+
await txn.put("run", run);
|
|
2147
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
2148
|
+
await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
2149
|
+
return cleanup;
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
async cleanupNow(request) {
|
|
2153
|
+
if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
|
|
2154
|
+
...request,
|
|
2155
|
+
retentionMs: Math.max(1, request.retentionMs)
|
|
2156
|
+
});
|
|
2157
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2158
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2159
|
+
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
2160
|
+
const dueAt = new Date(this.now());
|
|
2161
|
+
const run = await txn.get("run");
|
|
2162
|
+
if (run) await txn.put("run", WorkflowRunSchema.parse({
|
|
2163
|
+
...run,
|
|
2164
|
+
expiredAt: dueAt
|
|
2165
|
+
}));
|
|
2166
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2167
|
+
...cleanup,
|
|
2168
|
+
dueAt
|
|
2169
|
+
});
|
|
2170
|
+
await txn.setAlarm(this.now() + 1);
|
|
2171
|
+
});
|
|
2172
|
+
try {
|
|
2173
|
+
await this.executeCleanup();
|
|
2174
|
+
} catch (error) {
|
|
2175
|
+
await this.recordCleanupFailure(error);
|
|
2176
|
+
}
|
|
2177
|
+
return this.getCleanupStatus();
|
|
2178
|
+
}
|
|
2179
|
+
async rearmCleanup() {
|
|
2180
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
2181
|
+
if (!cleanup) return null;
|
|
2182
|
+
if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
2183
|
+
else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
|
|
2184
|
+
return cleanup;
|
|
2185
|
+
}
|
|
2186
|
+
async alarm() {
|
|
2187
|
+
try {
|
|
2188
|
+
await this.executeCleanup();
|
|
2189
|
+
} catch (error) {
|
|
2190
|
+
await this.recordCleanupFailure(error);
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
namespace(name) {
|
|
2194
|
+
const namespace = this.env?.[name];
|
|
2195
|
+
if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
|
|
2196
|
+
return namespace;
|
|
2197
|
+
}
|
|
2198
|
+
async setCleanupPhase(expected, next, updates = {}) {
|
|
2199
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2200
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2201
|
+
if (!cleanup || cleanup.phase !== expected) return;
|
|
2202
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2203
|
+
...cleanup,
|
|
2204
|
+
...updates,
|
|
2205
|
+
phase: next,
|
|
2206
|
+
attempts: 0,
|
|
2207
|
+
lastError: void 0
|
|
2208
|
+
});
|
|
2209
|
+
});
|
|
2210
|
+
}
|
|
2211
|
+
async executeCleanup() {
|
|
2212
|
+
for (;;) {
|
|
2213
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
2214
|
+
if (!cleanup) {
|
|
2215
|
+
await this.ctx.storage.deleteAlarm();
|
|
2216
|
+
return;
|
|
2217
|
+
}
|
|
2218
|
+
if (cleanup.phase === "tombstoned") {
|
|
2219
|
+
await this.ctx.storage.deleteAlarm();
|
|
2220
|
+
return;
|
|
2221
|
+
}
|
|
2222
|
+
if (cleanup.phase === "retained") {
|
|
2223
|
+
if (cleanup.dueAt.getTime() > this.now()) {
|
|
2224
|
+
await this.ctx.storage.setAlarm(cleanup.dueAt);
|
|
2225
|
+
return;
|
|
2226
|
+
}
|
|
2227
|
+
await this.setCleanupPhase("retained", "index");
|
|
2228
|
+
continue;
|
|
2229
|
+
}
|
|
2230
|
+
if (cleanup.phase === "index") {
|
|
2231
|
+
const [indexMarkers, hookMarkers] = await Promise.all([this.ctx.storage.list({ prefix: INDEX_MARKER_PREFIX }), this.ctx.storage.list({ prefix: HOOK_MARKER_PREFIX })]);
|
|
2232
|
+
const indexNamespace = this.namespace("WORKFLOW_INDEX");
|
|
2233
|
+
await indexNamespace.get(indexNamespace.idFromName("index")).expireRun({
|
|
2234
|
+
runId: cleanup.runId,
|
|
2235
|
+
keys: [
|
|
2236
|
+
workflowRunIndexKey(cleanup),
|
|
2237
|
+
globalRunIndexKey(cleanup),
|
|
2238
|
+
...indexMarkers.values()
|
|
2239
|
+
],
|
|
2240
|
+
hooks: Array.from(hookMarkers.values()),
|
|
2241
|
+
expiredAt: cleanup.dueAt.getTime()
|
|
2242
|
+
});
|
|
2243
|
+
await this.setCleanupPhase("index", "streams");
|
|
2244
|
+
continue;
|
|
2245
|
+
}
|
|
2246
|
+
if (cleanup.phase === "streams") {
|
|
2247
|
+
const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
|
|
2248
|
+
const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
|
|
2249
|
+
const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime());
|
|
2250
|
+
let deletedStreams = streams.length === 0 ? cleanup.deletedStreams : 0;
|
|
2251
|
+
for (let offset = 0; offset < streams.length; offset += STREAM_DELETE_CONCURRENCY) {
|
|
2252
|
+
const batch = streams.slice(offset, offset + STREAM_DELETE_CONCURRENCY);
|
|
2253
|
+
const results = await Promise.all(batch.map((name) => {
|
|
2254
|
+
return streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime());
|
|
2255
|
+
}));
|
|
2256
|
+
deletedStreams += results.filter((result) => result.deleted).length;
|
|
2257
|
+
}
|
|
2258
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2259
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
2260
|
+
if (current?.phase === "streams") await txn.put(CLEANUP_RECORD_KEY, {
|
|
2261
|
+
...current,
|
|
2262
|
+
deletedStreams
|
|
2263
|
+
});
|
|
2264
|
+
});
|
|
2265
|
+
await registry.finalizeRegistry(cleanup.runId);
|
|
2266
|
+
await this.setCleanupPhase("streams", "queues");
|
|
2267
|
+
continue;
|
|
2268
|
+
}
|
|
2269
|
+
if (cleanup.phase === "queues") {
|
|
2270
|
+
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
2271
|
+
let deletedQueueMessages = 0;
|
|
2272
|
+
for (let shard = 0; shard < cleanup.queueShards; shard++) {
|
|
2273
|
+
const result = await queues.get(queues.idFromName(`q:${shard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime());
|
|
2274
|
+
deletedQueueMessages += result.deleted;
|
|
2275
|
+
}
|
|
2276
|
+
await this.setCleanupPhase("queues", "payload", { deletedQueueMessages });
|
|
2277
|
+
continue;
|
|
2278
|
+
}
|
|
2279
|
+
if (cleanup.phase === "payload") {
|
|
2280
|
+
if (!await this.deletePayloadBatch()) return;
|
|
2281
|
+
}
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
async deletePayloadBatch() {
|
|
2285
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2286
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2287
|
+
if (!cleanup || cleanup.phase !== "payload") return true;
|
|
2288
|
+
const entries = await txn.list({ limit: 259 });
|
|
2289
|
+
const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
|
|
2290
|
+
const now = new Date(this.now());
|
|
2291
|
+
await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
|
|
2292
|
+
for (const key of keys) await txn.delete(key);
|
|
2293
|
+
if (keys.length > 0) {
|
|
2294
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2295
|
+
...cleanup,
|
|
2296
|
+
deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
|
|
2297
|
+
lastError: void 0
|
|
2298
|
+
});
|
|
2299
|
+
await txn.setAlarm(this.now() + 1);
|
|
2300
|
+
return false;
|
|
2301
|
+
}
|
|
2302
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2303
|
+
...cleanup,
|
|
2304
|
+
phase: "tombstoned",
|
|
2305
|
+
attempts: 0,
|
|
2306
|
+
lastError: void 0,
|
|
2307
|
+
tombstonedAt: now
|
|
2308
|
+
});
|
|
2309
|
+
await txn.deleteAlarm();
|
|
2310
|
+
return true;
|
|
2311
|
+
});
|
|
2312
|
+
}
|
|
2313
|
+
async recordCleanupFailure(error) {
|
|
2314
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2315
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2316
|
+
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
2317
|
+
const attempts = cleanup.attempts + 1;
|
|
2318
|
+
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
2319
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2320
|
+
...cleanup,
|
|
2321
|
+
attempts,
|
|
2322
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
2323
|
+
lastAttemptAt: new Date(this.now())
|
|
2324
|
+
});
|
|
2325
|
+
await txn.setAlarm(this.now() + delay);
|
|
1693
2326
|
});
|
|
1694
2327
|
}
|
|
1695
|
-
/**
|
|
1696
|
-
* Claim a queue-message dedup slot. Used on DO instances dedicated to a
|
|
1697
|
-
* single idempotency key (`claim:<queueName>:<idempotencyKey>`).
|
|
1698
|
-
*
|
|
1699
|
-
* - Unclaimed -> claim and proceed.
|
|
1700
|
-
* - Claimed by the SAME messageId -> redelivery of the same message
|
|
1701
|
-
* (previous attempt failed transiently) -> proceed.
|
|
1702
|
-
* - Claimed by a DIFFERENT messageId -> concurrent duplicate -> reject,
|
|
1703
|
-
* unless the claim is stale (holder crashed / went to the DLQ without
|
|
1704
|
-
* releasing), in which case the claim is stolen.
|
|
1705
|
-
*/
|
|
1706
2328
|
async claimInflight(params) {
|
|
1707
|
-
await this.ensureSchema();
|
|
1708
2329
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1709
2330
|
const existing = await txn.get("claim");
|
|
1710
|
-
const now =
|
|
2331
|
+
const now = this.now();
|
|
1711
2332
|
if (existing && existing.messageId !== params.messageId && now - existing.claimedAt < params.staleMs) return { claimed: false };
|
|
1712
2333
|
await txn.put("claim", {
|
|
1713
2334
|
messageId: params.messageId,
|
|
@@ -1716,11 +2337,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1716
2337
|
return { claimed: true };
|
|
1717
2338
|
});
|
|
1718
2339
|
}
|
|
1719
|
-
/**
|
|
1720
|
-
* Release a queue-message dedup claim after the message was fully handled.
|
|
1721
|
-
*/
|
|
1722
2340
|
async releaseInflight() {
|
|
1723
|
-
await this.ensureSchema();
|
|
1724
2341
|
await this.ctx.storage.delete("claim");
|
|
1725
2342
|
}
|
|
1726
2343
|
};
|
|
@@ -1731,13 +2348,21 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1731
2348
|
* behind a node:http server — the full wire protocol without a celld binary.
|
|
1732
2349
|
*/
|
|
1733
2350
|
async function startHarness(options = {}) {
|
|
2351
|
+
const cellEnv = { ...options.cellEnv };
|
|
1734
2352
|
const fleet = new FakeFleet({
|
|
1735
2353
|
runs: WorkflowRunDO,
|
|
1736
2354
|
streams: StreamDO,
|
|
1737
2355
|
index: IndexDO,
|
|
1738
2356
|
queue: QueueDO,
|
|
1739
2357
|
...options.extraClasses
|
|
1740
|
-
},
|
|
2358
|
+
}, cellEnv);
|
|
2359
|
+
cellEnv.clock ??= options.virtualClock ? () => fleet.now : () => Date.now();
|
|
2360
|
+
Object.assign(cellEnv, {
|
|
2361
|
+
WORKFLOW_DB: fleet.namespace("runs"),
|
|
2362
|
+
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
2363
|
+
WORKFLOW_INDEX: fleet.namespace("index"),
|
|
2364
|
+
WORKFLOW_QUEUE: fleet.namespace("queue")
|
|
2365
|
+
});
|
|
1741
2366
|
const router = createRouter({
|
|
1742
2367
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
1743
2368
|
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
@@ -1771,7 +2396,10 @@ async function startHarness(options = {}) {
|
|
|
1771
2396
|
return {
|
|
1772
2397
|
url: `http://127.0.0.1:${port}`,
|
|
1773
2398
|
fleet,
|
|
1774
|
-
close: () => new Promise((resolve, reject) =>
|
|
2399
|
+
close: () => new Promise((resolve, reject) => {
|
|
2400
|
+
server.close((err) => err ? reject(err) : resolve());
|
|
2401
|
+
server.closeAllConnections();
|
|
2402
|
+
})
|
|
1775
2403
|
};
|
|
1776
2404
|
}
|
|
1777
2405
|
//#endregion
|