@ewhauser/world-celld 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +61 -23
- package/dist/index.d.ts +126 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +206 -135
- package/dist/index.js.map +1 -1
- package/dist/{util-BvWlkqgV.js → retention-bLY1vcOu.js} +44 -2
- package/dist/retention-bLY1vcOu.js.map +1 -0
- package/dist/testing.d.ts +2 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +802 -174
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +133 -62
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +829 -170
- package/dist/worker.js.map +1 -1
- package/package.json +7 -7
- package/dist/util-BvWlkqgV.js.map +0 -1
package/dist/worker.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema,
|
|
1
|
+
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
2
2
|
import { DurableObject } from "cloudflare:workers";
|
|
3
3
|
//#region src/vendor/shared/serialization.ts
|
|
4
4
|
/** Standard Date fields used across all worlds */
|
|
@@ -162,7 +162,11 @@ const BINDINGS = {
|
|
|
162
162
|
"getEvent",
|
|
163
163
|
"listEvents",
|
|
164
164
|
"listSteps",
|
|
165
|
-
"listHooks"
|
|
165
|
+
"listHooks",
|
|
166
|
+
"getCleanupStatus",
|
|
167
|
+
"scheduleCleanup",
|
|
168
|
+
"cleanupNow",
|
|
169
|
+
"rearmCleanup"
|
|
166
170
|
])
|
|
167
171
|
},
|
|
168
172
|
streams: {
|
|
@@ -173,7 +177,10 @@ const BINDINGS = {
|
|
|
173
177
|
"getChunks",
|
|
174
178
|
"getInfo",
|
|
175
179
|
"registerStream",
|
|
176
|
-
"listStreams"
|
|
180
|
+
"listStreams",
|
|
181
|
+
"expireRegistry",
|
|
182
|
+
"finalizeRegistry",
|
|
183
|
+
"expireStream"
|
|
177
184
|
])
|
|
178
185
|
},
|
|
179
186
|
index: {
|
|
@@ -182,6 +189,8 @@ const BINDINGS = {
|
|
|
182
189
|
"get",
|
|
183
190
|
"put",
|
|
184
191
|
"delete",
|
|
192
|
+
"putOwned",
|
|
193
|
+
"expireRun",
|
|
185
194
|
"list",
|
|
186
195
|
"reserveHookToken",
|
|
187
196
|
"finalizeHookIndexes",
|
|
@@ -197,7 +206,8 @@ const BINDINGS = {
|
|
|
197
206
|
"listDeadLetters",
|
|
198
207
|
"redriveDeadLetter",
|
|
199
208
|
"purgeDeadLetters",
|
|
200
|
-
"rearmAlarm"
|
|
209
|
+
"rearmAlarm",
|
|
210
|
+
"expireRun"
|
|
201
211
|
])
|
|
202
212
|
}
|
|
203
213
|
};
|
|
@@ -320,6 +330,14 @@ var IndexDO = class extends DurableObject {
|
|
|
320
330
|
async put(key, value) {
|
|
321
331
|
await this.ctx.storage.put(key, value);
|
|
322
332
|
}
|
|
333
|
+
/** Publish a derived index only while its owning run is not expired. */
|
|
334
|
+
async putOwned(runId, key, value) {
|
|
335
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
336
|
+
if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
|
|
337
|
+
await txn.put(key, value);
|
|
338
|
+
return { stored: true };
|
|
339
|
+
});
|
|
340
|
+
}
|
|
323
341
|
async delete(key) {
|
|
324
342
|
await this.ctx.storage.delete(key);
|
|
325
343
|
}
|
|
@@ -365,6 +383,11 @@ var IndexDO = class extends DurableObject {
|
|
|
365
383
|
}
|
|
366
384
|
async finalizeHookIndexes(token, hookId, serializedHook, owner) {
|
|
367
385
|
await this.ctx.storage.transaction(async (txn) => {
|
|
386
|
+
if (await txn.get(`expired:${owner.runId}`) !== void 0) {
|
|
387
|
+
const claim = await txn.get(`hookclaim:${token}`);
|
|
388
|
+
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(`hookclaim:${token}`);
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
368
391
|
const indexedHook = await txn.get(`hook:${token}`);
|
|
369
392
|
if (indexedHook !== void 0 && !sameOwner(ownerFromHook(indexedHook), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
370
393
|
const claim = await txn.get(`hookclaim:${token}`);
|
|
@@ -396,6 +419,42 @@ var IndexDO = class extends DurableObject {
|
|
|
396
419
|
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
|
|
397
420
|
});
|
|
398
421
|
}
|
|
422
|
+
/**
|
|
423
|
+
* Fence a run and remove all of its known derived indexes atomically.
|
|
424
|
+
* Once the fence exists, putOwned() and hook finalization cannot resurrect
|
|
425
|
+
* entries from a delayed request.
|
|
426
|
+
*/
|
|
427
|
+
async expireRun(request) {
|
|
428
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
429
|
+
await txn.put(`expired:${request.runId}`, request.expiredAt);
|
|
430
|
+
let deleted = 0;
|
|
431
|
+
for (const key of new Set(request.keys)) if (await txn.delete(key)) deleted++;
|
|
432
|
+
for (const hook of request.hooks) {
|
|
433
|
+
const owner = {
|
|
434
|
+
runId: request.runId,
|
|
435
|
+
hookId: hook.hookId
|
|
436
|
+
};
|
|
437
|
+
const tokenKey = `hook:${hook.token}`;
|
|
438
|
+
const idKey = `hookid:${hook.hookId}`;
|
|
439
|
+
const claimKey = `hookclaim:${hook.token}`;
|
|
440
|
+
const [byToken, byId, claim] = await Promise.all([
|
|
441
|
+
txn.get(tokenKey),
|
|
442
|
+
txn.get(idKey),
|
|
443
|
+
txn.get(claimKey)
|
|
444
|
+
]);
|
|
445
|
+
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
|
|
446
|
+
if (await txn.delete(tokenKey)) deleted++;
|
|
447
|
+
}
|
|
448
|
+
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
|
|
449
|
+
if (await txn.delete(idKey)) deleted++;
|
|
450
|
+
}
|
|
451
|
+
if (claim !== void 0 && sameOwner(claim.owner, owner)) {
|
|
452
|
+
if (await txn.delete(claimKey)) deleted++;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return { deleted };
|
|
456
|
+
});
|
|
457
|
+
}
|
|
399
458
|
};
|
|
400
459
|
//#endregion
|
|
401
460
|
//#region src/worker/durable-objects/QueueDO.ts
|
|
@@ -418,6 +477,12 @@ function pad(ms) {
|
|
|
418
477
|
function dueKey(atMs, messageId) {
|
|
419
478
|
return `due:${pad(atMs)}:${messageId}`;
|
|
420
479
|
}
|
|
480
|
+
function runReferenceKey(runId, messageId) {
|
|
481
|
+
return `run:${runId}:${messageId}`;
|
|
482
|
+
}
|
|
483
|
+
function expiredRunKey(runId) {
|
|
484
|
+
return `expired-run:${runId}`;
|
|
485
|
+
}
|
|
421
486
|
function backoffSeconds(attempt) {
|
|
422
487
|
return Math.min(60, 2 ** attempt);
|
|
423
488
|
}
|
|
@@ -439,6 +504,11 @@ var QueueDO = class extends DurableObject {
|
|
|
439
504
|
const storage = this.ctx.storage;
|
|
440
505
|
const now = this.#now();
|
|
441
506
|
return await storage.transaction(async (txn) => {
|
|
507
|
+
if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
|
|
508
|
+
ok: false,
|
|
509
|
+
code: "RUN_EXPIRED",
|
|
510
|
+
message: `Workflow run "${request.runId}" has expired`
|
|
511
|
+
};
|
|
442
512
|
const pinned = await txn.get("cfg");
|
|
443
513
|
if (!pinned) await txn.put("cfg", {
|
|
444
514
|
queueShards: request.config.queueShards,
|
|
@@ -469,14 +539,21 @@ var QueueDO = class extends DurableObject {
|
|
|
469
539
|
queueName: request.queueName,
|
|
470
540
|
pathname: request.pathname,
|
|
471
541
|
body: request.body,
|
|
542
|
+
runId: request.runId,
|
|
472
543
|
targetBaseUrl: request.config.targetBaseUrl,
|
|
473
544
|
idempotencyKey: request.idempotencyKey,
|
|
474
545
|
attempt: 0,
|
|
475
546
|
enqueuedAt: now
|
|
476
547
|
};
|
|
477
548
|
const dueAt = now + Math.max(0, request.delaySeconds ?? 0) * 1e3;
|
|
549
|
+
const scheduleKey = dueKey(dueAt, row.messageId);
|
|
478
550
|
await txn.put(`msg:${row.messageId}`, row);
|
|
479
|
-
await txn.put(
|
|
551
|
+
await txn.put(scheduleKey, row.messageId);
|
|
552
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
553
|
+
messageId: row.messageId,
|
|
554
|
+
dueKey: scheduleKey,
|
|
555
|
+
idempotencyKey: row.idempotencyKey
|
|
556
|
+
});
|
|
480
557
|
await this.#armAlarmAtMost(txn, dueAt, now);
|
|
481
558
|
return {
|
|
482
559
|
ok: true,
|
|
@@ -514,7 +591,16 @@ var QueueDO = class extends DurableObject {
|
|
|
514
591
|
for (const [key, deadline] of inflight) if (deadline <= now) {
|
|
515
592
|
const messageId = key.slice(9);
|
|
516
593
|
await txn.delete(key);
|
|
517
|
-
|
|
594
|
+
const row = await txn.get(`msg:${messageId}`);
|
|
595
|
+
if (row) {
|
|
596
|
+
const scheduleKey = dueKey(now, messageId);
|
|
597
|
+
await txn.put(scheduleKey, messageId);
|
|
598
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
599
|
+
messageId: row.messageId,
|
|
600
|
+
dueKey: scheduleKey,
|
|
601
|
+
idempotencyKey: row.idempotencyKey
|
|
602
|
+
});
|
|
603
|
+
}
|
|
518
604
|
} else inflightCount++;
|
|
519
605
|
const rows = [];
|
|
520
606
|
if (inflightCount < maxInflight) {
|
|
@@ -569,8 +655,25 @@ var QueueDO = class extends DurableObject {
|
|
|
569
655
|
timeoutSeconds = void 0;
|
|
570
656
|
}
|
|
571
657
|
if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
|
|
658
|
+
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
659
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
660
|
+
await txn.delete(`inflight:${row.messageId}`);
|
|
661
|
+
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
662
|
+
if (row.idempotencyKey) {
|
|
663
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
664
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
665
|
+
}
|
|
666
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
572
669
|
await txn.delete(`inflight:${row.messageId}`);
|
|
573
|
-
|
|
670
|
+
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
671
|
+
await txn.put(scheduleKey, row.messageId);
|
|
672
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
673
|
+
messageId: row.messageId,
|
|
674
|
+
dueKey: scheduleKey,
|
|
675
|
+
idempotencyKey: row.idempotencyKey
|
|
676
|
+
});
|
|
574
677
|
await this.#scheduleNextAlarm(txn, now);
|
|
575
678
|
});
|
|
576
679
|
else await this.#retry(row, "HTTP 503");
|
|
@@ -587,6 +690,7 @@ var QueueDO = class extends DurableObject {
|
|
|
587
690
|
if (row.idempotencyKey) {
|
|
588
691
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
589
692
|
}
|
|
693
|
+
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
590
694
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
591
695
|
});
|
|
592
696
|
}
|
|
@@ -596,6 +700,17 @@ var QueueDO = class extends DurableObject {
|
|
|
596
700
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
597
701
|
const attempt = row.attempt + 1;
|
|
598
702
|
await storage.transaction(async (txn) => {
|
|
703
|
+
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
704
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
705
|
+
await txn.delete(`inflight:${row.messageId}`);
|
|
706
|
+
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
707
|
+
if (row.idempotencyKey) {
|
|
708
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
709
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
710
|
+
}
|
|
711
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
599
714
|
await txn.delete(`inflight:${row.messageId}`);
|
|
600
715
|
if (attempt >= maxAttempts) {
|
|
601
716
|
const dead = {
|
|
@@ -604,11 +719,17 @@ var QueueDO = class extends DurableObject {
|
|
|
604
719
|
lastError: reason,
|
|
605
720
|
failedAt: now
|
|
606
721
|
};
|
|
607
|
-
|
|
722
|
+
const deadKey = `dlq:${pad(now)}:${row.messageId}`;
|
|
723
|
+
await txn.put(deadKey, dead);
|
|
608
724
|
await txn.delete(`msg:${row.messageId}`);
|
|
609
725
|
if (row.idempotencyKey) {
|
|
610
726
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
611
727
|
}
|
|
728
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
729
|
+
messageId: row.messageId,
|
|
730
|
+
dlqKey: deadKey,
|
|
731
|
+
idempotencyKey: row.idempotencyKey
|
|
732
|
+
});
|
|
612
733
|
await this.#scheduleNextAlarm(txn, now);
|
|
613
734
|
return;
|
|
614
735
|
}
|
|
@@ -617,8 +738,14 @@ var QueueDO = class extends DurableObject {
|
|
|
617
738
|
attempt,
|
|
618
739
|
lastError: reason
|
|
619
740
|
};
|
|
741
|
+
const scheduleKey = dueKey(now + backoffSeconds(attempt) * 1e3, row.messageId);
|
|
620
742
|
await txn.put(`msg:${row.messageId}`, updated);
|
|
621
|
-
await txn.put(
|
|
743
|
+
await txn.put(scheduleKey, row.messageId);
|
|
744
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
745
|
+
messageId: row.messageId,
|
|
746
|
+
dueKey: scheduleKey,
|
|
747
|
+
idempotencyKey: row.idempotencyKey
|
|
748
|
+
});
|
|
622
749
|
await this.#scheduleNextAlarm(txn, now);
|
|
623
750
|
});
|
|
624
751
|
}
|
|
@@ -702,8 +829,14 @@ var QueueDO = class extends DurableObject {
|
|
|
702
829
|
await txn.put(dedupKey, row.messageId);
|
|
703
830
|
}
|
|
704
831
|
await txn.put(`msg:${row.messageId}`, row);
|
|
705
|
-
|
|
832
|
+
const scheduleKey = dueKey(now, row.messageId);
|
|
833
|
+
await txn.put(scheduleKey, row.messageId);
|
|
706
834
|
await txn.delete(key);
|
|
835
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
836
|
+
messageId: row.messageId,
|
|
837
|
+
dueKey: scheduleKey,
|
|
838
|
+
idempotencyKey: row.idempotencyKey
|
|
839
|
+
});
|
|
707
840
|
await this.#armAlarmAtMost(txn, now, now);
|
|
708
841
|
return { ok: true };
|
|
709
842
|
}
|
|
@@ -713,7 +846,11 @@ var QueueDO = class extends DurableObject {
|
|
|
713
846
|
async purgeDeadLetters() {
|
|
714
847
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
715
848
|
const dlq = await txn.list({ prefix: "dlq:" });
|
|
716
|
-
for (const key of dlq
|
|
849
|
+
for (const [key, value] of dlq) {
|
|
850
|
+
await txn.delete(key);
|
|
851
|
+
const dead = value;
|
|
852
|
+
if (dead.runId) await txn.delete(runReferenceKey(dead.runId, dead.messageId));
|
|
853
|
+
}
|
|
717
854
|
return { purged: dlq.size };
|
|
718
855
|
});
|
|
719
856
|
}
|
|
@@ -725,6 +862,32 @@ var QueueDO = class extends DurableObject {
|
|
|
725
862
|
await this.#rearm();
|
|
726
863
|
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
727
864
|
}
|
|
865
|
+
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
866
|
+
async expireRun(runId, expiredAt) {
|
|
867
|
+
return { deleted: await this.ctx.storage.transaction(async (txn) => {
|
|
868
|
+
const existing = await txn.get(expiredRunKey(runId));
|
|
869
|
+
const references = await txn.list({ prefix: `run:${runId}:` });
|
|
870
|
+
let count = existing?.deleted ?? 0;
|
|
871
|
+
for (const [key, reference] of references) {
|
|
872
|
+
await txn.delete(`msg:${reference.messageId}`);
|
|
873
|
+
await txn.delete(`inflight:${reference.messageId}`);
|
|
874
|
+
if (reference.dueKey) await txn.delete(reference.dueKey);
|
|
875
|
+
if (reference.dlqKey) await txn.delete(reference.dlqKey);
|
|
876
|
+
if (reference.idempotencyKey) {
|
|
877
|
+
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
878
|
+
if (await txn.get(dedupKey) === reference.messageId) await txn.delete(dedupKey);
|
|
879
|
+
}
|
|
880
|
+
await txn.delete(key);
|
|
881
|
+
count++;
|
|
882
|
+
}
|
|
883
|
+
await txn.put(expiredRunKey(runId), {
|
|
884
|
+
expiredAt,
|
|
885
|
+
deleted: count
|
|
886
|
+
});
|
|
887
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
888
|
+
return count;
|
|
889
|
+
}) };
|
|
890
|
+
}
|
|
728
891
|
};
|
|
729
892
|
//#endregion
|
|
730
893
|
//#region src/worker/durable-objects/StreamDO.ts
|
|
@@ -732,6 +895,8 @@ const META_KEY = "meta";
|
|
|
732
895
|
const CHUNK_KEY_PREFIX = "chunk:";
|
|
733
896
|
/** Registry keys used when this DO instance acts as a per-run stream index. */
|
|
734
897
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
898
|
+
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
899
|
+
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
735
900
|
/**
|
|
736
901
|
* Zero-pad chunk indexes so `storage.list({ prefix })` returns chunks in
|
|
737
902
|
* write order. 12 digits comfortably exceeds any realistic chunk count.
|
|
@@ -756,22 +921,28 @@ var StreamDO = class extends DurableObject {
|
|
|
756
921
|
closed: false
|
|
757
922
|
};
|
|
758
923
|
}
|
|
924
|
+
assertOwner(meta, runId) {
|
|
925
|
+
if (meta.expiredAt !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
926
|
+
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
927
|
+
}
|
|
759
928
|
/**
|
|
760
929
|
* Append a chunk. The index is allocated inside the transaction, so
|
|
761
930
|
* concurrent writers can never collide or skip.
|
|
762
931
|
*/
|
|
763
|
-
async writeChunk(data) {
|
|
932
|
+
async writeChunk(runId, data) {
|
|
764
933
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
765
934
|
const meta = await txn.get(META_KEY) ?? {
|
|
766
935
|
count: 0,
|
|
767
936
|
closed: false
|
|
768
937
|
};
|
|
938
|
+
this.assertOwner(meta, runId);
|
|
769
939
|
if (meta.closed) throw new Error("Cannot write to a closed stream");
|
|
770
940
|
const index = meta.count;
|
|
771
941
|
await txn.put(chunkKey(index), data);
|
|
772
942
|
await txn.put(META_KEY, {
|
|
773
943
|
count: index + 1,
|
|
774
|
-
closed: false
|
|
944
|
+
closed: false,
|
|
945
|
+
ownerRunId: runId
|
|
775
946
|
});
|
|
776
947
|
return index;
|
|
777
948
|
});
|
|
@@ -780,14 +951,16 @@ var StreamDO = class extends DurableObject {
|
|
|
780
951
|
* Close the stream (idempotent). Readers observe `done: true` once all
|
|
781
952
|
* previously written chunks have been consumed.
|
|
782
953
|
*/
|
|
783
|
-
async closeStream() {
|
|
954
|
+
async closeStream(runId) {
|
|
784
955
|
await this.ctx.storage.transaction(async (txn) => {
|
|
785
956
|
const meta = await txn.get(META_KEY) ?? {
|
|
786
957
|
count: 0,
|
|
787
958
|
closed: false
|
|
788
959
|
};
|
|
960
|
+
this.assertOwner(meta, runId);
|
|
789
961
|
await txn.put(META_KEY, {
|
|
790
962
|
...meta,
|
|
963
|
+
ownerRunId: runId,
|
|
791
964
|
closed: true
|
|
792
965
|
});
|
|
793
966
|
});
|
|
@@ -797,6 +970,7 @@ var StreamDO = class extends DurableObject {
|
|
|
797
970
|
*/
|
|
798
971
|
async getChunks(params) {
|
|
799
972
|
const meta = await this.getMeta();
|
|
973
|
+
if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
|
|
800
974
|
const start = Math.max(0, params.startIndex);
|
|
801
975
|
const entries = await this.ctx.storage.list({
|
|
802
976
|
prefix: CHUNK_KEY_PREFIX,
|
|
@@ -814,6 +988,7 @@ var StreamDO = class extends DurableObject {
|
|
|
814
988
|
*/
|
|
815
989
|
async getInfo() {
|
|
816
990
|
const meta = await this.getMeta();
|
|
991
|
+
if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
|
|
817
992
|
return {
|
|
818
993
|
tailIndex: meta.count - 1,
|
|
819
994
|
done: meta.closed
|
|
@@ -822,16 +997,66 @@ var StreamDO = class extends DurableObject {
|
|
|
822
997
|
/**
|
|
823
998
|
* Register a stream name against this per-run registry instance.
|
|
824
999
|
*/
|
|
825
|
-
async registerStream(name) {
|
|
826
|
-
await this.ctx.storage.
|
|
1000
|
+
async registerStream(runId, name) {
|
|
1001
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1002
|
+
if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
1003
|
+
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1004
|
+
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1005
|
+
await txn.put(REGISTRY_OWNER_KEY, runId);
|
|
1006
|
+
await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
|
|
1007
|
+
});
|
|
827
1008
|
}
|
|
828
1009
|
/**
|
|
829
1010
|
* List stream names registered against this per-run registry instance.
|
|
830
1011
|
*/
|
|
831
1012
|
async listStreams() {
|
|
1013
|
+
if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
|
|
832
1014
|
const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
833
1015
|
return Array.from(entries.keys()).map((key) => key.slice(7));
|
|
834
1016
|
}
|
|
1017
|
+
/** Fence a run's registry before any stream cells are removed. */
|
|
1018
|
+
async expireRegistry(runId, expiredAt) {
|
|
1019
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1020
|
+
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1021
|
+
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1022
|
+
await txn.put(REGISTRY_OWNER_KEY, runId);
|
|
1023
|
+
await txn.put(REGISTRY_EXPIRED_KEY, expiredAt);
|
|
1024
|
+
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1025
|
+
return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
/** Remove the registry payload after every referenced stream was fenced. */
|
|
1029
|
+
async finalizeRegistry(runId) {
|
|
1030
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1031
|
+
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1032
|
+
const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
|
|
1033
|
+
if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1034
|
+
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1035
|
+
for (const key of entries.keys()) await txn.delete(key);
|
|
1036
|
+
});
|
|
1037
|
+
}
|
|
1038
|
+
/** Delete stream payload while leaving a small ownership tombstone. */
|
|
1039
|
+
async expireStream(runId, expiredAt) {
|
|
1040
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1041
|
+
const meta = await txn.get(META_KEY) ?? {
|
|
1042
|
+
count: 0,
|
|
1043
|
+
closed: false
|
|
1044
|
+
};
|
|
1045
|
+
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
1046
|
+
const chunks = await txn.list({ prefix: CHUNK_KEY_PREFIX });
|
|
1047
|
+
for (const key of chunks.keys()) await txn.delete(key);
|
|
1048
|
+
await txn.put(META_KEY, {
|
|
1049
|
+
count: 0,
|
|
1050
|
+
closed: true,
|
|
1051
|
+
ownerRunId: runId,
|
|
1052
|
+
expiredAt
|
|
1053
|
+
});
|
|
1054
|
+
return {
|
|
1055
|
+
deleted: true,
|
|
1056
|
+
chunks: chunks.size
|
|
1057
|
+
};
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
835
1060
|
};
|
|
836
1061
|
//#endregion
|
|
837
1062
|
//#region src/apply-event.ts
|
|
@@ -841,6 +1066,7 @@ const RUN_KEY = "run";
|
|
|
841
1066
|
const EVENT_KEY_PREFIX = "event:";
|
|
842
1067
|
const STEP_KEY_PREFIX = "step:";
|
|
843
1068
|
const HOOK_KEY_PREFIX = "hook:";
|
|
1069
|
+
const WAIT_KEY_PREFIX = "wait:";
|
|
844
1070
|
const STEP_CREATED_KEY_PREFIX = "stepcreated:";
|
|
845
1071
|
const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
846
1072
|
/**
|
|
@@ -850,6 +1076,10 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
|
850
1076
|
* rejected as a duplicate instead of resurrecting the hook.
|
|
851
1077
|
*/
|
|
852
1078
|
const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
|
|
1079
|
+
/** Workflow-authored attr_set correlation claims. */
|
|
1080
|
+
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
1081
|
+
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
1082
|
+
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
853
1083
|
function failure(code, message, extra) {
|
|
854
1084
|
return {
|
|
855
1085
|
ok: false,
|
|
@@ -952,9 +1182,15 @@ async function releaseAllHooks(store) {
|
|
|
952
1182
|
}
|
|
953
1183
|
return released;
|
|
954
1184
|
}
|
|
1185
|
+
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
1186
|
+
async function releaseAllWaits(store) {
|
|
1187
|
+
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
1188
|
+
for (const key of waits.keys()) await store.delete(key);
|
|
1189
|
+
}
|
|
955
1190
|
async function applyEvent(store, ctx) {
|
|
956
1191
|
const { runId, data, now } = ctx;
|
|
957
1192
|
const specVersion = data.specVersion ?? SPEC_VERSION_CURRENT;
|
|
1193
|
+
if (specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: specVersion });
|
|
958
1194
|
/** Build and validate the stored event BEFORE anything is persisted. */
|
|
959
1195
|
const buildEvent = (record) => {
|
|
960
1196
|
const withMeta = {
|
|
@@ -962,9 +1198,16 @@ async function applyEvent(store, ctx) {
|
|
|
962
1198
|
runId,
|
|
963
1199
|
eventId: ctx.nextEventId(),
|
|
964
1200
|
createdAt: now,
|
|
1201
|
+
occurredAt: ctx.params?.occurredAt,
|
|
1202
|
+
resumeId: ctx.params?.resumeId,
|
|
965
1203
|
specVersion
|
|
966
1204
|
};
|
|
1205
|
+
if (record.eventType !== "hook_received") delete withMeta.resumeId;
|
|
967
1206
|
if (record.eventType === "run_started") delete withMeta.eventData;
|
|
1207
|
+
if (record.eventType === "step_started" && withMeta.eventData !== null && typeof withMeta.eventData === "object") {
|
|
1208
|
+
const { input: _input, ...eventData } = withMeta.eventData;
|
|
1209
|
+
withMeta.eventData = eventData;
|
|
1210
|
+
}
|
|
968
1211
|
return EventSchema.parse(compact(withMeta));
|
|
969
1212
|
};
|
|
970
1213
|
const putEvent = async (event) => {
|
|
@@ -975,14 +1218,21 @@ async function applyEvent(store, ctx) {
|
|
|
975
1218
|
if (!skipRunValidation) currentRun = await store.get(RUN_KEY);
|
|
976
1219
|
let bootstrapped;
|
|
977
1220
|
if (data.eventType === "run_started" && !currentRun && data.eventData) {
|
|
978
|
-
const { deploymentId, workflowName, input, executionContext } = data.eventData;
|
|
1221
|
+
const { deploymentId, workflowName, input, executionContext, attributes, allowReservedAttributes, encryptionPublicKey } = data.eventData;
|
|
979
1222
|
if (deploymentId && workflowName && input !== void 0) {
|
|
1223
|
+
const attributeChanges = Object.entries(attributes ?? {}).map(([key, value]) => ({
|
|
1224
|
+
key,
|
|
1225
|
+
value
|
|
1226
|
+
}));
|
|
1227
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes });
|
|
980
1228
|
const createdRun = WorkflowRunSchema.parse(compact({
|
|
981
1229
|
runId,
|
|
982
1230
|
deploymentId,
|
|
983
1231
|
workflowName,
|
|
984
1232
|
specVersion,
|
|
985
1233
|
executionContext,
|
|
1234
|
+
attributes,
|
|
1235
|
+
encryptionPublicKey,
|
|
986
1236
|
input,
|
|
987
1237
|
status: "pending",
|
|
988
1238
|
output: void 0,
|
|
@@ -998,7 +1248,10 @@ async function applyEvent(store, ctx) {
|
|
|
998
1248
|
deploymentId,
|
|
999
1249
|
workflowName,
|
|
1000
1250
|
input,
|
|
1001
|
-
executionContext
|
|
1251
|
+
executionContext,
|
|
1252
|
+
attributes,
|
|
1253
|
+
allowReservedAttributes,
|
|
1254
|
+
encryptionPublicKey
|
|
1002
1255
|
}
|
|
1003
1256
|
});
|
|
1004
1257
|
await store.put(RUN_KEY, createdRun);
|
|
@@ -1010,10 +1263,7 @@ async function applyEvent(store, ctx) {
|
|
|
1010
1263
|
};
|
|
1011
1264
|
}
|
|
1012
1265
|
}
|
|
1013
|
-
if (currentRun) {
|
|
1014
|
-
if (requiresNewerWorld(currentRun.specVersion)) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" requires a newer world (run specVersion ${currentRun.specVersion}, world ${SPEC_VERSION_CURRENT})`, { runSpecVersion: currentRun.specVersion });
|
|
1015
|
-
if (isLegacySpecVersion(currentRun.specVersion)) return failure("LEGACY_RUN_NOT_SUPPORTED", `Run "${runId}" was created with a legacy spec version (${currentRun.specVersion ?? "undefined"}) not supported by this world`);
|
|
1016
|
-
}
|
|
1266
|
+
if (currentRun && currentRun.specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${currentRun.specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: currentRun.specVersion });
|
|
1017
1267
|
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) {
|
|
1018
1268
|
if (data.eventType === "run_cancelled" && currentRun.status === "cancelled") {
|
|
1019
1269
|
const event = buildEvent({ ...data });
|
|
@@ -1027,14 +1277,32 @@ async function applyEvent(store, ctx) {
|
|
|
1027
1277
|
}
|
|
1028
1278
|
if (data.eventType === "run_started") return failure("RUN_EXPIRED", `Workflow run "${runId}" is already in terminal state "${currentRun.status}"`);
|
|
1029
1279
|
if (data.eventType === "run_completed" || data.eventType === "run_failed" || data.eventType === "run_cancelled") return failure("ENTITY_CONFLICT", `Cannot transition run from terminal state "${currentRun.status}"`);
|
|
1030
|
-
if (data
|
|
1280
|
+
if (isChildEntityCreationEvent(data)) return failure("ENTITY_CONFLICT", `Cannot create new entities on run in terminal state "${currentRun.status}"`);
|
|
1281
|
+
if (data.eventType === "attr_set") return failure("ENTITY_CONFLICT", `Cannot set attributes on run in terminal state "${currentRun.status}"`);
|
|
1031
1282
|
}
|
|
1032
1283
|
let validatedStep;
|
|
1033
1284
|
if (data.eventType === "step_started" || data.eventType === "step_completed" || data.eventType === "step_failed" || data.eventType === "step_retrying") {
|
|
1285
|
+
const lazyStepStart = data.eventType === "step_started" && isChildEntityCreationEvent(data);
|
|
1034
1286
|
validatedStep = await store.get(`${STEP_KEY_PREFIX}${data.correlationId}`);
|
|
1035
|
-
if (!validatedStep)
|
|
1036
|
-
|
|
1037
|
-
|
|
1287
|
+
if (!validatedStep) {
|
|
1288
|
+
if (!lazyStepStart) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1289
|
+
} else if (lazyStepStart) return failure("ENTITY_CONFLICT", `Step "${data.correlationId}" already exists`);
|
|
1290
|
+
if (validatedStep && isTerminalStepStatus(validatedStep.status)) return failure("ENTITY_CONFLICT", `Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
1291
|
+
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status) && validatedStep && validatedStep.status !== "running") return failure("RUN_EXPIRED", `Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
|
|
1292
|
+
}
|
|
1293
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) {
|
|
1294
|
+
const claim = await store.get(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`);
|
|
1295
|
+
if (claim) {
|
|
1296
|
+
if (claim.hookId !== data.correlationId) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded for a different hook`);
|
|
1297
|
+
if (claim.payloadDigest && ctx.params.resumePayloadDigest && claim.payloadDigest !== ctx.params.resumePayloadDigest) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded with a different payload`);
|
|
1298
|
+
const committed = await store.get(`${EVENT_KEY_PREFIX}${claim.eventId}`);
|
|
1299
|
+
if (committed?.eventType === "hook_received" && committed.correlationId === claim.hookId && committed.resumeId === ctx.params.resumeId) return {
|
|
1300
|
+
ok: true,
|
|
1301
|
+
event: committed,
|
|
1302
|
+
releasedHooks: []
|
|
1303
|
+
};
|
|
1304
|
+
if (committed) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" points to a different event`);
|
|
1305
|
+
}
|
|
1038
1306
|
}
|
|
1039
1307
|
if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
|
|
1040
1308
|
if (!await store.get(`hook:${data.correlationId}`)) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
@@ -1048,12 +1316,19 @@ async function applyEvent(store, ctx) {
|
|
|
1048
1316
|
run: existing,
|
|
1049
1317
|
releasedHooks: []
|
|
1050
1318
|
};
|
|
1319
|
+
const attributeChanges = Object.entries(data.eventData.attributes ?? {}).map(([key, value]) => ({
|
|
1320
|
+
key,
|
|
1321
|
+
value
|
|
1322
|
+
}));
|
|
1323
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes: data.eventData.allowReservedAttributes });
|
|
1051
1324
|
const run = WorkflowRunSchema.parse(compact({
|
|
1052
1325
|
runId,
|
|
1053
1326
|
deploymentId: data.eventData.deploymentId,
|
|
1054
1327
|
workflowName: data.eventData.workflowName,
|
|
1055
1328
|
specVersion,
|
|
1056
1329
|
executionContext: data.eventData.executionContext,
|
|
1330
|
+
attributes: data.eventData.attributes,
|
|
1331
|
+
encryptionPublicKey: data.eventData.encryptionPublicKey,
|
|
1057
1332
|
input: data.eventData.input,
|
|
1058
1333
|
status: "pending",
|
|
1059
1334
|
output: void 0,
|
|
@@ -1079,11 +1354,25 @@ async function applyEvent(store, ctx) {
|
|
|
1079
1354
|
}
|
|
1080
1355
|
case "run_started": {
|
|
1081
1356
|
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1082
|
-
if (currentRun.status === "running")
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1357
|
+
if (currentRun.status === "running") {
|
|
1358
|
+
const result = {
|
|
1359
|
+
ok: true,
|
|
1360
|
+
run: currentRun,
|
|
1361
|
+
releasedHooks: [],
|
|
1362
|
+
maxEvents: getMaxEventsPerRun()
|
|
1363
|
+
};
|
|
1364
|
+
if (ctx.params?.skipPreload) return result;
|
|
1365
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1366
|
+
limit: getMaxEventsPerRun(),
|
|
1367
|
+
sortOrder: "asc"
|
|
1368
|
+
}, (event) => event.eventId);
|
|
1369
|
+
return {
|
|
1370
|
+
...result,
|
|
1371
|
+
events: allEvents.data,
|
|
1372
|
+
cursor: allEvents.cursor,
|
|
1373
|
+
hasMore: allEvents.hasMore
|
|
1374
|
+
};
|
|
1375
|
+
}
|
|
1087
1376
|
const run = WorkflowRunSchema.parse(compact({
|
|
1088
1377
|
...currentRun,
|
|
1089
1378
|
status: "running",
|
|
@@ -1096,17 +1385,24 @@ async function applyEvent(store, ctx) {
|
|
|
1096
1385
|
const event = buildEvent({ ...data });
|
|
1097
1386
|
await store.put(RUN_KEY, run);
|
|
1098
1387
|
await putEvent(event);
|
|
1099
|
-
const
|
|
1100
|
-
limit: 1e3,
|
|
1101
|
-
sortOrder: "asc"
|
|
1102
|
-
}, (e) => e.eventId);
|
|
1103
|
-
return {
|
|
1388
|
+
const result = {
|
|
1104
1389
|
ok: true,
|
|
1105
1390
|
event,
|
|
1106
1391
|
run,
|
|
1107
1392
|
releasedHooks: [],
|
|
1108
1393
|
runCreated: bootstrapped,
|
|
1109
|
-
|
|
1394
|
+
maxEvents: getMaxEventsPerRun()
|
|
1395
|
+
};
|
|
1396
|
+
if (ctx.params?.skipPreload) return result;
|
|
1397
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1398
|
+
limit: getMaxEventsPerRun(),
|
|
1399
|
+
sortOrder: "asc"
|
|
1400
|
+
}, (e) => e.eventId);
|
|
1401
|
+
return {
|
|
1402
|
+
...result,
|
|
1403
|
+
events: allEvents.data,
|
|
1404
|
+
cursor: allEvents.cursor,
|
|
1405
|
+
hasMore: allEvents.hasMore
|
|
1110
1406
|
};
|
|
1111
1407
|
}
|
|
1112
1408
|
case "run_completed": {
|
|
@@ -1122,11 +1418,13 @@ async function applyEvent(store, ctx) {
|
|
|
1122
1418
|
const event = buildEvent({ ...data });
|
|
1123
1419
|
await store.put(RUN_KEY, run);
|
|
1124
1420
|
await putEvent(event);
|
|
1421
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
1422
|
+
await releaseAllWaits(store);
|
|
1125
1423
|
return {
|
|
1126
1424
|
ok: true,
|
|
1127
1425
|
event,
|
|
1128
1426
|
run,
|
|
1129
|
-
releasedHooks
|
|
1427
|
+
releasedHooks
|
|
1130
1428
|
};
|
|
1131
1429
|
}
|
|
1132
1430
|
case "run_failed": {
|
|
@@ -1142,11 +1440,13 @@ async function applyEvent(store, ctx) {
|
|
|
1142
1440
|
const event = buildEvent({ ...data });
|
|
1143
1441
|
await store.put(RUN_KEY, run);
|
|
1144
1442
|
await putEvent(event);
|
|
1443
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
1444
|
+
await releaseAllWaits(store);
|
|
1145
1445
|
return {
|
|
1146
1446
|
ok: true,
|
|
1147
1447
|
event,
|
|
1148
1448
|
run,
|
|
1149
|
-
releasedHooks
|
|
1449
|
+
releasedHooks
|
|
1150
1450
|
};
|
|
1151
1451
|
}
|
|
1152
1452
|
case "run_cancelled": {
|
|
@@ -1162,11 +1462,37 @@ async function applyEvent(store, ctx) {
|
|
|
1162
1462
|
const event = buildEvent({ ...data });
|
|
1163
1463
|
await store.put(RUN_KEY, run);
|
|
1164
1464
|
await putEvent(event);
|
|
1465
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
1466
|
+
await releaseAllWaits(store);
|
|
1165
1467
|
return {
|
|
1166
1468
|
ok: true,
|
|
1167
1469
|
event,
|
|
1168
1470
|
run,
|
|
1169
|
-
releasedHooks
|
|
1471
|
+
releasedHooks
|
|
1472
|
+
};
|
|
1473
|
+
}
|
|
1474
|
+
case "attr_set": {
|
|
1475
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1476
|
+
validateAttributeChanges(data.eventData.changes, {
|
|
1477
|
+
existingKeys: Object.keys(currentRun.attributes),
|
|
1478
|
+
allowReservedAttributes: data.eventData.allowReservedAttributes
|
|
1479
|
+
});
|
|
1480
|
+
const markerKey = data.correlationId && data.eventData.writer.type === "workflow" ? `${ATTRIBUTE_EVENT_MARKER_PREFIX}${data.correlationId}` : void 0;
|
|
1481
|
+
if (markerKey && await store.get(markerKey) !== void 0) return failure("ENTITY_CONFLICT", `Attribute event "${data.correlationId}" already exists`);
|
|
1482
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
1483
|
+
...currentRun,
|
|
1484
|
+
attributes: applyAttributeChanges(currentRun.attributes, data.eventData.changes),
|
|
1485
|
+
updatedAt: now
|
|
1486
|
+
}));
|
|
1487
|
+
const event = buildEvent({ ...data });
|
|
1488
|
+
await store.put(RUN_KEY, run);
|
|
1489
|
+
if (markerKey) await store.put(markerKey, event.eventId);
|
|
1490
|
+
await putEvent(event);
|
|
1491
|
+
return {
|
|
1492
|
+
ok: true,
|
|
1493
|
+
event,
|
|
1494
|
+
run,
|
|
1495
|
+
releasedHooks: []
|
|
1170
1496
|
};
|
|
1171
1497
|
}
|
|
1172
1498
|
case "step_created": {
|
|
@@ -1200,7 +1526,34 @@ async function applyEvent(store, ctx) {
|
|
|
1200
1526
|
};
|
|
1201
1527
|
}
|
|
1202
1528
|
case "step_started": {
|
|
1203
|
-
|
|
1529
|
+
let step = validatedStep;
|
|
1530
|
+
let stepCreated = false;
|
|
1531
|
+
if (!step && data.eventData?.input !== void 0 && data.eventData.stepName) {
|
|
1532
|
+
step = StepSchema.parse(compact({
|
|
1533
|
+
runId,
|
|
1534
|
+
stepId: data.correlationId,
|
|
1535
|
+
stepName: data.eventData.stepName,
|
|
1536
|
+
status: "pending",
|
|
1537
|
+
input: data.eventData.input,
|
|
1538
|
+
attempt: 0,
|
|
1539
|
+
createdAt: now,
|
|
1540
|
+
updatedAt: now,
|
|
1541
|
+
specVersion
|
|
1542
|
+
}));
|
|
1543
|
+
const createdEvent = buildEvent({
|
|
1544
|
+
eventType: "step_created",
|
|
1545
|
+
correlationId: data.correlationId,
|
|
1546
|
+
eventData: {
|
|
1547
|
+
stepName: data.eventData.stepName,
|
|
1548
|
+
workflowName: data.eventData.workflowName,
|
|
1549
|
+
input: data.eventData.input
|
|
1550
|
+
}
|
|
1551
|
+
});
|
|
1552
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, step);
|
|
1553
|
+
await store.put(creationIndexKey(STEP_CREATED_KEY_PREFIX, step.createdAt, step.stepId), step.stepId);
|
|
1554
|
+
await putEvent(createdEvent);
|
|
1555
|
+
stepCreated = true;
|
|
1556
|
+
}
|
|
1204
1557
|
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1205
1558
|
if (step.retryAfter && step.retryAfter.getTime() > now.getTime()) return failure("TOO_EARLY", `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, { retryAfterSeconds: Math.ceil((step.retryAfter.getTime() - now.getTime()) / 1e3) });
|
|
1206
1559
|
const updated = StepSchema.parse(compact({
|
|
@@ -1218,7 +1571,8 @@ async function applyEvent(store, ctx) {
|
|
|
1218
1571
|
ok: true,
|
|
1219
1572
|
event,
|
|
1220
1573
|
step: updated,
|
|
1221
|
-
releasedHooks: []
|
|
1574
|
+
releasedHooks: [],
|
|
1575
|
+
...stepCreated ? { stepCreated: true } : {}
|
|
1222
1576
|
};
|
|
1223
1577
|
}
|
|
1224
1578
|
case "step_completed": {
|
|
@@ -1247,7 +1601,7 @@ async function applyEvent(store, ctx) {
|
|
|
1247
1601
|
const updated = StepSchema.parse(compact({
|
|
1248
1602
|
...step,
|
|
1249
1603
|
status: "failed",
|
|
1250
|
-
error: toStructuredError(data.eventData.error
|
|
1604
|
+
error: toStructuredError(data.eventData.error),
|
|
1251
1605
|
completedAt: now,
|
|
1252
1606
|
updatedAt: now
|
|
1253
1607
|
}));
|
|
@@ -1267,7 +1621,7 @@ async function applyEvent(store, ctx) {
|
|
|
1267
1621
|
const updated = StepSchema.parse(compact({
|
|
1268
1622
|
...step,
|
|
1269
1623
|
status: "pending",
|
|
1270
|
-
error: toStructuredError(data.eventData.error
|
|
1624
|
+
error: toStructuredError(data.eventData.error),
|
|
1271
1625
|
retryAfter: data.eventData.retryAfter ? new Date(data.eventData.retryAfter) : void 0,
|
|
1272
1626
|
updatedAt: now
|
|
1273
1627
|
}));
|
|
@@ -1323,7 +1677,9 @@ async function applyEvent(store, ctx) {
|
|
|
1323
1677
|
environment: "",
|
|
1324
1678
|
createdAt: now,
|
|
1325
1679
|
specVersion,
|
|
1326
|
-
isWebhook: data.eventData.isWebhook ?? false
|
|
1680
|
+
isWebhook: data.eventData.isWebhook ?? false,
|
|
1681
|
+
isSystem: data.eventData.isSystem,
|
|
1682
|
+
tokenRetentionUntil: data.eventData.tokenRetentionUntil
|
|
1327
1683
|
}));
|
|
1328
1684
|
const event = buildEvent({ ...data });
|
|
1329
1685
|
await store.put(hookKey, hook);
|
|
@@ -1355,9 +1711,58 @@ async function applyEvent(store, ctx) {
|
|
|
1355
1711
|
}]
|
|
1356
1712
|
};
|
|
1357
1713
|
}
|
|
1714
|
+
case "wait_created": {
|
|
1715
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
1716
|
+
if (await store.get(waitKey) !== void 0) return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already exists`);
|
|
1717
|
+
const wait = WaitSchema.parse({
|
|
1718
|
+
waitId: `${runId}-${data.correlationId}`,
|
|
1719
|
+
runId,
|
|
1720
|
+
status: "waiting",
|
|
1721
|
+
resumeAt: data.eventData.resumeAt,
|
|
1722
|
+
createdAt: now,
|
|
1723
|
+
updatedAt: now,
|
|
1724
|
+
specVersion
|
|
1725
|
+
});
|
|
1726
|
+
const event = buildEvent({ ...data });
|
|
1727
|
+
await store.put(waitKey, wait);
|
|
1728
|
+
await putEvent(event);
|
|
1729
|
+
return {
|
|
1730
|
+
ok: true,
|
|
1731
|
+
event,
|
|
1732
|
+
wait,
|
|
1733
|
+
releasedHooks: []
|
|
1734
|
+
};
|
|
1735
|
+
}
|
|
1736
|
+
case "wait_completed": {
|
|
1737
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
1738
|
+
const existing = await store.get(waitKey);
|
|
1739
|
+
if (!existing) return failure("WAIT_NOT_FOUND", `Wait "${data.correlationId}" not found`);
|
|
1740
|
+
if (existing.status === "completed") return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already completed`);
|
|
1741
|
+
const event = buildEvent({ ...data });
|
|
1742
|
+
const wait = WaitSchema.parse({
|
|
1743
|
+
...existing,
|
|
1744
|
+
status: "completed",
|
|
1745
|
+
resumeAt: data.eventData?.resumeAt ?? existing.resumeAt,
|
|
1746
|
+
completedAt: now,
|
|
1747
|
+
updatedAt: now
|
|
1748
|
+
});
|
|
1749
|
+
await store.put(waitKey, wait);
|
|
1750
|
+
await putEvent(event);
|
|
1751
|
+
return {
|
|
1752
|
+
ok: true,
|
|
1753
|
+
event,
|
|
1754
|
+
wait,
|
|
1755
|
+
releasedHooks: []
|
|
1756
|
+
};
|
|
1757
|
+
}
|
|
1358
1758
|
default: {
|
|
1359
1759
|
const event = buildEvent({ ...data });
|
|
1360
1760
|
await putEvent(event);
|
|
1761
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) await store.put(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`, {
|
|
1762
|
+
hookId: data.correlationId,
|
|
1763
|
+
eventId: event.eventId,
|
|
1764
|
+
...ctx.params.resumePayloadDigest ? { payloadDigest: ctx.params.resumePayloadDigest } : {}
|
|
1765
|
+
});
|
|
1361
1766
|
return {
|
|
1362
1767
|
ok: true,
|
|
1363
1768
|
event,
|
|
@@ -1366,67 +1771,95 @@ async function applyEvent(store, ctx) {
|
|
|
1366
1771
|
}
|
|
1367
1772
|
}
|
|
1368
1773
|
}
|
|
1369
|
-
//#endregion
|
|
1370
|
-
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1371
|
-
/**
|
|
1372
|
-
* Current schema version for DO storage.
|
|
1373
|
-
* Increment when making breaking changes to the storage layout.
|
|
1374
|
-
* Migrations run lazily on first request after deploy.
|
|
1375
|
-
*
|
|
1376
|
-
* v2: one storage key per event ('event:<eventId>'), step ('step:<stepId>')
|
|
1377
|
-
* and hook ('hook:<hookId>') instead of single aggregate values. This package
|
|
1378
|
-
* is pre-production, so the migration simply drops the legacy aggregates.
|
|
1379
|
-
*/
|
|
1380
|
-
const CURRENT_SCHEMA_VERSION = 3;
|
|
1381
1774
|
/**
|
|
1382
|
-
*
|
|
1383
|
-
* is
|
|
1384
|
-
*
|
|
1775
|
+
* Complete Workflow 5's event-page response after an atomic apply. A celld
|
|
1776
|
+
* run is a single serialized writer, so the committed sequence is dense; a
|
|
1777
|
+
* stale eventCount therefore needs only the slots between the requested
|
|
1778
|
+
* position and the event this call committed.
|
|
1385
1779
|
*/
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
};
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
limit: 100
|
|
1406
|
-
});
|
|
1407
|
-
if (entities.size === 0) return;
|
|
1408
|
-
for (const [key, entity] of entities) {
|
|
1409
|
-
const id = getId(entity);
|
|
1410
|
-
await storage.put(`${indexPrefix}${entity.createdAt.toISOString()}:${id}`, id);
|
|
1411
|
-
startAfter = key;
|
|
1412
|
-
}
|
|
1413
|
-
if (entities.size < 100) return;
|
|
1780
|
+
async function finalizeEventPage(store, outcome, params) {
|
|
1781
|
+
if (!outcome.ok) return outcome;
|
|
1782
|
+
const result = outcome.run && outcome.maxEvents === void 0 ? {
|
|
1783
|
+
...outcome,
|
|
1784
|
+
maxEvents: getMaxEventsPerRun()
|
|
1785
|
+
} : outcome;
|
|
1786
|
+
if (!result.event) return result;
|
|
1787
|
+
if (typeof params?.sinceCursor === "string") {
|
|
1788
|
+
const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1789
|
+
limit: 100,
|
|
1790
|
+
cursor: params.sinceCursor,
|
|
1791
|
+
sortOrder: "asc"
|
|
1792
|
+
}, (event) => event.eventId);
|
|
1793
|
+
return {
|
|
1794
|
+
...result,
|
|
1795
|
+
events: page.data,
|
|
1796
|
+
cursor: page.cursor,
|
|
1797
|
+
hasMore: page.hasMore
|
|
1798
|
+
};
|
|
1414
1799
|
}
|
|
1800
|
+
if (params?.eventCount === void 0) return result;
|
|
1801
|
+
const committedSlot = eventIdToSlot(result.event.eventId);
|
|
1802
|
+
if (committedSlot === null || committedSlot <= params.eventCount + 1) return result;
|
|
1803
|
+
const skippedCount = committedSlot - params.eventCount - 1;
|
|
1804
|
+
const events = (await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1805
|
+
limit: skippedCount,
|
|
1806
|
+
cursor: params.eventCount > 0 ? slotToEventId(params.eventCount) : void 0,
|
|
1807
|
+
sortOrder: "asc"
|
|
1808
|
+
}, (event) => event.eventId)).data.filter((event) => event.eventId < result.event.eventId);
|
|
1809
|
+
return {
|
|
1810
|
+
...result,
|
|
1811
|
+
events,
|
|
1812
|
+
cursor: null,
|
|
1813
|
+
hasMore: events.length < skippedCount
|
|
1814
|
+
};
|
|
1415
1815
|
}
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1816
|
+
//#endregion
|
|
1817
|
+
//#region src/retention.ts
|
|
1818
|
+
const CLEANUP_RECORD_KEY = "retention:cleanup";
|
|
1819
|
+
const TOMBSTONE_KEY = "retention:tombstone";
|
|
1820
|
+
const INDEX_MARKER_PREFIX = "retention:index:";
|
|
1821
|
+
const HOOK_MARKER_PREFIX = "retention:hook:";
|
|
1822
|
+
function sortableTimestamp(date) {
|
|
1823
|
+
return date.getTime().toString().padStart(13, "0");
|
|
1824
|
+
}
|
|
1825
|
+
function workflowRunIndexKey(run) {
|
|
1826
|
+
return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
1827
|
+
}
|
|
1828
|
+
function globalRunIndexKey(run) {
|
|
1829
|
+
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
1830
|
+
}
|
|
1831
|
+
function correlationIndexKey(correlationId, createdAt, eventId, runId) {
|
|
1832
|
+
return `correlation:${encodeURIComponent(correlationId)}:${sortableTimestamp(createdAt)}:${eventId}:${runId}`;
|
|
1429
1833
|
}
|
|
1834
|
+
function indexMarkerKey(indexKey) {
|
|
1835
|
+
return `${INDEX_MARKER_PREFIX}${encodeURIComponent(indexKey)}`;
|
|
1836
|
+
}
|
|
1837
|
+
function hookMarkerKey(reference) {
|
|
1838
|
+
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
1839
|
+
}
|
|
1840
|
+
function cleanupTombstone(record, now) {
|
|
1841
|
+
return {
|
|
1842
|
+
version: 1,
|
|
1843
|
+
runId: record.runId,
|
|
1844
|
+
terminalStatus: record.terminalStatus,
|
|
1845
|
+
completedAt: record.completedAt,
|
|
1846
|
+
expiredAt: record.dueAt,
|
|
1847
|
+
tombstonedAt: now
|
|
1848
|
+
};
|
|
1849
|
+
}
|
|
1850
|
+
function expiredRead(tombstone) {
|
|
1851
|
+
return {
|
|
1852
|
+
ok: false,
|
|
1853
|
+
code: "RUN_EXPIRED",
|
|
1854
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
|
|
1855
|
+
tombstone
|
|
1856
|
+
};
|
|
1857
|
+
}
|
|
1858
|
+
//#endregion
|
|
1859
|
+
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1860
|
+
const PAYLOAD_DELETE_BATCH = 256;
|
|
1861
|
+
const STREAM_DELETE_CONCURRENCY = 16;
|
|
1862
|
+
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
1430
1863
|
/**
|
|
1431
1864
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
1432
1865
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -1451,105 +1884,335 @@ function storeFrom(storage) {
|
|
|
1451
1884
|
* queue-message dedup claims (see claimInflight/releaseInflight).
|
|
1452
1885
|
*/
|
|
1453
1886
|
var WorkflowRunDO = class extends DurableObject {
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
* Memoized so it only runs once per DO instantiation.
|
|
1458
|
-
*/
|
|
1459
|
-
ensureSchema() {
|
|
1460
|
-
if (!this.schemaReady) this.schemaReady = ensureSchemaUpToDate(this.ctx.storage);
|
|
1461
|
-
return this.schemaReady;
|
|
1887
|
+
now() {
|
|
1888
|
+
const clock = this.env?.clock;
|
|
1889
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
1462
1890
|
}
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1891
|
+
async expiredTombstone(storage, now = this.now()) {
|
|
1892
|
+
const tombstone = await storage.get(TOMBSTONE_KEY);
|
|
1893
|
+
if (tombstone) return tombstone;
|
|
1894
|
+
const cleanup = await storage.get(CLEANUP_RECORD_KEY);
|
|
1895
|
+
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
1896
|
+
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
1897
|
+
}
|
|
1898
|
+
async read(reader) {
|
|
1899
|
+
const tombstone = await this.expiredTombstone(this.ctx.storage);
|
|
1900
|
+
if (tombstone) return expiredRead(tombstone);
|
|
1901
|
+
return {
|
|
1902
|
+
ok: true,
|
|
1903
|
+
value: await reader()
|
|
1904
|
+
};
|
|
1905
|
+
}
|
|
1906
|
+
/** Apply an event and capture all retention metadata in the same transaction. */
|
|
1469
1907
|
async applyEvent(request) {
|
|
1470
|
-
await this.ensureSchema();
|
|
1471
1908
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1909
|
+
const now = new Date(this.now());
|
|
1910
|
+
const tombstone = await this.expiredTombstone(txn, now.getTime());
|
|
1911
|
+
if (tombstone) return {
|
|
1912
|
+
ok: false,
|
|
1913
|
+
code: "RUN_EXPIRED",
|
|
1914
|
+
message: `Workflow run "${request.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
1915
|
+
};
|
|
1472
1916
|
let eventSequence = await txn.get("event_sequence") ?? 0;
|
|
1473
1917
|
const outcome = await applyEvent(storeFrom(txn), {
|
|
1474
1918
|
...request,
|
|
1475
|
-
nextEventId: () =>
|
|
1476
|
-
now
|
|
1919
|
+
nextEventId: () => slotToEventId(++eventSequence),
|
|
1920
|
+
now
|
|
1477
1921
|
});
|
|
1478
|
-
if (outcome.ok)
|
|
1479
|
-
|
|
1922
|
+
if (!outcome.ok) return outcome;
|
|
1923
|
+
await txn.put("event_sequence", eventSequence);
|
|
1924
|
+
if (outcome.event?.correlationId) {
|
|
1925
|
+
const key = correlationIndexKey(outcome.event.correlationId, outcome.event.createdAt, outcome.event.eventId, request.runId);
|
|
1926
|
+
await txn.put(indexMarkerKey(key), key);
|
|
1927
|
+
}
|
|
1928
|
+
if (outcome.hookToIndex) {
|
|
1929
|
+
const reference = {
|
|
1930
|
+
hookId: outcome.hookToIndex.hookId,
|
|
1931
|
+
token: outcome.hookToIndex.token
|
|
1932
|
+
};
|
|
1933
|
+
await txn.put(hookMarkerKey(reference), reference);
|
|
1934
|
+
}
|
|
1935
|
+
for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
|
|
1936
|
+
const retentionMs = request.cleanup?.retentionMs ?? 0;
|
|
1937
|
+
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !await txn.get("retention:cleanup")) {
|
|
1938
|
+
const terminalRun = outcome.run;
|
|
1939
|
+
const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
|
|
1940
|
+
const run = WorkflowRunSchema.parse({
|
|
1941
|
+
...terminalRun,
|
|
1942
|
+
expiredAt: dueAt
|
|
1943
|
+
});
|
|
1944
|
+
const cleanup = {
|
|
1945
|
+
version: 1,
|
|
1946
|
+
runId: run.runId,
|
|
1947
|
+
workflowName: run.workflowName,
|
|
1948
|
+
createdAt: run.createdAt,
|
|
1949
|
+
completedAt: run.completedAt,
|
|
1950
|
+
terminalStatus: run.status,
|
|
1951
|
+
dueAt,
|
|
1952
|
+
queueShards: request.cleanup?.queueShards ?? 1,
|
|
1953
|
+
phase: "retained",
|
|
1954
|
+
attempts: 0,
|
|
1955
|
+
deletedPayloadKeys: 0,
|
|
1956
|
+
deletedStreams: 0,
|
|
1957
|
+
deletedQueueMessages: 0
|
|
1958
|
+
};
|
|
1959
|
+
await txn.put("run", run);
|
|
1960
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
1961
|
+
await txn.setAlarm(dueAt);
|
|
1962
|
+
outcome.run = run;
|
|
1963
|
+
}
|
|
1964
|
+
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
1480
1965
|
});
|
|
1481
1966
|
}
|
|
1482
|
-
/**
|
|
1483
|
-
* Get the workflow run
|
|
1484
|
-
*/
|
|
1485
1967
|
async getRun() {
|
|
1486
|
-
await this.
|
|
1487
|
-
return await this.ctx.storage.get("run") ?? null;
|
|
1968
|
+
return this.read(async () => await this.ctx.storage.get("run") ?? null);
|
|
1488
1969
|
}
|
|
1489
|
-
/**
|
|
1490
|
-
* Get a specific step
|
|
1491
|
-
*/
|
|
1492
1970
|
async getStep(stepId) {
|
|
1493
|
-
await this.
|
|
1494
|
-
return await this.ctx.storage.get(`step:${stepId}`) ?? null;
|
|
1971
|
+
return this.read(async () => await this.ctx.storage.get(`step:${stepId}`) ?? null);
|
|
1495
1972
|
}
|
|
1496
|
-
/**
|
|
1497
|
-
* List steps with real limit/cursor pagination (cursor = last stepId).
|
|
1498
|
-
*/
|
|
1499
1973
|
async listSteps(params) {
|
|
1500
|
-
|
|
1501
|
-
return listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
1974
|
+
return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
1502
1975
|
limit: params?.limit ?? 20,
|
|
1503
1976
|
cursor: params?.cursor,
|
|
1504
1977
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1505
|
-
});
|
|
1978
|
+
}));
|
|
1506
1979
|
}
|
|
1507
|
-
/**
|
|
1508
|
-
* Get a specific event
|
|
1509
|
-
*/
|
|
1510
1980
|
async getEvent(eventId) {
|
|
1511
|
-
await this.
|
|
1512
|
-
return await this.ctx.storage.get(`event:${eventId}`) ?? null;
|
|
1981
|
+
return this.read(async () => await this.ctx.storage.get(`event:${eventId}`) ?? null);
|
|
1513
1982
|
}
|
|
1514
|
-
/**
|
|
1515
|
-
* List events with real limit/cursor pagination. Events are keyed by their
|
|
1516
|
-
* monotonic ULID eventId, so storage order == event order.
|
|
1517
|
-
*/
|
|
1518
1983
|
async listEvents(params) {
|
|
1519
|
-
|
|
1520
|
-
return listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
|
|
1984
|
+
return this.read(() => listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
|
|
1521
1985
|
limit: params?.limit ?? 100,
|
|
1522
1986
|
cursor: params?.cursor,
|
|
1523
1987
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1524
|
-
}, (event) => event.eventId);
|
|
1988
|
+
}, (event) => event.eventId));
|
|
1525
1989
|
}
|
|
1526
|
-
/**
|
|
1527
|
-
* List hooks with real limit/cursor pagination (cursor = last hookId).
|
|
1528
|
-
*/
|
|
1529
1990
|
async listHooks(params) {
|
|
1530
|
-
|
|
1531
|
-
return listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
1991
|
+
return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
1532
1992
|
limit: params?.limit ?? 100,
|
|
1533
1993
|
cursor: params?.cursor,
|
|
1534
1994
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1995
|
+
}));
|
|
1996
|
+
}
|
|
1997
|
+
async getCleanupStatus() {
|
|
1998
|
+
return await this.ctx.storage.get("retention:cleanup") ?? null;
|
|
1999
|
+
}
|
|
2000
|
+
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
2001
|
+
async scheduleCleanup(request) {
|
|
2002
|
+
if (request.retentionMs <= 0) return this.getCleanupStatus();
|
|
2003
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2004
|
+
const existing = await txn.get(CLEANUP_RECORD_KEY);
|
|
2005
|
+
if (existing) return existing;
|
|
2006
|
+
if (await txn.get("retention:tombstone")) return null;
|
|
2007
|
+
const current = await txn.get("run");
|
|
2008
|
+
if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
|
|
2009
|
+
const terminalRun = current;
|
|
2010
|
+
const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
|
|
2011
|
+
const run = WorkflowRunSchema.parse({
|
|
2012
|
+
...terminalRun,
|
|
2013
|
+
expiredAt: dueAt
|
|
2014
|
+
});
|
|
2015
|
+
const cleanup = {
|
|
2016
|
+
version: 1,
|
|
2017
|
+
runId: run.runId,
|
|
2018
|
+
workflowName: run.workflowName,
|
|
2019
|
+
createdAt: run.createdAt,
|
|
2020
|
+
completedAt: run.completedAt,
|
|
2021
|
+
terminalStatus: run.status,
|
|
2022
|
+
dueAt,
|
|
2023
|
+
queueShards: request.queueShards,
|
|
2024
|
+
phase: "retained",
|
|
2025
|
+
attempts: 0,
|
|
2026
|
+
deletedPayloadKeys: 0,
|
|
2027
|
+
deletedStreams: 0,
|
|
2028
|
+
deletedQueueMessages: 0
|
|
2029
|
+
};
|
|
2030
|
+
await txn.put("run", run);
|
|
2031
|
+
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
2032
|
+
await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
2033
|
+
return cleanup;
|
|
2034
|
+
});
|
|
2035
|
+
}
|
|
2036
|
+
async cleanupNow(request) {
|
|
2037
|
+
if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
|
|
2038
|
+
...request,
|
|
2039
|
+
retentionMs: Math.max(1, request.retentionMs)
|
|
2040
|
+
});
|
|
2041
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2042
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2043
|
+
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
2044
|
+
const dueAt = new Date(this.now());
|
|
2045
|
+
const run = await txn.get("run");
|
|
2046
|
+
if (run) await txn.put("run", WorkflowRunSchema.parse({
|
|
2047
|
+
...run,
|
|
2048
|
+
expiredAt: dueAt
|
|
2049
|
+
}));
|
|
2050
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2051
|
+
...cleanup,
|
|
2052
|
+
dueAt
|
|
2053
|
+
});
|
|
2054
|
+
await txn.setAlarm(this.now() + 1);
|
|
2055
|
+
});
|
|
2056
|
+
try {
|
|
2057
|
+
await this.executeCleanup();
|
|
2058
|
+
} catch (error) {
|
|
2059
|
+
await this.recordCleanupFailure(error);
|
|
2060
|
+
}
|
|
2061
|
+
return this.getCleanupStatus();
|
|
2062
|
+
}
|
|
2063
|
+
async rearmCleanup() {
|
|
2064
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
2065
|
+
if (!cleanup) return null;
|
|
2066
|
+
if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
2067
|
+
else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
|
|
2068
|
+
return cleanup;
|
|
2069
|
+
}
|
|
2070
|
+
async alarm() {
|
|
2071
|
+
try {
|
|
2072
|
+
await this.executeCleanup();
|
|
2073
|
+
} catch (error) {
|
|
2074
|
+
await this.recordCleanupFailure(error);
|
|
2075
|
+
}
|
|
2076
|
+
}
|
|
2077
|
+
namespace(name) {
|
|
2078
|
+
const namespace = this.env?.[name];
|
|
2079
|
+
if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
|
|
2080
|
+
return namespace;
|
|
2081
|
+
}
|
|
2082
|
+
async setCleanupPhase(expected, next, updates = {}) {
|
|
2083
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2084
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2085
|
+
if (!cleanup || cleanup.phase !== expected) return;
|
|
2086
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2087
|
+
...cleanup,
|
|
2088
|
+
...updates,
|
|
2089
|
+
phase: next,
|
|
2090
|
+
attempts: 0,
|
|
2091
|
+
lastError: void 0
|
|
2092
|
+
});
|
|
2093
|
+
});
|
|
2094
|
+
}
|
|
2095
|
+
async executeCleanup() {
|
|
2096
|
+
for (;;) {
|
|
2097
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
2098
|
+
if (!cleanup) {
|
|
2099
|
+
await this.ctx.storage.deleteAlarm();
|
|
2100
|
+
return;
|
|
2101
|
+
}
|
|
2102
|
+
if (cleanup.phase === "tombstoned") {
|
|
2103
|
+
await this.ctx.storage.deleteAlarm();
|
|
2104
|
+
return;
|
|
2105
|
+
}
|
|
2106
|
+
if (cleanup.phase === "retained") {
|
|
2107
|
+
if (cleanup.dueAt.getTime() > this.now()) {
|
|
2108
|
+
await this.ctx.storage.setAlarm(cleanup.dueAt);
|
|
2109
|
+
return;
|
|
2110
|
+
}
|
|
2111
|
+
await this.setCleanupPhase("retained", "index");
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
2114
|
+
if (cleanup.phase === "index") {
|
|
2115
|
+
const [indexMarkers, hookMarkers] = await Promise.all([this.ctx.storage.list({ prefix: INDEX_MARKER_PREFIX }), this.ctx.storage.list({ prefix: HOOK_MARKER_PREFIX })]);
|
|
2116
|
+
const indexNamespace = this.namespace("WORKFLOW_INDEX");
|
|
2117
|
+
await indexNamespace.get(indexNamespace.idFromName("index")).expireRun({
|
|
2118
|
+
runId: cleanup.runId,
|
|
2119
|
+
keys: [
|
|
2120
|
+
workflowRunIndexKey(cleanup),
|
|
2121
|
+
globalRunIndexKey(cleanup),
|
|
2122
|
+
...indexMarkers.values()
|
|
2123
|
+
],
|
|
2124
|
+
hooks: Array.from(hookMarkers.values()),
|
|
2125
|
+
expiredAt: cleanup.dueAt.getTime()
|
|
2126
|
+
});
|
|
2127
|
+
await this.setCleanupPhase("index", "streams");
|
|
2128
|
+
continue;
|
|
2129
|
+
}
|
|
2130
|
+
if (cleanup.phase === "streams") {
|
|
2131
|
+
const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
|
|
2132
|
+
const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
|
|
2133
|
+
const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime());
|
|
2134
|
+
let deletedStreams = streams.length === 0 ? cleanup.deletedStreams : 0;
|
|
2135
|
+
for (let offset = 0; offset < streams.length; offset += STREAM_DELETE_CONCURRENCY) {
|
|
2136
|
+
const batch = streams.slice(offset, offset + STREAM_DELETE_CONCURRENCY);
|
|
2137
|
+
const results = await Promise.all(batch.map((name) => {
|
|
2138
|
+
return streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime());
|
|
2139
|
+
}));
|
|
2140
|
+
deletedStreams += results.filter((result) => result.deleted).length;
|
|
2141
|
+
}
|
|
2142
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2143
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
2144
|
+
if (current?.phase === "streams") await txn.put(CLEANUP_RECORD_KEY, {
|
|
2145
|
+
...current,
|
|
2146
|
+
deletedStreams
|
|
2147
|
+
});
|
|
2148
|
+
});
|
|
2149
|
+
await registry.finalizeRegistry(cleanup.runId);
|
|
2150
|
+
await this.setCleanupPhase("streams", "queues");
|
|
2151
|
+
continue;
|
|
2152
|
+
}
|
|
2153
|
+
if (cleanup.phase === "queues") {
|
|
2154
|
+
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
2155
|
+
let deletedQueueMessages = 0;
|
|
2156
|
+
for (let shard = 0; shard < cleanup.queueShards; shard++) {
|
|
2157
|
+
const result = await queues.get(queues.idFromName(`q:${shard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime());
|
|
2158
|
+
deletedQueueMessages += result.deleted;
|
|
2159
|
+
}
|
|
2160
|
+
await this.setCleanupPhase("queues", "payload", { deletedQueueMessages });
|
|
2161
|
+
continue;
|
|
2162
|
+
}
|
|
2163
|
+
if (cleanup.phase === "payload") {
|
|
2164
|
+
if (!await this.deletePayloadBatch()) return;
|
|
2165
|
+
}
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
async deletePayloadBatch() {
|
|
2169
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2170
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2171
|
+
if (!cleanup || cleanup.phase !== "payload") return true;
|
|
2172
|
+
const entries = await txn.list({ limit: 259 });
|
|
2173
|
+
const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
|
|
2174
|
+
const now = new Date(this.now());
|
|
2175
|
+
await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
|
|
2176
|
+
for (const key of keys) await txn.delete(key);
|
|
2177
|
+
if (keys.length > 0) {
|
|
2178
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2179
|
+
...cleanup,
|
|
2180
|
+
deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
|
|
2181
|
+
lastError: void 0
|
|
2182
|
+
});
|
|
2183
|
+
await txn.setAlarm(this.now() + 1);
|
|
2184
|
+
return false;
|
|
2185
|
+
}
|
|
2186
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2187
|
+
...cleanup,
|
|
2188
|
+
phase: "tombstoned",
|
|
2189
|
+
attempts: 0,
|
|
2190
|
+
lastError: void 0,
|
|
2191
|
+
tombstonedAt: now
|
|
2192
|
+
});
|
|
2193
|
+
await txn.deleteAlarm();
|
|
2194
|
+
return true;
|
|
2195
|
+
});
|
|
2196
|
+
}
|
|
2197
|
+
async recordCleanupFailure(error) {
|
|
2198
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
2199
|
+
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2200
|
+
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
2201
|
+
const attempts = cleanup.attempts + 1;
|
|
2202
|
+
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
2203
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2204
|
+
...cleanup,
|
|
2205
|
+
attempts,
|
|
2206
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
2207
|
+
lastAttemptAt: new Date(this.now())
|
|
2208
|
+
});
|
|
2209
|
+
await txn.setAlarm(this.now() + delay);
|
|
1535
2210
|
});
|
|
1536
2211
|
}
|
|
1537
|
-
/**
|
|
1538
|
-
* Claim a queue-message dedup slot. Used on DO instances dedicated to a
|
|
1539
|
-
* single idempotency key (`claim:<queueName>:<idempotencyKey>`).
|
|
1540
|
-
*
|
|
1541
|
-
* - Unclaimed -> claim and proceed.
|
|
1542
|
-
* - Claimed by the SAME messageId -> redelivery of the same message
|
|
1543
|
-
* (previous attempt failed transiently) -> proceed.
|
|
1544
|
-
* - Claimed by a DIFFERENT messageId -> concurrent duplicate -> reject,
|
|
1545
|
-
* unless the claim is stale (holder crashed / went to the DLQ without
|
|
1546
|
-
* releasing), in which case the claim is stolen.
|
|
1547
|
-
*/
|
|
1548
2212
|
async claimInflight(params) {
|
|
1549
|
-
await this.ensureSchema();
|
|
1550
2213
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1551
2214
|
const existing = await txn.get("claim");
|
|
1552
|
-
const now =
|
|
2215
|
+
const now = this.now();
|
|
1553
2216
|
if (existing && existing.messageId !== params.messageId && now - existing.claimedAt < params.staleMs) return { claimed: false };
|
|
1554
2217
|
await txn.put("claim", {
|
|
1555
2218
|
messageId: params.messageId,
|
|
@@ -1558,11 +2221,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1558
2221
|
return { claimed: true };
|
|
1559
2222
|
});
|
|
1560
2223
|
}
|
|
1561
|
-
/**
|
|
1562
|
-
* Release a queue-message dedup claim after the message was fully handled.
|
|
1563
|
-
*/
|
|
1564
2224
|
async releaseInflight() {
|
|
1565
|
-
await this.ensureSchema();
|
|
1566
2225
|
await this.ctx.storage.delete("claim");
|
|
1567
2226
|
}
|
|
1568
2227
|
};
|