@ewhauser/world-celld 0.1.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/testing.js CHANGED
@@ -1,5 +1,5 @@
1
- import { i as parse, n as rpcParse, r as rpcStringify, t as compact } from "./util-BvWlkqgV.js";
2
- import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema, isLegacySpecVersion, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld } from "@workflow/world";
1
+ import { a as cleanupTombstone, c as globalRunIndexKey, d as workflowRunIndexKey, f as compact, h as parse, i as TOMBSTONE_KEY, l as hookMarkerKey, m as rpcStringify, n as HOOK_MARKER_PREFIX, o as correlationIndexKey, p as rpcParse, r as INDEX_MARKER_PREFIX, s as expiredRead, t as CLEANUP_RECORD_KEY, u as indexMarkerKey } from "./retention-bLY1vcOu.js";
2
+ import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
3
3
  import http from "node:http";
4
4
  //#region src/testing/fake-cell.ts
5
5
  function applyListOptions(keys, options) {
@@ -309,7 +309,11 @@ const BINDINGS = {
309
309
  "getEvent",
310
310
  "listEvents",
311
311
  "listSteps",
312
- "listHooks"
312
+ "listHooks",
313
+ "getCleanupStatus",
314
+ "scheduleCleanup",
315
+ "cleanupNow",
316
+ "rearmCleanup"
313
317
  ])
314
318
  },
315
319
  streams: {
@@ -320,7 +324,10 @@ const BINDINGS = {
320
324
  "getChunks",
321
325
  "getInfo",
322
326
  "registerStream",
323
- "listStreams"
327
+ "listStreams",
328
+ "expireRegistry",
329
+ "finalizeRegistry",
330
+ "expireStream"
324
331
  ])
325
332
  },
326
333
  index: {
@@ -329,6 +336,8 @@ const BINDINGS = {
329
336
  "get",
330
337
  "put",
331
338
  "delete",
339
+ "putOwned",
340
+ "expireRun",
332
341
  "list",
333
342
  "reserveHookToken",
334
343
  "finalizeHookIndexes",
@@ -344,7 +353,8 @@ const BINDINGS = {
344
353
  "listDeadLetters",
345
354
  "redriveDeadLetter",
346
355
  "purgeDeadLetters",
347
- "rearmAlarm"
356
+ "rearmAlarm",
357
+ "expireRun"
348
358
  ])
349
359
  }
350
360
  };
@@ -478,6 +488,14 @@ var IndexDO = class extends DurableObject {
478
488
  async put(key, value) {
479
489
  await this.ctx.storage.put(key, value);
480
490
  }
491
+ /** Publish a derived index only while its owning run is not expired. */
492
+ async putOwned(runId, key, value) {
493
+ return await this.ctx.storage.transaction(async (txn) => {
494
+ if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
495
+ await txn.put(key, value);
496
+ return { stored: true };
497
+ });
498
+ }
481
499
  async delete(key) {
482
500
  await this.ctx.storage.delete(key);
483
501
  }
@@ -523,6 +541,11 @@ var IndexDO = class extends DurableObject {
523
541
  }
524
542
  async finalizeHookIndexes(token, hookId, serializedHook, owner) {
525
543
  await this.ctx.storage.transaction(async (txn) => {
544
+ if (await txn.get(`expired:${owner.runId}`) !== void 0) {
545
+ const claim = await txn.get(`hookclaim:${token}`);
546
+ if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(`hookclaim:${token}`);
547
+ return;
548
+ }
526
549
  const indexedHook = await txn.get(`hook:${token}`);
527
550
  if (indexedHook !== void 0 && !sameOwner(ownerFromHook(indexedHook), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
528
551
  const claim = await txn.get(`hookclaim:${token}`);
@@ -554,6 +577,42 @@ var IndexDO = class extends DurableObject {
554
577
  if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
555
578
  });
556
579
  }
580
+ /**
581
+ * Fence a run and remove all of its known derived indexes atomically.
582
+ * Once the fence exists, putOwned() and hook finalization cannot resurrect
583
+ * entries from a delayed request.
584
+ */
585
+ async expireRun(request) {
586
+ return await this.ctx.storage.transaction(async (txn) => {
587
+ await txn.put(`expired:${request.runId}`, request.expiredAt);
588
+ let deleted = 0;
589
+ for (const key of new Set(request.keys)) if (await txn.delete(key)) deleted++;
590
+ for (const hook of request.hooks) {
591
+ const owner = {
592
+ runId: request.runId,
593
+ hookId: hook.hookId
594
+ };
595
+ const tokenKey = `hook:${hook.token}`;
596
+ const idKey = `hookid:${hook.hookId}`;
597
+ const claimKey = `hookclaim:${hook.token}`;
598
+ const [byToken, byId, claim] = await Promise.all([
599
+ txn.get(tokenKey),
600
+ txn.get(idKey),
601
+ txn.get(claimKey)
602
+ ]);
603
+ if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
604
+ if (await txn.delete(tokenKey)) deleted++;
605
+ }
606
+ if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
607
+ if (await txn.delete(idKey)) deleted++;
608
+ }
609
+ if (claim !== void 0 && sameOwner(claim.owner, owner)) {
610
+ if (await txn.delete(claimKey)) deleted++;
611
+ }
612
+ }
613
+ return { deleted };
614
+ });
615
+ }
557
616
  };
558
617
  //#endregion
559
618
  //#region src/worker/durable-objects/QueueDO.ts
@@ -576,6 +635,12 @@ function pad(ms) {
576
635
  function dueKey(atMs, messageId) {
577
636
  return `due:${pad(atMs)}:${messageId}`;
578
637
  }
638
+ function runReferenceKey(runId, messageId) {
639
+ return `run:${runId}:${messageId}`;
640
+ }
641
+ function expiredRunKey(runId) {
642
+ return `expired-run:${runId}`;
643
+ }
579
644
  function backoffSeconds(attempt) {
580
645
  return Math.min(60, 2 ** attempt);
581
646
  }
@@ -597,6 +662,11 @@ var QueueDO = class extends DurableObject {
597
662
  const storage = this.ctx.storage;
598
663
  const now = this.#now();
599
664
  return await storage.transaction(async (txn) => {
665
+ if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
666
+ ok: false,
667
+ code: "RUN_EXPIRED",
668
+ message: `Workflow run "${request.runId}" has expired`
669
+ };
600
670
  const pinned = await txn.get("cfg");
601
671
  if (!pinned) await txn.put("cfg", {
602
672
  queueShards: request.config.queueShards,
@@ -627,14 +697,21 @@ var QueueDO = class extends DurableObject {
627
697
  queueName: request.queueName,
628
698
  pathname: request.pathname,
629
699
  body: request.body,
700
+ runId: request.runId,
630
701
  targetBaseUrl: request.config.targetBaseUrl,
631
702
  idempotencyKey: request.idempotencyKey,
632
703
  attempt: 0,
633
704
  enqueuedAt: now
634
705
  };
635
706
  const dueAt = now + Math.max(0, request.delaySeconds ?? 0) * 1e3;
707
+ const scheduleKey = dueKey(dueAt, row.messageId);
636
708
  await txn.put(`msg:${row.messageId}`, row);
637
- await txn.put(dueKey(dueAt, row.messageId), row.messageId);
709
+ await txn.put(scheduleKey, row.messageId);
710
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
711
+ messageId: row.messageId,
712
+ dueKey: scheduleKey,
713
+ idempotencyKey: row.idempotencyKey
714
+ });
638
715
  await this.#armAlarmAtMost(txn, dueAt, now);
639
716
  return {
640
717
  ok: true,
@@ -672,7 +749,16 @@ var QueueDO = class extends DurableObject {
672
749
  for (const [key, deadline] of inflight) if (deadline <= now) {
673
750
  const messageId = key.slice(9);
674
751
  await txn.delete(key);
675
- if (await txn.get(`msg:${messageId}`)) await txn.put(dueKey(now, messageId), messageId);
752
+ const row = await txn.get(`msg:${messageId}`);
753
+ if (row) {
754
+ const scheduleKey = dueKey(now, messageId);
755
+ await txn.put(scheduleKey, messageId);
756
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
757
+ messageId: row.messageId,
758
+ dueKey: scheduleKey,
759
+ idempotencyKey: row.idempotencyKey
760
+ });
761
+ }
676
762
  } else inflightCount++;
677
763
  const rows = [];
678
764
  if (inflightCount < maxInflight) {
@@ -727,8 +813,25 @@ var QueueDO = class extends DurableObject {
727
813
  timeoutSeconds = void 0;
728
814
  }
729
815
  if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
816
+ if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
817
+ await txn.delete(`msg:${row.messageId}`);
818
+ await txn.delete(`inflight:${row.messageId}`);
819
+ await txn.delete(runReferenceKey(row.runId, row.messageId));
820
+ if (row.idempotencyKey) {
821
+ const dedupKey = `key:${row.idempotencyKey}`;
822
+ if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
823
+ }
824
+ await this.#scheduleNextAlarm(txn, now);
825
+ return;
826
+ }
730
827
  await txn.delete(`inflight:${row.messageId}`);
731
- await txn.put(dueKey(now + timeoutSeconds * 1e3, row.messageId), row.messageId);
828
+ const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
829
+ await txn.put(scheduleKey, row.messageId);
830
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
831
+ messageId: row.messageId,
832
+ dueKey: scheduleKey,
833
+ idempotencyKey: row.idempotencyKey
834
+ });
732
835
  await this.#scheduleNextAlarm(txn, now);
733
836
  });
734
837
  else await this.#retry(row, "HTTP 503");
@@ -745,6 +848,7 @@ var QueueDO = class extends DurableObject {
745
848
  if (row.idempotencyKey) {
746
849
  if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
747
850
  }
851
+ if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
748
852
  await this.#scheduleNextAlarm(txn, this.#now());
749
853
  });
750
854
  }
@@ -754,6 +858,17 @@ var QueueDO = class extends DurableObject {
754
858
  const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
755
859
  const attempt = row.attempt + 1;
756
860
  await storage.transaction(async (txn) => {
861
+ if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
862
+ await txn.delete(`msg:${row.messageId}`);
863
+ await txn.delete(`inflight:${row.messageId}`);
864
+ await txn.delete(runReferenceKey(row.runId, row.messageId));
865
+ if (row.idempotencyKey) {
866
+ const dedupKey = `key:${row.idempotencyKey}`;
867
+ if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
868
+ }
869
+ await this.#scheduleNextAlarm(txn, now);
870
+ return;
871
+ }
757
872
  await txn.delete(`inflight:${row.messageId}`);
758
873
  if (attempt >= maxAttempts) {
759
874
  const dead = {
@@ -762,11 +877,17 @@ var QueueDO = class extends DurableObject {
762
877
  lastError: reason,
763
878
  failedAt: now
764
879
  };
765
- await txn.put(`dlq:${pad(now)}:${row.messageId}`, dead);
880
+ const deadKey = `dlq:${pad(now)}:${row.messageId}`;
881
+ await txn.put(deadKey, dead);
766
882
  await txn.delete(`msg:${row.messageId}`);
767
883
  if (row.idempotencyKey) {
768
884
  if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
769
885
  }
886
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
887
+ messageId: row.messageId,
888
+ dlqKey: deadKey,
889
+ idempotencyKey: row.idempotencyKey
890
+ });
770
891
  await this.#scheduleNextAlarm(txn, now);
771
892
  return;
772
893
  }
@@ -775,8 +896,14 @@ var QueueDO = class extends DurableObject {
775
896
  attempt,
776
897
  lastError: reason
777
898
  };
899
+ const scheduleKey = dueKey(now + backoffSeconds(attempt) * 1e3, row.messageId);
778
900
  await txn.put(`msg:${row.messageId}`, updated);
779
- await txn.put(dueKey(now + backoffSeconds(attempt) * 1e3, row.messageId), row.messageId);
901
+ await txn.put(scheduleKey, row.messageId);
902
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
903
+ messageId: row.messageId,
904
+ dueKey: scheduleKey,
905
+ idempotencyKey: row.idempotencyKey
906
+ });
780
907
  await this.#scheduleNextAlarm(txn, now);
781
908
  });
782
909
  }
@@ -860,8 +987,14 @@ var QueueDO = class extends DurableObject {
860
987
  await txn.put(dedupKey, row.messageId);
861
988
  }
862
989
  await txn.put(`msg:${row.messageId}`, row);
863
- await txn.put(dueKey(now, row.messageId), row.messageId);
990
+ const scheduleKey = dueKey(now, row.messageId);
991
+ await txn.put(scheduleKey, row.messageId);
864
992
  await txn.delete(key);
993
+ if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
994
+ messageId: row.messageId,
995
+ dueKey: scheduleKey,
996
+ idempotencyKey: row.idempotencyKey
997
+ });
865
998
  await this.#armAlarmAtMost(txn, now, now);
866
999
  return { ok: true };
867
1000
  }
@@ -871,7 +1004,11 @@ var QueueDO = class extends DurableObject {
871
1004
  async purgeDeadLetters() {
872
1005
  return await this.ctx.storage.transaction(async (txn) => {
873
1006
  const dlq = await txn.list({ prefix: "dlq:" });
874
- for (const key of dlq.keys()) await txn.delete(key);
1007
+ for (const [key, value] of dlq) {
1008
+ await txn.delete(key);
1009
+ const dead = value;
1010
+ if (dead.runId) await txn.delete(runReferenceKey(dead.runId, dead.messageId));
1011
+ }
875
1012
  return { purged: dlq.size };
876
1013
  });
877
1014
  }
@@ -883,6 +1020,32 @@ var QueueDO = class extends DurableObject {
883
1020
  await this.#rearm();
884
1021
  return { alarmAt: await this.ctx.storage.getAlarm() };
885
1022
  }
1023
+ /** Fence a run and purge every live, inflight, and dead-letter message it owns. */
1024
+ async expireRun(runId, expiredAt) {
1025
+ return { deleted: await this.ctx.storage.transaction(async (txn) => {
1026
+ const existing = await txn.get(expiredRunKey(runId));
1027
+ const references = await txn.list({ prefix: `run:${runId}:` });
1028
+ let count = existing?.deleted ?? 0;
1029
+ for (const [key, reference] of references) {
1030
+ await txn.delete(`msg:${reference.messageId}`);
1031
+ await txn.delete(`inflight:${reference.messageId}`);
1032
+ if (reference.dueKey) await txn.delete(reference.dueKey);
1033
+ if (reference.dlqKey) await txn.delete(reference.dlqKey);
1034
+ if (reference.idempotencyKey) {
1035
+ const dedupKey = `key:${reference.idempotencyKey}`;
1036
+ if (await txn.get(dedupKey) === reference.messageId) await txn.delete(dedupKey);
1037
+ }
1038
+ await txn.delete(key);
1039
+ count++;
1040
+ }
1041
+ await txn.put(expiredRunKey(runId), {
1042
+ expiredAt,
1043
+ deleted: count
1044
+ });
1045
+ await this.#scheduleNextAlarm(txn, this.#now());
1046
+ return count;
1047
+ }) };
1048
+ }
886
1049
  };
887
1050
  //#endregion
888
1051
  //#region src/worker/durable-objects/StreamDO.ts
@@ -890,6 +1053,8 @@ const META_KEY = "meta";
890
1053
  const CHUNK_KEY_PREFIX = "chunk:";
891
1054
  /** Registry keys used when this DO instance acts as a per-run stream index. */
892
1055
  const STREAM_REGISTRY_PREFIX = "stream:";
1056
+ const REGISTRY_OWNER_KEY = "registry:owner";
1057
+ const REGISTRY_EXPIRED_KEY = "registry:expired";
893
1058
  /**
894
1059
  * Zero-pad chunk indexes so `storage.list({ prefix })` returns chunks in
895
1060
  * write order. 12 digits comfortably exceeds any realistic chunk count.
@@ -914,22 +1079,28 @@ var StreamDO = class extends DurableObject {
914
1079
  closed: false
915
1080
  };
916
1081
  }
1082
+ assertOwner(meta, runId) {
1083
+ if (meta.expiredAt !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
1084
+ if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
1085
+ }
917
1086
  /**
918
1087
  * Append a chunk. The index is allocated inside the transaction, so
919
1088
  * concurrent writers can never collide or skip.
920
1089
  */
921
- async writeChunk(data) {
1090
+ async writeChunk(runId, data) {
922
1091
  return await this.ctx.storage.transaction(async (txn) => {
923
1092
  const meta = await txn.get(META_KEY) ?? {
924
1093
  count: 0,
925
1094
  closed: false
926
1095
  };
1096
+ this.assertOwner(meta, runId);
927
1097
  if (meta.closed) throw new Error("Cannot write to a closed stream");
928
1098
  const index = meta.count;
929
1099
  await txn.put(chunkKey(index), data);
930
1100
  await txn.put(META_KEY, {
931
1101
  count: index + 1,
932
- closed: false
1102
+ closed: false,
1103
+ ownerRunId: runId
933
1104
  });
934
1105
  return index;
935
1106
  });
@@ -938,14 +1109,16 @@ var StreamDO = class extends DurableObject {
938
1109
  * Close the stream (idempotent). Readers observe `done: true` once all
939
1110
  * previously written chunks have been consumed.
940
1111
  */
941
- async closeStream() {
1112
+ async closeStream(runId) {
942
1113
  await this.ctx.storage.transaction(async (txn) => {
943
1114
  const meta = await txn.get(META_KEY) ?? {
944
1115
  count: 0,
945
1116
  closed: false
946
1117
  };
1118
+ this.assertOwner(meta, runId);
947
1119
  await txn.put(META_KEY, {
948
1120
  ...meta,
1121
+ ownerRunId: runId,
949
1122
  closed: true
950
1123
  });
951
1124
  });
@@ -955,6 +1128,7 @@ var StreamDO = class extends DurableObject {
955
1128
  */
956
1129
  async getChunks(params) {
957
1130
  const meta = await this.getMeta();
1131
+ if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
958
1132
  const start = Math.max(0, params.startIndex);
959
1133
  const entries = await this.ctx.storage.list({
960
1134
  prefix: CHUNK_KEY_PREFIX,
@@ -972,6 +1146,7 @@ var StreamDO = class extends DurableObject {
972
1146
  */
973
1147
  async getInfo() {
974
1148
  const meta = await this.getMeta();
1149
+ if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
975
1150
  return {
976
1151
  tailIndex: meta.count - 1,
977
1152
  done: meta.closed
@@ -980,16 +1155,66 @@ var StreamDO = class extends DurableObject {
980
1155
  /**
981
1156
  * Register a stream name against this per-run registry instance.
982
1157
  */
983
- async registerStream(name) {
984
- await this.ctx.storage.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
1158
+ async registerStream(runId, name) {
1159
+ await this.ctx.storage.transaction(async (txn) => {
1160
+ if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
1161
+ const owner = await txn.get(REGISTRY_OWNER_KEY);
1162
+ if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
1163
+ await txn.put(REGISTRY_OWNER_KEY, runId);
1164
+ await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
1165
+ });
985
1166
  }
986
1167
  /**
987
1168
  * List stream names registered against this per-run registry instance.
988
1169
  */
989
1170
  async listStreams() {
1171
+ if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
990
1172
  const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
991
1173
  return Array.from(entries.keys()).map((key) => key.slice(7));
992
1174
  }
1175
+ /** Fence a run's registry before any stream cells are removed. */
1176
+ async expireRegistry(runId, expiredAt) {
1177
+ return await this.ctx.storage.transaction(async (txn) => {
1178
+ const owner = await txn.get(REGISTRY_OWNER_KEY);
1179
+ if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
1180
+ await txn.put(REGISTRY_OWNER_KEY, runId);
1181
+ await txn.put(REGISTRY_EXPIRED_KEY, expiredAt);
1182
+ const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
1183
+ return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
1184
+ });
1185
+ }
1186
+ /** Remove the registry payload after every referenced stream was fenced. */
1187
+ async finalizeRegistry(runId) {
1188
+ await this.ctx.storage.transaction(async (txn) => {
1189
+ const owner = await txn.get(REGISTRY_OWNER_KEY);
1190
+ const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
1191
+ if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
1192
+ const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
1193
+ for (const key of entries.keys()) await txn.delete(key);
1194
+ });
1195
+ }
1196
+ /** Delete stream payload while leaving a small ownership tombstone. */
1197
+ async expireStream(runId, expiredAt) {
1198
+ return await this.ctx.storage.transaction(async (txn) => {
1199
+ const meta = await txn.get(META_KEY) ?? {
1200
+ count: 0,
1201
+ closed: false
1202
+ };
1203
+ if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
1204
+ const chunks = await txn.list({ prefix: CHUNK_KEY_PREFIX });
1205
+ for (const key of chunks.keys()) await txn.delete(key);
1206
+ await txn.put(META_KEY, {
1207
+ count: 0,
1208
+ closed: true,
1209
+ ownerRunId: runId,
1210
+ expiredAt
1211
+ });
1212
+ return {
1213
+ deleted: true,
1214
+ chunks: chunks.size
1215
+ };
1216
+ });
1217
+ }
993
1218
  };
994
1219
  //#endregion
995
1220
  //#region src/apply-event.ts
@@ -999,6 +1224,7 @@ const RUN_KEY = "run";
999
1224
  const EVENT_KEY_PREFIX = "event:";
1000
1225
  const STEP_KEY_PREFIX = "step:";
1001
1226
  const HOOK_KEY_PREFIX = "hook:";
1227
+ const WAIT_KEY_PREFIX = "wait:";
1002
1228
  const STEP_CREATED_KEY_PREFIX = "stepcreated:";
1003
1229
  const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
1004
1230
  /**
@@ -1008,6 +1234,10 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
1008
1234
  * rejected as a duplicate instead of resurrecting the hook.
1009
1235
  */
1010
1236
  const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
1237
+ /** Workflow-authored attr_set correlation claims. */
1238
+ const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
1239
+ /** Durable lazy-hook resume claims, scoped by the containing run DO. */
1240
+ const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
1011
1241
  function failure(code, message, extra) {
1012
1242
  return {
1013
1243
  ok: false,
@@ -1110,9 +1340,15 @@ async function releaseAllHooks(store) {
1110
1340
  }
1111
1341
  return released;
1112
1342
  }
1343
+ /** Delete all persisted waits once their owning run becomes terminal. */
1344
+ async function releaseAllWaits(store) {
1345
+ const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
1346
+ for (const key of waits.keys()) await store.delete(key);
1347
+ }
1113
1348
  async function applyEvent(store, ctx) {
1114
1349
  const { runId, data, now } = ctx;
1115
1350
  const specVersion = data.specVersion ?? SPEC_VERSION_CURRENT;
1351
+ if (specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: specVersion });
1116
1352
  /** Build and validate the stored event BEFORE anything is persisted. */
1117
1353
  const buildEvent = (record) => {
1118
1354
  const withMeta = {
@@ -1120,9 +1356,16 @@ async function applyEvent(store, ctx) {
1120
1356
  runId,
1121
1357
  eventId: ctx.nextEventId(),
1122
1358
  createdAt: now,
1359
+ occurredAt: ctx.params?.occurredAt,
1360
+ resumeId: ctx.params?.resumeId,
1123
1361
  specVersion
1124
1362
  };
1363
+ if (record.eventType !== "hook_received") delete withMeta.resumeId;
1125
1364
  if (record.eventType === "run_started") delete withMeta.eventData;
1365
+ if (record.eventType === "step_started" && withMeta.eventData !== null && typeof withMeta.eventData === "object") {
1366
+ const { input: _input, ...eventData } = withMeta.eventData;
1367
+ withMeta.eventData = eventData;
1368
+ }
1126
1369
  return EventSchema.parse(compact(withMeta));
1127
1370
  };
1128
1371
  const putEvent = async (event) => {
@@ -1133,14 +1376,21 @@ async function applyEvent(store, ctx) {
1133
1376
  if (!skipRunValidation) currentRun = await store.get(RUN_KEY);
1134
1377
  let bootstrapped;
1135
1378
  if (data.eventType === "run_started" && !currentRun && data.eventData) {
1136
- const { deploymentId, workflowName, input, executionContext } = data.eventData;
1379
+ const { deploymentId, workflowName, input, executionContext, attributes, allowReservedAttributes, encryptionPublicKey } = data.eventData;
1137
1380
  if (deploymentId && workflowName && input !== void 0) {
1381
+ const attributeChanges = Object.entries(attributes ?? {}).map(([key, value]) => ({
1382
+ key,
1383
+ value
1384
+ }));
1385
+ validateAttributeChanges(attributeChanges, { allowReservedAttributes });
1138
1386
  const createdRun = WorkflowRunSchema.parse(compact({
1139
1387
  runId,
1140
1388
  deploymentId,
1141
1389
  workflowName,
1142
1390
  specVersion,
1143
1391
  executionContext,
1392
+ attributes,
1393
+ encryptionPublicKey,
1144
1394
  input,
1145
1395
  status: "pending",
1146
1396
  output: void 0,
@@ -1156,7 +1406,10 @@ async function applyEvent(store, ctx) {
1156
1406
  deploymentId,
1157
1407
  workflowName,
1158
1408
  input,
1159
- executionContext
1409
+ executionContext,
1410
+ attributes,
1411
+ allowReservedAttributes,
1412
+ encryptionPublicKey
1160
1413
  }
1161
1414
  });
1162
1415
  await store.put(RUN_KEY, createdRun);
@@ -1168,10 +1421,7 @@ async function applyEvent(store, ctx) {
1168
1421
  };
1169
1422
  }
1170
1423
  }
1171
- if (currentRun) {
1172
- if (requiresNewerWorld(currentRun.specVersion)) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" requires a newer world (run specVersion ${currentRun.specVersion}, world ${SPEC_VERSION_CURRENT})`, { runSpecVersion: currentRun.specVersion });
1173
- if (isLegacySpecVersion(currentRun.specVersion)) return failure("LEGACY_RUN_NOT_SUPPORTED", `Run "${runId}" was created with a legacy spec version (${currentRun.specVersion ?? "undefined"}) not supported by this world`);
1174
- }
1424
+ if (currentRun && currentRun.specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${currentRun.specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: currentRun.specVersion });
1175
1425
  if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) {
1176
1426
  if (data.eventType === "run_cancelled" && currentRun.status === "cancelled") {
1177
1427
  const event = buildEvent({ ...data });
@@ -1185,14 +1435,32 @@ async function applyEvent(store, ctx) {
1185
1435
  }
1186
1436
  if (data.eventType === "run_started") return failure("RUN_EXPIRED", `Workflow run "${runId}" is already in terminal state "${currentRun.status}"`);
1187
1437
  if (data.eventType === "run_completed" || data.eventType === "run_failed" || data.eventType === "run_cancelled") return failure("ENTITY_CONFLICT", `Cannot transition run from terminal state "${currentRun.status}"`);
1188
- if (data.eventType === "step_created" || data.eventType === "hook_created" || data.eventType === "wait_created") return failure("ENTITY_CONFLICT", `Cannot create new entities on run in terminal state "${currentRun.status}"`);
1438
+ if (isChildEntityCreationEvent(data)) return failure("ENTITY_CONFLICT", `Cannot create new entities on run in terminal state "${currentRun.status}"`);
1439
+ if (data.eventType === "attr_set") return failure("ENTITY_CONFLICT", `Cannot set attributes on run in terminal state "${currentRun.status}"`);
1189
1440
  }
1190
1441
  let validatedStep;
1191
1442
  if (data.eventType === "step_started" || data.eventType === "step_completed" || data.eventType === "step_failed" || data.eventType === "step_retrying") {
1443
+ const lazyStepStart = data.eventType === "step_started" && isChildEntityCreationEvent(data);
1192
1444
  validatedStep = await store.get(`${STEP_KEY_PREFIX}${data.correlationId}`);
1193
- if (!validatedStep) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
1194
- if (isTerminalStepStatus(validatedStep.status)) return failure("ENTITY_CONFLICT", `Cannot modify step in terminal state "${validatedStep.status}"`);
1195
- if (currentRun && isTerminalWorkflowRunStatus(currentRun.status) && validatedStep.status !== "running") return failure("RUN_EXPIRED", `Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
1445
+ if (!validatedStep) {
1446
+ if (!lazyStepStart) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
1447
+ } else if (lazyStepStart) return failure("ENTITY_CONFLICT", `Step "${data.correlationId}" already exists`);
1448
+ if (validatedStep && isTerminalStepStatus(validatedStep.status)) return failure("ENTITY_CONFLICT", `Cannot modify step in terminal state "${validatedStep.status}"`);
1449
+ if (currentRun && isTerminalWorkflowRunStatus(currentRun.status) && validatedStep && validatedStep.status !== "running") return failure("RUN_EXPIRED", `Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
1450
+ }
1451
+ if (data.eventType === "hook_received" && ctx.params?.resumeId) {
1452
+ const claim = await store.get(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`);
1453
+ if (claim) {
1454
+ if (claim.hookId !== data.correlationId) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded for a different hook`);
1455
+ if (claim.payloadDigest && ctx.params.resumePayloadDigest && claim.payloadDigest !== ctx.params.resumePayloadDigest) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded with a different payload`);
1456
+ const committed = await store.get(`${EVENT_KEY_PREFIX}${claim.eventId}`);
1457
+ if (committed?.eventType === "hook_received" && committed.correlationId === claim.hookId && committed.resumeId === ctx.params.resumeId) return {
1458
+ ok: true,
1459
+ event: committed,
1460
+ releasedHooks: []
1461
+ };
1462
+ if (committed) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" points to a different event`);
1463
+ }
1196
1464
  }
1197
1465
  if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
1198
1466
  if (!await store.get(`hook:${data.correlationId}`)) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
@@ -1206,12 +1474,19 @@ async function applyEvent(store, ctx) {
1206
1474
  run: existing,
1207
1475
  releasedHooks: []
1208
1476
  };
1477
+ const attributeChanges = Object.entries(data.eventData.attributes ?? {}).map(([key, value]) => ({
1478
+ key,
1479
+ value
1480
+ }));
1481
+ validateAttributeChanges(attributeChanges, { allowReservedAttributes: data.eventData.allowReservedAttributes });
1209
1482
  const run = WorkflowRunSchema.parse(compact({
1210
1483
  runId,
1211
1484
  deploymentId: data.eventData.deploymentId,
1212
1485
  workflowName: data.eventData.workflowName,
1213
1486
  specVersion,
1214
1487
  executionContext: data.eventData.executionContext,
1488
+ attributes: data.eventData.attributes,
1489
+ encryptionPublicKey: data.eventData.encryptionPublicKey,
1215
1490
  input: data.eventData.input,
1216
1491
  status: "pending",
1217
1492
  output: void 0,
@@ -1237,11 +1512,25 @@ async function applyEvent(store, ctx) {
1237
1512
  }
1238
1513
  case "run_started": {
1239
1514
  if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
1240
- if (currentRun.status === "running") return {
1241
- ok: true,
1242
- run: currentRun,
1243
- releasedHooks: []
1244
- };
1515
+ if (currentRun.status === "running") {
1516
+ const result = {
1517
+ ok: true,
1518
+ run: currentRun,
1519
+ releasedHooks: [],
1520
+ maxEvents: getMaxEventsPerRun()
1521
+ };
1522
+ if (ctx.params?.skipPreload) return result;
1523
+ const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
1524
+ limit: getMaxEventsPerRun(),
1525
+ sortOrder: "asc"
1526
+ }, (event) => event.eventId);
1527
+ return {
1528
+ ...result,
1529
+ events: allEvents.data,
1530
+ cursor: allEvents.cursor,
1531
+ hasMore: allEvents.hasMore
1532
+ };
1533
+ }
1245
1534
  const run = WorkflowRunSchema.parse(compact({
1246
1535
  ...currentRun,
1247
1536
  status: "running",
@@ -1254,17 +1543,24 @@ async function applyEvent(store, ctx) {
1254
1543
  const event = buildEvent({ ...data });
1255
1544
  await store.put(RUN_KEY, run);
1256
1545
  await putEvent(event);
1257
- const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
1258
- limit: 1e3,
1259
- sortOrder: "asc"
1260
- }, (e) => e.eventId);
1261
- return {
1546
+ const result = {
1262
1547
  ok: true,
1263
1548
  event,
1264
1549
  run,
1265
1550
  releasedHooks: [],
1266
1551
  runCreated: bootstrapped,
1267
- events: allEvents.data
1552
+ maxEvents: getMaxEventsPerRun()
1553
+ };
1554
+ if (ctx.params?.skipPreload) return result;
1555
+ const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
1556
+ limit: getMaxEventsPerRun(),
1557
+ sortOrder: "asc"
1558
+ }, (e) => e.eventId);
1559
+ return {
1560
+ ...result,
1561
+ events: allEvents.data,
1562
+ cursor: allEvents.cursor,
1563
+ hasMore: allEvents.hasMore
1268
1564
  };
1269
1565
  }
1270
1566
  case "run_completed": {
@@ -1280,11 +1576,13 @@ async function applyEvent(store, ctx) {
1280
1576
  const event = buildEvent({ ...data });
1281
1577
  await store.put(RUN_KEY, run);
1282
1578
  await putEvent(event);
1579
+ const releasedHooks = await releaseAllHooks(store);
1580
+ await releaseAllWaits(store);
1283
1581
  return {
1284
1582
  ok: true,
1285
1583
  event,
1286
1584
  run,
1287
- releasedHooks: await releaseAllHooks(store)
1585
+ releasedHooks
1288
1586
  };
1289
1587
  }
1290
1588
  case "run_failed": {
@@ -1300,11 +1598,13 @@ async function applyEvent(store, ctx) {
1300
1598
  const event = buildEvent({ ...data });
1301
1599
  await store.put(RUN_KEY, run);
1302
1600
  await putEvent(event);
1601
+ const releasedHooks = await releaseAllHooks(store);
1602
+ await releaseAllWaits(store);
1303
1603
  return {
1304
1604
  ok: true,
1305
1605
  event,
1306
1606
  run,
1307
- releasedHooks: await releaseAllHooks(store)
1607
+ releasedHooks
1308
1608
  };
1309
1609
  }
1310
1610
  case "run_cancelled": {
@@ -1320,11 +1620,37 @@ async function applyEvent(store, ctx) {
1320
1620
  const event = buildEvent({ ...data });
1321
1621
  await store.put(RUN_KEY, run);
1322
1622
  await putEvent(event);
1623
+ const releasedHooks = await releaseAllHooks(store);
1624
+ await releaseAllWaits(store);
1323
1625
  return {
1324
1626
  ok: true,
1325
1627
  event,
1326
1628
  run,
1327
- releasedHooks: await releaseAllHooks(store)
1629
+ releasedHooks
1630
+ };
1631
+ }
1632
+ case "attr_set": {
1633
+ if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
1634
+ validateAttributeChanges(data.eventData.changes, {
1635
+ existingKeys: Object.keys(currentRun.attributes),
1636
+ allowReservedAttributes: data.eventData.allowReservedAttributes
1637
+ });
1638
+ const markerKey = data.correlationId && data.eventData.writer.type === "workflow" ? `${ATTRIBUTE_EVENT_MARKER_PREFIX}${data.correlationId}` : void 0;
1639
+ if (markerKey && await store.get(markerKey) !== void 0) return failure("ENTITY_CONFLICT", `Attribute event "${data.correlationId}" already exists`);
1640
+ const run = WorkflowRunSchema.parse(compact({
1641
+ ...currentRun,
1642
+ attributes: applyAttributeChanges(currentRun.attributes, data.eventData.changes),
1643
+ updatedAt: now
1644
+ }));
1645
+ const event = buildEvent({ ...data });
1646
+ await store.put(RUN_KEY, run);
1647
+ if (markerKey) await store.put(markerKey, event.eventId);
1648
+ await putEvent(event);
1649
+ return {
1650
+ ok: true,
1651
+ event,
1652
+ run,
1653
+ releasedHooks: []
1328
1654
  };
1329
1655
  }
1330
1656
  case "step_created": {
@@ -1358,7 +1684,34 @@ async function applyEvent(store, ctx) {
1358
1684
  };
1359
1685
  }
1360
1686
  case "step_started": {
1361
- const step = validatedStep;
1687
+ let step = validatedStep;
1688
+ let stepCreated = false;
1689
+ if (!step && data.eventData?.input !== void 0 && data.eventData.stepName) {
1690
+ step = StepSchema.parse(compact({
1691
+ runId,
1692
+ stepId: data.correlationId,
1693
+ stepName: data.eventData.stepName,
1694
+ status: "pending",
1695
+ input: data.eventData.input,
1696
+ attempt: 0,
1697
+ createdAt: now,
1698
+ updatedAt: now,
1699
+ specVersion
1700
+ }));
1701
+ const createdEvent = buildEvent({
1702
+ eventType: "step_created",
1703
+ correlationId: data.correlationId,
1704
+ eventData: {
1705
+ stepName: data.eventData.stepName,
1706
+ workflowName: data.eventData.workflowName,
1707
+ input: data.eventData.input
1708
+ }
1709
+ });
1710
+ await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, step);
1711
+ await store.put(creationIndexKey(STEP_CREATED_KEY_PREFIX, step.createdAt, step.stepId), step.stepId);
1712
+ await putEvent(createdEvent);
1713
+ stepCreated = true;
1714
+ }
1362
1715
  if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
1363
1716
  if (step.retryAfter && step.retryAfter.getTime() > now.getTime()) return failure("TOO_EARLY", `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, { retryAfterSeconds: Math.ceil((step.retryAfter.getTime() - now.getTime()) / 1e3) });
1364
1717
  const updated = StepSchema.parse(compact({
@@ -1376,7 +1729,8 @@ async function applyEvent(store, ctx) {
1376
1729
  ok: true,
1377
1730
  event,
1378
1731
  step: updated,
1379
- releasedHooks: []
1732
+ releasedHooks: [],
1733
+ ...stepCreated ? { stepCreated: true } : {}
1380
1734
  };
1381
1735
  }
1382
1736
  case "step_completed": {
@@ -1405,7 +1759,7 @@ async function applyEvent(store, ctx) {
1405
1759
  const updated = StepSchema.parse(compact({
1406
1760
  ...step,
1407
1761
  status: "failed",
1408
- error: toStructuredError(data.eventData.error, { stack: data.eventData.stack }),
1762
+ error: toStructuredError(data.eventData.error),
1409
1763
  completedAt: now,
1410
1764
  updatedAt: now
1411
1765
  }));
@@ -1425,7 +1779,7 @@ async function applyEvent(store, ctx) {
1425
1779
  const updated = StepSchema.parse(compact({
1426
1780
  ...step,
1427
1781
  status: "pending",
1428
- error: toStructuredError(data.eventData.error, { stack: data.eventData.stack }),
1782
+ error: toStructuredError(data.eventData.error),
1429
1783
  retryAfter: data.eventData.retryAfter ? new Date(data.eventData.retryAfter) : void 0,
1430
1784
  updatedAt: now
1431
1785
  }));
@@ -1481,7 +1835,9 @@ async function applyEvent(store, ctx) {
1481
1835
  environment: "",
1482
1836
  createdAt: now,
1483
1837
  specVersion,
1484
- isWebhook: data.eventData.isWebhook ?? false
1838
+ isWebhook: data.eventData.isWebhook ?? false,
1839
+ isSystem: data.eventData.isSystem,
1840
+ tokenRetentionUntil: data.eventData.tokenRetentionUntil
1485
1841
  }));
1486
1842
  const event = buildEvent({ ...data });
1487
1843
  await store.put(hookKey, hook);
@@ -1513,78 +1869,113 @@ async function applyEvent(store, ctx) {
1513
1869
  }]
1514
1870
  };
1515
1871
  }
1516
- default: {
1872
+ case "wait_created": {
1873
+ const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
1874
+ if (await store.get(waitKey) !== void 0) return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already exists`);
1875
+ const wait = WaitSchema.parse({
1876
+ waitId: `${runId}-${data.correlationId}`,
1877
+ runId,
1878
+ status: "waiting",
1879
+ resumeAt: data.eventData.resumeAt,
1880
+ createdAt: now,
1881
+ updatedAt: now,
1882
+ specVersion
1883
+ });
1517
1884
  const event = buildEvent({ ...data });
1885
+ await store.put(waitKey, wait);
1518
1886
  await putEvent(event);
1519
1887
  return {
1520
1888
  ok: true,
1521
1889
  event,
1890
+ wait,
1522
1891
  releasedHooks: []
1523
1892
  };
1524
1893
  }
1525
- }
1526
- }
1527
- //#endregion
1528
- //#region src/worker/durable-objects/WorkflowRunDO.ts
1529
- /**
1530
- * Current schema version for DO storage.
1531
- * Increment when making breaking changes to the storage layout.
1532
- * Migrations run lazily on first request after deploy.
1533
- *
1534
- * v2: one storage key per event ('event:<eventId>'), step ('step:<stepId>')
1535
- * and hook ('hook:<hookId>') instead of single aggregate values. This package
1536
- * is pre-production, so the migration simply drops the legacy aggregates.
1537
- */
1538
- const CURRENT_SCHEMA_VERSION = 3;
1539
- /**
1540
- * Idempotent migration functions keyed by target version. The schema version
1541
- * is written only after the migration completes, so eviction safely retries
1542
- * an interrupted migration.
1543
- */
1544
- const MIGRATIONS = {
1545
- 2: async (storage) => {
1546
- await storage.transaction(async (txn) => {
1547
- await txn.delete("events");
1548
- await txn.delete("steps");
1549
- await txn.delete("hooks");
1550
- });
1551
- },
1552
- 3: async (storage) => {
1553
- await backfillCreationIndex(storage, STEP_KEY_PREFIX, STEP_CREATED_KEY_PREFIX, (step) => step.stepId);
1554
- await backfillCreationIndex(storage, HOOK_KEY_PREFIX, HOOK_CREATED_KEY_PREFIX, (hook) => hook.hookId);
1555
- }
1556
- };
1557
- async function backfillCreationIndex(storage, entityPrefix, indexPrefix, getId) {
1558
- let startAfter;
1559
- while (true) {
1560
- const entities = await storage.list({
1561
- prefix: entityPrefix,
1562
- startAfter,
1563
- limit: 100
1564
- });
1565
- if (entities.size === 0) return;
1566
- for (const [key, entity] of entities) {
1567
- const id = getId(entity);
1568
- await storage.put(`${indexPrefix}${entity.createdAt.toISOString()}:${id}`, id);
1569
- startAfter = key;
1894
+ case "wait_completed": {
1895
+ const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
1896
+ const existing = await store.get(waitKey);
1897
+ if (!existing) return failure("WAIT_NOT_FOUND", `Wait "${data.correlationId}" not found`);
1898
+ if (existing.status === "completed") return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already completed`);
1899
+ const event = buildEvent({ ...data });
1900
+ const wait = WaitSchema.parse({
1901
+ ...existing,
1902
+ status: "completed",
1903
+ resumeAt: data.eventData?.resumeAt ?? existing.resumeAt,
1904
+ completedAt: now,
1905
+ updatedAt: now
1906
+ });
1907
+ await store.put(waitKey, wait);
1908
+ await putEvent(event);
1909
+ return {
1910
+ ok: true,
1911
+ event,
1912
+ wait,
1913
+ releasedHooks: []
1914
+ };
1915
+ }
1916
+ default: {
1917
+ const event = buildEvent({ ...data });
1918
+ await putEvent(event);
1919
+ if (data.eventType === "hook_received" && ctx.params?.resumeId) await store.put(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`, {
1920
+ hookId: data.correlationId,
1921
+ eventId: event.eventId,
1922
+ ...ctx.params.resumePayloadDigest ? { payloadDigest: ctx.params.resumePayloadDigest } : {}
1923
+ });
1924
+ return {
1925
+ ok: true,
1926
+ event,
1927
+ releasedHooks: []
1928
+ };
1570
1929
  }
1571
- if (entities.size < 100) return;
1572
1930
  }
1573
1931
  }
1574
1932
  /**
1575
- * Ensure the DO storage schema is up to date.
1576
- * Runs any pending migrations sequentially inside transactions,
1577
- * persisting the version after each successful migration.
1578
- * Safe to call on every request -- it's a no-op when already current.
1933
+ * Complete Workflow 5's event-page response after an atomic apply. A celld
1934
+ * run is a single serialized writer, so the committed sequence is dense; a
1935
+ * stale eventCount therefore needs only the slots between the requested
1936
+ * position and the event this call committed.
1579
1937
  */
1580
- async function ensureSchemaUpToDate(storage) {
1581
- const stored = await storage.get("schema_version") ?? 1;
1582
- for (let v = stored + 1; v <= CURRENT_SCHEMA_VERSION; v++) {
1583
- const migration = MIGRATIONS[v];
1584
- if (migration) await migration(storage);
1585
- await storage.put("schema_version", v);
1938
+ async function finalizeEventPage(store, outcome, params) {
1939
+ if (!outcome.ok) return outcome;
1940
+ const result = outcome.run && outcome.maxEvents === void 0 ? {
1941
+ ...outcome,
1942
+ maxEvents: getMaxEventsPerRun()
1943
+ } : outcome;
1944
+ if (!result.event) return result;
1945
+ if (typeof params?.sinceCursor === "string") {
1946
+ const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
1947
+ limit: 100,
1948
+ cursor: params.sinceCursor,
1949
+ sortOrder: "asc"
1950
+ }, (event) => event.eventId);
1951
+ return {
1952
+ ...result,
1953
+ events: page.data,
1954
+ cursor: page.cursor,
1955
+ hasMore: page.hasMore
1956
+ };
1586
1957
  }
1958
+ if (params?.eventCount === void 0) return result;
1959
+ const committedSlot = eventIdToSlot(result.event.eventId);
1960
+ if (committedSlot === null || committedSlot <= params.eventCount + 1) return result;
1961
+ const skippedCount = committedSlot - params.eventCount - 1;
1962
+ const events = (await listByPrefix(store, EVENT_KEY_PREFIX, {
1963
+ limit: skippedCount,
1964
+ cursor: params.eventCount > 0 ? slotToEventId(params.eventCount) : void 0,
1965
+ sortOrder: "asc"
1966
+ }, (event) => event.eventId)).data.filter((event) => event.eventId < result.event.eventId);
1967
+ return {
1968
+ ...result,
1969
+ events,
1970
+ cursor: null,
1971
+ hasMore: events.length < skippedCount
1972
+ };
1587
1973
  }
1974
+ //#endregion
1975
+ //#region src/worker/durable-objects/WorkflowRunDO.ts
1976
+ const PAYLOAD_DELETE_BATCH = 256;
1977
+ const STREAM_DELETE_CONCURRENCY = 16;
1978
+ const CLEANUP_RETRY_MAX_MS = 36e5;
1588
1979
  /**
1589
1980
  * Adapt DO storage (or a transaction) to the {@link EventStore} interface
1590
1981
  * shared with the test mocks. No casts: the methods line up structurally,
@@ -1609,105 +2000,335 @@ function storeFrom(storage) {
1609
2000
  * queue-message dedup claims (see claimInflight/releaseInflight).
1610
2001
  */
1611
2002
  var WorkflowRunDO = class extends DurableObject {
1612
- schemaReady = null;
1613
- /**
1614
- * Lazily ensure schema migrations have run.
1615
- * Memoized so it only runs once per DO instantiation.
1616
- */
1617
- ensureSchema() {
1618
- if (!this.schemaReady) this.schemaReady = ensureSchemaUpToDate(this.ctx.storage);
1619
- return this.schemaReady;
2003
+ now() {
2004
+ const clock = this.env?.clock;
2005
+ return typeof clock === "function" ? clock() : Date.now();
1620
2006
  }
1621
- /**
1622
- * Apply an event: validate, guard, append the event and mutate the entity
1623
- * in ONE storage transaction. Guard failures are returned as structured
1624
- * outcomes (not thrown) because typed errors do not survive the RPC
1625
- * boundary with their `name` intact.
1626
- */
2007
+ async expiredTombstone(storage, now = this.now()) {
2008
+ const tombstone = await storage.get(TOMBSTONE_KEY);
2009
+ if (tombstone) return tombstone;
2010
+ const cleanup = await storage.get(CLEANUP_RECORD_KEY);
2011
+ if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
2012
+ return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
2013
+ }
2014
+ async read(reader) {
2015
+ const tombstone = await this.expiredTombstone(this.ctx.storage);
2016
+ if (tombstone) return expiredRead(tombstone);
2017
+ return {
2018
+ ok: true,
2019
+ value: await reader()
2020
+ };
2021
+ }
2022
+ /** Apply an event and capture all retention metadata in the same transaction. */
1627
2023
  async applyEvent(request) {
1628
- await this.ensureSchema();
1629
2024
  return await this.ctx.storage.transaction(async (txn) => {
2025
+ const now = new Date(this.now());
2026
+ const tombstone = await this.expiredTombstone(txn, now.getTime());
2027
+ if (tombstone) return {
2028
+ ok: false,
2029
+ code: "RUN_EXPIRED",
2030
+ message: `Workflow run "${request.runId}" expired at ${tombstone.expiredAt.toISOString()}`
2031
+ };
1630
2032
  let eventSequence = await txn.get("event_sequence") ?? 0;
1631
2033
  const outcome = await applyEvent(storeFrom(txn), {
1632
2034
  ...request,
1633
- nextEventId: () => `wevt_z${String(++eventSequence).padStart(20, "0")}`,
1634
- now: /* @__PURE__ */ new Date()
2035
+ nextEventId: () => slotToEventId(++eventSequence),
2036
+ now
1635
2037
  });
1636
- if (outcome.ok) await txn.put("event_sequence", eventSequence);
1637
- return outcome;
2038
+ if (!outcome.ok) return outcome;
2039
+ await txn.put("event_sequence", eventSequence);
2040
+ if (outcome.event?.correlationId) {
2041
+ const key = correlationIndexKey(outcome.event.correlationId, outcome.event.createdAt, outcome.event.eventId, request.runId);
2042
+ await txn.put(indexMarkerKey(key), key);
2043
+ }
2044
+ if (outcome.hookToIndex) {
2045
+ const reference = {
2046
+ hookId: outcome.hookToIndex.hookId,
2047
+ token: outcome.hookToIndex.token
2048
+ };
2049
+ await txn.put(hookMarkerKey(reference), reference);
2050
+ }
2051
+ for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
2052
+ const retentionMs = request.cleanup?.retentionMs ?? 0;
2053
+ if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !await txn.get("retention:cleanup")) {
2054
+ const terminalRun = outcome.run;
2055
+ const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
2056
+ const run = WorkflowRunSchema.parse({
2057
+ ...terminalRun,
2058
+ expiredAt: dueAt
2059
+ });
2060
+ const cleanup = {
2061
+ version: 1,
2062
+ runId: run.runId,
2063
+ workflowName: run.workflowName,
2064
+ createdAt: run.createdAt,
2065
+ completedAt: run.completedAt,
2066
+ terminalStatus: run.status,
2067
+ dueAt,
2068
+ queueShards: request.cleanup?.queueShards ?? 1,
2069
+ phase: "retained",
2070
+ attempts: 0,
2071
+ deletedPayloadKeys: 0,
2072
+ deletedStreams: 0,
2073
+ deletedQueueMessages: 0
2074
+ };
2075
+ await txn.put("run", run);
2076
+ await txn.put(CLEANUP_RECORD_KEY, cleanup);
2077
+ await txn.setAlarm(dueAt);
2078
+ outcome.run = run;
2079
+ }
2080
+ return finalizeEventPage(storeFrom(txn), outcome, request.params);
1638
2081
  });
1639
2082
  }
1640
- /**
1641
- * Get the workflow run
1642
- */
1643
2083
  async getRun() {
1644
- await this.ensureSchema();
1645
- return await this.ctx.storage.get("run") ?? null;
2084
+ return this.read(async () => await this.ctx.storage.get("run") ?? null);
1646
2085
  }
1647
- /**
1648
- * Get a specific step
1649
- */
1650
2086
  async getStep(stepId) {
1651
- await this.ensureSchema();
1652
- return await this.ctx.storage.get(`step:${stepId}`) ?? null;
2087
+ return this.read(async () => await this.ctx.storage.get(`step:${stepId}`) ?? null);
1653
2088
  }
1654
- /**
1655
- * List steps with real limit/cursor pagination (cursor = last stepId).
1656
- */
1657
2089
  async listSteps(params) {
1658
- await this.ensureSchema();
1659
- return listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
2090
+ return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
1660
2091
  limit: params?.limit ?? 20,
1661
2092
  cursor: params?.cursor,
1662
2093
  sortOrder: params?.sortOrder ?? "asc"
1663
- });
2094
+ }));
1664
2095
  }
1665
- /**
1666
- * Get a specific event
1667
- */
1668
2096
  async getEvent(eventId) {
1669
- await this.ensureSchema();
1670
- return await this.ctx.storage.get(`event:${eventId}`) ?? null;
2097
+ return this.read(async () => await this.ctx.storage.get(`event:${eventId}`) ?? null);
1671
2098
  }
1672
- /**
1673
- * List events with real limit/cursor pagination. Events are keyed by their
1674
- * monotonic ULID eventId, so storage order == event order.
1675
- */
1676
2099
  async listEvents(params) {
1677
- await this.ensureSchema();
1678
- return listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
2100
+ return this.read(() => listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
1679
2101
  limit: params?.limit ?? 100,
1680
2102
  cursor: params?.cursor,
1681
2103
  sortOrder: params?.sortOrder ?? "asc"
1682
- }, (event) => event.eventId);
2104
+ }, (event) => event.eventId));
1683
2105
  }
1684
- /**
1685
- * List hooks with real limit/cursor pagination (cursor = last hookId).
1686
- */
1687
2106
  async listHooks(params) {
1688
- await this.ensureSchema();
1689
- return listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
2107
+ return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
1690
2108
  limit: params?.limit ?? 100,
1691
2109
  cursor: params?.cursor,
1692
2110
  sortOrder: params?.sortOrder ?? "asc"
2111
+ }));
2112
+ }
2113
+ async getCleanupStatus() {
2114
+ return await this.ctx.storage.get("retention:cleanup") ?? null;
2115
+ }
2116
+ /** Explicitly schedule an existing terminal run, used for opt-in backfills. */
2117
+ async scheduleCleanup(request) {
2118
+ if (request.retentionMs <= 0) return this.getCleanupStatus();
2119
+ return await this.ctx.storage.transaction(async (txn) => {
2120
+ const existing = await txn.get(CLEANUP_RECORD_KEY);
2121
+ if (existing) return existing;
2122
+ if (await txn.get("retention:tombstone")) return null;
2123
+ const current = await txn.get("run");
2124
+ if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
2125
+ const terminalRun = current;
2126
+ const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
2127
+ const run = WorkflowRunSchema.parse({
2128
+ ...terminalRun,
2129
+ expiredAt: dueAt
2130
+ });
2131
+ const cleanup = {
2132
+ version: 1,
2133
+ runId: run.runId,
2134
+ workflowName: run.workflowName,
2135
+ createdAt: run.createdAt,
2136
+ completedAt: run.completedAt,
2137
+ terminalStatus: run.status,
2138
+ dueAt,
2139
+ queueShards: request.queueShards,
2140
+ phase: "retained",
2141
+ attempts: 0,
2142
+ deletedPayloadKeys: 0,
2143
+ deletedStreams: 0,
2144
+ deletedQueueMessages: 0
2145
+ };
2146
+ await txn.put("run", run);
2147
+ await txn.put(CLEANUP_RECORD_KEY, cleanup);
2148
+ await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
2149
+ return cleanup;
2150
+ });
2151
+ }
2152
+ async cleanupNow(request) {
2153
+ if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
2154
+ ...request,
2155
+ retentionMs: Math.max(1, request.retentionMs)
2156
+ });
2157
+ await this.ctx.storage.transaction(async (txn) => {
2158
+ const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2159
+ if (!cleanup || cleanup.phase === "tombstoned") return;
2160
+ const dueAt = new Date(this.now());
2161
+ const run = await txn.get("run");
2162
+ if (run) await txn.put("run", WorkflowRunSchema.parse({
2163
+ ...run,
2164
+ expiredAt: dueAt
2165
+ }));
2166
+ await txn.put(CLEANUP_RECORD_KEY, {
2167
+ ...cleanup,
2168
+ dueAt
2169
+ });
2170
+ await txn.setAlarm(this.now() + 1);
2171
+ });
2172
+ try {
2173
+ await this.executeCleanup();
2174
+ } catch (error) {
2175
+ await this.recordCleanupFailure(error);
2176
+ }
2177
+ return this.getCleanupStatus();
2178
+ }
2179
+ async rearmCleanup() {
2180
+ const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
2181
+ if (!cleanup) return null;
2182
+ if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
2183
+ else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
2184
+ return cleanup;
2185
+ }
2186
+ async alarm() {
2187
+ try {
2188
+ await this.executeCleanup();
2189
+ } catch (error) {
2190
+ await this.recordCleanupFailure(error);
2191
+ }
2192
+ }
2193
+ namespace(name) {
2194
+ const namespace = this.env?.[name];
2195
+ if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
2196
+ return namespace;
2197
+ }
2198
+ async setCleanupPhase(expected, next, updates = {}) {
2199
+ await this.ctx.storage.transaction(async (txn) => {
2200
+ const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2201
+ if (!cleanup || cleanup.phase !== expected) return;
2202
+ await txn.put(CLEANUP_RECORD_KEY, {
2203
+ ...cleanup,
2204
+ ...updates,
2205
+ phase: next,
2206
+ attempts: 0,
2207
+ lastError: void 0
2208
+ });
2209
+ });
2210
+ }
2211
+ async executeCleanup() {
2212
+ for (;;) {
2213
+ const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
2214
+ if (!cleanup) {
2215
+ await this.ctx.storage.deleteAlarm();
2216
+ return;
2217
+ }
2218
+ if (cleanup.phase === "tombstoned") {
2219
+ await this.ctx.storage.deleteAlarm();
2220
+ return;
2221
+ }
2222
+ if (cleanup.phase === "retained") {
2223
+ if (cleanup.dueAt.getTime() > this.now()) {
2224
+ await this.ctx.storage.setAlarm(cleanup.dueAt);
2225
+ return;
2226
+ }
2227
+ await this.setCleanupPhase("retained", "index");
2228
+ continue;
2229
+ }
2230
+ if (cleanup.phase === "index") {
2231
+ const [indexMarkers, hookMarkers] = await Promise.all([this.ctx.storage.list({ prefix: INDEX_MARKER_PREFIX }), this.ctx.storage.list({ prefix: HOOK_MARKER_PREFIX })]);
2232
+ const indexNamespace = this.namespace("WORKFLOW_INDEX");
2233
+ await indexNamespace.get(indexNamespace.idFromName("index")).expireRun({
2234
+ runId: cleanup.runId,
2235
+ keys: [
2236
+ workflowRunIndexKey(cleanup),
2237
+ globalRunIndexKey(cleanup),
2238
+ ...indexMarkers.values()
2239
+ ],
2240
+ hooks: Array.from(hookMarkers.values()),
2241
+ expiredAt: cleanup.dueAt.getTime()
2242
+ });
2243
+ await this.setCleanupPhase("index", "streams");
2244
+ continue;
2245
+ }
2246
+ if (cleanup.phase === "streams") {
2247
+ const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
2248
+ const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
2249
+ const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime());
2250
+ let deletedStreams = streams.length === 0 ? cleanup.deletedStreams : 0;
2251
+ for (let offset = 0; offset < streams.length; offset += STREAM_DELETE_CONCURRENCY) {
2252
+ const batch = streams.slice(offset, offset + STREAM_DELETE_CONCURRENCY);
2253
+ const results = await Promise.all(batch.map((name) => {
2254
+ return streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime());
2255
+ }));
2256
+ deletedStreams += results.filter((result) => result.deleted).length;
2257
+ }
2258
+ await this.ctx.storage.transaction(async (txn) => {
2259
+ const current = await txn.get(CLEANUP_RECORD_KEY);
2260
+ if (current?.phase === "streams") await txn.put(CLEANUP_RECORD_KEY, {
2261
+ ...current,
2262
+ deletedStreams
2263
+ });
2264
+ });
2265
+ await registry.finalizeRegistry(cleanup.runId);
2266
+ await this.setCleanupPhase("streams", "queues");
2267
+ continue;
2268
+ }
2269
+ if (cleanup.phase === "queues") {
2270
+ const queues = this.namespace("WORKFLOW_QUEUE");
2271
+ let deletedQueueMessages = 0;
2272
+ for (let shard = 0; shard < cleanup.queueShards; shard++) {
2273
+ const result = await queues.get(queues.idFromName(`q:${shard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime());
2274
+ deletedQueueMessages += result.deleted;
2275
+ }
2276
+ await this.setCleanupPhase("queues", "payload", { deletedQueueMessages });
2277
+ continue;
2278
+ }
2279
+ if (cleanup.phase === "payload") {
2280
+ if (!await this.deletePayloadBatch()) return;
2281
+ }
2282
+ }
2283
+ }
2284
+ async deletePayloadBatch() {
2285
+ return await this.ctx.storage.transaction(async (txn) => {
2286
+ const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2287
+ if (!cleanup || cleanup.phase !== "payload") return true;
2288
+ const entries = await txn.list({ limit: 259 });
2289
+ const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
2290
+ const now = new Date(this.now());
2291
+ await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
2292
+ for (const key of keys) await txn.delete(key);
2293
+ if (keys.length > 0) {
2294
+ await txn.put(CLEANUP_RECORD_KEY, {
2295
+ ...cleanup,
2296
+ deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
2297
+ lastError: void 0
2298
+ });
2299
+ await txn.setAlarm(this.now() + 1);
2300
+ return false;
2301
+ }
2302
+ await txn.put(CLEANUP_RECORD_KEY, {
2303
+ ...cleanup,
2304
+ phase: "tombstoned",
2305
+ attempts: 0,
2306
+ lastError: void 0,
2307
+ tombstonedAt: now
2308
+ });
2309
+ await txn.deleteAlarm();
2310
+ return true;
2311
+ });
2312
+ }
2313
+ async recordCleanupFailure(error) {
2314
+ await this.ctx.storage.transaction(async (txn) => {
2315
+ const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2316
+ if (!cleanup || cleanup.phase === "tombstoned") return;
2317
+ const attempts = cleanup.attempts + 1;
2318
+ const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
2319
+ await txn.put(CLEANUP_RECORD_KEY, {
2320
+ ...cleanup,
2321
+ attempts,
2322
+ lastError: error instanceof Error ? error.message : String(error),
2323
+ lastAttemptAt: new Date(this.now())
2324
+ });
2325
+ await txn.setAlarm(this.now() + delay);
1693
2326
  });
1694
2327
  }
1695
- /**
1696
- * Claim a queue-message dedup slot. Used on DO instances dedicated to a
1697
- * single idempotency key (`claim:<queueName>:<idempotencyKey>`).
1698
- *
1699
- * - Unclaimed -> claim and proceed.
1700
- * - Claimed by the SAME messageId -> redelivery of the same message
1701
- * (previous attempt failed transiently) -> proceed.
1702
- * - Claimed by a DIFFERENT messageId -> concurrent duplicate -> reject,
1703
- * unless the claim is stale (holder crashed / went to the DLQ without
1704
- * releasing), in which case the claim is stolen.
1705
- */
1706
2328
  async claimInflight(params) {
1707
- await this.ensureSchema();
1708
2329
  return await this.ctx.storage.transaction(async (txn) => {
1709
2330
  const existing = await txn.get("claim");
1710
- const now = Date.now();
2331
+ const now = this.now();
1711
2332
  if (existing && existing.messageId !== params.messageId && now - existing.claimedAt < params.staleMs) return { claimed: false };
1712
2333
  await txn.put("claim", {
1713
2334
  messageId: params.messageId,
@@ -1716,11 +2337,7 @@ var WorkflowRunDO = class extends DurableObject {
1716
2337
  return { claimed: true };
1717
2338
  });
1718
2339
  }
1719
- /**
1720
- * Release a queue-message dedup claim after the message was fully handled.
1721
- */
1722
2340
  async releaseInflight() {
1723
- await this.ensureSchema();
1724
2341
  await this.ctx.storage.delete("claim");
1725
2342
  }
1726
2343
  };
@@ -1731,13 +2348,21 @@ var WorkflowRunDO = class extends DurableObject {
1731
2348
  * behind a node:http server — the full wire protocol without a celld binary.
1732
2349
  */
1733
2350
  async function startHarness(options = {}) {
2351
+ const cellEnv = { ...options.cellEnv };
1734
2352
  const fleet = new FakeFleet({
1735
2353
  runs: WorkflowRunDO,
1736
2354
  streams: StreamDO,
1737
2355
  index: IndexDO,
1738
2356
  queue: QueueDO,
1739
2357
  ...options.extraClasses
1740
- }, options.cellEnv ?? {});
2358
+ }, cellEnv);
2359
+ cellEnv.clock ??= options.virtualClock ? () => fleet.now : () => Date.now();
2360
+ Object.assign(cellEnv, {
2361
+ WORKFLOW_DB: fleet.namespace("runs"),
2362
+ WORKFLOW_STREAMS: fleet.namespace("streams"),
2363
+ WORKFLOW_INDEX: fleet.namespace("index"),
2364
+ WORKFLOW_QUEUE: fleet.namespace("queue")
2365
+ });
1741
2366
  const router = createRouter({
1742
2367
  WORKFLOW_DB: fleet.namespace("runs"),
1743
2368
  WORKFLOW_STREAMS: fleet.namespace("streams"),
@@ -1771,7 +2396,10 @@ async function startHarness(options = {}) {
1771
2396
  return {
1772
2397
  url: `http://127.0.0.1:${port}`,
1773
2398
  fleet,
1774
- close: () => new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve()))
2399
+ close: () => new Promise((resolve, reject) => {
2400
+ server.close((err) => err ? reject(err) : resolve());
2401
+ server.closeAllConnections();
2402
+ })
1775
2403
  };
1776
2404
  }
1777
2405
  //#endregion