@ewhauser/world-celld 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/testing.js CHANGED
@@ -1,4 +1,4 @@
1
- import { a as cleanupTombstone, c as globalRunIndexKey, d as workflowRunIndexKey, f as compact, h as parse, i as TOMBSTONE_KEY, l as hookMarkerKey, m as rpcStringify, n as HOOK_MARKER_PREFIX, o as correlationIndexKey, p as rpcParse, r as INDEX_MARKER_PREFIX, s as expiredRead, t as CLEANUP_RECORD_KEY, u as indexMarkerKey } from "./retention-bLY1vcOu.js";
1
+ import { C as validateStreamReadRequest, S as normalizeStreamError, _ as decodeStreamWriteBatch, c as MAX_STREAM_BATCH_BYTES, d as MAX_STREAM_LONG_POLL_MS, f as MAX_STREAM_READ_BYTES, h as STREAM_BATCH_CONTENT_TYPE, i as parse, l as MAX_STREAM_CHUNK_BYTES, n as rpcParse, o as createWorkflowIndex, r as rpcStringify, s as runFenceCellName, t as compact, w as validateStreamWriteChunks, x as encodeStreamWriteResult, y as encodeStreamReadResult } from "./util-DmO0XkFw.js";
2
2
  import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
3
3
  import http from "node:http";
4
4
  //#region src/testing/fake-cell.ts
@@ -33,40 +33,109 @@ var FakeStorage = class {
33
33
  clock;
34
34
  data = /* @__PURE__ */ new Map();
35
35
  alarmAt = null;
36
+ getManyCalls = [];
37
+ operationCounts = {
38
+ get: 0,
39
+ getMany: 0,
40
+ list: 0,
41
+ put: 0,
42
+ putMany: 0,
43
+ delete: 0,
44
+ deleteMany: 0,
45
+ transaction: 0
46
+ };
47
+ transactionTail = Promise.resolve();
48
+ transactionDelayMs = 0;
49
+ /** Deterministic query-shape instrumentation for storage scaling tests. */
50
+ listCalls = [];
51
+ operationCalls = [];
36
52
  mutationFailure;
37
53
  constructor(clock = () => Date.now()) {
38
54
  this.clock = clock;
39
55
  }
40
- async get(key) {
41
- return this.data.get(key);
42
- }
43
- async put(key, value) {
44
- this.maybeFailMutation({
56
+ async get(keyOrKeys) {
57
+ if (Array.isArray(keyOrKeys)) {
58
+ this.operationCounts.getMany += 1;
59
+ this.getManyCalls.push([...keyOrKeys]);
60
+ this.recordStorageCall("get", keyOrKeys, false);
61
+ return new Map(keyOrKeys.filter((key) => this.data.has(key)).map((key) => [key, this.data.get(key)]));
62
+ }
63
+ this.operationCounts.get += 1;
64
+ this.recordStorageCall("get", [keyOrKeys], false);
65
+ return this.data.get(keyOrKeys);
66
+ }
67
+ async put(keyOrEntries, value) {
68
+ if (typeof keyOrEntries === "string") {
69
+ this.operationCounts.put += 1;
70
+ this.maybeFailMutation({
71
+ operation: "put",
72
+ key: keyOrEntries,
73
+ value
74
+ });
75
+ this.recordStorageCall("put", [keyOrEntries], false);
76
+ this.data.set(keyOrEntries, structuredClone(value));
77
+ return;
78
+ }
79
+ this.operationCounts.putMany += 1;
80
+ for (const [key, entryValue] of Object.entries(keyOrEntries)) this.maybeFailMutation({
45
81
  operation: "put",
46
82
  key,
47
- value
83
+ value: entryValue
48
84
  });
49
- this.data.set(key, structuredClone(value));
50
- }
51
- async delete(key) {
52
- this.maybeFailMutation({
85
+ this.recordStorageCall("put", Object.keys(keyOrEntries), false);
86
+ for (const [key, entryValue] of Object.entries(keyOrEntries)) this.data.set(key, structuredClone(entryValue));
87
+ }
88
+ async delete(keyOrKeys) {
89
+ if (typeof keyOrKeys === "string") {
90
+ this.operationCounts.delete += 1;
91
+ this.maybeFailMutation({
92
+ operation: "delete",
93
+ key: keyOrKeys
94
+ });
95
+ this.recordStorageCall("delete", [keyOrKeys], false);
96
+ return this.data.delete(keyOrKeys);
97
+ }
98
+ this.operationCounts.deleteMany += 1;
99
+ for (const key of keyOrKeys) this.maybeFailMutation({
53
100
  operation: "delete",
54
101
  key
55
102
  });
56
- return this.data.delete(key);
103
+ this.recordStorageCall("delete", keyOrKeys, false);
104
+ let deleted = 0;
105
+ for (const key of keyOrKeys) if (this.data.delete(key)) deleted += 1;
106
+ return deleted;
57
107
  }
58
108
  async deleteAll() {
59
109
  this.data.clear();
60
110
  }
61
111
  async list(options = {}) {
112
+ this.operationCounts.list += 1;
62
113
  const keys = applyListOptions(Array.from(this.data.keys()), options);
114
+ this.recordList(options, keys.length, false);
63
115
  return new Map(keys.map((k) => [k, this.data.get(k)]));
64
116
  }
65
117
  async transaction(cb) {
66
- const txn = new FakeTransaction(this);
67
- const result = await cb(txn);
68
- txn.commit();
69
- return result;
118
+ this.operationCounts.transaction += 1;
119
+ let release;
120
+ const previous = this.transactionTail;
121
+ this.transactionTail = new Promise((resolve) => {
122
+ release = resolve;
123
+ });
124
+ await previous;
125
+ try {
126
+ if (this.transactionDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, this.transactionDelayMs));
127
+ const txn = new FakeTransaction(this);
128
+ const result = await cb(txn);
129
+ txn.commit();
130
+ return result;
131
+ } finally {
132
+ release();
133
+ }
134
+ }
135
+ /** Add deterministic per-transaction work for contention benchmarks. */
136
+ setTransactionDelay(ms) {
137
+ if (!Number.isFinite(ms) || ms < 0) throw new Error("transaction delay must be non-negative");
138
+ this.transactionDelayMs = ms;
70
139
  }
71
140
  async getAlarm() {
72
141
  return this.alarmAt;
@@ -80,6 +149,24 @@ var FakeStorage = class {
80
149
  now() {
81
150
  return this.clock();
82
151
  }
152
+ resetOperationCounts() {
153
+ for (const key of Object.keys(this.operationCounts)) this.operationCounts[key] = 0;
154
+ this.getManyCalls.length = 0;
155
+ this.operationCalls.length = 0;
156
+ }
157
+ /** @internal Used by FakeTransaction to count the same platform operations. */
158
+ recordOperation(operation) {
159
+ this.operationCounts[operation] += 1;
160
+ }
161
+ /** @internal Shared with FakeTransaction for deterministic key tracing. */
162
+ recordStorageCall(operation, keys, transactional) {
163
+ this.operationCalls.push({
164
+ operation,
165
+ keys: [...keys],
166
+ transactional,
167
+ synchronous: false
168
+ });
169
+ }
83
170
  /** Inject one matching write failure, including writes inside transactions. */
84
171
  failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure")) {
85
172
  this.mutationFailure = {
@@ -94,6 +181,14 @@ var FakeStorage = class {
94
181
  this.mutationFailure = void 0;
95
182
  throw error;
96
183
  }
184
+ /** @internal Shared with FakeTransaction for deterministic query tracing. */
185
+ recordList(options, resultSize, transactional) {
186
+ this.listCalls.push({
187
+ options: { ...options },
188
+ resultSize,
189
+ transactional
190
+ });
191
+ }
97
192
  };
98
193
  var FakeTransaction = class {
99
194
  storage;
@@ -103,21 +198,64 @@ var FakeTransaction = class {
103
198
  constructor(storage) {
104
199
  this.storage = storage;
105
200
  }
106
- async get(key) {
201
+ async get(keyOrKeys) {
202
+ if (Array.isArray(keyOrKeys)) {
203
+ this.storage.recordOperation("getMany");
204
+ this.storage.getManyCalls.push([...keyOrKeys]);
205
+ this.storage.recordStorageCall("get", keyOrKeys, true);
206
+ const result = /* @__PURE__ */ new Map();
207
+ for (const key of keyOrKeys) {
208
+ if (this.deleted.has(key)) continue;
209
+ if (this.staged.has(key)) result.set(key, this.staged.get(key));
210
+ else if (this.storage.data.has(key)) result.set(key, this.storage.data.get(key));
211
+ }
212
+ return result;
213
+ }
214
+ this.storage.recordOperation("get");
215
+ const key = keyOrKeys;
216
+ this.storage.recordStorageCall("get", [key], true);
107
217
  if (this.deleted.has(key)) return void 0;
108
218
  if (this.staged.has(key)) return this.staged.get(key);
109
219
  return this.storage.data.get(key);
110
220
  }
111
- async put(key, value) {
112
- this.storage.maybeFailMutation({
221
+ async put(keyOrEntries, value) {
222
+ if (typeof keyOrEntries === "string") {
223
+ this.storage.recordOperation("put");
224
+ this.storage.maybeFailMutation({
225
+ operation: "put",
226
+ key: keyOrEntries,
227
+ value
228
+ });
229
+ this.storage.recordStorageCall("put", [keyOrEntries], true);
230
+ this.deleted.delete(keyOrEntries);
231
+ this.staged.set(keyOrEntries, structuredClone(value));
232
+ return;
233
+ }
234
+ this.storage.recordOperation("putMany");
235
+ for (const [key, entryValue] of Object.entries(keyOrEntries)) this.storage.maybeFailMutation({
113
236
  operation: "put",
114
237
  key,
115
- value
238
+ value: entryValue
116
239
  });
117
- this.deleted.delete(key);
118
- this.staged.set(key, structuredClone(value));
240
+ this.storage.recordStorageCall("put", Object.keys(keyOrEntries), true);
241
+ for (const [key, entryValue] of Object.entries(keyOrEntries)) {
242
+ this.deleted.delete(key);
243
+ this.staged.set(key, structuredClone(entryValue));
244
+ }
119
245
  }
120
- async delete(key) {
246
+ async delete(keyOrKeys) {
247
+ if (typeof keyOrKeys === "string") {
248
+ this.storage.recordOperation("delete");
249
+ this.storage.recordStorageCall("delete", [keyOrKeys], true);
250
+ return this.deleteOne(keyOrKeys);
251
+ }
252
+ this.storage.recordOperation("deleteMany");
253
+ this.storage.recordStorageCall("delete", keyOrKeys, true);
254
+ let deleted = 0;
255
+ for (const key of keyOrKeys) if (this.deleteOne(key)) deleted += 1;
256
+ return deleted;
257
+ }
258
+ deleteOne(key) {
121
259
  this.storage.maybeFailMutation({
122
260
  operation: "delete",
123
261
  key
@@ -128,10 +266,12 @@ var FakeTransaction = class {
128
266
  return existed;
129
267
  }
130
268
  async list(options = {}) {
269
+ this.storage.recordOperation("list");
131
270
  const merged = new Map(this.storage.data);
132
271
  for (const key of this.deleted) merged.delete(key);
133
272
  for (const [key, value] of this.staged) merged.set(key, value);
134
273
  const keys = applyListOptions(Array.from(merged.keys()), options);
274
+ this.storage.recordList(options, keys.length, true);
135
275
  return new Map(keys.map((k) => [k, merged.get(k)]));
136
276
  }
137
277
  async getAlarm() {
@@ -175,28 +315,31 @@ var FakeFleet = class {
175
315
  get: (id) => this.cell(bindingKey, id.toString()).instance
176
316
  };
177
317
  }
318
+ instantiate(bindingKey, name, storage, pendingWaits) {
319
+ const CellCtor = this.classes[bindingKey];
320
+ if (!CellCtor) throw new Error(`no cell class registered for binding: ${bindingKey}`);
321
+ return new CellCtor({
322
+ storage,
323
+ id: {
324
+ toString: () => name,
325
+ name
326
+ },
327
+ waitUntil(promise) {
328
+ pendingWaits.push(promise);
329
+ },
330
+ async blockConcurrencyWhile(cb) {
331
+ return cb();
332
+ }
333
+ }, this.cellEnv);
334
+ }
178
335
  cell(bindingKey, name) {
179
336
  const key = `${bindingKey}\0${name}`;
180
337
  let slot = this.cells.get(key);
181
338
  if (!slot) {
182
- const CellCtor = this.classes[bindingKey];
183
- if (!CellCtor) throw new Error(`no cell class registered for binding: ${bindingKey}`);
184
339
  const storage = new FakeStorage(() => this.now);
185
340
  const pendingWaits = [];
186
341
  slot = {
187
- instance: new CellCtor({
188
- storage,
189
- id: {
190
- toString: () => name,
191
- name
192
- },
193
- waitUntil(promise) {
194
- pendingWaits.push(promise);
195
- },
196
- async blockConcurrencyWhile(cb) {
197
- return cb();
198
- }
199
- }, this.cellEnv),
342
+ instance: this.instantiate(bindingKey, name, storage, pendingWaits),
200
343
  storage,
201
344
  pendingWaits,
202
345
  alarmRetryCount: 0,
@@ -206,6 +349,12 @@ var FakeFleet = class {
206
349
  }
207
350
  return slot;
208
351
  }
352
+ /** Recreate one in-memory DO instance while preserving its durable storage. */
353
+ restartCell(bindingKey, name) {
354
+ const slot = this.cell(bindingKey, name);
355
+ slot.instance = this.instantiate(bindingKey, name, slot.storage, slot.pendingWaits);
356
+ return slot.instance;
357
+ }
209
358
  advance(ms) {
210
359
  this.now += ms;
211
360
  }
@@ -292,13 +441,23 @@ async function authenticate(request, secret) {
292
441
  * Routes:
293
442
  * GET /v1/health — liveness + spec version (unauthenticated)
294
443
  * POST /v1/rpc/{binding}/{name}/{method} — DO RPC (bearer auth)
444
+ * POST /v1/index/{domain}/{method} — cohesive index operation
445
+ * GET /v1/streams/{name}/chunks — bounded binary long-poll read
446
+ * POST /v1/streams/{name}/chunks — bounded binary batch append
295
447
  *
296
- * The RPC body is an rpc-codec-encoded JSON array of arguments; the response
297
- * body is the rpc-codec-encoded return value. Only whitelisted methods
298
- * dispatch — everything else (including fetch/alarm) is unreachable.
448
+ * Generic RPC bodies use the tagged JSON codec. Stream chunk bodies use the
449
+ * compact binary stream protocol. Only whitelisted routes/methods dispatch.
299
450
  */
300
451
  const WORLD_NAME = "world-celld";
301
452
  const WORLD_VERSION = "0.1.0";
453
+ function workflowIndex(env) {
454
+ return createWorkflowIndex({
455
+ runCatalog: env.WORKFLOW_RUN_CATALOG,
456
+ runFences: env.WORKFLOW_RUN_FENCES,
457
+ hookTokens: env.WORKFLOW_HOOK_TOKENS,
458
+ hookIds: env.WORKFLOW_HOOK_IDS
459
+ });
460
+ }
302
461
  const BINDINGS = {
303
462
  runs: {
304
463
  env: "WORKFLOW_DB",
@@ -313,16 +472,15 @@ const BINDINGS = {
313
472
  "getCleanupStatus",
314
473
  "scheduleCleanup",
315
474
  "cleanupNow",
316
- "rearmCleanup"
475
+ "rearmCleanup",
476
+ "resolveHookTokenClaim"
317
477
  ])
318
478
  },
319
479
  streams: {
320
480
  env: "WORKFLOW_STREAMS",
321
481
  methods: /* @__PURE__ */ new Set([
322
- "writeChunk",
323
482
  "closeStream",
324
- "getChunks",
325
- "getInfo",
483
+ "failStream",
326
484
  "registerStream",
327
485
  "listStreams",
328
486
  "expireRegistry",
@@ -330,20 +488,13 @@ const BINDINGS = {
330
488
  "expireStream"
331
489
  ])
332
490
  },
333
- index: {
334
- env: "WORKFLOW_INDEX",
335
- methods: /* @__PURE__ */ new Set([
336
- "get",
337
- "put",
338
- "delete",
339
- "putOwned",
340
- "expireRun",
341
- "list",
342
- "reserveHookToken",
343
- "finalizeHookIndexes",
344
- "releaseHookToken",
345
- "deleteHookIndexes"
346
- ])
491
+ "hook-tokens": {
492
+ env: "WORKFLOW_HOOK_TOKENS",
493
+ methods: /* @__PURE__ */ new Set(["get"])
494
+ },
495
+ "hook-ids": {
496
+ env: "WORKFLOW_HOOK_IDS",
497
+ methods: /* @__PURE__ */ new Set(["get"])
347
498
  },
348
499
  queue: {
349
500
  env: "WORKFLOW_QUEUE",
@@ -360,6 +511,7 @@ const BINDINGS = {
360
511
  };
361
512
  /** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
362
513
  const MAX_BODY_BYTES = 33554432;
514
+ const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 1024;
363
515
  function errorResponse(status, name, message) {
364
516
  return Response.json({ error: {
365
517
  name,
@@ -388,6 +540,49 @@ async function readBoundedBody(request) {
388
540
  reader.releaseLock();
389
541
  }
390
542
  }
543
+ async function readBoundedBytes(request, maxBytes) {
544
+ if (!request.body) return /* @__PURE__ */ new Uint8Array();
545
+ const reader = request.body.getReader();
546
+ const chunks = [];
547
+ let total = 0;
548
+ try {
549
+ for (;;) {
550
+ const { done, value } = await reader.read();
551
+ if (done) break;
552
+ total += value.byteLength;
553
+ if (total > maxBytes) {
554
+ await reader.cancel("stream body exceeds configured limit").catch(() => void 0);
555
+ return null;
556
+ }
557
+ chunks.push(value);
558
+ }
559
+ } finally {
560
+ reader.releaseLock();
561
+ }
562
+ const body = new Uint8Array(total);
563
+ let offset = 0;
564
+ for (const chunk of chunks) {
565
+ body.set(chunk, offset);
566
+ offset += chunk.byteLength;
567
+ }
568
+ return body;
569
+ }
570
+ function parseBoundedInteger(url, name, minimum, maximum) {
571
+ const raw = url.searchParams.get(name);
572
+ const value = raw === null ? NaN : Number(raw);
573
+ if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
574
+ const error = /* @__PURE__ */ new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
575
+ error.name = "StreamProtocolError";
576
+ throw error;
577
+ }
578
+ return value;
579
+ }
580
+ function streamResponse(body) {
581
+ return new Response(body, {
582
+ status: 200,
583
+ headers: { "content-type": STREAM_BATCH_CONTENT_TYPE }
584
+ });
585
+ }
391
586
  function createRouter(env) {
392
587
  return async (request) => {
393
588
  const url = new URL(request.url);
@@ -399,9 +594,83 @@ function createRouter(env) {
399
594
  version: WORLD_VERSION,
400
595
  specVersion: SPEC_VERSION_CURRENT
401
596
  });
402
- if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
403
597
  const auth = await authenticate(request, env.WORLD_SECRET);
404
598
  if (!auth.ok) return auth.response;
599
+ if (parts[1] === "index" && parts.length === 4) {
600
+ if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
601
+ if (Number(request.headers.get("content-length") ?? "0") > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
602
+ let args;
603
+ try {
604
+ const text = await readBoundedBody(request);
605
+ if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
606
+ const parsed = rpcParse(text);
607
+ if (!Array.isArray(parsed)) return errorResponse(400, "BadRequest", "body must be an argument array");
608
+ args = parsed;
609
+ } catch {
610
+ return errorResponse(400, "BadRequest", "malformed rpc body");
611
+ }
612
+ try {
613
+ const index = workflowIndex(env);
614
+ let result;
615
+ const operation = `${parts[2]}.${parts[3]}`;
616
+ if (operation === "runs.list") result = await index.listRuns(args[0]);
617
+ else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
618
+ else if (operation === "runs.expire") result = await index.expireRun(args[0]);
619
+ else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
620
+ else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
621
+ else if (operation === "hooks.release-reservation") result = await index.releaseHookReservation(args[0], args[1], args[2]);
622
+ else if (operation === "hooks.release") result = await index.releaseHookIndexes(args[0]);
623
+ else return errorResponse(404, "NotFound", `unknown index operation: ${operation}`);
624
+ return new Response(rpcStringify(result ?? null), {
625
+ status: 200,
626
+ headers: { "content-type": "application/json" }
627
+ });
628
+ } catch (error) {
629
+ const err = error;
630
+ return Response.json({ error: {
631
+ name: err?.name ?? "Error",
632
+ message: err?.message ?? String(error),
633
+ status: typeof err?.status === "number" ? err.status : void 0
634
+ } }, { status: 500 });
635
+ }
636
+ }
637
+ if (parts[1] === "streams" && parts.length === 4 && parts[3] === "chunks") {
638
+ const namespace = env.WORKFLOW_STREAMS;
639
+ if (!namespace) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_STREAMS");
640
+ const name = decodeURIComponent(parts[2]);
641
+ const runId = url.searchParams.get("runId");
642
+ if (!runId) return errorResponse(400, "BadRequest", "runId is required");
643
+ const stub = namespace.get(namespace.idFromName(name));
644
+ try {
645
+ if (request.method === "POST") {
646
+ if (!request.headers.get("content-type")?.startsWith("application/vnd.world-celld.stream-batch.v1")) return errorResponse(415, "UnsupportedMediaType", STREAM_BATCH_CONTENT_TYPE);
647
+ if (Number(request.headers.get("content-length") ?? "0") > MAX_STREAM_WRITE_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
648
+ const body = await readBoundedBytes(request, MAX_STREAM_WRITE_BODY_BYTES);
649
+ if (body === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
650
+ const result = await stub.writeChunks(runId, decodeStreamWriteBatch(body));
651
+ return streamResponse(encodeStreamWriteResult(result));
652
+ }
653
+ if (request.method === "GET") {
654
+ const result = await stub.readChunks({
655
+ runId,
656
+ startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
657
+ maxChunks: parseBoundedInteger(url, "maxChunks", 0, 32),
658
+ maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
659
+ waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
660
+ }, request.signal);
661
+ return streamResponse(encodeStreamReadResult(result));
662
+ }
663
+ return errorResponse(405, "MethodNotAllowed", "expected GET or POST");
664
+ } catch (error) {
665
+ const err = error;
666
+ return Response.json({ error: {
667
+ name: err?.name ?? "Error",
668
+ message: err?.message ?? String(error),
669
+ status: typeof err?.status === "number" ? err.status : void 0
670
+ } }, { status: err?.name === "StreamProtocolError" ? 400 : 500 });
671
+ }
672
+ }
673
+ if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
405
674
  if (request.method !== "POST" || parts.length !== 5) return errorResponse(404, "NotFound", "expected POST /v1/rpc/{binding}/{name}/{method}");
406
675
  const [, , bindingKey, encodedName, method] = parts;
407
676
  const binding = BINDINGS[bindingKey];
@@ -457,160 +726,242 @@ var DurableObject = class {
457
726
  }
458
727
  };
459
728
  //#endregion
460
- //#region src/worker/durable-objects/IndexDO.ts
729
+ //#region src/worker/durable-objects/hook-index-shared.ts
461
730
  function sameOwner(left, right) {
462
731
  return left.runId === right.runId && left.hookId === right.hookId;
463
732
  }
464
- function ownerFromHook(raw) {
733
+ function ownerFromSerializedHook(raw) {
465
734
  const hook = parse(raw);
466
735
  return {
467
736
  runId: hook.runId,
468
737
  hookId: hook.hookId
469
738
  };
470
739
  }
471
- /**
472
- * Global index cell, replacing the Workers KV namespace world-cloudflare
473
- * uses for run listing and hook token/id lookup.
474
- *
475
- * A single cell (named `index`) serializes every read and write. Hook-token
476
- * reservation, publication, and release use storage transactions so
477
- * concurrent runs cannot both acquire the same token.
478
- *
479
- * The RPC surface matches the storage-layer KVNamespace interface bit-for-bit
480
- * (including MockKVNamespace's pagination contract: cursor = last returned
481
- * key, `list_complete` reflects whether further keys exist), so the vendored
482
- * storage.ts works against it unchanged.
483
- */
484
- var IndexDO = class extends DurableObject {
485
- async get(key) {
486
- return await this.ctx.storage.get(key) ?? null;
487
- }
488
- async put(key, value) {
489
- await this.ctx.storage.put(key, value);
740
+ function indexKey(prefix, value) {
741
+ return `${prefix}:${encodeURIComponent(value)}`;
742
+ }
743
+ /** Compact shard-local fence for every hook owned by one terminal or expired run. */
744
+ function shardRunFenceKey(runId) {
745
+ return `runfence:${encodeURIComponent(runId)}`;
746
+ }
747
+ async function runIsActive(env, runId) {
748
+ const namespace = env.WORKFLOW_RUN_FENCES;
749
+ if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
750
+ return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
751
+ }
752
+ //#endregion
753
+ //#region src/worker/durable-objects/HookIdDO.ts
754
+ const MAX_RELEASE_BATCH$1 = 64;
755
+ function recordKey$1(hookId) {
756
+ return indexKey("hookid", hookId);
757
+ }
758
+ function claimKey$1(hookId) {
759
+ return indexKey("claim", hookId);
760
+ }
761
+ /** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
762
+ var HookIdDO = class extends DurableObject {
763
+ async get(hookId) {
764
+ const raw = await this.ctx.storage.get(recordKey$1(hookId));
765
+ if (!raw) return null;
766
+ const owner = ownerFromSerializedHook(raw);
767
+ return await runIsActive(this.env, owner.runId) ? raw : null;
768
+ }
769
+ async reserve(hookId, owner, proposedClaimId) {
770
+ return await this.ctx.storage.transaction(async (txn) => {
771
+ const record = recordKey$1(hookId);
772
+ const claim = claimKey$1(hookId);
773
+ const values = await txn.get([record, claim]);
774
+ const indexed = values.get(record);
775
+ if (indexed !== void 0) return {
776
+ claimed: false,
777
+ holder: ownerFromSerializedHook(indexed)
778
+ };
779
+ const existing = values.get(claim);
780
+ if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
781
+ claimed: false,
782
+ holder: existing.owner
783
+ };
784
+ if (existing === void 0 || existing.claimId !== proposedClaimId) {
785
+ await txn.put(claim, {
786
+ owner,
787
+ claimId: proposedClaimId,
788
+ reservedAt: Date.now()
789
+ });
790
+ return {
791
+ claimed: true,
792
+ claimId: proposedClaimId
793
+ };
794
+ }
795
+ return {
796
+ claimed: true,
797
+ claimId: existing.claimId
798
+ };
799
+ });
490
800
  }
491
- /** Publish a derived index only while its owning run is not expired. */
492
- async putOwned(runId, key, value) {
801
+ async publish(hookId, serializedHook, owner, claimId) {
802
+ if (!await runIsActive(this.env, owner.runId)) {
803
+ await this.releaseBatch([{
804
+ hookId,
805
+ owner
806
+ }], Date.now(), true);
807
+ return { stored: false };
808
+ }
493
809
  return await this.ctx.storage.transaction(async (txn) => {
494
- if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
495
- await txn.put(key, value);
810
+ const record = recordKey$1(hookId);
811
+ const claim = claimKey$1(hookId);
812
+ const runFence = shardRunFenceKey(owner.runId);
813
+ const values = await txn.get([
814
+ record,
815
+ claim,
816
+ runFence
817
+ ]);
818
+ if (values.has(runFence)) {
819
+ const existingClaim = values.get(claim);
820
+ if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
821
+ return { stored: false };
822
+ }
823
+ const indexed = values.get(record);
824
+ if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
825
+ const existingClaim = values.get(claim);
826
+ if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
827
+ if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
828
+ await txn.put(record, serializedHook);
829
+ if (existingClaim !== void 0) await txn.delete(claim);
496
830
  return { stored: true };
497
831
  });
498
832
  }
499
- async delete(key) {
500
- await this.ctx.storage.delete(key);
833
+ async releaseClaim(hookId, owner, claimId) {
834
+ await this.ctx.storage.transaction(async (txn) => {
835
+ const key = claimKey$1(hookId);
836
+ const claim = await txn.get(key);
837
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
838
+ });
501
839
  }
502
- async list(options) {
503
- const prefix = options?.prefix ?? "";
504
- const limit = options?.limit ?? 1e3;
505
- const entries = await this.ctx.storage.list({
506
- prefix,
507
- ...options?.reverse ? {
508
- reverse: true,
509
- end: options.cursor
510
- } : { startAfter: options?.cursor },
511
- limit: limit + 1
840
+ async releaseBatch(hooks, fencedAt, runFenced) {
841
+ if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
842
+ if (hooks.length === 0) return { deleted: 0 };
843
+ return await this.ctx.storage.transaction(async (txn) => {
844
+ const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
845
+ const values = await txn.get(keys);
846
+ const deletes = [];
847
+ for (const { hookId, owner } of hooks) {
848
+ const record = recordKey$1(hookId);
849
+ const claim = claimKey$1(hookId);
850
+ const indexed = values.get(record);
851
+ const existingClaim = values.get(claim);
852
+ if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
853
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
854
+ }
855
+ if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
856
+ return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
512
857
  });
513
- const names = Array.from(entries.keys());
514
- const page = names.slice(0, limit);
515
- const listComplete = names.length <= limit;
516
- return {
517
- keys: page.map((name) => ({ name })),
518
- list_complete: listComplete,
519
- cursor: listComplete ? void 0 : page.at(-1)
520
- };
521
858
  }
522
- async reserveHookToken(token, owner) {
523
- return this.ctx.storage.transaction(async (txn) => {
524
- const indexedHook = await txn.get(`hook:${token}`);
525
- if (indexedHook !== void 0) return {
859
+ };
860
+ //#endregion
861
+ //#region src/worker/durable-objects/HookTokenDO.ts
862
+ const MAX_RELEASE_BATCH = 64;
863
+ function recordKey(token) {
864
+ return indexKey("hook", token);
865
+ }
866
+ function claimKey(token) {
867
+ return indexKey("claim", token);
868
+ }
869
+ /** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
870
+ var HookTokenDO = class extends DurableObject {
871
+ async get(token) {
872
+ const raw = await this.ctx.storage.get(recordKey(token));
873
+ if (!raw) return null;
874
+ const owner = ownerFromSerializedHook(raw);
875
+ return await runIsActive(this.env, owner.runId) ? raw : null;
876
+ }
877
+ async reserve(token, owner, proposedClaimId) {
878
+ return await this.ctx.storage.transaction(async (txn) => {
879
+ const record = recordKey(token);
880
+ const claim = claimKey(token);
881
+ const values = await txn.get([record, claim]);
882
+ const indexed = values.get(record);
883
+ if (indexed !== void 0) return {
526
884
  claimed: false,
527
- holder: ownerFromHook(indexedHook)
885
+ holder: ownerFromSerializedHook(indexed)
528
886
  };
529
- const claimKey = `hookclaim:${token}`;
530
- const claim = await txn.get(claimKey);
531
- if (claim !== void 0 && !sameOwner(claim.owner, owner)) return {
887
+ const existing = values.get(claim);
888
+ if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
532
889
  claimed: false,
533
- holder: claim.owner
890
+ holder: existing.owner
891
+ };
892
+ if (existing === void 0) {
893
+ await txn.put(claim, {
894
+ owner,
895
+ claimId: proposedClaimId,
896
+ reservedAt: Date.now()
897
+ });
898
+ return {
899
+ claimed: true,
900
+ claimId: proposedClaimId
901
+ };
902
+ }
903
+ return {
904
+ claimed: true,
905
+ claimId: existing.claimId
534
906
  };
535
- await txn.put(claimKey, {
536
- owner,
537
- reservedAt: Date.now()
538
- });
539
- return { claimed: true };
540
907
  });
541
908
  }
542
- async finalizeHookIndexes(token, hookId, serializedHook, owner) {
543
- await this.ctx.storage.transaction(async (txn) => {
544
- if (await txn.get(`expired:${owner.runId}`) !== void 0) {
545
- const claim = await txn.get(`hookclaim:${token}`);
546
- if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(`hookclaim:${token}`);
547
- return;
909
+ async finalize(token, serializedHook, owner, claimId) {
910
+ if (!await runIsActive(this.env, owner.runId)) {
911
+ await this.releaseBatch([{
912
+ token,
913
+ owner
914
+ }], Date.now(), true);
915
+ return { stored: false };
916
+ }
917
+ return await this.ctx.storage.transaction(async (txn) => {
918
+ const record = recordKey(token);
919
+ const claim = claimKey(token);
920
+ const runFence = shardRunFenceKey(owner.runId);
921
+ const values = await txn.get([
922
+ record,
923
+ claim,
924
+ runFence
925
+ ]);
926
+ if (values.has(runFence)) {
927
+ const existingClaim = values.get(claim);
928
+ if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
929
+ return { stored: false };
548
930
  }
549
- const indexedHook = await txn.get(`hook:${token}`);
550
- if (indexedHook !== void 0 && !sameOwner(ownerFromHook(indexedHook), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
551
- const claim = await txn.get(`hookclaim:${token}`);
552
- if (claim !== void 0 && !sameOwner(claim.owner, owner)) throw new Error(`Hook token ${token} is reserved by another hook`);
553
- await txn.put(`hook:${token}`, serializedHook);
554
- await txn.put(`hookid:${hookId}`, serializedHook);
555
- if (claim !== void 0) await txn.delete(`hookclaim:${token}`);
931
+ const indexed = values.get(record);
932
+ if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
933
+ const existingClaim = values.get(claim);
934
+ if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
935
+ if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
936
+ await txn.put(record, serializedHook);
937
+ if (existingClaim !== void 0) await txn.delete(claim);
938
+ return { stored: true };
556
939
  });
557
940
  }
558
- async releaseHookToken(token, owner) {
941
+ async releaseClaim(token, owner, claimId) {
559
942
  await this.ctx.storage.transaction(async (txn) => {
560
- const key = `hookclaim:${token}`;
943
+ const key = claimKey(token);
561
944
  const claim = await txn.get(key);
562
- if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(key);
945
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
563
946
  });
564
947
  }
565
- async deleteHookIndexes(token, hookId, owner) {
566
- await this.ctx.storage.transaction(async (txn) => {
567
- const tokenKey = `hook:${token}`;
568
- const idKey = `hookid:${hookId}`;
569
- const claimKey = `hookclaim:${token}`;
570
- const [byToken, byId, claim] = await Promise.all([
571
- txn.get(tokenKey),
572
- txn.get(idKey),
573
- txn.get(claimKey)
574
- ]);
575
- if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) await txn.delete(tokenKey);
576
- if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) await txn.delete(idKey);
577
- if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
578
- });
579
- }
580
- /**
581
- * Fence a run and remove all of its known derived indexes atomically.
582
- * Once the fence exists, putOwned() and hook finalization cannot resurrect
583
- * entries from a delayed request.
584
- */
585
- async expireRun(request) {
948
+ async releaseBatch(hooks, fencedAt, runFenced) {
949
+ if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
950
+ if (hooks.length === 0) return { deleted: 0 };
586
951
  return await this.ctx.storage.transaction(async (txn) => {
587
- await txn.put(`expired:${request.runId}`, request.expiredAt);
588
- let deleted = 0;
589
- for (const key of new Set(request.keys)) if (await txn.delete(key)) deleted++;
590
- for (const hook of request.hooks) {
591
- const owner = {
592
- runId: request.runId,
593
- hookId: hook.hookId
594
- };
595
- const tokenKey = `hook:${hook.token}`;
596
- const idKey = `hookid:${hook.hookId}`;
597
- const claimKey = `hookclaim:${hook.token}`;
598
- const [byToken, byId, claim] = await Promise.all([
599
- txn.get(tokenKey),
600
- txn.get(idKey),
601
- txn.get(claimKey)
602
- ]);
603
- if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
604
- if (await txn.delete(tokenKey)) deleted++;
605
- }
606
- if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
607
- if (await txn.delete(idKey)) deleted++;
608
- }
609
- if (claim !== void 0 && sameOwner(claim.owner, owner)) {
610
- if (await txn.delete(claimKey)) deleted++;
611
- }
952
+ const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
953
+ const values = await txn.get(keys);
954
+ const deletes = [];
955
+ for (const { token, owner } of hooks) {
956
+ const record = recordKey(token);
957
+ const claim = claimKey(token);
958
+ const indexed = values.get(record);
959
+ const existingClaim = values.get(claim);
960
+ if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
961
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
612
962
  }
613
- return { deleted };
963
+ if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
964
+ return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
614
965
  });
615
966
  }
616
967
  };
@@ -629,21 +980,61 @@ const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
629
980
  const INFLIGHT_GRACE_MS = 3e4;
630
981
  /** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
631
982
  const MIN_ALARM_DELAY_MS = 1;
983
+ /** Bound expiry recovery so one alarm cannot monopolize the cell. */
984
+ const EXPIRED_INFLIGHT_BATCH = 128;
985
+ const STORAGE_BATCH_SIZE$2 = 128;
986
+ const EXPIRE_RUN_REFERENCE_BATCH = 64;
987
+ const PURGE_DEAD_LETTER_BATCH = 128;
988
+ const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
632
989
  function pad(ms) {
633
990
  return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
634
991
  }
635
992
  function dueKey(atMs, messageId) {
636
993
  return `due:${pad(atMs)}:${messageId}`;
637
994
  }
995
+ function inflightKey(deadlineMs, messageId) {
996
+ return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
997
+ }
998
+ function deadlineFromInflightKey(key) {
999
+ return Number.parseInt(key.slice(18, 31), 10);
1000
+ }
638
1001
  function runReferenceKey(runId, messageId) {
639
1002
  return `run:${runId}:${messageId}`;
640
1003
  }
641
1004
  function expiredRunKey(runId) {
642
1005
  return `expired-run:${runId}`;
643
1006
  }
1007
+ function boundedLimit$1(value, fallback, maximum) {
1008
+ if (value === void 0 || !Number.isFinite(value)) return fallback;
1009
+ return Math.max(1, Math.min(maximum, Math.floor(value)));
1010
+ }
1011
+ async function getMany$1(storage, keys) {
1012
+ const result = /* @__PURE__ */ new Map();
1013
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$2) {
1014
+ const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$2));
1015
+ for (const [key, value] of page) result.set(key, value);
1016
+ }
1017
+ return result;
1018
+ }
1019
+ async function putMany(storage, entries) {
1020
+ const all = Array.from(entries);
1021
+ for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$2) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$2)));
1022
+ }
1023
+ async function deleteMany$1(storage, keys) {
1024
+ let deleted = 0;
1025
+ const unique = Array.from(new Set(keys));
1026
+ for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$2) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$2));
1027
+ return deleted;
1028
+ }
644
1029
  function backoffSeconds(attempt) {
645
1030
  return Math.min(60, 2 ** attempt);
646
1031
  }
1032
+ async function cancelUnusedResponseBody(response) {
1033
+ try {
1034
+ await response.body?.cancel();
1035
+ } catch {}
1036
+ }
1037
+ const MAX_QUEUE_INFLIGHT = 128;
647
1038
  var QueueDO = class extends DurableObject {
648
1039
  #now() {
649
1040
  const clock = this.env?.clock;
@@ -658,7 +1049,13 @@ var QueueDO = class extends DurableObject {
658
1049
  const parsed = raw ? Number.parseInt(raw, 10) : NaN;
659
1050
  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
660
1051
  }
1052
+ #maxInflight() {
1053
+ const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
1054
+ if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
1055
+ return configured;
1056
+ }
661
1057
  async enqueue(request) {
1058
+ this.#maxInflight();
662
1059
  const storage = this.ctx.storage;
663
1060
  const now = this.#now();
664
1061
  return await storage.transaction(async (txn) => {
@@ -733,7 +1130,7 @@ var QueueDO = class extends DurableObject {
733
1130
  if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
734
1131
  else await attempt();
735
1132
  if (failure !== null) throw failure;
736
- for (const row of claimed) this.ctx.waitUntil(this.#deliver(row));
1133
+ for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
737
1134
  }
738
1135
  /**
739
1136
  * Storage-only: recover lost inflight claims, claim due messages up to the
@@ -742,24 +1139,38 @@ var QueueDO = class extends DurableObject {
742
1139
  async #claimPhase() {
743
1140
  const storage = this.ctx.storage;
744
1141
  const now = this.#now();
745
- const maxInflight = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
1142
+ const maxInflight = this.#maxInflight();
746
1143
  return await storage.transaction(async (txn) => {
747
- const inflight = await txn.list({ prefix: "inflight:" });
748
- let inflightCount = 0;
749
- for (const [key, deadline] of inflight) if (deadline <= now) {
750
- const messageId = key.slice(9);
751
- await txn.delete(key);
752
- const row = await txn.get(`msg:${messageId}`);
1144
+ const expiredPage = await txn.list({
1145
+ prefix: INFLIGHT_DEADLINE_PREFIX,
1146
+ end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
1147
+ limit: 129
1148
+ });
1149
+ const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
1150
+ const expiredRows = await getMany$1(txn, expired.map(([, messageId]) => `msg:${messageId}`));
1151
+ await deleteMany$1(txn, expired.map(([key]) => key));
1152
+ const recoveryWrites = [];
1153
+ for (const [, messageId] of expired) {
1154
+ const row = expiredRows.get(`msg:${messageId}`);
753
1155
  if (row) {
754
1156
  const scheduleKey = dueKey(now, messageId);
755
- await txn.put(scheduleKey, messageId);
756
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1157
+ recoveryWrites.push([scheduleKey, messageId]);
1158
+ if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
757
1159
  messageId: row.messageId,
758
1160
  dueKey: scheduleKey,
759
1161
  idempotencyKey: row.idempotencyKey
760
- });
1162
+ }]);
761
1163
  }
762
- } else inflightCount++;
1164
+ }
1165
+ await putMany(txn, recoveryWrites);
1166
+ if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
1167
+ await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
1168
+ return [];
1169
+ }
1170
+ const inflightCount = (await txn.list({
1171
+ prefix: INFLIGHT_DEADLINE_PREFIX,
1172
+ limit: maxInflight
1173
+ })).size;
763
1174
  const rows = [];
764
1175
  if (inflightCount < maxInflight) {
765
1176
  const due = await txn.list({
@@ -768,19 +1179,31 @@ var QueueDO = class extends DurableObject {
768
1179
  limit: maxInflight - inflightCount
769
1180
  });
770
1181
  const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
771
- for (const [key, messageId] of due) {
772
- await txn.delete(key);
773
- const row = await txn.get(`msg:${messageId}`);
1182
+ const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
1183
+ await deleteMany$1(txn, Array.from(due.keys()));
1184
+ const claimWrites = [];
1185
+ for (const [, messageId] of due) {
1186
+ const row = dueRows.get(`msg:${messageId}`);
774
1187
  if (!row) continue;
775
- await txn.put(`inflight:${messageId}`, now + timeoutMs + INFLIGHT_GRACE_MS);
776
- rows.push(row);
1188
+ const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
1189
+ claimWrites.push([claimKey, messageId]);
1190
+ if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
1191
+ messageId: row.messageId,
1192
+ inflightKey: claimKey,
1193
+ idempotencyKey: row.idempotencyKey
1194
+ }]);
1195
+ rows.push({
1196
+ row,
1197
+ inflightKey: claimKey
1198
+ });
777
1199
  }
1200
+ await putMany(txn, claimWrites);
778
1201
  }
779
1202
  await this.#scheduleNextAlarm(txn, now);
780
1203
  return rows;
781
1204
  });
782
1205
  }
783
- async #deliver(row) {
1206
+ async #deliver({ row, inflightKey: claimKey }) {
784
1207
  const storage = this.ctx.storage;
785
1208
  const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
786
1209
  const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
@@ -804,8 +1227,10 @@ var QueueDO = class extends DurableObject {
804
1227
  transportError = error;
805
1228
  }
806
1229
  const now = this.#now();
807
- if (response?.ok) await this.#ack(row);
808
- else if (response && response.status === 503) {
1230
+ if (response?.ok) {
1231
+ await cancelUnusedResponseBody(response);
1232
+ await this.#ack(row, claimKey);
1233
+ } else if (response && response.status === 503) {
809
1234
  let timeoutSeconds;
810
1235
  try {
811
1236
  timeoutSeconds = (await response.json())?.timeoutSeconds;
@@ -813,9 +1238,9 @@ var QueueDO = class extends DurableObject {
813
1238
  timeoutSeconds = void 0;
814
1239
  }
815
1240
  if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
1241
+ if (!await txn.delete(claimKey)) return;
816
1242
  if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
817
1243
  await txn.delete(`msg:${row.messageId}`);
818
- await txn.delete(`inflight:${row.messageId}`);
819
1244
  await txn.delete(runReferenceKey(row.runId, row.messageId));
820
1245
  if (row.idempotencyKey) {
821
1246
  const dedupKey = `key:${row.idempotencyKey}`;
@@ -824,7 +1249,6 @@ var QueueDO = class extends DurableObject {
824
1249
  await this.#scheduleNextAlarm(txn, now);
825
1250
  return;
826
1251
  }
827
- await txn.delete(`inflight:${row.messageId}`);
828
1252
  const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
829
1253
  await txn.put(scheduleKey, row.messageId);
830
1254
  if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
@@ -834,17 +1258,20 @@ var QueueDO = class extends DurableObject {
834
1258
  });
835
1259
  await this.#scheduleNextAlarm(txn, now);
836
1260
  });
837
- else await this.#retry(row, "HTTP 503");
838
- } else if (response && PERMANENT_STATUSES.has(response.status)) await this.#ack(row);
839
- else {
1261
+ else await this.#retry(row, claimKey, "HTTP 503");
1262
+ } else if (response && PERMANENT_STATUSES.has(response.status)) {
1263
+ await cancelUnusedResponseBody(response);
1264
+ await this.#ack(row, claimKey);
1265
+ } else {
840
1266
  const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
841
- await this.#retry(row, reason);
1267
+ if (response) await cancelUnusedResponseBody(response);
1268
+ await this.#retry(row, claimKey, reason);
842
1269
  }
843
1270
  }
844
- async #ack(row) {
1271
+ async #ack(row, claimKey) {
845
1272
  await this.ctx.storage.transaction(async (txn) => {
1273
+ if (!await txn.delete(claimKey)) return;
846
1274
  await txn.delete(`msg:${row.messageId}`);
847
- await txn.delete(`inflight:${row.messageId}`);
848
1275
  if (row.idempotencyKey) {
849
1276
  if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
850
1277
  }
@@ -852,15 +1279,15 @@ var QueueDO = class extends DurableObject {
852
1279
  await this.#scheduleNextAlarm(txn, this.#now());
853
1280
  });
854
1281
  }
855
- async #retry(row, reason) {
1282
+ async #retry(row, claimKey, reason) {
856
1283
  const storage = this.ctx.storage;
857
1284
  const now = this.#now();
858
1285
  const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
859
1286
  const attempt = row.attempt + 1;
860
1287
  await storage.transaction(async (txn) => {
1288
+ if (!await txn.delete(claimKey)) return;
861
1289
  if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
862
1290
  await txn.delete(`msg:${row.messageId}`);
863
- await txn.delete(`inflight:${row.messageId}`);
864
1291
  await txn.delete(runReferenceKey(row.runId, row.messageId));
865
1292
  if (row.idempotencyKey) {
866
1293
  const dedupKey = `key:${row.idempotencyKey}`;
@@ -869,7 +1296,6 @@ var QueueDO = class extends DurableObject {
869
1296
  await this.#scheduleNextAlarm(txn, now);
870
1297
  return;
871
1298
  }
872
- await txn.delete(`inflight:${row.messageId}`);
873
1299
  if (attempt >= maxAttempts) {
874
1300
  const dead = {
875
1301
  ...row,
@@ -929,8 +1355,14 @@ var QueueDO = class extends DurableObject {
929
1355
  limit: 1
930
1356
  });
931
1357
  for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
932
- const inflight = await storage.list({ prefix: "inflight:" });
933
- for (const deadline of inflight.values()) if (next === null || deadline < next) next = deadline;
1358
+ const inflight = await storage.list({
1359
+ prefix: INFLIGHT_DEADLINE_PREFIX,
1360
+ limit: 1
1361
+ });
1362
+ for (const key of inflight.keys()) {
1363
+ const deadline = deadlineFromInflightKey(key);
1364
+ if (next === null || deadline < next) next = deadline;
1365
+ }
934
1366
  if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
935
1367
  else await storage.deleteAlarm();
936
1368
  }
@@ -938,7 +1370,7 @@ var QueueDO = class extends DurableObject {
938
1370
  const storage = this.ctx.storage;
939
1371
  const [due, inflight, dlq, alarmAt] = await Promise.all([
940
1372
  storage.list({ prefix: "due:" }),
941
- storage.list({ prefix: "inflight:" }),
1373
+ storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
942
1374
  storage.list({ prefix: "dlq:" }),
943
1375
  storage.getAlarm()
944
1376
  ]);
@@ -1003,13 +1435,16 @@ var QueueDO = class extends DurableObject {
1003
1435
  }
1004
1436
  async purgeDeadLetters() {
1005
1437
  return await this.ctx.storage.transaction(async (txn) => {
1006
- const dlq = await txn.list({ prefix: "dlq:" });
1007
- for (const [key, value] of dlq) {
1008
- await txn.delete(key);
1009
- const dead = value;
1010
- if (dead.runId) await txn.delete(runReferenceKey(dead.runId, dead.messageId));
1011
- }
1012
- return { purged: dlq.size };
1438
+ const entries = await txn.list({
1439
+ prefix: "dlq:",
1440
+ limit: 129
1441
+ });
1442
+ const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
1443
+ await deleteMany$1(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
1444
+ return {
1445
+ purged: page.length,
1446
+ hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
1447
+ };
1013
1448
  });
1014
1449
  }
1015
1450
  /**
@@ -1021,140 +1456,384 @@ var QueueDO = class extends DurableObject {
1021
1456
  return { alarmAt: await this.ctx.storage.getAlarm() };
1022
1457
  }
1023
1458
  /** Fence a run and purge every live, inflight, and dead-letter message it owns. */
1024
- async expireRun(runId, expiredAt) {
1025
- return { deleted: await this.ctx.storage.transaction(async (txn) => {
1459
+ async expireRun(runId, expiredAt, options) {
1460
+ const storage = this.ctx.storage;
1461
+ const limit = boundedLimit$1(options?.limit, EXPIRE_RUN_REFERENCE_BATCH, EXPIRE_RUN_REFERENCE_BATCH);
1462
+ return await storage.transaction(async (txn) => {
1026
1463
  const existing = await txn.get(expiredRunKey(runId));
1027
- const references = await txn.list({ prefix: `run:${runId}:` });
1464
+ const references = await txn.list({
1465
+ prefix: `run:${runId}:`,
1466
+ limit: limit + 1
1467
+ });
1468
+ const page = Array.from(references).slice(0, limit);
1469
+ const dedupHolders = await getMany$1(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
1470
+ const keys = [];
1028
1471
  let count = existing?.deleted ?? 0;
1029
- for (const [key, reference] of references) {
1030
- await txn.delete(`msg:${reference.messageId}`);
1031
- await txn.delete(`inflight:${reference.messageId}`);
1032
- if (reference.dueKey) await txn.delete(reference.dueKey);
1033
- if (reference.dlqKey) await txn.delete(reference.dlqKey);
1472
+ for (const [key, reference] of page) {
1473
+ keys.push(`msg:${reference.messageId}`, key);
1474
+ if (reference.inflightKey) keys.push(reference.inflightKey);
1475
+ if (reference.dueKey) keys.push(reference.dueKey);
1476
+ if (reference.dlqKey) keys.push(reference.dlqKey);
1034
1477
  if (reference.idempotencyKey) {
1035
1478
  const dedupKey = `key:${reference.idempotencyKey}`;
1036
- if (await txn.get(dedupKey) === reference.messageId) await txn.delete(dedupKey);
1479
+ if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
1037
1480
  }
1038
- await txn.delete(key);
1039
1481
  count++;
1040
1482
  }
1483
+ await deleteMany$1(txn, keys);
1041
1484
  await txn.put(expiredRunKey(runId), {
1042
1485
  expiredAt,
1043
1486
  deleted: count
1044
1487
  });
1045
1488
  await this.#scheduleNextAlarm(txn, this.#now());
1046
- return count;
1047
- }) };
1489
+ return {
1490
+ deleted: count,
1491
+ done: references.size <= limit
1492
+ };
1493
+ });
1494
+ }
1495
+ };
1496
+ //#endregion
1497
+ //#region src/worker/durable-objects/RunCatalogDO.ts
1498
+ const MAX_INDEX_LIST_SIZE = 1001;
1499
+ /** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
1500
+ var RunCatalogDO = class extends DurableObject {
1501
+ async upsertRun(runId, keys, serializedMetadata) {
1502
+ if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
1503
+ return await this.ctx.storage.transaction(async (txn) => {
1504
+ if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
1505
+ await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
1506
+ return { stored: true };
1507
+ });
1508
+ }
1509
+ async list(options = {}) {
1510
+ const suppliedLimit = options.limit;
1511
+ const limit = Math.min(MAX_INDEX_LIST_SIZE, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : MAX_INDEX_LIST_SIZE));
1512
+ const entries = await this.ctx.storage.list({
1513
+ prefix: options.prefix ?? "",
1514
+ ...options.reverse ? {
1515
+ reverse: true,
1516
+ end: options.cursor
1517
+ } : { startAfter: options.cursor },
1518
+ limit: limit + 1
1519
+ });
1520
+ const page = Array.from(entries).slice(0, limit);
1521
+ const listComplete = entries.size <= limit;
1522
+ return {
1523
+ keys: page.map(([name, value]) => ({
1524
+ name,
1525
+ value
1526
+ })),
1527
+ list_complete: listComplete,
1528
+ cursor: listComplete ? void 0 : page.at(-1)?.[0]
1529
+ };
1530
+ }
1531
+ async expireRun(runId, keys, expiredAt) {
1532
+ if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
1533
+ return await this.ctx.storage.transaction(async (txn) => {
1534
+ await txn.put(`expired:${runId}`, expiredAt);
1535
+ return { deleted: await txn.delete(keys) };
1536
+ });
1537
+ }
1538
+ };
1539
+ //#endregion
1540
+ //#region src/worker/durable-objects/RunFenceDO.ts
1541
+ /** Per-run lifecycle fence used by hook shards without a global lookup cell. */
1542
+ var RunFenceDO = class extends DurableObject {
1543
+ async getStatus() {
1544
+ return (await this.ctx.storage.get("fence"))?.status ?? "active";
1545
+ }
1546
+ async fenceTerminal(at) {
1547
+ return await this.ctx.storage.transaction(async (txn) => {
1548
+ const current = await txn.get("fence");
1549
+ if (current) return current.status;
1550
+ await txn.put("fence", {
1551
+ status: "terminal",
1552
+ at
1553
+ });
1554
+ return "terminal";
1555
+ });
1556
+ }
1557
+ async fenceExpired(at) {
1558
+ return await this.ctx.storage.transaction(async (txn) => {
1559
+ const current = await txn.get("fence");
1560
+ if (current?.status === "expired") return current.status;
1561
+ await txn.put("fence", {
1562
+ status: "expired",
1563
+ at
1564
+ });
1565
+ return "expired";
1566
+ });
1048
1567
  }
1049
1568
  };
1050
1569
  //#endregion
1051
1570
  //#region src/worker/durable-objects/StreamDO.ts
1052
1571
  const META_KEY = "meta";
1053
1572
  const CHUNK_KEY_PREFIX = "chunk:";
1573
+ const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
1054
1574
  /** Registry keys used when this DO instance acts as a per-run stream index. */
1055
1575
  const STREAM_REGISTRY_PREFIX = "stream:";
1056
1576
  const REGISTRY_OWNER_KEY = "registry:owner";
1057
1577
  const REGISTRY_EXPIRED_KEY = "registry:expired";
1058
- /**
1059
- * Zero-pad chunk indexes so `storage.list({ prefix })` returns chunks in
1060
- * write order. 12 digits comfortably exceeds any realistic chunk count.
1061
- */
1578
+ /** Each chunk occupies a payload key and a size key; one delete accepts at most 128 keys. */
1579
+ const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
1580
+ const MAX_EXPIRE_CHUNK_LIMIT = 64;
1581
+ /** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
1582
+ const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
1583
+ const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
1584
+ function boundedLimit(value, fallback, maximum) {
1585
+ if (value === void 0 || !Number.isFinite(value)) return fallback;
1586
+ return Math.max(1, Math.min(maximum, Math.floor(value)));
1587
+ }
1588
+ function emptyMeta() {
1589
+ return {
1590
+ count: 0,
1591
+ state: "open"
1592
+ };
1593
+ }
1594
+ function validateMeta(meta) {
1595
+ if (!Number.isSafeInteger(meta.count) || meta.count < 0 || meta.count > 2147483647 || ![
1596
+ "open",
1597
+ "closed",
1598
+ "errored",
1599
+ "expired"
1600
+ ].includes(meta.state)) throw new Error("Invalid persisted stream metadata");
1601
+ if (meta.state === "errored" && !meta.error) throw new Error("Invalid persisted stream error metadata");
1602
+ return meta;
1603
+ }
1604
+ /** Zero-padding keeps storage.list() results in stream offset order. */
1062
1605
  function chunkKey(index) {
1063
1606
  return `${CHUNK_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
1064
1607
  }
1608
+ function chunkSizeKey(index) {
1609
+ return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
1610
+ }
1611
+ function indexFromChunkSizeKey(key) {
1612
+ const suffix = key.slice(11);
1613
+ const index = Number(suffix);
1614
+ if (!/^\d{12}$/.test(suffix) || !Number.isSafeInteger(index) || index < 0) throw new Error(`Invalid persisted stream chunk size key "${key}"`);
1615
+ return index;
1616
+ }
1617
+ function validateChunkSize(size, index) {
1618
+ if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
1619
+ return size;
1620
+ }
1621
+ function compactChunk(chunk) {
1622
+ return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
1623
+ }
1624
+ function abortReason(signal) {
1625
+ return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
1626
+ }
1065
1627
  /**
1066
1628
  * Durable Object backing workflow streams.
1067
1629
  *
1068
- * Two roles, selected by the DO name:
1069
- * - `stream:<name>`: chunk storage for a single stream. A monotonic
1070
- * per-stream counter (maintained transactionally inside the DO) assigns
1071
- * each chunk its index; one chunk per storage key.
1072
- * - `run-streams:<runId>`: registry of stream names owned by a run,
1073
- * powering `listStreamsByRunId`.
1630
+ * Stream cells persist binary chunks under monotonic offset keys. Public
1631
+ * reads are bounded long polls, and writes append a bounded ordered batch in
1632
+ * one storage transaction. Per-run registry cells retain their existing role.
1074
1633
  */
1075
1634
  var StreamDO = class extends DurableObject {
1635
+ meta;
1636
+ metaLoad;
1637
+ mutationTail = Promise.resolve();
1638
+ changeVersion = 0;
1639
+ waiters = /* @__PURE__ */ new Set();
1076
1640
  async getMeta() {
1077
- return await this.ctx.storage.get(META_KEY) ?? {
1078
- count: 0,
1079
- closed: false
1080
- };
1641
+ if (this.meta) return this.meta;
1642
+ this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
1643
+ const loaded = meta ? validateMeta(meta) : emptyMeta();
1644
+ this.meta = loaded;
1645
+ return loaded;
1646
+ });
1647
+ try {
1648
+ return await this.metaLoad;
1649
+ } catch (error) {
1650
+ this.metaLoad = void 0;
1651
+ throw error;
1652
+ }
1081
1653
  }
1082
1654
  assertOwner(meta, runId) {
1083
- if (meta.expiredAt !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
1655
+ if (!runId) throw new Error("Stream runId is required");
1084
1656
  if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
1085
1657
  }
1086
- /**
1087
- * Append a chunk. The index is allocated inside the transaction, so
1088
- * concurrent writers can never collide or skip.
1089
- */
1090
- async writeChunk(runId, data) {
1091
- return await this.ctx.storage.transaction(async (txn) => {
1092
- const meta = await txn.get(META_KEY) ?? {
1093
- count: 0,
1094
- closed: false
1658
+ assertWritable(meta, runId) {
1659
+ this.assertOwner(meta, runId);
1660
+ if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
1661
+ if (meta.state === "closed") throw new Error("Cannot write to a closed stream");
1662
+ if (meta.state === "errored") throw new Error("Cannot write to an errored stream");
1663
+ }
1664
+ runMutation(operation) {
1665
+ const result = this.mutationTail.then(operation, operation);
1666
+ this.mutationTail = result.then(() => void 0, () => void 0);
1667
+ return result;
1668
+ }
1669
+ wakeReaders() {
1670
+ this.changeVersion += 1;
1671
+ for (const waiter of Array.from(this.waiters)) waiter.resolve("change");
1672
+ }
1673
+ waitForChange(observedVersion, waitMs, signal) {
1674
+ if (signal?.aborted) return Promise.reject(abortReason(signal));
1675
+ return new Promise((resolve, reject) => {
1676
+ let settled = false;
1677
+ const waiter = {
1678
+ timer: setTimeout(() => finish("timeout"), waitMs),
1679
+ signal,
1680
+ resolve: (reason) => finish(reason)
1681
+ };
1682
+ const cleanup = () => {
1683
+ clearTimeout(waiter.timer);
1684
+ this.waiters.delete(waiter);
1685
+ if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
1686
+ };
1687
+ const finish = (reason) => {
1688
+ if (settled) return;
1689
+ settled = true;
1690
+ cleanup();
1691
+ resolve(reason);
1095
1692
  };
1096
- this.assertOwner(meta, runId);
1097
- if (meta.closed) throw new Error("Cannot write to a closed stream");
1098
- const index = meta.count;
1099
- await txn.put(chunkKey(index), data);
1100
- await txn.put(META_KEY, {
1101
- count: index + 1,
1102
- closed: false,
1103
- ownerRunId: runId
1693
+ waiter.onAbort = () => {
1694
+ if (settled) return;
1695
+ settled = true;
1696
+ cleanup();
1697
+ reject(abortReason(signal));
1698
+ };
1699
+ this.waiters.add(waiter);
1700
+ signal?.addEventListener("abort", waiter.onAbort, { once: true });
1701
+ if (this.changeVersion !== observedVersion) finish("change");
1702
+ });
1703
+ }
1704
+ async readSnapshot(request, timedOut) {
1705
+ const meta = await this.getMeta();
1706
+ this.assertOwner(meta, request.runId);
1707
+ const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
1708
+ const chunks = [];
1709
+ if (available > 0) {
1710
+ const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
1711
+ const sizeKeys = indexes.map(chunkSizeKey);
1712
+ const storedSizes = await this.ctx.storage.get(sizeKeys);
1713
+ const selectedIndexes = [];
1714
+ const selectedSizes = [];
1715
+ let bytes = 0;
1716
+ for (const index of indexes) {
1717
+ const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
1718
+ if (bytes + size > request.maxBytes) break;
1719
+ selectedIndexes.push(index);
1720
+ selectedSizes.push(size);
1721
+ bytes += size;
1722
+ }
1723
+ const payloadKeys = selectedIndexes.map(chunkKey);
1724
+ const storedChunks = await this.ctx.storage.get(payloadKeys);
1725
+ for (let offset = 0; offset < payloadKeys.length; offset++) {
1726
+ const chunk = storedChunks.get(payloadKeys[offset]);
1727
+ if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
1728
+ chunks.push(chunk);
1729
+ }
1730
+ }
1731
+ return {
1732
+ startIndex: request.startIndex,
1733
+ tailIndex: meta.count - 1,
1734
+ chunks,
1735
+ state: meta.state,
1736
+ timedOut,
1737
+ error: meta.error
1738
+ };
1739
+ }
1740
+ /** Append one bounded binary batch with contiguous indexes. */
1741
+ async writeChunks(runId, chunks) {
1742
+ validateStreamWriteChunks(chunks);
1743
+ const storedChunks = chunks.map(compactChunk);
1744
+ return await this.runMutation(async () => {
1745
+ const committed = await this.ctx.storage.transaction(async (txn) => {
1746
+ const stored = await txn.get(META_KEY);
1747
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1748
+ this.assertWritable(meta, runId);
1749
+ const startIndex = meta.count;
1750
+ if (startIndex + chunks.length > 2147483647) throw new Error("Stream offset limit exceeded");
1751
+ const nextMeta = {
1752
+ count: startIndex + chunks.length,
1753
+ state: "open",
1754
+ ownerRunId: runId
1755
+ };
1756
+ const entries = { [META_KEY]: nextMeta };
1757
+ for (let offset = 0; offset < storedChunks.length; offset++) {
1758
+ const index = startIndex + offset;
1759
+ entries[chunkKey(index)] = storedChunks[offset];
1760
+ entries[chunkSizeKey(index)] = storedChunks[offset].byteLength;
1761
+ }
1762
+ await txn.put(entries);
1763
+ return {
1764
+ meta: nextMeta,
1765
+ result: {
1766
+ startIndex,
1767
+ count: chunks.length,
1768
+ tailIndex: nextMeta.count - 1
1769
+ }
1770
+ };
1104
1771
  });
1105
- return index;
1772
+ this.meta = committed.meta;
1773
+ this.metaLoad = Promise.resolve(committed.meta);
1774
+ this.wakeReaders();
1775
+ return committed.result;
1106
1776
  });
1107
1777
  }
1108
1778
  /**
1109
- * Close the stream (idempotent). Readers observe `done: true` once all
1110
- * previously written chunks have been consumed.
1779
+ * Return available binary chunks and terminal metadata in one bounded
1780
+ * operation, waiting at most waitMs when the requested offset is idle.
1111
1781
  */
1782
+ async readChunks(request, signal) {
1783
+ validateStreamReadRequest(request);
1784
+ const observedVersion = this.changeVersion;
1785
+ const immediate = await this.readSnapshot(request, false);
1786
+ if (immediate.chunks.length > 0 || immediate.state !== "open" || request.waitMs === 0 || request.maxChunks === 0) return immediate;
1787
+ const reason = await this.waitForChange(observedVersion, request.waitMs, signal);
1788
+ return await this.readSnapshot(request, reason === "timeout");
1789
+ }
1790
+ /** Close idempotently; the first terminal state wins. */
1112
1791
  async closeStream(runId) {
1113
- await this.ctx.storage.transaction(async (txn) => {
1114
- const meta = await txn.get(META_KEY) ?? {
1115
- count: 0,
1116
- closed: false
1117
- };
1118
- this.assertOwner(meta, runId);
1119
- await txn.put(META_KEY, {
1120
- ...meta,
1121
- ownerRunId: runId,
1122
- closed: true
1792
+ await this.runMutation(async () => {
1793
+ const nextMeta = await this.ctx.storage.transaction(async (txn) => {
1794
+ const stored = await txn.get(META_KEY);
1795
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1796
+ this.assertOwner(meta, runId);
1797
+ if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
1798
+ if (meta.state !== "open") return meta;
1799
+ const closed = {
1800
+ ...meta,
1801
+ ownerRunId: runId,
1802
+ state: "closed"
1803
+ };
1804
+ await txn.put(META_KEY, closed);
1805
+ return closed;
1123
1806
  });
1807
+ this.meta = nextMeta;
1808
+ this.metaLoad = Promise.resolve(nextMeta);
1809
+ this.wakeReaders();
1124
1810
  });
1125
1811
  }
1126
- /**
1127
- * Read up to `limit` chunks starting at `startIndex` (0-based, inclusive).
1128
- */
1129
- async getChunks(params) {
1130
- const meta = await this.getMeta();
1131
- if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
1132
- const start = Math.max(0, params.startIndex);
1133
- const entries = await this.ctx.storage.list({
1134
- prefix: CHUNK_KEY_PREFIX,
1135
- start: chunkKey(start),
1136
- limit: params.limit
1812
+ /** Persist a terminal stream failure and wake every pending reader. */
1813
+ async failStream(runId, error) {
1814
+ const normalized = normalizeStreamError(error);
1815
+ await this.runMutation(async () => {
1816
+ const nextMeta = await this.ctx.storage.transaction(async (txn) => {
1817
+ const stored = await txn.get(META_KEY);
1818
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1819
+ this.assertOwner(meta, runId);
1820
+ if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
1821
+ if (meta.state !== "open") return meta;
1822
+ const failed = {
1823
+ ...meta,
1824
+ ownerRunId: runId,
1825
+ state: "errored",
1826
+ error: normalized
1827
+ };
1828
+ await txn.put(META_KEY, failed);
1829
+ return failed;
1830
+ });
1831
+ this.meta = nextMeta;
1832
+ this.metaLoad = Promise.resolve(nextMeta);
1833
+ this.wakeReaders();
1137
1834
  });
1138
- return {
1139
- chunks: Array.from(entries.values()),
1140
- done: meta.closed,
1141
- tailIndex: meta.count - 1
1142
- };
1143
- }
1144
- /**
1145
- * Lightweight stream metadata: last chunk index and completion flag.
1146
- */
1147
- async getInfo() {
1148
- const meta = await this.getMeta();
1149
- if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
1150
- return {
1151
- tailIndex: meta.count - 1,
1152
- done: meta.closed
1153
- };
1154
1835
  }
1155
- /**
1156
- * Register a stream name against this per-run registry instance.
1157
- */
1836
+ /** Register a stream name against this per-run registry instance. */
1158
1837
  async registerStream(runId, name) {
1159
1838
  await this.ctx.storage.transaction(async (txn) => {
1160
1839
  if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
@@ -1164,57 +1843,124 @@ var StreamDO = class extends DurableObject {
1164
1843
  await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
1165
1844
  });
1166
1845
  }
1167
- /**
1168
- * List stream names registered against this per-run registry instance.
1169
- */
1170
1846
  async listStreams() {
1171
1847
  if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
1172
1848
  const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
1173
1849
  return Array.from(entries.keys()).map((key) => key.slice(7));
1174
1850
  }
1175
1851
  /** Fence a run's registry before any stream cells are removed. */
1176
- async expireRegistry(runId, expiredAt) {
1852
+ async expireRegistry(runId, expiredAt, options) {
1853
+ const limit = boundedLimit(options?.limit, 16, 128);
1177
1854
  return await this.ctx.storage.transaction(async (txn) => {
1178
- const owner = await txn.get(REGISTRY_OWNER_KEY);
1855
+ const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
1856
+ const owner = state.get(REGISTRY_OWNER_KEY);
1179
1857
  if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
1180
- await txn.put(REGISTRY_OWNER_KEY, runId);
1181
- await txn.put(REGISTRY_EXPIRED_KEY, expiredAt);
1182
- const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
1858
+ const expiry = state.get(REGISTRY_EXPIRED_KEY);
1859
+ await txn.put({
1860
+ [REGISTRY_OWNER_KEY]: runId,
1861
+ [REGISTRY_EXPIRED_KEY]: expiry ?? {
1862
+ expiredAt,
1863
+ deleted: 0
1864
+ }
1865
+ });
1866
+ const entries = await txn.list({
1867
+ prefix: STREAM_REGISTRY_PREFIX,
1868
+ limit
1869
+ });
1183
1870
  return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
1184
1871
  });
1185
1872
  }
1186
- /** Remove the registry payload after every referenced stream was fenced. */
1187
- async finalizeRegistry(runId) {
1188
- await this.ctx.storage.transaction(async (txn) => {
1189
- const owner = await txn.get(REGISTRY_OWNER_KEY);
1190
- const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
1191
- if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
1192
- const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
1193
- for (const key of entries.keys()) await txn.delete(key);
1194
- });
1195
- }
1196
- /** Delete stream payload while leaving a small ownership tombstone. */
1197
- async expireStream(runId, expiredAt) {
1873
+ /** Remove completed registry entries in a bounded, retry-safe batch. */
1874
+ async finalizeRegistry(runId, streams) {
1198
1875
  return await this.ctx.storage.transaction(async (txn) => {
1199
- const meta = await txn.get(META_KEY) ?? {
1200
- count: 0,
1201
- closed: false
1202
- };
1203
- if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
1204
- const chunks = await txn.list({ prefix: CHUNK_KEY_PREFIX });
1205
- for (const key of chunks.keys()) await txn.delete(key);
1206
- await txn.put(META_KEY, {
1207
- count: 0,
1208
- closed: true,
1209
- ownerRunId: runId,
1210
- expiredAt
1876
+ const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
1877
+ const owner = state.get(REGISTRY_OWNER_KEY);
1878
+ const expiry = state.get(REGISTRY_EXPIRED_KEY);
1879
+ if (owner !== runId || expiry === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
1880
+ const keys = Array.from(new Set(streams.map((name) => `${STREAM_REGISTRY_PREFIX}${name}`)));
1881
+ const removed = keys.length === 0 ? 0 : await txn.delete(keys);
1882
+ const deleted = expiry.deleted + removed;
1883
+ await txn.put(REGISTRY_EXPIRED_KEY, {
1884
+ ...expiry,
1885
+ deleted
1211
1886
  });
1212
1887
  return {
1213
- deleted: true,
1214
- chunks: chunks.size
1888
+ deleted,
1889
+ done: (await txn.list({
1890
+ prefix: STREAM_REGISTRY_PREFIX,
1891
+ limit: 1
1892
+ })).size === 0
1215
1893
  };
1216
1894
  });
1217
1895
  }
1896
+ /**
1897
+ * Fence the stream and delete one bounded page. Listing size keys avoids
1898
+ * loading payloads, and the transaction makes a crash retry resume safely.
1899
+ */
1900
+ async expireStream(runId, expiredAt, options) {
1901
+ const limit = boundedLimit(options?.limit, DEFAULT_EXPIRE_CHUNK_LIMIT, MAX_EXPIRE_CHUNK_LIMIT);
1902
+ const byteLimit = boundedLimit(options?.byteLimit, DEFAULT_EXPIRE_BYTE_LIMIT, MAX_EXPIRE_BYTE_LIMIT);
1903
+ return await this.runMutation(async () => {
1904
+ const result = await this.ctx.storage.transaction(async (txn) => {
1905
+ const stored = await txn.get(META_KEY);
1906
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1907
+ this.assertOwner(meta, runId);
1908
+ const firstFence = meta.state !== "expired";
1909
+ if (meta.state === "expired" && meta.payloadDeleted) return {
1910
+ meta,
1911
+ result: {
1912
+ deleted: false,
1913
+ chunks: 0,
1914
+ bytes: 0,
1915
+ done: true
1916
+ }
1917
+ };
1918
+ const candidates = await txn.list({
1919
+ prefix: CHUNK_SIZE_KEY_PREFIX,
1920
+ limit: limit + 1
1921
+ });
1922
+ const page = [];
1923
+ let bytes = 0;
1924
+ for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
1925
+ const index = indexFromChunkSizeKey(key);
1926
+ const size = validateChunkSize(storedSize, index);
1927
+ if (page.length > 0 && bytes + size > byteLimit) break;
1928
+ page.push({
1929
+ index,
1930
+ size
1931
+ });
1932
+ bytes += size;
1933
+ }
1934
+ const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
1935
+ if (keys.length > 0) await txn.delete(keys);
1936
+ const done = candidates.size === page.length;
1937
+ const nextMeta = {
1938
+ ...meta,
1939
+ count: done ? 0 : meta.count,
1940
+ ownerRunId: runId,
1941
+ state: "expired",
1942
+ error: void 0,
1943
+ expiredAt: meta.expiredAt ?? expiredAt,
1944
+ expiredChunkCount: meta.expiredChunkCount ?? meta.count,
1945
+ payloadDeleted: done
1946
+ };
1947
+ await txn.put(META_KEY, nextMeta);
1948
+ return {
1949
+ meta: nextMeta,
1950
+ result: {
1951
+ deleted: firstFence,
1952
+ chunks: page.length,
1953
+ bytes,
1954
+ done
1955
+ }
1956
+ };
1957
+ });
1958
+ this.meta = result.meta;
1959
+ this.metaLoad = Promise.resolve(result.meta);
1960
+ this.wakeReaders();
1961
+ return result.result;
1962
+ });
1963
+ }
1218
1964
  };
1219
1965
  //#endregion
1220
1966
  //#region src/apply-event.ts
@@ -1234,10 +1980,36 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
1234
1980
  * rejected as a duplicate instead of resurrecting the hook.
1235
1981
  */
1236
1982
  const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
1983
+ /** Durable repair record for idempotent post-commit hook-index release. */
1984
+ const HOOK_RELEASE_MARKER_PREFIX = "hookreleased:";
1985
+ /** Exact ambiguous reservations canceled by a serialized RunDO resolution. */
1986
+ const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
1237
1987
  /** Workflow-authored attr_set correlation claims. */
1238
1988
  const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
1239
1989
  /** Durable lazy-hook resume claims, scoped by the containing run DO. */
1240
1990
  const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
1991
+ const STORAGE_BATCH_SIZE$1 = 128;
1992
+ const MAX_ENTITY_PAGE_SIZE = 1e3;
1993
+ async function getMany(store, keys) {
1994
+ if (store.getMany) {
1995
+ const result = /* @__PURE__ */ new Map();
1996
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
1997
+ const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
1998
+ for (const entry of page) result.set(...entry);
1999
+ }
2000
+ return result;
2001
+ }
2002
+ const values = await Promise.all(keys.map((key) => store.get(key)));
2003
+ return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
2004
+ }
2005
+ async function deleteMany(store, keys) {
2006
+ if (store.deleteMany) {
2007
+ let deleted = 0;
2008
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
2009
+ return deleted;
2010
+ }
2011
+ return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
2012
+ }
1241
2013
  function failure(code, message, extra) {
1242
2014
  return {
1243
2015
  ok: false,
@@ -1246,6 +2018,9 @@ function failure(code, message, extra) {
1246
2018
  ...extra
1247
2019
  };
1248
2020
  }
2021
+ function hookClaimCancellationKey(claimId) {
2022
+ return `${HOOK_CLAIM_CANCELLATION_PREFIX}${encodeURIComponent(claimId)}`;
2023
+ }
1249
2024
  function readStringProp(value, key) {
1250
2025
  if (value !== null && typeof value === "object") {
1251
2026
  const prop = value[key];
@@ -1301,7 +2076,9 @@ function creationIndexKey(prefix, createdAt, id) {
1301
2076
  * requested ordering.
1302
2077
  */
1303
2078
  async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
1304
- const { limit, cursor, sortOrder = "asc" } = params;
2079
+ const requestedLimit = Number.isFinite(params.limit) ? Math.floor(params.limit) : 1;
2080
+ const limit = Math.min(MAX_ENTITY_PAGE_SIZE, Math.max(1, requestedLimit));
2081
+ const { cursor, sortOrder = "asc" } = params;
1305
2082
  const entries = await store.list(sortOrder === "desc" ? {
1306
2083
  prefix: indexPrefix,
1307
2084
  reverse: true,
@@ -1313,11 +2090,12 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
1313
2090
  startAfter: cursor ? `${indexPrefix}${cursor}` : void 0
1314
2091
  });
1315
2092
  const page = Array.from(entries.entries()).slice(0, limit);
1316
- const data = [];
1317
- for (const [, id] of page) {
1318
- const item = await store.get(`${entityPrefix}${id}`);
1319
- if (item !== void 0) data.push(item);
1320
- }
2093
+ const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
2094
+ const entities = await getMany(store, entityKeys);
2095
+ const data = entityKeys.flatMap((key) => {
2096
+ const item = entities.get(key);
2097
+ return item === void 0 ? [] : [item];
2098
+ });
1321
2099
  const hasMore = entries.size > limit;
1322
2100
  const lastKey = page.at(-1)?.[0];
1323
2101
  return {
@@ -1328,22 +2106,26 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
1328
2106
  }
1329
2107
  /** Delete all hook entities for the run, returning the released tokens. */
1330
2108
  async function releaseAllHooks(store) {
2109
+ if (store.deferTerminalCleanup) return [];
1331
2110
  const hooks = await store.list({ prefix: HOOK_KEY_PREFIX });
1332
2111
  const released = [];
2112
+ const keys = [];
1333
2113
  for (const hook of hooks.values()) {
1334
- await store.delete(`${HOOK_KEY_PREFIX}${hook.hookId}`);
1335
- await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
2114
+ keys.push(`${HOOK_KEY_PREFIX}${hook.hookId}`);
2115
+ keys.push(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
1336
2116
  released.push({
1337
2117
  hookId: hook.hookId,
1338
2118
  token: hook.token
1339
2119
  });
1340
2120
  }
2121
+ await deleteMany(store, keys);
1341
2122
  return released;
1342
2123
  }
1343
2124
  /** Delete all persisted waits once their owning run becomes terminal. */
1344
2125
  async function releaseAllWaits(store) {
2126
+ if (store.deferTerminalCleanup) return;
1345
2127
  const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
1346
- for (const key of waits.keys()) await store.delete(key);
2128
+ await deleteMany(store, Array.from(waits.keys()));
1347
2129
  }
1348
2130
  async function applyEvent(store, ctx) {
1349
2131
  const { runId, data, now } = ctx;
@@ -1463,7 +2245,16 @@ async function applyEvent(store, ctx) {
1463
2245
  }
1464
2246
  }
1465
2247
  if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
1466
- if (!await store.get(`hook:${data.correlationId}`)) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
2248
+ if (!await store.get(`hook:${data.correlationId}`)) {
2249
+ if (data.eventType === "hook_disposed") {
2250
+ const released = await store.get(`${HOOK_RELEASE_MARKER_PREFIX}${data.correlationId}`);
2251
+ if (released) return {
2252
+ ok: true,
2253
+ releasedHooks: [released]
2254
+ };
2255
+ }
2256
+ return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
2257
+ }
1467
2258
  }
1468
2259
  if (!skipRunValidation && data.eventType !== "run_started" && !currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
1469
2260
  switch (data.eventType) {
@@ -1799,6 +2590,7 @@ async function applyEvent(store, ctx) {
1799
2590
  const holder = ctx.tokenHolder ?? null;
1800
2591
  const markerKey = `${HOOK_EVENT_MARKER_PREFIX}${hookId}`;
1801
2592
  const hookKey = `${HOOK_KEY_PREFIX}${hookId}`;
2593
+ if (ctx.hookClaimId && await store.get(hookClaimCancellationKey(ctx.hookClaimId)) !== void 0) return failure("HOOK_CLAIM_CANCELLED", `Hook reservation for "${hookId}" was canceled before commit`);
1802
2594
  if (holder && (holder.runId !== runId || holder.hookId !== hookId)) {
1803
2595
  const event = buildEvent({
1804
2596
  eventType: "hook_conflict",
@@ -1858,6 +2650,10 @@ async function applyEvent(store, ctx) {
1858
2650
  if (!hook) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
1859
2651
  await store.delete(hookKey);
1860
2652
  await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
2653
+ await store.put(`${HOOK_RELEASE_MARKER_PREFIX}${hook.hookId}`, {
2654
+ hookId: hook.hookId,
2655
+ token: hook.token
2656
+ });
1861
2657
  const event = buildEvent({ ...data });
1862
2658
  await putEvent(event);
1863
2659
  return {
@@ -1972,10 +2768,55 @@ async function finalizeEventPage(store, outcome, params) {
1972
2768
  };
1973
2769
  }
1974
2770
  //#endregion
2771
+ //#region src/retention.ts
2772
+ const CLEANUP_RECORD_KEY = "retention:cleanup";
2773
+ const TOMBSTONE_KEY = "retention:tombstone";
2774
+ const HOOK_MARKER_PREFIX = "retention:hook:";
2775
+ const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
2776
+ function sortableTimestamp(date) {
2777
+ return date.getTime().toString().padStart(13, "0");
2778
+ }
2779
+ function workflowRunIndexKey(run) {
2780
+ return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
2781
+ }
2782
+ function globalRunIndexKey(run) {
2783
+ return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
2784
+ }
2785
+ function hookMarkerKey(reference) {
2786
+ return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
2787
+ }
2788
+ function cleanupTombstone(record, now) {
2789
+ return {
2790
+ version: 1,
2791
+ runId: record.runId,
2792
+ terminalStatus: record.terminalStatus,
2793
+ completedAt: record.completedAt,
2794
+ expiredAt: record.dueAt,
2795
+ tombstonedAt: now
2796
+ };
2797
+ }
2798
+ function expiredRead(tombstone) {
2799
+ return {
2800
+ ok: false,
2801
+ code: "RUN_EXPIRED",
2802
+ message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
2803
+ tombstone
2804
+ };
2805
+ }
2806
+ //#endregion
1975
2807
  //#region src/worker/durable-objects/WorkflowRunDO.ts
1976
- const PAYLOAD_DELETE_BATCH = 256;
1977
- const STREAM_DELETE_CONCURRENCY = 16;
2808
+ const STORAGE_BATCH_SIZE = 128;
2809
+ const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
2810
+ const HOOK_MARKER_PAGE_SIZE = 64;
2811
+ const TERMINAL_HOOK_PAGE_SIZE = 64;
2812
+ const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
2813
+ const STREAMS_PER_CLEANUP_PAGE = 16;
2814
+ const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
2815
+ const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
2816
+ const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
2817
+ const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
1978
2818
  const CLEANUP_RETRY_MAX_MS = 36e5;
2819
+ const CLEANUP_PROGRESS_KEY = "retention:progress";
1979
2820
  /**
1980
2821
  * Adapt DO storage (or a transaction) to the {@link EventStore} interface
1981
2822
  * shared with the test mocks. No casts: the methods line up structurally,
@@ -1984,11 +2825,21 @@ const CLEANUP_RETRY_MAX_MS = 36e5;
1984
2825
  function storeFrom(storage) {
1985
2826
  return {
1986
2827
  get: (key) => storage.get(key),
2828
+ getMany: (keys) => storage.get(keys),
1987
2829
  put: (key, value) => storage.put(key, value),
1988
2830
  delete: (key) => storage.delete(key),
1989
- list: (options) => storage.list(options)
2831
+ deleteMany: (keys) => storage.delete(keys),
2832
+ list: (options) => storage.list(options),
2833
+ deferTerminalCleanup: true
1990
2834
  };
1991
2835
  }
2836
+ function hookCreationIndexKey(hook) {
2837
+ return `${HOOK_CREATED_KEY_PREFIX}${hook.createdAt.toISOString()}:${hook.hookId}`;
2838
+ }
2839
+ async function putStorageEntries(storage, entries) {
2840
+ const all = Array.from(entries);
2841
+ for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
2842
+ }
1992
2843
  /**
1993
2844
  * Durable Object for managing a single workflow run's state.
1994
2845
  *
@@ -2004,26 +2855,51 @@ var WorkflowRunDO = class extends DurableObject {
2004
2855
  const clock = this.env?.clock;
2005
2856
  return typeof clock === "function" ? clock() : Date.now();
2006
2857
  }
2007
- async expiredTombstone(storage, now = this.now()) {
2008
- const tombstone = await storage.get(TOMBSTONE_KEY);
2858
+ tombstoneFrom(values, now = this.now()) {
2859
+ const tombstone = values.get(TOMBSTONE_KEY);
2009
2860
  if (tombstone) return tombstone;
2010
- const cleanup = await storage.get(CLEANUP_RECORD_KEY);
2861
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
2011
2862
  if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
2012
2863
  return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
2013
2864
  }
2014
- async read(reader) {
2015
- const tombstone = await this.expiredTombstone(this.ctx.storage);
2016
- if (tombstone) return expiredRead(tombstone);
2865
+ async retentionState(storage, now = this.now()) {
2866
+ const values = await storage.get([TOMBSTONE_KEY, CLEANUP_RECORD_KEY]);
2017
2867
  return {
2018
- ok: true,
2019
- value: await reader()
2868
+ cleanup: values.get(CLEANUP_RECORD_KEY),
2869
+ tombstone: this.tombstoneFrom(values, now)
2020
2870
  };
2021
2871
  }
2872
+ async read(reader) {
2873
+ return await this.ctx.storage.transaction(async (txn) => {
2874
+ const { tombstone } = await this.retentionState(txn);
2875
+ if (tombstone) return expiredRead(tombstone);
2876
+ return {
2877
+ ok: true,
2878
+ value: await reader(txn)
2879
+ };
2880
+ });
2881
+ }
2882
+ async readKey(key) {
2883
+ return await this.ctx.storage.transaction(async (txn) => {
2884
+ const values = await txn.get([
2885
+ TOMBSTONE_KEY,
2886
+ CLEANUP_RECORD_KEY,
2887
+ key
2888
+ ]);
2889
+ const tombstone = this.tombstoneFrom(values);
2890
+ if (tombstone) return expiredRead(tombstone);
2891
+ return {
2892
+ ok: true,
2893
+ value: values.get(key) ?? null
2894
+ };
2895
+ });
2896
+ }
2022
2897
  /** Apply an event and capture all retention metadata in the same transaction. */
2023
2898
  async applyEvent(request) {
2024
2899
  return await this.ctx.storage.transaction(async (txn) => {
2025
2900
  const now = new Date(this.now());
2026
- const tombstone = await this.expiredTombstone(txn, now.getTime());
2901
+ const retention = await this.retentionState(txn, now.getTime());
2902
+ const { tombstone } = retention;
2027
2903
  if (tombstone) return {
2028
2904
  ok: false,
2029
2905
  code: "RUN_EXPIRED",
@@ -2037,20 +2913,12 @@ var WorkflowRunDO = class extends DurableObject {
2037
2913
  });
2038
2914
  if (!outcome.ok) return outcome;
2039
2915
  await txn.put("event_sequence", eventSequence);
2040
- if (outcome.event?.correlationId) {
2041
- const key = correlationIndexKey(outcome.event.correlationId, outcome.event.createdAt, outcome.event.eventId, request.runId);
2042
- await txn.put(indexMarkerKey(key), key);
2043
- }
2044
- if (outcome.hookToIndex) {
2045
- const reference = {
2046
- hookId: outcome.hookToIndex.hookId,
2047
- token: outcome.hookToIndex.token
2048
- };
2049
- await txn.put(hookMarkerKey(reference), reference);
2050
- }
2051
- for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
2916
+ await putStorageEntries(txn, (outcome.hookToIndex ? [{
2917
+ hookId: outcome.hookToIndex.hookId,
2918
+ token: outcome.hookToIndex.token
2919
+ }, ...outcome.releasedHooks] : outcome.releasedHooks).map((reference) => [hookMarkerKey(reference), reference]));
2052
2920
  const retentionMs = request.cleanup?.retentionMs ?? 0;
2053
- if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !await txn.get("retention:cleanup")) {
2921
+ if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !retention.cleanup) {
2054
2922
  const terminalRun = outcome.run;
2055
2923
  const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
2056
2924
  const run = WorkflowRunSchema.parse({
@@ -2067,6 +2935,7 @@ var WorkflowRunDO = class extends DurableObject {
2067
2935
  dueAt,
2068
2936
  queueShards: request.cleanup?.queueShards ?? 1,
2069
2937
  phase: "retained",
2938
+ generation: 0,
2070
2939
  attempts: 0,
2071
2940
  deletedPayloadKeys: 0,
2072
2941
  deletedStreams: 0,
@@ -2077,34 +2946,60 @@ var WorkflowRunDO = class extends DurableObject {
2077
2946
  await txn.setAlarm(dueAt);
2078
2947
  outcome.run = run;
2079
2948
  }
2949
+ if (outcome.run && isTerminalWorkflowRunStatus(outcome.run.status)) {
2950
+ if (!await txn.get("terminal:cleanup")) await txn.put(TERMINAL_CLEANUP_KEY, {
2951
+ version: 1,
2952
+ runId: outcome.run.runId,
2953
+ phase: "hooks",
2954
+ generation: 0,
2955
+ attempts: 0
2956
+ });
2957
+ await txn.setAlarm(this.now() + 1);
2958
+ }
2080
2959
  return finalizeEventPage(storeFrom(txn), outcome, request.params);
2081
2960
  });
2082
2961
  }
2962
+ /**
2963
+ * Resolve an ambiguous hook-creation RPC in the same serialization domain as
2964
+ * applyEvent. If the hook is absent, the exact reservation is durably fenced
2965
+ * before the caller releases its token and hook-ID claims.
2966
+ */
2967
+ async resolveHookTokenClaim(request) {
2968
+ return await this.ctx.storage.transaction(async (txn) => {
2969
+ if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
2970
+ await txn.put(hookClaimCancellationKey(request.claimId), {
2971
+ hookId: request.hookId,
2972
+ token: request.token,
2973
+ canceledAt: this.now()
2974
+ });
2975
+ return { committed: false };
2976
+ });
2977
+ }
2083
2978
  async getRun() {
2084
- return this.read(async () => await this.ctx.storage.get("run") ?? null);
2979
+ return this.readKey("run");
2085
2980
  }
2086
2981
  async getStep(stepId) {
2087
- return this.read(async () => await this.ctx.storage.get(`step:${stepId}`) ?? null);
2982
+ return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
2088
2983
  }
2089
2984
  async listSteps(params) {
2090
- return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
2985
+ return this.read((txn) => listByCreationTime(storeFrom(txn), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
2091
2986
  limit: params?.limit ?? 20,
2092
2987
  cursor: params?.cursor,
2093
2988
  sortOrder: params?.sortOrder ?? "asc"
2094
2989
  }));
2095
2990
  }
2096
2991
  async getEvent(eventId) {
2097
- return this.read(async () => await this.ctx.storage.get(`event:${eventId}`) ?? null);
2992
+ return this.readKey(`${EVENT_KEY_PREFIX}${eventId}`);
2098
2993
  }
2099
2994
  async listEvents(params) {
2100
- return this.read(() => listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
2995
+ return this.read((txn) => listByPrefix(storeFrom(txn), EVENT_KEY_PREFIX, {
2101
2996
  limit: params?.limit ?? 100,
2102
2997
  cursor: params?.cursor,
2103
2998
  sortOrder: params?.sortOrder ?? "asc"
2104
2999
  }, (event) => event.eventId));
2105
3000
  }
2106
3001
  async listHooks(params) {
2107
- return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
3002
+ return this.read((txn) => listByCreationTime(storeFrom(txn), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
2108
3003
  limit: params?.limit ?? 100,
2109
3004
  cursor: params?.cursor,
2110
3005
  sortOrder: params?.sortOrder ?? "asc"
@@ -2117,10 +3012,16 @@ var WorkflowRunDO = class extends DurableObject {
2117
3012
  async scheduleCleanup(request) {
2118
3013
  if (request.retentionMs <= 0) return this.getCleanupStatus();
2119
3014
  return await this.ctx.storage.transaction(async (txn) => {
2120
- const existing = await txn.get(CLEANUP_RECORD_KEY);
3015
+ const values = await txn.get([
3016
+ CLEANUP_RECORD_KEY,
3017
+ TOMBSTONE_KEY,
3018
+ TERMINAL_CLEANUP_KEY,
3019
+ "run"
3020
+ ]);
3021
+ const existing = values.get(CLEANUP_RECORD_KEY);
2121
3022
  if (existing) return existing;
2122
- if (await txn.get("retention:tombstone")) return null;
2123
- const current = await txn.get("run");
3023
+ if (values.get("retention:tombstone")) return null;
3024
+ const current = values.get("run");
2124
3025
  if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
2125
3026
  const terminalRun = current;
2126
3027
  const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
@@ -2138,6 +3039,7 @@ var WorkflowRunDO = class extends DurableObject {
2138
3039
  dueAt,
2139
3040
  queueShards: request.queueShards,
2140
3041
  phase: "retained",
3042
+ generation: 0,
2141
3043
  attempts: 0,
2142
3044
  deletedPayloadKeys: 0,
2143
3045
  deletedStreams: 0,
@@ -2145,7 +3047,7 @@ var WorkflowRunDO = class extends DurableObject {
2145
3047
  };
2146
3048
  await txn.put("run", run);
2147
3049
  await txn.put(CLEANUP_RECORD_KEY, cleanup);
2148
- await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
3050
+ await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
2149
3051
  return cleanup;
2150
3052
  });
2151
3053
  }
@@ -2165,135 +3067,334 @@ var WorkflowRunDO = class extends DurableObject {
2165
3067
  }));
2166
3068
  await txn.put(CLEANUP_RECORD_KEY, {
2167
3069
  ...cleanup,
2168
- dueAt
3070
+ dueAt,
3071
+ generation: cleanup.generation + 1
2169
3072
  });
2170
3073
  await txn.setAlarm(this.now() + 1);
2171
3074
  });
2172
- try {
2173
- await this.executeCleanup();
2174
- } catch (error) {
2175
- await this.recordCleanupFailure(error);
2176
- }
3075
+ await this.executeScheduledCleanupPage();
2177
3076
  return this.getCleanupStatus();
2178
3077
  }
2179
3078
  async rearmCleanup() {
2180
- const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
2181
- if (!cleanup) return null;
2182
- if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
3079
+ const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
3080
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
3081
+ if (values.get("terminal:cleanup")) await this.ctx.storage.setAlarm(this.now() + 1);
3082
+ else if (!cleanup || cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
2183
3083
  else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
2184
- return cleanup;
3084
+ return cleanup ?? null;
2185
3085
  }
2186
3086
  async alarm() {
2187
- try {
2188
- await this.executeCleanup();
2189
- } catch (error) {
2190
- await this.recordCleanupFailure(error);
2191
- }
3087
+ await this.executeScheduledCleanupPage();
2192
3088
  }
2193
3089
  namespace(name) {
2194
3090
  const namespace = this.env?.[name];
2195
3091
  if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
2196
3092
  return namespace;
2197
3093
  }
2198
- async setCleanupPhase(expected, next, updates = {}) {
3094
+ workflowIndex() {
3095
+ return createWorkflowIndex({
3096
+ runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
3097
+ runFences: this.namespace("WORKFLOW_RUN_FENCES"),
3098
+ hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
3099
+ hookIds: this.namespace("WORKFLOW_HOOK_IDS")
3100
+ });
3101
+ }
3102
+ async setCleanupPhase(expected, next, updates = {}, continueAt) {
2199
3103
  await this.ctx.storage.transaction(async (txn) => {
2200
3104
  const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2201
- if (!cleanup || cleanup.phase !== expected) return;
3105
+ if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
2202
3106
  await txn.put(CLEANUP_RECORD_KEY, {
2203
3107
  ...cleanup,
2204
3108
  ...updates,
2205
3109
  phase: next,
3110
+ generation: cleanup.generation + 1,
2206
3111
  attempts: 0,
2207
3112
  lastError: void 0
2208
3113
  });
3114
+ if (continueAt !== void 0) await txn.setAlarm(continueAt);
2209
3115
  });
2210
3116
  }
2211
- async executeCleanup() {
2212
- for (;;) {
2213
- const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
2214
- if (!cleanup) {
2215
- await this.ctx.storage.deleteAlarm();
2216
- return;
3117
+ async executeScheduledCleanupPage() {
3118
+ const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
3119
+ if (terminalCleanup) {
3120
+ try {
3121
+ await this.executeTerminalCleanup(terminalCleanup);
3122
+ } catch (error) {
3123
+ await this.recordTerminalCleanupFailure(error, terminalCleanup);
2217
3124
  }
2218
- if (cleanup.phase === "tombstoned") {
2219
- await this.ctx.storage.deleteAlarm();
3125
+ return;
3126
+ }
3127
+ const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
3128
+ if (!cleanup || cleanup.phase === "tombstoned") {
3129
+ await this.ctx.storage.deleteAlarm();
3130
+ return;
3131
+ }
3132
+ try {
3133
+ await this.executeCleanup(cleanup);
3134
+ } catch (error) {
3135
+ await this.recordCleanupFailure(error, cleanup);
3136
+ }
3137
+ }
3138
+ async executeCleanup(cleanup) {
3139
+ if (cleanup.phase === "retained") {
3140
+ if (cleanup.dueAt.getTime() > this.now()) {
3141
+ await this.ctx.storage.setAlarm(cleanup.dueAt);
2220
3142
  return;
2221
3143
  }
2222
- if (cleanup.phase === "retained") {
2223
- if (cleanup.dueAt.getTime() > this.now()) {
2224
- await this.ctx.storage.setAlarm(cleanup.dueAt);
2225
- return;
2226
- }
2227
- await this.setCleanupPhase("retained", "index");
2228
- continue;
2229
- }
2230
- if (cleanup.phase === "index") {
2231
- const [indexMarkers, hookMarkers] = await Promise.all([this.ctx.storage.list({ prefix: INDEX_MARKER_PREFIX }), this.ctx.storage.list({ prefix: HOOK_MARKER_PREFIX })]);
2232
- const indexNamespace = this.namespace("WORKFLOW_INDEX");
2233
- await indexNamespace.get(indexNamespace.idFromName("index")).expireRun({
2234
- runId: cleanup.runId,
2235
- keys: [
2236
- workflowRunIndexKey(cleanup),
2237
- globalRunIndexKey(cleanup),
2238
- ...indexMarkers.values()
2239
- ],
2240
- hooks: Array.from(hookMarkers.values()),
2241
- expiredAt: cleanup.dueAt.getTime()
3144
+ await this.setCleanupPhase(cleanup, "index", {}, this.now() + 1);
3145
+ return;
3146
+ }
3147
+ if (cleanup.phase === "index") {
3148
+ await this.deleteIndexPage(cleanup);
3149
+ return;
3150
+ }
3151
+ if (cleanup.phase === "streams") {
3152
+ await this.deleteStreamPage(cleanup);
3153
+ return;
3154
+ }
3155
+ if (cleanup.phase === "queues") {
3156
+ await this.deleteQueuePage(cleanup);
3157
+ return;
3158
+ }
3159
+ if (cleanup.phase === "payload") await this.deletePayloadBatch();
3160
+ }
3161
+ async executeTerminalCleanup(cleanup) {
3162
+ if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
3163
+ else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
3164
+ else await this.deleteTerminalWaitPage(cleanup);
3165
+ }
3166
+ async deleteTerminalHookPage(cleanup) {
3167
+ const entries = await this.ctx.storage.list({
3168
+ prefix: HOOK_KEY_PREFIX,
3169
+ limit: 65
3170
+ });
3171
+ const page = Array.from(entries.values()).slice(0, TERMINAL_HOOK_PAGE_SIZE);
3172
+ await this.workflowIndex().releaseHookIndexes({
3173
+ runId: cleanup.runId,
3174
+ hooks: page.map((hook) => ({
3175
+ hookId: hook.hookId,
3176
+ token: hook.token
3177
+ })),
3178
+ terminal: true
3179
+ });
3180
+ await this.ctx.storage.transaction(async (txn) => {
3181
+ const current = await txn.get(TERMINAL_CLEANUP_KEY);
3182
+ if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
3183
+ const keys = page.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]);
3184
+ if (keys.length > 0) await txn.delete(keys);
3185
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3186
+ ...current,
3187
+ phase: entries.size <= TERMINAL_HOOK_PAGE_SIZE ? "markers" : "hooks",
3188
+ generation: current.generation + 1,
3189
+ attempts: 0,
3190
+ lastError: void 0
3191
+ });
3192
+ await txn.setAlarm(this.now() + 1);
3193
+ });
3194
+ }
3195
+ async deleteTerminalHookMarkerPage(cleanup) {
3196
+ const entries = await this.ctx.storage.list({
3197
+ prefix: HOOK_MARKER_PREFIX,
3198
+ limit: 65
3199
+ });
3200
+ const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
3201
+ await this.workflowIndex().releaseHookIndexes({
3202
+ runId: cleanup.runId,
3203
+ hooks: page.map(([, reference]) => reference),
3204
+ terminal: true
3205
+ });
3206
+ await this.ctx.storage.transaction(async (txn) => {
3207
+ const current = await txn.get(TERMINAL_CLEANUP_KEY);
3208
+ if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
3209
+ const keys = page.map(([key]) => key);
3210
+ if (keys.length > 0) await txn.delete(keys);
3211
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3212
+ ...current,
3213
+ phase: entries.size <= HOOK_MARKER_PAGE_SIZE ? "waits" : "markers",
3214
+ generation: current.generation + 1,
3215
+ attempts: 0,
3216
+ lastError: void 0
3217
+ });
3218
+ await txn.setAlarm(this.now() + 1);
3219
+ });
3220
+ }
3221
+ async deleteTerminalWaitPage(cleanup) {
3222
+ await this.ctx.storage.transaction(async (txn) => {
3223
+ const current = await txn.get(TERMINAL_CLEANUP_KEY);
3224
+ if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
3225
+ const entries = await txn.list({
3226
+ prefix: WAIT_KEY_PREFIX,
3227
+ limit: 129
3228
+ });
3229
+ const keys = Array.from(entries.keys()).slice(0, TERMINAL_WAIT_PAGE_SIZE);
3230
+ if (keys.length > 0) await txn.delete(keys);
3231
+ if (entries.size > TERMINAL_WAIT_PAGE_SIZE) {
3232
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3233
+ ...current,
3234
+ generation: current.generation + 1,
3235
+ attempts: 0,
3236
+ lastError: void 0
2242
3237
  });
2243
- await this.setCleanupPhase("index", "streams");
2244
- continue;
3238
+ await txn.setAlarm(this.now() + 1);
3239
+ return;
2245
3240
  }
2246
- if (cleanup.phase === "streams") {
2247
- const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
2248
- const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
2249
- const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime());
2250
- let deletedStreams = streams.length === 0 ? cleanup.deletedStreams : 0;
2251
- for (let offset = 0; offset < streams.length; offset += STREAM_DELETE_CONCURRENCY) {
2252
- const batch = streams.slice(offset, offset + STREAM_DELETE_CONCURRENCY);
2253
- const results = await Promise.all(batch.map((name) => {
2254
- return streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime());
2255
- }));
2256
- deletedStreams += results.filter((result) => result.deleted).length;
2257
- }
2258
- await this.ctx.storage.transaction(async (txn) => {
2259
- const current = await txn.get(CLEANUP_RECORD_KEY);
2260
- if (current?.phase === "streams") await txn.put(CLEANUP_RECORD_KEY, {
2261
- ...current,
2262
- deletedStreams
2263
- });
3241
+ await txn.delete(TERMINAL_CLEANUP_KEY);
3242
+ const retention = await txn.get(CLEANUP_RECORD_KEY);
3243
+ if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
3244
+ else await txn.deleteAlarm();
3245
+ });
3246
+ }
3247
+ async deleteIndexPage(cleanup) {
3248
+ const hookEntries = await this.ctx.storage.list({
3249
+ prefix: HOOK_MARKER_PREFIX,
3250
+ limit: 65
3251
+ });
3252
+ const hookMarkers = Array.from(hookEntries).slice(0, HOOK_MARKER_PAGE_SIZE);
3253
+ await this.workflowIndex().expireRun({
3254
+ runId: cleanup.runId,
3255
+ keys: [workflowRunIndexKey(cleanup), globalRunIndexKey(cleanup)],
3256
+ hooks: hookMarkers.map(([, value]) => value),
3257
+ expiredAt: cleanup.dueAt.getTime()
3258
+ });
3259
+ const markerKeys = hookMarkers.map(([key]) => key);
3260
+ await this.ctx.storage.transaction(async (txn) => {
3261
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3262
+ if (current?.phase !== "index" || current.generation !== cleanup.generation) return;
3263
+ if (markerKeys.length > 0) await txn.delete(markerKeys);
3264
+ if (hookEntries.size <= HOOK_MARKER_PAGE_SIZE) await txn.put(CLEANUP_RECORD_KEY, {
3265
+ ...current,
3266
+ phase: "streams",
3267
+ generation: current.generation + 1,
3268
+ attempts: 0,
3269
+ lastError: void 0
3270
+ });
3271
+ else await txn.put(CLEANUP_RECORD_KEY, {
3272
+ ...current,
3273
+ generation: current.generation + 1,
3274
+ attempts: 0,
3275
+ lastError: void 0
3276
+ });
3277
+ await txn.setAlarm(this.now() + 1);
3278
+ });
3279
+ }
3280
+ async deleteStreamPage(cleanup) {
3281
+ const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
3282
+ const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
3283
+ const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime(), { limit: STREAMS_PER_CLEANUP_PAGE });
3284
+ const completed = [];
3285
+ let remainingChunks = STREAM_CHUNKS_PER_CLEANUP_PAGE;
3286
+ let remainingBytes = STREAM_BYTES_PER_CLEANUP_PAGE;
3287
+ for (const name of streams) {
3288
+ if (remainingChunks <= 0 || remainingBytes <= 0) break;
3289
+ const result = await streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime(), {
3290
+ limit: remainingChunks,
3291
+ byteLimit: remainingBytes
3292
+ });
3293
+ remainingChunks -= result.chunks;
3294
+ remainingBytes -= result.bytes;
3295
+ if (result.done) completed.push(name);
3296
+ }
3297
+ const registryResult = await registry.finalizeRegistry(cleanup.runId, completed);
3298
+ await this.ctx.storage.transaction(async (txn) => {
3299
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3300
+ if (current?.phase !== "streams" || current.generation !== cleanup.generation) return;
3301
+ await txn.put(CLEANUP_RECORD_KEY, {
3302
+ ...current,
3303
+ deletedStreams: registryResult.deleted,
3304
+ phase: registryResult.done ? "queues" : "streams",
3305
+ generation: current.generation + 1,
3306
+ attempts: 0,
3307
+ lastError: void 0
3308
+ });
3309
+ await txn.setAlarm(this.now() + 1);
3310
+ });
3311
+ }
3312
+ async deleteQueuePage(cleanup) {
3313
+ const queues = this.namespace("WORKFLOW_QUEUE");
3314
+ let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3315
+ queueShard: 0,
3316
+ queueShardDeleted: 0
3317
+ };
3318
+ let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
3319
+ let expectedGeneration = cleanup.generation;
3320
+ while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
3321
+ const requestedShard = progress.queueShard;
3322
+ const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
3323
+ const reconciled = await this.ctx.storage.transaction(async (txn) => {
3324
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3325
+ if (current?.phase !== "queues") return null;
3326
+ if (current.generation !== expectedGeneration) return { stale: true };
3327
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3328
+ queueShard: 0,
3329
+ queueShardDeleted: 0
3330
+ };
3331
+ if (persisted.queueShard !== requestedShard) return { stale: true };
3332
+ if (result.deleted < persisted.queueShardDeleted) return { stale: true };
3333
+ const next = result.done ? {
3334
+ queueShard: persisted.queueShard + 1,
3335
+ queueShardDeleted: 0
3336
+ } : {
3337
+ ...persisted,
3338
+ queueShardDeleted: result.deleted
3339
+ };
3340
+ const delta = result.deleted - persisted.queueShardDeleted;
3341
+ await txn.put(CLEANUP_RECORD_KEY, {
3342
+ ...current,
3343
+ deletedQueueMessages: current.deletedQueueMessages + delta,
3344
+ generation: current.generation + 1,
3345
+ attempts: 0,
3346
+ lastError: void 0
2264
3347
  });
2265
- await registry.finalizeRegistry(cleanup.runId);
2266
- await this.setCleanupPhase("streams", "queues");
2267
- continue;
2268
- }
2269
- if (cleanup.phase === "queues") {
2270
- const queues = this.namespace("WORKFLOW_QUEUE");
2271
- let deletedQueueMessages = 0;
2272
- for (let shard = 0; shard < cleanup.queueShards; shard++) {
2273
- const result = await queues.get(queues.idFromName(`q:${shard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime());
2274
- deletedQueueMessages += result.deleted;
2275
- }
2276
- await this.setCleanupPhase("queues", "payload", { deletedQueueMessages });
2277
- continue;
2278
- }
2279
- if (cleanup.phase === "payload") {
2280
- if (!await this.deletePayloadBatch()) return;
2281
- }
3348
+ await txn.put(CLEANUP_PROGRESS_KEY, next);
3349
+ await txn.setAlarm(this.now() + 1);
3350
+ return {
3351
+ stale: false,
3352
+ progress: next,
3353
+ generation: current.generation + 1,
3354
+ applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
3355
+ };
3356
+ });
3357
+ if (!reconciled) return;
3358
+ if (reconciled.stale) return;
3359
+ progress = reconciled.progress;
3360
+ expectedGeneration = reconciled.generation;
3361
+ if (!reconciled.applied && progress.queueShard === requestedShard) break;
3362
+ if (!result.done && progress.queueShard === requestedShard) break;
2282
3363
  }
3364
+ await this.ctx.storage.transaction(async (txn) => {
3365
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3366
+ if (current?.phase !== "queues") return;
3367
+ if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3368
+ queueShard: 0,
3369
+ queueShardDeleted: 0
3370
+ }).queueShard >= current.queueShards) {
3371
+ await txn.put(CLEANUP_RECORD_KEY, {
3372
+ ...current,
3373
+ phase: "payload",
3374
+ generation: current.generation + 1
3375
+ });
3376
+ await txn.delete(CLEANUP_PROGRESS_KEY);
3377
+ await txn.setAlarm(this.now() + 1);
3378
+ }
3379
+ });
2283
3380
  }
2284
3381
  async deletePayloadBatch() {
2285
3382
  return await this.ctx.storage.transaction(async (txn) => {
2286
3383
  const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2287
3384
  if (!cleanup || cleanup.phase !== "payload") return true;
2288
- const entries = await txn.list({ limit: 259 });
3385
+ const entries = await txn.list({
3386
+ limit: 131,
3387
+ noCache: true
3388
+ });
2289
3389
  const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
2290
3390
  const now = new Date(this.now());
2291
3391
  await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
2292
- for (const key of keys) await txn.delete(key);
3392
+ if (keys.length > 0) await txn.delete(keys);
2293
3393
  if (keys.length > 0) {
2294
3394
  await txn.put(CLEANUP_RECORD_KEY, {
2295
3395
  ...cleanup,
2296
3396
  deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
3397
+ generation: cleanup.generation + 1,
2297
3398
  lastError: void 0
2298
3399
  });
2299
3400
  await txn.setAlarm(this.now() + 1);
@@ -2302,6 +3403,7 @@ var WorkflowRunDO = class extends DurableObject {
2302
3403
  await txn.put(CLEANUP_RECORD_KEY, {
2303
3404
  ...cleanup,
2304
3405
  phase: "tombstoned",
3406
+ generation: cleanup.generation + 1,
2305
3407
  attempts: 0,
2306
3408
  lastError: void 0,
2307
3409
  tombstonedAt: now
@@ -2310,14 +3412,31 @@ var WorkflowRunDO = class extends DurableObject {
2310
3412
  return true;
2311
3413
  });
2312
3414
  }
2313
- async recordCleanupFailure(error) {
3415
+ async recordCleanupFailure(error, expected) {
2314
3416
  await this.ctx.storage.transaction(async (txn) => {
2315
3417
  const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2316
- if (!cleanup || cleanup.phase === "tombstoned") return;
3418
+ if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation || cleanup.phase === "tombstoned") return;
2317
3419
  const attempts = cleanup.attempts + 1;
2318
3420
  const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
2319
3421
  await txn.put(CLEANUP_RECORD_KEY, {
2320
3422
  ...cleanup,
3423
+ generation: cleanup.generation + 1,
3424
+ attempts,
3425
+ lastError: error instanceof Error ? error.message : String(error),
3426
+ lastAttemptAt: new Date(this.now())
3427
+ });
3428
+ await txn.setAlarm(this.now() + delay);
3429
+ });
3430
+ }
3431
+ async recordTerminalCleanupFailure(error, expected) {
3432
+ await this.ctx.storage.transaction(async (txn) => {
3433
+ const cleanup = await txn.get(TERMINAL_CLEANUP_KEY);
3434
+ if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
3435
+ const attempts = cleanup.attempts + 1;
3436
+ const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
3437
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3438
+ ...cleanup,
3439
+ generation: cleanup.generation + 1,
2321
3440
  attempts,
2322
3441
  lastError: error instanceof Error ? error.message : String(error),
2323
3442
  lastAttemptAt: new Date(this.now())
@@ -2352,7 +3471,10 @@ async function startHarness(options = {}) {
2352
3471
  const fleet = new FakeFleet({
2353
3472
  runs: WorkflowRunDO,
2354
3473
  streams: StreamDO,
2355
- index: IndexDO,
3474
+ "run-catalog": RunCatalogDO,
3475
+ "run-fences": RunFenceDO,
3476
+ "hook-tokens": HookTokenDO,
3477
+ "hook-ids": HookIdDO,
2356
3478
  queue: QueueDO,
2357
3479
  ...options.extraClasses
2358
3480
  }, cellEnv);
@@ -2360,17 +3482,32 @@ async function startHarness(options = {}) {
2360
3482
  Object.assign(cellEnv, {
2361
3483
  WORKFLOW_DB: fleet.namespace("runs"),
2362
3484
  WORKFLOW_STREAMS: fleet.namespace("streams"),
2363
- WORKFLOW_INDEX: fleet.namespace("index"),
3485
+ WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
3486
+ WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
3487
+ WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
3488
+ WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
2364
3489
  WORKFLOW_QUEUE: fleet.namespace("queue")
2365
3490
  });
2366
3491
  const router = createRouter({
2367
3492
  WORKFLOW_DB: fleet.namespace("runs"),
2368
3493
  WORKFLOW_STREAMS: fleet.namespace("streams"),
2369
- WORKFLOW_INDEX: fleet.namespace("index"),
3494
+ WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
3495
+ WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
3496
+ WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
3497
+ WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
2370
3498
  WORKFLOW_QUEUE: fleet.namespace("queue"),
2371
3499
  WORLD_SECRET: options.secret
2372
3500
  });
2373
3501
  const server = http.createServer(async (req, res) => {
3502
+ const requestAbort = new AbortController();
3503
+ const abortRequest = () => {
3504
+ if (!requestAbort.signal.aborted) requestAbort.abort(new DOMException("HTTP client disconnected", "AbortError"));
3505
+ };
3506
+ const abortOnPrematureClose = () => {
3507
+ if (!res.writableEnded) abortRequest();
3508
+ };
3509
+ req.once("aborted", abortRequest);
3510
+ res.once("close", abortOnPrematureClose);
2374
3511
  try {
2375
3512
  const chunks = [];
2376
3513
  for await (const chunk of req) chunks.push(chunk);
@@ -2378,17 +3515,23 @@ async function startHarness(options = {}) {
2378
3515
  const request = new Request(`http://127.0.0.1${req.url ?? "/"}`, {
2379
3516
  method: req.method,
2380
3517
  headers: req.headers,
2381
- body: req.method === "GET" || req.method === "HEAD" || body.length === 0 ? null : body
3518
+ body: req.method === "GET" || req.method === "HEAD" || body.length === 0 ? null : body,
3519
+ signal: requestAbort.signal
2382
3520
  });
2383
3521
  const response = await router(request);
3522
+ if (requestAbort.signal.aborted || res.destroyed) return;
2384
3523
  res.writeHead(response.status, Object.fromEntries(response.headers.entries()));
2385
3524
  res.end(Buffer.from(await response.arrayBuffer()));
2386
3525
  } catch (error) {
3526
+ if (requestAbort.signal.aborted || res.destroyed) return;
2387
3527
  res.writeHead(500, { "content-type": "application/json" });
2388
3528
  res.end(JSON.stringify({ error: {
2389
3529
  name: "HarnessError",
2390
3530
  message: String(error)
2391
3531
  } }));
3532
+ } finally {
3533
+ req.off("aborted", abortRequest);
3534
+ res.off("close", abortOnPrematureClose);
2392
3535
  }
2393
3536
  });
2394
3537
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));