@ewhauser/world-celld 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/worker.js CHANGED
@@ -96,6 +96,342 @@ function rpcStringify(value) {
96
96
  function rpcParse(text) {
97
97
  return JSON.parse(text, rpcReviver);
98
98
  }
99
+ const INDEX_PROTOCOL_VERSION = "v1";
100
+ const HOOK_RELEASE_BATCH = 64;
101
+ /** Stable FNV-1a routing hash. Collisions share a shard but never a storage key. */
102
+ function stableIndexHash(value) {
103
+ let hash = 2166136261;
104
+ for (const byte of new TextEncoder().encode(value)) {
105
+ hash ^= byte;
106
+ hash = Math.imul(hash, 16777619);
107
+ }
108
+ return hash >>> 0;
109
+ }
110
+ function shardName(domain, value, cardinality) {
111
+ const width = Math.max(2, (cardinality - 1).toString(16).length);
112
+ const shard = stableIndexHash(value) % cardinality;
113
+ return `${domain}:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(width, "0")}`;
114
+ }
115
+ function runCatalogShardName(runId) {
116
+ return shardName("run-catalog", runId, 16);
117
+ }
118
+ function hookTokenShardName(token) {
119
+ return shardName("hook-token", token, 32);
120
+ }
121
+ function hookIdShardName(hookId) {
122
+ return shardName("hook-id", hookId, 32);
123
+ }
124
+ function runFenceCellName(runId) {
125
+ return `run-fence:${INDEX_PROTOCOL_VERSION}:${runId}`;
126
+ }
127
+ function allRunCatalogShardNames() {
128
+ return Array.from({ length: 16 }, (_, shard) => `run-catalog:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(2, "0")}`);
129
+ }
130
+ function stub(namespace, name) {
131
+ return namespace.get(namespace.idFromName(name));
132
+ }
133
+ function runKeys(run) {
134
+ const timestamp = run.createdAt.getTime().toString().padStart(13, "0");
135
+ return [`run:${run.workflowName}:${timestamp}:${run.runId}`, `runall:${timestamp}:${run.runId}`];
136
+ }
137
+ function groupByShard(values, route) {
138
+ const groups = /* @__PURE__ */ new Map();
139
+ for (const value of values) {
140
+ const name = route(value);
141
+ const group = groups.get(name);
142
+ if (group) group.push(value);
143
+ else groups.set(name, [value]);
144
+ }
145
+ return groups;
146
+ }
147
+ function sameOwner$1(left, right) {
148
+ return left.runId === right.runId && left.hookId === right.hookId;
149
+ }
150
+ function compareBytes(left, right) {
151
+ const length = Math.min(left.byteLength, right.byteLength);
152
+ for (let index = 0; index < length; index++) {
153
+ const difference = left[index] - right[index];
154
+ if (difference !== 0) return difference;
155
+ }
156
+ return left.byteLength - right.byteLength;
157
+ }
158
+ async function inBatches(values, operation) {
159
+ let deleted = 0;
160
+ for (let offset = 0; offset < values.length; offset += HOOK_RELEASE_BATCH) deleted += (await operation(values.slice(offset, offset + HOOK_RELEASE_BATCH))).deleted;
161
+ return deleted;
162
+ }
163
+ function createWorkflowIndex(bindings) {
164
+ const fence = (runId) => stub(bindings.runFences, runFenceCellName(runId));
165
+ const releaseHooks = async (request, fencedAt) => {
166
+ const tokenGroups = groupByShard(request.hooks, (hook) => hookTokenShardName(hook.token));
167
+ const idGroups = groupByShard(request.hooks, (hook) => hookIdShardName(hook.hookId));
168
+ const releases = [];
169
+ for (const [name, hooks] of tokenGroups) {
170
+ const target = stub(bindings.hookTokens, name);
171
+ releases.push(inBatches(hooks.map((hook) => ({
172
+ token: hook.token,
173
+ owner: {
174
+ runId: request.runId,
175
+ hookId: hook.hookId
176
+ }
177
+ })), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
178
+ }
179
+ for (const [name, hooks] of idGroups) {
180
+ const target = stub(bindings.hookIds, name);
181
+ releases.push(inBatches(hooks.map((hook) => ({
182
+ hookId: hook.hookId,
183
+ owner: {
184
+ runId: request.runId,
185
+ hookId: hook.hookId
186
+ }
187
+ })), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
188
+ }
189
+ return (await Promise.all(releases)).reduce((total, count) => total + count, 0);
190
+ };
191
+ return {
192
+ async commitRun(run, serializedMetadata) {
193
+ if (isTerminalWorkflowRunStatus(run.status)) await fence(run.runId).fenceTerminal(Date.now());
194
+ return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata);
195
+ },
196
+ async listRuns(options) {
197
+ const suppliedLimit = options?.limit;
198
+ const limit = Math.min(1e3, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : 1e3));
199
+ const pages = await Promise.all(allRunCatalogShardNames().map((name) => stub(bindings.runCatalog, name).list({
200
+ ...options,
201
+ limit: limit + 1
202
+ })));
203
+ const encoder = new TextEncoder();
204
+ const candidates = pages.flatMap((shardPage) => shardPage.keys.map((entry) => ({
205
+ entry,
206
+ encodedName: encoder.encode(entry.name)
207
+ }))).toSorted((left, right) => {
208
+ const compared = compareBytes(left.encodedName, right.encodedName);
209
+ return options?.reverse ? -compared : compared;
210
+ });
211
+ const page = candidates.slice(0, limit).map(({ entry }) => entry);
212
+ const listComplete = candidates.length <= limit && pages.every((entry) => entry.list_complete);
213
+ return {
214
+ keys: page,
215
+ list_complete: listComplete,
216
+ cursor: listComplete ? void 0 : page.at(-1)?.name
217
+ };
218
+ },
219
+ getHookByToken(token) {
220
+ return stub(bindings.hookTokens, hookTokenShardName(token)).get(token);
221
+ },
222
+ getHookById(hookId) {
223
+ return stub(bindings.hookIds, hookIdShardName(hookId)).get(hookId);
224
+ },
225
+ async reserveHook(token, owner) {
226
+ const proposedClaimId = crypto.randomUUID();
227
+ const tokenTarget = stub(bindings.hookTokens, hookTokenShardName(token));
228
+ const tokenClaim = await tokenTarget.reserve(token, owner, proposedClaimId);
229
+ if (!tokenClaim.claimed && !sameOwner$1(tokenClaim.holder, owner)) return {
230
+ admitted: false,
231
+ holder: tokenClaim.holder
232
+ };
233
+ const tokenClaimId = tokenClaim.claimed ? tokenClaim.claimId : void 0;
234
+ const hookIdTarget = stub(bindings.hookIds, hookIdShardName(owner.hookId));
235
+ const sharedClaimId = tokenClaimId ?? proposedClaimId;
236
+ let hookIdClaim;
237
+ try {
238
+ hookIdClaim = await hookIdTarget.reserve(owner.hookId, owner, sharedClaimId);
239
+ } catch (error) {
240
+ await Promise.allSettled([...tokenClaimId ? [tokenTarget.releaseClaim(token, owner, tokenClaimId)] : [], hookIdTarget.releaseClaim(owner.hookId, owner, sharedClaimId)]);
241
+ throw error;
242
+ }
243
+ if (!hookIdClaim.claimed) {
244
+ if (tokenClaimId) await tokenTarget.releaseClaim(token, owner, tokenClaimId);
245
+ return {
246
+ admitted: false,
247
+ holder: hookIdClaim.holder
248
+ };
249
+ }
250
+ return {
251
+ admitted: true,
252
+ reservation: {
253
+ claimId: `${tokenClaimId ?? "-"}:${hookIdClaim.claimId}`,
254
+ tokenClaimId,
255
+ hookIdClaimId: hookIdClaim.claimId
256
+ }
257
+ };
258
+ },
259
+ async finalizeHookIndexes(token, hookId, serializedHook, owner, reservation) {
260
+ if (!(await stub(bindings.hookTokens, hookTokenShardName(token)).finalize(token, serializedHook, owner, reservation?.tokenClaimId)).stored) return;
261
+ await stub(bindings.hookIds, hookIdShardName(hookId)).publish(hookId, serializedHook, owner, reservation?.hookIdClaimId);
262
+ },
263
+ async releaseHookReservation(token, owner, reservation) {
264
+ const releases = [];
265
+ if (reservation.tokenClaimId) releases.push(stub(bindings.hookTokens, hookTokenShardName(token)).releaseClaim(token, owner, reservation.tokenClaimId));
266
+ if (reservation.hookIdClaimId) releases.push(stub(bindings.hookIds, hookIdShardName(owner.hookId)).releaseClaim(owner.hookId, owner, reservation.hookIdClaimId));
267
+ await Promise.all(releases);
268
+ },
269
+ async releaseHookIndexes(request) {
270
+ if (request.terminal) await fence(request.runId).fenceTerminal(Date.now());
271
+ return { deleted: await releaseHooks(request, Date.now()) };
272
+ },
273
+ async expireRun(request) {
274
+ await fence(request.runId).fenceExpired(request.expiredAt);
275
+ const [catalog, hookDeletes] = await Promise.all([stub(bindings.runCatalog, runCatalogShardName(request.runId)).expireRun(request.runId, request.keys, request.expiredAt), releaseHooks({
276
+ runId: request.runId,
277
+ hooks: request.hooks,
278
+ terminal: true
279
+ }, request.expiredAt)]);
280
+ return { deleted: catalog.deleted + hookDeletes };
281
+ }
282
+ };
283
+ }
284
+ //#endregion
285
+ //#region src/stream-protocol.ts
286
+ /**
287
+ * Hard-cutover stream protocol shared by the client, Worker router, and
288
+ * StreamDO. Chunk payloads use a compact binary envelope on the public HTTP
289
+ * boundary and Uint8Array values on the DO/storage boundary.
290
+ */
291
+ const STREAM_BATCH_CONTENT_TYPE = "application/vnd.world-celld.stream-batch.v1";
292
+ /** A chunk remains below the 2 MiB SQLite-backed DO key/value limit. */
293
+ const MAX_STREAM_CHUNK_BYTES = 1048576;
294
+ /** Keeps serialized RPC bodies well below the 32 MiB Workers RPC limit. */
295
+ const MAX_STREAM_BATCH_BYTES = 8388608;
296
+ const MAX_STREAM_READ_BYTES = 8388608;
297
+ const MAX_STREAM_ERROR_BYTES = 16384;
298
+ /** Must remain below the default 30 second fleet request deadline. */
299
+ const MAX_STREAM_LONG_POLL_MS = 2e4;
300
+ const WRITE_MAGIC = 1464030039;
301
+ const WRITE_ACK_MAGIC = 1464030017;
302
+ const READ_MAGIC = 1464030034;
303
+ const PROTOCOL_VERSION = 1;
304
+ const WRITE_HEADER_BYTES = 8;
305
+ const WRITE_ACK_BYTES = 16;
306
+ const READ_HEADER_BYTES = 28;
307
+ const STATE_TO_WIRE = {
308
+ open: 0,
309
+ closed: 1,
310
+ errored: 2,
311
+ expired: 3
312
+ };
313
+ function protocolError(message) {
314
+ const error = /* @__PURE__ */ new Error(`world-celld: invalid stream protocol: ${message}`);
315
+ error.name = "StreamProtocolError";
316
+ return error;
317
+ }
318
+ function requireSafeIndex(value, field) {
319
+ if (!Number.isSafeInteger(value) || value < -1 || value > 2147483647) throw protocolError(`${field} is out of range`);
320
+ }
321
+ function validateStreamWriteChunks(chunks) {
322
+ if (chunks.length === 0) throw protocolError("batch must contain at least one chunk");
323
+ if (chunks.length > 32) throw protocolError(`batch exceeds 32 chunks`);
324
+ let totalBytes = 0;
325
+ for (const chunk of chunks) {
326
+ if (!(chunk instanceof Uint8Array)) throw protocolError("chunk is not binary data");
327
+ if (chunk.byteLength > 1048576) throw protocolError(`chunk exceeds ${MAX_STREAM_CHUNK_BYTES} bytes`);
328
+ totalBytes += chunk.byteLength;
329
+ if (totalBytes > 8388608) throw protocolError(`batch exceeds ${MAX_STREAM_BATCH_BYTES} bytes`);
330
+ }
331
+ return totalBytes;
332
+ }
333
+ function validateStreamReadRequest(request) {
334
+ if (!request.runId) throw protocolError("runId is required");
335
+ if (!Number.isSafeInteger(request.startIndex) || request.startIndex < 0) throw protocolError("startIndex must be a non-negative safe integer");
336
+ if (!Number.isSafeInteger(request.maxChunks) || request.maxChunks < 0 || request.maxChunks > 32) throw protocolError(`maxChunks must be between 0 and 32`);
337
+ if (!Number.isSafeInteger(request.maxBytes) || request.maxBytes < 1048576 || request.maxBytes > 8388608) throw protocolError(`maxBytes must be between ${MAX_STREAM_CHUNK_BYTES} and ${MAX_STREAM_READ_BYTES}`);
338
+ if (!Number.isSafeInteger(request.waitMs) || request.waitMs < 0 || request.waitMs > 2e4) throw protocolError(`waitMs must be between 0 and ${MAX_STREAM_LONG_POLL_MS}`);
339
+ }
340
+ function decodeStreamWriteBatch(encoded) {
341
+ if (encoded.byteLength < WRITE_HEADER_BYTES) throw protocolError("truncated write header");
342
+ const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength);
343
+ if (view.getUint32(0) !== WRITE_MAGIC) throw protocolError("bad write magic");
344
+ if (view.getUint16(4) !== PROTOCOL_VERSION) throw protocolError("unsupported write version");
345
+ const count = view.getUint16(6);
346
+ const chunks = [];
347
+ let offset = WRITE_HEADER_BYTES;
348
+ for (let index = 0; index < count; index++) {
349
+ if (offset + 4 > encoded.byteLength) throw protocolError("truncated chunk length");
350
+ const length = view.getUint32(offset);
351
+ offset += 4;
352
+ if (offset + length > encoded.byteLength) throw protocolError("truncated chunk data");
353
+ chunks.push(new Uint8Array(encoded.subarray(offset, offset + length)));
354
+ offset += length;
355
+ }
356
+ if (offset !== encoded.byteLength) throw protocolError("trailing write data");
357
+ validateStreamWriteChunks(chunks);
358
+ return chunks;
359
+ }
360
+ function encodeStreamWriteResult(result) {
361
+ requireSafeIndex(result.startIndex, "startIndex");
362
+ requireSafeIndex(result.tailIndex, "tailIndex");
363
+ if (!Number.isSafeInteger(result.count) || result.count < 1 || result.count > 32) throw protocolError("write count is out of range");
364
+ const encoded = new Uint8Array(WRITE_ACK_BYTES);
365
+ const view = new DataView(encoded.buffer);
366
+ view.setUint32(0, WRITE_ACK_MAGIC);
367
+ view.setUint16(4, PROTOCOL_VERSION);
368
+ view.setUint16(6, result.count);
369
+ view.setInt32(8, result.startIndex);
370
+ view.setInt32(12, result.tailIndex);
371
+ return encoded;
372
+ }
373
+ function encodeStreamReadResult(result) {
374
+ requireSafeIndex(result.startIndex, "startIndex");
375
+ requireSafeIndex(result.tailIndex, "tailIndex");
376
+ let chunkBytes = 0;
377
+ for (const chunk of result.chunks) chunkBytes += 4 + chunk.byteLength;
378
+ if (result.chunks.length > 32 || chunkBytes > 8388736) throw protocolError("read result exceeds configured bounds");
379
+ const encoder = new TextEncoder();
380
+ const errorName = result.error ? encoder.encode(result.error.name) : /* @__PURE__ */ new Uint8Array();
381
+ const errorMessage = result.error ? encoder.encode(result.error.message) : /* @__PURE__ */ new Uint8Array();
382
+ if (errorName.byteLength + errorMessage.byteLength > 16384) throw protocolError(`stream error exceeds ${MAX_STREAM_ERROR_BYTES} bytes`);
383
+ const encoded = new Uint8Array(READ_HEADER_BYTES + chunkBytes + errorName.byteLength + errorMessage.byteLength);
384
+ const view = new DataView(encoded.buffer);
385
+ view.setUint32(0, READ_MAGIC);
386
+ view.setUint8(4, PROTOCOL_VERSION);
387
+ const state = STATE_TO_WIRE[result.state];
388
+ if (state === void 0) throw protocolError("unknown stream state");
389
+ view.setUint8(5, state);
390
+ view.setUint8(6, result.timedOut ? 1 : 0);
391
+ view.setUint8(7, 0);
392
+ view.setInt32(8, result.startIndex);
393
+ view.setInt32(12, result.tailIndex);
394
+ view.setUint32(16, result.chunks.length);
395
+ view.setUint32(20, errorName.byteLength);
396
+ view.setUint32(24, errorMessage.byteLength);
397
+ let offset = READ_HEADER_BYTES;
398
+ for (const chunk of result.chunks) {
399
+ view.setUint32(offset, chunk.byteLength);
400
+ offset += 4;
401
+ encoded.set(chunk, offset);
402
+ offset += chunk.byteLength;
403
+ }
404
+ encoded.set(errorName, offset);
405
+ offset += errorName.byteLength;
406
+ encoded.set(errorMessage, offset);
407
+ return encoded;
408
+ }
409
+ function normalizeStreamError(error) {
410
+ let normalized;
411
+ if (error instanceof Error) normalized = {
412
+ name: error.name || "Error",
413
+ message: error.message
414
+ };
415
+ else if (typeof error === "string") normalized = {
416
+ name: "Error",
417
+ message: error
418
+ };
419
+ else if (error && typeof error === "object" && typeof error.name === "string" && typeof error.message === "string") normalized = {
420
+ name: error.name,
421
+ message: error.message
422
+ };
423
+ else normalized = {
424
+ name: "Error",
425
+ message: String(error)
426
+ };
427
+ normalized.name = normalized.name.slice(0, 1024);
428
+ normalized.message = normalized.message.slice(0, 4096);
429
+ while (new TextEncoder().encode(normalized.name).byteLength + new TextEncoder().encode(normalized.message).byteLength > MAX_STREAM_ERROR_BYTES) {
430
+ normalized.message = normalized.message.slice(0, Math.max(0, normalized.message.length - 256));
431
+ if (normalized.message.length === 0) normalized.name = normalized.name.slice(0, Math.max(0, normalized.name.length - 256));
432
+ }
433
+ return normalized;
434
+ }
99
435
  //#endregion
100
436
  //#region src/worker/auth.ts
101
437
  /**
@@ -145,13 +481,23 @@ async function authenticate(request, secret) {
145
481
  * Routes:
146
482
  * GET /v1/health — liveness + spec version (unauthenticated)
147
483
  * POST /v1/rpc/{binding}/{name}/{method} — DO RPC (bearer auth)
484
+ * POST /v1/index/{domain}/{method} — cohesive index operation
485
+ * GET /v1/streams/{name}/chunks — bounded binary long-poll read
486
+ * POST /v1/streams/{name}/chunks — bounded binary batch append
148
487
  *
149
- * The RPC body is an rpc-codec-encoded JSON array of arguments; the response
150
- * body is the rpc-codec-encoded return value. Only whitelisted methods
151
- * dispatch — everything else (including fetch/alarm) is unreachable.
488
+ * Generic RPC bodies use the tagged JSON codec. Stream chunk bodies use the
489
+ * compact binary stream protocol. Only whitelisted routes/methods dispatch.
152
490
  */
153
491
  const WORLD_NAME = "world-celld";
154
492
  const WORLD_VERSION = "0.1.0";
493
+ function workflowIndex(env) {
494
+ return createWorkflowIndex({
495
+ runCatalog: env.WORKFLOW_RUN_CATALOG,
496
+ runFences: env.WORKFLOW_RUN_FENCES,
497
+ hookTokens: env.WORKFLOW_HOOK_TOKENS,
498
+ hookIds: env.WORKFLOW_HOOK_IDS
499
+ });
500
+ }
155
501
  const BINDINGS = {
156
502
  runs: {
157
503
  env: "WORKFLOW_DB",
@@ -166,16 +512,15 @@ const BINDINGS = {
166
512
  "getCleanupStatus",
167
513
  "scheduleCleanup",
168
514
  "cleanupNow",
169
- "rearmCleanup"
515
+ "rearmCleanup",
516
+ "resolveHookTokenClaim"
170
517
  ])
171
518
  },
172
519
  streams: {
173
520
  env: "WORKFLOW_STREAMS",
174
521
  methods: /* @__PURE__ */ new Set([
175
- "writeChunk",
176
522
  "closeStream",
177
- "getChunks",
178
- "getInfo",
523
+ "failStream",
179
524
  "registerStream",
180
525
  "listStreams",
181
526
  "expireRegistry",
@@ -183,20 +528,13 @@ const BINDINGS = {
183
528
  "expireStream"
184
529
  ])
185
530
  },
186
- index: {
187
- env: "WORKFLOW_INDEX",
188
- methods: /* @__PURE__ */ new Set([
189
- "get",
190
- "put",
191
- "delete",
192
- "putOwned",
193
- "expireRun",
194
- "list",
195
- "reserveHookToken",
196
- "finalizeHookIndexes",
197
- "releaseHookToken",
198
- "deleteHookIndexes"
199
- ])
531
+ "hook-tokens": {
532
+ env: "WORKFLOW_HOOK_TOKENS",
533
+ methods: /* @__PURE__ */ new Set(["get"])
534
+ },
535
+ "hook-ids": {
536
+ env: "WORKFLOW_HOOK_IDS",
537
+ methods: /* @__PURE__ */ new Set(["get"])
200
538
  },
201
539
  queue: {
202
540
  env: "WORKFLOW_QUEUE",
@@ -213,6 +551,7 @@ const BINDINGS = {
213
551
  };
214
552
  /** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
215
553
  const MAX_BODY_BYTES = 33554432;
554
+ const MAX_STREAM_WRITE_BODY_BYTES = 8389632;
216
555
  function errorResponse(status, name, message) {
217
556
  return Response.json({ error: {
218
557
  name,
@@ -241,6 +580,49 @@ async function readBoundedBody(request) {
241
580
  reader.releaseLock();
242
581
  }
243
582
  }
583
+ async function readBoundedBytes(request, maxBytes) {
584
+ if (!request.body) return /* @__PURE__ */ new Uint8Array();
585
+ const reader = request.body.getReader();
586
+ const chunks = [];
587
+ let total = 0;
588
+ try {
589
+ for (;;) {
590
+ const { done, value } = await reader.read();
591
+ if (done) break;
592
+ total += value.byteLength;
593
+ if (total > maxBytes) {
594
+ await reader.cancel("stream body exceeds configured limit").catch(() => void 0);
595
+ return null;
596
+ }
597
+ chunks.push(value);
598
+ }
599
+ } finally {
600
+ reader.releaseLock();
601
+ }
602
+ const body = new Uint8Array(total);
603
+ let offset = 0;
604
+ for (const chunk of chunks) {
605
+ body.set(chunk, offset);
606
+ offset += chunk.byteLength;
607
+ }
608
+ return body;
609
+ }
610
+ function parseBoundedInteger(url, name, minimum, maximum) {
611
+ const raw = url.searchParams.get(name);
612
+ const value = raw === null ? NaN : Number(raw);
613
+ if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
614
+ const error = /* @__PURE__ */ new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
615
+ error.name = "StreamProtocolError";
616
+ throw error;
617
+ }
618
+ return value;
619
+ }
620
+ function streamResponse(body) {
621
+ return new Response(body, {
622
+ status: 200,
623
+ headers: { "content-type": STREAM_BATCH_CONTENT_TYPE }
624
+ });
625
+ }
244
626
  function createRouter(env) {
245
627
  return async (request) => {
246
628
  const url = new URL(request.url);
@@ -252,9 +634,79 @@ function createRouter(env) {
252
634
  version: WORLD_VERSION,
253
635
  specVersion: SPEC_VERSION_CURRENT
254
636
  });
255
- if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
256
637
  const auth = await authenticate(request, env.WORLD_SECRET);
257
638
  if (!auth.ok) return auth.response;
639
+ if (parts[1] === "index" && parts.length === 4) {
640
+ if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
641
+ if (Number(request.headers.get("content-length") ?? "0") > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
642
+ let args;
643
+ try {
644
+ const text = await readBoundedBody(request);
645
+ if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
646
+ const parsed = rpcParse(text);
647
+ if (!Array.isArray(parsed)) return errorResponse(400, "BadRequest", "body must be an argument array");
648
+ args = parsed;
649
+ } catch {
650
+ return errorResponse(400, "BadRequest", "malformed rpc body");
651
+ }
652
+ try {
653
+ const index = workflowIndex(env);
654
+ let result;
655
+ const operation = `${parts[2]}.${parts[3]}`;
656
+ if (operation === "runs.list") result = await index.listRuns(args[0]);
657
+ else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
658
+ else if (operation === "runs.expire") result = await index.expireRun(args[0]);
659
+ else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
660
+ else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
661
+ else if (operation === "hooks.release-reservation") result = await index.releaseHookReservation(args[0], args[1], args[2]);
662
+ else if (operation === "hooks.release") result = await index.releaseHookIndexes(args[0]);
663
+ else return errorResponse(404, "NotFound", `unknown index operation: ${operation}`);
664
+ return new Response(rpcStringify(result ?? null), {
665
+ status: 200,
666
+ headers: { "content-type": "application/json" }
667
+ });
668
+ } catch (error) {
669
+ const err = error;
670
+ return Response.json({ error: {
671
+ name: err?.name ?? "Error",
672
+ message: err?.message ?? String(error),
673
+ status: typeof err?.status === "number" ? err.status : void 0
674
+ } }, { status: 500 });
675
+ }
676
+ }
677
+ if (parts[1] === "streams" && parts.length === 4 && parts[3] === "chunks") {
678
+ const namespace = env.WORKFLOW_STREAMS;
679
+ if (!namespace) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_STREAMS");
680
+ const name = decodeURIComponent(parts[2]);
681
+ const runId = url.searchParams.get("runId");
682
+ if (!runId) return errorResponse(400, "BadRequest", "runId is required");
683
+ const stub = namespace.get(namespace.idFromName(name));
684
+ try {
685
+ if (request.method === "POST") {
686
+ if (!request.headers.get("content-type")?.startsWith("application/vnd.world-celld.stream-batch.v1")) return errorResponse(415, "UnsupportedMediaType", STREAM_BATCH_CONTENT_TYPE);
687
+ if (Number(request.headers.get("content-length") ?? "0") > MAX_STREAM_WRITE_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
688
+ const body = await readBoundedBytes(request, MAX_STREAM_WRITE_BODY_BYTES);
689
+ if (body === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
690
+ return streamResponse(encodeStreamWriteResult(await stub.writeChunks(runId, decodeStreamWriteBatch(body))));
691
+ }
692
+ if (request.method === "GET") return streamResponse(encodeStreamReadResult(await stub.readChunks({
693
+ runId,
694
+ startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
695
+ maxChunks: parseBoundedInteger(url, "maxChunks", 0, 32),
696
+ maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
697
+ waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
698
+ }, request.signal)));
699
+ return errorResponse(405, "MethodNotAllowed", "expected GET or POST");
700
+ } catch (error) {
701
+ const err = error;
702
+ return Response.json({ error: {
703
+ name: err?.name ?? "Error",
704
+ message: err?.message ?? String(error),
705
+ status: typeof err?.status === "number" ? err.status : void 0
706
+ } }, { status: err?.name === "StreamProtocolError" ? 400 : 500 });
707
+ }
708
+ }
709
+ if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
258
710
  if (request.method !== "POST" || parts.length !== 5) return errorResponse(404, "NotFound", "expected POST /v1/rpc/{binding}/{name}/{method}");
259
711
  const [, , bindingKey, encodedName, method] = parts;
260
712
  const binding = BINDINGS[bindingKey];
@@ -299,160 +751,242 @@ function compact(obj) {
299
751
  return value;
300
752
  }
301
753
  //#endregion
302
- //#region src/worker/durable-objects/IndexDO.ts
754
+ //#region src/worker/durable-objects/hook-index-shared.ts
303
755
  function sameOwner(left, right) {
304
756
  return left.runId === right.runId && left.hookId === right.hookId;
305
757
  }
306
- function ownerFromHook(raw) {
758
+ function ownerFromSerializedHook(raw) {
307
759
  const hook = parse(raw);
308
760
  return {
309
761
  runId: hook.runId,
310
762
  hookId: hook.hookId
311
763
  };
312
764
  }
313
- /**
314
- * Global index cell, replacing the Workers KV namespace world-cloudflare
315
- * uses for run listing and hook token/id lookup.
316
- *
317
- * A single cell (named `index`) serializes every read and write. Hook-token
318
- * reservation, publication, and release use storage transactions so
319
- * concurrent runs cannot both acquire the same token.
320
- *
321
- * The RPC surface matches the storage-layer KVNamespace interface bit-for-bit
322
- * (including MockKVNamespace's pagination contract: cursor = last returned
323
- * key, `list_complete` reflects whether further keys exist), so the vendored
324
- * storage.ts works against it unchanged.
325
- */
326
- var IndexDO = class extends DurableObject {
327
- async get(key) {
328
- return await this.ctx.storage.get(key) ?? null;
329
- }
330
- async put(key, value) {
331
- await this.ctx.storage.put(key, value);
765
+ function indexKey(prefix, value) {
766
+ return `${prefix}:${encodeURIComponent(value)}`;
767
+ }
768
+ /** Compact shard-local fence for every hook owned by one terminal or expired run. */
769
+ function shardRunFenceKey(runId) {
770
+ return `runfence:${encodeURIComponent(runId)}`;
771
+ }
772
+ async function runIsActive(env, runId) {
773
+ const namespace = env.WORKFLOW_RUN_FENCES;
774
+ if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
775
+ return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
776
+ }
777
+ //#endregion
778
+ //#region src/worker/durable-objects/HookIdDO.ts
779
+ const MAX_RELEASE_BATCH$1 = 64;
780
+ function recordKey$1(hookId) {
781
+ return indexKey("hookid", hookId);
782
+ }
783
+ function claimKey$1(hookId) {
784
+ return indexKey("claim", hookId);
785
+ }
786
+ /** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
787
+ var HookIdDO = class extends DurableObject {
788
+ async get(hookId) {
789
+ const raw = await this.ctx.storage.get(recordKey$1(hookId));
790
+ if (!raw) return null;
791
+ const owner = ownerFromSerializedHook(raw);
792
+ return await runIsActive(this.env, owner.runId) ? raw : null;
793
+ }
794
+ async reserve(hookId, owner, proposedClaimId) {
795
+ return await this.ctx.storage.transaction(async (txn) => {
796
+ const record = recordKey$1(hookId);
797
+ const claim = claimKey$1(hookId);
798
+ const values = await txn.get([record, claim]);
799
+ const indexed = values.get(record);
800
+ if (indexed !== void 0) return {
801
+ claimed: false,
802
+ holder: ownerFromSerializedHook(indexed)
803
+ };
804
+ const existing = values.get(claim);
805
+ if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
806
+ claimed: false,
807
+ holder: existing.owner
808
+ };
809
+ if (existing === void 0 || existing.claimId !== proposedClaimId) {
810
+ await txn.put(claim, {
811
+ owner,
812
+ claimId: proposedClaimId,
813
+ reservedAt: Date.now()
814
+ });
815
+ return {
816
+ claimed: true,
817
+ claimId: proposedClaimId
818
+ };
819
+ }
820
+ return {
821
+ claimed: true,
822
+ claimId: existing.claimId
823
+ };
824
+ });
332
825
  }
333
- /** Publish a derived index only while its owning run is not expired. */
334
- async putOwned(runId, key, value) {
826
+ async publish(hookId, serializedHook, owner, claimId) {
827
+ if (!await runIsActive(this.env, owner.runId)) {
828
+ await this.releaseBatch([{
829
+ hookId,
830
+ owner
831
+ }], Date.now(), true);
832
+ return { stored: false };
833
+ }
335
834
  return await this.ctx.storage.transaction(async (txn) => {
336
- if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
337
- await txn.put(key, value);
835
+ const record = recordKey$1(hookId);
836
+ const claim = claimKey$1(hookId);
837
+ const runFence = shardRunFenceKey(owner.runId);
838
+ const values = await txn.get([
839
+ record,
840
+ claim,
841
+ runFence
842
+ ]);
843
+ if (values.has(runFence)) {
844
+ const existingClaim = values.get(claim);
845
+ if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
846
+ return { stored: false };
847
+ }
848
+ const indexed = values.get(record);
849
+ if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
850
+ const existingClaim = values.get(claim);
851
+ if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
852
+ if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
853
+ await txn.put(record, serializedHook);
854
+ if (existingClaim !== void 0) await txn.delete(claim);
338
855
  return { stored: true };
339
856
  });
340
857
  }
341
- async delete(key) {
342
- await this.ctx.storage.delete(key);
858
+ async releaseClaim(hookId, owner, claimId) {
859
+ await this.ctx.storage.transaction(async (txn) => {
860
+ const key = claimKey$1(hookId);
861
+ const claim = await txn.get(key);
862
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
863
+ });
343
864
  }
344
- async list(options) {
345
- const prefix = options?.prefix ?? "";
346
- const limit = options?.limit ?? 1e3;
347
- const entries = await this.ctx.storage.list({
348
- prefix,
349
- ...options?.reverse ? {
350
- reverse: true,
351
- end: options.cursor
352
- } : { startAfter: options?.cursor },
353
- limit: limit + 1
865
+ async releaseBatch(hooks, fencedAt, runFenced) {
866
+ if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
867
+ if (hooks.length === 0) return { deleted: 0 };
868
+ return await this.ctx.storage.transaction(async (txn) => {
869
+ const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
870
+ const values = await txn.get(keys);
871
+ const deletes = [];
872
+ for (const { hookId, owner } of hooks) {
873
+ const record = recordKey$1(hookId);
874
+ const claim = claimKey$1(hookId);
875
+ const indexed = values.get(record);
876
+ const existingClaim = values.get(claim);
877
+ if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
878
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
879
+ }
880
+ if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
881
+ return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
354
882
  });
355
- const names = Array.from(entries.keys());
356
- const page = names.slice(0, limit);
357
- const listComplete = names.length <= limit;
358
- return {
359
- keys: page.map((name) => ({ name })),
360
- list_complete: listComplete,
361
- cursor: listComplete ? void 0 : page.at(-1)
362
- };
363
883
  }
364
- async reserveHookToken(token, owner) {
365
- return this.ctx.storage.transaction(async (txn) => {
366
- const indexedHook = await txn.get(`hook:${token}`);
367
- if (indexedHook !== void 0) return {
884
+ };
885
+ //#endregion
886
+ //#region src/worker/durable-objects/HookTokenDO.ts
887
+ const MAX_RELEASE_BATCH = 64;
888
+ function recordKey(token) {
889
+ return indexKey("hook", token);
890
+ }
891
+ function claimKey(token) {
892
+ return indexKey("claim", token);
893
+ }
894
+ /** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
895
+ var HookTokenDO = class extends DurableObject {
896
+ async get(token) {
897
+ const raw = await this.ctx.storage.get(recordKey(token));
898
+ if (!raw) return null;
899
+ const owner = ownerFromSerializedHook(raw);
900
+ return await runIsActive(this.env, owner.runId) ? raw : null;
901
+ }
902
+ async reserve(token, owner, proposedClaimId) {
903
+ return await this.ctx.storage.transaction(async (txn) => {
904
+ const record = recordKey(token);
905
+ const claim = claimKey(token);
906
+ const values = await txn.get([record, claim]);
907
+ const indexed = values.get(record);
908
+ if (indexed !== void 0) return {
368
909
  claimed: false,
369
- holder: ownerFromHook(indexedHook)
910
+ holder: ownerFromSerializedHook(indexed)
370
911
  };
371
- const claimKey = `hookclaim:${token}`;
372
- const claim = await txn.get(claimKey);
373
- if (claim !== void 0 && !sameOwner(claim.owner, owner)) return {
912
+ const existing = values.get(claim);
913
+ if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
374
914
  claimed: false,
375
- holder: claim.owner
915
+ holder: existing.owner
916
+ };
917
+ if (existing === void 0) {
918
+ await txn.put(claim, {
919
+ owner,
920
+ claimId: proposedClaimId,
921
+ reservedAt: Date.now()
922
+ });
923
+ return {
924
+ claimed: true,
925
+ claimId: proposedClaimId
926
+ };
927
+ }
928
+ return {
929
+ claimed: true,
930
+ claimId: existing.claimId
376
931
  };
377
- await txn.put(claimKey, {
378
- owner,
379
- reservedAt: Date.now()
380
- });
381
- return { claimed: true };
382
932
  });
383
933
  }
384
- async finalizeHookIndexes(token, hookId, serializedHook, owner) {
385
- await this.ctx.storage.transaction(async (txn) => {
386
- if (await txn.get(`expired:${owner.runId}`) !== void 0) {
387
- const claim = await txn.get(`hookclaim:${token}`);
388
- if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(`hookclaim:${token}`);
389
- return;
934
+ async finalize(token, serializedHook, owner, claimId) {
935
+ if (!await runIsActive(this.env, owner.runId)) {
936
+ await this.releaseBatch([{
937
+ token,
938
+ owner
939
+ }], Date.now(), true);
940
+ return { stored: false };
941
+ }
942
+ return await this.ctx.storage.transaction(async (txn) => {
943
+ const record = recordKey(token);
944
+ const claim = claimKey(token);
945
+ const runFence = shardRunFenceKey(owner.runId);
946
+ const values = await txn.get([
947
+ record,
948
+ claim,
949
+ runFence
950
+ ]);
951
+ if (values.has(runFence)) {
952
+ const existingClaim = values.get(claim);
953
+ if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
954
+ return { stored: false };
390
955
  }
391
- const indexedHook = await txn.get(`hook:${token}`);
392
- if (indexedHook !== void 0 && !sameOwner(ownerFromHook(indexedHook), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
393
- const claim = await txn.get(`hookclaim:${token}`);
394
- if (claim !== void 0 && !sameOwner(claim.owner, owner)) throw new Error(`Hook token ${token} is reserved by another hook`);
395
- await txn.put(`hook:${token}`, serializedHook);
396
- await txn.put(`hookid:${hookId}`, serializedHook);
397
- if (claim !== void 0) await txn.delete(`hookclaim:${token}`);
956
+ const indexed = values.get(record);
957
+ if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
958
+ const existingClaim = values.get(claim);
959
+ if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
960
+ if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
961
+ await txn.put(record, serializedHook);
962
+ if (existingClaim !== void 0) await txn.delete(claim);
963
+ return { stored: true };
398
964
  });
399
965
  }
400
- async releaseHookToken(token, owner) {
966
+ async releaseClaim(token, owner, claimId) {
401
967
  await this.ctx.storage.transaction(async (txn) => {
402
- const key = `hookclaim:${token}`;
968
+ const key = claimKey(token);
403
969
  const claim = await txn.get(key);
404
- if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(key);
970
+ if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
405
971
  });
406
972
  }
407
- async deleteHookIndexes(token, hookId, owner) {
408
- await this.ctx.storage.transaction(async (txn) => {
409
- const tokenKey = `hook:${token}`;
410
- const idKey = `hookid:${hookId}`;
411
- const claimKey = `hookclaim:${token}`;
412
- const [byToken, byId, claim] = await Promise.all([
413
- txn.get(tokenKey),
414
- txn.get(idKey),
415
- txn.get(claimKey)
416
- ]);
417
- if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) await txn.delete(tokenKey);
418
- if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) await txn.delete(idKey);
419
- if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
420
- });
421
- }
422
- /**
423
- * Fence a run and remove all of its known derived indexes atomically.
424
- * Once the fence exists, putOwned() and hook finalization cannot resurrect
425
- * entries from a delayed request.
426
- */
427
- async expireRun(request) {
973
+ async releaseBatch(hooks, fencedAt, runFenced) {
974
+ if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
975
+ if (hooks.length === 0) return { deleted: 0 };
428
976
  return await this.ctx.storage.transaction(async (txn) => {
429
- await txn.put(`expired:${request.runId}`, request.expiredAt);
430
- let deleted = 0;
431
- for (const key of new Set(request.keys)) if (await txn.delete(key)) deleted++;
432
- for (const hook of request.hooks) {
433
- const owner = {
434
- runId: request.runId,
435
- hookId: hook.hookId
436
- };
437
- const tokenKey = `hook:${hook.token}`;
438
- const idKey = `hookid:${hook.hookId}`;
439
- const claimKey = `hookclaim:${hook.token}`;
440
- const [byToken, byId, claim] = await Promise.all([
441
- txn.get(tokenKey),
442
- txn.get(idKey),
443
- txn.get(claimKey)
444
- ]);
445
- if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
446
- if (await txn.delete(tokenKey)) deleted++;
447
- }
448
- if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
449
- if (await txn.delete(idKey)) deleted++;
450
- }
451
- if (claim !== void 0 && sameOwner(claim.owner, owner)) {
452
- if (await txn.delete(claimKey)) deleted++;
453
- }
977
+ const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
978
+ const values = await txn.get(keys);
979
+ const deletes = [];
980
+ for (const { token, owner } of hooks) {
981
+ const record = recordKey(token);
982
+ const claim = claimKey(token);
983
+ const indexed = values.get(record);
984
+ const existingClaim = values.get(claim);
985
+ if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
986
+ if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
454
987
  }
455
- return { deleted };
988
+ if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
989
+ return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
456
990
  });
457
991
  }
458
992
  };
@@ -471,21 +1005,61 @@ const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
471
1005
  const INFLIGHT_GRACE_MS = 3e4;
472
1006
  /** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
473
1007
  const MIN_ALARM_DELAY_MS = 1;
1008
+ /** Bound expiry recovery so one alarm cannot monopolize the cell. */
1009
+ const EXPIRED_INFLIGHT_BATCH = 128;
1010
+ const STORAGE_BATCH_SIZE$2 = 128;
1011
+ const EXPIRE_RUN_REFERENCE_BATCH = 64;
1012
+ const PURGE_DEAD_LETTER_BATCH = 128;
1013
+ const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
474
1014
  function pad(ms) {
475
1015
  return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
476
1016
  }
477
1017
  function dueKey(atMs, messageId) {
478
1018
  return `due:${pad(atMs)}:${messageId}`;
479
1019
  }
1020
+ function inflightKey(deadlineMs, messageId) {
1021
+ return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
1022
+ }
1023
+ function deadlineFromInflightKey(key) {
1024
+ return Number.parseInt(key.slice(18, 31), 10);
1025
+ }
480
1026
  function runReferenceKey(runId, messageId) {
481
1027
  return `run:${runId}:${messageId}`;
482
1028
  }
483
1029
  function expiredRunKey(runId) {
484
1030
  return `expired-run:${runId}`;
485
1031
  }
1032
+ function boundedLimit$1(value, fallback, maximum) {
1033
+ if (value === void 0 || !Number.isFinite(value)) return fallback;
1034
+ return Math.max(1, Math.min(maximum, Math.floor(value)));
1035
+ }
1036
+ async function getMany$1(storage, keys) {
1037
+ const result = /* @__PURE__ */ new Map();
1038
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$2) {
1039
+ const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$2));
1040
+ for (const [key, value] of page) result.set(key, value);
1041
+ }
1042
+ return result;
1043
+ }
1044
+ async function putMany(storage, entries) {
1045
+ const all = Array.from(entries);
1046
+ for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$2) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$2)));
1047
+ }
1048
+ async function deleteMany$1(storage, keys) {
1049
+ let deleted = 0;
1050
+ const unique = Array.from(new Set(keys));
1051
+ for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$2) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$2));
1052
+ return deleted;
1053
+ }
486
1054
  function backoffSeconds(attempt) {
487
1055
  return Math.min(60, 2 ** attempt);
488
1056
  }
1057
+ async function cancelUnusedResponseBody(response) {
1058
+ try {
1059
+ await response.body?.cancel();
1060
+ } catch {}
1061
+ }
1062
+ const MAX_QUEUE_INFLIGHT = 128;
489
1063
  var QueueDO = class extends DurableObject {
490
1064
  #now() {
491
1065
  const clock = this.env?.clock;
@@ -500,7 +1074,13 @@ var QueueDO = class extends DurableObject {
500
1074
  const parsed = raw ? Number.parseInt(raw, 10) : NaN;
501
1075
  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
502
1076
  }
1077
+ #maxInflight() {
1078
+ const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
1079
+ if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
1080
+ return configured;
1081
+ }
503
1082
  async enqueue(request) {
1083
+ this.#maxInflight();
504
1084
  const storage = this.ctx.storage;
505
1085
  const now = this.#now();
506
1086
  return await storage.transaction(async (txn) => {
@@ -575,7 +1155,7 @@ var QueueDO = class extends DurableObject {
575
1155
  if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
576
1156
  else await attempt();
577
1157
  if (failure !== null) throw failure;
578
- for (const row of claimed) this.ctx.waitUntil(this.#deliver(row));
1158
+ for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
579
1159
  }
580
1160
  /**
581
1161
  * Storage-only: recover lost inflight claims, claim due messages up to the
@@ -584,24 +1164,38 @@ var QueueDO = class extends DurableObject {
584
1164
  async #claimPhase() {
585
1165
  const storage = this.ctx.storage;
586
1166
  const now = this.#now();
587
- const maxInflight = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
1167
+ const maxInflight = this.#maxInflight();
588
1168
  return await storage.transaction(async (txn) => {
589
- const inflight = await txn.list({ prefix: "inflight:" });
590
- let inflightCount = 0;
591
- for (const [key, deadline] of inflight) if (deadline <= now) {
592
- const messageId = key.slice(9);
593
- await txn.delete(key);
594
- const row = await txn.get(`msg:${messageId}`);
1169
+ const expiredPage = await txn.list({
1170
+ prefix: INFLIGHT_DEADLINE_PREFIX,
1171
+ end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
1172
+ limit: 129
1173
+ });
1174
+ const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
1175
+ const expiredRows = await getMany$1(txn, expired.map(([, messageId]) => `msg:${messageId}`));
1176
+ await deleteMany$1(txn, expired.map(([key]) => key));
1177
+ const recoveryWrites = [];
1178
+ for (const [, messageId] of expired) {
1179
+ const row = expiredRows.get(`msg:${messageId}`);
595
1180
  if (row) {
596
1181
  const scheduleKey = dueKey(now, messageId);
597
- await txn.put(scheduleKey, messageId);
598
- if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
1182
+ recoveryWrites.push([scheduleKey, messageId]);
1183
+ if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
599
1184
  messageId: row.messageId,
600
1185
  dueKey: scheduleKey,
601
1186
  idempotencyKey: row.idempotencyKey
602
- });
1187
+ }]);
603
1188
  }
604
- } else inflightCount++;
1189
+ }
1190
+ await putMany(txn, recoveryWrites);
1191
+ if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
1192
+ await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
1193
+ return [];
1194
+ }
1195
+ const inflightCount = (await txn.list({
1196
+ prefix: INFLIGHT_DEADLINE_PREFIX,
1197
+ limit: maxInflight
1198
+ })).size;
605
1199
  const rows = [];
606
1200
  if (inflightCount < maxInflight) {
607
1201
  const due = await txn.list({
@@ -610,19 +1204,31 @@ var QueueDO = class extends DurableObject {
610
1204
  limit: maxInflight - inflightCount
611
1205
  });
612
1206
  const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
613
- for (const [key, messageId] of due) {
614
- await txn.delete(key);
615
- const row = await txn.get(`msg:${messageId}`);
1207
+ const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
1208
+ await deleteMany$1(txn, Array.from(due.keys()));
1209
+ const claimWrites = [];
1210
+ for (const [, messageId] of due) {
1211
+ const row = dueRows.get(`msg:${messageId}`);
616
1212
  if (!row) continue;
617
- await txn.put(`inflight:${messageId}`, now + timeoutMs + INFLIGHT_GRACE_MS);
618
- rows.push(row);
1213
+ const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
1214
+ claimWrites.push([claimKey, messageId]);
1215
+ if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
1216
+ messageId: row.messageId,
1217
+ inflightKey: claimKey,
1218
+ idempotencyKey: row.idempotencyKey
1219
+ }]);
1220
+ rows.push({
1221
+ row,
1222
+ inflightKey: claimKey
1223
+ });
619
1224
  }
1225
+ await putMany(txn, claimWrites);
620
1226
  }
621
1227
  await this.#scheduleNextAlarm(txn, now);
622
1228
  return rows;
623
1229
  });
624
1230
  }
625
- async #deliver(row) {
1231
+ async #deliver({ row, inflightKey: claimKey }) {
626
1232
  const storage = this.ctx.storage;
627
1233
  const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
628
1234
  const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
@@ -646,8 +1252,10 @@ var QueueDO = class extends DurableObject {
646
1252
  transportError = error;
647
1253
  }
648
1254
  const now = this.#now();
649
- if (response?.ok) await this.#ack(row);
650
- else if (response && response.status === 503) {
1255
+ if (response?.ok) {
1256
+ await cancelUnusedResponseBody(response);
1257
+ await this.#ack(row, claimKey);
1258
+ } else if (response && response.status === 503) {
651
1259
  let timeoutSeconds;
652
1260
  try {
653
1261
  timeoutSeconds = (await response.json())?.timeoutSeconds;
@@ -655,9 +1263,9 @@ var QueueDO = class extends DurableObject {
655
1263
  timeoutSeconds = void 0;
656
1264
  }
657
1265
  if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
1266
+ if (!await txn.delete(claimKey)) return;
658
1267
  if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
659
1268
  await txn.delete(`msg:${row.messageId}`);
660
- await txn.delete(`inflight:${row.messageId}`);
661
1269
  await txn.delete(runReferenceKey(row.runId, row.messageId));
662
1270
  if (row.idempotencyKey) {
663
1271
  const dedupKey = `key:${row.idempotencyKey}`;
@@ -666,7 +1274,6 @@ var QueueDO = class extends DurableObject {
666
1274
  await this.#scheduleNextAlarm(txn, now);
667
1275
  return;
668
1276
  }
669
- await txn.delete(`inflight:${row.messageId}`);
670
1277
  const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
671
1278
  await txn.put(scheduleKey, row.messageId);
672
1279
  if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
@@ -676,17 +1283,20 @@ var QueueDO = class extends DurableObject {
676
1283
  });
677
1284
  await this.#scheduleNextAlarm(txn, now);
678
1285
  });
679
- else await this.#retry(row, "HTTP 503");
680
- } else if (response && PERMANENT_STATUSES.has(response.status)) await this.#ack(row);
681
- else {
1286
+ else await this.#retry(row, claimKey, "HTTP 503");
1287
+ } else if (response && PERMANENT_STATUSES.has(response.status)) {
1288
+ await cancelUnusedResponseBody(response);
1289
+ await this.#ack(row, claimKey);
1290
+ } else {
682
1291
  const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
683
- await this.#retry(row, reason);
1292
+ if (response) await cancelUnusedResponseBody(response);
1293
+ await this.#retry(row, claimKey, reason);
684
1294
  }
685
1295
  }
686
- async #ack(row) {
1296
+ async #ack(row, claimKey) {
687
1297
  await this.ctx.storage.transaction(async (txn) => {
1298
+ if (!await txn.delete(claimKey)) return;
688
1299
  await txn.delete(`msg:${row.messageId}`);
689
- await txn.delete(`inflight:${row.messageId}`);
690
1300
  if (row.idempotencyKey) {
691
1301
  if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
692
1302
  }
@@ -694,15 +1304,15 @@ var QueueDO = class extends DurableObject {
694
1304
  await this.#scheduleNextAlarm(txn, this.#now());
695
1305
  });
696
1306
  }
697
- async #retry(row, reason) {
1307
+ async #retry(row, claimKey, reason) {
698
1308
  const storage = this.ctx.storage;
699
1309
  const now = this.#now();
700
1310
  const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
701
1311
  const attempt = row.attempt + 1;
702
1312
  await storage.transaction(async (txn) => {
1313
+ if (!await txn.delete(claimKey)) return;
703
1314
  if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
704
1315
  await txn.delete(`msg:${row.messageId}`);
705
- await txn.delete(`inflight:${row.messageId}`);
706
1316
  await txn.delete(runReferenceKey(row.runId, row.messageId));
707
1317
  if (row.idempotencyKey) {
708
1318
  const dedupKey = `key:${row.idempotencyKey}`;
@@ -711,7 +1321,6 @@ var QueueDO = class extends DurableObject {
711
1321
  await this.#scheduleNextAlarm(txn, now);
712
1322
  return;
713
1323
  }
714
- await txn.delete(`inflight:${row.messageId}`);
715
1324
  if (attempt >= maxAttempts) {
716
1325
  const dead = {
717
1326
  ...row,
@@ -771,8 +1380,14 @@ var QueueDO = class extends DurableObject {
771
1380
  limit: 1
772
1381
  });
773
1382
  for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
774
- const inflight = await storage.list({ prefix: "inflight:" });
775
- for (const deadline of inflight.values()) if (next === null || deadline < next) next = deadline;
1383
+ const inflight = await storage.list({
1384
+ prefix: INFLIGHT_DEADLINE_PREFIX,
1385
+ limit: 1
1386
+ });
1387
+ for (const key of inflight.keys()) {
1388
+ const deadline = deadlineFromInflightKey(key);
1389
+ if (next === null || deadline < next) next = deadline;
1390
+ }
776
1391
  if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
777
1392
  else await storage.deleteAlarm();
778
1393
  }
@@ -780,7 +1395,7 @@ var QueueDO = class extends DurableObject {
780
1395
  const storage = this.ctx.storage;
781
1396
  const [due, inflight, dlq, alarmAt] = await Promise.all([
782
1397
  storage.list({ prefix: "due:" }),
783
- storage.list({ prefix: "inflight:" }),
1398
+ storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
784
1399
  storage.list({ prefix: "dlq:" }),
785
1400
  storage.getAlarm()
786
1401
  ]);
@@ -845,13 +1460,16 @@ var QueueDO = class extends DurableObject {
845
1460
  }
846
1461
  async purgeDeadLetters() {
847
1462
  return await this.ctx.storage.transaction(async (txn) => {
848
- const dlq = await txn.list({ prefix: "dlq:" });
849
- for (const [key, value] of dlq) {
850
- await txn.delete(key);
851
- const dead = value;
852
- if (dead.runId) await txn.delete(runReferenceKey(dead.runId, dead.messageId));
853
- }
854
- return { purged: dlq.size };
1463
+ const entries = await txn.list({
1464
+ prefix: "dlq:",
1465
+ limit: 129
1466
+ });
1467
+ const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
1468
+ await deleteMany$1(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
1469
+ return {
1470
+ purged: page.length,
1471
+ hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
1472
+ };
855
1473
  });
856
1474
  }
857
1475
  /**
@@ -863,140 +1481,384 @@ var QueueDO = class extends DurableObject {
863
1481
  return { alarmAt: await this.ctx.storage.getAlarm() };
864
1482
  }
865
1483
  /** Fence a run and purge every live, inflight, and dead-letter message it owns. */
866
- async expireRun(runId, expiredAt) {
867
- return { deleted: await this.ctx.storage.transaction(async (txn) => {
1484
+ async expireRun(runId, expiredAt, options) {
1485
+ const storage = this.ctx.storage;
1486
+ const limit = boundedLimit$1(options?.limit, EXPIRE_RUN_REFERENCE_BATCH, EXPIRE_RUN_REFERENCE_BATCH);
1487
+ return await storage.transaction(async (txn) => {
868
1488
  const existing = await txn.get(expiredRunKey(runId));
869
- const references = await txn.list({ prefix: `run:${runId}:` });
1489
+ const references = await txn.list({
1490
+ prefix: `run:${runId}:`,
1491
+ limit: limit + 1
1492
+ });
1493
+ const page = Array.from(references).slice(0, limit);
1494
+ const dedupHolders = await getMany$1(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
1495
+ const keys = [];
870
1496
  let count = existing?.deleted ?? 0;
871
- for (const [key, reference] of references) {
872
- await txn.delete(`msg:${reference.messageId}`);
873
- await txn.delete(`inflight:${reference.messageId}`);
874
- if (reference.dueKey) await txn.delete(reference.dueKey);
875
- if (reference.dlqKey) await txn.delete(reference.dlqKey);
1497
+ for (const [key, reference] of page) {
1498
+ keys.push(`msg:${reference.messageId}`, key);
1499
+ if (reference.inflightKey) keys.push(reference.inflightKey);
1500
+ if (reference.dueKey) keys.push(reference.dueKey);
1501
+ if (reference.dlqKey) keys.push(reference.dlqKey);
876
1502
  if (reference.idempotencyKey) {
877
1503
  const dedupKey = `key:${reference.idempotencyKey}`;
878
- if (await txn.get(dedupKey) === reference.messageId) await txn.delete(dedupKey);
1504
+ if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
879
1505
  }
880
- await txn.delete(key);
881
1506
  count++;
882
1507
  }
1508
+ await deleteMany$1(txn, keys);
883
1509
  await txn.put(expiredRunKey(runId), {
884
1510
  expiredAt,
885
1511
  deleted: count
886
1512
  });
887
1513
  await this.#scheduleNextAlarm(txn, this.#now());
888
- return count;
889
- }) };
1514
+ return {
1515
+ deleted: count,
1516
+ done: references.size <= limit
1517
+ };
1518
+ });
1519
+ }
1520
+ };
1521
+ //#endregion
1522
+ //#region src/worker/durable-objects/RunCatalogDO.ts
1523
+ const MAX_INDEX_LIST_SIZE = 1001;
1524
+ /** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
1525
+ var RunCatalogDO = class extends DurableObject {
1526
+ async upsertRun(runId, keys, serializedMetadata) {
1527
+ if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
1528
+ return await this.ctx.storage.transaction(async (txn) => {
1529
+ if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
1530
+ await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
1531
+ return { stored: true };
1532
+ });
1533
+ }
1534
+ async list(options = {}) {
1535
+ const suppliedLimit = options.limit;
1536
+ const limit = Math.min(MAX_INDEX_LIST_SIZE, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : MAX_INDEX_LIST_SIZE));
1537
+ const entries = await this.ctx.storage.list({
1538
+ prefix: options.prefix ?? "",
1539
+ ...options.reverse ? {
1540
+ reverse: true,
1541
+ end: options.cursor
1542
+ } : { startAfter: options.cursor },
1543
+ limit: limit + 1
1544
+ });
1545
+ const page = Array.from(entries).slice(0, limit);
1546
+ const listComplete = entries.size <= limit;
1547
+ return {
1548
+ keys: page.map(([name, value]) => ({
1549
+ name,
1550
+ value
1551
+ })),
1552
+ list_complete: listComplete,
1553
+ cursor: listComplete ? void 0 : page.at(-1)?.[0]
1554
+ };
1555
+ }
1556
+ async expireRun(runId, keys, expiredAt) {
1557
+ if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
1558
+ return await this.ctx.storage.transaction(async (txn) => {
1559
+ await txn.put(`expired:${runId}`, expiredAt);
1560
+ return { deleted: await txn.delete(keys) };
1561
+ });
1562
+ }
1563
+ };
1564
+ //#endregion
1565
+ //#region src/worker/durable-objects/RunFenceDO.ts
1566
+ /** Per-run lifecycle fence used by hook shards without a global lookup cell. */
1567
+ var RunFenceDO = class extends DurableObject {
1568
+ async getStatus() {
1569
+ return (await this.ctx.storage.get("fence"))?.status ?? "active";
1570
+ }
1571
+ async fenceTerminal(at) {
1572
+ return await this.ctx.storage.transaction(async (txn) => {
1573
+ const current = await txn.get("fence");
1574
+ if (current) return current.status;
1575
+ await txn.put("fence", {
1576
+ status: "terminal",
1577
+ at
1578
+ });
1579
+ return "terminal";
1580
+ });
1581
+ }
1582
+ async fenceExpired(at) {
1583
+ return await this.ctx.storage.transaction(async (txn) => {
1584
+ const current = await txn.get("fence");
1585
+ if (current?.status === "expired") return current.status;
1586
+ await txn.put("fence", {
1587
+ status: "expired",
1588
+ at
1589
+ });
1590
+ return "expired";
1591
+ });
890
1592
  }
891
1593
  };
892
1594
  //#endregion
893
1595
  //#region src/worker/durable-objects/StreamDO.ts
894
1596
  const META_KEY = "meta";
895
1597
  const CHUNK_KEY_PREFIX = "chunk:";
1598
+ const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
896
1599
  /** Registry keys used when this DO instance acts as a per-run stream index. */
897
1600
  const STREAM_REGISTRY_PREFIX = "stream:";
898
1601
  const REGISTRY_OWNER_KEY = "registry:owner";
899
1602
  const REGISTRY_EXPIRED_KEY = "registry:expired";
900
- /**
901
- * Zero-pad chunk indexes so `storage.list({ prefix })` returns chunks in
902
- * write order. 12 digits comfortably exceeds any realistic chunk count.
903
- */
1603
+ /** Each chunk occupies a payload key and a size key; one delete accepts at most 128 keys. */
1604
+ const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
1605
+ const MAX_EXPIRE_CHUNK_LIMIT = 64;
1606
+ /** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
1607
+ const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
1608
+ const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
1609
+ function boundedLimit(value, fallback, maximum) {
1610
+ if (value === void 0 || !Number.isFinite(value)) return fallback;
1611
+ return Math.max(1, Math.min(maximum, Math.floor(value)));
1612
+ }
1613
+ function emptyMeta() {
1614
+ return {
1615
+ count: 0,
1616
+ state: "open"
1617
+ };
1618
+ }
1619
+ function validateMeta(meta) {
1620
+ if (!Number.isSafeInteger(meta.count) || meta.count < 0 || meta.count > 2147483647 || ![
1621
+ "open",
1622
+ "closed",
1623
+ "errored",
1624
+ "expired"
1625
+ ].includes(meta.state)) throw new Error("Invalid persisted stream metadata");
1626
+ if (meta.state === "errored" && !meta.error) throw new Error("Invalid persisted stream error metadata");
1627
+ return meta;
1628
+ }
1629
+ /** Zero-padding keeps storage.list() results in stream offset order. */
904
1630
  function chunkKey(index) {
905
1631
  return `${CHUNK_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
906
1632
  }
1633
+ function chunkSizeKey(index) {
1634
+ return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
1635
+ }
1636
+ function indexFromChunkSizeKey(key) {
1637
+ const suffix = key.slice(11);
1638
+ const index = Number(suffix);
1639
+ if (!/^\d{12}$/.test(suffix) || !Number.isSafeInteger(index) || index < 0) throw new Error(`Invalid persisted stream chunk size key "${key}"`);
1640
+ return index;
1641
+ }
1642
+ function validateChunkSize(size, index) {
1643
+ if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
1644
+ return size;
1645
+ }
1646
+ function compactChunk(chunk) {
1647
+ return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
1648
+ }
1649
+ function abortReason(signal) {
1650
+ return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
1651
+ }
907
1652
  /**
908
1653
  * Durable Object backing workflow streams.
909
1654
  *
910
- * Two roles, selected by the DO name:
911
- * - `stream:<name>`: chunk storage for a single stream. A monotonic
912
- * per-stream counter (maintained transactionally inside the DO) assigns
913
- * each chunk its index; one chunk per storage key.
914
- * - `run-streams:<runId>`: registry of stream names owned by a run,
915
- * powering `listStreamsByRunId`.
1655
+ * Stream cells persist binary chunks under monotonic offset keys. Public
1656
+ * reads are bounded long polls, and writes append a bounded ordered batch in
1657
+ * one storage transaction. Per-run registry cells retain their existing role.
916
1658
  */
917
1659
  var StreamDO = class extends DurableObject {
1660
+ meta;
1661
+ metaLoad;
1662
+ mutationTail = Promise.resolve();
1663
+ changeVersion = 0;
1664
+ waiters = /* @__PURE__ */ new Set();
918
1665
  async getMeta() {
919
- return await this.ctx.storage.get(META_KEY) ?? {
920
- count: 0,
921
- closed: false
922
- };
1666
+ if (this.meta) return this.meta;
1667
+ this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
1668
+ const loaded = meta ? validateMeta(meta) : emptyMeta();
1669
+ this.meta = loaded;
1670
+ return loaded;
1671
+ });
1672
+ try {
1673
+ return await this.metaLoad;
1674
+ } catch (error) {
1675
+ this.metaLoad = void 0;
1676
+ throw error;
1677
+ }
923
1678
  }
924
1679
  assertOwner(meta, runId) {
925
- if (meta.expiredAt !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
1680
+ if (!runId) throw new Error("Stream runId is required");
926
1681
  if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
927
1682
  }
928
- /**
929
- * Append a chunk. The index is allocated inside the transaction, so
930
- * concurrent writers can never collide or skip.
931
- */
932
- async writeChunk(runId, data) {
933
- return await this.ctx.storage.transaction(async (txn) => {
934
- const meta = await txn.get(META_KEY) ?? {
935
- count: 0,
936
- closed: false
1683
+ assertWritable(meta, runId) {
1684
+ this.assertOwner(meta, runId);
1685
+ if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
1686
+ if (meta.state === "closed") throw new Error("Cannot write to a closed stream");
1687
+ if (meta.state === "errored") throw new Error("Cannot write to an errored stream");
1688
+ }
1689
+ runMutation(operation) {
1690
+ const result = this.mutationTail.then(operation, operation);
1691
+ this.mutationTail = result.then(() => void 0, () => void 0);
1692
+ return result;
1693
+ }
1694
+ wakeReaders() {
1695
+ this.changeVersion += 1;
1696
+ for (const waiter of Array.from(this.waiters)) waiter.resolve("change");
1697
+ }
1698
+ waitForChange(observedVersion, waitMs, signal) {
1699
+ if (signal?.aborted) return Promise.reject(abortReason(signal));
1700
+ return new Promise((resolve, reject) => {
1701
+ let settled = false;
1702
+ const waiter = {
1703
+ timer: setTimeout(() => finish("timeout"), waitMs),
1704
+ signal,
1705
+ resolve: (reason) => finish(reason)
937
1706
  };
938
- this.assertOwner(meta, runId);
939
- if (meta.closed) throw new Error("Cannot write to a closed stream");
940
- const index = meta.count;
941
- await txn.put(chunkKey(index), data);
942
- await txn.put(META_KEY, {
943
- count: index + 1,
944
- closed: false,
945
- ownerRunId: runId
1707
+ const cleanup = () => {
1708
+ clearTimeout(waiter.timer);
1709
+ this.waiters.delete(waiter);
1710
+ if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
1711
+ };
1712
+ const finish = (reason) => {
1713
+ if (settled) return;
1714
+ settled = true;
1715
+ cleanup();
1716
+ resolve(reason);
1717
+ };
1718
+ waiter.onAbort = () => {
1719
+ if (settled) return;
1720
+ settled = true;
1721
+ cleanup();
1722
+ reject(abortReason(signal));
1723
+ };
1724
+ this.waiters.add(waiter);
1725
+ signal?.addEventListener("abort", waiter.onAbort, { once: true });
1726
+ if (this.changeVersion !== observedVersion) finish("change");
1727
+ });
1728
+ }
1729
+ async readSnapshot(request, timedOut) {
1730
+ const meta = await this.getMeta();
1731
+ this.assertOwner(meta, request.runId);
1732
+ const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
1733
+ const chunks = [];
1734
+ if (available > 0) {
1735
+ const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
1736
+ const sizeKeys = indexes.map(chunkSizeKey);
1737
+ const storedSizes = await this.ctx.storage.get(sizeKeys);
1738
+ const selectedIndexes = [];
1739
+ const selectedSizes = [];
1740
+ let bytes = 0;
1741
+ for (const index of indexes) {
1742
+ const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
1743
+ if (bytes + size > request.maxBytes) break;
1744
+ selectedIndexes.push(index);
1745
+ selectedSizes.push(size);
1746
+ bytes += size;
1747
+ }
1748
+ const payloadKeys = selectedIndexes.map(chunkKey);
1749
+ const storedChunks = await this.ctx.storage.get(payloadKeys);
1750
+ for (let offset = 0; offset < payloadKeys.length; offset++) {
1751
+ const chunk = storedChunks.get(payloadKeys[offset]);
1752
+ if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
1753
+ chunks.push(chunk);
1754
+ }
1755
+ }
1756
+ return {
1757
+ startIndex: request.startIndex,
1758
+ tailIndex: meta.count - 1,
1759
+ chunks,
1760
+ state: meta.state,
1761
+ timedOut,
1762
+ error: meta.error
1763
+ };
1764
+ }
1765
+ /** Append one bounded binary batch with contiguous indexes. */
1766
+ async writeChunks(runId, chunks) {
1767
+ validateStreamWriteChunks(chunks);
1768
+ const storedChunks = chunks.map(compactChunk);
1769
+ return await this.runMutation(async () => {
1770
+ const committed = await this.ctx.storage.transaction(async (txn) => {
1771
+ const stored = await txn.get(META_KEY);
1772
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1773
+ this.assertWritable(meta, runId);
1774
+ const startIndex = meta.count;
1775
+ if (startIndex + chunks.length > 2147483647) throw new Error("Stream offset limit exceeded");
1776
+ const nextMeta = {
1777
+ count: startIndex + chunks.length,
1778
+ state: "open",
1779
+ ownerRunId: runId
1780
+ };
1781
+ const entries = { [META_KEY]: nextMeta };
1782
+ for (let offset = 0; offset < storedChunks.length; offset++) {
1783
+ const index = startIndex + offset;
1784
+ entries[chunkKey(index)] = storedChunks[offset];
1785
+ entries[chunkSizeKey(index)] = storedChunks[offset].byteLength;
1786
+ }
1787
+ await txn.put(entries);
1788
+ return {
1789
+ meta: nextMeta,
1790
+ result: {
1791
+ startIndex,
1792
+ count: chunks.length,
1793
+ tailIndex: nextMeta.count - 1
1794
+ }
1795
+ };
946
1796
  });
947
- return index;
1797
+ this.meta = committed.meta;
1798
+ this.metaLoad = Promise.resolve(committed.meta);
1799
+ this.wakeReaders();
1800
+ return committed.result;
948
1801
  });
949
1802
  }
950
1803
  /**
951
- * Close the stream (idempotent). Readers observe `done: true` once all
952
- * previously written chunks have been consumed.
1804
+ * Return available binary chunks and terminal metadata in one bounded
1805
+ * operation, waiting at most waitMs when the requested offset is idle.
953
1806
  */
1807
+ async readChunks(request, signal) {
1808
+ validateStreamReadRequest(request);
1809
+ const observedVersion = this.changeVersion;
1810
+ const immediate = await this.readSnapshot(request, false);
1811
+ if (immediate.chunks.length > 0 || immediate.state !== "open" || request.waitMs === 0 || request.maxChunks === 0) return immediate;
1812
+ const reason = await this.waitForChange(observedVersion, request.waitMs, signal);
1813
+ return await this.readSnapshot(request, reason === "timeout");
1814
+ }
1815
+ /** Close idempotently; the first terminal state wins. */
954
1816
  async closeStream(runId) {
955
- await this.ctx.storage.transaction(async (txn) => {
956
- const meta = await txn.get(META_KEY) ?? {
957
- count: 0,
958
- closed: false
959
- };
960
- this.assertOwner(meta, runId);
961
- await txn.put(META_KEY, {
962
- ...meta,
963
- ownerRunId: runId,
964
- closed: true
1817
+ await this.runMutation(async () => {
1818
+ const nextMeta = await this.ctx.storage.transaction(async (txn) => {
1819
+ const stored = await txn.get(META_KEY);
1820
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1821
+ this.assertOwner(meta, runId);
1822
+ if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
1823
+ if (meta.state !== "open") return meta;
1824
+ const closed = {
1825
+ ...meta,
1826
+ ownerRunId: runId,
1827
+ state: "closed"
1828
+ };
1829
+ await txn.put(META_KEY, closed);
1830
+ return closed;
965
1831
  });
1832
+ this.meta = nextMeta;
1833
+ this.metaLoad = Promise.resolve(nextMeta);
1834
+ this.wakeReaders();
966
1835
  });
967
1836
  }
968
- /**
969
- * Read up to `limit` chunks starting at `startIndex` (0-based, inclusive).
970
- */
971
- async getChunks(params) {
972
- const meta = await this.getMeta();
973
- if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
974
- const start = Math.max(0, params.startIndex);
975
- const entries = await this.ctx.storage.list({
976
- prefix: CHUNK_KEY_PREFIX,
977
- start: chunkKey(start),
978
- limit: params.limit
1837
+ /** Persist a terminal stream failure and wake every pending reader. */
1838
+ async failStream(runId, error) {
1839
+ const normalized = normalizeStreamError(error);
1840
+ await this.runMutation(async () => {
1841
+ const nextMeta = await this.ctx.storage.transaction(async (txn) => {
1842
+ const stored = await txn.get(META_KEY);
1843
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1844
+ this.assertOwner(meta, runId);
1845
+ if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
1846
+ if (meta.state !== "open") return meta;
1847
+ const failed = {
1848
+ ...meta,
1849
+ ownerRunId: runId,
1850
+ state: "errored",
1851
+ error: normalized
1852
+ };
1853
+ await txn.put(META_KEY, failed);
1854
+ return failed;
1855
+ });
1856
+ this.meta = nextMeta;
1857
+ this.metaLoad = Promise.resolve(nextMeta);
1858
+ this.wakeReaders();
979
1859
  });
980
- return {
981
- chunks: Array.from(entries.values()),
982
- done: meta.closed,
983
- tailIndex: meta.count - 1
984
- };
985
1860
  }
986
- /**
987
- * Lightweight stream metadata: last chunk index and completion flag.
988
- */
989
- async getInfo() {
990
- const meta = await this.getMeta();
991
- if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
992
- return {
993
- tailIndex: meta.count - 1,
994
- done: meta.closed
995
- };
996
- }
997
- /**
998
- * Register a stream name against this per-run registry instance.
999
- */
1861
+ /** Register a stream name against this per-run registry instance. */
1000
1862
  async registerStream(runId, name) {
1001
1863
  await this.ctx.storage.transaction(async (txn) => {
1002
1864
  if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
@@ -1006,57 +1868,124 @@ var StreamDO = class extends DurableObject {
1006
1868
  await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
1007
1869
  });
1008
1870
  }
1009
- /**
1010
- * List stream names registered against this per-run registry instance.
1011
- */
1012
1871
  async listStreams() {
1013
1872
  if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
1014
1873
  const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
1015
1874
  return Array.from(entries.keys()).map((key) => key.slice(7));
1016
1875
  }
1017
1876
  /** Fence a run's registry before any stream cells are removed. */
1018
- async expireRegistry(runId, expiredAt) {
1877
+ async expireRegistry(runId, expiredAt, options) {
1878
+ const limit = boundedLimit(options?.limit, 16, 128);
1019
1879
  return await this.ctx.storage.transaction(async (txn) => {
1020
- const owner = await txn.get(REGISTRY_OWNER_KEY);
1880
+ const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
1881
+ const owner = state.get(REGISTRY_OWNER_KEY);
1021
1882
  if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
1022
- await txn.put(REGISTRY_OWNER_KEY, runId);
1023
- await txn.put(REGISTRY_EXPIRED_KEY, expiredAt);
1024
- const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
1883
+ const expiry = state.get(REGISTRY_EXPIRED_KEY);
1884
+ await txn.put({
1885
+ [REGISTRY_OWNER_KEY]: runId,
1886
+ [REGISTRY_EXPIRED_KEY]: expiry ?? {
1887
+ expiredAt,
1888
+ deleted: 0
1889
+ }
1890
+ });
1891
+ const entries = await txn.list({
1892
+ prefix: STREAM_REGISTRY_PREFIX,
1893
+ limit
1894
+ });
1025
1895
  return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
1026
1896
  });
1027
1897
  }
1028
- /** Remove the registry payload after every referenced stream was fenced. */
1029
- async finalizeRegistry(runId) {
1030
- await this.ctx.storage.transaction(async (txn) => {
1031
- const owner = await txn.get(REGISTRY_OWNER_KEY);
1032
- const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
1033
- if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
1034
- const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
1035
- for (const key of entries.keys()) await txn.delete(key);
1036
- });
1037
- }
1038
- /** Delete stream payload while leaving a small ownership tombstone. */
1039
- async expireStream(runId, expiredAt) {
1898
+ /** Remove completed registry entries in a bounded, retry-safe batch. */
1899
+ async finalizeRegistry(runId, streams) {
1040
1900
  return await this.ctx.storage.transaction(async (txn) => {
1041
- const meta = await txn.get(META_KEY) ?? {
1042
- count: 0,
1043
- closed: false
1044
- };
1045
- if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
1046
- const chunks = await txn.list({ prefix: CHUNK_KEY_PREFIX });
1047
- for (const key of chunks.keys()) await txn.delete(key);
1048
- await txn.put(META_KEY, {
1049
- count: 0,
1050
- closed: true,
1051
- ownerRunId: runId,
1052
- expiredAt
1901
+ const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
1902
+ const owner = state.get(REGISTRY_OWNER_KEY);
1903
+ const expiry = state.get(REGISTRY_EXPIRED_KEY);
1904
+ if (owner !== runId || expiry === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
1905
+ const keys = Array.from(new Set(streams.map((name) => `${STREAM_REGISTRY_PREFIX}${name}`)));
1906
+ const removed = keys.length === 0 ? 0 : await txn.delete(keys);
1907
+ const deleted = expiry.deleted + removed;
1908
+ await txn.put(REGISTRY_EXPIRED_KEY, {
1909
+ ...expiry,
1910
+ deleted
1053
1911
  });
1054
1912
  return {
1055
- deleted: true,
1056
- chunks: chunks.size
1913
+ deleted,
1914
+ done: (await txn.list({
1915
+ prefix: STREAM_REGISTRY_PREFIX,
1916
+ limit: 1
1917
+ })).size === 0
1057
1918
  };
1058
1919
  });
1059
1920
  }
1921
+ /**
1922
+ * Fence the stream and delete one bounded page. Listing size keys avoids
1923
+ * loading payloads, and the transaction makes a crash retry resume safely.
1924
+ */
1925
+ async expireStream(runId, expiredAt, options) {
1926
+ const limit = boundedLimit(options?.limit, DEFAULT_EXPIRE_CHUNK_LIMIT, MAX_EXPIRE_CHUNK_LIMIT);
1927
+ const byteLimit = boundedLimit(options?.byteLimit, DEFAULT_EXPIRE_BYTE_LIMIT, MAX_EXPIRE_BYTE_LIMIT);
1928
+ return await this.runMutation(async () => {
1929
+ const result = await this.ctx.storage.transaction(async (txn) => {
1930
+ const stored = await txn.get(META_KEY);
1931
+ const meta = stored ? validateMeta(stored) : emptyMeta();
1932
+ this.assertOwner(meta, runId);
1933
+ const firstFence = meta.state !== "expired";
1934
+ if (meta.state === "expired" && meta.payloadDeleted) return {
1935
+ meta,
1936
+ result: {
1937
+ deleted: false,
1938
+ chunks: 0,
1939
+ bytes: 0,
1940
+ done: true
1941
+ }
1942
+ };
1943
+ const candidates = await txn.list({
1944
+ prefix: CHUNK_SIZE_KEY_PREFIX,
1945
+ limit: limit + 1
1946
+ });
1947
+ const page = [];
1948
+ let bytes = 0;
1949
+ for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
1950
+ const index = indexFromChunkSizeKey(key);
1951
+ const size = validateChunkSize(storedSize, index);
1952
+ if (page.length > 0 && bytes + size > byteLimit) break;
1953
+ page.push({
1954
+ index,
1955
+ size
1956
+ });
1957
+ bytes += size;
1958
+ }
1959
+ const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
1960
+ if (keys.length > 0) await txn.delete(keys);
1961
+ const done = candidates.size === page.length;
1962
+ const nextMeta = {
1963
+ ...meta,
1964
+ count: done ? 0 : meta.count,
1965
+ ownerRunId: runId,
1966
+ state: "expired",
1967
+ error: void 0,
1968
+ expiredAt: meta.expiredAt ?? expiredAt,
1969
+ expiredChunkCount: meta.expiredChunkCount ?? meta.count,
1970
+ payloadDeleted: done
1971
+ };
1972
+ await txn.put(META_KEY, nextMeta);
1973
+ return {
1974
+ meta: nextMeta,
1975
+ result: {
1976
+ deleted: firstFence,
1977
+ chunks: page.length,
1978
+ bytes,
1979
+ done
1980
+ }
1981
+ };
1982
+ });
1983
+ this.meta = result.meta;
1984
+ this.metaLoad = Promise.resolve(result.meta);
1985
+ this.wakeReaders();
1986
+ return result.result;
1987
+ });
1988
+ }
1060
1989
  };
1061
1990
  //#endregion
1062
1991
  //#region src/apply-event.ts
@@ -1076,10 +2005,36 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
1076
2005
  * rejected as a duplicate instead of resurrecting the hook.
1077
2006
  */
1078
2007
  const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
2008
+ /** Durable repair record for idempotent post-commit hook-index release. */
2009
+ const HOOK_RELEASE_MARKER_PREFIX = "hookreleased:";
2010
+ /** Exact ambiguous reservations canceled by a serialized RunDO resolution. */
2011
+ const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
1079
2012
  /** Workflow-authored attr_set correlation claims. */
1080
2013
  const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
1081
2014
  /** Durable lazy-hook resume claims, scoped by the containing run DO. */
1082
2015
  const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
2016
+ const STORAGE_BATCH_SIZE$1 = 128;
2017
+ const MAX_ENTITY_PAGE_SIZE = 1e3;
2018
+ async function getMany(store, keys) {
2019
+ if (store.getMany) {
2020
+ const result = /* @__PURE__ */ new Map();
2021
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
2022
+ const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
2023
+ for (const entry of page) result.set(...entry);
2024
+ }
2025
+ return result;
2026
+ }
2027
+ const values = await Promise.all(keys.map((key) => store.get(key)));
2028
+ return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
2029
+ }
2030
+ async function deleteMany(store, keys) {
2031
+ if (store.deleteMany) {
2032
+ let deleted = 0;
2033
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
2034
+ return deleted;
2035
+ }
2036
+ return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
2037
+ }
1083
2038
  function failure(code, message, extra) {
1084
2039
  return {
1085
2040
  ok: false,
@@ -1088,6 +2043,9 @@ function failure(code, message, extra) {
1088
2043
  ...extra
1089
2044
  };
1090
2045
  }
2046
+ function hookClaimCancellationKey(claimId) {
2047
+ return `${HOOK_CLAIM_CANCELLATION_PREFIX}${encodeURIComponent(claimId)}`;
2048
+ }
1091
2049
  function readStringProp(value, key) {
1092
2050
  if (value !== null && typeof value === "object") {
1093
2051
  const prop = value[key];
@@ -1143,7 +2101,9 @@ function creationIndexKey(prefix, createdAt, id) {
1143
2101
  * requested ordering.
1144
2102
  */
1145
2103
  async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
1146
- const { limit, cursor, sortOrder = "asc" } = params;
2104
+ const requestedLimit = Number.isFinite(params.limit) ? Math.floor(params.limit) : 1;
2105
+ const limit = Math.min(MAX_ENTITY_PAGE_SIZE, Math.max(1, requestedLimit));
2106
+ const { cursor, sortOrder = "asc" } = params;
1147
2107
  const entries = await store.list(sortOrder === "desc" ? {
1148
2108
  prefix: indexPrefix,
1149
2109
  reverse: true,
@@ -1155,11 +2115,12 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
1155
2115
  startAfter: cursor ? `${indexPrefix}${cursor}` : void 0
1156
2116
  });
1157
2117
  const page = Array.from(entries.entries()).slice(0, limit);
1158
- const data = [];
1159
- for (const [, id] of page) {
1160
- const item = await store.get(`${entityPrefix}${id}`);
1161
- if (item !== void 0) data.push(item);
1162
- }
2118
+ const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
2119
+ const entities = await getMany(store, entityKeys);
2120
+ const data = entityKeys.flatMap((key) => {
2121
+ const item = entities.get(key);
2122
+ return item === void 0 ? [] : [item];
2123
+ });
1163
2124
  const hasMore = entries.size > limit;
1164
2125
  const lastKey = page.at(-1)?.[0];
1165
2126
  return {
@@ -1170,22 +2131,26 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
1170
2131
  }
1171
2132
  /** Delete all hook entities for the run, returning the released tokens. */
1172
2133
  async function releaseAllHooks(store) {
2134
+ if (store.deferTerminalCleanup) return [];
1173
2135
  const hooks = await store.list({ prefix: HOOK_KEY_PREFIX });
1174
2136
  const released = [];
2137
+ const keys = [];
1175
2138
  for (const hook of hooks.values()) {
1176
- await store.delete(`${HOOK_KEY_PREFIX}${hook.hookId}`);
1177
- await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
2139
+ keys.push(`${HOOK_KEY_PREFIX}${hook.hookId}`);
2140
+ keys.push(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
1178
2141
  released.push({
1179
2142
  hookId: hook.hookId,
1180
2143
  token: hook.token
1181
2144
  });
1182
2145
  }
2146
+ await deleteMany(store, keys);
1183
2147
  return released;
1184
2148
  }
1185
2149
  /** Delete all persisted waits once their owning run becomes terminal. */
1186
2150
  async function releaseAllWaits(store) {
2151
+ if (store.deferTerminalCleanup) return;
1187
2152
  const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
1188
- for (const key of waits.keys()) await store.delete(key);
2153
+ await deleteMany(store, Array.from(waits.keys()));
1189
2154
  }
1190
2155
  async function applyEvent(store, ctx) {
1191
2156
  const { runId, data, now } = ctx;
@@ -1305,7 +2270,16 @@ async function applyEvent(store, ctx) {
1305
2270
  }
1306
2271
  }
1307
2272
  if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
1308
- if (!await store.get(`hook:${data.correlationId}`)) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
2273
+ if (!await store.get(`hook:${data.correlationId}`)) {
2274
+ if (data.eventType === "hook_disposed") {
2275
+ const released = await store.get(`${HOOK_RELEASE_MARKER_PREFIX}${data.correlationId}`);
2276
+ if (released) return {
2277
+ ok: true,
2278
+ releasedHooks: [released]
2279
+ };
2280
+ }
2281
+ return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
2282
+ }
1309
2283
  }
1310
2284
  if (!skipRunValidation && data.eventType !== "run_started" && !currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
1311
2285
  switch (data.eventType) {
@@ -1641,6 +2615,7 @@ async function applyEvent(store, ctx) {
1641
2615
  const holder = ctx.tokenHolder ?? null;
1642
2616
  const markerKey = `${HOOK_EVENT_MARKER_PREFIX}${hookId}`;
1643
2617
  const hookKey = `${HOOK_KEY_PREFIX}${hookId}`;
2618
+ if (ctx.hookClaimId && await store.get(hookClaimCancellationKey(ctx.hookClaimId)) !== void 0) return failure("HOOK_CLAIM_CANCELLED", `Hook reservation for "${hookId}" was canceled before commit`);
1644
2619
  if (holder && (holder.runId !== runId || holder.hookId !== hookId)) {
1645
2620
  const event = buildEvent({
1646
2621
  eventType: "hook_conflict",
@@ -1700,6 +2675,10 @@ async function applyEvent(store, ctx) {
1700
2675
  if (!hook) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
1701
2676
  await store.delete(hookKey);
1702
2677
  await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
2678
+ await store.put(`${HOOK_RELEASE_MARKER_PREFIX}${hook.hookId}`, {
2679
+ hookId: hook.hookId,
2680
+ token: hook.token
2681
+ });
1703
2682
  const event = buildEvent({ ...data });
1704
2683
  await putEvent(event);
1705
2684
  return {
@@ -1817,8 +2796,8 @@ async function finalizeEventPage(store, outcome, params) {
1817
2796
  //#region src/retention.ts
1818
2797
  const CLEANUP_RECORD_KEY = "retention:cleanup";
1819
2798
  const TOMBSTONE_KEY = "retention:tombstone";
1820
- const INDEX_MARKER_PREFIX = "retention:index:";
1821
2799
  const HOOK_MARKER_PREFIX = "retention:hook:";
2800
+ const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
1822
2801
  function sortableTimestamp(date) {
1823
2802
  return date.getTime().toString().padStart(13, "0");
1824
2803
  }
@@ -1828,12 +2807,6 @@ function workflowRunIndexKey(run) {
1828
2807
  function globalRunIndexKey(run) {
1829
2808
  return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
1830
2809
  }
1831
- function correlationIndexKey(correlationId, createdAt, eventId, runId) {
1832
- return `correlation:${encodeURIComponent(correlationId)}:${sortableTimestamp(createdAt)}:${eventId}:${runId}`;
1833
- }
1834
- function indexMarkerKey(indexKey) {
1835
- return `${INDEX_MARKER_PREFIX}${encodeURIComponent(indexKey)}`;
1836
- }
1837
2810
  function hookMarkerKey(reference) {
1838
2811
  return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
1839
2812
  }
@@ -1857,9 +2830,18 @@ function expiredRead(tombstone) {
1857
2830
  }
1858
2831
  //#endregion
1859
2832
  //#region src/worker/durable-objects/WorkflowRunDO.ts
1860
- const PAYLOAD_DELETE_BATCH = 256;
1861
- const STREAM_DELETE_CONCURRENCY = 16;
2833
+ const STORAGE_BATCH_SIZE = 128;
2834
+ const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
2835
+ const HOOK_MARKER_PAGE_SIZE = 64;
2836
+ const TERMINAL_HOOK_PAGE_SIZE = 64;
2837
+ const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
2838
+ const STREAMS_PER_CLEANUP_PAGE = 16;
2839
+ const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
2840
+ const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
2841
+ const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
2842
+ const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
1862
2843
  const CLEANUP_RETRY_MAX_MS = 36e5;
2844
+ const CLEANUP_PROGRESS_KEY = "retention:progress";
1863
2845
  /**
1864
2846
  * Adapt DO storage (or a transaction) to the {@link EventStore} interface
1865
2847
  * shared with the test mocks. No casts: the methods line up structurally,
@@ -1868,11 +2850,21 @@ const CLEANUP_RETRY_MAX_MS = 36e5;
1868
2850
  function storeFrom(storage) {
1869
2851
  return {
1870
2852
  get: (key) => storage.get(key),
2853
+ getMany: (keys) => storage.get(keys),
1871
2854
  put: (key, value) => storage.put(key, value),
1872
2855
  delete: (key) => storage.delete(key),
1873
- list: (options) => storage.list(options)
2856
+ deleteMany: (keys) => storage.delete(keys),
2857
+ list: (options) => storage.list(options),
2858
+ deferTerminalCleanup: true
1874
2859
  };
1875
2860
  }
2861
+ function hookCreationIndexKey(hook) {
2862
+ return `${HOOK_CREATED_KEY_PREFIX}${hook.createdAt.toISOString()}:${hook.hookId}`;
2863
+ }
2864
+ async function putStorageEntries(storage, entries) {
2865
+ const all = Array.from(entries);
2866
+ for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
2867
+ }
1876
2868
  /**
1877
2869
  * Durable Object for managing a single workflow run's state.
1878
2870
  *
@@ -1888,26 +2880,51 @@ var WorkflowRunDO = class extends DurableObject {
1888
2880
  const clock = this.env?.clock;
1889
2881
  return typeof clock === "function" ? clock() : Date.now();
1890
2882
  }
1891
- async expiredTombstone(storage, now = this.now()) {
1892
- const tombstone = await storage.get(TOMBSTONE_KEY);
2883
+ tombstoneFrom(values, now = this.now()) {
2884
+ const tombstone = values.get(TOMBSTONE_KEY);
1893
2885
  if (tombstone) return tombstone;
1894
- const cleanup = await storage.get(CLEANUP_RECORD_KEY);
2886
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
1895
2887
  if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
1896
2888
  return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
1897
2889
  }
1898
- async read(reader) {
1899
- const tombstone = await this.expiredTombstone(this.ctx.storage);
1900
- if (tombstone) return expiredRead(tombstone);
2890
+ async retentionState(storage, now = this.now()) {
2891
+ const values = await storage.get([TOMBSTONE_KEY, CLEANUP_RECORD_KEY]);
1901
2892
  return {
1902
- ok: true,
1903
- value: await reader()
2893
+ cleanup: values.get(CLEANUP_RECORD_KEY),
2894
+ tombstone: this.tombstoneFrom(values, now)
1904
2895
  };
1905
2896
  }
2897
+ async read(reader) {
2898
+ return await this.ctx.storage.transaction(async (txn) => {
2899
+ const { tombstone } = await this.retentionState(txn);
2900
+ if (tombstone) return expiredRead(tombstone);
2901
+ return {
2902
+ ok: true,
2903
+ value: await reader(txn)
2904
+ };
2905
+ });
2906
+ }
2907
+ async readKey(key) {
2908
+ return await this.ctx.storage.transaction(async (txn) => {
2909
+ const values = await txn.get([
2910
+ TOMBSTONE_KEY,
2911
+ CLEANUP_RECORD_KEY,
2912
+ key
2913
+ ]);
2914
+ const tombstone = this.tombstoneFrom(values);
2915
+ if (tombstone) return expiredRead(tombstone);
2916
+ return {
2917
+ ok: true,
2918
+ value: values.get(key) ?? null
2919
+ };
2920
+ });
2921
+ }
1906
2922
  /** Apply an event and capture all retention metadata in the same transaction. */
1907
2923
  async applyEvent(request) {
1908
2924
  return await this.ctx.storage.transaction(async (txn) => {
1909
2925
  const now = new Date(this.now());
1910
- const tombstone = await this.expiredTombstone(txn, now.getTime());
2926
+ const retention = await this.retentionState(txn, now.getTime());
2927
+ const { tombstone } = retention;
1911
2928
  if (tombstone) return {
1912
2929
  ok: false,
1913
2930
  code: "RUN_EXPIRED",
@@ -1921,20 +2938,12 @@ var WorkflowRunDO = class extends DurableObject {
1921
2938
  });
1922
2939
  if (!outcome.ok) return outcome;
1923
2940
  await txn.put("event_sequence", eventSequence);
1924
- if (outcome.event?.correlationId) {
1925
- const key = correlationIndexKey(outcome.event.correlationId, outcome.event.createdAt, outcome.event.eventId, request.runId);
1926
- await txn.put(indexMarkerKey(key), key);
1927
- }
1928
- if (outcome.hookToIndex) {
1929
- const reference = {
1930
- hookId: outcome.hookToIndex.hookId,
1931
- token: outcome.hookToIndex.token
1932
- };
1933
- await txn.put(hookMarkerKey(reference), reference);
1934
- }
1935
- for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
2941
+ await putStorageEntries(txn, (outcome.hookToIndex ? [{
2942
+ hookId: outcome.hookToIndex.hookId,
2943
+ token: outcome.hookToIndex.token
2944
+ }, ...outcome.releasedHooks] : outcome.releasedHooks).map((reference) => [hookMarkerKey(reference), reference]));
1936
2945
  const retentionMs = request.cleanup?.retentionMs ?? 0;
1937
- if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !await txn.get("retention:cleanup")) {
2946
+ if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !retention.cleanup) {
1938
2947
  const terminalRun = outcome.run;
1939
2948
  const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
1940
2949
  const run = WorkflowRunSchema.parse({
@@ -1951,6 +2960,7 @@ var WorkflowRunDO = class extends DurableObject {
1951
2960
  dueAt,
1952
2961
  queueShards: request.cleanup?.queueShards ?? 1,
1953
2962
  phase: "retained",
2963
+ generation: 0,
1954
2964
  attempts: 0,
1955
2965
  deletedPayloadKeys: 0,
1956
2966
  deletedStreams: 0,
@@ -1961,34 +2971,60 @@ var WorkflowRunDO = class extends DurableObject {
1961
2971
  await txn.setAlarm(dueAt);
1962
2972
  outcome.run = run;
1963
2973
  }
2974
+ if (outcome.run && isTerminalWorkflowRunStatus(outcome.run.status)) {
2975
+ if (!await txn.get("terminal:cleanup")) await txn.put(TERMINAL_CLEANUP_KEY, {
2976
+ version: 1,
2977
+ runId: outcome.run.runId,
2978
+ phase: "hooks",
2979
+ generation: 0,
2980
+ attempts: 0
2981
+ });
2982
+ await txn.setAlarm(this.now() + 1);
2983
+ }
1964
2984
  return finalizeEventPage(storeFrom(txn), outcome, request.params);
1965
2985
  });
1966
2986
  }
2987
+ /**
2988
+ * Resolve an ambiguous hook-creation RPC in the same serialization domain as
2989
+ * applyEvent. If the hook is absent, the exact reservation is durably fenced
2990
+ * before the caller releases its token and hook-ID claims.
2991
+ */
2992
+ async resolveHookTokenClaim(request) {
2993
+ return await this.ctx.storage.transaction(async (txn) => {
2994
+ if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
2995
+ await txn.put(hookClaimCancellationKey(request.claimId), {
2996
+ hookId: request.hookId,
2997
+ token: request.token,
2998
+ canceledAt: this.now()
2999
+ });
3000
+ return { committed: false };
3001
+ });
3002
+ }
1967
3003
  async getRun() {
1968
- return this.read(async () => await this.ctx.storage.get("run") ?? null);
3004
+ return this.readKey("run");
1969
3005
  }
1970
3006
  async getStep(stepId) {
1971
- return this.read(async () => await this.ctx.storage.get(`step:${stepId}`) ?? null);
3007
+ return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
1972
3008
  }
1973
3009
  async listSteps(params) {
1974
- return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
3010
+ return this.read((txn) => listByCreationTime(storeFrom(txn), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
1975
3011
  limit: params?.limit ?? 20,
1976
3012
  cursor: params?.cursor,
1977
3013
  sortOrder: params?.sortOrder ?? "asc"
1978
3014
  }));
1979
3015
  }
1980
3016
  async getEvent(eventId) {
1981
- return this.read(async () => await this.ctx.storage.get(`event:${eventId}`) ?? null);
3017
+ return this.readKey(`${EVENT_KEY_PREFIX}${eventId}`);
1982
3018
  }
1983
3019
  async listEvents(params) {
1984
- return this.read(() => listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
3020
+ return this.read((txn) => listByPrefix(storeFrom(txn), EVENT_KEY_PREFIX, {
1985
3021
  limit: params?.limit ?? 100,
1986
3022
  cursor: params?.cursor,
1987
3023
  sortOrder: params?.sortOrder ?? "asc"
1988
3024
  }, (event) => event.eventId));
1989
3025
  }
1990
3026
  async listHooks(params) {
1991
- return this.read(() => listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
3027
+ return this.read((txn) => listByCreationTime(storeFrom(txn), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
1992
3028
  limit: params?.limit ?? 100,
1993
3029
  cursor: params?.cursor,
1994
3030
  sortOrder: params?.sortOrder ?? "asc"
@@ -2001,10 +3037,16 @@ var WorkflowRunDO = class extends DurableObject {
2001
3037
  async scheduleCleanup(request) {
2002
3038
  if (request.retentionMs <= 0) return this.getCleanupStatus();
2003
3039
  return await this.ctx.storage.transaction(async (txn) => {
2004
- const existing = await txn.get(CLEANUP_RECORD_KEY);
3040
+ const values = await txn.get([
3041
+ CLEANUP_RECORD_KEY,
3042
+ TOMBSTONE_KEY,
3043
+ TERMINAL_CLEANUP_KEY,
3044
+ "run"
3045
+ ]);
3046
+ const existing = values.get(CLEANUP_RECORD_KEY);
2005
3047
  if (existing) return existing;
2006
- if (await txn.get("retention:tombstone")) return null;
2007
- const current = await txn.get("run");
3048
+ if (values.get("retention:tombstone")) return null;
3049
+ const current = values.get("run");
2008
3050
  if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
2009
3051
  const terminalRun = current;
2010
3052
  const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
@@ -2022,6 +3064,7 @@ var WorkflowRunDO = class extends DurableObject {
2022
3064
  dueAt,
2023
3065
  queueShards: request.queueShards,
2024
3066
  phase: "retained",
3067
+ generation: 0,
2025
3068
  attempts: 0,
2026
3069
  deletedPayloadKeys: 0,
2027
3070
  deletedStreams: 0,
@@ -2029,7 +3072,7 @@ var WorkflowRunDO = class extends DurableObject {
2029
3072
  };
2030
3073
  await txn.put("run", run);
2031
3074
  await txn.put(CLEANUP_RECORD_KEY, cleanup);
2032
- await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
3075
+ await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
2033
3076
  return cleanup;
2034
3077
  });
2035
3078
  }
@@ -2049,135 +3092,334 @@ var WorkflowRunDO = class extends DurableObject {
2049
3092
  }));
2050
3093
  await txn.put(CLEANUP_RECORD_KEY, {
2051
3094
  ...cleanup,
2052
- dueAt
3095
+ dueAt,
3096
+ generation: cleanup.generation + 1
2053
3097
  });
2054
3098
  await txn.setAlarm(this.now() + 1);
2055
3099
  });
2056
- try {
2057
- await this.executeCleanup();
2058
- } catch (error) {
2059
- await this.recordCleanupFailure(error);
2060
- }
3100
+ await this.executeScheduledCleanupPage();
2061
3101
  return this.getCleanupStatus();
2062
3102
  }
2063
3103
  async rearmCleanup() {
2064
- const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
2065
- if (!cleanup) return null;
2066
- if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
3104
+ const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
3105
+ const cleanup = values.get(CLEANUP_RECORD_KEY);
3106
+ if (values.get("terminal:cleanup")) await this.ctx.storage.setAlarm(this.now() + 1);
3107
+ else if (!cleanup || cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
2067
3108
  else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
2068
- return cleanup;
3109
+ return cleanup ?? null;
2069
3110
  }
2070
3111
  async alarm() {
2071
- try {
2072
- await this.executeCleanup();
2073
- } catch (error) {
2074
- await this.recordCleanupFailure(error);
2075
- }
3112
+ await this.executeScheduledCleanupPage();
2076
3113
  }
2077
3114
  namespace(name) {
2078
3115
  const namespace = this.env?.[name];
2079
3116
  if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
2080
3117
  return namespace;
2081
3118
  }
2082
- async setCleanupPhase(expected, next, updates = {}) {
3119
+ workflowIndex() {
3120
+ return createWorkflowIndex({
3121
+ runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
3122
+ runFences: this.namespace("WORKFLOW_RUN_FENCES"),
3123
+ hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
3124
+ hookIds: this.namespace("WORKFLOW_HOOK_IDS")
3125
+ });
3126
+ }
3127
+ async setCleanupPhase(expected, next, updates = {}, continueAt) {
2083
3128
  await this.ctx.storage.transaction(async (txn) => {
2084
3129
  const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2085
- if (!cleanup || cleanup.phase !== expected) return;
3130
+ if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
2086
3131
  await txn.put(CLEANUP_RECORD_KEY, {
2087
3132
  ...cleanup,
2088
3133
  ...updates,
2089
3134
  phase: next,
3135
+ generation: cleanup.generation + 1,
2090
3136
  attempts: 0,
2091
3137
  lastError: void 0
2092
3138
  });
3139
+ if (continueAt !== void 0) await txn.setAlarm(continueAt);
2093
3140
  });
2094
3141
  }
2095
- async executeCleanup() {
2096
- for (;;) {
2097
- const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
2098
- if (!cleanup) {
2099
- await this.ctx.storage.deleteAlarm();
2100
- return;
3142
+ async executeScheduledCleanupPage() {
3143
+ const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
3144
+ if (terminalCleanup) {
3145
+ try {
3146
+ await this.executeTerminalCleanup(terminalCleanup);
3147
+ } catch (error) {
3148
+ await this.recordTerminalCleanupFailure(error, terminalCleanup);
2101
3149
  }
2102
- if (cleanup.phase === "tombstoned") {
2103
- await this.ctx.storage.deleteAlarm();
3150
+ return;
3151
+ }
3152
+ const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
3153
+ if (!cleanup || cleanup.phase === "tombstoned") {
3154
+ await this.ctx.storage.deleteAlarm();
3155
+ return;
3156
+ }
3157
+ try {
3158
+ await this.executeCleanup(cleanup);
3159
+ } catch (error) {
3160
+ await this.recordCleanupFailure(error, cleanup);
3161
+ }
3162
+ }
3163
+ async executeCleanup(cleanup) {
3164
+ if (cleanup.phase === "retained") {
3165
+ if (cleanup.dueAt.getTime() > this.now()) {
3166
+ await this.ctx.storage.setAlarm(cleanup.dueAt);
2104
3167
  return;
2105
3168
  }
2106
- if (cleanup.phase === "retained") {
2107
- if (cleanup.dueAt.getTime() > this.now()) {
2108
- await this.ctx.storage.setAlarm(cleanup.dueAt);
2109
- return;
2110
- }
2111
- await this.setCleanupPhase("retained", "index");
2112
- continue;
2113
- }
2114
- if (cleanup.phase === "index") {
2115
- const [indexMarkers, hookMarkers] = await Promise.all([this.ctx.storage.list({ prefix: INDEX_MARKER_PREFIX }), this.ctx.storage.list({ prefix: HOOK_MARKER_PREFIX })]);
2116
- const indexNamespace = this.namespace("WORKFLOW_INDEX");
2117
- await indexNamespace.get(indexNamespace.idFromName("index")).expireRun({
2118
- runId: cleanup.runId,
2119
- keys: [
2120
- workflowRunIndexKey(cleanup),
2121
- globalRunIndexKey(cleanup),
2122
- ...indexMarkers.values()
2123
- ],
2124
- hooks: Array.from(hookMarkers.values()),
2125
- expiredAt: cleanup.dueAt.getTime()
3169
+ await this.setCleanupPhase(cleanup, "index", {}, this.now() + 1);
3170
+ return;
3171
+ }
3172
+ if (cleanup.phase === "index") {
3173
+ await this.deleteIndexPage(cleanup);
3174
+ return;
3175
+ }
3176
+ if (cleanup.phase === "streams") {
3177
+ await this.deleteStreamPage(cleanup);
3178
+ return;
3179
+ }
3180
+ if (cleanup.phase === "queues") {
3181
+ await this.deleteQueuePage(cleanup);
3182
+ return;
3183
+ }
3184
+ if (cleanup.phase === "payload") await this.deletePayloadBatch();
3185
+ }
3186
+ async executeTerminalCleanup(cleanup) {
3187
+ if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
3188
+ else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
3189
+ else await this.deleteTerminalWaitPage(cleanup);
3190
+ }
3191
+ async deleteTerminalHookPage(cleanup) {
3192
+ const entries = await this.ctx.storage.list({
3193
+ prefix: HOOK_KEY_PREFIX,
3194
+ limit: 65
3195
+ });
3196
+ const page = Array.from(entries.values()).slice(0, TERMINAL_HOOK_PAGE_SIZE);
3197
+ await this.workflowIndex().releaseHookIndexes({
3198
+ runId: cleanup.runId,
3199
+ hooks: page.map((hook) => ({
3200
+ hookId: hook.hookId,
3201
+ token: hook.token
3202
+ })),
3203
+ terminal: true
3204
+ });
3205
+ await this.ctx.storage.transaction(async (txn) => {
3206
+ const current = await txn.get(TERMINAL_CLEANUP_KEY);
3207
+ if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
3208
+ const keys = page.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]);
3209
+ if (keys.length > 0) await txn.delete(keys);
3210
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3211
+ ...current,
3212
+ phase: entries.size <= TERMINAL_HOOK_PAGE_SIZE ? "markers" : "hooks",
3213
+ generation: current.generation + 1,
3214
+ attempts: 0,
3215
+ lastError: void 0
3216
+ });
3217
+ await txn.setAlarm(this.now() + 1);
3218
+ });
3219
+ }
3220
+ async deleteTerminalHookMarkerPage(cleanup) {
3221
+ const entries = await this.ctx.storage.list({
3222
+ prefix: HOOK_MARKER_PREFIX,
3223
+ limit: 65
3224
+ });
3225
+ const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
3226
+ await this.workflowIndex().releaseHookIndexes({
3227
+ runId: cleanup.runId,
3228
+ hooks: page.map(([, reference]) => reference),
3229
+ terminal: true
3230
+ });
3231
+ await this.ctx.storage.transaction(async (txn) => {
3232
+ const current = await txn.get(TERMINAL_CLEANUP_KEY);
3233
+ if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
3234
+ const keys = page.map(([key]) => key);
3235
+ if (keys.length > 0) await txn.delete(keys);
3236
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3237
+ ...current,
3238
+ phase: entries.size <= HOOK_MARKER_PAGE_SIZE ? "waits" : "markers",
3239
+ generation: current.generation + 1,
3240
+ attempts: 0,
3241
+ lastError: void 0
3242
+ });
3243
+ await txn.setAlarm(this.now() + 1);
3244
+ });
3245
+ }
3246
+ async deleteTerminalWaitPage(cleanup) {
3247
+ await this.ctx.storage.transaction(async (txn) => {
3248
+ const current = await txn.get(TERMINAL_CLEANUP_KEY);
3249
+ if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
3250
+ const entries = await txn.list({
3251
+ prefix: WAIT_KEY_PREFIX,
3252
+ limit: 129
3253
+ });
3254
+ const keys = Array.from(entries.keys()).slice(0, TERMINAL_WAIT_PAGE_SIZE);
3255
+ if (keys.length > 0) await txn.delete(keys);
3256
+ if (entries.size > TERMINAL_WAIT_PAGE_SIZE) {
3257
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3258
+ ...current,
3259
+ generation: current.generation + 1,
3260
+ attempts: 0,
3261
+ lastError: void 0
2126
3262
  });
2127
- await this.setCleanupPhase("index", "streams");
2128
- continue;
3263
+ await txn.setAlarm(this.now() + 1);
3264
+ return;
2129
3265
  }
2130
- if (cleanup.phase === "streams") {
2131
- const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
2132
- const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
2133
- const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime());
2134
- let deletedStreams = streams.length === 0 ? cleanup.deletedStreams : 0;
2135
- for (let offset = 0; offset < streams.length; offset += STREAM_DELETE_CONCURRENCY) {
2136
- const batch = streams.slice(offset, offset + STREAM_DELETE_CONCURRENCY);
2137
- const results = await Promise.all(batch.map((name) => {
2138
- return streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime());
2139
- }));
2140
- deletedStreams += results.filter((result) => result.deleted).length;
2141
- }
2142
- await this.ctx.storage.transaction(async (txn) => {
2143
- const current = await txn.get(CLEANUP_RECORD_KEY);
2144
- if (current?.phase === "streams") await txn.put(CLEANUP_RECORD_KEY, {
2145
- ...current,
2146
- deletedStreams
2147
- });
3266
+ await txn.delete(TERMINAL_CLEANUP_KEY);
3267
+ const retention = await txn.get(CLEANUP_RECORD_KEY);
3268
+ if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
3269
+ else await txn.deleteAlarm();
3270
+ });
3271
+ }
3272
+ async deleteIndexPage(cleanup) {
3273
+ const hookEntries = await this.ctx.storage.list({
3274
+ prefix: HOOK_MARKER_PREFIX,
3275
+ limit: 65
3276
+ });
3277
+ const hookMarkers = Array.from(hookEntries).slice(0, HOOK_MARKER_PAGE_SIZE);
3278
+ await this.workflowIndex().expireRun({
3279
+ runId: cleanup.runId,
3280
+ keys: [workflowRunIndexKey(cleanup), globalRunIndexKey(cleanup)],
3281
+ hooks: hookMarkers.map(([, value]) => value),
3282
+ expiredAt: cleanup.dueAt.getTime()
3283
+ });
3284
+ const markerKeys = hookMarkers.map(([key]) => key);
3285
+ await this.ctx.storage.transaction(async (txn) => {
3286
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3287
+ if (current?.phase !== "index" || current.generation !== cleanup.generation) return;
3288
+ if (markerKeys.length > 0) await txn.delete(markerKeys);
3289
+ if (hookEntries.size <= HOOK_MARKER_PAGE_SIZE) await txn.put(CLEANUP_RECORD_KEY, {
3290
+ ...current,
3291
+ phase: "streams",
3292
+ generation: current.generation + 1,
3293
+ attempts: 0,
3294
+ lastError: void 0
3295
+ });
3296
+ else await txn.put(CLEANUP_RECORD_KEY, {
3297
+ ...current,
3298
+ generation: current.generation + 1,
3299
+ attempts: 0,
3300
+ lastError: void 0
3301
+ });
3302
+ await txn.setAlarm(this.now() + 1);
3303
+ });
3304
+ }
3305
+ async deleteStreamPage(cleanup) {
3306
+ const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
3307
+ const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
3308
+ const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime(), { limit: STREAMS_PER_CLEANUP_PAGE });
3309
+ const completed = [];
3310
+ let remainingChunks = STREAM_CHUNKS_PER_CLEANUP_PAGE;
3311
+ let remainingBytes = STREAM_BYTES_PER_CLEANUP_PAGE;
3312
+ for (const name of streams) {
3313
+ if (remainingChunks <= 0 || remainingBytes <= 0) break;
3314
+ const result = await streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime(), {
3315
+ limit: remainingChunks,
3316
+ byteLimit: remainingBytes
3317
+ });
3318
+ remainingChunks -= result.chunks;
3319
+ remainingBytes -= result.bytes;
3320
+ if (result.done) completed.push(name);
3321
+ }
3322
+ const registryResult = await registry.finalizeRegistry(cleanup.runId, completed);
3323
+ await this.ctx.storage.transaction(async (txn) => {
3324
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3325
+ if (current?.phase !== "streams" || current.generation !== cleanup.generation) return;
3326
+ await txn.put(CLEANUP_RECORD_KEY, {
3327
+ ...current,
3328
+ deletedStreams: registryResult.deleted,
3329
+ phase: registryResult.done ? "queues" : "streams",
3330
+ generation: current.generation + 1,
3331
+ attempts: 0,
3332
+ lastError: void 0
3333
+ });
3334
+ await txn.setAlarm(this.now() + 1);
3335
+ });
3336
+ }
3337
+ async deleteQueuePage(cleanup) {
3338
+ const queues = this.namespace("WORKFLOW_QUEUE");
3339
+ let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
3340
+ queueShard: 0,
3341
+ queueShardDeleted: 0
3342
+ };
3343
+ let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
3344
+ let expectedGeneration = cleanup.generation;
3345
+ while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
3346
+ const requestedShard = progress.queueShard;
3347
+ const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
3348
+ const reconciled = await this.ctx.storage.transaction(async (txn) => {
3349
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3350
+ if (current?.phase !== "queues") return null;
3351
+ if (current.generation !== expectedGeneration) return { stale: true };
3352
+ const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3353
+ queueShard: 0,
3354
+ queueShardDeleted: 0
3355
+ };
3356
+ if (persisted.queueShard !== requestedShard) return { stale: true };
3357
+ if (result.deleted < persisted.queueShardDeleted) return { stale: true };
3358
+ const next = result.done ? {
3359
+ queueShard: persisted.queueShard + 1,
3360
+ queueShardDeleted: 0
3361
+ } : {
3362
+ ...persisted,
3363
+ queueShardDeleted: result.deleted
3364
+ };
3365
+ const delta = result.deleted - persisted.queueShardDeleted;
3366
+ await txn.put(CLEANUP_RECORD_KEY, {
3367
+ ...current,
3368
+ deletedQueueMessages: current.deletedQueueMessages + delta,
3369
+ generation: current.generation + 1,
3370
+ attempts: 0,
3371
+ lastError: void 0
2148
3372
  });
2149
- await registry.finalizeRegistry(cleanup.runId);
2150
- await this.setCleanupPhase("streams", "queues");
2151
- continue;
2152
- }
2153
- if (cleanup.phase === "queues") {
2154
- const queues = this.namespace("WORKFLOW_QUEUE");
2155
- let deletedQueueMessages = 0;
2156
- for (let shard = 0; shard < cleanup.queueShards; shard++) {
2157
- const result = await queues.get(queues.idFromName(`q:${shard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime());
2158
- deletedQueueMessages += result.deleted;
2159
- }
2160
- await this.setCleanupPhase("queues", "payload", { deletedQueueMessages });
2161
- continue;
2162
- }
2163
- if (cleanup.phase === "payload") {
2164
- if (!await this.deletePayloadBatch()) return;
2165
- }
3373
+ await txn.put(CLEANUP_PROGRESS_KEY, next);
3374
+ await txn.setAlarm(this.now() + 1);
3375
+ return {
3376
+ stale: false,
3377
+ progress: next,
3378
+ generation: current.generation + 1,
3379
+ applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
3380
+ };
3381
+ });
3382
+ if (!reconciled) return;
3383
+ if (reconciled.stale) return;
3384
+ progress = reconciled.progress;
3385
+ expectedGeneration = reconciled.generation;
3386
+ if (!reconciled.applied && progress.queueShard === requestedShard) break;
3387
+ if (!result.done && progress.queueShard === requestedShard) break;
2166
3388
  }
3389
+ await this.ctx.storage.transaction(async (txn) => {
3390
+ const current = await txn.get(CLEANUP_RECORD_KEY);
3391
+ if (current?.phase !== "queues") return;
3392
+ if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
3393
+ queueShard: 0,
3394
+ queueShardDeleted: 0
3395
+ }).queueShard >= current.queueShards) {
3396
+ await txn.put(CLEANUP_RECORD_KEY, {
3397
+ ...current,
3398
+ phase: "payload",
3399
+ generation: current.generation + 1
3400
+ });
3401
+ await txn.delete(CLEANUP_PROGRESS_KEY);
3402
+ await txn.setAlarm(this.now() + 1);
3403
+ }
3404
+ });
2167
3405
  }
2168
3406
  async deletePayloadBatch() {
2169
3407
  return await this.ctx.storage.transaction(async (txn) => {
2170
3408
  const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2171
3409
  if (!cleanup || cleanup.phase !== "payload") return true;
2172
- const entries = await txn.list({ limit: 259 });
3410
+ const entries = await txn.list({
3411
+ limit: 131,
3412
+ noCache: true
3413
+ });
2173
3414
  const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
2174
3415
  const now = new Date(this.now());
2175
3416
  await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
2176
- for (const key of keys) await txn.delete(key);
3417
+ if (keys.length > 0) await txn.delete(keys);
2177
3418
  if (keys.length > 0) {
2178
3419
  await txn.put(CLEANUP_RECORD_KEY, {
2179
3420
  ...cleanup,
2180
3421
  deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
3422
+ generation: cleanup.generation + 1,
2181
3423
  lastError: void 0
2182
3424
  });
2183
3425
  await txn.setAlarm(this.now() + 1);
@@ -2186,6 +3428,7 @@ var WorkflowRunDO = class extends DurableObject {
2186
3428
  await txn.put(CLEANUP_RECORD_KEY, {
2187
3429
  ...cleanup,
2188
3430
  phase: "tombstoned",
3431
+ generation: cleanup.generation + 1,
2189
3432
  attempts: 0,
2190
3433
  lastError: void 0,
2191
3434
  tombstonedAt: now
@@ -2194,14 +3437,31 @@ var WorkflowRunDO = class extends DurableObject {
2194
3437
  return true;
2195
3438
  });
2196
3439
  }
2197
- async recordCleanupFailure(error) {
3440
+ async recordCleanupFailure(error, expected) {
2198
3441
  await this.ctx.storage.transaction(async (txn) => {
2199
3442
  const cleanup = await txn.get(CLEANUP_RECORD_KEY);
2200
- if (!cleanup || cleanup.phase === "tombstoned") return;
3443
+ if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation || cleanup.phase === "tombstoned") return;
2201
3444
  const attempts = cleanup.attempts + 1;
2202
3445
  const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
2203
3446
  await txn.put(CLEANUP_RECORD_KEY, {
2204
3447
  ...cleanup,
3448
+ generation: cleanup.generation + 1,
3449
+ attempts,
3450
+ lastError: error instanceof Error ? error.message : String(error),
3451
+ lastAttemptAt: new Date(this.now())
3452
+ });
3453
+ await txn.setAlarm(this.now() + delay);
3454
+ });
3455
+ }
3456
+ async recordTerminalCleanupFailure(error, expected) {
3457
+ await this.ctx.storage.transaction(async (txn) => {
3458
+ const cleanup = await txn.get(TERMINAL_CLEANUP_KEY);
3459
+ if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
3460
+ const attempts = cleanup.attempts + 1;
3461
+ const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
3462
+ await txn.put(TERMINAL_CLEANUP_KEY, {
3463
+ ...cleanup,
3464
+ generation: cleanup.generation + 1,
2205
3465
  attempts,
2206
3466
  lastError: error instanceof Error ? error.message : String(error),
2207
3467
  lastAttemptAt: new Date(this.now())
@@ -2236,6 +3496,6 @@ var worker_default = { async fetch(request, env) {
2236
3496
  return createRouter(env)(request);
2237
3497
  } };
2238
3498
  //#endregion
2239
- export { IndexDO, QueueDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
3499
+ export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, RunFenceDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
2240
3500
 
2241
3501
  //# sourceMappingURL=worker.js.map