@ewhauser/world-celld 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/testing.js CHANGED
@@ -1,4 +1,4 @@
1
- import { A as MAX_STREAM_BATCH_BYTES, D as isRecord$1, E as isPositiveSafeInteger, G as validateStreamReadRequest, K as validateStreamWriteChunks, L as STREAM_BATCH_CONTENT_TYPE, N as MAX_STREAM_LONG_POLL_MS, P as MAX_STREAM_READ_BYTES, S as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, T as isNonNegativeSafeInteger, U as encodeStreamWriteResult, V as encodeStreamReadResult, W as normalizeStreamError, _ as HOOK_CLAIM_LEASE_MS, a as queueOrphanName, c as validateNativeQueueEnvelope, d as rpcParse, f as rpcStringify, h as createWorkflowIndex, i as queueClaimName, j as MAX_STREAM_CHUNK_BYTES, l as validateNativeQueueSendOptions, n as QUEUE_PAYLOAD_REGISTRY_PREFIX, o as queuePayloadObjectKey, p as parse, s as queuePayloadRegistryKey, t as NATIVE_QUEUE_MAX_MESSAGE_BYTES, u as compact, v as LIFECYCLE_COMPACTION_RETRY_MS, z as decodeStreamWriteBatch } from "./queue-protocol-CmJNMH_5.js";
1
+ import { A as isPositiveSafeInteger, E as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, F as MAX_STREAM_CHUNK_BYTES, H as STREAM_CHUNK_LIMIT_HEADER, J as encodeStreamWriteResult, K as encodeStreamReadResult, L as MAX_STREAM_LONG_POLL_MS, P as MAX_STREAM_BATCH_BYTES, Q as validateStreamWriteChunks, R as MAX_STREAM_READ_BYTES, S as LIFECYCLE_COMPACTION_RETRY_MS, V as STREAM_BATCH_CONTENT_TYPE, W as decodeStreamWriteBatch, X as normalizeStreamError, Z as validateStreamReadRequest, a as isInlineRunBody, c as queueOrphanName, d as validateNativeQueueEnvelope, f as validateNativeQueueSendOptions, g as parse, h as rpcStringify, i as QUEUE_PAYLOAD_REGISTRY_PREFIX, j as isRecord$1, k as isNonNegativeSafeInteger, l as queuePayloadObjectKey, m as rpcParse, n as listRunsPage, p as compact, r as NATIVE_QUEUE_MAX_MESSAGE_BYTES, s as queueClaimName, t as MAX_RUN_LIST_LIMIT, u as queuePayloadRegistryKey, v as createWorkflowIndex, x as HOOK_CLAIM_LEASE_MS, y as runIndexMetadata } from "./run-list-C987PkJ2.js";
2
2
  import { CreateEventSchema, EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
3
3
  import http from "node:http";
4
4
  //#region src/testing/fake-cell.ts
@@ -180,11 +180,12 @@ var FakeStorage = class {
180
180
  synchronous: false
181
181
  });
182
182
  }
183
- /** Inject one matching write failure, including writes inside transactions. */
184
- failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure")) {
183
+ /** Inject matching write failures (one by default), including writes inside transactions. */
184
+ failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure"), times = 1) {
185
185
  this.mutationFailure = {
186
186
  predicate,
187
- error
187
+ error,
188
+ remaining: times
188
189
  };
189
190
  }
190
191
  /** Inject one matching storage read failure, including reads inside transactions. */
@@ -198,7 +199,8 @@ var FakeStorage = class {
198
199
  maybeFailMutation(mutation) {
199
200
  if (!this.mutationFailure?.predicate(mutation)) return;
200
201
  const { error } = this.mutationFailure;
201
- this.mutationFailure = void 0;
202
+ this.mutationFailure.remaining -= 1;
203
+ if (this.mutationFailure.remaining <= 0) this.mutationFailure = void 0;
202
204
  throw error;
203
205
  }
204
206
  /** @internal Shared with FakeTransaction so read failures model transactional reads too. */
@@ -371,6 +373,10 @@ var FakeFleet = class {
371
373
  }
372
374
  }, this.cellEnv);
373
375
  }
376
+ /** Whether a cell has been activated, without activating it. */
377
+ hasCell(bindingKey, name) {
378
+ return this.cells.has(`${bindingKey}\0${name}`);
379
+ }
374
380
  cell(bindingKey, name) {
375
381
  const key = `${bindingKey}\0${name}`;
376
382
  let slot = this.cells.get(key);
@@ -1250,6 +1256,7 @@ async function finalizeEventPage(store, outcome, params) {
1250
1256
  maxEvents: getMaxEventsPerRun()
1251
1257
  } : outcome;
1252
1258
  if (!result.event) return result;
1259
+ if (params?.preloadEvents && result.event.eventType === "hook_received") return await attachHookReplayPreload(store, result);
1253
1260
  if (typeof params?.sinceCursor === "string") {
1254
1261
  const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
1255
1262
  limit: 100,
@@ -1279,6 +1286,32 @@ async function finalizeEventPage(store, outcome, params) {
1279
1286
  hasMore: events.length < skippedCount
1280
1287
  };
1281
1288
  }
1289
+ /**
1290
+ * Lazy hook resume preload: return the run and its complete replay log with
1291
+ * the hook_received write, read in the same transaction, so the runtime can
1292
+ * skip its run_started write and the index publication that follows it. The
1293
+ * runtime trusts only a complete log with a non-null cursor, so a run that is
1294
+ * not started, not running, or over the event ceiling gets the plain result.
1295
+ */
1296
+ async function attachHookReplayPreload(store, result) {
1297
+ const run = await store.get(RUN_KEY);
1298
+ if (!run?.startedAt || run.status !== "running") return result;
1299
+ const maxEvents = getMaxEventsPerRun();
1300
+ const log = await listByPrefix(store, EVENT_KEY_PREFIX, {
1301
+ limit: maxEvents,
1302
+ sortOrder: "asc"
1303
+ }, (event) => event.eventId);
1304
+ const last = log.data.at(-1);
1305
+ if (log.hasMore || last === void 0) return result;
1306
+ return {
1307
+ ...result,
1308
+ replayRun: run,
1309
+ maxEvents,
1310
+ events: log.data,
1311
+ cursor: last.eventId,
1312
+ hasMore: false
1313
+ };
1314
+ }
1282
1315
  //#endregion
1283
1316
  //#region src/worker/auth.ts
1284
1317
  /**
@@ -1321,6 +1354,46 @@ async function authenticate(request, secret) {
1321
1354
  return { ok: true };
1322
1355
  }
1323
1356
  //#endregion
1357
+ //#region src/worker/queue-claims.ts
1358
+ function cellQueueClaim(cell) {
1359
+ return {
1360
+ claim: (messageId, staleMs) => cell.claimInflight({
1361
+ messageId,
1362
+ staleMs
1363
+ }),
1364
+ hold: (params) => cell.holdInflight(params),
1365
+ release: (messageId) => cell.releaseInflight(messageId),
1366
+ complete: (messageId) => cell.completeQueueMessage(messageId)
1367
+ };
1368
+ }
1369
+ function runQueueClaim(run, claimName) {
1370
+ return {
1371
+ claim: async (messageId, staleMs) => {
1372
+ const result = await run.claimRunQueueMessage({
1373
+ claimName,
1374
+ messageId,
1375
+ staleMs
1376
+ });
1377
+ return result.expired ? {
1378
+ expired: true,
1379
+ claimed: false
1380
+ } : result;
1381
+ },
1382
+ hold: (params) => run.holdRunQueueMessage({
1383
+ claimName,
1384
+ ...params
1385
+ }),
1386
+ release: (messageId) => run.releaseRunQueueMessage({
1387
+ claimName,
1388
+ messageId
1389
+ }),
1390
+ complete: (messageId) => run.completeRunQueueMessage({
1391
+ claimName,
1392
+ messageId
1393
+ })
1394
+ };
1395
+ }
1396
+ //#endregion
1324
1397
  //#region src/worker/index-validation.ts
1325
1398
  function invalid(message) {
1326
1399
  const error = /* @__PURE__ */ new Error(`invalid index RPC arguments: ${message}`);
@@ -1415,6 +1488,34 @@ function listOptions(value) {
1415
1488
  }
1416
1489
  return parsed;
1417
1490
  }
1491
+ const RUN_STATUSES = /* @__PURE__ */ new Set([
1492
+ "pending",
1493
+ "running",
1494
+ "completed",
1495
+ "failed",
1496
+ "cancelled"
1497
+ ]);
1498
+ function runListRequest(value) {
1499
+ if (!isRecord$1(value)) invalid("run list request must be an object");
1500
+ const limit = value.limit;
1501
+ if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1e3) invalid(`request.limit must be an integer between 1 and ${MAX_RUN_LIST_LIMIT}`);
1502
+ const parsed = { limit };
1503
+ if (value.workflowName !== void 0) parsed.workflowName = nonEmptyString(value.workflowName, "request.workflowName");
1504
+ if (value.cursor !== void 0) parsed.cursor = stringValue(value.cursor, "request.cursor");
1505
+ if (value.statuses !== void 0) {
1506
+ if (!Array.isArray(value.statuses) || value.statuses.length === 0 || !value.statuses.every((status) => RUN_STATUSES.has(status))) invalid("request.statuses must be a non-empty array of run statuses");
1507
+ parsed.statuses = value.statuses;
1508
+ }
1509
+ if (value.sortOrder !== void 0) {
1510
+ if (value.sortOrder !== "asc" && value.sortOrder !== "desc") invalid("request.sortOrder must be 'asc' or 'desc'");
1511
+ parsed.sortOrder = value.sortOrder;
1512
+ }
1513
+ if (value.resolveData !== void 0) {
1514
+ if (value.resolveData !== "none" && value.resolveData !== "all") invalid("request.resolveData must be 'none' or 'all'");
1515
+ parsed.resolveData = value.resolveData;
1516
+ }
1517
+ return parsed;
1518
+ }
1418
1519
  function runCommit(args) {
1419
1520
  exactArgs(args, 3);
1420
1521
  const parsedRun = WorkflowRunSchema.safeParse(args[0]);
@@ -1475,6 +1576,12 @@ function validateIndexRequest(operation, args) {
1475
1576
  operation,
1476
1577
  args: [listOptions(args[0])]
1477
1578
  };
1579
+ case "runs.list-resolved":
1580
+ exactArgs(args, 1);
1581
+ return {
1582
+ operation,
1583
+ args: [runListRequest(args[0])]
1584
+ };
1478
1585
  case "runs.commit": return runCommit(args);
1479
1586
  case "runs.expire":
1480
1587
  exactArgs(args, 1);
@@ -1510,6 +1617,7 @@ function validateIndexRequest(operation, args) {
1510
1617
  }
1511
1618
  const INDEX_OPERATIONS = /* @__PURE__ */ new Set([
1512
1619
  "runs.list",
1620
+ "runs.list-resolved",
1513
1621
  "runs.commit",
1514
1622
  "runs.expire",
1515
1623
  "hooks.reserve",
@@ -1534,6 +1642,18 @@ const INDEX_OPERATIONS = /* @__PURE__ */ new Set([
1534
1642
  */
1535
1643
  const WORLD_NAME = "world-celld";
1536
1644
  const WORLD_VERSION = "0.1.0";
1645
+ /**
1646
+ * Per-isolate record of runs whose Queue idempotency claims stay in claim
1647
+ * cells (created before run-scoped claims) or live in the run cell. A run's
1648
+ * scope never changes, so the cache only saves the run-cell question.
1649
+ */
1650
+ const runClaimScopes = /* @__PURE__ */ new Map();
1651
+ const MAX_REMEMBERED_RUN_CLAIM_SCOPES = 4096;
1652
+ function rememberRunClaimScope(runId, scope) {
1653
+ if (runClaimScopes.has(runId)) return;
1654
+ if (runClaimScopes.size >= MAX_REMEMBERED_RUN_CLAIM_SCOPES) runClaimScopes.delete(runClaimScopes.keys().next().value);
1655
+ runClaimScopes.set(runId, scope);
1656
+ }
1537
1657
  function workflowIndex(env) {
1538
1658
  return createWorkflowIndex({
1539
1659
  runCatalog: env.WORKFLOW_RUN_CATALOG,
@@ -1541,6 +1661,37 @@ function workflowIndex(env) {
1541
1661
  hookIds: env.WORKFLOW_HOOK_IDS
1542
1662
  });
1543
1663
  }
1664
+ /**
1665
+ * Publish a committed applyEvent outcome to the derivative indexes inside the
1666
+ * fleet, sparing the client a round trip per run mutation. Each index write is
1667
+ * idempotent under the outcome's publication lease. A failed write leaves its
1668
+ * flag unset, so the client publishes it itself exactly as it would without
1669
+ * this step; older clients ignore the flags and publish again.
1670
+ */
1671
+ async function publishApplyEventIndexes(env, runId, outcome) {
1672
+ if (typeof outcome !== "object" || outcome === null) return;
1673
+ const success = outcome;
1674
+ if (success.ok !== true) return;
1675
+ const index = workflowIndex(env);
1676
+ const publications = [];
1677
+ if (success.run && typeof success.indexPublicationExpiresAt === "number") {
1678
+ const run = success.run;
1679
+ const lease = success.indexPublicationExpiresAt;
1680
+ publications.push((async () => {
1681
+ await index.commitRun(run, runIndexMetadata(run), lease);
1682
+ success.runIndexPublished = true;
1683
+ })());
1684
+ }
1685
+ const releasedHooks = success.releasedHooks;
1686
+ if (Array.isArray(releasedHooks) && releasedHooks.length > 0) publications.push((async () => {
1687
+ await index.releaseHookIndexes({
1688
+ runId,
1689
+ hooks: releasedHooks
1690
+ });
1691
+ success.hookIndexesReleased = true;
1692
+ })());
1693
+ await Promise.allSettled(publications);
1694
+ }
1544
1695
  const BINDINGS = {
1545
1696
  runs: {
1546
1697
  env: "WORKFLOW_DB",
@@ -1583,7 +1734,7 @@ const BINDINGS = {
1583
1734
  };
1584
1735
  /** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
1585
1736
  const MAX_BODY_BYTES = 33554432;
1586
- const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 1024;
1737
+ const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 2048 + 1024;
1587
1738
  const QUEUE_ORPHAN_GRACE_MS = 432e6;
1588
1739
  function errorResponse(status, name, message) {
1589
1740
  return Response.json({ error: {
@@ -1700,7 +1851,10 @@ function parseBoundedInteger(url, name, minimum, maximum) {
1700
1851
  function streamResponse(body) {
1701
1852
  return new Response(body, {
1702
1853
  status: 200,
1703
- headers: { "content-type": STREAM_BATCH_CONTENT_TYPE }
1854
+ headers: {
1855
+ "content-type": STREAM_BATCH_CONTENT_TYPE,
1856
+ [STREAM_CHUNK_LIMIT_HEADER]: String(512)
1857
+ }
1704
1858
  });
1705
1859
  }
1706
1860
  function createRouter(env) {
@@ -1737,31 +1891,71 @@ function createRouter(env) {
1737
1891
  try {
1738
1892
  const namespace = env.WORKFLOW_DB;
1739
1893
  const payloadStore = env.WORKFLOW_QUEUE_PAYLOADS;
1740
- const payloadKey = envelope.runId ? queuePayloadObjectKey(envelope.runId, envelope.messageId) : void 0;
1741
- const brokerEnvelope = payloadKey ? {
1894
+ const inlineRunBody = envelope.runId !== void 0 && isInlineRunBody(envelope.body);
1895
+ const payloadKey = envelope.runId && !inlineRunBody ? queuePayloadObjectKey(envelope.runId, envelope.messageId) : void 0;
1896
+ let brokerEnvelope = payloadKey ? {
1742
1897
  ...envelope,
1743
1898
  payloadKey,
1744
1899
  body: void 0
1745
1900
  } : envelope;
1746
- const encoded = JSON.stringify(brokerEnvelope);
1747
- if (new TextEncoder().encode(encoded).byteLength > 128e3) return errorResponse(413, "PayloadTooLarge", `native queue envelope exceeds ${NATIVE_QUEUE_MAX_MESSAGE_BYTES} bytes`);
1901
+ const sized = JSON.stringify({
1902
+ ...brokerEnvelope,
1903
+ claimScope: "run"
1904
+ });
1905
+ if (new TextEncoder().encode(sized).byteLength > 128e3) return errorResponse(413, "PayloadTooLarge", `native queue envelope exceeds ${NATIVE_QUEUE_MAX_MESSAGE_BYTES} bytes`);
1748
1906
  const now = Date.now();
1749
1907
  const reservationExpiresAt = Math.max(now, envelope.notBefore ?? now) + QUEUE_ORPHAN_GRACE_MS;
1908
+ const run = envelope.runId ? namespace.get(namespace.idFromName(envelope.runId)) : void 0;
1750
1909
  let claim;
1910
+ let admission;
1751
1911
  if (envelope.idempotencyKey) {
1752
- claim = namespace.get(namespace.idFromName(queueClaimName(envelope.queueName, envelope.idempotencyKey)));
1753
- const reservation = await claim.reserveQueueMessage({
1754
- messageId: envelope.messageId,
1755
- expiresAt: reservationExpiresAt
1756
- });
1912
+ const claimName = queueClaimName(envelope.queueName, envelope.idempotencyKey);
1913
+ let scope = run && envelope.runId ? runClaimScopes.get(envelope.runId) : "cell";
1914
+ let reservation;
1915
+ if (run && envelope.runId && scope !== "cell") {
1916
+ const reserved = await run.reserveRunQueueMessage({
1917
+ claimName,
1918
+ messageId: envelope.messageId,
1919
+ expiresAt: reservationExpiresAt
1920
+ });
1921
+ if (!reserved.ok) return errorResponse(410, "RunExpiredError", reserved.message);
1922
+ scope = reserved.scope;
1923
+ rememberRunClaimScope(envelope.runId, reserved.scope);
1924
+ if (reserved.scope === "run") {
1925
+ reservation = reserved;
1926
+ claim = runQueueClaim(run, claimName);
1927
+ brokerEnvelope = {
1928
+ ...brokerEnvelope,
1929
+ claimScope: "run"
1930
+ };
1931
+ }
1932
+ } else if (inlineRunBody && run) {
1933
+ admission = run.getQueueAdmission();
1934
+ admission.catch(() => void 0);
1935
+ }
1936
+ if (!reservation) {
1937
+ const cell = namespace.get(namespace.idFromName(claimName));
1938
+ reservation = await cell.reserveQueueMessage({
1939
+ messageId: envelope.messageId,
1940
+ expiresAt: reservationExpiresAt
1941
+ });
1942
+ claim = cellQueueClaim(cell);
1943
+ }
1757
1944
  if (!reservation.admitted) return new Response(rpcStringify({ messageId: reservation.messageId }), {
1758
1945
  status: 200,
1759
1946
  headers: { "content-type": "application/json" }
1760
1947
  });
1948
+ } else if (inlineRunBody && run) admission = run.getQueueAdmission();
1949
+ const encoded = JSON.stringify(brokerEnvelope);
1950
+ if (admission) {
1951
+ const admitted = await admission;
1952
+ if (!admitted.ok) {
1953
+ await claim?.complete(envelope.messageId);
1954
+ return errorResponse(410, "RunExpiredError", admitted.message);
1955
+ }
1761
1956
  }
1762
- if (envelope.runId && payloadKey) {
1957
+ if (run && envelope.runId && payloadKey) {
1763
1958
  if (!payloadStore) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_QUEUE_PAYLOADS");
1764
- const run = namespace.get(namespace.idFromName(envelope.runId));
1765
1959
  const orphanExpiresAt = reservationExpiresAt;
1766
1960
  const registered = await run.registerQueuePayload({
1767
1961
  messageId: envelope.messageId,
@@ -1769,7 +1963,7 @@ function createRouter(env) {
1769
1963
  orphanExpiresAt
1770
1964
  });
1771
1965
  if (!registered.ok) {
1772
- await claim?.completeQueueMessage(envelope.messageId);
1966
+ await claim?.complete(envelope.messageId);
1773
1967
  return errorResponse(410, "RunExpiredError", registered.message);
1774
1968
  }
1775
1969
  const orphan = namespace.get(namespace.idFromName(queueOrphanName(envelope.messageId)));
@@ -1782,7 +1976,7 @@ function createRouter(env) {
1782
1976
  });
1783
1977
  } catch (error) {
1784
1978
  await run.unregisterQueuePayload(envelope.messageId);
1785
- await claim?.completeQueueMessage(envelope.messageId);
1979
+ await claim?.complete(envelope.messageId);
1786
1980
  throw error;
1787
1981
  }
1788
1982
  try {
@@ -1791,21 +1985,21 @@ function createRouter(env) {
1791
1985
  messageId: envelope.messageId
1792
1986
  });
1793
1987
  } catch (error) {
1794
- await claim?.completeQueueMessage(envelope.messageId);
1988
+ await claim?.complete(envelope.messageId);
1795
1989
  throw error;
1796
1990
  }
1797
1991
  let finalized;
1798
1992
  try {
1799
1993
  finalized = await run.finalizeQueuePayload(envelope.messageId);
1800
1994
  } catch (error) {
1801
- await claim?.completeQueueMessage(envelope.messageId);
1995
+ await claim?.complete(envelope.messageId);
1802
1996
  throw error;
1803
1997
  }
1804
1998
  if (!finalized.ok) {
1805
1999
  await payloadStore.delete(payloadKey);
1806
2000
  await run.unregisterQueuePayload(envelope.messageId);
1807
2001
  await orphan.cancelQueuePayloadOrphan(envelope.messageId);
1808
- await claim?.completeQueueMessage(envelope.messageId);
2002
+ await claim?.complete(envelope.messageId);
1809
2003
  return errorResponse(410, "RunExpiredError", finalized.message);
1810
2004
  }
1811
2005
  }
@@ -1864,6 +2058,18 @@ function createRouter(env) {
1864
2058
  case "runs.list":
1865
2059
  result = await index.listRuns(...validated.args);
1866
2060
  break;
2061
+ case "runs.list-resolved": {
2062
+ const runs = env.WORKFLOW_DB;
2063
+ if (!runs) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_DB");
2064
+ result = await listRunsPage({
2065
+ listRuns: (options) => index.listRuns(options),
2066
+ readRun: async (runId) => {
2067
+ const outcome = await runs.get(runs.idFromName(runId)).getRun();
2068
+ return outcome.ok ? outcome.value : null;
2069
+ }
2070
+ }, ...validated.args);
2071
+ break;
2072
+ }
1867
2073
  case "runs.commit":
1868
2074
  result = await index.commitRun(...validated.args);
1869
2075
  break;
@@ -1920,7 +2126,7 @@ function createRouter(env) {
1920
2126
  const readRequest = {
1921
2127
  runId,
1922
2128
  startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
1923
- maxChunks: parseBoundedInteger(url, "maxChunks", 0, 32),
2129
+ maxChunks: parseBoundedInteger(url, "maxChunks", 0, 512),
1924
2130
  maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
1925
2131
  waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
1926
2132
  };
@@ -1984,6 +2190,7 @@ function createRouter(env) {
1984
2190
  }
1985
2191
  try {
1986
2192
  const result = await namespace.get(namespace.idFromName(name))[method](...args);
2193
+ if (bindingKey === "runs" && method === "applyEvent") await publishApplyEventIndexes(env, name, result);
1987
2194
  return new Response(rpcStringify(result ?? null), {
1988
2195
  status: 200,
1989
2196
  headers: { "content-type": "application/json" }
@@ -2413,9 +2620,18 @@ var RunCatalogDO = class extends DurableObject {
2413
2620
  async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
2414
2621
  const canonicalKeys = canonicalCatalogKeys(runId, keys);
2415
2622
  return await this.ctx.storage.transaction(async (txn) => {
2416
- if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
2623
+ const recordKey = catalogKeysKey(runId);
2624
+ const existing = await txn.get([
2625
+ expiredKey(runId),
2626
+ canonicalKeys[0],
2627
+ canonicalKeys[1],
2628
+ recordKey
2629
+ ]);
2630
+ if (existing.get(expiredKey(runId)) !== void 0) return { stored: false };
2417
2631
  const now = this.now();
2418
2632
  if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
2633
+ const record = existing.get(recordKey);
2634
+ if (existing.get(canonicalKeys[0]) === serializedMetadata && existing.get(canonicalKeys[1]) === serializedMetadata && record?.keys.length === canonicalKeys.length && record.keys.every((key, index) => key === canonicalKeys[index])) return { stored: true };
2419
2635
  await txn.put({
2420
2636
  [canonicalKeys[0]]: serializedMetadata,
2421
2637
  [canonicalKeys[1]]: serializedMetadata,
@@ -2527,13 +2743,23 @@ const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
2527
2743
  const STREAM_REGISTRY_PREFIX = "stream:";
2528
2744
  const REGISTRY_OWNER_KEY = "registry:owner";
2529
2745
  const REGISTRY_EXPIRED_KEY = "registry:expired";
2530
- /** Each chunk occupies a payload key and a size key; one delete accepts at most 128 keys. */
2746
+ /** Each chunk occupies a payload key and a size key, deleted in calls of at most 128 keys. */
2531
2747
  const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
2532
- const MAX_EXPIRE_CHUNK_LIMIT = 64;
2748
+ const MAX_EXPIRE_CHUNK_LIMIT = 256;
2749
+ const DELETE_BATCH_KEYS = 128;
2533
2750
  /** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
2534
2751
  const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
2535
2752
  const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
2536
2753
  const MAX_STREAM_INDEX = 2147483647;
2754
+ const SEGMENT_LAYOUT = 2;
2755
+ const SEGMENT_KEY_PREFIX = "seg:";
2756
+ const SEGMENT_SIZE_KEY_PREFIX = "segsize:";
2757
+ /** Payload bytes per segment row, well below the 2 MiB per-value storage limit. */
2758
+ const MAX_SEGMENT_BYTES = 1048576;
2759
+ /** Layout of a stream's first write. Existing streams keep the layout they started with. */
2760
+ const NEW_STREAM_LAYOUT = SEGMENT_LAYOUT;
2761
+ /** One storage get or put call accepts at most 128 keys. */
2762
+ const STORAGE_BATCH_KEYS = 128;
2537
2763
  function boundedLimit(value, fallback, maximum) {
2538
2764
  if (value === void 0 || !Number.isFinite(value)) return fallback;
2539
2765
  return Math.max(1, Math.min(maximum, Math.floor(value)));
@@ -2552,7 +2778,7 @@ function validateMeta(meta) {
2552
2778
  "closed",
2553
2779
  "errored",
2554
2780
  "expired"
2555
- ].includes(candidate.state) || candidate.ownerRunId !== void 0 && typeof candidate.ownerRunId !== "string" || candidate.payloadDeleted !== void 0 && typeof candidate.payloadDeleted !== "boolean") throw new Error("Invalid persisted stream metadata");
2781
+ ].includes(candidate.state) || candidate.ownerRunId !== void 0 && typeof candidate.ownerRunId !== "string" || candidate.payloadDeleted !== void 0 && typeof candidate.payloadDeleted !== "boolean" || candidate.layout !== void 0 && candidate.layout !== SEGMENT_LAYOUT) throw new Error("Invalid persisted stream metadata");
2556
2782
  const error = candidate.error;
2557
2783
  if (error !== void 0 && (!error || typeof error !== "object" || typeof error.name !== "string" || typeof error.message !== "string") || candidate.state === "errored" !== (error !== void 0)) throw new Error("Invalid persisted stream error metadata");
2558
2784
  return candidate;
@@ -2564,6 +2790,80 @@ function chunkKey(index) {
2564
2790
  function chunkSizeKey(index) {
2565
2791
  return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
2566
2792
  }
2793
+ function segmentKey(start) {
2794
+ return `${SEGMENT_KEY_PREFIX}${start.toString().padStart(12, "0")}`;
2795
+ }
2796
+ function segmentSizeKey(start) {
2797
+ return `${SEGMENT_SIZE_KEY_PREFIX}${start.toString().padStart(12, "0")}`;
2798
+ }
2799
+ function startFromSegmentKey(key, prefix) {
2800
+ const suffix = key.slice(prefix.length);
2801
+ const start = Number(suffix);
2802
+ if (!key.startsWith(prefix) || !/^\d{12}$/.test(suffix) || !Number.isSafeInteger(start) || start > MAX_STREAM_INDEX) throw new Error(`Invalid persisted stream segment key "${key}"`);
2803
+ return start;
2804
+ }
2805
+ function validateSegmentSize(value, start) {
2806
+ const size = value;
2807
+ if (!size || !Number.isSafeInteger(size.count) || size.count < 1 || !Number.isSafeInteger(size.bytes) || size.bytes < 0) throw new Error(`Invalid persisted stream segment size at index ${start}`);
2808
+ return size;
2809
+ }
2810
+ /** Pack chunks as `[u32 count][u32 length]*count` followed by the payloads. */
2811
+ function encodeSegment(chunks) {
2812
+ const payloadBytes = chunks.reduce((total, chunk) => total + chunk.byteLength, 0);
2813
+ const header = 4 + 4 * chunks.length;
2814
+ const encoded = new Uint8Array(header + payloadBytes);
2815
+ const view = new DataView(encoded.buffer);
2816
+ view.setUint32(0, chunks.length);
2817
+ let offset = header;
2818
+ for (let index = 0; index < chunks.length; index++) {
2819
+ view.setUint32(4 + 4 * index, chunks[index].byteLength);
2820
+ encoded.set(chunks[index], offset);
2821
+ offset += chunks[index].byteLength;
2822
+ }
2823
+ return encoded;
2824
+ }
2825
+ function decodeSegment(value, start, size) {
2826
+ if (!(value instanceof Uint8Array) || value.byteLength < 4) throw new Error(`Invalid persisted stream segment at index ${start}`);
2827
+ const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
2828
+ const count = view.getUint32(0);
2829
+ const header = 4 + 4 * count;
2830
+ if (count !== size.count || header + size.bytes !== value.byteLength) throw new Error(`Invalid persisted stream segment at index ${start}`);
2831
+ const chunks = [];
2832
+ let offset = header;
2833
+ for (let index = 0; index < count; index++) {
2834
+ const length = view.getUint32(4 + 4 * index);
2835
+ if (offset + length > value.byteLength) throw new Error(`Invalid persisted stream segment at index ${start}`);
2836
+ chunks.push(value.subarray(offset, offset + length));
2837
+ offset += length;
2838
+ }
2839
+ if (offset !== value.byteLength) throw new Error(`Invalid persisted stream segment at index ${start}`);
2840
+ return chunks;
2841
+ }
2842
+ /** Group one write's chunks into segments of at most MAX_SEGMENT_BYTES payload bytes. */
2843
+ function packSegments(startIndex, chunks) {
2844
+ const segments = [];
2845
+ let current = [];
2846
+ let bytes = 0;
2847
+ let start = startIndex;
2848
+ for (const chunk of chunks) {
2849
+ if (current.length > 0 && bytes + chunk.byteLength > MAX_SEGMENT_BYTES) {
2850
+ segments.push({
2851
+ start,
2852
+ chunks: current
2853
+ });
2854
+ start += current.length;
2855
+ current = [];
2856
+ bytes = 0;
2857
+ }
2858
+ current.push(chunk);
2859
+ bytes += chunk.byteLength;
2860
+ }
2861
+ if (current.length > 0) segments.push({
2862
+ start,
2863
+ chunks: current
2864
+ });
2865
+ return segments;
2866
+ }
2567
2867
  function indexFromChunkSizeKey(key) {
2568
2868
  const suffix = key.slice(11);
2569
2869
  const index = Number(suffix);
@@ -2580,6 +2880,9 @@ function validateChunkSize(size, index) {
2580
2880
  if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
2581
2881
  return size;
2582
2882
  }
2883
+ async function deleteInBatches(txn, keys) {
2884
+ for (let offset = 0; offset < keys.length; offset += DELETE_BATCH_KEYS) await txn.delete(keys.slice(offset, offset + DELETE_BATCH_KEYS));
2885
+ }
2583
2886
  function compactChunk(chunk) {
2584
2887
  return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
2585
2888
  }
@@ -2599,6 +2902,15 @@ var StreamDO = class extends DurableObject {
2599
2902
  mutationTail = Promise.resolve();
2600
2903
  changeVersion = 0;
2601
2904
  waiters = /* @__PURE__ */ new Set();
2905
+ /** Multi-key read split into calls within the per-call key limit. */
2906
+ async getMany(keys) {
2907
+ if (keys.length <= STORAGE_BATCH_KEYS) return await this.ctx.storage.get(keys);
2908
+ const groups = [];
2909
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_KEYS) groups.push(keys.slice(offset, offset + STORAGE_BATCH_KEYS));
2910
+ const merged = /* @__PURE__ */ new Map();
2911
+ for (const result of await Promise.all(groups.map((group) => this.ctx.storage.get(group)))) for (const [key, value] of result) merged.set(key, value);
2912
+ return merged;
2913
+ }
2602
2914
  async getMeta() {
2603
2915
  if (this.meta) return this.meta;
2604
2916
  this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
@@ -2667,29 +2979,8 @@ var StreamDO = class extends DurableObject {
2667
2979
  const meta = await this.getMeta();
2668
2980
  this.assertOwner(meta, request.runId);
2669
2981
  const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
2670
- const chunks = [];
2671
- if (available > 0) {
2672
- const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
2673
- const sizeKeys = indexes.map(chunkSizeKey);
2674
- const storedSizes = await this.ctx.storage.get(sizeKeys);
2675
- const selectedIndexes = [];
2676
- const selectedSizes = [];
2677
- let bytes = 0;
2678
- for (const index of indexes) {
2679
- const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
2680
- if (bytes + size > request.maxBytes) break;
2681
- selectedIndexes.push(index);
2682
- selectedSizes.push(size);
2683
- bytes += size;
2684
- }
2685
- const payloadKeys = selectedIndexes.map(chunkKey);
2686
- const storedChunks = await this.ctx.storage.get(payloadKeys);
2687
- for (let offset = 0; offset < payloadKeys.length; offset++) {
2688
- const chunk = storedChunks.get(payloadKeys[offset]);
2689
- if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
2690
- chunks.push(chunk);
2691
- }
2692
- }
2982
+ let chunks = [];
2983
+ if (available > 0) chunks = meta.layout === SEGMENT_LAYOUT ? await this.readSegmentRows(request.startIndex, available, request.maxBytes) : await this.readChunkRows(request.startIndex, available, request.maxBytes);
2693
2984
  return {
2694
2985
  startIndex: request.startIndex,
2695
2986
  tailIndex: meta.count - 1,
@@ -2699,10 +2990,91 @@ var StreamDO = class extends DurableObject {
2699
2990
  error: meta.error
2700
2991
  };
2701
2992
  }
2993
+ async readChunkRows(startIndex, available, maxBytes) {
2994
+ const indexes = Array.from({ length: available }, (_, offset) => startIndex + offset);
2995
+ const sizeKeys = indexes.map(chunkSizeKey);
2996
+ const storedSizes = await this.getMany(sizeKeys);
2997
+ const selectedIndexes = [];
2998
+ const selectedSizes = [];
2999
+ let bytes = 0;
3000
+ for (const index of indexes) {
3001
+ const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
3002
+ if (bytes + size > maxBytes) break;
3003
+ selectedIndexes.push(index);
3004
+ selectedSizes.push(size);
3005
+ bytes += size;
3006
+ }
3007
+ const payloadKeys = selectedIndexes.map(chunkKey);
3008
+ const storedChunks = await this.getMany(payloadKeys);
3009
+ const chunks = [];
3010
+ for (let offset = 0; offset < payloadKeys.length; offset++) {
3011
+ const chunk = storedChunks.get(payloadKeys[offset]);
3012
+ if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
3013
+ chunks.push(chunk);
3014
+ }
3015
+ return chunks;
3016
+ }
3017
+ /**
3018
+ * Plan from the small segment size rows, then load only the segments that
3019
+ * cover the requested range. A read that starts on a segment boundary (the
3020
+ * common sequential case) needs one list and one multi-get.
3021
+ */
3022
+ async readSegmentRows(startIndex, available, maxBytes) {
3023
+ const endIndex = startIndex + available;
3024
+ const listed = await this.ctx.storage.list({
3025
+ prefix: SEGMENT_SIZE_KEY_PREFIX,
3026
+ start: segmentSizeKey(startIndex),
3027
+ end: segmentSizeKey(endIndex),
3028
+ limit: available
3029
+ });
3030
+ const planned = [];
3031
+ for (const [key, value] of listed) {
3032
+ const start = startFromSegmentKey(key, SEGMENT_SIZE_KEY_PREFIX);
3033
+ planned.push({
3034
+ start,
3035
+ size: validateSegmentSize(value, start)
3036
+ });
3037
+ }
3038
+ if (planned[0]?.start !== startIndex) {
3039
+ const entry = (await this.ctx.storage.list({
3040
+ prefix: SEGMENT_SIZE_KEY_PREFIX,
3041
+ end: segmentSizeKey(startIndex + 1),
3042
+ reverse: true,
3043
+ limit: 1
3044
+ })).entries().next().value;
3045
+ if (!entry) throw new Error(`Missing persisted stream segment at index ${startIndex}`);
3046
+ const start = startFromSegmentKey(entry[0], SEGMENT_SIZE_KEY_PREFIX);
3047
+ planned.unshift({
3048
+ start,
3049
+ size: validateSegmentSize(entry[1], start)
3050
+ });
3051
+ }
3052
+ const selected = [];
3053
+ let plannedBytes = 0;
3054
+ for (const segment of planned) {
3055
+ if (selected.length > 0 && plannedBytes >= maxBytes) break;
3056
+ selected.push(segment);
3057
+ plannedBytes += segment.size.bytes;
3058
+ }
3059
+ const values = await this.getMany(selected.map(({ start }) => segmentKey(start)));
3060
+ const chunks = [];
3061
+ let bytes = 0;
3062
+ let expected = startIndex;
3063
+ for (const { start, size } of selected) {
3064
+ if (start > expected) throw new Error(`Missing persisted stream segment at index ${expected}`);
3065
+ const segment = decodeSegment(values.get(segmentKey(start)), start, size);
3066
+ for (let offset = expected - start; offset < segment.length; offset++) {
3067
+ if (expected >= endIndex || bytes + segment[offset].byteLength > maxBytes) return chunks;
3068
+ chunks.push(segment[offset]);
3069
+ bytes += segment[offset].byteLength;
3070
+ expected++;
3071
+ }
3072
+ }
3073
+ return chunks;
3074
+ }
2702
3075
  /** Append one bounded binary batch with contiguous indexes. */
2703
3076
  async writeChunks(runId, chunks) {
2704
3077
  validateStreamWriteChunks(chunks);
2705
- const storedChunks = chunks.map(compactChunk);
2706
3078
  return await this.runMutation(async () => {
2707
3079
  const committed = await this.ctx.storage.transaction(async (txn) => {
2708
3080
  const stored = await txn.get(META_KEY);
@@ -2710,18 +3082,30 @@ var StreamDO = class extends DurableObject {
2710
3082
  this.assertWritable(meta, runId);
2711
3083
  const startIndex = meta.count;
2712
3084
  if (startIndex + chunks.length > MAX_STREAM_INDEX) throw new Error("Stream offset limit exceeded");
3085
+ const layout = stored === void 0 ? NEW_STREAM_LAYOUT : meta.layout;
2713
3086
  const nextMeta = {
2714
3087
  count: startIndex + chunks.length,
2715
3088
  state: "open",
2716
- ownerRunId: runId
3089
+ ownerRunId: runId,
3090
+ ...layout === void 0 ? {} : { layout }
2717
3091
  };
2718
- const entries = { [META_KEY]: nextMeta };
2719
- for (let offset = 0; offset < storedChunks.length; offset++) {
2720
- const index = startIndex + offset;
2721
- entries[chunkKey(index)] = storedChunks[offset];
2722
- entries[chunkSizeKey(index)] = storedChunks[offset].byteLength;
3092
+ const entries = [[META_KEY, nextMeta]];
3093
+ if (layout === SEGMENT_LAYOUT) for (const segment of packSegments(startIndex, chunks)) {
3094
+ entries.push([segmentKey(segment.start), encodeSegment(segment.chunks)]);
3095
+ entries.push([segmentSizeKey(segment.start), {
3096
+ count: segment.chunks.length,
3097
+ bytes: segment.chunks.reduce((total, chunk) => total + chunk.byteLength, 0)
3098
+ }]);
3099
+ }
3100
+ else {
3101
+ const storedChunks = chunks.map(compactChunk);
3102
+ for (let offset = 0; offset < storedChunks.length; offset++) {
3103
+ const index = startIndex + offset;
3104
+ entries.push([chunkKey(index), storedChunks[offset]]);
3105
+ entries.push([chunkSizeKey(index), storedChunks[offset].byteLength]);
3106
+ }
2723
3107
  }
2724
- await txn.put(entries);
3108
+ for (let offset = 0; offset < entries.length; offset += STORAGE_BATCH_KEYS) await txn.put(Object.fromEntries(entries.slice(offset, offset + STORAGE_BATCH_KEYS)));
2725
3109
  return {
2726
3110
  meta: nextMeta,
2727
3111
  result: {
@@ -2855,6 +3239,102 @@ var StreamDO = class extends DurableObject {
2855
3239
  };
2856
3240
  });
2857
3241
  }
3242
+ /** Delete one page of per-chunk rows, listing size rows to avoid loading payloads. */
3243
+ async expireChunkPage(txn, limit, byteLimit) {
3244
+ const candidates = await txn.list({
3245
+ prefix: CHUNK_SIZE_KEY_PREFIX,
3246
+ limit: limit + 1
3247
+ });
3248
+ const page = [];
3249
+ let bytes = 0;
3250
+ for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
3251
+ const index = indexFromChunkSizeKey(key);
3252
+ const size = validateChunkSize(storedSize, index);
3253
+ if (page.length > 0 && bytes + size > byteLimit) break;
3254
+ page.push({
3255
+ index,
3256
+ size
3257
+ });
3258
+ bytes += size;
3259
+ }
3260
+ let payloadFallback = false;
3261
+ if (candidates.size === 0) {
3262
+ const first = (await txn.list({
3263
+ prefix: CHUNK_KEY_PREFIX,
3264
+ limit: 1
3265
+ })).entries().next().value;
3266
+ if (first) {
3267
+ const [key, payload] = first;
3268
+ page.push({
3269
+ index: indexFromChunkKey(key),
3270
+ size: payload instanceof Uint8Array ? payload.byteLength : 0
3271
+ });
3272
+ bytes = page[0].size;
3273
+ payloadFallback = true;
3274
+ }
3275
+ }
3276
+ await deleteInBatches(txn, page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]));
3277
+ let done = candidates.size === page.length;
3278
+ if (payloadFallback) done = (await txn.list({
3279
+ prefix: CHUNK_KEY_PREFIX,
3280
+ limit: 1
3281
+ })).size === 0;
3282
+ else if (done && candidates.size > 0) done = (await txn.list({
3283
+ prefix: CHUNK_KEY_PREFIX,
3284
+ limit: 1
3285
+ })).size === 0;
3286
+ return {
3287
+ chunks: page.length,
3288
+ bytes,
3289
+ done
3290
+ };
3291
+ }
3292
+ /**
3293
+ * Delete one page of segment rows, listing segment size rows to avoid
3294
+ * loading payloads. One segment always makes progress, even when it holds
3295
+ * more chunks or bytes than the page allows.
3296
+ */
3297
+ async expireSegmentPage(txn, limit, byteLimit) {
3298
+ const candidates = await txn.list({
3299
+ prefix: SEGMENT_SIZE_KEY_PREFIX,
3300
+ limit: limit + 1
3301
+ });
3302
+ const page = [];
3303
+ let chunks = 0;
3304
+ let bytes = 0;
3305
+ for (const [key, value] of candidates) {
3306
+ const start = startFromSegmentKey(key, SEGMENT_SIZE_KEY_PREFIX);
3307
+ const size = validateSegmentSize(value, start);
3308
+ if (page.length > 0 && (chunks + size.count > limit || bytes + size.bytes > byteLimit)) break;
3309
+ page.push(start);
3310
+ chunks += size.count;
3311
+ bytes += size.bytes;
3312
+ }
3313
+ let orphan = false;
3314
+ if (candidates.size === 0) {
3315
+ const first = (await txn.list({
3316
+ prefix: SEGMENT_KEY_PREFIX,
3317
+ limit: 1
3318
+ })).entries().next().value;
3319
+ if (first) {
3320
+ page.push(startFromSegmentKey(first[0], SEGMENT_KEY_PREFIX));
3321
+ bytes = first[1] instanceof Uint8Array ? first[1].byteLength : 0;
3322
+ chunks = 1;
3323
+ orphan = true;
3324
+ }
3325
+ }
3326
+ await deleteInBatches(txn, page.flatMap((start) => [segmentKey(start), segmentSizeKey(start)]));
3327
+ let done = !orphan && candidates.size === page.length;
3328
+ if (orphan || done) done = (await txn.list({
3329
+ prefix: SEGMENT_KEY_PREFIX,
3330
+ limit: 1
3331
+ })).size === 0;
3332
+ return {
3333
+ chunks,
3334
+ bytes,
3335
+ done
3336
+ };
3337
+ }
2858
3338
  /**
2859
3339
  * Fence the stream and delete one bounded page. Listing size keys avoids
2860
3340
  * loading payloads, and the transaction makes a crash retry resume safely.
@@ -2868,50 +3348,8 @@ var StreamDO = class extends DurableObject {
2868
3348
  const meta = stored === void 0 ? emptyMeta() : validateMeta(stored);
2869
3349
  this.assertOwner(meta, runId);
2870
3350
  const firstFence = meta.state !== "expired";
2871
- const candidates = await txn.list({
2872
- prefix: CHUNK_SIZE_KEY_PREFIX,
2873
- limit: limit + 1
2874
- });
2875
- const page = [];
2876
- let bytes = 0;
2877
- for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
2878
- const index = indexFromChunkSizeKey(key);
2879
- const size = validateChunkSize(storedSize, index);
2880
- if (page.length > 0 && bytes + size > byteLimit) break;
2881
- page.push({
2882
- index,
2883
- size
2884
- });
2885
- bytes += size;
2886
- }
2887
- let payloadFallback = false;
2888
- if (candidates.size === 0) {
2889
- const first = (await txn.list({
2890
- prefix: CHUNK_KEY_PREFIX,
2891
- limit: 1
2892
- })).entries().next().value;
2893
- if (first) {
2894
- const [key, payload] = first;
2895
- page.push({
2896
- index: indexFromChunkKey(key),
2897
- size: payload instanceof Uint8Array ? payload.byteLength : 0
2898
- });
2899
- bytes = page[0].size;
2900
- payloadFallback = true;
2901
- }
2902
- }
2903
- const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
2904
- if (keys.length > 0) await txn.delete(keys);
2905
- let done = candidates.size === page.length;
2906
- if (payloadFallback) done = (await txn.list({
2907
- prefix: CHUNK_KEY_PREFIX,
2908
- limit: 1
2909
- })).size === 0;
2910
- else if (done && candidates.size > 0) done = (await txn.list({
2911
- prefix: CHUNK_KEY_PREFIX,
2912
- limit: 1
2913
- })).size === 0;
2914
- if (done && page.length === 0 && meta.state === "expired" && meta.payloadDeleted) return {
3351
+ const { chunks, bytes, done } = meta.layout === SEGMENT_LAYOUT ? await this.expireSegmentPage(txn, limit, byteLimit) : await this.expireChunkPage(txn, limit, byteLimit);
3352
+ if (done && chunks === 0 && meta.state === "expired" && meta.payloadDeleted) return {
2915
3353
  meta,
2916
3354
  result: {
2917
3355
  deleted: false,
@@ -2935,7 +3373,7 @@ var StreamDO = class extends DurableObject {
2935
3373
  meta: nextMeta,
2936
3374
  result: {
2937
3375
  deleted: firstFence,
2938
- chunks: page.length,
3376
+ chunks,
2939
3377
  bytes,
2940
3378
  done
2941
3379
  }
@@ -3005,19 +3443,29 @@ async function deleteQueuePayloadObjects(binding, keys) {
3005
3443
  }
3006
3444
  //#endregion
3007
3445
  //#region src/worker/durable-objects/WorkflowRunDO.ts
3446
+ const EVENT_SEQUENCE_KEY = "event_sequence";
3008
3447
  const STORAGE_BATCH_SIZE = 128;
3009
3448
  const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
3010
3449
  const HOOK_MARKER_PAGE_SIZE = 64;
3011
3450
  const TERMINAL_HOOK_PAGE_SIZE = 64;
3012
3451
  const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
3013
3452
  const STREAMS_PER_CLEANUP_PAGE = 16;
3014
- const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
3453
+ /** StreamDO deletes a page in 128-key calls within one transaction. */
3454
+ const STREAM_CHUNKS_PER_CLEANUP_PAGE = 256;
3015
3455
  const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
3016
3456
  const QUEUE_PAYLOADS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
3017
3457
  const CLEANUP_RETRY_MAX_MS = 36e5;
3018
3458
  const MAX_DATE_MS = 864e13;
3019
3459
  const QUEUE_ORPHAN_KEY = "queue-orphan";
3020
3460
  const QUEUE_RESERVATION_KEY = "queue-reservation";
3461
+ /** Run-scoped Queue idempotency state, one key per claim name, in the run's own cell. */
3462
+ const RUN_QUEUE_CLAIM_PREFIX = "queue-claim:";
3463
+ /**
3464
+ * Present when every idempotency claim of this run lives in the run's own
3465
+ * cell. Written only when the run is created, so a run keeps one claim scope
3466
+ * for its whole life and claims made before the marker existed stay valid.
3467
+ */
3468
+ const RUN_QUEUE_CLAIM_SCOPE_KEY = "queue-claim-scope";
3021
3469
  function validateCleanupRequest(request, allowDisabled) {
3022
3470
  if (!isRecord$1(request)) throw new Error("world-celld retention request must be an object");
3023
3471
  if (!(allowDisabled ? isNonNegativeSafeInteger(request.retentionMs) : isPositiveSafeInteger(request.retentionMs))) throw new Error(`world-celld retentionMs must be a ${allowDisabled ? "non-negative" : "positive"} safe integer`);
@@ -3071,6 +3519,12 @@ async function putStorageEntries(storage, entries) {
3071
3519
  const all = Array.from(entries);
3072
3520
  for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
3073
3521
  }
3522
+ function runQueueClaimKey(claimName) {
3523
+ return `${RUN_QUEUE_CLAIM_PREFIX}${claimName}`;
3524
+ }
3525
+ function validRunQueueClaimName(claimName) {
3526
+ return typeof claimName === "string" && claimName.startsWith("claim:");
3527
+ }
3074
3528
  /**
3075
3529
  * Durable Object for managing a single workflow run's state.
3076
3530
  *
@@ -3093,11 +3547,14 @@ var WorkflowRunDO = class extends DurableObject {
3093
3547
  if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
3094
3548
  return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
3095
3549
  }
3096
- async retentionState(storage, now = this.now()) {
3097
- const values = await storage.get([TOMBSTONE_KEY, CLEANUP_RECORD_KEY]);
3550
+ async retentionState(storage, now = this.now(), options) {
3551
+ const keys = [TOMBSTONE_KEY, CLEANUP_RECORD_KEY];
3552
+ if (options?.withEventSequence) keys.push(EVENT_SEQUENCE_KEY);
3553
+ const values = await storage.get(keys);
3098
3554
  return {
3099
3555
  cleanup: values.get(CLEANUP_RECORD_KEY),
3100
- tombstone: this.tombstoneFrom(values, now)
3556
+ tombstone: this.tombstoneFrom(values, now),
3557
+ eventSequence: values.get(EVENT_SEQUENCE_KEY) ?? 0
3101
3558
  };
3102
3559
  }
3103
3560
  async read(reader) {
@@ -3132,21 +3589,23 @@ var WorkflowRunDO = class extends DurableObject {
3132
3589
  if (request.cleanup !== void 0) validateCleanupRequest(request.cleanup, true);
3133
3590
  return await this.ctx.storage.transaction(async (txn) => {
3134
3591
  const now = new Date(this.now());
3135
- const retention = await this.retentionState(txn, now.getTime());
3592
+ const retention = await this.retentionState(txn, now.getTime(), { withEventSequence: true });
3136
3593
  const { tombstone } = retention;
3137
3594
  if (tombstone) return {
3138
3595
  ok: false,
3139
3596
  code: "RUN_EXPIRED",
3140
3597
  message: `Workflow run "${request.runId}" expired at ${tombstone.expiredAt.toISOString()}`
3141
3598
  };
3142
- let eventSequence = await txn.get("event_sequence") ?? 0;
3599
+ const storedSequence = retention.eventSequence;
3600
+ let eventSequence = storedSequence;
3143
3601
  const outcome = await applyEvent(storeFrom(txn), {
3144
3602
  ...request,
3145
3603
  nextEventId: () => slotToEventId(++eventSequence),
3146
3604
  now
3147
3605
  });
3148
3606
  if (!outcome.ok) return outcome;
3149
- await txn.put("event_sequence", eventSequence);
3607
+ if (eventSequence !== storedSequence) await txn.put(EVENT_SEQUENCE_KEY, eventSequence);
3608
+ if (outcome.runCreated) await txn.put(RUN_QUEUE_CLAIM_SCOPE_KEY, "run");
3150
3609
  await putStorageEntries(txn, (outcome.hookToIndex ? [{
3151
3610
  hookId: outcome.hookToIndex.hookId,
3152
3611
  token: outcome.hookToIndex.token
@@ -3526,10 +3985,58 @@ var WorkflowRunDO = class extends DurableObject {
3526
3985
  if (cleanup.phase === "payload") await this.deletePayloadBatch();
3527
3986
  }
3528
3987
  async executeTerminalCleanup(cleanup) {
3988
+ if (cleanup.phase === "hooks" && await this.completeSmallTerminalCleanup(cleanup)) return;
3529
3989
  if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
3530
3990
  else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
3531
3991
  else await this.deleteTerminalWaitPage(cleanup);
3532
3992
  }
3993
+ /**
3994
+ * Most runs end with a handful of hooks and waits. When every terminal
3995
+ * phase fits in one page, release the hook indexes once and delete hooks,
3996
+ * markers, and waits in a single transaction instead of one alarm and one
3997
+ * transaction per phase. Returns false, having changed nothing locally, when
3998
+ * any phase needs paging; the paged phases then run as before.
3999
+ */
4000
+ async completeSmallTerminalCleanup(cleanup) {
4001
+ const [hookEntries, markerEntries] = await Promise.all([this.ctx.storage.list({
4002
+ prefix: HOOK_KEY_PREFIX,
4003
+ limit: 65
4004
+ }), this.ctx.storage.list({
4005
+ prefix: HOOK_MARKER_PREFIX,
4006
+ limit: 65
4007
+ })]);
4008
+ if (hookEntries.size > TERMINAL_HOOK_PAGE_SIZE || markerEntries.size > HOOK_MARKER_PAGE_SIZE) return false;
4009
+ const hooks = Array.from(hookEntries.values());
4010
+ const references = /* @__PURE__ */ new Map();
4011
+ for (const reference of [...hooks.map((hook) => ({
4012
+ hookId: hook.hookId,
4013
+ token: hook.token
4014
+ })), ...markerEntries.values()]) references.set(`${reference.hookId}\u0000${reference.token}`, reference);
4015
+ if (references.size > 0) await this.workflowIndex().releaseHookIndexes({
4016
+ runId: cleanup.runId,
4017
+ hooks: Array.from(references.values())
4018
+ });
4019
+ return await this.ctx.storage.transaction(async (txn) => {
4020
+ const current = await txn.get(TERMINAL_CLEANUP_KEY);
4021
+ if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return true;
4022
+ const waits = await txn.list({
4023
+ prefix: WAIT_KEY_PREFIX,
4024
+ limit: 129
4025
+ });
4026
+ if (waits.size > TERMINAL_WAIT_PAGE_SIZE) return false;
4027
+ const keys = [
4028
+ ...hooks.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]),
4029
+ ...markerEntries.keys(),
4030
+ ...waits.keys()
4031
+ ];
4032
+ for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE) await txn.delete(keys.slice(offset, offset + STORAGE_BATCH_SIZE));
4033
+ await txn.delete(TERMINAL_CLEANUP_KEY);
4034
+ const retention = await txn.get(CLEANUP_RECORD_KEY);
4035
+ if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
4036
+ else await txn.deleteAlarm();
4037
+ return true;
4038
+ });
4039
+ }
3533
4040
  async deleteTerminalHookPage(cleanup) {
3534
4041
  const entries = await this.ctx.storage.list({
3535
4042
  prefix: HOOK_KEY_PREFIX,
@@ -3765,6 +4272,150 @@ var WorkflowRunDO = class extends DurableObject {
3765
4272
  await txn.setAlarm(this.now() + delay);
3766
4273
  });
3767
4274
  }
4275
+ /**
4276
+ * Read-only admission for a run-bearing Queue message whose body travels
4277
+ * inline in the broker envelope. It answers what payload registration does
4278
+ * for offloaded bodies, without writing anything.
4279
+ */
4280
+ async getQueueAdmission() {
4281
+ const { tombstone } = await this.retentionState(this.ctx.storage);
4282
+ return tombstone ? {
4283
+ ok: false,
4284
+ message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
4285
+ } : { ok: true };
4286
+ }
4287
+ /**
4288
+ * Enqueue-side admission and idempotency reservation in one transaction,
4289
+ * for a run-scoped claim. Replaces a separate claim cell's reservation plus
4290
+ * the run expiry check.
4291
+ */
4292
+ async reserveRunQueueMessage(params) {
4293
+ if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.expiresAt)) throw new TypeError("world-celld run queue reservation is invalid");
4294
+ return await this.ctx.storage.transaction(async (txn) => {
4295
+ const key = runQueueClaimKey(params.claimName);
4296
+ const values = await txn.get([
4297
+ TOMBSTONE_KEY,
4298
+ CLEANUP_RECORD_KEY,
4299
+ RUN_QUEUE_CLAIM_SCOPE_KEY,
4300
+ key
4301
+ ]);
4302
+ const tombstone = this.tombstoneFrom(values);
4303
+ if (tombstone) return {
4304
+ ok: false,
4305
+ message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
4306
+ };
4307
+ if (values.get("queue-claim-scope") !== "run") return {
4308
+ ok: true,
4309
+ scope: "cell"
4310
+ };
4311
+ const state = values.get(key) ?? {};
4312
+ const existing = state.reservation;
4313
+ if (existing && existing.expiresAt > this.now() && existing.messageId !== params.messageId) return {
4314
+ ok: true,
4315
+ scope: "run",
4316
+ admitted: false,
4317
+ messageId: existing.messageId
4318
+ };
4319
+ await txn.put(key, {
4320
+ ...state,
4321
+ reservation: {
4322
+ messageId: params.messageId,
4323
+ expiresAt: Math.max(existing?.expiresAt ?? 0, params.expiresAt)
4324
+ }
4325
+ });
4326
+ return {
4327
+ ok: true,
4328
+ scope: "run",
4329
+ admitted: true,
4330
+ messageId: params.messageId
4331
+ };
4332
+ });
4333
+ }
4334
+ /** Delivery-side run expiry check and in-flight claim in one transaction. */
4335
+ async claimRunQueueMessage(params) {
4336
+ if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string" || params.messageId.length === 0 || !isPositiveSafeInteger(params.staleMs)) throw new TypeError("world-celld run queue claim is invalid");
4337
+ return await this.ctx.storage.transaction(async (txn) => {
4338
+ const key = runQueueClaimKey(params.claimName);
4339
+ const values = await txn.get([
4340
+ TOMBSTONE_KEY,
4341
+ CLEANUP_RECORD_KEY,
4342
+ key
4343
+ ]);
4344
+ if (this.tombstoneFrom(values)) return { expired: true };
4345
+ const state = values.get(key) ?? {};
4346
+ const existing = state.claim;
4347
+ const now = this.now();
4348
+ if (existing && existing.expiresAt > now) {
4349
+ if (existing.messageId !== params.messageId) return {
4350
+ expired: false,
4351
+ claimed: false
4352
+ };
4353
+ if (existing.retryAt === void 0 || existing.retryAt > now) return {
4354
+ expired: false,
4355
+ claimed: false,
4356
+ retryAt: existing.retryAt ?? existing.expiresAt
4357
+ };
4358
+ }
4359
+ await txn.put(key, {
4360
+ ...state,
4361
+ claim: {
4362
+ messageId: params.messageId,
4363
+ expiresAt: now + params.staleMs
4364
+ }
4365
+ });
4366
+ return {
4367
+ expired: false,
4368
+ claimed: true
4369
+ };
4370
+ });
4371
+ }
4372
+ async holdRunQueueMessage(params) {
4373
+ if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.retryAt) || !isNonNegativeSafeInteger(params.expiresAt) || params.reservationExpiresAt !== void 0 && !isNonNegativeSafeInteger(params.reservationExpiresAt)) throw new TypeError("world-celld run queue claim hold is invalid");
4374
+ return await this.ctx.storage.transaction(async (txn) => {
4375
+ const key = runQueueClaimKey(params.claimName);
4376
+ const state = await txn.get(key) ?? {};
4377
+ const existing = state.claim;
4378
+ if (!existing || existing.messageId !== params.messageId) return { held: false };
4379
+ const next = {
4380
+ ...state,
4381
+ claim: {
4382
+ ...existing,
4383
+ retryAt: params.retryAt,
4384
+ expiresAt: Math.max(existing.expiresAt, params.expiresAt)
4385
+ }
4386
+ };
4387
+ if (state.reservation?.messageId === params.messageId && params.reservationExpiresAt !== void 0 && params.reservationExpiresAt > state.reservation.expiresAt) next.reservation = {
4388
+ ...state.reservation,
4389
+ expiresAt: params.reservationExpiresAt
4390
+ };
4391
+ await txn.put(key, next);
4392
+ return { held: true };
4393
+ });
4394
+ }
4395
+ async releaseRunQueueMessage(params) {
4396
+ if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string") throw new TypeError("world-celld run queue claim release is invalid");
4397
+ await this.ctx.storage.transaction(async (txn) => {
4398
+ const key = runQueueClaimKey(params.claimName);
4399
+ const state = await txn.get(key);
4400
+ if (state?.claim?.messageId !== params.messageId) return;
4401
+ const { claim: _released, ...rest } = state;
4402
+ if (rest.reservation) await txn.put(key, rest);
4403
+ else await txn.delete(key);
4404
+ });
4405
+ }
4406
+ async completeRunQueueMessage(params) {
4407
+ if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string") throw new TypeError("world-celld run queue completion is invalid");
4408
+ await this.ctx.storage.transaction(async (txn) => {
4409
+ const key = runQueueClaimKey(params.claimName);
4410
+ const state = await txn.get(key);
4411
+ if (!state) return;
4412
+ const next = {};
4413
+ if (state.reservation && state.reservation.messageId !== params.messageId) next.reservation = state.reservation;
4414
+ if (state.claim && state.claim.messageId !== params.messageId) next.claim = state.claim;
4415
+ if (next.reservation || next.claim) await txn.put(key, next);
4416
+ else await txn.delete(key);
4417
+ });
4418
+ }
3768
4419
  async registerQueuePayload(registration) {
3769
4420
  if (!registration || typeof registration.messageId !== "string" || registration.messageId.length === 0 || typeof registration.key !== "string" || registration.key.length === 0 || !isNonNegativeSafeInteger(registration.orphanExpiresAt)) throw new TypeError("world-celld queue payload registration is invalid");
3770
4421
  return await this.ctx.storage.transaction(async (txn) => {