@ewhauser/world-celld 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +32 -15
- package/celld-queue-worker/README.md +2 -2
- package/celld-queue-worker/wrangler.jsonc +3 -1
- package/celld-worker/README.md +3 -3
- package/dist/index.d.ts +54 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +113 -100
- package/dist/index.js.map +1 -1
- package/dist/queue-consumer.d.ts +1 -1
- package/dist/queue-consumer.js +1 -1
- package/dist/{queue-delivery-DzNOtCYA.d.ts → queue-delivery-BdeNUbRN.d.ts} +29 -3
- package/dist/queue-delivery-BdeNUbRN.d.ts.map +1 -0
- package/dist/{queue-protocol-Ds44uuND.js → queue-protocol-DKYt8qZJ.js} +26 -3
- package/dist/queue-protocol-DKYt8qZJ.js.map +1 -0
- package/dist/{queue-protocol-CmJNMH_5.js → run-list-C987PkJ2.js} +147 -18
- package/dist/run-list-C987PkJ2.js.map +1 -0
- package/dist/testing.d.ts +4 -2
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +762 -111
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +85 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +888 -130
- package/dist/worker.js.map +1 -1
- package/docs/celld-v0.6-upgrade.md +121 -0
- package/package.json +4 -2
- package/dist/queue-delivery-DzNOtCYA.d.ts.map +0 -1
- package/dist/queue-protocol-CmJNMH_5.js.map +0 -1
- package/dist/queue-protocol-Ds44uuND.js.map +0 -1
- package/docs/celld-v0.5-upgrade.md +0 -130
package/dist/testing.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as isPositiveSafeInteger, E as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, F as MAX_STREAM_CHUNK_BYTES, H as STREAM_CHUNK_LIMIT_HEADER, J as encodeStreamWriteResult, K as encodeStreamReadResult, L as MAX_STREAM_LONG_POLL_MS, P as MAX_STREAM_BATCH_BYTES, Q as validateStreamWriteChunks, R as MAX_STREAM_READ_BYTES, S as LIFECYCLE_COMPACTION_RETRY_MS, V as STREAM_BATCH_CONTENT_TYPE, W as decodeStreamWriteBatch, X as normalizeStreamError, Z as validateStreamReadRequest, a as isInlineRunBody, c as queueOrphanName, d as validateNativeQueueEnvelope, f as validateNativeQueueSendOptions, g as parse, h as rpcStringify, i as QUEUE_PAYLOAD_REGISTRY_PREFIX, j as isRecord$1, k as isNonNegativeSafeInteger, l as queuePayloadObjectKey, m as rpcParse, n as listRunsPage, p as compact, r as NATIVE_QUEUE_MAX_MESSAGE_BYTES, s as queueClaimName, t as MAX_RUN_LIST_LIMIT, u as queuePayloadRegistryKey, v as createWorkflowIndex, x as HOOK_CLAIM_LEASE_MS, y as runIndexMetadata } from "./run-list-C987PkJ2.js";
|
|
2
2
|
import { CreateEventSchema, EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
//#region src/testing/fake-cell.ts
|
|
@@ -180,11 +180,12 @@ var FakeStorage = class {
|
|
|
180
180
|
synchronous: false
|
|
181
181
|
});
|
|
182
182
|
}
|
|
183
|
-
/** Inject
|
|
184
|
-
failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure")) {
|
|
183
|
+
/** Inject matching write failures (one by default), including writes inside transactions. */
|
|
184
|
+
failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure"), times = 1) {
|
|
185
185
|
this.mutationFailure = {
|
|
186
186
|
predicate,
|
|
187
|
-
error
|
|
187
|
+
error,
|
|
188
|
+
remaining: times
|
|
188
189
|
};
|
|
189
190
|
}
|
|
190
191
|
/** Inject one matching storage read failure, including reads inside transactions. */
|
|
@@ -198,7 +199,8 @@ var FakeStorage = class {
|
|
|
198
199
|
maybeFailMutation(mutation) {
|
|
199
200
|
if (!this.mutationFailure?.predicate(mutation)) return;
|
|
200
201
|
const { error } = this.mutationFailure;
|
|
201
|
-
this.mutationFailure
|
|
202
|
+
this.mutationFailure.remaining -= 1;
|
|
203
|
+
if (this.mutationFailure.remaining <= 0) this.mutationFailure = void 0;
|
|
202
204
|
throw error;
|
|
203
205
|
}
|
|
204
206
|
/** @internal Shared with FakeTransaction so read failures model transactional reads too. */
|
|
@@ -371,6 +373,10 @@ var FakeFleet = class {
|
|
|
371
373
|
}
|
|
372
374
|
}, this.cellEnv);
|
|
373
375
|
}
|
|
376
|
+
/** Whether a cell has been activated, without activating it. */
|
|
377
|
+
hasCell(bindingKey, name) {
|
|
378
|
+
return this.cells.has(`${bindingKey}\0${name}`);
|
|
379
|
+
}
|
|
374
380
|
cell(bindingKey, name) {
|
|
375
381
|
const key = `${bindingKey}\0${name}`;
|
|
376
382
|
let slot = this.cells.get(key);
|
|
@@ -1250,6 +1256,7 @@ async function finalizeEventPage(store, outcome, params) {
|
|
|
1250
1256
|
maxEvents: getMaxEventsPerRun()
|
|
1251
1257
|
} : outcome;
|
|
1252
1258
|
if (!result.event) return result;
|
|
1259
|
+
if (params?.preloadEvents && result.event.eventType === "hook_received") return await attachHookReplayPreload(store, result);
|
|
1253
1260
|
if (typeof params?.sinceCursor === "string") {
|
|
1254
1261
|
const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1255
1262
|
limit: 100,
|
|
@@ -1279,6 +1286,32 @@ async function finalizeEventPage(store, outcome, params) {
|
|
|
1279
1286
|
hasMore: events.length < skippedCount
|
|
1280
1287
|
};
|
|
1281
1288
|
}
|
|
1289
|
+
/**
|
|
1290
|
+
* Lazy hook resume preload: return the run and its complete replay log with
|
|
1291
|
+
* the hook_received write, read in the same transaction, so the runtime can
|
|
1292
|
+
* skip its run_started write and the index publication that follows it. The
|
|
1293
|
+
* runtime trusts only a complete log with a non-null cursor, so a run that is
|
|
1294
|
+
* not started, not running, or over the event ceiling gets the plain result.
|
|
1295
|
+
*/
|
|
1296
|
+
async function attachHookReplayPreload(store, result) {
|
|
1297
|
+
const run = await store.get(RUN_KEY);
|
|
1298
|
+
if (!run?.startedAt || run.status !== "running") return result;
|
|
1299
|
+
const maxEvents = getMaxEventsPerRun();
|
|
1300
|
+
const log = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1301
|
+
limit: maxEvents,
|
|
1302
|
+
sortOrder: "asc"
|
|
1303
|
+
}, (event) => event.eventId);
|
|
1304
|
+
const last = log.data.at(-1);
|
|
1305
|
+
if (log.hasMore || last === void 0) return result;
|
|
1306
|
+
return {
|
|
1307
|
+
...result,
|
|
1308
|
+
replayRun: run,
|
|
1309
|
+
maxEvents,
|
|
1310
|
+
events: log.data,
|
|
1311
|
+
cursor: last.eventId,
|
|
1312
|
+
hasMore: false
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1282
1315
|
//#endregion
|
|
1283
1316
|
//#region src/worker/auth.ts
|
|
1284
1317
|
/**
|
|
@@ -1321,6 +1354,46 @@ async function authenticate(request, secret) {
|
|
|
1321
1354
|
return { ok: true };
|
|
1322
1355
|
}
|
|
1323
1356
|
//#endregion
|
|
1357
|
+
//#region src/worker/queue-claims.ts
|
|
1358
|
+
function cellQueueClaim(cell) {
|
|
1359
|
+
return {
|
|
1360
|
+
claim: (messageId, staleMs) => cell.claimInflight({
|
|
1361
|
+
messageId,
|
|
1362
|
+
staleMs
|
|
1363
|
+
}),
|
|
1364
|
+
hold: (params) => cell.holdInflight(params),
|
|
1365
|
+
release: (messageId) => cell.releaseInflight(messageId),
|
|
1366
|
+
complete: (messageId) => cell.completeQueueMessage(messageId)
|
|
1367
|
+
};
|
|
1368
|
+
}
|
|
1369
|
+
function runQueueClaim(run, claimName) {
|
|
1370
|
+
return {
|
|
1371
|
+
claim: async (messageId, staleMs) => {
|
|
1372
|
+
const result = await run.claimRunQueueMessage({
|
|
1373
|
+
claimName,
|
|
1374
|
+
messageId,
|
|
1375
|
+
staleMs
|
|
1376
|
+
});
|
|
1377
|
+
return result.expired ? {
|
|
1378
|
+
expired: true,
|
|
1379
|
+
claimed: false
|
|
1380
|
+
} : result;
|
|
1381
|
+
},
|
|
1382
|
+
hold: (params) => run.holdRunQueueMessage({
|
|
1383
|
+
claimName,
|
|
1384
|
+
...params
|
|
1385
|
+
}),
|
|
1386
|
+
release: (messageId) => run.releaseRunQueueMessage({
|
|
1387
|
+
claimName,
|
|
1388
|
+
messageId
|
|
1389
|
+
}),
|
|
1390
|
+
complete: (messageId) => run.completeRunQueueMessage({
|
|
1391
|
+
claimName,
|
|
1392
|
+
messageId
|
|
1393
|
+
})
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
//#endregion
|
|
1324
1397
|
//#region src/worker/index-validation.ts
|
|
1325
1398
|
function invalid(message) {
|
|
1326
1399
|
const error = /* @__PURE__ */ new Error(`invalid index RPC arguments: ${message}`);
|
|
@@ -1415,6 +1488,34 @@ function listOptions(value) {
|
|
|
1415
1488
|
}
|
|
1416
1489
|
return parsed;
|
|
1417
1490
|
}
|
|
1491
|
+
const RUN_STATUSES = /* @__PURE__ */ new Set([
|
|
1492
|
+
"pending",
|
|
1493
|
+
"running",
|
|
1494
|
+
"completed",
|
|
1495
|
+
"failed",
|
|
1496
|
+
"cancelled"
|
|
1497
|
+
]);
|
|
1498
|
+
function runListRequest(value) {
|
|
1499
|
+
if (!isRecord$1(value)) invalid("run list request must be an object");
|
|
1500
|
+
const limit = value.limit;
|
|
1501
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1e3) invalid(`request.limit must be an integer between 1 and ${MAX_RUN_LIST_LIMIT}`);
|
|
1502
|
+
const parsed = { limit };
|
|
1503
|
+
if (value.workflowName !== void 0) parsed.workflowName = nonEmptyString(value.workflowName, "request.workflowName");
|
|
1504
|
+
if (value.cursor !== void 0) parsed.cursor = stringValue(value.cursor, "request.cursor");
|
|
1505
|
+
if (value.statuses !== void 0) {
|
|
1506
|
+
if (!Array.isArray(value.statuses) || value.statuses.length === 0 || !value.statuses.every((status) => RUN_STATUSES.has(status))) invalid("request.statuses must be a non-empty array of run statuses");
|
|
1507
|
+
parsed.statuses = value.statuses;
|
|
1508
|
+
}
|
|
1509
|
+
if (value.sortOrder !== void 0) {
|
|
1510
|
+
if (value.sortOrder !== "asc" && value.sortOrder !== "desc") invalid("request.sortOrder must be 'asc' or 'desc'");
|
|
1511
|
+
parsed.sortOrder = value.sortOrder;
|
|
1512
|
+
}
|
|
1513
|
+
if (value.resolveData !== void 0) {
|
|
1514
|
+
if (value.resolveData !== "none" && value.resolveData !== "all") invalid("request.resolveData must be 'none' or 'all'");
|
|
1515
|
+
parsed.resolveData = value.resolveData;
|
|
1516
|
+
}
|
|
1517
|
+
return parsed;
|
|
1518
|
+
}
|
|
1418
1519
|
function runCommit(args) {
|
|
1419
1520
|
exactArgs(args, 3);
|
|
1420
1521
|
const parsedRun = WorkflowRunSchema.safeParse(args[0]);
|
|
@@ -1475,6 +1576,12 @@ function validateIndexRequest(operation, args) {
|
|
|
1475
1576
|
operation,
|
|
1476
1577
|
args: [listOptions(args[0])]
|
|
1477
1578
|
};
|
|
1579
|
+
case "runs.list-resolved":
|
|
1580
|
+
exactArgs(args, 1);
|
|
1581
|
+
return {
|
|
1582
|
+
operation,
|
|
1583
|
+
args: [runListRequest(args[0])]
|
|
1584
|
+
};
|
|
1478
1585
|
case "runs.commit": return runCommit(args);
|
|
1479
1586
|
case "runs.expire":
|
|
1480
1587
|
exactArgs(args, 1);
|
|
@@ -1510,6 +1617,7 @@ function validateIndexRequest(operation, args) {
|
|
|
1510
1617
|
}
|
|
1511
1618
|
const INDEX_OPERATIONS = /* @__PURE__ */ new Set([
|
|
1512
1619
|
"runs.list",
|
|
1620
|
+
"runs.list-resolved",
|
|
1513
1621
|
"runs.commit",
|
|
1514
1622
|
"runs.expire",
|
|
1515
1623
|
"hooks.reserve",
|
|
@@ -1534,6 +1642,18 @@ const INDEX_OPERATIONS = /* @__PURE__ */ new Set([
|
|
|
1534
1642
|
*/
|
|
1535
1643
|
const WORLD_NAME = "world-celld";
|
|
1536
1644
|
const WORLD_VERSION = "0.1.0";
|
|
1645
|
+
/**
|
|
1646
|
+
* Per-isolate record of runs whose Queue idempotency claims stay in claim
|
|
1647
|
+
* cells (created before run-scoped claims) or live in the run cell. A run's
|
|
1648
|
+
* scope never changes, so the cache only saves the run-cell question.
|
|
1649
|
+
*/
|
|
1650
|
+
const runClaimScopes = /* @__PURE__ */ new Map();
|
|
1651
|
+
const MAX_REMEMBERED_RUN_CLAIM_SCOPES = 4096;
|
|
1652
|
+
function rememberRunClaimScope(runId, scope) {
|
|
1653
|
+
if (runClaimScopes.has(runId)) return;
|
|
1654
|
+
if (runClaimScopes.size >= MAX_REMEMBERED_RUN_CLAIM_SCOPES) runClaimScopes.delete(runClaimScopes.keys().next().value);
|
|
1655
|
+
runClaimScopes.set(runId, scope);
|
|
1656
|
+
}
|
|
1537
1657
|
function workflowIndex(env) {
|
|
1538
1658
|
return createWorkflowIndex({
|
|
1539
1659
|
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
@@ -1541,6 +1661,37 @@ function workflowIndex(env) {
|
|
|
1541
1661
|
hookIds: env.WORKFLOW_HOOK_IDS
|
|
1542
1662
|
});
|
|
1543
1663
|
}
|
|
1664
|
+
/**
|
|
1665
|
+
* Publish a committed applyEvent outcome to the derivative indexes inside the
|
|
1666
|
+
* fleet, sparing the client a round trip per run mutation. Each index write is
|
|
1667
|
+
* idempotent under the outcome's publication lease. A failed write leaves its
|
|
1668
|
+
* flag unset, so the client publishes it itself exactly as it would without
|
|
1669
|
+
* this step; older clients ignore the flags and publish again.
|
|
1670
|
+
*/
|
|
1671
|
+
async function publishApplyEventIndexes(env, runId, outcome) {
|
|
1672
|
+
if (typeof outcome !== "object" || outcome === null) return;
|
|
1673
|
+
const success = outcome;
|
|
1674
|
+
if (success.ok !== true) return;
|
|
1675
|
+
const index = workflowIndex(env);
|
|
1676
|
+
const publications = [];
|
|
1677
|
+
if (success.run && typeof success.indexPublicationExpiresAt === "number") {
|
|
1678
|
+
const run = success.run;
|
|
1679
|
+
const lease = success.indexPublicationExpiresAt;
|
|
1680
|
+
publications.push((async () => {
|
|
1681
|
+
await index.commitRun(run, runIndexMetadata(run), lease);
|
|
1682
|
+
success.runIndexPublished = true;
|
|
1683
|
+
})());
|
|
1684
|
+
}
|
|
1685
|
+
const releasedHooks = success.releasedHooks;
|
|
1686
|
+
if (Array.isArray(releasedHooks) && releasedHooks.length > 0) publications.push((async () => {
|
|
1687
|
+
await index.releaseHookIndexes({
|
|
1688
|
+
runId,
|
|
1689
|
+
hooks: releasedHooks
|
|
1690
|
+
});
|
|
1691
|
+
success.hookIndexesReleased = true;
|
|
1692
|
+
})());
|
|
1693
|
+
await Promise.allSettled(publications);
|
|
1694
|
+
}
|
|
1544
1695
|
const BINDINGS = {
|
|
1545
1696
|
runs: {
|
|
1546
1697
|
env: "WORKFLOW_DB",
|
|
@@ -1583,7 +1734,7 @@ const BINDINGS = {
|
|
|
1583
1734
|
};
|
|
1584
1735
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
1585
1736
|
const MAX_BODY_BYTES = 33554432;
|
|
1586
|
-
const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 1024;
|
|
1737
|
+
const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 2048 + 1024;
|
|
1587
1738
|
const QUEUE_ORPHAN_GRACE_MS = 432e6;
|
|
1588
1739
|
function errorResponse(status, name, message) {
|
|
1589
1740
|
return Response.json({ error: {
|
|
@@ -1700,7 +1851,10 @@ function parseBoundedInteger(url, name, minimum, maximum) {
|
|
|
1700
1851
|
function streamResponse(body) {
|
|
1701
1852
|
return new Response(body, {
|
|
1702
1853
|
status: 200,
|
|
1703
|
-
headers: {
|
|
1854
|
+
headers: {
|
|
1855
|
+
"content-type": STREAM_BATCH_CONTENT_TYPE,
|
|
1856
|
+
[STREAM_CHUNK_LIMIT_HEADER]: String(512)
|
|
1857
|
+
}
|
|
1704
1858
|
});
|
|
1705
1859
|
}
|
|
1706
1860
|
function createRouter(env) {
|
|
@@ -1737,31 +1891,71 @@ function createRouter(env) {
|
|
|
1737
1891
|
try {
|
|
1738
1892
|
const namespace = env.WORKFLOW_DB;
|
|
1739
1893
|
const payloadStore = env.WORKFLOW_QUEUE_PAYLOADS;
|
|
1740
|
-
const
|
|
1741
|
-
const
|
|
1894
|
+
const inlineRunBody = envelope.runId !== void 0 && isInlineRunBody(envelope.body);
|
|
1895
|
+
const payloadKey = envelope.runId && !inlineRunBody ? queuePayloadObjectKey(envelope.runId, envelope.messageId) : void 0;
|
|
1896
|
+
let brokerEnvelope = payloadKey ? {
|
|
1742
1897
|
...envelope,
|
|
1743
1898
|
payloadKey,
|
|
1744
1899
|
body: void 0
|
|
1745
1900
|
} : envelope;
|
|
1746
|
-
const
|
|
1747
|
-
|
|
1901
|
+
const sized = JSON.stringify({
|
|
1902
|
+
...brokerEnvelope,
|
|
1903
|
+
claimScope: "run"
|
|
1904
|
+
});
|
|
1905
|
+
if (new TextEncoder().encode(sized).byteLength > 128e3) return errorResponse(413, "PayloadTooLarge", `native queue envelope exceeds ${NATIVE_QUEUE_MAX_MESSAGE_BYTES} bytes`);
|
|
1748
1906
|
const now = Date.now();
|
|
1749
1907
|
const reservationExpiresAt = Math.max(now, envelope.notBefore ?? now) + QUEUE_ORPHAN_GRACE_MS;
|
|
1908
|
+
const run = envelope.runId ? namespace.get(namespace.idFromName(envelope.runId)) : void 0;
|
|
1750
1909
|
let claim;
|
|
1910
|
+
let admission;
|
|
1751
1911
|
if (envelope.idempotencyKey) {
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1912
|
+
const claimName = queueClaimName(envelope.queueName, envelope.idempotencyKey);
|
|
1913
|
+
let scope = run && envelope.runId ? runClaimScopes.get(envelope.runId) : "cell";
|
|
1914
|
+
let reservation;
|
|
1915
|
+
if (run && envelope.runId && scope !== "cell") {
|
|
1916
|
+
const reserved = await run.reserveRunQueueMessage({
|
|
1917
|
+
claimName,
|
|
1918
|
+
messageId: envelope.messageId,
|
|
1919
|
+
expiresAt: reservationExpiresAt
|
|
1920
|
+
});
|
|
1921
|
+
if (!reserved.ok) return errorResponse(410, "RunExpiredError", reserved.message);
|
|
1922
|
+
scope = reserved.scope;
|
|
1923
|
+
rememberRunClaimScope(envelope.runId, reserved.scope);
|
|
1924
|
+
if (reserved.scope === "run") {
|
|
1925
|
+
reservation = reserved;
|
|
1926
|
+
claim = runQueueClaim(run, claimName);
|
|
1927
|
+
brokerEnvelope = {
|
|
1928
|
+
...brokerEnvelope,
|
|
1929
|
+
claimScope: "run"
|
|
1930
|
+
};
|
|
1931
|
+
}
|
|
1932
|
+
} else if (inlineRunBody && run) {
|
|
1933
|
+
admission = run.getQueueAdmission();
|
|
1934
|
+
admission.catch(() => void 0);
|
|
1935
|
+
}
|
|
1936
|
+
if (!reservation) {
|
|
1937
|
+
const cell = namespace.get(namespace.idFromName(claimName));
|
|
1938
|
+
reservation = await cell.reserveQueueMessage({
|
|
1939
|
+
messageId: envelope.messageId,
|
|
1940
|
+
expiresAt: reservationExpiresAt
|
|
1941
|
+
});
|
|
1942
|
+
claim = cellQueueClaim(cell);
|
|
1943
|
+
}
|
|
1757
1944
|
if (!reservation.admitted) return new Response(rpcStringify({ messageId: reservation.messageId }), {
|
|
1758
1945
|
status: 200,
|
|
1759
1946
|
headers: { "content-type": "application/json" }
|
|
1760
1947
|
});
|
|
1948
|
+
} else if (inlineRunBody && run) admission = run.getQueueAdmission();
|
|
1949
|
+
const encoded = JSON.stringify(brokerEnvelope);
|
|
1950
|
+
if (admission) {
|
|
1951
|
+
const admitted = await admission;
|
|
1952
|
+
if (!admitted.ok) {
|
|
1953
|
+
await claim?.complete(envelope.messageId);
|
|
1954
|
+
return errorResponse(410, "RunExpiredError", admitted.message);
|
|
1955
|
+
}
|
|
1761
1956
|
}
|
|
1762
|
-
if (envelope.runId && payloadKey) {
|
|
1957
|
+
if (run && envelope.runId && payloadKey) {
|
|
1763
1958
|
if (!payloadStore) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_QUEUE_PAYLOADS");
|
|
1764
|
-
const run = namespace.get(namespace.idFromName(envelope.runId));
|
|
1765
1959
|
const orphanExpiresAt = reservationExpiresAt;
|
|
1766
1960
|
const registered = await run.registerQueuePayload({
|
|
1767
1961
|
messageId: envelope.messageId,
|
|
@@ -1769,7 +1963,7 @@ function createRouter(env) {
|
|
|
1769
1963
|
orphanExpiresAt
|
|
1770
1964
|
});
|
|
1771
1965
|
if (!registered.ok) {
|
|
1772
|
-
await claim?.
|
|
1966
|
+
await claim?.complete(envelope.messageId);
|
|
1773
1967
|
return errorResponse(410, "RunExpiredError", registered.message);
|
|
1774
1968
|
}
|
|
1775
1969
|
const orphan = namespace.get(namespace.idFromName(queueOrphanName(envelope.messageId)));
|
|
@@ -1782,7 +1976,7 @@ function createRouter(env) {
|
|
|
1782
1976
|
});
|
|
1783
1977
|
} catch (error) {
|
|
1784
1978
|
await run.unregisterQueuePayload(envelope.messageId);
|
|
1785
|
-
await claim?.
|
|
1979
|
+
await claim?.complete(envelope.messageId);
|
|
1786
1980
|
throw error;
|
|
1787
1981
|
}
|
|
1788
1982
|
try {
|
|
@@ -1791,21 +1985,21 @@ function createRouter(env) {
|
|
|
1791
1985
|
messageId: envelope.messageId
|
|
1792
1986
|
});
|
|
1793
1987
|
} catch (error) {
|
|
1794
|
-
await claim?.
|
|
1988
|
+
await claim?.complete(envelope.messageId);
|
|
1795
1989
|
throw error;
|
|
1796
1990
|
}
|
|
1797
1991
|
let finalized;
|
|
1798
1992
|
try {
|
|
1799
1993
|
finalized = await run.finalizeQueuePayload(envelope.messageId);
|
|
1800
1994
|
} catch (error) {
|
|
1801
|
-
await claim?.
|
|
1995
|
+
await claim?.complete(envelope.messageId);
|
|
1802
1996
|
throw error;
|
|
1803
1997
|
}
|
|
1804
1998
|
if (!finalized.ok) {
|
|
1805
1999
|
await payloadStore.delete(payloadKey);
|
|
1806
2000
|
await run.unregisterQueuePayload(envelope.messageId);
|
|
1807
2001
|
await orphan.cancelQueuePayloadOrphan(envelope.messageId);
|
|
1808
|
-
await claim?.
|
|
2002
|
+
await claim?.complete(envelope.messageId);
|
|
1809
2003
|
return errorResponse(410, "RunExpiredError", finalized.message);
|
|
1810
2004
|
}
|
|
1811
2005
|
}
|
|
@@ -1864,6 +2058,18 @@ function createRouter(env) {
|
|
|
1864
2058
|
case "runs.list":
|
|
1865
2059
|
result = await index.listRuns(...validated.args);
|
|
1866
2060
|
break;
|
|
2061
|
+
case "runs.list-resolved": {
|
|
2062
|
+
const runs = env.WORKFLOW_DB;
|
|
2063
|
+
if (!runs) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_DB");
|
|
2064
|
+
result = await listRunsPage({
|
|
2065
|
+
listRuns: (options) => index.listRuns(options),
|
|
2066
|
+
readRun: async (runId) => {
|
|
2067
|
+
const outcome = await runs.get(runs.idFromName(runId)).getRun();
|
|
2068
|
+
return outcome.ok ? outcome.value : null;
|
|
2069
|
+
}
|
|
2070
|
+
}, ...validated.args);
|
|
2071
|
+
break;
|
|
2072
|
+
}
|
|
1867
2073
|
case "runs.commit":
|
|
1868
2074
|
result = await index.commitRun(...validated.args);
|
|
1869
2075
|
break;
|
|
@@ -1920,7 +2126,7 @@ function createRouter(env) {
|
|
|
1920
2126
|
const readRequest = {
|
|
1921
2127
|
runId,
|
|
1922
2128
|
startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
|
|
1923
|
-
maxChunks: parseBoundedInteger(url, "maxChunks", 0,
|
|
2129
|
+
maxChunks: parseBoundedInteger(url, "maxChunks", 0, 512),
|
|
1924
2130
|
maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
|
|
1925
2131
|
waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
|
|
1926
2132
|
};
|
|
@@ -1984,6 +2190,7 @@ function createRouter(env) {
|
|
|
1984
2190
|
}
|
|
1985
2191
|
try {
|
|
1986
2192
|
const result = await namespace.get(namespace.idFromName(name))[method](...args);
|
|
2193
|
+
if (bindingKey === "runs" && method === "applyEvent") await publishApplyEventIndexes(env, name, result);
|
|
1987
2194
|
return new Response(rpcStringify(result ?? null), {
|
|
1988
2195
|
status: 200,
|
|
1989
2196
|
headers: { "content-type": "application/json" }
|
|
@@ -2413,9 +2620,18 @@ var RunCatalogDO = class extends DurableObject {
|
|
|
2413
2620
|
async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
|
|
2414
2621
|
const canonicalKeys = canonicalCatalogKeys(runId, keys);
|
|
2415
2622
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2416
|
-
|
|
2623
|
+
const recordKey = catalogKeysKey(runId);
|
|
2624
|
+
const existing = await txn.get([
|
|
2625
|
+
expiredKey(runId),
|
|
2626
|
+
canonicalKeys[0],
|
|
2627
|
+
canonicalKeys[1],
|
|
2628
|
+
recordKey
|
|
2629
|
+
]);
|
|
2630
|
+
if (existing.get(expiredKey(runId)) !== void 0) return { stored: false };
|
|
2417
2631
|
const now = this.now();
|
|
2418
2632
|
if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
|
|
2633
|
+
const record = existing.get(recordKey);
|
|
2634
|
+
if (existing.get(canonicalKeys[0]) === serializedMetadata && existing.get(canonicalKeys[1]) === serializedMetadata && record?.keys.length === canonicalKeys.length && record.keys.every((key, index) => key === canonicalKeys[index])) return { stored: true };
|
|
2419
2635
|
await txn.put({
|
|
2420
2636
|
[canonicalKeys[0]]: serializedMetadata,
|
|
2421
2637
|
[canonicalKeys[1]]: serializedMetadata,
|
|
@@ -2527,13 +2743,23 @@ const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
|
|
|
2527
2743
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
2528
2744
|
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
2529
2745
|
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
2530
|
-
/** Each chunk occupies a payload key and a size key
|
|
2746
|
+
/** Each chunk occupies a payload key and a size key, deleted in calls of at most 128 keys. */
|
|
2531
2747
|
const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
|
|
2532
|
-
const MAX_EXPIRE_CHUNK_LIMIT =
|
|
2748
|
+
const MAX_EXPIRE_CHUNK_LIMIT = 256;
|
|
2749
|
+
const DELETE_BATCH_KEYS = 128;
|
|
2533
2750
|
/** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
|
|
2534
2751
|
const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
|
|
2535
2752
|
const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
|
|
2536
2753
|
const MAX_STREAM_INDEX = 2147483647;
|
|
2754
|
+
const SEGMENT_LAYOUT = 2;
|
|
2755
|
+
const SEGMENT_KEY_PREFIX = "seg:";
|
|
2756
|
+
const SEGMENT_SIZE_KEY_PREFIX = "segsize:";
|
|
2757
|
+
/** Payload bytes per segment row, well below the 2 MiB per-value storage limit. */
|
|
2758
|
+
const MAX_SEGMENT_BYTES = 1048576;
|
|
2759
|
+
/** Layout of a stream's first write. Existing streams keep the layout they started with. */
|
|
2760
|
+
const NEW_STREAM_LAYOUT = SEGMENT_LAYOUT;
|
|
2761
|
+
/** One storage get or put call accepts at most 128 keys. */
|
|
2762
|
+
const STORAGE_BATCH_KEYS = 128;
|
|
2537
2763
|
function boundedLimit(value, fallback, maximum) {
|
|
2538
2764
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
2539
2765
|
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
@@ -2552,7 +2778,7 @@ function validateMeta(meta) {
|
|
|
2552
2778
|
"closed",
|
|
2553
2779
|
"errored",
|
|
2554
2780
|
"expired"
|
|
2555
|
-
].includes(candidate.state) || candidate.ownerRunId !== void 0 && typeof candidate.ownerRunId !== "string" || candidate.payloadDeleted !== void 0 && typeof candidate.payloadDeleted !== "boolean") throw new Error("Invalid persisted stream metadata");
|
|
2781
|
+
].includes(candidate.state) || candidate.ownerRunId !== void 0 && typeof candidate.ownerRunId !== "string" || candidate.payloadDeleted !== void 0 && typeof candidate.payloadDeleted !== "boolean" || candidate.layout !== void 0 && candidate.layout !== SEGMENT_LAYOUT) throw new Error("Invalid persisted stream metadata");
|
|
2556
2782
|
const error = candidate.error;
|
|
2557
2783
|
if (error !== void 0 && (!error || typeof error !== "object" || typeof error.name !== "string" || typeof error.message !== "string") || candidate.state === "errored" !== (error !== void 0)) throw new Error("Invalid persisted stream error metadata");
|
|
2558
2784
|
return candidate;
|
|
@@ -2564,6 +2790,80 @@ function chunkKey(index) {
|
|
|
2564
2790
|
function chunkSizeKey(index) {
|
|
2565
2791
|
return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
2566
2792
|
}
|
|
2793
|
+
function segmentKey(start) {
|
|
2794
|
+
return `${SEGMENT_KEY_PREFIX}${start.toString().padStart(12, "0")}`;
|
|
2795
|
+
}
|
|
2796
|
+
function segmentSizeKey(start) {
|
|
2797
|
+
return `${SEGMENT_SIZE_KEY_PREFIX}${start.toString().padStart(12, "0")}`;
|
|
2798
|
+
}
|
|
2799
|
+
function startFromSegmentKey(key, prefix) {
|
|
2800
|
+
const suffix = key.slice(prefix.length);
|
|
2801
|
+
const start = Number(suffix);
|
|
2802
|
+
if (!key.startsWith(prefix) || !/^\d{12}$/.test(suffix) || !Number.isSafeInteger(start) || start > MAX_STREAM_INDEX) throw new Error(`Invalid persisted stream segment key "${key}"`);
|
|
2803
|
+
return start;
|
|
2804
|
+
}
|
|
2805
|
+
function validateSegmentSize(value, start) {
|
|
2806
|
+
const size = value;
|
|
2807
|
+
if (!size || !Number.isSafeInteger(size.count) || size.count < 1 || !Number.isSafeInteger(size.bytes) || size.bytes < 0) throw new Error(`Invalid persisted stream segment size at index ${start}`);
|
|
2808
|
+
return size;
|
|
2809
|
+
}
|
|
2810
|
+
/** Pack chunks as `[u32 count][u32 length]*count` followed by the payloads. */
|
|
2811
|
+
function encodeSegment(chunks) {
|
|
2812
|
+
const payloadBytes = chunks.reduce((total, chunk) => total + chunk.byteLength, 0);
|
|
2813
|
+
const header = 4 + 4 * chunks.length;
|
|
2814
|
+
const encoded = new Uint8Array(header + payloadBytes);
|
|
2815
|
+
const view = new DataView(encoded.buffer);
|
|
2816
|
+
view.setUint32(0, chunks.length);
|
|
2817
|
+
let offset = header;
|
|
2818
|
+
for (let index = 0; index < chunks.length; index++) {
|
|
2819
|
+
view.setUint32(4 + 4 * index, chunks[index].byteLength);
|
|
2820
|
+
encoded.set(chunks[index], offset);
|
|
2821
|
+
offset += chunks[index].byteLength;
|
|
2822
|
+
}
|
|
2823
|
+
return encoded;
|
|
2824
|
+
}
|
|
2825
|
+
function decodeSegment(value, start, size) {
|
|
2826
|
+
if (!(value instanceof Uint8Array) || value.byteLength < 4) throw new Error(`Invalid persisted stream segment at index ${start}`);
|
|
2827
|
+
const view = new DataView(value.buffer, value.byteOffset, value.byteLength);
|
|
2828
|
+
const count = view.getUint32(0);
|
|
2829
|
+
const header = 4 + 4 * count;
|
|
2830
|
+
if (count !== size.count || header + size.bytes !== value.byteLength) throw new Error(`Invalid persisted stream segment at index ${start}`);
|
|
2831
|
+
const chunks = [];
|
|
2832
|
+
let offset = header;
|
|
2833
|
+
for (let index = 0; index < count; index++) {
|
|
2834
|
+
const length = view.getUint32(4 + 4 * index);
|
|
2835
|
+
if (offset + length > value.byteLength) throw new Error(`Invalid persisted stream segment at index ${start}`);
|
|
2836
|
+
chunks.push(value.subarray(offset, offset + length));
|
|
2837
|
+
offset += length;
|
|
2838
|
+
}
|
|
2839
|
+
if (offset !== value.byteLength) throw new Error(`Invalid persisted stream segment at index ${start}`);
|
|
2840
|
+
return chunks;
|
|
2841
|
+
}
|
|
2842
|
+
/** Group one write's chunks into segments of at most MAX_SEGMENT_BYTES payload bytes. */
|
|
2843
|
+
function packSegments(startIndex, chunks) {
|
|
2844
|
+
const segments = [];
|
|
2845
|
+
let current = [];
|
|
2846
|
+
let bytes = 0;
|
|
2847
|
+
let start = startIndex;
|
|
2848
|
+
for (const chunk of chunks) {
|
|
2849
|
+
if (current.length > 0 && bytes + chunk.byteLength > MAX_SEGMENT_BYTES) {
|
|
2850
|
+
segments.push({
|
|
2851
|
+
start,
|
|
2852
|
+
chunks: current
|
|
2853
|
+
});
|
|
2854
|
+
start += current.length;
|
|
2855
|
+
current = [];
|
|
2856
|
+
bytes = 0;
|
|
2857
|
+
}
|
|
2858
|
+
current.push(chunk);
|
|
2859
|
+
bytes += chunk.byteLength;
|
|
2860
|
+
}
|
|
2861
|
+
if (current.length > 0) segments.push({
|
|
2862
|
+
start,
|
|
2863
|
+
chunks: current
|
|
2864
|
+
});
|
|
2865
|
+
return segments;
|
|
2866
|
+
}
|
|
2567
2867
|
function indexFromChunkSizeKey(key) {
|
|
2568
2868
|
const suffix = key.slice(11);
|
|
2569
2869
|
const index = Number(suffix);
|
|
@@ -2580,6 +2880,9 @@ function validateChunkSize(size, index) {
|
|
|
2580
2880
|
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
|
|
2581
2881
|
return size;
|
|
2582
2882
|
}
|
|
2883
|
+
async function deleteInBatches(txn, keys) {
|
|
2884
|
+
for (let offset = 0; offset < keys.length; offset += DELETE_BATCH_KEYS) await txn.delete(keys.slice(offset, offset + DELETE_BATCH_KEYS));
|
|
2885
|
+
}
|
|
2583
2886
|
function compactChunk(chunk) {
|
|
2584
2887
|
return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
|
|
2585
2888
|
}
|
|
@@ -2599,6 +2902,15 @@ var StreamDO = class extends DurableObject {
|
|
|
2599
2902
|
mutationTail = Promise.resolve();
|
|
2600
2903
|
changeVersion = 0;
|
|
2601
2904
|
waiters = /* @__PURE__ */ new Set();
|
|
2905
|
+
/** Multi-key read split into calls within the per-call key limit. */
|
|
2906
|
+
async getMany(keys) {
|
|
2907
|
+
if (keys.length <= STORAGE_BATCH_KEYS) return await this.ctx.storage.get(keys);
|
|
2908
|
+
const groups = [];
|
|
2909
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_KEYS) groups.push(keys.slice(offset, offset + STORAGE_BATCH_KEYS));
|
|
2910
|
+
const merged = /* @__PURE__ */ new Map();
|
|
2911
|
+
for (const result of await Promise.all(groups.map((group) => this.ctx.storage.get(group)))) for (const [key, value] of result) merged.set(key, value);
|
|
2912
|
+
return merged;
|
|
2913
|
+
}
|
|
2602
2914
|
async getMeta() {
|
|
2603
2915
|
if (this.meta) return this.meta;
|
|
2604
2916
|
this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
|
|
@@ -2667,29 +2979,8 @@ var StreamDO = class extends DurableObject {
|
|
|
2667
2979
|
const meta = await this.getMeta();
|
|
2668
2980
|
this.assertOwner(meta, request.runId);
|
|
2669
2981
|
const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
|
|
2670
|
-
|
|
2671
|
-
if (available > 0)
|
|
2672
|
-
const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
|
|
2673
|
-
const sizeKeys = indexes.map(chunkSizeKey);
|
|
2674
|
-
const storedSizes = await this.ctx.storage.get(sizeKeys);
|
|
2675
|
-
const selectedIndexes = [];
|
|
2676
|
-
const selectedSizes = [];
|
|
2677
|
-
let bytes = 0;
|
|
2678
|
-
for (const index of indexes) {
|
|
2679
|
-
const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
|
|
2680
|
-
if (bytes + size > request.maxBytes) break;
|
|
2681
|
-
selectedIndexes.push(index);
|
|
2682
|
-
selectedSizes.push(size);
|
|
2683
|
-
bytes += size;
|
|
2684
|
-
}
|
|
2685
|
-
const payloadKeys = selectedIndexes.map(chunkKey);
|
|
2686
|
-
const storedChunks = await this.ctx.storage.get(payloadKeys);
|
|
2687
|
-
for (let offset = 0; offset < payloadKeys.length; offset++) {
|
|
2688
|
-
const chunk = storedChunks.get(payloadKeys[offset]);
|
|
2689
|
-
if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
|
|
2690
|
-
chunks.push(chunk);
|
|
2691
|
-
}
|
|
2692
|
-
}
|
|
2982
|
+
let chunks = [];
|
|
2983
|
+
if (available > 0) chunks = meta.layout === SEGMENT_LAYOUT ? await this.readSegmentRows(request.startIndex, available, request.maxBytes) : await this.readChunkRows(request.startIndex, available, request.maxBytes);
|
|
2693
2984
|
return {
|
|
2694
2985
|
startIndex: request.startIndex,
|
|
2695
2986
|
tailIndex: meta.count - 1,
|
|
@@ -2699,10 +2990,91 @@ var StreamDO = class extends DurableObject {
|
|
|
2699
2990
|
error: meta.error
|
|
2700
2991
|
};
|
|
2701
2992
|
}
|
|
2993
|
+
async readChunkRows(startIndex, available, maxBytes) {
|
|
2994
|
+
const indexes = Array.from({ length: available }, (_, offset) => startIndex + offset);
|
|
2995
|
+
const sizeKeys = indexes.map(chunkSizeKey);
|
|
2996
|
+
const storedSizes = await this.getMany(sizeKeys);
|
|
2997
|
+
const selectedIndexes = [];
|
|
2998
|
+
const selectedSizes = [];
|
|
2999
|
+
let bytes = 0;
|
|
3000
|
+
for (const index of indexes) {
|
|
3001
|
+
const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
|
|
3002
|
+
if (bytes + size > maxBytes) break;
|
|
3003
|
+
selectedIndexes.push(index);
|
|
3004
|
+
selectedSizes.push(size);
|
|
3005
|
+
bytes += size;
|
|
3006
|
+
}
|
|
3007
|
+
const payloadKeys = selectedIndexes.map(chunkKey);
|
|
3008
|
+
const storedChunks = await this.getMany(payloadKeys);
|
|
3009
|
+
const chunks = [];
|
|
3010
|
+
for (let offset = 0; offset < payloadKeys.length; offset++) {
|
|
3011
|
+
const chunk = storedChunks.get(payloadKeys[offset]);
|
|
3012
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
|
|
3013
|
+
chunks.push(chunk);
|
|
3014
|
+
}
|
|
3015
|
+
return chunks;
|
|
3016
|
+
}
|
|
3017
|
+
/**
|
|
3018
|
+
* Plan from the small segment size rows, then load only the segments that
|
|
3019
|
+
* cover the requested range. A read that starts on a segment boundary (the
|
|
3020
|
+
* common sequential case) needs one list and one multi-get.
|
|
3021
|
+
*/
|
|
3022
|
+
async readSegmentRows(startIndex, available, maxBytes) {
|
|
3023
|
+
const endIndex = startIndex + available;
|
|
3024
|
+
const listed = await this.ctx.storage.list({
|
|
3025
|
+
prefix: SEGMENT_SIZE_KEY_PREFIX,
|
|
3026
|
+
start: segmentSizeKey(startIndex),
|
|
3027
|
+
end: segmentSizeKey(endIndex),
|
|
3028
|
+
limit: available
|
|
3029
|
+
});
|
|
3030
|
+
const planned = [];
|
|
3031
|
+
for (const [key, value] of listed) {
|
|
3032
|
+
const start = startFromSegmentKey(key, SEGMENT_SIZE_KEY_PREFIX);
|
|
3033
|
+
planned.push({
|
|
3034
|
+
start,
|
|
3035
|
+
size: validateSegmentSize(value, start)
|
|
3036
|
+
});
|
|
3037
|
+
}
|
|
3038
|
+
if (planned[0]?.start !== startIndex) {
|
|
3039
|
+
const entry = (await this.ctx.storage.list({
|
|
3040
|
+
prefix: SEGMENT_SIZE_KEY_PREFIX,
|
|
3041
|
+
end: segmentSizeKey(startIndex + 1),
|
|
3042
|
+
reverse: true,
|
|
3043
|
+
limit: 1
|
|
3044
|
+
})).entries().next().value;
|
|
3045
|
+
if (!entry) throw new Error(`Missing persisted stream segment at index ${startIndex}`);
|
|
3046
|
+
const start = startFromSegmentKey(entry[0], SEGMENT_SIZE_KEY_PREFIX);
|
|
3047
|
+
planned.unshift({
|
|
3048
|
+
start,
|
|
3049
|
+
size: validateSegmentSize(entry[1], start)
|
|
3050
|
+
});
|
|
3051
|
+
}
|
|
3052
|
+
const selected = [];
|
|
3053
|
+
let plannedBytes = 0;
|
|
3054
|
+
for (const segment of planned) {
|
|
3055
|
+
if (selected.length > 0 && plannedBytes >= maxBytes) break;
|
|
3056
|
+
selected.push(segment);
|
|
3057
|
+
plannedBytes += segment.size.bytes;
|
|
3058
|
+
}
|
|
3059
|
+
const values = await this.getMany(selected.map(({ start }) => segmentKey(start)));
|
|
3060
|
+
const chunks = [];
|
|
3061
|
+
let bytes = 0;
|
|
3062
|
+
let expected = startIndex;
|
|
3063
|
+
for (const { start, size } of selected) {
|
|
3064
|
+
if (start > expected) throw new Error(`Missing persisted stream segment at index ${expected}`);
|
|
3065
|
+
const segment = decodeSegment(values.get(segmentKey(start)), start, size);
|
|
3066
|
+
for (let offset = expected - start; offset < segment.length; offset++) {
|
|
3067
|
+
if (expected >= endIndex || bytes + segment[offset].byteLength > maxBytes) return chunks;
|
|
3068
|
+
chunks.push(segment[offset]);
|
|
3069
|
+
bytes += segment[offset].byteLength;
|
|
3070
|
+
expected++;
|
|
3071
|
+
}
|
|
3072
|
+
}
|
|
3073
|
+
return chunks;
|
|
3074
|
+
}
|
|
2702
3075
|
/** Append one bounded binary batch with contiguous indexes. */
|
|
2703
3076
|
async writeChunks(runId, chunks) {
|
|
2704
3077
|
validateStreamWriteChunks(chunks);
|
|
2705
|
-
const storedChunks = chunks.map(compactChunk);
|
|
2706
3078
|
return await this.runMutation(async () => {
|
|
2707
3079
|
const committed = await this.ctx.storage.transaction(async (txn) => {
|
|
2708
3080
|
const stored = await txn.get(META_KEY);
|
|
@@ -2710,18 +3082,30 @@ var StreamDO = class extends DurableObject {
|
|
|
2710
3082
|
this.assertWritable(meta, runId);
|
|
2711
3083
|
const startIndex = meta.count;
|
|
2712
3084
|
if (startIndex + chunks.length > MAX_STREAM_INDEX) throw new Error("Stream offset limit exceeded");
|
|
3085
|
+
const layout = stored === void 0 ? NEW_STREAM_LAYOUT : meta.layout;
|
|
2713
3086
|
const nextMeta = {
|
|
2714
3087
|
count: startIndex + chunks.length,
|
|
2715
3088
|
state: "open",
|
|
2716
|
-
ownerRunId: runId
|
|
3089
|
+
ownerRunId: runId,
|
|
3090
|
+
...layout === void 0 ? {} : { layout }
|
|
2717
3091
|
};
|
|
2718
|
-
const entries =
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
entries[
|
|
2722
|
-
|
|
3092
|
+
const entries = [[META_KEY, nextMeta]];
|
|
3093
|
+
if (layout === SEGMENT_LAYOUT) for (const segment of packSegments(startIndex, chunks)) {
|
|
3094
|
+
entries.push([segmentKey(segment.start), encodeSegment(segment.chunks)]);
|
|
3095
|
+
entries.push([segmentSizeKey(segment.start), {
|
|
3096
|
+
count: segment.chunks.length,
|
|
3097
|
+
bytes: segment.chunks.reduce((total, chunk) => total + chunk.byteLength, 0)
|
|
3098
|
+
}]);
|
|
3099
|
+
}
|
|
3100
|
+
else {
|
|
3101
|
+
const storedChunks = chunks.map(compactChunk);
|
|
3102
|
+
for (let offset = 0; offset < storedChunks.length; offset++) {
|
|
3103
|
+
const index = startIndex + offset;
|
|
3104
|
+
entries.push([chunkKey(index), storedChunks[offset]]);
|
|
3105
|
+
entries.push([chunkSizeKey(index), storedChunks[offset].byteLength]);
|
|
3106
|
+
}
|
|
2723
3107
|
}
|
|
2724
|
-
await txn.put(entries);
|
|
3108
|
+
for (let offset = 0; offset < entries.length; offset += STORAGE_BATCH_KEYS) await txn.put(Object.fromEntries(entries.slice(offset, offset + STORAGE_BATCH_KEYS)));
|
|
2725
3109
|
return {
|
|
2726
3110
|
meta: nextMeta,
|
|
2727
3111
|
result: {
|
|
@@ -2855,6 +3239,102 @@ var StreamDO = class extends DurableObject {
|
|
|
2855
3239
|
};
|
|
2856
3240
|
});
|
|
2857
3241
|
}
|
|
3242
|
+
/** Delete one page of per-chunk rows, listing size rows to avoid loading payloads. */
|
|
3243
|
+
async expireChunkPage(txn, limit, byteLimit) {
|
|
3244
|
+
const candidates = await txn.list({
|
|
3245
|
+
prefix: CHUNK_SIZE_KEY_PREFIX,
|
|
3246
|
+
limit: limit + 1
|
|
3247
|
+
});
|
|
3248
|
+
const page = [];
|
|
3249
|
+
let bytes = 0;
|
|
3250
|
+
for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
|
|
3251
|
+
const index = indexFromChunkSizeKey(key);
|
|
3252
|
+
const size = validateChunkSize(storedSize, index);
|
|
3253
|
+
if (page.length > 0 && bytes + size > byteLimit) break;
|
|
3254
|
+
page.push({
|
|
3255
|
+
index,
|
|
3256
|
+
size
|
|
3257
|
+
});
|
|
3258
|
+
bytes += size;
|
|
3259
|
+
}
|
|
3260
|
+
let payloadFallback = false;
|
|
3261
|
+
if (candidates.size === 0) {
|
|
3262
|
+
const first = (await txn.list({
|
|
3263
|
+
prefix: CHUNK_KEY_PREFIX,
|
|
3264
|
+
limit: 1
|
|
3265
|
+
})).entries().next().value;
|
|
3266
|
+
if (first) {
|
|
3267
|
+
const [key, payload] = first;
|
|
3268
|
+
page.push({
|
|
3269
|
+
index: indexFromChunkKey(key),
|
|
3270
|
+
size: payload instanceof Uint8Array ? payload.byteLength : 0
|
|
3271
|
+
});
|
|
3272
|
+
bytes = page[0].size;
|
|
3273
|
+
payloadFallback = true;
|
|
3274
|
+
}
|
|
3275
|
+
}
|
|
3276
|
+
await deleteInBatches(txn, page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]));
|
|
3277
|
+
let done = candidates.size === page.length;
|
|
3278
|
+
if (payloadFallback) done = (await txn.list({
|
|
3279
|
+
prefix: CHUNK_KEY_PREFIX,
|
|
3280
|
+
limit: 1
|
|
3281
|
+
})).size === 0;
|
|
3282
|
+
else if (done && candidates.size > 0) done = (await txn.list({
|
|
3283
|
+
prefix: CHUNK_KEY_PREFIX,
|
|
3284
|
+
limit: 1
|
|
3285
|
+
})).size === 0;
|
|
3286
|
+
return {
|
|
3287
|
+
chunks: page.length,
|
|
3288
|
+
bytes,
|
|
3289
|
+
done
|
|
3290
|
+
};
|
|
3291
|
+
}
|
|
3292
|
+
/**
|
|
3293
|
+
* Delete one page of segment rows, listing segment size rows to avoid
|
|
3294
|
+
* loading payloads. One segment always makes progress, even when it holds
|
|
3295
|
+
* more chunks or bytes than the page allows.
|
|
3296
|
+
*/
|
|
3297
|
+
async expireSegmentPage(txn, limit, byteLimit) {
|
|
3298
|
+
const candidates = await txn.list({
|
|
3299
|
+
prefix: SEGMENT_SIZE_KEY_PREFIX,
|
|
3300
|
+
limit: limit + 1
|
|
3301
|
+
});
|
|
3302
|
+
const page = [];
|
|
3303
|
+
let chunks = 0;
|
|
3304
|
+
let bytes = 0;
|
|
3305
|
+
for (const [key, value] of candidates) {
|
|
3306
|
+
const start = startFromSegmentKey(key, SEGMENT_SIZE_KEY_PREFIX);
|
|
3307
|
+
const size = validateSegmentSize(value, start);
|
|
3308
|
+
if (page.length > 0 && (chunks + size.count > limit || bytes + size.bytes > byteLimit)) break;
|
|
3309
|
+
page.push(start);
|
|
3310
|
+
chunks += size.count;
|
|
3311
|
+
bytes += size.bytes;
|
|
3312
|
+
}
|
|
3313
|
+
let orphan = false;
|
|
3314
|
+
if (candidates.size === 0) {
|
|
3315
|
+
const first = (await txn.list({
|
|
3316
|
+
prefix: SEGMENT_KEY_PREFIX,
|
|
3317
|
+
limit: 1
|
|
3318
|
+
})).entries().next().value;
|
|
3319
|
+
if (first) {
|
|
3320
|
+
page.push(startFromSegmentKey(first[0], SEGMENT_KEY_PREFIX));
|
|
3321
|
+
bytes = first[1] instanceof Uint8Array ? first[1].byteLength : 0;
|
|
3322
|
+
chunks = 1;
|
|
3323
|
+
orphan = true;
|
|
3324
|
+
}
|
|
3325
|
+
}
|
|
3326
|
+
await deleteInBatches(txn, page.flatMap((start) => [segmentKey(start), segmentSizeKey(start)]));
|
|
3327
|
+
let done = !orphan && candidates.size === page.length;
|
|
3328
|
+
if (orphan || done) done = (await txn.list({
|
|
3329
|
+
prefix: SEGMENT_KEY_PREFIX,
|
|
3330
|
+
limit: 1
|
|
3331
|
+
})).size === 0;
|
|
3332
|
+
return {
|
|
3333
|
+
chunks,
|
|
3334
|
+
bytes,
|
|
3335
|
+
done
|
|
3336
|
+
};
|
|
3337
|
+
}
|
|
2858
3338
|
/**
|
|
2859
3339
|
* Fence the stream and delete one bounded page. Listing size keys avoids
|
|
2860
3340
|
* loading payloads, and the transaction makes a crash retry resume safely.
|
|
@@ -2868,50 +3348,8 @@ var StreamDO = class extends DurableObject {
|
|
|
2868
3348
|
const meta = stored === void 0 ? emptyMeta() : validateMeta(stored);
|
|
2869
3349
|
this.assertOwner(meta, runId);
|
|
2870
3350
|
const firstFence = meta.state !== "expired";
|
|
2871
|
-
const
|
|
2872
|
-
|
|
2873
|
-
limit: limit + 1
|
|
2874
|
-
});
|
|
2875
|
-
const page = [];
|
|
2876
|
-
let bytes = 0;
|
|
2877
|
-
for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
|
|
2878
|
-
const index = indexFromChunkSizeKey(key);
|
|
2879
|
-
const size = validateChunkSize(storedSize, index);
|
|
2880
|
-
if (page.length > 0 && bytes + size > byteLimit) break;
|
|
2881
|
-
page.push({
|
|
2882
|
-
index,
|
|
2883
|
-
size
|
|
2884
|
-
});
|
|
2885
|
-
bytes += size;
|
|
2886
|
-
}
|
|
2887
|
-
let payloadFallback = false;
|
|
2888
|
-
if (candidates.size === 0) {
|
|
2889
|
-
const first = (await txn.list({
|
|
2890
|
-
prefix: CHUNK_KEY_PREFIX,
|
|
2891
|
-
limit: 1
|
|
2892
|
-
})).entries().next().value;
|
|
2893
|
-
if (first) {
|
|
2894
|
-
const [key, payload] = first;
|
|
2895
|
-
page.push({
|
|
2896
|
-
index: indexFromChunkKey(key),
|
|
2897
|
-
size: payload instanceof Uint8Array ? payload.byteLength : 0
|
|
2898
|
-
});
|
|
2899
|
-
bytes = page[0].size;
|
|
2900
|
-
payloadFallback = true;
|
|
2901
|
-
}
|
|
2902
|
-
}
|
|
2903
|
-
const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
|
|
2904
|
-
if (keys.length > 0) await txn.delete(keys);
|
|
2905
|
-
let done = candidates.size === page.length;
|
|
2906
|
-
if (payloadFallback) done = (await txn.list({
|
|
2907
|
-
prefix: CHUNK_KEY_PREFIX,
|
|
2908
|
-
limit: 1
|
|
2909
|
-
})).size === 0;
|
|
2910
|
-
else if (done && candidates.size > 0) done = (await txn.list({
|
|
2911
|
-
prefix: CHUNK_KEY_PREFIX,
|
|
2912
|
-
limit: 1
|
|
2913
|
-
})).size === 0;
|
|
2914
|
-
if (done && page.length === 0 && meta.state === "expired" && meta.payloadDeleted) return {
|
|
3351
|
+
const { chunks, bytes, done } = meta.layout === SEGMENT_LAYOUT ? await this.expireSegmentPage(txn, limit, byteLimit) : await this.expireChunkPage(txn, limit, byteLimit);
|
|
3352
|
+
if (done && chunks === 0 && meta.state === "expired" && meta.payloadDeleted) return {
|
|
2915
3353
|
meta,
|
|
2916
3354
|
result: {
|
|
2917
3355
|
deleted: false,
|
|
@@ -2935,7 +3373,7 @@ var StreamDO = class extends DurableObject {
|
|
|
2935
3373
|
meta: nextMeta,
|
|
2936
3374
|
result: {
|
|
2937
3375
|
deleted: firstFence,
|
|
2938
|
-
chunks
|
|
3376
|
+
chunks,
|
|
2939
3377
|
bytes,
|
|
2940
3378
|
done
|
|
2941
3379
|
}
|
|
@@ -3005,19 +3443,29 @@ async function deleteQueuePayloadObjects(binding, keys) {
|
|
|
3005
3443
|
}
|
|
3006
3444
|
//#endregion
|
|
3007
3445
|
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
3446
|
+
const EVENT_SEQUENCE_KEY = "event_sequence";
|
|
3008
3447
|
const STORAGE_BATCH_SIZE = 128;
|
|
3009
3448
|
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
3010
3449
|
const HOOK_MARKER_PAGE_SIZE = 64;
|
|
3011
3450
|
const TERMINAL_HOOK_PAGE_SIZE = 64;
|
|
3012
3451
|
const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
3013
3452
|
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
3014
|
-
|
|
3453
|
+
/** StreamDO deletes a page in 128-key calls within one transaction. */
|
|
3454
|
+
const STREAM_CHUNKS_PER_CLEANUP_PAGE = 256;
|
|
3015
3455
|
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
3016
3456
|
const QUEUE_PAYLOADS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3017
3457
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
3018
3458
|
const MAX_DATE_MS = 864e13;
|
|
3019
3459
|
const QUEUE_ORPHAN_KEY = "queue-orphan";
|
|
3020
3460
|
const QUEUE_RESERVATION_KEY = "queue-reservation";
|
|
3461
|
+
/** Run-scoped Queue idempotency state, one key per claim name, in the run's own cell. */
|
|
3462
|
+
const RUN_QUEUE_CLAIM_PREFIX = "queue-claim:";
|
|
3463
|
+
/**
|
|
3464
|
+
* Present when every idempotency claim of this run lives in the run's own
|
|
3465
|
+
* cell. Written only when the run is created, so a run keeps one claim scope
|
|
3466
|
+
* for its whole life and claims made before the marker existed stay valid.
|
|
3467
|
+
*/
|
|
3468
|
+
const RUN_QUEUE_CLAIM_SCOPE_KEY = "queue-claim-scope";
|
|
3021
3469
|
function validateCleanupRequest(request, allowDisabled) {
|
|
3022
3470
|
if (!isRecord$1(request)) throw new Error("world-celld retention request must be an object");
|
|
3023
3471
|
if (!(allowDisabled ? isNonNegativeSafeInteger(request.retentionMs) : isPositiveSafeInteger(request.retentionMs))) throw new Error(`world-celld retentionMs must be a ${allowDisabled ? "non-negative" : "positive"} safe integer`);
|
|
@@ -3071,6 +3519,12 @@ async function putStorageEntries(storage, entries) {
|
|
|
3071
3519
|
const all = Array.from(entries);
|
|
3072
3520
|
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
|
|
3073
3521
|
}
|
|
3522
|
+
function runQueueClaimKey(claimName) {
|
|
3523
|
+
return `${RUN_QUEUE_CLAIM_PREFIX}${claimName}`;
|
|
3524
|
+
}
|
|
3525
|
+
function validRunQueueClaimName(claimName) {
|
|
3526
|
+
return typeof claimName === "string" && claimName.startsWith("claim:");
|
|
3527
|
+
}
|
|
3074
3528
|
/**
|
|
3075
3529
|
* Durable Object for managing a single workflow run's state.
|
|
3076
3530
|
*
|
|
@@ -3093,11 +3547,14 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3093
3547
|
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
3094
3548
|
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
3095
3549
|
}
|
|
3096
|
-
async retentionState(storage, now = this.now()) {
|
|
3097
|
-
const
|
|
3550
|
+
async retentionState(storage, now = this.now(), options) {
|
|
3551
|
+
const keys = [TOMBSTONE_KEY, CLEANUP_RECORD_KEY];
|
|
3552
|
+
if (options?.withEventSequence) keys.push(EVENT_SEQUENCE_KEY);
|
|
3553
|
+
const values = await storage.get(keys);
|
|
3098
3554
|
return {
|
|
3099
3555
|
cleanup: values.get(CLEANUP_RECORD_KEY),
|
|
3100
|
-
tombstone: this.tombstoneFrom(values, now)
|
|
3556
|
+
tombstone: this.tombstoneFrom(values, now),
|
|
3557
|
+
eventSequence: values.get(EVENT_SEQUENCE_KEY) ?? 0
|
|
3101
3558
|
};
|
|
3102
3559
|
}
|
|
3103
3560
|
async read(reader) {
|
|
@@ -3132,21 +3589,23 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3132
3589
|
if (request.cleanup !== void 0) validateCleanupRequest(request.cleanup, true);
|
|
3133
3590
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
3134
3591
|
const now = new Date(this.now());
|
|
3135
|
-
const retention = await this.retentionState(txn, now.getTime());
|
|
3592
|
+
const retention = await this.retentionState(txn, now.getTime(), { withEventSequence: true });
|
|
3136
3593
|
const { tombstone } = retention;
|
|
3137
3594
|
if (tombstone) return {
|
|
3138
3595
|
ok: false,
|
|
3139
3596
|
code: "RUN_EXPIRED",
|
|
3140
3597
|
message: `Workflow run "${request.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
3141
3598
|
};
|
|
3142
|
-
|
|
3599
|
+
const storedSequence = retention.eventSequence;
|
|
3600
|
+
let eventSequence = storedSequence;
|
|
3143
3601
|
const outcome = await applyEvent(storeFrom(txn), {
|
|
3144
3602
|
...request,
|
|
3145
3603
|
nextEventId: () => slotToEventId(++eventSequence),
|
|
3146
3604
|
now
|
|
3147
3605
|
});
|
|
3148
3606
|
if (!outcome.ok) return outcome;
|
|
3149
|
-
await txn.put(
|
|
3607
|
+
if (eventSequence !== storedSequence) await txn.put(EVENT_SEQUENCE_KEY, eventSequence);
|
|
3608
|
+
if (outcome.runCreated) await txn.put(RUN_QUEUE_CLAIM_SCOPE_KEY, "run");
|
|
3150
3609
|
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
3151
3610
|
hookId: outcome.hookToIndex.hookId,
|
|
3152
3611
|
token: outcome.hookToIndex.token
|
|
@@ -3526,10 +3985,58 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3526
3985
|
if (cleanup.phase === "payload") await this.deletePayloadBatch();
|
|
3527
3986
|
}
|
|
3528
3987
|
async executeTerminalCleanup(cleanup) {
|
|
3988
|
+
if (cleanup.phase === "hooks" && await this.completeSmallTerminalCleanup(cleanup)) return;
|
|
3529
3989
|
if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
|
|
3530
3990
|
else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
|
|
3531
3991
|
else await this.deleteTerminalWaitPage(cleanup);
|
|
3532
3992
|
}
|
|
3993
|
+
/**
|
|
3994
|
+
* Most runs end with a handful of hooks and waits. When every terminal
|
|
3995
|
+
* phase fits in one page, release the hook indexes once and delete hooks,
|
|
3996
|
+
* markers, and waits in a single transaction instead of one alarm and one
|
|
3997
|
+
* transaction per phase. Returns false, having changed nothing locally, when
|
|
3998
|
+
* any phase needs paging; the paged phases then run as before.
|
|
3999
|
+
*/
|
|
4000
|
+
async completeSmallTerminalCleanup(cleanup) {
|
|
4001
|
+
const [hookEntries, markerEntries] = await Promise.all([this.ctx.storage.list({
|
|
4002
|
+
prefix: HOOK_KEY_PREFIX,
|
|
4003
|
+
limit: 65
|
|
4004
|
+
}), this.ctx.storage.list({
|
|
4005
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
4006
|
+
limit: 65
|
|
4007
|
+
})]);
|
|
4008
|
+
if (hookEntries.size > TERMINAL_HOOK_PAGE_SIZE || markerEntries.size > HOOK_MARKER_PAGE_SIZE) return false;
|
|
4009
|
+
const hooks = Array.from(hookEntries.values());
|
|
4010
|
+
const references = /* @__PURE__ */ new Map();
|
|
4011
|
+
for (const reference of [...hooks.map((hook) => ({
|
|
4012
|
+
hookId: hook.hookId,
|
|
4013
|
+
token: hook.token
|
|
4014
|
+
})), ...markerEntries.values()]) references.set(`${reference.hookId}\u0000${reference.token}`, reference);
|
|
4015
|
+
if (references.size > 0) await this.workflowIndex().releaseHookIndexes({
|
|
4016
|
+
runId: cleanup.runId,
|
|
4017
|
+
hooks: Array.from(references.values())
|
|
4018
|
+
});
|
|
4019
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4020
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
4021
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return true;
|
|
4022
|
+
const waits = await txn.list({
|
|
4023
|
+
prefix: WAIT_KEY_PREFIX,
|
|
4024
|
+
limit: 129
|
|
4025
|
+
});
|
|
4026
|
+
if (waits.size > TERMINAL_WAIT_PAGE_SIZE) return false;
|
|
4027
|
+
const keys = [
|
|
4028
|
+
...hooks.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]),
|
|
4029
|
+
...markerEntries.keys(),
|
|
4030
|
+
...waits.keys()
|
|
4031
|
+
];
|
|
4032
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE) await txn.delete(keys.slice(offset, offset + STORAGE_BATCH_SIZE));
|
|
4033
|
+
await txn.delete(TERMINAL_CLEANUP_KEY);
|
|
4034
|
+
const retention = await txn.get(CLEANUP_RECORD_KEY);
|
|
4035
|
+
if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
|
|
4036
|
+
else await txn.deleteAlarm();
|
|
4037
|
+
return true;
|
|
4038
|
+
});
|
|
4039
|
+
}
|
|
3533
4040
|
async deleteTerminalHookPage(cleanup) {
|
|
3534
4041
|
const entries = await this.ctx.storage.list({
|
|
3535
4042
|
prefix: HOOK_KEY_PREFIX,
|
|
@@ -3765,6 +4272,150 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3765
4272
|
await txn.setAlarm(this.now() + delay);
|
|
3766
4273
|
});
|
|
3767
4274
|
}
|
|
4275
|
+
/**
|
|
4276
|
+
* Read-only admission for a run-bearing Queue message whose body travels
|
|
4277
|
+
* inline in the broker envelope. It answers what payload registration does
|
|
4278
|
+
* for offloaded bodies, without writing anything.
|
|
4279
|
+
*/
|
|
4280
|
+
async getQueueAdmission() {
|
|
4281
|
+
const { tombstone } = await this.retentionState(this.ctx.storage);
|
|
4282
|
+
return tombstone ? {
|
|
4283
|
+
ok: false,
|
|
4284
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
4285
|
+
} : { ok: true };
|
|
4286
|
+
}
|
|
4287
|
+
/**
|
|
4288
|
+
* Enqueue-side admission and idempotency reservation in one transaction,
|
|
4289
|
+
* for a run-scoped claim. Replaces a separate claim cell's reservation plus
|
|
4290
|
+
* the run expiry check.
|
|
4291
|
+
*/
|
|
4292
|
+
async reserveRunQueueMessage(params) {
|
|
4293
|
+
if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.expiresAt)) throw new TypeError("world-celld run queue reservation is invalid");
|
|
4294
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4295
|
+
const key = runQueueClaimKey(params.claimName);
|
|
4296
|
+
const values = await txn.get([
|
|
4297
|
+
TOMBSTONE_KEY,
|
|
4298
|
+
CLEANUP_RECORD_KEY,
|
|
4299
|
+
RUN_QUEUE_CLAIM_SCOPE_KEY,
|
|
4300
|
+
key
|
|
4301
|
+
]);
|
|
4302
|
+
const tombstone = this.tombstoneFrom(values);
|
|
4303
|
+
if (tombstone) return {
|
|
4304
|
+
ok: false,
|
|
4305
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
4306
|
+
};
|
|
4307
|
+
if (values.get("queue-claim-scope") !== "run") return {
|
|
4308
|
+
ok: true,
|
|
4309
|
+
scope: "cell"
|
|
4310
|
+
};
|
|
4311
|
+
const state = values.get(key) ?? {};
|
|
4312
|
+
const existing = state.reservation;
|
|
4313
|
+
if (existing && existing.expiresAt > this.now() && existing.messageId !== params.messageId) return {
|
|
4314
|
+
ok: true,
|
|
4315
|
+
scope: "run",
|
|
4316
|
+
admitted: false,
|
|
4317
|
+
messageId: existing.messageId
|
|
4318
|
+
};
|
|
4319
|
+
await txn.put(key, {
|
|
4320
|
+
...state,
|
|
4321
|
+
reservation: {
|
|
4322
|
+
messageId: params.messageId,
|
|
4323
|
+
expiresAt: Math.max(existing?.expiresAt ?? 0, params.expiresAt)
|
|
4324
|
+
}
|
|
4325
|
+
});
|
|
4326
|
+
return {
|
|
4327
|
+
ok: true,
|
|
4328
|
+
scope: "run",
|
|
4329
|
+
admitted: true,
|
|
4330
|
+
messageId: params.messageId
|
|
4331
|
+
};
|
|
4332
|
+
});
|
|
4333
|
+
}
|
|
4334
|
+
/** Delivery-side run expiry check and in-flight claim in one transaction. */
|
|
4335
|
+
async claimRunQueueMessage(params) {
|
|
4336
|
+
if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string" || params.messageId.length === 0 || !isPositiveSafeInteger(params.staleMs)) throw new TypeError("world-celld run queue claim is invalid");
|
|
4337
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4338
|
+
const key = runQueueClaimKey(params.claimName);
|
|
4339
|
+
const values = await txn.get([
|
|
4340
|
+
TOMBSTONE_KEY,
|
|
4341
|
+
CLEANUP_RECORD_KEY,
|
|
4342
|
+
key
|
|
4343
|
+
]);
|
|
4344
|
+
if (this.tombstoneFrom(values)) return { expired: true };
|
|
4345
|
+
const state = values.get(key) ?? {};
|
|
4346
|
+
const existing = state.claim;
|
|
4347
|
+
const now = this.now();
|
|
4348
|
+
if (existing && existing.expiresAt > now) {
|
|
4349
|
+
if (existing.messageId !== params.messageId) return {
|
|
4350
|
+
expired: false,
|
|
4351
|
+
claimed: false
|
|
4352
|
+
};
|
|
4353
|
+
if (existing.retryAt === void 0 || existing.retryAt > now) return {
|
|
4354
|
+
expired: false,
|
|
4355
|
+
claimed: false,
|
|
4356
|
+
retryAt: existing.retryAt ?? existing.expiresAt
|
|
4357
|
+
};
|
|
4358
|
+
}
|
|
4359
|
+
await txn.put(key, {
|
|
4360
|
+
...state,
|
|
4361
|
+
claim: {
|
|
4362
|
+
messageId: params.messageId,
|
|
4363
|
+
expiresAt: now + params.staleMs
|
|
4364
|
+
}
|
|
4365
|
+
});
|
|
4366
|
+
return {
|
|
4367
|
+
expired: false,
|
|
4368
|
+
claimed: true
|
|
4369
|
+
};
|
|
4370
|
+
});
|
|
4371
|
+
}
|
|
4372
|
+
async holdRunQueueMessage(params) {
|
|
4373
|
+
if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.retryAt) || !isNonNegativeSafeInteger(params.expiresAt) || params.reservationExpiresAt !== void 0 && !isNonNegativeSafeInteger(params.reservationExpiresAt)) throw new TypeError("world-celld run queue claim hold is invalid");
|
|
4374
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4375
|
+
const key = runQueueClaimKey(params.claimName);
|
|
4376
|
+
const state = await txn.get(key) ?? {};
|
|
4377
|
+
const existing = state.claim;
|
|
4378
|
+
if (!existing || existing.messageId !== params.messageId) return { held: false };
|
|
4379
|
+
const next = {
|
|
4380
|
+
...state,
|
|
4381
|
+
claim: {
|
|
4382
|
+
...existing,
|
|
4383
|
+
retryAt: params.retryAt,
|
|
4384
|
+
expiresAt: Math.max(existing.expiresAt, params.expiresAt)
|
|
4385
|
+
}
|
|
4386
|
+
};
|
|
4387
|
+
if (state.reservation?.messageId === params.messageId && params.reservationExpiresAt !== void 0 && params.reservationExpiresAt > state.reservation.expiresAt) next.reservation = {
|
|
4388
|
+
...state.reservation,
|
|
4389
|
+
expiresAt: params.reservationExpiresAt
|
|
4390
|
+
};
|
|
4391
|
+
await txn.put(key, next);
|
|
4392
|
+
return { held: true };
|
|
4393
|
+
});
|
|
4394
|
+
}
|
|
4395
|
+
async releaseRunQueueMessage(params) {
|
|
4396
|
+
if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string") throw new TypeError("world-celld run queue claim release is invalid");
|
|
4397
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
4398
|
+
const key = runQueueClaimKey(params.claimName);
|
|
4399
|
+
const state = await txn.get(key);
|
|
4400
|
+
if (state?.claim?.messageId !== params.messageId) return;
|
|
4401
|
+
const { claim: _released, ...rest } = state;
|
|
4402
|
+
if (rest.reservation) await txn.put(key, rest);
|
|
4403
|
+
else await txn.delete(key);
|
|
4404
|
+
});
|
|
4405
|
+
}
|
|
4406
|
+
async completeRunQueueMessage(params) {
|
|
4407
|
+
if (!validRunQueueClaimName(params?.claimName) || typeof params.messageId !== "string") throw new TypeError("world-celld run queue completion is invalid");
|
|
4408
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
4409
|
+
const key = runQueueClaimKey(params.claimName);
|
|
4410
|
+
const state = await txn.get(key);
|
|
4411
|
+
if (!state) return;
|
|
4412
|
+
const next = {};
|
|
4413
|
+
if (state.reservation && state.reservation.messageId !== params.messageId) next.reservation = state.reservation;
|
|
4414
|
+
if (state.claim && state.claim.messageId !== params.messageId) next.claim = state.claim;
|
|
4415
|
+
if (next.reservation || next.claim) await txn.put(key, next);
|
|
4416
|
+
else await txn.delete(key);
|
|
4417
|
+
});
|
|
4418
|
+
}
|
|
3768
4419
|
async registerQueuePayload(registration) {
|
|
3769
4420
|
if (!registration || typeof registration.messageId !== "string" || registration.messageId.length === 0 || typeof registration.key !== "string" || registration.key.length === 0 || !isNonNegativeSafeInteger(registration.orphanExpiresAt)) throw new TypeError("world-celld queue payload registration is invalid");
|
|
3770
4421
|
return await this.ctx.storage.transaction(async (txn) => {
|