@ewhauser/world-celld 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +43 -15
- package/celld-queue-worker/README.md +2 -2
- package/celld-queue-worker/wrangler.jsonc +3 -1
- package/celld-worker/README.md +3 -3
- package/dist/index.d.ts +44 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +113 -99
- package/dist/index.js.map +1 -1
- package/dist/queue-consumer.js +1 -1
- package/dist/{queue-protocol-Ds44uuND.js → queue-protocol-CtH7UUCO.js} +2 -2
- package/dist/{queue-protocol-Ds44uuND.js.map → queue-protocol-CtH7UUCO.js.map} +1 -1
- package/dist/{queue-protocol-CmJNMH_5.js → run-list-B__jKuE7.js} +124 -18
- package/dist/run-list-B__jKuE7.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +211 -25
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +19 -0
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +313 -33
- package/dist/worker.js.map +1 -1
- package/docs/celld-v0.6-upgrade.md +121 -0
- package/package.json +7 -5
- package/dist/queue-protocol-CmJNMH_5.js.map +0 -1
- package/docs/celld-v0.5-upgrade.md +0 -130
package/dist/testing.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as isRecord$1, B as STREAM_BATCH_CONTENT_TYPE, G as encodeStreamReadResult, I as MAX_STREAM_LONG_POLL_MS, L as MAX_STREAM_READ_BYTES, N as MAX_STREAM_BATCH_BYTES, O as isNonNegativeSafeInteger, P as MAX_STREAM_CHUNK_BYTES, T as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, U as decodeStreamWriteBatch, V as STREAM_CHUNK_LIMIT_HEADER, X as validateStreamReadRequest, Y as normalizeStreamError, Z as validateStreamWriteChunks, _ as createWorkflowIndex, b as HOOK_CLAIM_LEASE_MS, c as queuePayloadObjectKey, d as validateNativeQueueSendOptions, f as compact, h as parse, i as QUEUE_PAYLOAD_REGISTRY_PREFIX, k as isPositiveSafeInteger, l as queuePayloadRegistryKey, m as rpcStringify, n as listRunsPage, o as queueClaimName, p as rpcParse, q as encodeStreamWriteResult, r as NATIVE_QUEUE_MAX_MESSAGE_BYTES, s as queueOrphanName, t as MAX_RUN_LIST_LIMIT, u as validateNativeQueueEnvelope, v as runIndexMetadata, x as LIFECYCLE_COMPACTION_RETRY_MS } from "./run-list-B__jKuE7.js";
|
|
2
2
|
import { CreateEventSchema, EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
//#region src/testing/fake-cell.ts
|
|
@@ -180,11 +180,12 @@ var FakeStorage = class {
|
|
|
180
180
|
synchronous: false
|
|
181
181
|
});
|
|
182
182
|
}
|
|
183
|
-
/** Inject
|
|
184
|
-
failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure")) {
|
|
183
|
+
/** Inject matching write failures (one by default), including writes inside transactions. */
|
|
184
|
+
failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure"), times = 1) {
|
|
185
185
|
this.mutationFailure = {
|
|
186
186
|
predicate,
|
|
187
|
-
error
|
|
187
|
+
error,
|
|
188
|
+
remaining: times
|
|
188
189
|
};
|
|
189
190
|
}
|
|
190
191
|
/** Inject one matching storage read failure, including reads inside transactions. */
|
|
@@ -198,7 +199,8 @@ var FakeStorage = class {
|
|
|
198
199
|
maybeFailMutation(mutation) {
|
|
199
200
|
if (!this.mutationFailure?.predicate(mutation)) return;
|
|
200
201
|
const { error } = this.mutationFailure;
|
|
201
|
-
this.mutationFailure
|
|
202
|
+
this.mutationFailure.remaining -= 1;
|
|
203
|
+
if (this.mutationFailure.remaining <= 0) this.mutationFailure = void 0;
|
|
202
204
|
throw error;
|
|
203
205
|
}
|
|
204
206
|
/** @internal Shared with FakeTransaction so read failures model transactional reads too. */
|
|
@@ -1250,6 +1252,7 @@ async function finalizeEventPage(store, outcome, params) {
|
|
|
1250
1252
|
maxEvents: getMaxEventsPerRun()
|
|
1251
1253
|
} : outcome;
|
|
1252
1254
|
if (!result.event) return result;
|
|
1255
|
+
if (params?.preloadEvents && result.event.eventType === "hook_received") return await attachHookReplayPreload(store, result);
|
|
1253
1256
|
if (typeof params?.sinceCursor === "string") {
|
|
1254
1257
|
const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1255
1258
|
limit: 100,
|
|
@@ -1279,6 +1282,32 @@ async function finalizeEventPage(store, outcome, params) {
|
|
|
1279
1282
|
hasMore: events.length < skippedCount
|
|
1280
1283
|
};
|
|
1281
1284
|
}
|
|
1285
|
+
/**
|
|
1286
|
+
* Lazy hook resume preload: return the run and its complete replay log with
|
|
1287
|
+
* the hook_received write, read in the same transaction, so the runtime can
|
|
1288
|
+
* skip its run_started write and the index publication that follows it. The
|
|
1289
|
+
* runtime trusts only a complete log with a non-null cursor, so a run that is
|
|
1290
|
+
* not started, not running, or over the event ceiling gets the plain result.
|
|
1291
|
+
*/
|
|
1292
|
+
async function attachHookReplayPreload(store, result) {
|
|
1293
|
+
const run = await store.get(RUN_KEY);
|
|
1294
|
+
if (!run?.startedAt || run.status !== "running") return result;
|
|
1295
|
+
const maxEvents = getMaxEventsPerRun();
|
|
1296
|
+
const log = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1297
|
+
limit: maxEvents,
|
|
1298
|
+
sortOrder: "asc"
|
|
1299
|
+
}, (event) => event.eventId);
|
|
1300
|
+
const last = log.data.at(-1);
|
|
1301
|
+
if (log.hasMore || last === void 0) return result;
|
|
1302
|
+
return {
|
|
1303
|
+
...result,
|
|
1304
|
+
replayRun: run,
|
|
1305
|
+
maxEvents,
|
|
1306
|
+
events: log.data,
|
|
1307
|
+
cursor: last.eventId,
|
|
1308
|
+
hasMore: false
|
|
1309
|
+
};
|
|
1310
|
+
}
|
|
1282
1311
|
//#endregion
|
|
1283
1312
|
//#region src/worker/auth.ts
|
|
1284
1313
|
/**
|
|
@@ -1415,6 +1444,34 @@ function listOptions(value) {
|
|
|
1415
1444
|
}
|
|
1416
1445
|
return parsed;
|
|
1417
1446
|
}
|
|
1447
|
+
const RUN_STATUSES = /* @__PURE__ */ new Set([
|
|
1448
|
+
"pending",
|
|
1449
|
+
"running",
|
|
1450
|
+
"completed",
|
|
1451
|
+
"failed",
|
|
1452
|
+
"cancelled"
|
|
1453
|
+
]);
|
|
1454
|
+
function runListRequest(value) {
|
|
1455
|
+
if (!isRecord$1(value)) invalid("run list request must be an object");
|
|
1456
|
+
const limit = value.limit;
|
|
1457
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1e3) invalid(`request.limit must be an integer between 1 and ${MAX_RUN_LIST_LIMIT}`);
|
|
1458
|
+
const parsed = { limit };
|
|
1459
|
+
if (value.workflowName !== void 0) parsed.workflowName = nonEmptyString(value.workflowName, "request.workflowName");
|
|
1460
|
+
if (value.cursor !== void 0) parsed.cursor = stringValue(value.cursor, "request.cursor");
|
|
1461
|
+
if (value.statuses !== void 0) {
|
|
1462
|
+
if (!Array.isArray(value.statuses) || value.statuses.length === 0 || !value.statuses.every((status) => RUN_STATUSES.has(status))) invalid("request.statuses must be a non-empty array of run statuses");
|
|
1463
|
+
parsed.statuses = value.statuses;
|
|
1464
|
+
}
|
|
1465
|
+
if (value.sortOrder !== void 0) {
|
|
1466
|
+
if (value.sortOrder !== "asc" && value.sortOrder !== "desc") invalid("request.sortOrder must be 'asc' or 'desc'");
|
|
1467
|
+
parsed.sortOrder = value.sortOrder;
|
|
1468
|
+
}
|
|
1469
|
+
if (value.resolveData !== void 0) {
|
|
1470
|
+
if (value.resolveData !== "none" && value.resolveData !== "all") invalid("request.resolveData must be 'none' or 'all'");
|
|
1471
|
+
parsed.resolveData = value.resolveData;
|
|
1472
|
+
}
|
|
1473
|
+
return parsed;
|
|
1474
|
+
}
|
|
1418
1475
|
function runCommit(args) {
|
|
1419
1476
|
exactArgs(args, 3);
|
|
1420
1477
|
const parsedRun = WorkflowRunSchema.safeParse(args[0]);
|
|
@@ -1475,6 +1532,12 @@ function validateIndexRequest(operation, args) {
|
|
|
1475
1532
|
operation,
|
|
1476
1533
|
args: [listOptions(args[0])]
|
|
1477
1534
|
};
|
|
1535
|
+
case "runs.list-resolved":
|
|
1536
|
+
exactArgs(args, 1);
|
|
1537
|
+
return {
|
|
1538
|
+
operation,
|
|
1539
|
+
args: [runListRequest(args[0])]
|
|
1540
|
+
};
|
|
1478
1541
|
case "runs.commit": return runCommit(args);
|
|
1479
1542
|
case "runs.expire":
|
|
1480
1543
|
exactArgs(args, 1);
|
|
@@ -1510,6 +1573,7 @@ function validateIndexRequest(operation, args) {
|
|
|
1510
1573
|
}
|
|
1511
1574
|
const INDEX_OPERATIONS = /* @__PURE__ */ new Set([
|
|
1512
1575
|
"runs.list",
|
|
1576
|
+
"runs.list-resolved",
|
|
1513
1577
|
"runs.commit",
|
|
1514
1578
|
"runs.expire",
|
|
1515
1579
|
"hooks.reserve",
|
|
@@ -1541,6 +1605,37 @@ function workflowIndex(env) {
|
|
|
1541
1605
|
hookIds: env.WORKFLOW_HOOK_IDS
|
|
1542
1606
|
});
|
|
1543
1607
|
}
|
|
1608
|
+
/**
|
|
1609
|
+
* Publish a committed applyEvent outcome to the derivative indexes inside the
|
|
1610
|
+
* fleet, sparing the client a round trip per run mutation. Each index write is
|
|
1611
|
+
* idempotent under the outcome's publication lease. A failed write leaves its
|
|
1612
|
+
* flag unset, so the client publishes it itself exactly as it would without
|
|
1613
|
+
* this step; older clients ignore the flags and publish again.
|
|
1614
|
+
*/
|
|
1615
|
+
async function publishApplyEventIndexes(env, runId, outcome) {
|
|
1616
|
+
if (typeof outcome !== "object" || outcome === null) return;
|
|
1617
|
+
const success = outcome;
|
|
1618
|
+
if (success.ok !== true) return;
|
|
1619
|
+
const index = workflowIndex(env);
|
|
1620
|
+
const publications = [];
|
|
1621
|
+
if (success.run && typeof success.indexPublicationExpiresAt === "number") {
|
|
1622
|
+
const run = success.run;
|
|
1623
|
+
const lease = success.indexPublicationExpiresAt;
|
|
1624
|
+
publications.push((async () => {
|
|
1625
|
+
await index.commitRun(run, runIndexMetadata(run), lease);
|
|
1626
|
+
success.runIndexPublished = true;
|
|
1627
|
+
})());
|
|
1628
|
+
}
|
|
1629
|
+
const releasedHooks = success.releasedHooks;
|
|
1630
|
+
if (Array.isArray(releasedHooks) && releasedHooks.length > 0) publications.push((async () => {
|
|
1631
|
+
await index.releaseHookIndexes({
|
|
1632
|
+
runId,
|
|
1633
|
+
hooks: releasedHooks
|
|
1634
|
+
});
|
|
1635
|
+
success.hookIndexesReleased = true;
|
|
1636
|
+
})());
|
|
1637
|
+
await Promise.allSettled(publications);
|
|
1638
|
+
}
|
|
1544
1639
|
const BINDINGS = {
|
|
1545
1640
|
runs: {
|
|
1546
1641
|
env: "WORKFLOW_DB",
|
|
@@ -1583,7 +1678,7 @@ const BINDINGS = {
|
|
|
1583
1678
|
};
|
|
1584
1679
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
1585
1680
|
const MAX_BODY_BYTES = 33554432;
|
|
1586
|
-
const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 1024;
|
|
1681
|
+
const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 2048 + 1024;
|
|
1587
1682
|
const QUEUE_ORPHAN_GRACE_MS = 432e6;
|
|
1588
1683
|
function errorResponse(status, name, message) {
|
|
1589
1684
|
return Response.json({ error: {
|
|
@@ -1700,7 +1795,10 @@ function parseBoundedInteger(url, name, minimum, maximum) {
|
|
|
1700
1795
|
function streamResponse(body) {
|
|
1701
1796
|
return new Response(body, {
|
|
1702
1797
|
status: 200,
|
|
1703
|
-
headers: {
|
|
1798
|
+
headers: {
|
|
1799
|
+
"content-type": STREAM_BATCH_CONTENT_TYPE,
|
|
1800
|
+
[STREAM_CHUNK_LIMIT_HEADER]: String(512)
|
|
1801
|
+
}
|
|
1704
1802
|
});
|
|
1705
1803
|
}
|
|
1706
1804
|
function createRouter(env) {
|
|
@@ -1864,6 +1962,18 @@ function createRouter(env) {
|
|
|
1864
1962
|
case "runs.list":
|
|
1865
1963
|
result = await index.listRuns(...validated.args);
|
|
1866
1964
|
break;
|
|
1965
|
+
case "runs.list-resolved": {
|
|
1966
|
+
const runs = env.WORKFLOW_DB;
|
|
1967
|
+
if (!runs) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_DB");
|
|
1968
|
+
result = await listRunsPage({
|
|
1969
|
+
listRuns: (options) => index.listRuns(options),
|
|
1970
|
+
readRun: async (runId) => {
|
|
1971
|
+
const outcome = await runs.get(runs.idFromName(runId)).getRun();
|
|
1972
|
+
return outcome.ok ? outcome.value : null;
|
|
1973
|
+
}
|
|
1974
|
+
}, ...validated.args);
|
|
1975
|
+
break;
|
|
1976
|
+
}
|
|
1867
1977
|
case "runs.commit":
|
|
1868
1978
|
result = await index.commitRun(...validated.args);
|
|
1869
1979
|
break;
|
|
@@ -1920,7 +2030,7 @@ function createRouter(env) {
|
|
|
1920
2030
|
const readRequest = {
|
|
1921
2031
|
runId,
|
|
1922
2032
|
startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
|
|
1923
|
-
maxChunks: parseBoundedInteger(url, "maxChunks", 0,
|
|
2033
|
+
maxChunks: parseBoundedInteger(url, "maxChunks", 0, 512),
|
|
1924
2034
|
maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
|
|
1925
2035
|
waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
|
|
1926
2036
|
};
|
|
@@ -1984,6 +2094,7 @@ function createRouter(env) {
|
|
|
1984
2094
|
}
|
|
1985
2095
|
try {
|
|
1986
2096
|
const result = await namespace.get(namespace.idFromName(name))[method](...args);
|
|
2097
|
+
if (bindingKey === "runs" && method === "applyEvent") await publishApplyEventIndexes(env, name, result);
|
|
1987
2098
|
return new Response(rpcStringify(result ?? null), {
|
|
1988
2099
|
status: 200,
|
|
1989
2100
|
headers: { "content-type": "application/json" }
|
|
@@ -2413,9 +2524,18 @@ var RunCatalogDO = class extends DurableObject {
|
|
|
2413
2524
|
async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
|
|
2414
2525
|
const canonicalKeys = canonicalCatalogKeys(runId, keys);
|
|
2415
2526
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2416
|
-
|
|
2527
|
+
const recordKey = catalogKeysKey(runId);
|
|
2528
|
+
const existing = await txn.get([
|
|
2529
|
+
expiredKey(runId),
|
|
2530
|
+
canonicalKeys[0],
|
|
2531
|
+
canonicalKeys[1],
|
|
2532
|
+
recordKey
|
|
2533
|
+
]);
|
|
2534
|
+
if (existing.get(expiredKey(runId)) !== void 0) return { stored: false };
|
|
2417
2535
|
const now = this.now();
|
|
2418
2536
|
if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
|
|
2537
|
+
const record = existing.get(recordKey);
|
|
2538
|
+
if (existing.get(canonicalKeys[0]) === serializedMetadata && existing.get(canonicalKeys[1]) === serializedMetadata && record?.keys.length === canonicalKeys.length && record.keys.every((key, index) => key === canonicalKeys[index])) return { stored: true };
|
|
2419
2539
|
await txn.put({
|
|
2420
2540
|
[canonicalKeys[0]]: serializedMetadata,
|
|
2421
2541
|
[canonicalKeys[1]]: serializedMetadata,
|
|
@@ -2527,13 +2647,16 @@ const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
|
|
|
2527
2647
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
2528
2648
|
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
2529
2649
|
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
2530
|
-
/** Each chunk occupies a payload key and a size key
|
|
2650
|
+
/** Each chunk occupies a payload key and a size key, deleted in calls of at most 128 keys. */
|
|
2531
2651
|
const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
|
|
2532
|
-
const MAX_EXPIRE_CHUNK_LIMIT =
|
|
2652
|
+
const MAX_EXPIRE_CHUNK_LIMIT = 256;
|
|
2653
|
+
const DELETE_BATCH_KEYS = 128;
|
|
2533
2654
|
/** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
|
|
2534
2655
|
const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
|
|
2535
2656
|
const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
|
|
2536
2657
|
const MAX_STREAM_INDEX = 2147483647;
|
|
2658
|
+
/** One storage get or put call accepts at most 128 keys. */
|
|
2659
|
+
const STORAGE_BATCH_KEYS = 128;
|
|
2537
2660
|
function boundedLimit(value, fallback, maximum) {
|
|
2538
2661
|
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
2539
2662
|
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
@@ -2599,6 +2722,15 @@ var StreamDO = class extends DurableObject {
|
|
|
2599
2722
|
mutationTail = Promise.resolve();
|
|
2600
2723
|
changeVersion = 0;
|
|
2601
2724
|
waiters = /* @__PURE__ */ new Set();
|
|
2725
|
+
/** Multi-key read split into calls within the per-call key limit. */
|
|
2726
|
+
async getMany(keys) {
|
|
2727
|
+
if (keys.length <= STORAGE_BATCH_KEYS) return await this.ctx.storage.get(keys);
|
|
2728
|
+
const groups = [];
|
|
2729
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_KEYS) groups.push(keys.slice(offset, offset + STORAGE_BATCH_KEYS));
|
|
2730
|
+
const merged = /* @__PURE__ */ new Map();
|
|
2731
|
+
for (const result of await Promise.all(groups.map((group) => this.ctx.storage.get(group)))) for (const [key, value] of result) merged.set(key, value);
|
|
2732
|
+
return merged;
|
|
2733
|
+
}
|
|
2602
2734
|
async getMeta() {
|
|
2603
2735
|
if (this.meta) return this.meta;
|
|
2604
2736
|
this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
|
|
@@ -2671,7 +2803,7 @@ var StreamDO = class extends DurableObject {
|
|
|
2671
2803
|
if (available > 0) {
|
|
2672
2804
|
const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
|
|
2673
2805
|
const sizeKeys = indexes.map(chunkSizeKey);
|
|
2674
|
-
const storedSizes = await this.
|
|
2806
|
+
const storedSizes = await this.getMany(sizeKeys);
|
|
2675
2807
|
const selectedIndexes = [];
|
|
2676
2808
|
const selectedSizes = [];
|
|
2677
2809
|
let bytes = 0;
|
|
@@ -2683,7 +2815,7 @@ var StreamDO = class extends DurableObject {
|
|
|
2683
2815
|
bytes += size;
|
|
2684
2816
|
}
|
|
2685
2817
|
const payloadKeys = selectedIndexes.map(chunkKey);
|
|
2686
|
-
const storedChunks = await this.
|
|
2818
|
+
const storedChunks = await this.getMany(payloadKeys);
|
|
2687
2819
|
for (let offset = 0; offset < payloadKeys.length; offset++) {
|
|
2688
2820
|
const chunk = storedChunks.get(payloadKeys[offset]);
|
|
2689
2821
|
if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
|
|
@@ -2715,13 +2847,13 @@ var StreamDO = class extends DurableObject {
|
|
|
2715
2847
|
state: "open",
|
|
2716
2848
|
ownerRunId: runId
|
|
2717
2849
|
};
|
|
2718
|
-
const entries =
|
|
2850
|
+
const entries = [[META_KEY, nextMeta]];
|
|
2719
2851
|
for (let offset = 0; offset < storedChunks.length; offset++) {
|
|
2720
2852
|
const index = startIndex + offset;
|
|
2721
|
-
entries[chunkKey(index)
|
|
2722
|
-
entries[chunkSizeKey(index)
|
|
2853
|
+
entries.push([chunkKey(index), storedChunks[offset]]);
|
|
2854
|
+
entries.push([chunkSizeKey(index), storedChunks[offset].byteLength]);
|
|
2723
2855
|
}
|
|
2724
|
-
await txn.put(entries);
|
|
2856
|
+
for (let offset = 0; offset < entries.length; offset += STORAGE_BATCH_KEYS) await txn.put(Object.fromEntries(entries.slice(offset, offset + STORAGE_BATCH_KEYS)));
|
|
2725
2857
|
return {
|
|
2726
2858
|
meta: nextMeta,
|
|
2727
2859
|
result: {
|
|
@@ -2901,7 +3033,7 @@ var StreamDO = class extends DurableObject {
|
|
|
2901
3033
|
}
|
|
2902
3034
|
}
|
|
2903
3035
|
const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
|
|
2904
|
-
|
|
3036
|
+
for (let offset = 0; offset < keys.length; offset += DELETE_BATCH_KEYS) await txn.delete(keys.slice(offset, offset + DELETE_BATCH_KEYS));
|
|
2905
3037
|
let done = candidates.size === page.length;
|
|
2906
3038
|
if (payloadFallback) done = (await txn.list({
|
|
2907
3039
|
prefix: CHUNK_KEY_PREFIX,
|
|
@@ -3005,13 +3137,15 @@ async function deleteQueuePayloadObjects(binding, keys) {
|
|
|
3005
3137
|
}
|
|
3006
3138
|
//#endregion
|
|
3007
3139
|
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
3140
|
+
const EVENT_SEQUENCE_KEY = "event_sequence";
|
|
3008
3141
|
const STORAGE_BATCH_SIZE = 128;
|
|
3009
3142
|
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
3010
3143
|
const HOOK_MARKER_PAGE_SIZE = 64;
|
|
3011
3144
|
const TERMINAL_HOOK_PAGE_SIZE = 64;
|
|
3012
3145
|
const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
3013
3146
|
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
3014
|
-
|
|
3147
|
+
/** StreamDO deletes a page in 128-key calls within one transaction. */
|
|
3148
|
+
const STREAM_CHUNKS_PER_CLEANUP_PAGE = 256;
|
|
3015
3149
|
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
3016
3150
|
const QUEUE_PAYLOADS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3017
3151
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
@@ -3093,11 +3227,14 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3093
3227
|
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
3094
3228
|
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
3095
3229
|
}
|
|
3096
|
-
async retentionState(storage, now = this.now()) {
|
|
3097
|
-
const
|
|
3230
|
+
async retentionState(storage, now = this.now(), options) {
|
|
3231
|
+
const keys = [TOMBSTONE_KEY, CLEANUP_RECORD_KEY];
|
|
3232
|
+
if (options?.withEventSequence) keys.push(EVENT_SEQUENCE_KEY);
|
|
3233
|
+
const values = await storage.get(keys);
|
|
3098
3234
|
return {
|
|
3099
3235
|
cleanup: values.get(CLEANUP_RECORD_KEY),
|
|
3100
|
-
tombstone: this.tombstoneFrom(values, now)
|
|
3236
|
+
tombstone: this.tombstoneFrom(values, now),
|
|
3237
|
+
eventSequence: values.get(EVENT_SEQUENCE_KEY) ?? 0
|
|
3101
3238
|
};
|
|
3102
3239
|
}
|
|
3103
3240
|
async read(reader) {
|
|
@@ -3132,21 +3269,22 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3132
3269
|
if (request.cleanup !== void 0) validateCleanupRequest(request.cleanup, true);
|
|
3133
3270
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
3134
3271
|
const now = new Date(this.now());
|
|
3135
|
-
const retention = await this.retentionState(txn, now.getTime());
|
|
3272
|
+
const retention = await this.retentionState(txn, now.getTime(), { withEventSequence: true });
|
|
3136
3273
|
const { tombstone } = retention;
|
|
3137
3274
|
if (tombstone) return {
|
|
3138
3275
|
ok: false,
|
|
3139
3276
|
code: "RUN_EXPIRED",
|
|
3140
3277
|
message: `Workflow run "${request.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
3141
3278
|
};
|
|
3142
|
-
|
|
3279
|
+
const storedSequence = retention.eventSequence;
|
|
3280
|
+
let eventSequence = storedSequence;
|
|
3143
3281
|
const outcome = await applyEvent(storeFrom(txn), {
|
|
3144
3282
|
...request,
|
|
3145
3283
|
nextEventId: () => slotToEventId(++eventSequence),
|
|
3146
3284
|
now
|
|
3147
3285
|
});
|
|
3148
3286
|
if (!outcome.ok) return outcome;
|
|
3149
|
-
await txn.put(
|
|
3287
|
+
if (eventSequence !== storedSequence) await txn.put(EVENT_SEQUENCE_KEY, eventSequence);
|
|
3150
3288
|
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
3151
3289
|
hookId: outcome.hookToIndex.hookId,
|
|
3152
3290
|
token: outcome.hookToIndex.token
|
|
@@ -3526,10 +3664,58 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3526
3664
|
if (cleanup.phase === "payload") await this.deletePayloadBatch();
|
|
3527
3665
|
}
|
|
3528
3666
|
async executeTerminalCleanup(cleanup) {
|
|
3667
|
+
if (cleanup.phase === "hooks" && await this.completeSmallTerminalCleanup(cleanup)) return;
|
|
3529
3668
|
if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
|
|
3530
3669
|
else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
|
|
3531
3670
|
else await this.deleteTerminalWaitPage(cleanup);
|
|
3532
3671
|
}
|
|
3672
|
+
/**
|
|
3673
|
+
* Most runs end with a handful of hooks and waits. When every terminal
|
|
3674
|
+
* phase fits in one page, release the hook indexes once and delete hooks,
|
|
3675
|
+
* markers, and waits in a single transaction instead of one alarm and one
|
|
3676
|
+
* transaction per phase. Returns false, having changed nothing locally, when
|
|
3677
|
+
* any phase needs paging; the paged phases then run as before.
|
|
3678
|
+
*/
|
|
3679
|
+
async completeSmallTerminalCleanup(cleanup) {
|
|
3680
|
+
const [hookEntries, markerEntries] = await Promise.all([this.ctx.storage.list({
|
|
3681
|
+
prefix: HOOK_KEY_PREFIX,
|
|
3682
|
+
limit: 65
|
|
3683
|
+
}), this.ctx.storage.list({
|
|
3684
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3685
|
+
limit: 65
|
|
3686
|
+
})]);
|
|
3687
|
+
if (hookEntries.size > TERMINAL_HOOK_PAGE_SIZE || markerEntries.size > HOOK_MARKER_PAGE_SIZE) return false;
|
|
3688
|
+
const hooks = Array.from(hookEntries.values());
|
|
3689
|
+
const references = /* @__PURE__ */ new Map();
|
|
3690
|
+
for (const reference of [...hooks.map((hook) => ({
|
|
3691
|
+
hookId: hook.hookId,
|
|
3692
|
+
token: hook.token
|
|
3693
|
+
})), ...markerEntries.values()]) references.set(`${reference.hookId}\u0000${reference.token}`, reference);
|
|
3694
|
+
if (references.size > 0) await this.workflowIndex().releaseHookIndexes({
|
|
3695
|
+
runId: cleanup.runId,
|
|
3696
|
+
hooks: Array.from(references.values())
|
|
3697
|
+
});
|
|
3698
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3699
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3700
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return true;
|
|
3701
|
+
const waits = await txn.list({
|
|
3702
|
+
prefix: WAIT_KEY_PREFIX,
|
|
3703
|
+
limit: 129
|
|
3704
|
+
});
|
|
3705
|
+
if (waits.size > TERMINAL_WAIT_PAGE_SIZE) return false;
|
|
3706
|
+
const keys = [
|
|
3707
|
+
...hooks.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]),
|
|
3708
|
+
...markerEntries.keys(),
|
|
3709
|
+
...waits.keys()
|
|
3710
|
+
];
|
|
3711
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE) await txn.delete(keys.slice(offset, offset + STORAGE_BATCH_SIZE));
|
|
3712
|
+
await txn.delete(TERMINAL_CLEANUP_KEY);
|
|
3713
|
+
const retention = await txn.get(CLEANUP_RECORD_KEY);
|
|
3714
|
+
if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
|
|
3715
|
+
else await txn.deleteAlarm();
|
|
3716
|
+
return true;
|
|
3717
|
+
});
|
|
3718
|
+
}
|
|
3533
3719
|
async deleteTerminalHookPage(cleanup) {
|
|
3534
3720
|
const entries = await this.ctx.storage.list({
|
|
3535
3721
|
prefix: HOOK_KEY_PREFIX,
|