@ewhauser/world-celld 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/NOTICE +1 -1
- package/README.md +117 -106
- package/celld-queue-worker/README.md +26 -0
- package/celld-queue-worker/worker.ts +3 -0
- package/celld-queue-worker/wrangler.jsonc +26 -0
- package/celld-worker/README.md +33 -25
- package/celld-worker/worker.ts +1 -1
- package/celld-worker/wrangler.jsonc +12 -8
- package/dist/index.d.ts +32 -70
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +33 -90
- package/dist/index.js.map +1 -1
- package/dist/queue-consumer.d.ts +33 -0
- package/dist/queue-consumer.d.ts.map +1 -0
- package/dist/queue-consumer.js +75 -0
- package/dist/queue-consumer.js.map +1 -0
- package/dist/queue-delivery-DzNOtCYA.d.ts +49 -0
- package/dist/queue-delivery-DzNOtCYA.d.ts.map +1 -0
- package/dist/{util-Cu09mVzK.js → queue-protocol-CmJNMH_5.js} +60 -12
- package/dist/queue-protocol-CmJNMH_5.js.map +1 -0
- package/dist/queue-protocol-Ds44uuND.js +104 -0
- package/dist/queue-protocol-Ds44uuND.js.map +1 -0
- package/dist/testing.d.ts +3 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +420 -833
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +106 -181
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +602 -955
- package/dist/worker.js.map +1 -1
- package/docs/celld-v0.5-upgrade.md +130 -0
- package/package.json +14 -6
- package/dist/util-Cu09mVzK.js.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { CreateEventSchema, EventSchema, HookSchema,
|
|
3
|
-
import { WorkflowWorldError } from "@workflow/errors";
|
|
1
|
+
import { A as MAX_STREAM_BATCH_BYTES, D as isRecord$1, E as isPositiveSafeInteger, G as validateStreamReadRequest, K as validateStreamWriteChunks, L as STREAM_BATCH_CONTENT_TYPE, N as MAX_STREAM_LONG_POLL_MS, P as MAX_STREAM_READ_BYTES, S as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, T as isNonNegativeSafeInteger, U as encodeStreamWriteResult, V as encodeStreamReadResult, W as normalizeStreamError, _ as HOOK_CLAIM_LEASE_MS, a as queueOrphanName, c as validateNativeQueueEnvelope, d as rpcParse, f as rpcStringify, h as createWorkflowIndex, i as queueClaimName, j as MAX_STREAM_CHUNK_BYTES, l as validateNativeQueueSendOptions, n as QUEUE_PAYLOAD_REGISTRY_PREFIX, o as queuePayloadObjectKey, p as parse, s as queuePayloadRegistryKey, t as NATIVE_QUEUE_MAX_MESSAGE_BYTES, u as compact, v as LIFECYCLE_COMPACTION_RETRY_MS, z as decodeStreamWriteBatch } from "./queue-protocol-CmJNMH_5.js";
|
|
2
|
+
import { CreateEventSchema, EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
4
3
|
import http from "node:http";
|
|
5
4
|
//#region src/testing/fake-cell.ts
|
|
6
5
|
function applyListOptions(keys, options) {
|
|
@@ -458,13 +457,13 @@ const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
|
|
|
458
457
|
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
459
458
|
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
460
459
|
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
461
|
-
const STORAGE_BATCH_SIZE$
|
|
460
|
+
const STORAGE_BATCH_SIZE$1 = 128;
|
|
462
461
|
const MAX_ENTITY_PAGE_SIZE = 1e3;
|
|
463
|
-
async function getMany
|
|
462
|
+
async function getMany(store, keys) {
|
|
464
463
|
if (store.getMany) {
|
|
465
464
|
const result = /* @__PURE__ */ new Map();
|
|
466
|
-
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$
|
|
467
|
-
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$
|
|
465
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
466
|
+
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
468
467
|
for (const entry of page) result.set(...entry);
|
|
469
468
|
}
|
|
470
469
|
return result;
|
|
@@ -472,10 +471,10 @@ async function getMany$1(store, keys) {
|
|
|
472
471
|
const values = await Promise.all(keys.map((key) => store.get(key)));
|
|
473
472
|
return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
|
|
474
473
|
}
|
|
475
|
-
async function deleteMany
|
|
474
|
+
async function deleteMany(store, keys) {
|
|
476
475
|
if (store.deleteMany) {
|
|
477
476
|
let deleted = 0;
|
|
478
|
-
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$
|
|
477
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
479
478
|
return deleted;
|
|
480
479
|
}
|
|
481
480
|
return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
|
|
@@ -604,7 +603,7 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
604
603
|
});
|
|
605
604
|
const page = Array.from(entries.entries()).slice(0, limit);
|
|
606
605
|
const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
|
|
607
|
-
const entities = await getMany
|
|
606
|
+
const entities = await getMany(store, entityKeys);
|
|
608
607
|
const data = entityKeys.flatMap((key) => {
|
|
609
608
|
const item = entities.get(key);
|
|
610
609
|
return item === void 0 ? [] : [item];
|
|
@@ -631,14 +630,14 @@ async function releaseAllHooks(store) {
|
|
|
631
630
|
token: hook.token
|
|
632
631
|
});
|
|
633
632
|
}
|
|
634
|
-
await deleteMany
|
|
633
|
+
await deleteMany(store, keys);
|
|
635
634
|
return released;
|
|
636
635
|
}
|
|
637
636
|
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
638
637
|
async function releaseAllWaits(store) {
|
|
639
638
|
if (store.deferTerminalCleanup) return;
|
|
640
639
|
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
641
|
-
await deleteMany
|
|
640
|
+
await deleteMany(store, Array.from(waits.keys()));
|
|
642
641
|
}
|
|
643
642
|
async function applyEvent(store, ctx) {
|
|
644
643
|
const { runId, data, now } = ctx;
|
|
@@ -1580,24 +1579,12 @@ const BINDINGS = {
|
|
|
1580
1579
|
"hook-ids": {
|
|
1581
1580
|
env: "WORKFLOW_HOOK_IDS",
|
|
1582
1581
|
methods: /* @__PURE__ */ new Set(["get"])
|
|
1583
|
-
},
|
|
1584
|
-
queue: {
|
|
1585
|
-
env: "WORKFLOW_QUEUE",
|
|
1586
|
-
methods: /* @__PURE__ */ new Set([
|
|
1587
|
-
"enqueue",
|
|
1588
|
-
"stats",
|
|
1589
|
-
"listDeadLetters",
|
|
1590
|
-
"redriveDeadLetter",
|
|
1591
|
-
"purgeDeadLetters",
|
|
1592
|
-
"rearmAlarm",
|
|
1593
|
-
"expireRun",
|
|
1594
|
-
"acknowledgeExpireRun"
|
|
1595
|
-
])
|
|
1596
1582
|
}
|
|
1597
1583
|
};
|
|
1598
1584
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
1599
1585
|
const MAX_BODY_BYTES = 33554432;
|
|
1600
1586
|
const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 1024;
|
|
1587
|
+
const QUEUE_ORPHAN_GRACE_MS = 432e6;
|
|
1601
1588
|
function errorResponse(status, name, message) {
|
|
1602
1589
|
return Response.json({ error: {
|
|
1603
1590
|
name,
|
|
@@ -1681,6 +1668,25 @@ async function readBoundedBytes(request, maxBytes) {
|
|
|
1681
1668
|
}
|
|
1682
1669
|
return body;
|
|
1683
1670
|
}
|
|
1671
|
+
async function parseRpcArguments(request) {
|
|
1672
|
+
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
1673
|
+
if (!hasContentType(request, "application/json")) return errorResponse(415, "UnsupportedMediaType", "expected application/json");
|
|
1674
|
+
let contentLength;
|
|
1675
|
+
try {
|
|
1676
|
+
contentLength = declaredContentLength(request);
|
|
1677
|
+
} catch (error) {
|
|
1678
|
+
return errorResponse(400, "BadRequest", error.message);
|
|
1679
|
+
}
|
|
1680
|
+
if (contentLength !== null && contentLength > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
1681
|
+
try {
|
|
1682
|
+
const text = await readBoundedBody(request);
|
|
1683
|
+
if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
1684
|
+
const parsed = rpcParse(text);
|
|
1685
|
+
return Array.isArray(parsed) ? parsed : errorResponse(400, "BadRequest", "body must be an argument array");
|
|
1686
|
+
} catch {
|
|
1687
|
+
return errorResponse(400, "BadRequest", "malformed rpc body");
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1684
1690
|
function parseBoundedInteger(url, name, minimum, maximum) {
|
|
1685
1691
|
const raw = url.searchParams.get(name);
|
|
1686
1692
|
const value = raw !== null && /^\d+$/.test(raw) ? Number(raw) : NaN;
|
|
@@ -1713,6 +1719,111 @@ function createRouter(env) {
|
|
|
1713
1719
|
}
|
|
1714
1720
|
const auth = await authenticate(request, env.WORLD_SECRET);
|
|
1715
1721
|
if (!auth.ok) return auth.response;
|
|
1722
|
+
if (parts[1] === "queue" && parts.length === 3) {
|
|
1723
|
+
const parsed = await parseRpcArguments(request);
|
|
1724
|
+
if (parsed instanceof Response) return parsed;
|
|
1725
|
+
if (parts[2] === "send") {
|
|
1726
|
+
if (parsed.length < 1 || parsed.length > 2) return errorResponse(400, "BadRequest", "queue send expects an envelope and options");
|
|
1727
|
+
if (!env.WORKFLOW_QUEUE) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_QUEUE");
|
|
1728
|
+
let envelope;
|
|
1729
|
+
let options;
|
|
1730
|
+
try {
|
|
1731
|
+
envelope = validateNativeQueueEnvelope(parsed[0]);
|
|
1732
|
+
options = validateNativeQueueSendOptions(parsed[1]);
|
|
1733
|
+
if (envelope.body === void 0 || envelope.payloadKey !== void 0) throw new TypeError("queue send requires an inline body");
|
|
1734
|
+
} catch (error) {
|
|
1735
|
+
return errorResponse(400, "BadRequest", error.message);
|
|
1736
|
+
}
|
|
1737
|
+
try {
|
|
1738
|
+
const namespace = env.WORKFLOW_DB;
|
|
1739
|
+
const payloadStore = env.WORKFLOW_QUEUE_PAYLOADS;
|
|
1740
|
+
const payloadKey = envelope.runId ? queuePayloadObjectKey(envelope.runId, envelope.messageId) : void 0;
|
|
1741
|
+
const brokerEnvelope = payloadKey ? {
|
|
1742
|
+
...envelope,
|
|
1743
|
+
payloadKey,
|
|
1744
|
+
body: void 0
|
|
1745
|
+
} : envelope;
|
|
1746
|
+
const encoded = JSON.stringify(brokerEnvelope);
|
|
1747
|
+
if (new TextEncoder().encode(encoded).byteLength > 128e3) return errorResponse(413, "PayloadTooLarge", `native queue envelope exceeds ${NATIVE_QUEUE_MAX_MESSAGE_BYTES} bytes`);
|
|
1748
|
+
const now = Date.now();
|
|
1749
|
+
const reservationExpiresAt = Math.max(now, envelope.notBefore ?? now) + QUEUE_ORPHAN_GRACE_MS;
|
|
1750
|
+
let claim;
|
|
1751
|
+
if (envelope.idempotencyKey) {
|
|
1752
|
+
claim = namespace.get(namespace.idFromName(queueClaimName(envelope.queueName, envelope.idempotencyKey)));
|
|
1753
|
+
const reservation = await claim.reserveQueueMessage({
|
|
1754
|
+
messageId: envelope.messageId,
|
|
1755
|
+
expiresAt: reservationExpiresAt
|
|
1756
|
+
});
|
|
1757
|
+
if (!reservation.admitted) return new Response(rpcStringify({ messageId: reservation.messageId }), {
|
|
1758
|
+
status: 200,
|
|
1759
|
+
headers: { "content-type": "application/json" }
|
|
1760
|
+
});
|
|
1761
|
+
}
|
|
1762
|
+
if (envelope.runId && payloadKey) {
|
|
1763
|
+
if (!payloadStore) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_QUEUE_PAYLOADS");
|
|
1764
|
+
const run = namespace.get(namespace.idFromName(envelope.runId));
|
|
1765
|
+
const orphanExpiresAt = reservationExpiresAt;
|
|
1766
|
+
const registered = await run.registerQueuePayload({
|
|
1767
|
+
messageId: envelope.messageId,
|
|
1768
|
+
key: payloadKey,
|
|
1769
|
+
orphanExpiresAt
|
|
1770
|
+
});
|
|
1771
|
+
if (!registered.ok) {
|
|
1772
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1773
|
+
return errorResponse(410, "RunExpiredError", registered.message);
|
|
1774
|
+
}
|
|
1775
|
+
const orphan = namespace.get(namespace.idFromName(queueOrphanName(envelope.messageId)));
|
|
1776
|
+
try {
|
|
1777
|
+
await orphan.scheduleQueuePayloadOrphan({
|
|
1778
|
+
messageId: envelope.messageId,
|
|
1779
|
+
runId: envelope.runId,
|
|
1780
|
+
key: payloadKey,
|
|
1781
|
+
expiresAt: orphanExpiresAt
|
|
1782
|
+
});
|
|
1783
|
+
} catch (error) {
|
|
1784
|
+
await run.unregisterQueuePayload(envelope.messageId);
|
|
1785
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1786
|
+
throw error;
|
|
1787
|
+
}
|
|
1788
|
+
try {
|
|
1789
|
+
await payloadStore.write(payloadKey, envelope.body, {
|
|
1790
|
+
runId: envelope.runId,
|
|
1791
|
+
messageId: envelope.messageId
|
|
1792
|
+
});
|
|
1793
|
+
} catch (error) {
|
|
1794
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1795
|
+
throw error;
|
|
1796
|
+
}
|
|
1797
|
+
let finalized;
|
|
1798
|
+
try {
|
|
1799
|
+
finalized = await run.finalizeQueuePayload(envelope.messageId);
|
|
1800
|
+
} catch (error) {
|
|
1801
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1802
|
+
throw error;
|
|
1803
|
+
}
|
|
1804
|
+
if (!finalized.ok) {
|
|
1805
|
+
await payloadStore.delete(payloadKey);
|
|
1806
|
+
await run.unregisterQueuePayload(envelope.messageId);
|
|
1807
|
+
await orphan.cancelQueuePayloadOrphan(envelope.messageId);
|
|
1808
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1809
|
+
return errorResponse(410, "RunExpiredError", finalized.message);
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
await env.WORKFLOW_QUEUE.send(encoded, {
|
|
1813
|
+
contentType: "text",
|
|
1814
|
+
...options
|
|
1815
|
+
});
|
|
1816
|
+
return new Response(rpcStringify({ messageId: envelope.messageId }), {
|
|
1817
|
+
status: 200,
|
|
1818
|
+
headers: { "content-type": "application/json" }
|
|
1819
|
+
});
|
|
1820
|
+
} catch (error) {
|
|
1821
|
+
const err = error;
|
|
1822
|
+
return errorResponse(500, err.name ?? "Error", err.message ?? String(error));
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
return errorResponse(404, "NotFound", `unknown queue operation: ${parts[2]}`);
|
|
1826
|
+
}
|
|
1716
1827
|
if (parts[1] === "index" && parts.length === 4) {
|
|
1717
1828
|
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
1718
1829
|
if (!hasContentType(request, "application/json")) return errorResponse(415, "UnsupportedMediaType", "expected application/json");
|
|
@@ -1930,7 +2041,7 @@ async function runIsActive(env, runId) {
|
|
|
1930
2041
|
//#region src/worker/durable-objects/HookIdDO.ts
|
|
1931
2042
|
const MAX_RELEASE_BATCH$1 = 64;
|
|
1932
2043
|
const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
|
|
1933
|
-
function pad$
|
|
2044
|
+
function pad$2(ms) {
|
|
1934
2045
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1935
2046
|
}
|
|
1936
2047
|
function recordKey$1(hookId) {
|
|
@@ -1940,7 +2051,7 @@ function claimKey$1(hookId) {
|
|
|
1940
2051
|
return indexKey("claim", hookId);
|
|
1941
2052
|
}
|
|
1942
2053
|
function claimDeadlineKey$1(hookId, expiresAt) {
|
|
1943
|
-
return `${CLAIM_DEADLINE_PREFIX$1}${pad$
|
|
2054
|
+
return `${CLAIM_DEADLINE_PREFIX$1}${pad$2(expiresAt)}:${encodeURIComponent(hookId)}`;
|
|
1944
2055
|
}
|
|
1945
2056
|
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
1946
2057
|
var HookIdDO = class extends DurableObject {
|
|
@@ -2074,7 +2185,7 @@ var HookIdDO = class extends DurableObject {
|
|
|
2074
2185
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2075
2186
|
const due = await txn.list({
|
|
2076
2187
|
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
2077
|
-
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$
|
|
2188
|
+
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$2(now + 1)}`,
|
|
2078
2189
|
limit: 129
|
|
2079
2190
|
});
|
|
2080
2191
|
const page = Array.from(due).slice(0, 128);
|
|
@@ -2102,7 +2213,7 @@ var HookIdDO = class extends DurableObject {
|
|
|
2102
2213
|
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
2103
2214
|
const MAX_RELEASE_BATCH = 64;
|
|
2104
2215
|
const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
|
|
2105
|
-
function pad$
|
|
2216
|
+
function pad$1(ms) {
|
|
2106
2217
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
2107
2218
|
}
|
|
2108
2219
|
function recordKey(token) {
|
|
@@ -2112,7 +2223,7 @@ function claimKey(token) {
|
|
|
2112
2223
|
return indexKey("claim", token);
|
|
2113
2224
|
}
|
|
2114
2225
|
function claimDeadlineKey(token, expiresAt) {
|
|
2115
|
-
return `${CLAIM_DEADLINE_PREFIX}${pad$
|
|
2226
|
+
return `${CLAIM_DEADLINE_PREFIX}${pad$1(expiresAt)}:${encodeURIComponent(token)}`;
|
|
2116
2227
|
}
|
|
2117
2228
|
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
2118
2229
|
var HookTokenDO = class extends DurableObject {
|
|
@@ -2245,7 +2356,7 @@ var HookTokenDO = class extends DurableObject {
|
|
|
2245
2356
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2246
2357
|
const due = await txn.list({
|
|
2247
2358
|
prefix: CLAIM_DEADLINE_PREFIX,
|
|
2248
|
-
end: `${CLAIM_DEADLINE_PREFIX}${pad$
|
|
2359
|
+
end: `${CLAIM_DEADLINE_PREFIX}${pad$1(now + 1)}`,
|
|
2249
2360
|
limit: 129
|
|
2250
2361
|
});
|
|
2251
2362
|
const page = Array.from(due).slice(0, 128);
|
|
@@ -2270,705 +2381,6 @@ var HookTokenDO = class extends DurableObject {
|
|
|
2270
2381
|
}
|
|
2271
2382
|
};
|
|
2272
2383
|
//#endregion
|
|
2273
|
-
//#region src/worker/durable-objects/QueueDO.ts
|
|
2274
|
-
const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
2275
|
-
404,
|
|
2276
|
-
409,
|
|
2277
|
-
410,
|
|
2278
|
-
422
|
|
2279
|
-
]);
|
|
2280
|
-
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
2281
|
-
const DEFAULT_MAX_INFLIGHT = 5;
|
|
2282
|
-
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
2283
|
-
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
2284
|
-
const MIN_ALARM_DELAY_MS = 1;
|
|
2285
|
-
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
2286
|
-
const EXPIRED_INFLIGHT_BATCH = 128;
|
|
2287
|
-
const STORAGE_BATCH_SIZE$1 = 128;
|
|
2288
|
-
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
2289
|
-
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
2290
|
-
const MAX_DEAD_LETTER_LIST_SIZE = 1e3;
|
|
2291
|
-
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
2292
|
-
const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
|
|
2293
|
-
function pad$1(ms) {
|
|
2294
|
-
if (!isNonNegativeSafeInteger(ms) || ms > 9999999999999) throw new Error("queue timestamp must be a non-negative 13-digit integer");
|
|
2295
|
-
return String(ms).padStart(13, "0");
|
|
2296
|
-
}
|
|
2297
|
-
function dueKey(atMs, messageId) {
|
|
2298
|
-
return `due:${pad$1(atMs)}:${messageId}`;
|
|
2299
|
-
}
|
|
2300
|
-
function inflightKey(deadlineMs, messageId) {
|
|
2301
|
-
return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
|
|
2302
|
-
}
|
|
2303
|
-
function deadlineFromInflightKey(key) {
|
|
2304
|
-
return Number.parseInt(key.slice(18, 31), 10);
|
|
2305
|
-
}
|
|
2306
|
-
function inclusiveTimestampEnd(prefix, now) {
|
|
2307
|
-
return now >= 9999999999999 ? `${prefix.slice(0, -1)};` : `${prefix}${pad$1(now + 1)}`;
|
|
2308
|
-
}
|
|
2309
|
-
function runReferenceKey(runId, messageId) {
|
|
2310
|
-
return `run:${runId}:${messageId}`;
|
|
2311
|
-
}
|
|
2312
|
-
function expiredRunKey(runId) {
|
|
2313
|
-
return `expired-run:${runId}`;
|
|
2314
|
-
}
|
|
2315
|
-
function expiredRunGcKey(runId, compactAt) {
|
|
2316
|
-
return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
|
|
2317
|
-
}
|
|
2318
|
-
function validateEnqueueRequest(request) {
|
|
2319
|
-
if (!isRecord$1(request)) throw new Error("world-celld queue enqueue request must be an object");
|
|
2320
|
-
for (const field of [
|
|
2321
|
-
"messageId",
|
|
2322
|
-
"queueName",
|
|
2323
|
-
"body"
|
|
2324
|
-
]) if (typeof request[field] !== "string" || request[field].length === 0) throw new Error(`world-celld queue enqueue ${field} must be a non-empty string`);
|
|
2325
|
-
if (request.pathname !== "flow") throw new Error("world-celld queue enqueue pathname must be flow");
|
|
2326
|
-
if (!isRecord$1(request.config) || typeof request.config.targetBaseUrl !== "string" || request.config.targetBaseUrl.length === 0) throw new Error("world-celld queue enqueue config is invalid");
|
|
2327
|
-
if (!isPositiveSafeInteger(request.config.queueShards)) throw new Error("world-celld queue enqueue queueShards must be a positive safe integer");
|
|
2328
|
-
if (request.delaySeconds !== void 0 && !isValidQueueDelaySeconds(request.delaySeconds)) throw new Error("world-celld queue enqueue delaySeconds is out of range");
|
|
2329
|
-
if (request.runId !== void 0 && (typeof request.runId !== "string" || request.runId.length === 0)) throw new Error("world-celld queue enqueue runId must be a non-empty string");
|
|
2330
|
-
if (request.idempotencyKey !== void 0 && (typeof request.idempotencyKey !== "string" || request.idempotencyKey.length === 0)) throw new Error("world-celld queue enqueue idempotencyKey must be a non-empty string");
|
|
2331
|
-
let body;
|
|
2332
|
-
try {
|
|
2333
|
-
body = parse(request.body);
|
|
2334
|
-
} catch {
|
|
2335
|
-
throw new WorkflowWorldError("world-celld queue enqueue body must be valid tagged JSON", { status: 422 });
|
|
2336
|
-
}
|
|
2337
|
-
const payload = QueuePayloadSchema.safeParse(body);
|
|
2338
|
-
if (!payload.success) throw new WorkflowWorldError("world-celld queue enqueue body is an invalid queue payload", { status: 422 });
|
|
2339
|
-
const payloadRunId = "runId" in payload.data && typeof payload.data.runId === "string" ? payload.data.runId : void 0;
|
|
2340
|
-
if (request.runId !== payloadRunId) throw new WorkflowWorldError("world-celld queue enqueue runId does not match body", { status: 422 });
|
|
2341
|
-
}
|
|
2342
|
-
async function getMany(storage, keys) {
|
|
2343
|
-
const result = /* @__PURE__ */ new Map();
|
|
2344
|
-
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
2345
|
-
const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2346
|
-
for (const [key, value] of page) result.set(key, value);
|
|
2347
|
-
}
|
|
2348
|
-
return result;
|
|
2349
|
-
}
|
|
2350
|
-
async function putMany(storage, entries) {
|
|
2351
|
-
const all = Array.from(entries);
|
|
2352
|
-
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$1) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$1)));
|
|
2353
|
-
}
|
|
2354
|
-
async function deleteMany(storage, keys) {
|
|
2355
|
-
let deleted = 0;
|
|
2356
|
-
const unique = Array.from(new Set(keys));
|
|
2357
|
-
for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$1) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2358
|
-
return deleted;
|
|
2359
|
-
}
|
|
2360
|
-
function backoffSeconds(attempt) {
|
|
2361
|
-
return Math.min(60, 2 ** attempt);
|
|
2362
|
-
}
|
|
2363
|
-
async function cancelUnusedResponseBody(response) {
|
|
2364
|
-
try {
|
|
2365
|
-
await response.body?.cancel();
|
|
2366
|
-
} catch {}
|
|
2367
|
-
}
|
|
2368
|
-
const MAX_QUEUE_INFLIGHT = 128;
|
|
2369
|
-
var QueueDO = class extends DurableObject {
|
|
2370
|
-
#now() {
|
|
2371
|
-
const clock = this.env?.clock;
|
|
2372
|
-
return typeof clock === "function" ? clock() : Date.now();
|
|
2373
|
-
}
|
|
2374
|
-
#fetch() {
|
|
2375
|
-
const impl = this.env?.fetch;
|
|
2376
|
-
return typeof impl === "function" ? impl : fetch;
|
|
2377
|
-
}
|
|
2378
|
-
#intVar(name, fallback) {
|
|
2379
|
-
const raw = this.env?.[name];
|
|
2380
|
-
return strictIntegerSetting(name, raw, fallback, 1);
|
|
2381
|
-
}
|
|
2382
|
-
#maxAttempts() {
|
|
2383
|
-
return this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
2384
|
-
}
|
|
2385
|
-
#maxInflight() {
|
|
2386
|
-
const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
|
|
2387
|
-
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
2388
|
-
return configured;
|
|
2389
|
-
}
|
|
2390
|
-
#deliveryTimeoutMs() {
|
|
2391
|
-
const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
2392
|
-
if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
|
|
2393
|
-
return configured;
|
|
2394
|
-
}
|
|
2395
|
-
#validateRuntimeConfig() {
|
|
2396
|
-
this.#maxAttempts();
|
|
2397
|
-
this.#maxInflight();
|
|
2398
|
-
this.#deliveryTimeoutMs();
|
|
2399
|
-
}
|
|
2400
|
-
async #authoritativeRunExpired(runId) {
|
|
2401
|
-
const namespace = this.env?.WORKFLOW_DB;
|
|
2402
|
-
if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
|
|
2403
|
-
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
|
|
2404
|
-
}
|
|
2405
|
-
async #withRunMutation(runId, onExpired, onAllowed) {
|
|
2406
|
-
const expired = await this.#authoritativeRunExpired(runId);
|
|
2407
|
-
return await this.ctx.storage.transaction(async (txn) => {
|
|
2408
|
-
if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
|
|
2409
|
-
return await onAllowed(txn);
|
|
2410
|
-
});
|
|
2411
|
-
}
|
|
2412
|
-
async enqueue(request) {
|
|
2413
|
-
this.#validateRuntimeConfig();
|
|
2414
|
-
validateEnqueueRequest(request);
|
|
2415
|
-
const storage = this.ctx.storage;
|
|
2416
|
-
const now = this.#now();
|
|
2417
|
-
const dueAt = queueDelayDeadline(now, request.delaySeconds ?? 0, 0, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2418
|
-
if (dueAt === null) throw new Error("world-celld queue enqueue delaySeconds lacks required delivery headroom");
|
|
2419
|
-
const persist = async (txn) => {
|
|
2420
|
-
const pinned = await txn.get("cfg");
|
|
2421
|
-
if (!pinned) await txn.put("cfg", {
|
|
2422
|
-
queueShards: request.config.queueShards,
|
|
2423
|
-
pinnedAt: now
|
|
2424
|
-
});
|
|
2425
|
-
else if (pinned.queueShards !== request.config.queueShards) return {
|
|
2426
|
-
ok: false,
|
|
2427
|
-
code: "CONFIG_MISMATCH",
|
|
2428
|
-
message: `queue cell shard-count mismatch: pinned ${pinned.queueShards}, got ${request.config.queueShards} — changing queueShards requires draining the fleet`
|
|
2429
|
-
};
|
|
2430
|
-
if (request.idempotencyKey) {
|
|
2431
|
-
const existing = await txn.get(`key:${request.idempotencyKey}`);
|
|
2432
|
-
if (existing) {
|
|
2433
|
-
if (await txn.get(`msg:${existing}`)) {
|
|
2434
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2435
|
-
return {
|
|
2436
|
-
ok: true,
|
|
2437
|
-
messageId: existing,
|
|
2438
|
-
deduped: true
|
|
2439
|
-
};
|
|
2440
|
-
}
|
|
2441
|
-
await txn.delete(`key:${request.idempotencyKey}`);
|
|
2442
|
-
}
|
|
2443
|
-
await txn.put(`key:${request.idempotencyKey}`, request.messageId);
|
|
2444
|
-
}
|
|
2445
|
-
const row = {
|
|
2446
|
-
messageId: request.messageId,
|
|
2447
|
-
queueName: request.queueName,
|
|
2448
|
-
pathname: request.pathname,
|
|
2449
|
-
body: request.body,
|
|
2450
|
-
runId: request.runId,
|
|
2451
|
-
targetBaseUrl: request.config.targetBaseUrl,
|
|
2452
|
-
idempotencyKey: request.idempotencyKey,
|
|
2453
|
-
attempt: 0,
|
|
2454
|
-
enqueuedAt: now
|
|
2455
|
-
};
|
|
2456
|
-
const scheduleKey = dueKey(dueAt, row.messageId);
|
|
2457
|
-
await txn.put(`msg:${row.messageId}`, row);
|
|
2458
|
-
await txn.put(scheduleKey, row.messageId);
|
|
2459
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2460
|
-
messageId: row.messageId,
|
|
2461
|
-
dueKey: scheduleKey,
|
|
2462
|
-
idempotencyKey: row.idempotencyKey
|
|
2463
|
-
});
|
|
2464
|
-
await this.#armAlarmAtMost(txn, dueAt, now);
|
|
2465
|
-
return {
|
|
2466
|
-
ok: true,
|
|
2467
|
-
messageId: row.messageId,
|
|
2468
|
-
deduped: false
|
|
2469
|
-
};
|
|
2470
|
-
};
|
|
2471
|
-
return request.runId ? await this.#withRunMutation(request.runId, async () => ({
|
|
2472
|
-
ok: false,
|
|
2473
|
-
code: "RUN_EXPIRED",
|
|
2474
|
-
message: `Workflow run "${request.runId}" has expired`
|
|
2475
|
-
}), persist) : await storage.transaction(persist);
|
|
2476
|
-
}
|
|
2477
|
-
async alarm() {
|
|
2478
|
-
this.#validateRuntimeConfig();
|
|
2479
|
-
let claimed = [];
|
|
2480
|
-
let failure = null;
|
|
2481
|
-
const attempt = async () => {
|
|
2482
|
-
try {
|
|
2483
|
-
claimed = await this.#claimPhase();
|
|
2484
|
-
} catch (error) {
|
|
2485
|
-
failure = error;
|
|
2486
|
-
}
|
|
2487
|
-
};
|
|
2488
|
-
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
2489
|
-
else await attempt();
|
|
2490
|
-
if (failure !== null) {
|
|
2491
|
-
const retryAt = checkedQueueTimestampAdd(this.#now(), LIFECYCLE_COMPACTION_RETRY_MS);
|
|
2492
|
-
if (retryAt === null) await this.ctx.storage.deleteAlarm();
|
|
2493
|
-
else await this.ctx.storage.setAlarm(retryAt);
|
|
2494
|
-
throw failure;
|
|
2495
|
-
}
|
|
2496
|
-
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
2497
|
-
}
|
|
2498
|
-
/**
|
|
2499
|
-
* Storage-only: recover lost inflight claims, claim due messages up to the
|
|
2500
|
-
* inflight cap, and re-arm the alarm. Never throws for data conditions.
|
|
2501
|
-
*/
|
|
2502
|
-
async #claimPhase() {
|
|
2503
|
-
const storage = this.ctx.storage;
|
|
2504
|
-
const now = this.#now();
|
|
2505
|
-
const maxInflight = this.#maxInflight();
|
|
2506
|
-
return await storage.transaction(async (txn) => {
|
|
2507
|
-
if (await this.#compactExpiredRunFences(txn, now)) {
|
|
2508
|
-
const nextAt = checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS);
|
|
2509
|
-
if (nextAt === null) await txn.deleteAlarm();
|
|
2510
|
-
else await txn.setAlarm(nextAt);
|
|
2511
|
-
return [];
|
|
2512
|
-
}
|
|
2513
|
-
const expiredPage = await txn.list({
|
|
2514
|
-
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
2515
|
-
end: inclusiveTimestampEnd(INFLIGHT_DEADLINE_PREFIX, now),
|
|
2516
|
-
limit: 129
|
|
2517
|
-
});
|
|
2518
|
-
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
2519
|
-
const expiredRows = await getMany(txn, expired.map(([, messageId]) => `msg:${messageId}`));
|
|
2520
|
-
await deleteMany(txn, expired.map(([key]) => key));
|
|
2521
|
-
const recoveryWrites = [];
|
|
2522
|
-
for (const [, messageId] of expired) {
|
|
2523
|
-
const row = expiredRows.get(`msg:${messageId}`);
|
|
2524
|
-
if (row) {
|
|
2525
|
-
const recoveryAt = checkedQueueTimestampAdd(now, 0, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2526
|
-
if (recoveryAt === null) {
|
|
2527
|
-
await this.#deadLetter(txn, row, row.attempt, "inflight recovery exceeded the queue timestamp horizon", now);
|
|
2528
|
-
continue;
|
|
2529
|
-
}
|
|
2530
|
-
const scheduleKey = dueKey(recoveryAt, messageId);
|
|
2531
|
-
recoveryWrites.push([scheduleKey, messageId]);
|
|
2532
|
-
if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
2533
|
-
messageId: row.messageId,
|
|
2534
|
-
dueKey: scheduleKey,
|
|
2535
|
-
idempotencyKey: row.idempotencyKey
|
|
2536
|
-
}]);
|
|
2537
|
-
}
|
|
2538
|
-
}
|
|
2539
|
-
await putMany(txn, recoveryWrites);
|
|
2540
|
-
if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
|
|
2541
|
-
const nextAt = checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS);
|
|
2542
|
-
if (nextAt === null) await txn.deleteAlarm();
|
|
2543
|
-
else await txn.setAlarm(nextAt);
|
|
2544
|
-
return [];
|
|
2545
|
-
}
|
|
2546
|
-
const inflightCount = (await txn.list({
|
|
2547
|
-
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
2548
|
-
limit: maxInflight
|
|
2549
|
-
})).size;
|
|
2550
|
-
const rows = [];
|
|
2551
|
-
if (inflightCount < maxInflight) {
|
|
2552
|
-
const due = await txn.list({
|
|
2553
|
-
prefix: "due:",
|
|
2554
|
-
end: inclusiveTimestampEnd("due:", now),
|
|
2555
|
-
limit: maxInflight - inflightCount
|
|
2556
|
-
});
|
|
2557
|
-
const timeoutMs = this.#deliveryTimeoutMs();
|
|
2558
|
-
const dueRows = await getMany(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
2559
|
-
await deleteMany(txn, Array.from(due.keys()));
|
|
2560
|
-
const claimWrites = [];
|
|
2561
|
-
for (const [, messageId] of due) {
|
|
2562
|
-
const row = dueRows.get(`msg:${messageId}`);
|
|
2563
|
-
if (!row) continue;
|
|
2564
|
-
const claimDeadline = checkedQueueTimestampAdd(now, timeoutMs + QUEUE_INFLIGHT_GRACE_MS);
|
|
2565
|
-
if (claimDeadline === null) {
|
|
2566
|
-
await this.#deadLetter(txn, row, row.attempt, "delivery lease exceeded the queue timestamp horizon", now);
|
|
2567
|
-
continue;
|
|
2568
|
-
}
|
|
2569
|
-
const claimKey = inflightKey(claimDeadline, messageId);
|
|
2570
|
-
claimWrites.push([claimKey, messageId]);
|
|
2571
|
-
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
2572
|
-
messageId: row.messageId,
|
|
2573
|
-
inflightKey: claimKey,
|
|
2574
|
-
idempotencyKey: row.idempotencyKey
|
|
2575
|
-
}]);
|
|
2576
|
-
rows.push({
|
|
2577
|
-
row,
|
|
2578
|
-
inflightKey: claimKey
|
|
2579
|
-
});
|
|
2580
|
-
}
|
|
2581
|
-
await putMany(txn, claimWrites);
|
|
2582
|
-
}
|
|
2583
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2584
|
-
return rows;
|
|
2585
|
-
});
|
|
2586
|
-
}
|
|
2587
|
-
async #deliver({ row, inflightKey: claimKey }) {
|
|
2588
|
-
const storage = this.ctx.storage;
|
|
2589
|
-
const timeoutMs = this.#deliveryTimeoutMs();
|
|
2590
|
-
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
2591
|
-
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
2592
|
-
if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
|
|
2593
|
-
await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
|
|
2594
|
-
return;
|
|
2595
|
-
}
|
|
2596
|
-
let response = null;
|
|
2597
|
-
let transportError = null;
|
|
2598
|
-
try {
|
|
2599
|
-
response = await this.#fetch()(url, {
|
|
2600
|
-
method: "POST",
|
|
2601
|
-
headers: {
|
|
2602
|
-
"content-type": "application/json",
|
|
2603
|
-
"x-vqs-queue-name": row.queueName,
|
|
2604
|
-
"x-vqs-message-id": row.messageId,
|
|
2605
|
-
"x-vqs-message-attempt": String(row.attempt + 1),
|
|
2606
|
-
...callbackSecret ? { "x-workflow-callback-secret": callbackSecret } : {}
|
|
2607
|
-
},
|
|
2608
|
-
body: row.body,
|
|
2609
|
-
signal: AbortSignal.timeout(timeoutMs)
|
|
2610
|
-
});
|
|
2611
|
-
} catch (error) {
|
|
2612
|
-
transportError = error;
|
|
2613
|
-
}
|
|
2614
|
-
const now = this.#now();
|
|
2615
|
-
if (response?.ok) {
|
|
2616
|
-
await cancelUnusedResponseBody(response);
|
|
2617
|
-
await this.#ack(row, claimKey);
|
|
2618
|
-
} else if (response && response.status === 503) {
|
|
2619
|
-
let timeoutSeconds;
|
|
2620
|
-
try {
|
|
2621
|
-
timeoutSeconds = (await response.json())?.timeoutSeconds;
|
|
2622
|
-
} catch {
|
|
2623
|
-
timeoutSeconds = void 0;
|
|
2624
|
-
}
|
|
2625
|
-
const redeliveryAt = queueDelayDeadline(now, timeoutSeconds, 1, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2626
|
-
if (redeliveryAt !== null) {
|
|
2627
|
-
const reschedule = async (txn) => {
|
|
2628
|
-
if (!await txn.delete(claimKey)) return;
|
|
2629
|
-
const scheduleKey = dueKey(redeliveryAt, row.messageId);
|
|
2630
|
-
await txn.put(scheduleKey, row.messageId);
|
|
2631
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2632
|
-
messageId: row.messageId,
|
|
2633
|
-
dueKey: scheduleKey,
|
|
2634
|
-
idempotencyKey: row.idempotencyKey
|
|
2635
|
-
});
|
|
2636
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2637
|
-
};
|
|
2638
|
-
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
|
|
2639
|
-
else await storage.transaction(reschedule);
|
|
2640
|
-
} else await this.#retry(row, claimKey, "HTTP 503");
|
|
2641
|
-
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
2642
|
-
await cancelUnusedResponseBody(response);
|
|
2643
|
-
await this.#ack(row, claimKey);
|
|
2644
|
-
} else {
|
|
2645
|
-
const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
|
|
2646
|
-
if (response) await cancelUnusedResponseBody(response);
|
|
2647
|
-
await this.#retry(row, claimKey, reason);
|
|
2648
|
-
}
|
|
2649
|
-
}
|
|
2650
|
-
async #ack(row, claimKey) {
|
|
2651
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
2652
|
-
if (!await txn.delete(claimKey)) return;
|
|
2653
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
2654
|
-
if (row.idempotencyKey) {
|
|
2655
|
-
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
2656
|
-
}
|
|
2657
|
-
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
2658
|
-
await this.#scheduleNextAlarm(txn, this.#now());
|
|
2659
|
-
});
|
|
2660
|
-
}
|
|
2661
|
-
async #dropExpiredMessage(txn, row, claimKey, now) {
|
|
2662
|
-
if (!await txn.delete(claimKey)) return;
|
|
2663
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
2664
|
-
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
2665
|
-
if (row.idempotencyKey) {
|
|
2666
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
2667
|
-
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
2668
|
-
}
|
|
2669
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2670
|
-
}
|
|
2671
|
-
async #retry(row, claimKey, reason) {
|
|
2672
|
-
const storage = this.ctx.storage;
|
|
2673
|
-
const now = this.#now();
|
|
2674
|
-
const maxAttempts = this.#maxAttempts();
|
|
2675
|
-
const attempt = row.attempt + 1;
|
|
2676
|
-
const retryAt = checkedQueueTimestampAdd(now, backoffSeconds(attempt) * 1e3, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2677
|
-
const persistRetry = async (txn) => {
|
|
2678
|
-
if (!await txn.delete(claimKey)) return;
|
|
2679
|
-
if (attempt >= maxAttempts || retryAt === null) {
|
|
2680
|
-
await this.#deadLetter(txn, row, attempt, retryAt === null ? `${reason}; retry exceeded the queue timestamp horizon` : reason, now);
|
|
2681
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2682
|
-
return;
|
|
2683
|
-
}
|
|
2684
|
-
const updated = {
|
|
2685
|
-
...row,
|
|
2686
|
-
attempt,
|
|
2687
|
-
lastError: reason
|
|
2688
|
-
};
|
|
2689
|
-
const scheduleKey = dueKey(retryAt, row.messageId);
|
|
2690
|
-
await txn.put(`msg:${row.messageId}`, updated);
|
|
2691
|
-
await txn.put(scheduleKey, row.messageId);
|
|
2692
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2693
|
-
messageId: row.messageId,
|
|
2694
|
-
dueKey: scheduleKey,
|
|
2695
|
-
idempotencyKey: row.idempotencyKey
|
|
2696
|
-
});
|
|
2697
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2698
|
-
};
|
|
2699
|
-
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
|
|
2700
|
-
else await storage.transaction(persistRetry);
|
|
2701
|
-
}
|
|
2702
|
-
async #deadLetter(txn, row, attempt, reason, now) {
|
|
2703
|
-
const failedAt = checkedQueueTimestampAdd(now, 0);
|
|
2704
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
2705
|
-
if (row.idempotencyKey) {
|
|
2706
|
-
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
2707
|
-
}
|
|
2708
|
-
if (failedAt === null) {
|
|
2709
|
-
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
2710
|
-
return;
|
|
2711
|
-
}
|
|
2712
|
-
const dead = {
|
|
2713
|
-
...row,
|
|
2714
|
-
attempt,
|
|
2715
|
-
lastError: reason,
|
|
2716
|
-
failedAt
|
|
2717
|
-
};
|
|
2718
|
-
const deadKey = `dlq:${pad$1(failedAt)}:${row.messageId}`;
|
|
2719
|
-
await txn.put(deadKey, dead);
|
|
2720
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2721
|
-
messageId: row.messageId,
|
|
2722
|
-
dlqKey: deadKey,
|
|
2723
|
-
idempotencyKey: row.idempotencyKey
|
|
2724
|
-
});
|
|
2725
|
-
}
|
|
2726
|
-
/** Return true when another due compaction page must run immediately. */
|
|
2727
|
-
async #compactExpiredRunFences(txn, now) {
|
|
2728
|
-
const due = await txn.list({
|
|
2729
|
-
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
2730
|
-
end: inclusiveTimestampEnd(EXPIRED_RUN_GC_PREFIX, now),
|
|
2731
|
-
limit: 129
|
|
2732
|
-
});
|
|
2733
|
-
const page = Array.from(due).slice(0, 128);
|
|
2734
|
-
if (page.length === 0) return false;
|
|
2735
|
-
const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
|
|
2736
|
-
const markers = await getMany(txn, markerKeys);
|
|
2737
|
-
const deletes = [];
|
|
2738
|
-
for (let index = 0; index < page.length; index++) {
|
|
2739
|
-
const [gcKey, item] = page[index];
|
|
2740
|
-
const markerKey = markerKeys[index];
|
|
2741
|
-
const marker = markers.get(markerKey);
|
|
2742
|
-
deletes.push(gcKey);
|
|
2743
|
-
if (marker?.compactAt !== item.compactAt) continue;
|
|
2744
|
-
deletes.push(markerKey);
|
|
2745
|
-
}
|
|
2746
|
-
await deleteMany(txn, deletes);
|
|
2747
|
-
return due.size > 128;
|
|
2748
|
-
}
|
|
2749
|
-
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
2750
|
-
async #armAlarmAtMost(storage, atMs, now) {
|
|
2751
|
-
const current = await storage.getAlarm();
|
|
2752
|
-
const immediate = checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS);
|
|
2753
|
-
if (current !== null && current <= now) {
|
|
2754
|
-
if (immediate === null) await storage.deleteAlarm();
|
|
2755
|
-
else await storage.setAlarm(immediate);
|
|
2756
|
-
return;
|
|
2757
|
-
}
|
|
2758
|
-
const target = atMs <= now ? immediate : atMs;
|
|
2759
|
-
if (target === null) {
|
|
2760
|
-
await storage.deleteAlarm();
|
|
2761
|
-
return;
|
|
2762
|
-
}
|
|
2763
|
-
if (current === null || target < current) await storage.setAlarm(target);
|
|
2764
|
-
}
|
|
2765
|
-
/** Recompute the alarm from the earliest due entry / inflight deadline. */
|
|
2766
|
-
async #rearm() {
|
|
2767
|
-
await this.#scheduleNextAlarm(this.ctx.storage, this.#now());
|
|
2768
|
-
}
|
|
2769
|
-
/** Keep the alarm index consistent with message state in the same transaction. */
|
|
2770
|
-
async #scheduleNextAlarm(storage, now) {
|
|
2771
|
-
let next = null;
|
|
2772
|
-
const due = await storage.list({
|
|
2773
|
-
prefix: "due:",
|
|
2774
|
-
limit: 1
|
|
2775
|
-
});
|
|
2776
|
-
for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
|
|
2777
|
-
const inflight = await storage.list({
|
|
2778
|
-
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
2779
|
-
limit: 1
|
|
2780
|
-
});
|
|
2781
|
-
for (const key of inflight.keys()) {
|
|
2782
|
-
const deadline = deadlineFromInflightKey(key);
|
|
2783
|
-
if (next === null || deadline < next) next = deadline;
|
|
2784
|
-
}
|
|
2785
|
-
const expiryGc = await storage.list({
|
|
2786
|
-
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
2787
|
-
limit: 1
|
|
2788
|
-
});
|
|
2789
|
-
for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
|
|
2790
|
-
if (next !== null) {
|
|
2791
|
-
const target = next <= now ? checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS) : next;
|
|
2792
|
-
if (target === null) await storage.deleteAlarm();
|
|
2793
|
-
else await storage.setAlarm(target);
|
|
2794
|
-
} else await storage.deleteAlarm();
|
|
2795
|
-
}
|
|
2796
|
-
async stats() {
|
|
2797
|
-
const storage = this.ctx.storage;
|
|
2798
|
-
const [due, inflight, dlq, alarmAt] = await Promise.all([
|
|
2799
|
-
storage.list({ prefix: "due:" }),
|
|
2800
|
-
storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
|
|
2801
|
-
storage.list({ prefix: "dlq:" }),
|
|
2802
|
-
storage.getAlarm()
|
|
2803
|
-
]);
|
|
2804
|
-
return {
|
|
2805
|
-
pending: due.size,
|
|
2806
|
-
inflight: inflight.size,
|
|
2807
|
-
deadLetters: dlq.size,
|
|
2808
|
-
alarmAt
|
|
2809
|
-
};
|
|
2810
|
-
}
|
|
2811
|
-
async listDeadLetters(params) {
|
|
2812
|
-
if (params !== void 0 && !isRecord$1(params)) throw new Error("world-celld queue dead-letter list options must be an object");
|
|
2813
|
-
const limit = boundedIntegerOption("world-celld queue dead-letter limit", params?.limit, 100, 1, MAX_DEAD_LETTER_LIST_SIZE);
|
|
2814
|
-
if (params?.cursor !== void 0 && typeof params.cursor !== "string") throw new Error("world-celld queue dead-letter cursor must be a string");
|
|
2815
|
-
const entries = await this.ctx.storage.list({
|
|
2816
|
-
prefix: "dlq:",
|
|
2817
|
-
startAfter: params?.cursor,
|
|
2818
|
-
limit: limit + 1
|
|
2819
|
-
});
|
|
2820
|
-
const keys = Array.from(entries.keys());
|
|
2821
|
-
const page = keys.slice(0, limit);
|
|
2822
|
-
const hasMore = keys.length > limit;
|
|
2823
|
-
return {
|
|
2824
|
-
data: page.map((k) => entries.get(k)),
|
|
2825
|
-
cursor: hasMore ? page.at(-1) : null,
|
|
2826
|
-
hasMore
|
|
2827
|
-
};
|
|
2828
|
-
}
|
|
2829
|
-
/** Move a dead letter back to the live queue (attempt count reset). */
|
|
2830
|
-
async redriveDeadLetter(messageId) {
|
|
2831
|
-
if (typeof messageId !== "string" || messageId.length === 0) throw new Error("world-celld queue redrive messageId must be a non-empty string");
|
|
2832
|
-
const storage = this.ctx.storage;
|
|
2833
|
-
const now = this.#now();
|
|
2834
|
-
if (checkedQueueTimestampAdd(now, 0, 330001) === null) return { ok: false };
|
|
2835
|
-
const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
|
|
2836
|
-
if (!found) return { ok: false };
|
|
2837
|
-
const [deadKey, candidate] = found;
|
|
2838
|
-
const redrive = async (txn) => {
|
|
2839
|
-
const dead = await txn.get(deadKey);
|
|
2840
|
-
if (!dead || dead.messageId !== messageId) return { ok: false };
|
|
2841
|
-
const { failedAt: _failedAt, ...rest } = dead;
|
|
2842
|
-
const row = {
|
|
2843
|
-
...rest,
|
|
2844
|
-
attempt: 0,
|
|
2845
|
-
lastError: void 0
|
|
2846
|
-
};
|
|
2847
|
-
if (row.idempotencyKey) {
|
|
2848
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
2849
|
-
const holder = await txn.get(dedupKey);
|
|
2850
|
-
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
2851
|
-
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
2852
|
-
await txn.put(dedupKey, row.messageId);
|
|
2853
|
-
}
|
|
2854
|
-
await txn.put(`msg:${row.messageId}`, row);
|
|
2855
|
-
const scheduleKey = dueKey(now, row.messageId);
|
|
2856
|
-
await txn.put(scheduleKey, row.messageId);
|
|
2857
|
-
await txn.delete(deadKey);
|
|
2858
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2859
|
-
messageId: row.messageId,
|
|
2860
|
-
dueKey: scheduleKey,
|
|
2861
|
-
idempotencyKey: row.idempotencyKey
|
|
2862
|
-
});
|
|
2863
|
-
await this.#armAlarmAtMost(txn, now, now);
|
|
2864
|
-
return { ok: true };
|
|
2865
|
-
};
|
|
2866
|
-
const rejectExpired = async (txn) => {
|
|
2867
|
-
if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
|
|
2868
|
-
return { ok: false };
|
|
2869
|
-
};
|
|
2870
|
-
return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
|
|
2871
|
-
}
|
|
2872
|
-
async purgeDeadLetters() {
|
|
2873
|
-
return await this.ctx.storage.transaction(async (txn) => {
|
|
2874
|
-
const entries = await txn.list({
|
|
2875
|
-
prefix: "dlq:",
|
|
2876
|
-
limit: 129
|
|
2877
|
-
});
|
|
2878
|
-
const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
|
|
2879
|
-
await deleteMany(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
|
|
2880
|
-
return {
|
|
2881
|
-
purged: page.length,
|
|
2882
|
-
hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
|
|
2883
|
-
};
|
|
2884
|
-
});
|
|
2885
|
-
}
|
|
2886
|
-
/**
|
|
2887
|
-
* Recovery hatch for celld's six-failure alarm abandonment: derive the due
|
|
2888
|
-
* time again and arm it.
|
|
2889
|
-
*/
|
|
2890
|
-
async rearmAlarm() {
|
|
2891
|
-
await this.#rearm();
|
|
2892
|
-
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
2893
|
-
}
|
|
2894
|
-
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
2895
|
-
async expireRun(runId, expiredAt, options) {
|
|
2896
|
-
if (typeof runId !== "string" || runId.length === 0) throw new Error("world-celld queue expiry runId must be a non-empty string");
|
|
2897
|
-
if (!isNonNegativeSafeInteger(expiredAt)) throw new Error("world-celld queue expiry expiredAt must be a non-negative safe integer");
|
|
2898
|
-
if (options !== void 0 && !isRecord$1(options)) throw new Error("world-celld queue expiry options must be an object");
|
|
2899
|
-
const storage = this.ctx.storage;
|
|
2900
|
-
const limit = boundedIntegerOption("world-celld queue expiry limit", options?.limit, EXPIRE_RUN_REFERENCE_BATCH, 1, EXPIRE_RUN_REFERENCE_BATCH);
|
|
2901
|
-
return await storage.transaction(async (txn) => {
|
|
2902
|
-
const existing = await txn.get(expiredRunKey(runId));
|
|
2903
|
-
const references = await txn.list({
|
|
2904
|
-
prefix: `run:${runId}:`,
|
|
2905
|
-
limit: limit + 1
|
|
2906
|
-
});
|
|
2907
|
-
const page = Array.from(references).slice(0, limit);
|
|
2908
|
-
const dedupHolders = await getMany(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
|
|
2909
|
-
const keys = [];
|
|
2910
|
-
let count = existing?.deleted ?? 0;
|
|
2911
|
-
for (const [key, reference] of page) {
|
|
2912
|
-
keys.push(`msg:${reference.messageId}`, key);
|
|
2913
|
-
if (reference.inflightKey) keys.push(reference.inflightKey);
|
|
2914
|
-
if (reference.dueKey) keys.push(reference.dueKey);
|
|
2915
|
-
if (reference.dlqKey) keys.push(reference.dlqKey);
|
|
2916
|
-
if (reference.idempotencyKey) {
|
|
2917
|
-
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
2918
|
-
if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
|
|
2919
|
-
}
|
|
2920
|
-
count++;
|
|
2921
|
-
}
|
|
2922
|
-
await deleteMany(txn, keys);
|
|
2923
|
-
const done = references.size <= limit;
|
|
2924
|
-
const markerExpiredAt = existing?.expiredAt ?? expiredAt;
|
|
2925
|
-
await txn.put(expiredRunKey(runId), {
|
|
2926
|
-
expiredAt: markerExpiredAt,
|
|
2927
|
-
deleted: count,
|
|
2928
|
-
compactAt: existing?.compactAt ?? null
|
|
2929
|
-
});
|
|
2930
|
-
await this.#scheduleNextAlarm(txn, this.#now());
|
|
2931
|
-
return done ? {
|
|
2932
|
-
deleted: count,
|
|
2933
|
-
done: true,
|
|
2934
|
-
receipt: {
|
|
2935
|
-
expiredAt: markerExpiredAt,
|
|
2936
|
-
deleted: count
|
|
2937
|
-
}
|
|
2938
|
-
} : {
|
|
2939
|
-
deleted: count,
|
|
2940
|
-
done: false
|
|
2941
|
-
};
|
|
2942
|
-
});
|
|
2943
|
-
}
|
|
2944
|
-
/** Acknowledge a final cleanup receipt before making its marker compactable. */
|
|
2945
|
-
async acknowledgeExpireRun(runId, receipt) {
|
|
2946
|
-
if (typeof runId !== "string" || runId.length === 0) throw new Error("world-celld queue expiry runId must be a non-empty string");
|
|
2947
|
-
if (!isRecord$1(receipt) || !isNonNegativeSafeInteger(receipt.expiredAt) || !isNonNegativeSafeInteger(receipt.deleted)) throw new Error("world-celld queue expiry receipt is invalid");
|
|
2948
|
-
return await this.ctx.storage.transaction(async (txn) => {
|
|
2949
|
-
const key = expiredRunKey(runId);
|
|
2950
|
-
const marker = await txn.get(key);
|
|
2951
|
-
if (!marker) return { acknowledged: true };
|
|
2952
|
-
if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
|
|
2953
|
-
if (marker.compactAt === null) {
|
|
2954
|
-
const compactAt = checkedQueueTimestampAdd(this.#now(), QUEUE_FENCE_GRACE_MS);
|
|
2955
|
-
if (compactAt !== null) {
|
|
2956
|
-
await txn.put(key, {
|
|
2957
|
-
...marker,
|
|
2958
|
-
compactAt
|
|
2959
|
-
});
|
|
2960
|
-
await txn.put(expiredRunGcKey(runId, compactAt), {
|
|
2961
|
-
runId,
|
|
2962
|
-
compactAt
|
|
2963
|
-
});
|
|
2964
|
-
}
|
|
2965
|
-
}
|
|
2966
|
-
await this.#scheduleNextAlarm(txn, this.#now());
|
|
2967
|
-
return { acknowledged: true };
|
|
2968
|
-
});
|
|
2969
|
-
}
|
|
2970
|
-
};
|
|
2971
|
-
//#endregion
|
|
2972
2384
|
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
2973
2385
|
const MAX_INDEX_LIST_SIZE = 1001;
|
|
2974
2386
|
const EXPIRY_GC_PREFIX = "expiry-gc:";
|
|
@@ -3574,6 +2986,24 @@ function expiredRead(tombstone) {
|
|
|
3574
2986
|
};
|
|
3575
2987
|
}
|
|
3576
2988
|
//#endregion
|
|
2989
|
+
//#region src/worker/queue-payload-store.ts
|
|
2990
|
+
function createQueuePayloadStore(binding) {
|
|
2991
|
+
return {
|
|
2992
|
+
async write(key, value, metadata) {
|
|
2993
|
+
await binding.put(key, value, { customMetadata: metadata });
|
|
2994
|
+
},
|
|
2995
|
+
async read(key) {
|
|
2996
|
+
return (await binding.get(key))?.text() ?? null;
|
|
2997
|
+
},
|
|
2998
|
+
async delete(keys) {
|
|
2999
|
+
await binding.delete(keys);
|
|
3000
|
+
}
|
|
3001
|
+
};
|
|
3002
|
+
}
|
|
3003
|
+
async function deleteQueuePayloadObjects(binding, keys) {
|
|
3004
|
+
await binding.delete(keys);
|
|
3005
|
+
}
|
|
3006
|
+
//#endregion
|
|
3577
3007
|
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
3578
3008
|
const STORAGE_BATCH_SIZE = 128;
|
|
3579
3009
|
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
@@ -3583,15 +3013,14 @@ const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
|
3583
3013
|
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
3584
3014
|
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3585
3015
|
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
3586
|
-
const
|
|
3016
|
+
const QUEUE_PAYLOADS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3587
3017
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
3588
|
-
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
3589
|
-
const RUN_QUEUE_SHARDS_KEY = "retention:queue-shards";
|
|
3590
3018
|
const MAX_DATE_MS = 864e13;
|
|
3019
|
+
const QUEUE_ORPHAN_KEY = "queue-orphan";
|
|
3020
|
+
const QUEUE_RESERVATION_KEY = "queue-reservation";
|
|
3591
3021
|
function validateCleanupRequest(request, allowDisabled) {
|
|
3592
3022
|
if (!isRecord$1(request)) throw new Error("world-celld retention request must be an object");
|
|
3593
3023
|
if (!(allowDisabled ? isNonNegativeSafeInteger(request.retentionMs) : isPositiveSafeInteger(request.retentionMs))) throw new Error(`world-celld retentionMs must be a ${allowDisabled ? "non-negative" : "positive"} safe integer`);
|
|
3594
|
-
if (!isPositiveSafeInteger(request.queueShards)) throw new Error("world-celld retention queueShards must be a positive safe integer");
|
|
3595
3024
|
return request;
|
|
3596
3025
|
}
|
|
3597
3026
|
function retentionDeadline(baseMs, retentionMs, runId) {
|
|
@@ -3599,7 +3028,7 @@ function retentionDeadline(baseMs, retentionMs, runId) {
|
|
|
3599
3028
|
if (!Number.isSafeInteger(dueAtMs) || dueAtMs > MAX_DATE_MS) throw new Error(`world-celld retention deadline is out of range for ${runId}`);
|
|
3600
3029
|
return dueAtMs;
|
|
3601
3030
|
}
|
|
3602
|
-
function cleanupRecord(run, dueAt,
|
|
3031
|
+
function cleanupRecord(run, dueAt, reason) {
|
|
3603
3032
|
const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
|
|
3604
3033
|
return {
|
|
3605
3034
|
version: 1,
|
|
@@ -3611,13 +3040,12 @@ function cleanupRecord(run, dueAt, queueShards, reason) {
|
|
|
3611
3040
|
terminalStatus: terminal?.status,
|
|
3612
3041
|
reason,
|
|
3613
3042
|
dueAt,
|
|
3614
|
-
queueShards,
|
|
3615
3043
|
phase: "retained",
|
|
3616
3044
|
generation: 0,
|
|
3617
3045
|
attempts: 0,
|
|
3618
3046
|
deletedPayloadKeys: 0,
|
|
3619
3047
|
deletedStreams: 0,
|
|
3620
|
-
|
|
3048
|
+
deletedQueuePayloads: 0
|
|
3621
3049
|
};
|
|
3622
3050
|
}
|
|
3623
3051
|
/**
|
|
@@ -3650,8 +3078,8 @@ async function putStorageEntries(storage, entries) {
|
|
|
3650
3078
|
* storage transaction: guard checks, the event append, and the entity
|
|
3651
3079
|
* mutation are atomic even if the DO is evicted mid-operation.
|
|
3652
3080
|
*
|
|
3653
|
-
* Instances named `claim:<queueName>:<idempotencyKey>`
|
|
3654
|
-
*
|
|
3081
|
+
* Instances named `claim:<queueName>:<idempotencyKey>` reserve one message ID
|
|
3082
|
+
* at publication time and serialize its deliveries until completion.
|
|
3655
3083
|
*/
|
|
3656
3084
|
var WorkflowRunDO = class extends DurableObject {
|
|
3657
3085
|
now() {
|
|
@@ -3718,7 +3146,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3718
3146
|
now
|
|
3719
3147
|
});
|
|
3720
3148
|
if (!outcome.ok) return outcome;
|
|
3721
|
-
if (outcome.runCreated && request.cleanup) await txn.put(RUN_QUEUE_SHARDS_KEY, request.cleanup.queueShards);
|
|
3722
3149
|
await txn.put("event_sequence", eventSequence);
|
|
3723
3150
|
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
3724
3151
|
hookId: outcome.hookToIndex.hookId,
|
|
@@ -3732,7 +3159,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3732
3159
|
...terminalRun,
|
|
3733
3160
|
expiredAt: dueAt
|
|
3734
3161
|
});
|
|
3735
|
-
const cleanup = cleanupRecord(run, dueAt,
|
|
3162
|
+
const cleanup = cleanupRecord(run, dueAt, "terminal-retention");
|
|
3736
3163
|
await txn.put("run", run);
|
|
3737
3164
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3738
3165
|
await txn.setAlarm(dueAt);
|
|
@@ -3842,7 +3269,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3842
3269
|
...terminalRun,
|
|
3843
3270
|
expiredAt: dueAt
|
|
3844
3271
|
});
|
|
3845
|
-
const cleanup = cleanupRecord(run, dueAt,
|
|
3272
|
+
const cleanup = cleanupRecord(run, dueAt, "manual");
|
|
3846
3273
|
await txn.put("run", run);
|
|
3847
3274
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3848
3275
|
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
@@ -3863,7 +3290,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3863
3290
|
const values = await txn.get([
|
|
3864
3291
|
CLEANUP_RECORD_KEY,
|
|
3865
3292
|
TOMBSTONE_KEY,
|
|
3866
|
-
RUN_QUEUE_SHARDS_KEY,
|
|
3867
3293
|
"run"
|
|
3868
3294
|
]);
|
|
3869
3295
|
const tombstone = values.get(TOMBSTONE_KEY);
|
|
@@ -3882,12 +3308,9 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3882
3308
|
cleanup: null
|
|
3883
3309
|
};
|
|
3884
3310
|
const dueAt = new Date(dueAtMs);
|
|
3885
|
-
|
|
3886
|
-
const storedQueueShards = values.get(RUN_QUEUE_SHARDS_KEY);
|
|
3887
|
-
const queueShards = Number.isSafeInteger(storedQueueShards) && storedQueueShards > 0 ? storedQueueShards : request.queueShards;
|
|
3888
|
-
let cleanup = existing;
|
|
3311
|
+
let cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3889
3312
|
if (!cleanup) {
|
|
3890
|
-
cleanup = cleanupRecord(run, dueAt,
|
|
3313
|
+
cleanup = cleanupRecord(run, dueAt, "maximum-age");
|
|
3891
3314
|
await txn.put("run", WorkflowRunSchema.parse({
|
|
3892
3315
|
...run,
|
|
3893
3316
|
expiredAt: dueAt
|
|
@@ -3990,9 +3413,69 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3990
3413
|
});
|
|
3991
3414
|
}
|
|
3992
3415
|
async executeScheduledCleanupPage() {
|
|
3993
|
-
const values = await this.ctx.storage.get([
|
|
3416
|
+
const values = await this.ctx.storage.get([
|
|
3417
|
+
CLEANUP_RECORD_KEY,
|
|
3418
|
+
TERMINAL_CLEANUP_KEY,
|
|
3419
|
+
"claim",
|
|
3420
|
+
QUEUE_RESERVATION_KEY,
|
|
3421
|
+
QUEUE_ORPHAN_KEY,
|
|
3422
|
+
"run"
|
|
3423
|
+
]);
|
|
3994
3424
|
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3995
3425
|
const terminalCleanup = values.get(TERMINAL_CLEANUP_KEY);
|
|
3426
|
+
const claim = values.get("claim");
|
|
3427
|
+
const reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
3428
|
+
if (values.get(QUEUE_ORPHAN_KEY)) {
|
|
3429
|
+
const leased = await this.ctx.storage.transaction(async (txn) => {
|
|
3430
|
+
const current = await txn.get(QUEUE_ORPHAN_KEY);
|
|
3431
|
+
if (!current) return null;
|
|
3432
|
+
if (current.expiresAt > this.now()) {
|
|
3433
|
+
await txn.setAlarm(current.expiresAt);
|
|
3434
|
+
return null;
|
|
3435
|
+
}
|
|
3436
|
+
if (current.deleting) return current;
|
|
3437
|
+
const next = {
|
|
3438
|
+
...current,
|
|
3439
|
+
generation: current.generation + 1,
|
|
3440
|
+
deleting: true
|
|
3441
|
+
};
|
|
3442
|
+
await txn.put(QUEUE_ORPHAN_KEY, next);
|
|
3443
|
+
return next;
|
|
3444
|
+
});
|
|
3445
|
+
if (!leased) return;
|
|
3446
|
+
const bucket = this.env?.WORKFLOW_QUEUE_PAYLOADS;
|
|
3447
|
+
const runs = this.env?.WORKFLOW_DB;
|
|
3448
|
+
if (!bucket || !runs) throw new Error("world-celld queue orphan cleanup is missing required bindings");
|
|
3449
|
+
await deleteQueuePayloadObjects(bucket, leased.key);
|
|
3450
|
+
await runs.get(runs.idFromName(leased.runId)).unregisterQueuePayload(leased.messageId);
|
|
3451
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3452
|
+
const current = await txn.get(QUEUE_ORPHAN_KEY);
|
|
3453
|
+
if (current?.messageId !== leased.messageId || current.generation !== leased.generation || !current.deleting) return;
|
|
3454
|
+
await txn.delete(QUEUE_ORPHAN_KEY);
|
|
3455
|
+
await txn.deleteAlarm();
|
|
3456
|
+
});
|
|
3457
|
+
return;
|
|
3458
|
+
}
|
|
3459
|
+
if (claim || reservation) {
|
|
3460
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3461
|
+
const current = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
3462
|
+
let nextClaim = current.get("claim");
|
|
3463
|
+
let nextReservation = current.get(QUEUE_RESERVATION_KEY);
|
|
3464
|
+
const now = this.now();
|
|
3465
|
+
if (nextClaim && nextClaim.expiresAt <= now) {
|
|
3466
|
+
await txn.delete("claim");
|
|
3467
|
+
nextClaim = void 0;
|
|
3468
|
+
}
|
|
3469
|
+
if (nextReservation && nextReservation.expiresAt <= now) {
|
|
3470
|
+
await txn.delete(QUEUE_RESERVATION_KEY);
|
|
3471
|
+
nextReservation = void 0;
|
|
3472
|
+
}
|
|
3473
|
+
const nextAlarm = Math.min(nextClaim?.expiresAt ?? Number.POSITIVE_INFINITY, nextReservation?.expiresAt ?? Number.POSITIVE_INFINITY);
|
|
3474
|
+
if (Number.isFinite(nextAlarm)) await txn.setAlarm(nextAlarm);
|
|
3475
|
+
else await txn.deleteAlarm();
|
|
3476
|
+
});
|
|
3477
|
+
return;
|
|
3478
|
+
}
|
|
3996
3479
|
if (cleanup && cleanup.phase !== "tombstoned" && cleanup.dueAt.getTime() <= this.now()) {
|
|
3997
3480
|
try {
|
|
3998
3481
|
await this.executeCleanup(cleanup);
|
|
@@ -4191,85 +3674,27 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4191
3674
|
});
|
|
4192
3675
|
}
|
|
4193
3676
|
async deleteQueuePage(cleanup) {
|
|
4194
|
-
const
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
4203
|
-
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
4204
|
-
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
4205
|
-
const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
|
|
4206
|
-
if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
|
|
4207
|
-
await txn.put(CLEANUP_RECORD_KEY, {
|
|
4208
|
-
...current,
|
|
4209
|
-
generation: current.generation + 1,
|
|
4210
|
-
attempts: 0,
|
|
4211
|
-
lastError: void 0
|
|
4212
|
-
});
|
|
4213
|
-
await txn.put(CLEANUP_PROGRESS_KEY, {
|
|
4214
|
-
queueShard: queueShard + 1,
|
|
4215
|
-
queueShardDeleted: 0
|
|
4216
|
-
});
|
|
4217
|
-
await txn.setAlarm(this.now() + 1);
|
|
4218
|
-
});
|
|
4219
|
-
return;
|
|
4220
|
-
}
|
|
4221
|
-
if (progress.queueShard < cleanup.queueShards) {
|
|
4222
|
-
const requestedShard = progress.queueShard;
|
|
4223
|
-
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
4224
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
4225
|
-
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
4226
|
-
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
4227
|
-
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
4228
|
-
queueShard: 0,
|
|
4229
|
-
queueShardDeleted: 0
|
|
4230
|
-
};
|
|
4231
|
-
if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
|
|
4232
|
-
if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
|
|
4233
|
-
const delta = result.deleted - persisted.queueShardDeleted;
|
|
4234
|
-
const next = result.done ? {
|
|
4235
|
-
queueShard: requestedShard,
|
|
4236
|
-
queueShardDeleted: result.deleted,
|
|
4237
|
-
pendingAck: {
|
|
4238
|
-
queueShard: requestedShard,
|
|
4239
|
-
receipt: result.receipt
|
|
4240
|
-
}
|
|
4241
|
-
} : {
|
|
4242
|
-
queueShard: requestedShard,
|
|
4243
|
-
queueShardDeleted: result.deleted
|
|
4244
|
-
};
|
|
4245
|
-
await txn.put(CLEANUP_RECORD_KEY, {
|
|
4246
|
-
...current,
|
|
4247
|
-
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
4248
|
-
generation: current.generation + 1,
|
|
4249
|
-
attempts: 0,
|
|
4250
|
-
lastError: void 0
|
|
4251
|
-
});
|
|
4252
|
-
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
4253
|
-
await txn.setAlarm(this.now() + 1);
|
|
4254
|
-
});
|
|
4255
|
-
return;
|
|
4256
|
-
}
|
|
3677
|
+
const entries = await this.ctx.storage.list({
|
|
3678
|
+
prefix: QUEUE_PAYLOAD_REGISTRY_PREFIX,
|
|
3679
|
+
limit: 129
|
|
3680
|
+
});
|
|
3681
|
+
const page = Array.from(entries).slice(0, QUEUE_PAYLOADS_PER_CLEANUP_PAGE);
|
|
3682
|
+
const bucket = this.env?.WORKFLOW_QUEUE_PAYLOADS;
|
|
3683
|
+
if (!bucket) throw new Error("world-celld retention missing binding WORKFLOW_QUEUE_PAYLOADS");
|
|
3684
|
+
if (page.length > 0) await deleteQueuePayloadObjects(bucket, page.map(([, registration]) => registration.key));
|
|
4257
3685
|
await this.ctx.storage.transaction(async (txn) => {
|
|
4258
3686
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
4259
3687
|
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
4260
|
-
|
|
4261
|
-
|
|
4262
|
-
|
|
4263
|
-
|
|
4264
|
-
|
|
4265
|
-
|
|
4266
|
-
|
|
4267
|
-
|
|
4268
|
-
|
|
4269
|
-
|
|
4270
|
-
await txn.delete(CLEANUP_PROGRESS_KEY);
|
|
4271
|
-
await txn.setAlarm(this.now() + 1);
|
|
4272
|
-
}
|
|
3688
|
+
if (page.length > 0) await txn.delete(page.map(([key]) => key));
|
|
3689
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3690
|
+
...current,
|
|
3691
|
+
deletedQueuePayloads: current.deletedQueuePayloads + page.length,
|
|
3692
|
+
phase: entries.size <= QUEUE_PAYLOADS_PER_CLEANUP_PAGE ? "payload" : "queues",
|
|
3693
|
+
generation: current.generation + 1,
|
|
3694
|
+
attempts: 0,
|
|
3695
|
+
lastError: void 0
|
|
3696
|
+
});
|
|
3697
|
+
await txn.setAlarm(this.now() + 1);
|
|
4273
3698
|
});
|
|
4274
3699
|
}
|
|
4275
3700
|
async deletePayloadBatch() {
|
|
@@ -4340,20 +3765,163 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4340
3765
|
await txn.setAlarm(this.now() + delay);
|
|
4341
3766
|
});
|
|
4342
3767
|
}
|
|
3768
|
+
async registerQueuePayload(registration) {
|
|
3769
|
+
if (!registration || typeof registration.messageId !== "string" || registration.messageId.length === 0 || typeof registration.key !== "string" || registration.key.length === 0 || !isNonNegativeSafeInteger(registration.orphanExpiresAt)) throw new TypeError("world-celld queue payload registration is invalid");
|
|
3770
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3771
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3772
|
+
if (tombstone) return {
|
|
3773
|
+
ok: false,
|
|
3774
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
3775
|
+
};
|
|
3776
|
+
await txn.put(queuePayloadRegistryKey(registration.messageId), registration);
|
|
3777
|
+
return { ok: true };
|
|
3778
|
+
});
|
|
3779
|
+
}
|
|
3780
|
+
async finalizeQueuePayload(messageId) {
|
|
3781
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue payload messageId must be a non-empty string");
|
|
3782
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3783
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3784
|
+
if (tombstone) return {
|
|
3785
|
+
ok: false,
|
|
3786
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
3787
|
+
};
|
|
3788
|
+
return await txn.get(queuePayloadRegistryKey(messageId)) ? { ok: true } : {
|
|
3789
|
+
ok: false,
|
|
3790
|
+
message: "workflow queue payload registration was removed before publication"
|
|
3791
|
+
};
|
|
3792
|
+
});
|
|
3793
|
+
}
|
|
3794
|
+
async unregisterQueuePayload(messageId) {
|
|
3795
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue payload messageId must be a non-empty string");
|
|
3796
|
+
await this.ctx.storage.delete(queuePayloadRegistryKey(messageId));
|
|
3797
|
+
}
|
|
3798
|
+
async scheduleQueuePayloadOrphan(orphan) {
|
|
3799
|
+
if (!orphan || typeof orphan.messageId !== "string" || orphan.messageId.length === 0 || typeof orphan.runId !== "string" || orphan.runId.length === 0 || typeof orphan.key !== "string" || orphan.key.length === 0 || !isNonNegativeSafeInteger(orphan.expiresAt)) throw new TypeError("world-celld queue payload orphan is invalid");
|
|
3800
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3801
|
+
const existing = await txn.get(QUEUE_ORPHAN_KEY);
|
|
3802
|
+
if (existing && existing.messageId !== orphan.messageId) throw new Error("world-celld queue payload orphan identity mismatch");
|
|
3803
|
+
if (existing?.deleting) throw new Error("world-celld queue payload orphan deletion is in progress");
|
|
3804
|
+
const next = {
|
|
3805
|
+
...orphan,
|
|
3806
|
+
expiresAt: Math.max(existing?.expiresAt ?? 0, orphan.expiresAt),
|
|
3807
|
+
generation: (existing?.generation ?? 0) + 1,
|
|
3808
|
+
deleting: false
|
|
3809
|
+
};
|
|
3810
|
+
await txn.put(QUEUE_ORPHAN_KEY, next);
|
|
3811
|
+
await txn.setAlarm(next.expiresAt);
|
|
3812
|
+
});
|
|
3813
|
+
}
|
|
3814
|
+
async cancelQueuePayloadOrphan(messageId) {
|
|
3815
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue payload orphan messageId must be a non-empty string");
|
|
3816
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3817
|
+
const existing = await txn.get(QUEUE_ORPHAN_KEY);
|
|
3818
|
+
if (!existing || existing.messageId !== messageId) return;
|
|
3819
|
+
await txn.delete(QUEUE_ORPHAN_KEY);
|
|
3820
|
+
await txn.deleteAlarm();
|
|
3821
|
+
});
|
|
3822
|
+
}
|
|
3823
|
+
async reserveQueueMessage(params) {
|
|
3824
|
+
if (typeof params?.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.expiresAt)) throw new TypeError("world-celld queue message reservation is invalid");
|
|
3825
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3826
|
+
const values = await txn.get([QUEUE_RESERVATION_KEY, "claim"]);
|
|
3827
|
+
const existing = values.get(QUEUE_RESERVATION_KEY);
|
|
3828
|
+
const claim = values.get("claim");
|
|
3829
|
+
const now = this.now();
|
|
3830
|
+
if (existing && existing.expiresAt > now && existing.messageId !== params.messageId) {
|
|
3831
|
+
await txn.setAlarm(Math.min(existing.expiresAt, claim?.expiresAt ?? existing.expiresAt));
|
|
3832
|
+
return {
|
|
3833
|
+
admitted: false,
|
|
3834
|
+
messageId: existing.messageId
|
|
3835
|
+
};
|
|
3836
|
+
}
|
|
3837
|
+
const reservation = {
|
|
3838
|
+
messageId: params.messageId,
|
|
3839
|
+
expiresAt: Math.max(existing?.expiresAt ?? 0, params.expiresAt)
|
|
3840
|
+
};
|
|
3841
|
+
await txn.put(QUEUE_RESERVATION_KEY, reservation);
|
|
3842
|
+
await txn.setAlarm(Math.min(reservation.expiresAt, claim?.expiresAt ?? reservation.expiresAt));
|
|
3843
|
+
return {
|
|
3844
|
+
admitted: true,
|
|
3845
|
+
messageId: params.messageId
|
|
3846
|
+
};
|
|
3847
|
+
});
|
|
3848
|
+
}
|
|
3849
|
+
async completeQueueMessage(messageId) {
|
|
3850
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue messageId must be a non-empty string");
|
|
3851
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3852
|
+
const values = await txn.get([QUEUE_RESERVATION_KEY, "claim"]);
|
|
3853
|
+
let reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
3854
|
+
let claim = values.get("claim");
|
|
3855
|
+
if (reservation?.messageId === messageId) {
|
|
3856
|
+
await txn.delete(QUEUE_RESERVATION_KEY);
|
|
3857
|
+
reservation = void 0;
|
|
3858
|
+
}
|
|
3859
|
+
if (claim?.messageId === messageId) {
|
|
3860
|
+
await txn.delete("claim");
|
|
3861
|
+
claim = void 0;
|
|
3862
|
+
}
|
|
3863
|
+
const nextAlarm = Math.min(reservation?.expiresAt ?? Number.POSITIVE_INFINITY, claim?.expiresAt ?? Number.POSITIVE_INFINITY);
|
|
3864
|
+
if (Number.isFinite(nextAlarm)) await txn.setAlarm(nextAlarm);
|
|
3865
|
+
else await txn.deleteAlarm();
|
|
3866
|
+
});
|
|
3867
|
+
}
|
|
4343
3868
|
async claimInflight(params) {
|
|
3869
|
+
if (typeof params?.messageId !== "string" || params.messageId.length === 0 || !isPositiveSafeInteger(params.staleMs)) throw new TypeError("world-celld queue claim is invalid");
|
|
4344
3870
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
4345
|
-
const
|
|
3871
|
+
const values = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
3872
|
+
const existing = values.get("claim");
|
|
3873
|
+
const reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
4346
3874
|
const now = this.now();
|
|
4347
|
-
if (existing && existing.
|
|
3875
|
+
if (existing && existing.expiresAt > now) {
|
|
3876
|
+
if (existing.messageId !== params.messageId) return { claimed: false };
|
|
3877
|
+
if (existing.retryAt === void 0 || existing.retryAt > now) return {
|
|
3878
|
+
claimed: false,
|
|
3879
|
+
retryAt: existing.retryAt ?? existing.expiresAt
|
|
3880
|
+
};
|
|
3881
|
+
}
|
|
3882
|
+
const expiresAt = now + params.staleMs;
|
|
4348
3883
|
await txn.put("claim", {
|
|
4349
3884
|
messageId: params.messageId,
|
|
4350
|
-
|
|
3885
|
+
expiresAt
|
|
4351
3886
|
});
|
|
3887
|
+
await txn.setAlarm(Math.min(expiresAt, reservation?.expiresAt ?? expiresAt));
|
|
4352
3888
|
return { claimed: true };
|
|
4353
3889
|
});
|
|
4354
3890
|
}
|
|
4355
|
-
async
|
|
4356
|
-
|
|
3891
|
+
async holdInflight(params) {
|
|
3892
|
+
if (typeof params?.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.retryAt) || !isNonNegativeSafeInteger(params.expiresAt) || params.reservationExpiresAt !== void 0 && !isNonNegativeSafeInteger(params.reservationExpiresAt)) throw new TypeError("world-celld queue claim hold is invalid");
|
|
3893
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3894
|
+
const values = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
3895
|
+
const existing = values.get("claim");
|
|
3896
|
+
let reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
3897
|
+
if (!existing || existing.messageId !== params.messageId) return { held: false };
|
|
3898
|
+
const expiresAt = Math.max(existing.expiresAt, params.expiresAt);
|
|
3899
|
+
await txn.put("claim", {
|
|
3900
|
+
...existing,
|
|
3901
|
+
retryAt: params.retryAt,
|
|
3902
|
+
expiresAt
|
|
3903
|
+
});
|
|
3904
|
+
if (reservation?.messageId === params.messageId && params.reservationExpiresAt !== void 0 && params.reservationExpiresAt > reservation.expiresAt) {
|
|
3905
|
+
reservation = {
|
|
3906
|
+
...reservation,
|
|
3907
|
+
expiresAt: params.reservationExpiresAt
|
|
3908
|
+
};
|
|
3909
|
+
await txn.put(QUEUE_RESERVATION_KEY, reservation);
|
|
3910
|
+
}
|
|
3911
|
+
await txn.setAlarm(Math.min(expiresAt, reservation?.expiresAt ?? expiresAt));
|
|
3912
|
+
return { held: true };
|
|
3913
|
+
});
|
|
3914
|
+
}
|
|
3915
|
+
async releaseInflight(messageId) {
|
|
3916
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3917
|
+
const values = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
3918
|
+
const existing = values.get("claim");
|
|
3919
|
+
const reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
3920
|
+
if (!existing || messageId !== void 0 && existing.messageId !== messageId) return;
|
|
3921
|
+
await txn.delete("claim");
|
|
3922
|
+
if (reservation) await txn.setAlarm(reservation.expiresAt);
|
|
3923
|
+
else await txn.deleteAlarm();
|
|
3924
|
+
});
|
|
4357
3925
|
}
|
|
4358
3926
|
};
|
|
4359
3927
|
//#endregion
|
|
@@ -4364,13 +3932,29 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4364
3932
|
*/
|
|
4365
3933
|
async function startHarness(options = {}) {
|
|
4366
3934
|
const cellEnv = { ...options.cellEnv };
|
|
3935
|
+
const queueMessages = [];
|
|
3936
|
+
const queuePayloads = /* @__PURE__ */ new Map();
|
|
3937
|
+
const queuePayloadBucket = {
|
|
3938
|
+
async put(key, value) {
|
|
3939
|
+
queuePayloads.set(key, value);
|
|
3940
|
+
},
|
|
3941
|
+
async get(key) {
|
|
3942
|
+
const value = queuePayloads.get(key);
|
|
3943
|
+
return value === void 0 ? null : { text: async () => value };
|
|
3944
|
+
},
|
|
3945
|
+
async delete(keys) {
|
|
3946
|
+
for (const key of Array.isArray(keys) ? keys : [keys]) queuePayloads.delete(key);
|
|
3947
|
+
}
|
|
3948
|
+
};
|
|
3949
|
+
const nativeQueue = { async send(body) {
|
|
3950
|
+
queueMessages.push(body);
|
|
3951
|
+
} };
|
|
4367
3952
|
const fleet = new FakeFleet({
|
|
4368
3953
|
runs: WorkflowRunDO,
|
|
4369
3954
|
streams: StreamDO,
|
|
4370
3955
|
"run-catalog": RunCatalogDO,
|
|
4371
3956
|
"hook-tokens": HookTokenDO,
|
|
4372
3957
|
"hook-ids": HookIdDO,
|
|
4373
|
-
queue: QueueDO,
|
|
4374
3958
|
...options.extraClasses
|
|
4375
3959
|
}, cellEnv);
|
|
4376
3960
|
cellEnv.clock ??= options.virtualClock ? () => fleet.now : () => Date.now();
|
|
@@ -4380,7 +3964,7 @@ async function startHarness(options = {}) {
|
|
|
4380
3964
|
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
4381
3965
|
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
4382
3966
|
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
4383
|
-
|
|
3967
|
+
WORKFLOW_QUEUE_PAYLOADS: queuePayloadBucket
|
|
4384
3968
|
});
|
|
4385
3969
|
const router = createRouter({
|
|
4386
3970
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
@@ -4388,7 +3972,8 @@ async function startHarness(options = {}) {
|
|
|
4388
3972
|
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
4389
3973
|
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
4390
3974
|
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
4391
|
-
WORKFLOW_QUEUE:
|
|
3975
|
+
WORKFLOW_QUEUE: nativeQueue,
|
|
3976
|
+
WORKFLOW_QUEUE_PAYLOADS: createQueuePayloadStore(queuePayloadBucket),
|
|
4392
3977
|
WORLD_SECRET: options.secret
|
|
4393
3978
|
});
|
|
4394
3979
|
const server = http.createServer(async (req, res) => {
|
|
@@ -4432,6 +4017,8 @@ async function startHarness(options = {}) {
|
|
|
4432
4017
|
return {
|
|
4433
4018
|
url: `http://127.0.0.1:${port}`,
|
|
4434
4019
|
fleet,
|
|
4020
|
+
queueMessages,
|
|
4021
|
+
queuePayloads,
|
|
4435
4022
|
close: () => new Promise((resolve, reject) => {
|
|
4436
4023
|
server.close((err) => err ? reject(err) : resolve());
|
|
4437
4024
|
server.closeAllConnections();
|