@ewhauser/world-celld 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/NOTICE +1 -1
- package/README.md +106 -106
- package/celld-queue-worker/README.md +26 -0
- package/celld-queue-worker/worker.ts +3 -0
- package/celld-queue-worker/wrangler.jsonc +26 -0
- package/celld-worker/README.md +33 -25
- package/celld-worker/worker.ts +1 -1
- package/celld-worker/wrangler.jsonc +12 -8
- package/dist/index.d.ts +32 -70
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +30 -88
- package/dist/index.js.map +1 -1
- package/dist/queue-consumer.d.ts +33 -0
- package/dist/queue-consumer.d.ts.map +1 -0
- package/dist/queue-consumer.js +75 -0
- package/dist/queue-consumer.js.map +1 -0
- package/dist/queue-delivery-DzNOtCYA.d.ts +49 -0
- package/dist/queue-delivery-DzNOtCYA.d.ts.map +1 -0
- package/dist/{util-Cu09mVzK.js → queue-protocol-CmJNMH_5.js} +60 -12
- package/dist/queue-protocol-CmJNMH_5.js.map +1 -0
- package/dist/queue-protocol-Ds44uuND.js +104 -0
- package/dist/queue-protocol-Ds44uuND.js.map +1 -0
- package/dist/testing.d.ts +3 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +420 -833
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +106 -181
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +602 -955
- package/dist/worker.js.map +1 -1
- package/docs/celld-v0.5-upgrade.md +130 -0
- package/package.json +11 -3
- package/dist/util-Cu09mVzK.js.map +0 -1
package/dist/worker.js
CHANGED
|
@@ -1,6 +1,181 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { DurableObject } from "cloudflare:workers";
|
|
3
|
-
import {
|
|
1
|
+
import { _ as isRecord$1, a as queueOrphanName, c as validateNativeQueueEnvelope, d as HOOK_CLAIM_LEASE_MS, f as LIFECYCLE_COMPACTION_RETRY_MS, g as isPositiveSafeInteger, h as isNonNegativeSafeInteger, i as queueClaimName, l as validateNativeQueueSendOptions, m as QUEUE_CLAIM_STALE_MS, n as QUEUE_PAYLOAD_REGISTRY_PREFIX, o as queuePayloadObjectKey, p as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, s as queuePayloadRegistryKey, t as NATIVE_QUEUE_MAX_MESSAGE_BYTES, v as queueDelayDeadline, y as strictIntegerSetting } from "./queue-protocol-Ds44uuND.js";
|
|
2
|
+
import { DurableObject, WorkerEntrypoint } from "cloudflare:workers";
|
|
3
|
+
import { CreateEventSchema, EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
4
|
+
//#region src/worker/auth.ts
|
|
5
|
+
/**
|
|
6
|
+
* Bearer-token auth for the worker router.
|
|
7
|
+
*
|
|
8
|
+
* Fails closed: when the WORLD_SECRET var is not configured, every protected
|
|
9
|
+
* route returns 503 instead of forwarding unauthenticated traffic (the
|
|
10
|
+
* eve-ambient celld-worker pattern).
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Constant-time string comparison via SHA-256 digests (digest-then-compare
|
|
14
|
+
* is length-independent). crypto.subtle is global in both workerd and Node.
|
|
15
|
+
*/
|
|
16
|
+
async function timingSafeEqual(a, b) {
|
|
17
|
+
const enc = new TextEncoder();
|
|
18
|
+
const [da, db] = await Promise.all([crypto.subtle.digest("SHA-256", enc.encode(a)), crypto.subtle.digest("SHA-256", enc.encode(b))]);
|
|
19
|
+
const va = new Uint8Array(da);
|
|
20
|
+
const vb = new Uint8Array(db);
|
|
21
|
+
let diff = 0;
|
|
22
|
+
for (let i = 0; i < va.length; i++) diff |= va[i] ^ vb[i];
|
|
23
|
+
return diff === 0;
|
|
24
|
+
}
|
|
25
|
+
async function authenticate(request, secret) {
|
|
26
|
+
if (!secret) return {
|
|
27
|
+
ok: false,
|
|
28
|
+
response: Response.json({ error: {
|
|
29
|
+
name: "WorldNotConfigured",
|
|
30
|
+
message: "WORLD_SECRET is not configured"
|
|
31
|
+
} }, { status: 503 })
|
|
32
|
+
};
|
|
33
|
+
const header = request.headers.get("authorization") ?? "";
|
|
34
|
+
const token = header.startsWith("Bearer ") ? header.slice(7) : "";
|
|
35
|
+
if (!token || !await timingSafeEqual(token, secret)) return {
|
|
36
|
+
ok: false,
|
|
37
|
+
response: Response.json({ error: {
|
|
38
|
+
name: "Unauthorized",
|
|
39
|
+
message: "invalid or missing bearer token"
|
|
40
|
+
} }, { status: 401 })
|
|
41
|
+
};
|
|
42
|
+
return { ok: true };
|
|
43
|
+
}
|
|
44
|
+
//#endregion
|
|
45
|
+
//#region src/worker/queue-delivery.ts
|
|
46
|
+
/** Internal delivery contract. No Request/Response objects cross the RPC boundary. */
|
|
47
|
+
const MAX_QUEUE_DELIVERY_RESPONSE_BYTES = 65536;
|
|
48
|
+
const QUEUE_ORPHAN_GRACE_MS$1 = 432e6;
|
|
49
|
+
const PERMANENT_QUEUE_STATUSES = /* @__PURE__ */ new Set([
|
|
50
|
+
404,
|
|
51
|
+
409,
|
|
52
|
+
410,
|
|
53
|
+
422
|
|
54
|
+
]);
|
|
55
|
+
async function readResponseText(response) {
|
|
56
|
+
if (!response.body) return "";
|
|
57
|
+
const reader = response.body.getReader();
|
|
58
|
+
const decoder = new TextDecoder();
|
|
59
|
+
let total = 0;
|
|
60
|
+
let text = "";
|
|
61
|
+
try {
|
|
62
|
+
for (;;) {
|
|
63
|
+
const { done, value } = await reader.read();
|
|
64
|
+
if (done) break;
|
|
65
|
+
total += value.byteLength;
|
|
66
|
+
if (total > MAX_QUEUE_DELIVERY_RESPONSE_BYTES) {
|
|
67
|
+
await reader.cancel("queue delivery response is too large").catch(() => void 0);
|
|
68
|
+
throw new Error("workflow queue handler response exceeds configured limit");
|
|
69
|
+
}
|
|
70
|
+
text += decoder.decode(value, { stream: true });
|
|
71
|
+
}
|
|
72
|
+
return text + decoder.decode();
|
|
73
|
+
} finally {
|
|
74
|
+
reader.releaseLock();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/** Validate at the service boundary; callers cannot bypass auth via RPC. */
|
|
78
|
+
async function deliverQueueMessage(env, secret, input, attempt) {
|
|
79
|
+
if (!env.WORLD_SECRET) throw new Error("WORLD_SECRET is not configured");
|
|
80
|
+
if (typeof secret !== "string" || !secret || !await timingSafeEqual(secret, env.WORLD_SECRET)) throw new Error("Unauthorized Queue delivery");
|
|
81
|
+
const envelope = validateNativeQueueEnvelope(input);
|
|
82
|
+
if (!Number.isSafeInteger(attempt) || attempt < 1) throw new TypeError("queue delivery attempt must be a positive safe integer");
|
|
83
|
+
if (!env.WORKFLOW_DB) throw new Error("missing binding: WORKFLOW_DB");
|
|
84
|
+
const namespace = env.WORKFLOW_DB;
|
|
85
|
+
const payloadStore = env.WORKFLOW_QUEUE_PAYLOADS;
|
|
86
|
+
const runStub = (name) => namespace.get(namespace.idFromName(name));
|
|
87
|
+
let claim;
|
|
88
|
+
let claimed = false;
|
|
89
|
+
async function releaseClaim() {
|
|
90
|
+
if (claimed) await claim.releaseInflight(envelope.messageId);
|
|
91
|
+
}
|
|
92
|
+
async function unregisterPayload() {
|
|
93
|
+
if (envelope.runId) {
|
|
94
|
+
await runStub(envelope.runId).unregisterQueuePayload(envelope.messageId);
|
|
95
|
+
await runStub(queueOrphanName(envelope.messageId)).cancelQueuePayloadOrphan(envelope.messageId);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
if (envelope.idempotencyKey) {
|
|
100
|
+
claim = runStub(queueClaimName(envelope.queueName, envelope.idempotencyKey));
|
|
101
|
+
const result = await claim.claimInflight({
|
|
102
|
+
messageId: envelope.messageId,
|
|
103
|
+
staleMs: QUEUE_CLAIM_STALE_MS
|
|
104
|
+
});
|
|
105
|
+
if (!result.claimed) {
|
|
106
|
+
const now = Date.now();
|
|
107
|
+
if (result.retryAt !== void 0 && result.retryAt > now) return {
|
|
108
|
+
kind: "suspend",
|
|
109
|
+
timeoutSeconds: Math.max(1, Math.ceil((result.retryAt - now) / 1e3))
|
|
110
|
+
};
|
|
111
|
+
return { kind: "complete" };
|
|
112
|
+
}
|
|
113
|
+
claimed = true;
|
|
114
|
+
}
|
|
115
|
+
let body = envelope.body;
|
|
116
|
+
if (envelope.payloadKey) {
|
|
117
|
+
if (!payloadStore) throw new Error("missing binding: WORKFLOW_QUEUE_PAYLOADS");
|
|
118
|
+
body = await payloadStore.read(envelope.payloadKey) ?? void 0;
|
|
119
|
+
if (body === void 0) {
|
|
120
|
+
await unregisterPayload();
|
|
121
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
122
|
+
return { kind: "complete" };
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const headers = {
|
|
126
|
+
"content-type": "application/json",
|
|
127
|
+
"x-vqs-queue-name": envelope.queueName,
|
|
128
|
+
"x-vqs-message-id": envelope.messageId,
|
|
129
|
+
"x-vqs-message-attempt": String(attempt)
|
|
130
|
+
};
|
|
131
|
+
if (env.WORKFLOW_CALLBACK_SECRET) headers["x-workflow-callback-secret"] = env.WORKFLOW_CALLBACK_SECRET;
|
|
132
|
+
const callback = await fetch(`${envelope.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/flow`, {
|
|
133
|
+
method: "POST",
|
|
134
|
+
headers,
|
|
135
|
+
body,
|
|
136
|
+
signal: AbortSignal.timeout(3e5)
|
|
137
|
+
});
|
|
138
|
+
if (callback.ok || PERMANENT_QUEUE_STATUSES.has(callback.status)) {
|
|
139
|
+
await callback.body?.cancel().catch(() => void 0);
|
|
140
|
+
if (envelope.payloadKey) {
|
|
141
|
+
await payloadStore.delete(envelope.payloadKey);
|
|
142
|
+
await unregisterPayload();
|
|
143
|
+
}
|
|
144
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
145
|
+
return { kind: "complete" };
|
|
146
|
+
}
|
|
147
|
+
if (callback.status === 503) {
|
|
148
|
+
const responseBody = await readResponseText(callback);
|
|
149
|
+
const timeoutSeconds = JSON.parse(responseBody).timeoutSeconds;
|
|
150
|
+
const notBefore = queueDelayDeadline(Date.now(), timeoutSeconds, 1, QUEUE_CLAIM_STALE_MS);
|
|
151
|
+
if (notBefore === null) throw new Error("workflow queue suspension deadline is invalid");
|
|
152
|
+
if (envelope.payloadKey && envelope.runId) await runStub(queueOrphanName(envelope.messageId)).scheduleQueuePayloadOrphan({
|
|
153
|
+
messageId: envelope.messageId,
|
|
154
|
+
runId: envelope.runId,
|
|
155
|
+
key: envelope.payloadKey,
|
|
156
|
+
expiresAt: notBefore + QUEUE_ORPHAN_GRACE_MS$1
|
|
157
|
+
});
|
|
158
|
+
if (claim) await claim.holdInflight({
|
|
159
|
+
messageId: envelope.messageId,
|
|
160
|
+
retryAt: notBefore,
|
|
161
|
+
expiresAt: notBefore + QUEUE_CLAIM_STALE_MS,
|
|
162
|
+
reservationExpiresAt: notBefore + QUEUE_ORPHAN_GRACE_MS$1
|
|
163
|
+
});
|
|
164
|
+
return {
|
|
165
|
+
kind: "suspend",
|
|
166
|
+
timeoutSeconds
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
await callback.body?.cancel().catch(() => void 0);
|
|
170
|
+
await releaseClaim();
|
|
171
|
+
claimed = false;
|
|
172
|
+
return { kind: "retry" };
|
|
173
|
+
} catch (error) {
|
|
174
|
+
await releaseClaim();
|
|
175
|
+
throw error;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
//#endregion
|
|
4
179
|
//#region src/vendor/shared/serialization.ts
|
|
5
180
|
/** Standard Date fields used across all worlds */
|
|
6
181
|
const DATE_FIELDS = /* @__PURE__ */ new Set([
|
|
@@ -100,13 +275,13 @@ const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
|
|
|
100
275
|
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
101
276
|
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
102
277
|
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
103
|
-
const STORAGE_BATCH_SIZE$
|
|
278
|
+
const STORAGE_BATCH_SIZE$1 = 128;
|
|
104
279
|
const MAX_ENTITY_PAGE_SIZE = 1e3;
|
|
105
|
-
async function getMany
|
|
280
|
+
async function getMany(store, keys) {
|
|
106
281
|
if (store.getMany) {
|
|
107
282
|
const result = /* @__PURE__ */ new Map();
|
|
108
|
-
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$
|
|
109
|
-
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$
|
|
283
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
284
|
+
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
110
285
|
for (const entry of page) result.set(...entry);
|
|
111
286
|
}
|
|
112
287
|
return result;
|
|
@@ -114,15 +289,15 @@ async function getMany$1(store, keys) {
|
|
|
114
289
|
const values = await Promise.all(keys.map((key) => store.get(key)));
|
|
115
290
|
return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
|
|
116
291
|
}
|
|
117
|
-
async function deleteMany
|
|
292
|
+
async function deleteMany(store, keys) {
|
|
118
293
|
if (store.deleteMany) {
|
|
119
294
|
let deleted = 0;
|
|
120
|
-
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$
|
|
295
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
121
296
|
return deleted;
|
|
122
297
|
}
|
|
123
298
|
return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
|
|
124
299
|
}
|
|
125
|
-
function isRecord
|
|
300
|
+
function isRecord(value) {
|
|
126
301
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
127
302
|
}
|
|
128
303
|
/**
|
|
@@ -131,12 +306,12 @@ function isRecord$1(value) {
|
|
|
131
306
|
* events cannot allocate a sequence or touch hook/entity storage.
|
|
132
307
|
*/
|
|
133
308
|
function parseApplyEventRequest(value) {
|
|
134
|
-
if (!isRecord
|
|
309
|
+
if (!isRecord(value)) throw new TypeError("applyEvent request must be an object");
|
|
135
310
|
if (typeof value.runId !== "string" || value.runId.length === 0) throw new TypeError("applyEvent runId must be a non-empty string");
|
|
136
|
-
const data = CreateEventSchema.parse(isRecord
|
|
311
|
+
const data = CreateEventSchema.parse(isRecord(value.data) ? compact(value.data) : value.data);
|
|
137
312
|
const params = value.params;
|
|
138
313
|
if (params !== void 0) {
|
|
139
|
-
if (!isRecord
|
|
314
|
+
if (!isRecord(params)) throw new TypeError("applyEvent params must be an object");
|
|
140
315
|
if (params.occurredAt !== void 0 && (!(params.occurredAt instanceof Date) || Number.isNaN(params.occurredAt.getTime()))) throw new TypeError("applyEvent params.occurredAt must be a valid Date");
|
|
141
316
|
for (const key of [
|
|
142
317
|
"resumeId",
|
|
@@ -149,7 +324,7 @@ function parseApplyEventRequest(value) {
|
|
|
149
324
|
let tokenHolder;
|
|
150
325
|
if (value.tokenHolder === null) tokenHolder = null;
|
|
151
326
|
else if (value.tokenHolder !== void 0) {
|
|
152
|
-
if (!isRecord
|
|
327
|
+
if (!isRecord(value.tokenHolder) || typeof value.tokenHolder.runId !== "string" || value.tokenHolder.runId.length === 0 || typeof value.tokenHolder.hookId !== "string" || value.tokenHolder.hookId.length === 0) throw new TypeError("applyEvent tokenHolder must contain non-empty runId and hookId strings");
|
|
153
328
|
tokenHolder = {
|
|
154
329
|
runId: value.tokenHolder.runId,
|
|
155
330
|
hookId: value.tokenHolder.hookId
|
|
@@ -246,7 +421,7 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
246
421
|
});
|
|
247
422
|
const page = Array.from(entries.entries()).slice(0, limit);
|
|
248
423
|
const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
|
|
249
|
-
const entities = await getMany
|
|
424
|
+
const entities = await getMany(store, entityKeys);
|
|
250
425
|
const data = entityKeys.flatMap((key) => {
|
|
251
426
|
const item = entities.get(key);
|
|
252
427
|
return item === void 0 ? [] : [item];
|
|
@@ -273,14 +448,14 @@ async function releaseAllHooks(store) {
|
|
|
273
448
|
token: hook.token
|
|
274
449
|
});
|
|
275
450
|
}
|
|
276
|
-
await deleteMany
|
|
451
|
+
await deleteMany(store, keys);
|
|
277
452
|
return released;
|
|
278
453
|
}
|
|
279
454
|
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
280
455
|
async function releaseAllWaits(store) {
|
|
281
456
|
if (store.deferTerminalCleanup) return;
|
|
282
457
|
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
283
|
-
await deleteMany
|
|
458
|
+
await deleteMany(store, Array.from(waits.keys()));
|
|
284
459
|
}
|
|
285
460
|
async function applyEvent(store, ctx) {
|
|
286
461
|
const { runId, data, now } = ctx;
|
|
@@ -1308,91 +1483,6 @@ function normalizeStreamError(error) {
|
|
|
1308
1483
|
return normalized;
|
|
1309
1484
|
}
|
|
1310
1485
|
//#endregion
|
|
1311
|
-
//#region src/worker/auth.ts
|
|
1312
|
-
/**
|
|
1313
|
-
* Bearer-token auth for the worker router.
|
|
1314
|
-
*
|
|
1315
|
-
* Fails closed: when the WORLD_SECRET var is not configured, every protected
|
|
1316
|
-
* route returns 503 instead of forwarding unauthenticated traffic (the
|
|
1317
|
-
* eve-ambient celld-worker pattern).
|
|
1318
|
-
*/
|
|
1319
|
-
/**
|
|
1320
|
-
* Constant-time string comparison via SHA-256 digests (digest-then-compare
|
|
1321
|
-
* is length-independent). crypto.subtle is global in both workerd and Node.
|
|
1322
|
-
*/
|
|
1323
|
-
async function timingSafeEqual(a, b) {
|
|
1324
|
-
const enc = new TextEncoder();
|
|
1325
|
-
const [da, db] = await Promise.all([crypto.subtle.digest("SHA-256", enc.encode(a)), crypto.subtle.digest("SHA-256", enc.encode(b))]);
|
|
1326
|
-
const va = new Uint8Array(da);
|
|
1327
|
-
const vb = new Uint8Array(db);
|
|
1328
|
-
let diff = 0;
|
|
1329
|
-
for (let i = 0; i < va.length; i++) diff |= va[i] ^ vb[i];
|
|
1330
|
-
return diff === 0;
|
|
1331
|
-
}
|
|
1332
|
-
async function authenticate(request, secret) {
|
|
1333
|
-
if (!secret) return {
|
|
1334
|
-
ok: false,
|
|
1335
|
-
response: Response.json({ error: {
|
|
1336
|
-
name: "WorldNotConfigured",
|
|
1337
|
-
message: "WORLD_SECRET is not configured"
|
|
1338
|
-
} }, { status: 503 })
|
|
1339
|
-
};
|
|
1340
|
-
const header = request.headers.get("authorization") ?? "";
|
|
1341
|
-
const token = header.startsWith("Bearer ") ? header.slice(7) : "";
|
|
1342
|
-
if (!token || !await timingSafeEqual(token, secret)) return {
|
|
1343
|
-
ok: false,
|
|
1344
|
-
response: Response.json({ error: {
|
|
1345
|
-
name: "Unauthorized",
|
|
1346
|
-
message: "invalid or missing bearer token"
|
|
1347
|
-
} }, { status: 401 })
|
|
1348
|
-
};
|
|
1349
|
-
return { ok: true };
|
|
1350
|
-
}
|
|
1351
|
-
const MAX_QUEUE_DELAY_SECONDS = Math.floor(9999999999999 / 1e3);
|
|
1352
|
-
function isRecord(value) {
|
|
1353
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1354
|
-
}
|
|
1355
|
-
function isNonNegativeSafeInteger(value) {
|
|
1356
|
-
return Number.isSafeInteger(value) && value >= 0;
|
|
1357
|
-
}
|
|
1358
|
-
function isPositiveSafeInteger(value) {
|
|
1359
|
-
return Number.isSafeInteger(value) && value > 0;
|
|
1360
|
-
}
|
|
1361
|
-
function boundedIntegerOption(name, value, fallback, minimum, maximum = Number.MAX_SAFE_INTEGER) {
|
|
1362
|
-
if (value === void 0) return fallback;
|
|
1363
|
-
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
|
|
1364
|
-
return value;
|
|
1365
|
-
}
|
|
1366
|
-
/**
|
|
1367
|
-
* Parse a deployment/runtime integer setting without accepting whitespace,
|
|
1368
|
-
* exponents, fractions, or numeric prefixes such as `5junk`.
|
|
1369
|
-
*/
|
|
1370
|
-
function strictIntegerSetting(name, raw, fallback, minimum, maximum = Number.MAX_SAFE_INTEGER) {
|
|
1371
|
-
if (raw === void 0) return fallback;
|
|
1372
|
-
const value = typeof raw === "number" ? raw : /^\d+$/.test(raw) ? Number(raw) : NaN;
|
|
1373
|
-
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
|
|
1374
|
-
return value;
|
|
1375
|
-
}
|
|
1376
|
-
function isValidQueueDelaySeconds(value, minimum = 0) {
|
|
1377
|
-
return Number.isSafeInteger(value) && value >= minimum && value <= MAX_QUEUE_DELAY_SECONDS;
|
|
1378
|
-
}
|
|
1379
|
-
function queueDelayDeadline(now, delaySeconds, minimum = 0, requiredHeadroomMs = 0) {
|
|
1380
|
-
if (!isNonNegativeSafeInteger(now) || !isValidQueueDelaySeconds(delaySeconds, minimum)) return null;
|
|
1381
|
-
return checkedQueueTimestampAdd(now, delaySeconds * 1e3, requiredHeadroomMs);
|
|
1382
|
-
}
|
|
1383
|
-
/**
|
|
1384
|
-
* Add an offset to a queue timestamp while preserving its 13-digit key order.
|
|
1385
|
-
* Optional headroom reserves space for a later persisted derivative deadline.
|
|
1386
|
-
*/
|
|
1387
|
-
function checkedQueueTimestampAdd(timestampMs, offsetMs, requiredHeadroomMs = 0) {
|
|
1388
|
-
if (!isNonNegativeSafeInteger(timestampMs) || !isNonNegativeSafeInteger(offsetMs) || !isNonNegativeSafeInteger(requiredHeadroomMs)) return null;
|
|
1389
|
-
const timestamp = timestampMs;
|
|
1390
|
-
const offset = offsetMs;
|
|
1391
|
-
if (timestamp > 9999999999999 || offset > 9999999999999 - timestamp) return null;
|
|
1392
|
-
const result = timestamp + offset;
|
|
1393
|
-
return requiredHeadroomMs <= 9999999999999 - result ? result : null;
|
|
1394
|
-
}
|
|
1395
|
-
//#endregion
|
|
1396
1486
|
//#region src/worker/index-validation.ts
|
|
1397
1487
|
function invalid(message) {
|
|
1398
1488
|
const error = /* @__PURE__ */ new Error(`invalid index RPC arguments: ${message}`);
|
|
@@ -1420,14 +1510,14 @@ function positiveSafeInteger(value, name) {
|
|
|
1420
1510
|
return parsed;
|
|
1421
1511
|
}
|
|
1422
1512
|
function owner(value, name = "owner") {
|
|
1423
|
-
if (!isRecord(value)) invalid(`${name} must be an object`);
|
|
1513
|
+
if (!isRecord$1(value)) invalid(`${name} must be an object`);
|
|
1424
1514
|
return {
|
|
1425
1515
|
runId: nonEmptyString(value.runId, `${name}.runId`),
|
|
1426
1516
|
hookId: nonEmptyString(value.hookId, `${name}.hookId`)
|
|
1427
1517
|
};
|
|
1428
1518
|
}
|
|
1429
1519
|
function reservation(value) {
|
|
1430
|
-
if (!isRecord(value)) invalid("reservation must be an object");
|
|
1520
|
+
if (!isRecord$1(value)) invalid("reservation must be an object");
|
|
1431
1521
|
const parsed = { claimId: nonEmptyString(value.claimId, "reservation.claimId") };
|
|
1432
1522
|
if (value.tokenClaimId !== void 0) parsed.tokenClaimId = nonEmptyString(value.tokenClaimId, "reservation.tokenClaimId");
|
|
1433
1523
|
if (value.hookIdClaimId !== void 0) parsed.hookIdClaimId = nonEmptyString(value.hookIdClaimId, "reservation.hookIdClaimId");
|
|
@@ -1436,7 +1526,7 @@ function reservation(value) {
|
|
|
1436
1526
|
function hooks(value) {
|
|
1437
1527
|
if (!Array.isArray(value)) invalid("hooks must be an array");
|
|
1438
1528
|
return value.map((entry, index) => {
|
|
1439
|
-
if (!isRecord(entry)) invalid(`hooks[${index}] must be an object`);
|
|
1529
|
+
if (!isRecord$1(entry)) invalid(`hooks[${index}] must be an object`);
|
|
1440
1530
|
return {
|
|
1441
1531
|
hookId: nonEmptyString(entry.hookId, `hooks[${index}].hookId`),
|
|
1442
1532
|
token: nonEmptyString(entry.token, `hooks[${index}].token`)
|
|
@@ -1448,14 +1538,14 @@ function exactProperties(value, allowed, name) {
|
|
|
1448
1538
|
if (unexpected !== void 0) invalid(`${name}.${unexpected} is not allowed`);
|
|
1449
1539
|
}
|
|
1450
1540
|
function releaseRequest(value) {
|
|
1451
|
-
if (!isRecord(value)) invalid("release request must be an object");
|
|
1541
|
+
if (!isRecord$1(value)) invalid("release request must be an object");
|
|
1452
1542
|
return {
|
|
1453
1543
|
runId: nonEmptyString(value.runId, "request.runId"),
|
|
1454
1544
|
hooks: hooks(value.hooks)
|
|
1455
1545
|
};
|
|
1456
1546
|
}
|
|
1457
1547
|
function expireRequest(value) {
|
|
1458
|
-
if (!isRecord(value)) invalid("expiry request must be an object");
|
|
1548
|
+
if (!isRecord$1(value)) invalid("expiry request must be an object");
|
|
1459
1549
|
exactProperties(value, [
|
|
1460
1550
|
"runId",
|
|
1461
1551
|
"hooks",
|
|
@@ -1469,7 +1559,7 @@ function expireRequest(value) {
|
|
|
1469
1559
|
}
|
|
1470
1560
|
function listOptions(value) {
|
|
1471
1561
|
if (value === void 0 || value === null) return void 0;
|
|
1472
|
-
if (!isRecord(value)) invalid("list options must be an object");
|
|
1562
|
+
if (!isRecord$1(value)) invalid("list options must be an object");
|
|
1473
1563
|
const parsed = {};
|
|
1474
1564
|
for (const name of [
|
|
1475
1565
|
"prefix",
|
|
@@ -1502,7 +1592,7 @@ function runCommit(args) {
|
|
|
1502
1592
|
} catch {
|
|
1503
1593
|
invalid("serializedMetadata must be valid JSON");
|
|
1504
1594
|
}
|
|
1505
|
-
if (!isRecord(metadata) || metadata.runId !== run.runId || metadata.status !== run.status || metadata.createdAt !== run.createdAt.toISOString()) invalid("serializedMetadata must match the committed run");
|
|
1595
|
+
if (!isRecord$1(metadata) || metadata.runId !== run.runId || metadata.status !== run.status || metadata.createdAt !== run.createdAt.toISOString()) invalid("serializedMetadata must match the committed run");
|
|
1506
1596
|
return {
|
|
1507
1597
|
operation: "runs.commit",
|
|
1508
1598
|
args: [
|
|
@@ -1651,24 +1741,12 @@ const BINDINGS = {
|
|
|
1651
1741
|
"hook-ids": {
|
|
1652
1742
|
env: "WORKFLOW_HOOK_IDS",
|
|
1653
1743
|
methods: /* @__PURE__ */ new Set(["get"])
|
|
1654
|
-
},
|
|
1655
|
-
queue: {
|
|
1656
|
-
env: "WORKFLOW_QUEUE",
|
|
1657
|
-
methods: /* @__PURE__ */ new Set([
|
|
1658
|
-
"enqueue",
|
|
1659
|
-
"stats",
|
|
1660
|
-
"listDeadLetters",
|
|
1661
|
-
"redriveDeadLetter",
|
|
1662
|
-
"purgeDeadLetters",
|
|
1663
|
-
"rearmAlarm",
|
|
1664
|
-
"expireRun",
|
|
1665
|
-
"acknowledgeExpireRun"
|
|
1666
|
-
])
|
|
1667
1744
|
}
|
|
1668
1745
|
};
|
|
1669
1746
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
1670
1747
|
const MAX_BODY_BYTES = 33554432;
|
|
1671
1748
|
const MAX_STREAM_WRITE_BODY_BYTES = 8389632;
|
|
1749
|
+
const QUEUE_ORPHAN_GRACE_MS = 432e6;
|
|
1672
1750
|
function errorResponse(status, name, message) {
|
|
1673
1751
|
return Response.json({ error: {
|
|
1674
1752
|
name,
|
|
@@ -1752,6 +1830,25 @@ async function readBoundedBytes(request, maxBytes) {
|
|
|
1752
1830
|
}
|
|
1753
1831
|
return body;
|
|
1754
1832
|
}
|
|
1833
|
+
async function parseRpcArguments(request) {
|
|
1834
|
+
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
1835
|
+
if (!hasContentType(request, "application/json")) return errorResponse(415, "UnsupportedMediaType", "expected application/json");
|
|
1836
|
+
let contentLength;
|
|
1837
|
+
try {
|
|
1838
|
+
contentLength = declaredContentLength(request);
|
|
1839
|
+
} catch (error) {
|
|
1840
|
+
return errorResponse(400, "BadRequest", error.message);
|
|
1841
|
+
}
|
|
1842
|
+
if (contentLength !== null && contentLength > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
1843
|
+
try {
|
|
1844
|
+
const text = await readBoundedBody(request);
|
|
1845
|
+
if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
1846
|
+
const parsed = rpcParse(text);
|
|
1847
|
+
return Array.isArray(parsed) ? parsed : errorResponse(400, "BadRequest", "body must be an argument array");
|
|
1848
|
+
} catch {
|
|
1849
|
+
return errorResponse(400, "BadRequest", "malformed rpc body");
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1755
1852
|
function parseBoundedInteger(url, name, minimum, maximum) {
|
|
1756
1853
|
const raw = url.searchParams.get(name);
|
|
1757
1854
|
const value = raw !== null && /^\d+$/.test(raw) ? Number(raw) : NaN;
|
|
@@ -1784,6 +1881,111 @@ function createRouter(env) {
|
|
|
1784
1881
|
}
|
|
1785
1882
|
const auth = await authenticate(request, env.WORLD_SECRET);
|
|
1786
1883
|
if (!auth.ok) return auth.response;
|
|
1884
|
+
if (parts[1] === "queue" && parts.length === 3) {
|
|
1885
|
+
const parsed = await parseRpcArguments(request);
|
|
1886
|
+
if (parsed instanceof Response) return parsed;
|
|
1887
|
+
if (parts[2] === "send") {
|
|
1888
|
+
if (parsed.length < 1 || parsed.length > 2) return errorResponse(400, "BadRequest", "queue send expects an envelope and options");
|
|
1889
|
+
if (!env.WORKFLOW_QUEUE) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_QUEUE");
|
|
1890
|
+
let envelope;
|
|
1891
|
+
let options;
|
|
1892
|
+
try {
|
|
1893
|
+
envelope = validateNativeQueueEnvelope(parsed[0]);
|
|
1894
|
+
options = validateNativeQueueSendOptions(parsed[1]);
|
|
1895
|
+
if (envelope.body === void 0 || envelope.payloadKey !== void 0) throw new TypeError("queue send requires an inline body");
|
|
1896
|
+
} catch (error) {
|
|
1897
|
+
return errorResponse(400, "BadRequest", error.message);
|
|
1898
|
+
}
|
|
1899
|
+
try {
|
|
1900
|
+
const namespace = env.WORKFLOW_DB;
|
|
1901
|
+
const payloadStore = env.WORKFLOW_QUEUE_PAYLOADS;
|
|
1902
|
+
const payloadKey = envelope.runId ? queuePayloadObjectKey(envelope.runId, envelope.messageId) : void 0;
|
|
1903
|
+
const brokerEnvelope = payloadKey ? {
|
|
1904
|
+
...envelope,
|
|
1905
|
+
payloadKey,
|
|
1906
|
+
body: void 0
|
|
1907
|
+
} : envelope;
|
|
1908
|
+
const encoded = JSON.stringify(brokerEnvelope);
|
|
1909
|
+
if (new TextEncoder().encode(encoded).byteLength > 128e3) return errorResponse(413, "PayloadTooLarge", `native queue envelope exceeds ${NATIVE_QUEUE_MAX_MESSAGE_BYTES} bytes`);
|
|
1910
|
+
const now = Date.now();
|
|
1911
|
+
const reservationExpiresAt = Math.max(now, envelope.notBefore ?? now) + QUEUE_ORPHAN_GRACE_MS;
|
|
1912
|
+
let claim;
|
|
1913
|
+
if (envelope.idempotencyKey) {
|
|
1914
|
+
claim = namespace.get(namespace.idFromName(queueClaimName(envelope.queueName, envelope.idempotencyKey)));
|
|
1915
|
+
const reservation = await claim.reserveQueueMessage({
|
|
1916
|
+
messageId: envelope.messageId,
|
|
1917
|
+
expiresAt: reservationExpiresAt
|
|
1918
|
+
});
|
|
1919
|
+
if (!reservation.admitted) return new Response(rpcStringify({ messageId: reservation.messageId }), {
|
|
1920
|
+
status: 200,
|
|
1921
|
+
headers: { "content-type": "application/json" }
|
|
1922
|
+
});
|
|
1923
|
+
}
|
|
1924
|
+
if (envelope.runId && payloadKey) {
|
|
1925
|
+
if (!payloadStore) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_QUEUE_PAYLOADS");
|
|
1926
|
+
const run = namespace.get(namespace.idFromName(envelope.runId));
|
|
1927
|
+
const orphanExpiresAt = reservationExpiresAt;
|
|
1928
|
+
const registered = await run.registerQueuePayload({
|
|
1929
|
+
messageId: envelope.messageId,
|
|
1930
|
+
key: payloadKey,
|
|
1931
|
+
orphanExpiresAt
|
|
1932
|
+
});
|
|
1933
|
+
if (!registered.ok) {
|
|
1934
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1935
|
+
return errorResponse(410, "RunExpiredError", registered.message);
|
|
1936
|
+
}
|
|
1937
|
+
const orphan = namespace.get(namespace.idFromName(queueOrphanName(envelope.messageId)));
|
|
1938
|
+
try {
|
|
1939
|
+
await orphan.scheduleQueuePayloadOrphan({
|
|
1940
|
+
messageId: envelope.messageId,
|
|
1941
|
+
runId: envelope.runId,
|
|
1942
|
+
key: payloadKey,
|
|
1943
|
+
expiresAt: orphanExpiresAt
|
|
1944
|
+
});
|
|
1945
|
+
} catch (error) {
|
|
1946
|
+
await run.unregisterQueuePayload(envelope.messageId);
|
|
1947
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1948
|
+
throw error;
|
|
1949
|
+
}
|
|
1950
|
+
try {
|
|
1951
|
+
await payloadStore.write(payloadKey, envelope.body, {
|
|
1952
|
+
runId: envelope.runId,
|
|
1953
|
+
messageId: envelope.messageId
|
|
1954
|
+
});
|
|
1955
|
+
} catch (error) {
|
|
1956
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1957
|
+
throw error;
|
|
1958
|
+
}
|
|
1959
|
+
let finalized;
|
|
1960
|
+
try {
|
|
1961
|
+
finalized = await run.finalizeQueuePayload(envelope.messageId);
|
|
1962
|
+
} catch (error) {
|
|
1963
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1964
|
+
throw error;
|
|
1965
|
+
}
|
|
1966
|
+
if (!finalized.ok) {
|
|
1967
|
+
await payloadStore.delete(payloadKey);
|
|
1968
|
+
await run.unregisterQueuePayload(envelope.messageId);
|
|
1969
|
+
await orphan.cancelQueuePayloadOrphan(envelope.messageId);
|
|
1970
|
+
await claim?.completeQueueMessage(envelope.messageId);
|
|
1971
|
+
return errorResponse(410, "RunExpiredError", finalized.message);
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
await env.WORKFLOW_QUEUE.send(encoded, {
|
|
1975
|
+
contentType: "text",
|
|
1976
|
+
...options
|
|
1977
|
+
});
|
|
1978
|
+
return new Response(rpcStringify({ messageId: envelope.messageId }), {
|
|
1979
|
+
status: 200,
|
|
1980
|
+
headers: { "content-type": "application/json" }
|
|
1981
|
+
});
|
|
1982
|
+
} catch (error) {
|
|
1983
|
+
const err = error;
|
|
1984
|
+
return errorResponse(500, err.name ?? "Error", err.message ?? String(error));
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
return errorResponse(404, "NotFound", `unknown queue operation: ${parts[2]}`);
|
|
1988
|
+
}
|
|
1787
1989
|
if (parts[1] === "index" && parts.length === 4) {
|
|
1788
1990
|
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
1789
1991
|
if (!hasContentType(request, "application/json")) return errorResponse(415, "UnsupportedMediaType", "expected application/json");
|
|
@@ -2010,7 +2212,7 @@ function catalogCandidate(name, value) {
|
|
|
2010
2212
|
const parsed = JSON.parse(value);
|
|
2011
2213
|
return {
|
|
2012
2214
|
runId,
|
|
2013
|
-
invalid: !isRecord(parsed) || parsed.runId !== runId
|
|
2215
|
+
invalid: !isRecord$1(parsed) || parsed.runId !== runId
|
|
2014
2216
|
};
|
|
2015
2217
|
} catch {
|
|
2016
2218
|
return {
|
|
@@ -2038,7 +2240,6 @@ async function runRetentionSweep(scheduledTime, env) {
|
|
|
2038
2240
|
preserved: 0
|
|
2039
2241
|
};
|
|
2040
2242
|
const batchSize = strictIntegerSetting("world-celld: WORKFLOW_RETENTION_BATCH_SIZE", env.WORKFLOW_RETENTION_BATCH_SIZE, 128, 1, MAX_RETENTION_SWEEP_BATCH_SIZE);
|
|
2041
|
-
const queueShards = strictIntegerSetting("world-celld: WORKFLOW_RETENTION_QUEUE_SHARDS", env.WORKFLOW_RETENTION_QUEUE_SHARDS, 1, 1);
|
|
2042
2243
|
const cutoff = scheduledTime - retentionMs;
|
|
2043
2244
|
const runNamespace = requireNamespace("WORKFLOW_DB", env.WORKFLOW_DB);
|
|
2044
2245
|
const catalogNamespace = requireNamespace("WORKFLOW_RUN_CATALOG", env.WORKFLOW_RUN_CATALOG);
|
|
@@ -2075,7 +2276,6 @@ async function runRetentionSweep(scheduledTime, env) {
|
|
|
2075
2276
|
};
|
|
2076
2277
|
const outcome = await runNamespace.get(runNamespace.idFromName(runId)).enforceRetention({
|
|
2077
2278
|
retentionMs,
|
|
2078
|
-
queueShards,
|
|
2079
2279
|
scheduledTime
|
|
2080
2280
|
});
|
|
2081
2281
|
if (outcome.state === "missing" || outcome.state === "not-due") {
|
|
@@ -2100,6 +2300,24 @@ async function runRetentionSweep(scheduledTime, env) {
|
|
|
2100
2300
|
return result;
|
|
2101
2301
|
}
|
|
2102
2302
|
//#endregion
|
|
2303
|
+
//#region src/worker/queue-payload-store.ts
|
|
2304
|
+
function createQueuePayloadStore(binding) {
|
|
2305
|
+
return {
|
|
2306
|
+
async write(key, value, metadata) {
|
|
2307
|
+
await binding.put(key, value, { customMetadata: metadata });
|
|
2308
|
+
},
|
|
2309
|
+
async read(key) {
|
|
2310
|
+
return (await binding.get(key))?.text() ?? null;
|
|
2311
|
+
},
|
|
2312
|
+
async delete(keys) {
|
|
2313
|
+
await binding.delete(keys);
|
|
2314
|
+
}
|
|
2315
|
+
};
|
|
2316
|
+
}
|
|
2317
|
+
async function deleteQueuePayloadObjects(binding, keys) {
|
|
2318
|
+
await binding.delete(keys);
|
|
2319
|
+
}
|
|
2320
|
+
//#endregion
|
|
2103
2321
|
//#region src/worker/durable-objects/hook-index-shared.ts
|
|
2104
2322
|
function sameOwner(left, right) {
|
|
2105
2323
|
return left.runId === right.runId && left.hookId === right.hookId;
|
|
@@ -2120,32 +2338,10 @@ async function runIsActive(env, runId) {
|
|
|
2120
2338
|
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
|
|
2121
2339
|
}
|
|
2122
2340
|
//#endregion
|
|
2123
|
-
//#region src/lifecycle.ts
|
|
2124
|
-
/**
|
|
2125
|
-
* Hard protocol bounds used by lifecycle leases and derivative-fence cleanup.
|
|
2126
|
-
*
|
|
2127
|
-
* These are not retention TTLs. They bound how long a request which was
|
|
2128
|
-
* admitted before expiry may still mutate a derivative Durable Object:
|
|
2129
|
-
*
|
|
2130
|
-
* - fleet calls have at most three 5-minute attempts plus bounded retry delay;
|
|
2131
|
-
* - one queue delivery owns its lease for at most 5 minutes plus 30 seconds;
|
|
2132
|
-
* - hook admission spans reserve retries, one authoritative apply, and
|
|
2133
|
-
* finalize retries before the exact claim must be reacquired.
|
|
2134
|
-
*/
|
|
2135
|
-
const MAX_FLEET_RPC_TIMEOUT_MS = 3e5;
|
|
2136
|
-
const MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS = 1200900;
|
|
2137
|
-
const MAX_QUEUE_DELIVERY_TIMEOUT_MS = 3e5;
|
|
2138
|
-
const QUEUE_INFLIGHT_GRACE_MS = 3e4;
|
|
2139
|
-
const MAX_QUEUE_INFLIGHT_LEASE_MS = 33e4;
|
|
2140
|
-
const MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS = 330001;
|
|
2141
|
-
const QUEUE_FENCE_GRACE_MS = Math.max(MAX_FLEET_RPC_TIMEOUT_MS, MAX_QUEUE_INFLIGHT_LEASE_MS);
|
|
2142
|
-
const HOOK_CLAIM_LEASE_MS = 2101800;
|
|
2143
|
-
const LIFECYCLE_COMPACTION_RETRY_MS = 1e3;
|
|
2144
|
-
//#endregion
|
|
2145
2341
|
//#region src/worker/durable-objects/HookIdDO.ts
|
|
2146
2342
|
const MAX_RELEASE_BATCH$1 = 64;
|
|
2147
2343
|
const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
|
|
2148
|
-
function pad$
|
|
2344
|
+
function pad$2(ms) {
|
|
2149
2345
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
2150
2346
|
}
|
|
2151
2347
|
function recordKey$1(hookId) {
|
|
@@ -2155,7 +2351,7 @@ function claimKey$1(hookId) {
|
|
|
2155
2351
|
return indexKey("claim", hookId);
|
|
2156
2352
|
}
|
|
2157
2353
|
function claimDeadlineKey$1(hookId, expiresAt) {
|
|
2158
|
-
return `${CLAIM_DEADLINE_PREFIX$1}${pad$
|
|
2354
|
+
return `${CLAIM_DEADLINE_PREFIX$1}${pad$2(expiresAt)}:${encodeURIComponent(hookId)}`;
|
|
2159
2355
|
}
|
|
2160
2356
|
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
2161
2357
|
var HookIdDO = class extends DurableObject {
|
|
@@ -2289,7 +2485,7 @@ var HookIdDO = class extends DurableObject {
|
|
|
2289
2485
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2290
2486
|
const due = await txn.list({
|
|
2291
2487
|
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
2292
|
-
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$
|
|
2488
|
+
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$2(now + 1)}`,
|
|
2293
2489
|
limit: 129
|
|
2294
2490
|
});
|
|
2295
2491
|
const page = Array.from(due).slice(0, 128);
|
|
@@ -2317,7 +2513,7 @@ var HookIdDO = class extends DurableObject {
|
|
|
2317
2513
|
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
2318
2514
|
const MAX_RELEASE_BATCH = 64;
|
|
2319
2515
|
const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
|
|
2320
|
-
function pad$
|
|
2516
|
+
function pad$1(ms) {
|
|
2321
2517
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
2322
2518
|
}
|
|
2323
2519
|
function recordKey(token) {
|
|
@@ -2327,7 +2523,7 @@ function claimKey(token) {
|
|
|
2327
2523
|
return indexKey("claim", token);
|
|
2328
2524
|
}
|
|
2329
2525
|
function claimDeadlineKey(token, expiresAt) {
|
|
2330
|
-
return `${CLAIM_DEADLINE_PREFIX}${pad$
|
|
2526
|
+
return `${CLAIM_DEADLINE_PREFIX}${pad$1(expiresAt)}:${encodeURIComponent(token)}`;
|
|
2331
2527
|
}
|
|
2332
2528
|
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
2333
2529
|
var HookTokenDO = class extends DurableObject {
|
|
@@ -2460,7 +2656,7 @@ var HookTokenDO = class extends DurableObject {
|
|
|
2460
2656
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2461
2657
|
const due = await txn.list({
|
|
2462
2658
|
prefix: CLAIM_DEADLINE_PREFIX,
|
|
2463
|
-
end: `${CLAIM_DEADLINE_PREFIX}${pad$
|
|
2659
|
+
end: `${CLAIM_DEADLINE_PREFIX}${pad$1(now + 1)}`,
|
|
2464
2660
|
limit: 129
|
|
2465
2661
|
});
|
|
2466
2662
|
const page = Array.from(due).slice(0, 128);
|
|
@@ -2485,705 +2681,6 @@ var HookTokenDO = class extends DurableObject {
|
|
|
2485
2681
|
}
|
|
2486
2682
|
};
|
|
2487
2683
|
//#endregion
|
|
2488
|
-
//#region src/worker/durable-objects/QueueDO.ts
|
|
2489
|
-
const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
2490
|
-
404,
|
|
2491
|
-
409,
|
|
2492
|
-
410,
|
|
2493
|
-
422
|
|
2494
|
-
]);
|
|
2495
|
-
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
2496
|
-
const DEFAULT_MAX_INFLIGHT = 5;
|
|
2497
|
-
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
2498
|
-
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
2499
|
-
const MIN_ALARM_DELAY_MS = 1;
|
|
2500
|
-
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
2501
|
-
const EXPIRED_INFLIGHT_BATCH = 128;
|
|
2502
|
-
const STORAGE_BATCH_SIZE$1 = 128;
|
|
2503
|
-
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
2504
|
-
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
2505
|
-
const MAX_DEAD_LETTER_LIST_SIZE = 1e3;
|
|
2506
|
-
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
2507
|
-
const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
|
|
2508
|
-
function pad$1(ms) {
|
|
2509
|
-
if (!isNonNegativeSafeInteger(ms) || ms > 9999999999999) throw new Error("queue timestamp must be a non-negative 13-digit integer");
|
|
2510
|
-
return String(ms).padStart(13, "0");
|
|
2511
|
-
}
|
|
2512
|
-
function dueKey(atMs, messageId) {
|
|
2513
|
-
return `due:${pad$1(atMs)}:${messageId}`;
|
|
2514
|
-
}
|
|
2515
|
-
function inflightKey(deadlineMs, messageId) {
|
|
2516
|
-
return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
|
|
2517
|
-
}
|
|
2518
|
-
function deadlineFromInflightKey(key) {
|
|
2519
|
-
return Number.parseInt(key.slice(18, 31), 10);
|
|
2520
|
-
}
|
|
2521
|
-
function inclusiveTimestampEnd(prefix, now) {
|
|
2522
|
-
return now >= 9999999999999 ? `${prefix.slice(0, -1)};` : `${prefix}${pad$1(now + 1)}`;
|
|
2523
|
-
}
|
|
2524
|
-
function runReferenceKey(runId, messageId) {
|
|
2525
|
-
return `run:${runId}:${messageId}`;
|
|
2526
|
-
}
|
|
2527
|
-
function expiredRunKey(runId) {
|
|
2528
|
-
return `expired-run:${runId}`;
|
|
2529
|
-
}
|
|
2530
|
-
function expiredRunGcKey(runId, compactAt) {
|
|
2531
|
-
return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
|
|
2532
|
-
}
|
|
2533
|
-
function validateEnqueueRequest(request) {
|
|
2534
|
-
if (!isRecord(request)) throw new Error("world-celld queue enqueue request must be an object");
|
|
2535
|
-
for (const field of [
|
|
2536
|
-
"messageId",
|
|
2537
|
-
"queueName",
|
|
2538
|
-
"body"
|
|
2539
|
-
]) if (typeof request[field] !== "string" || request[field].length === 0) throw new Error(`world-celld queue enqueue ${field} must be a non-empty string`);
|
|
2540
|
-
if (request.pathname !== "flow") throw new Error("world-celld queue enqueue pathname must be flow");
|
|
2541
|
-
if (!isRecord(request.config) || typeof request.config.targetBaseUrl !== "string" || request.config.targetBaseUrl.length === 0) throw new Error("world-celld queue enqueue config is invalid");
|
|
2542
|
-
if (!isPositiveSafeInteger(request.config.queueShards)) throw new Error("world-celld queue enqueue queueShards must be a positive safe integer");
|
|
2543
|
-
if (request.delaySeconds !== void 0 && !isValidQueueDelaySeconds(request.delaySeconds)) throw new Error("world-celld queue enqueue delaySeconds is out of range");
|
|
2544
|
-
if (request.runId !== void 0 && (typeof request.runId !== "string" || request.runId.length === 0)) throw new Error("world-celld queue enqueue runId must be a non-empty string");
|
|
2545
|
-
if (request.idempotencyKey !== void 0 && (typeof request.idempotencyKey !== "string" || request.idempotencyKey.length === 0)) throw new Error("world-celld queue enqueue idempotencyKey must be a non-empty string");
|
|
2546
|
-
let body;
|
|
2547
|
-
try {
|
|
2548
|
-
body = parse(request.body);
|
|
2549
|
-
} catch {
|
|
2550
|
-
throw new WorkflowWorldError("world-celld queue enqueue body must be valid tagged JSON", { status: 422 });
|
|
2551
|
-
}
|
|
2552
|
-
const payload = QueuePayloadSchema.safeParse(body);
|
|
2553
|
-
if (!payload.success) throw new WorkflowWorldError("world-celld queue enqueue body is an invalid queue payload", { status: 422 });
|
|
2554
|
-
const payloadRunId = "runId" in payload.data && typeof payload.data.runId === "string" ? payload.data.runId : void 0;
|
|
2555
|
-
if (request.runId !== payloadRunId) throw new WorkflowWorldError("world-celld queue enqueue runId does not match body", { status: 422 });
|
|
2556
|
-
}
|
|
2557
|
-
async function getMany(storage, keys) {
|
|
2558
|
-
const result = /* @__PURE__ */ new Map();
|
|
2559
|
-
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
2560
|
-
const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2561
|
-
for (const [key, value] of page) result.set(key, value);
|
|
2562
|
-
}
|
|
2563
|
-
return result;
|
|
2564
|
-
}
|
|
2565
|
-
async function putMany(storage, entries) {
|
|
2566
|
-
const all = Array.from(entries);
|
|
2567
|
-
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$1) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$1)));
|
|
2568
|
-
}
|
|
2569
|
-
async function deleteMany(storage, keys) {
|
|
2570
|
-
let deleted = 0;
|
|
2571
|
-
const unique = Array.from(new Set(keys));
|
|
2572
|
-
for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$1) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2573
|
-
return deleted;
|
|
2574
|
-
}
|
|
2575
|
-
function backoffSeconds(attempt) {
|
|
2576
|
-
return Math.min(60, 2 ** attempt);
|
|
2577
|
-
}
|
|
2578
|
-
async function cancelUnusedResponseBody(response) {
|
|
2579
|
-
try {
|
|
2580
|
-
await response.body?.cancel();
|
|
2581
|
-
} catch {}
|
|
2582
|
-
}
|
|
2583
|
-
const MAX_QUEUE_INFLIGHT = 128;
|
|
2584
|
-
var QueueDO = class extends DurableObject {
|
|
2585
|
-
#now() {
|
|
2586
|
-
const clock = this.env?.clock;
|
|
2587
|
-
return typeof clock === "function" ? clock() : Date.now();
|
|
2588
|
-
}
|
|
2589
|
-
#fetch() {
|
|
2590
|
-
const impl = this.env?.fetch;
|
|
2591
|
-
return typeof impl === "function" ? impl : fetch;
|
|
2592
|
-
}
|
|
2593
|
-
#intVar(name, fallback) {
|
|
2594
|
-
const raw = this.env?.[name];
|
|
2595
|
-
return strictIntegerSetting(name, raw, fallback, 1);
|
|
2596
|
-
}
|
|
2597
|
-
#maxAttempts() {
|
|
2598
|
-
return this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
2599
|
-
}
|
|
2600
|
-
#maxInflight() {
|
|
2601
|
-
const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
|
|
2602
|
-
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
2603
|
-
return configured;
|
|
2604
|
-
}
|
|
2605
|
-
#deliveryTimeoutMs() {
|
|
2606
|
-
const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
2607
|
-
if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
|
|
2608
|
-
return configured;
|
|
2609
|
-
}
|
|
2610
|
-
#validateRuntimeConfig() {
|
|
2611
|
-
this.#maxAttempts();
|
|
2612
|
-
this.#maxInflight();
|
|
2613
|
-
this.#deliveryTimeoutMs();
|
|
2614
|
-
}
|
|
2615
|
-
async #authoritativeRunExpired(runId) {
|
|
2616
|
-
const namespace = this.env?.WORKFLOW_DB;
|
|
2617
|
-
if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
|
|
2618
|
-
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
|
|
2619
|
-
}
|
|
2620
|
-
async #withRunMutation(runId, onExpired, onAllowed) {
|
|
2621
|
-
const expired = await this.#authoritativeRunExpired(runId);
|
|
2622
|
-
return await this.ctx.storage.transaction(async (txn) => {
|
|
2623
|
-
if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
|
|
2624
|
-
return await onAllowed(txn);
|
|
2625
|
-
});
|
|
2626
|
-
}
|
|
2627
|
-
async enqueue(request) {
|
|
2628
|
-
this.#validateRuntimeConfig();
|
|
2629
|
-
validateEnqueueRequest(request);
|
|
2630
|
-
const storage = this.ctx.storage;
|
|
2631
|
-
const now = this.#now();
|
|
2632
|
-
const dueAt = queueDelayDeadline(now, request.delaySeconds ?? 0, 0, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2633
|
-
if (dueAt === null) throw new Error("world-celld queue enqueue delaySeconds lacks required delivery headroom");
|
|
2634
|
-
const persist = async (txn) => {
|
|
2635
|
-
const pinned = await txn.get("cfg");
|
|
2636
|
-
if (!pinned) await txn.put("cfg", {
|
|
2637
|
-
queueShards: request.config.queueShards,
|
|
2638
|
-
pinnedAt: now
|
|
2639
|
-
});
|
|
2640
|
-
else if (pinned.queueShards !== request.config.queueShards) return {
|
|
2641
|
-
ok: false,
|
|
2642
|
-
code: "CONFIG_MISMATCH",
|
|
2643
|
-
message: `queue cell shard-count mismatch: pinned ${pinned.queueShards}, got ${request.config.queueShards} — changing queueShards requires draining the fleet`
|
|
2644
|
-
};
|
|
2645
|
-
if (request.idempotencyKey) {
|
|
2646
|
-
const existing = await txn.get(`key:${request.idempotencyKey}`);
|
|
2647
|
-
if (existing) {
|
|
2648
|
-
if (await txn.get(`msg:${existing}`)) {
|
|
2649
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2650
|
-
return {
|
|
2651
|
-
ok: true,
|
|
2652
|
-
messageId: existing,
|
|
2653
|
-
deduped: true
|
|
2654
|
-
};
|
|
2655
|
-
}
|
|
2656
|
-
await txn.delete(`key:${request.idempotencyKey}`);
|
|
2657
|
-
}
|
|
2658
|
-
await txn.put(`key:${request.idempotencyKey}`, request.messageId);
|
|
2659
|
-
}
|
|
2660
|
-
const row = {
|
|
2661
|
-
messageId: request.messageId,
|
|
2662
|
-
queueName: request.queueName,
|
|
2663
|
-
pathname: request.pathname,
|
|
2664
|
-
body: request.body,
|
|
2665
|
-
runId: request.runId,
|
|
2666
|
-
targetBaseUrl: request.config.targetBaseUrl,
|
|
2667
|
-
idempotencyKey: request.idempotencyKey,
|
|
2668
|
-
attempt: 0,
|
|
2669
|
-
enqueuedAt: now
|
|
2670
|
-
};
|
|
2671
|
-
const scheduleKey = dueKey(dueAt, row.messageId);
|
|
2672
|
-
await txn.put(`msg:${row.messageId}`, row);
|
|
2673
|
-
await txn.put(scheduleKey, row.messageId);
|
|
2674
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2675
|
-
messageId: row.messageId,
|
|
2676
|
-
dueKey: scheduleKey,
|
|
2677
|
-
idempotencyKey: row.idempotencyKey
|
|
2678
|
-
});
|
|
2679
|
-
await this.#armAlarmAtMost(txn, dueAt, now);
|
|
2680
|
-
return {
|
|
2681
|
-
ok: true,
|
|
2682
|
-
messageId: row.messageId,
|
|
2683
|
-
deduped: false
|
|
2684
|
-
};
|
|
2685
|
-
};
|
|
2686
|
-
return request.runId ? await this.#withRunMutation(request.runId, async () => ({
|
|
2687
|
-
ok: false,
|
|
2688
|
-
code: "RUN_EXPIRED",
|
|
2689
|
-
message: `Workflow run "${request.runId}" has expired`
|
|
2690
|
-
}), persist) : await storage.transaction(persist);
|
|
2691
|
-
}
|
|
2692
|
-
async alarm() {
|
|
2693
|
-
this.#validateRuntimeConfig();
|
|
2694
|
-
let claimed = [];
|
|
2695
|
-
let failure = null;
|
|
2696
|
-
const attempt = async () => {
|
|
2697
|
-
try {
|
|
2698
|
-
claimed = await this.#claimPhase();
|
|
2699
|
-
} catch (error) {
|
|
2700
|
-
failure = error;
|
|
2701
|
-
}
|
|
2702
|
-
};
|
|
2703
|
-
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
2704
|
-
else await attempt();
|
|
2705
|
-
if (failure !== null) {
|
|
2706
|
-
const retryAt = checkedQueueTimestampAdd(this.#now(), LIFECYCLE_COMPACTION_RETRY_MS);
|
|
2707
|
-
if (retryAt === null) await this.ctx.storage.deleteAlarm();
|
|
2708
|
-
else await this.ctx.storage.setAlarm(retryAt);
|
|
2709
|
-
throw failure;
|
|
2710
|
-
}
|
|
2711
|
-
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
2712
|
-
}
|
|
2713
|
-
/**
|
|
2714
|
-
* Storage-only: recover lost inflight claims, claim due messages up to the
|
|
2715
|
-
* inflight cap, and re-arm the alarm. Never throws for data conditions.
|
|
2716
|
-
*/
|
|
2717
|
-
async #claimPhase() {
|
|
2718
|
-
const storage = this.ctx.storage;
|
|
2719
|
-
const now = this.#now();
|
|
2720
|
-
const maxInflight = this.#maxInflight();
|
|
2721
|
-
return await storage.transaction(async (txn) => {
|
|
2722
|
-
if (await this.#compactExpiredRunFences(txn, now)) {
|
|
2723
|
-
const nextAt = checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS);
|
|
2724
|
-
if (nextAt === null) await txn.deleteAlarm();
|
|
2725
|
-
else await txn.setAlarm(nextAt);
|
|
2726
|
-
return [];
|
|
2727
|
-
}
|
|
2728
|
-
const expiredPage = await txn.list({
|
|
2729
|
-
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
2730
|
-
end: inclusiveTimestampEnd(INFLIGHT_DEADLINE_PREFIX, now),
|
|
2731
|
-
limit: 129
|
|
2732
|
-
});
|
|
2733
|
-
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
2734
|
-
const expiredRows = await getMany(txn, expired.map(([, messageId]) => `msg:${messageId}`));
|
|
2735
|
-
await deleteMany(txn, expired.map(([key]) => key));
|
|
2736
|
-
const recoveryWrites = [];
|
|
2737
|
-
for (const [, messageId] of expired) {
|
|
2738
|
-
const row = expiredRows.get(`msg:${messageId}`);
|
|
2739
|
-
if (row) {
|
|
2740
|
-
const recoveryAt = checkedQueueTimestampAdd(now, 0, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2741
|
-
if (recoveryAt === null) {
|
|
2742
|
-
await this.#deadLetter(txn, row, row.attempt, "inflight recovery exceeded the queue timestamp horizon", now);
|
|
2743
|
-
continue;
|
|
2744
|
-
}
|
|
2745
|
-
const scheduleKey = dueKey(recoveryAt, messageId);
|
|
2746
|
-
recoveryWrites.push([scheduleKey, messageId]);
|
|
2747
|
-
if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
2748
|
-
messageId: row.messageId,
|
|
2749
|
-
dueKey: scheduleKey,
|
|
2750
|
-
idempotencyKey: row.idempotencyKey
|
|
2751
|
-
}]);
|
|
2752
|
-
}
|
|
2753
|
-
}
|
|
2754
|
-
await putMany(txn, recoveryWrites);
|
|
2755
|
-
if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
|
|
2756
|
-
const nextAt = checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS);
|
|
2757
|
-
if (nextAt === null) await txn.deleteAlarm();
|
|
2758
|
-
else await txn.setAlarm(nextAt);
|
|
2759
|
-
return [];
|
|
2760
|
-
}
|
|
2761
|
-
const inflightCount = (await txn.list({
|
|
2762
|
-
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
2763
|
-
limit: maxInflight
|
|
2764
|
-
})).size;
|
|
2765
|
-
const rows = [];
|
|
2766
|
-
if (inflightCount < maxInflight) {
|
|
2767
|
-
const due = await txn.list({
|
|
2768
|
-
prefix: "due:",
|
|
2769
|
-
end: inclusiveTimestampEnd("due:", now),
|
|
2770
|
-
limit: maxInflight - inflightCount
|
|
2771
|
-
});
|
|
2772
|
-
const timeoutMs = this.#deliveryTimeoutMs();
|
|
2773
|
-
const dueRows = await getMany(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
2774
|
-
await deleteMany(txn, Array.from(due.keys()));
|
|
2775
|
-
const claimWrites = [];
|
|
2776
|
-
for (const [, messageId] of due) {
|
|
2777
|
-
const row = dueRows.get(`msg:${messageId}`);
|
|
2778
|
-
if (!row) continue;
|
|
2779
|
-
const claimDeadline = checkedQueueTimestampAdd(now, timeoutMs + QUEUE_INFLIGHT_GRACE_MS);
|
|
2780
|
-
if (claimDeadline === null) {
|
|
2781
|
-
await this.#deadLetter(txn, row, row.attempt, "delivery lease exceeded the queue timestamp horizon", now);
|
|
2782
|
-
continue;
|
|
2783
|
-
}
|
|
2784
|
-
const claimKey = inflightKey(claimDeadline, messageId);
|
|
2785
|
-
claimWrites.push([claimKey, messageId]);
|
|
2786
|
-
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
2787
|
-
messageId: row.messageId,
|
|
2788
|
-
inflightKey: claimKey,
|
|
2789
|
-
idempotencyKey: row.idempotencyKey
|
|
2790
|
-
}]);
|
|
2791
|
-
rows.push({
|
|
2792
|
-
row,
|
|
2793
|
-
inflightKey: claimKey
|
|
2794
|
-
});
|
|
2795
|
-
}
|
|
2796
|
-
await putMany(txn, claimWrites);
|
|
2797
|
-
}
|
|
2798
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2799
|
-
return rows;
|
|
2800
|
-
});
|
|
2801
|
-
}
|
|
2802
|
-
async #deliver({ row, inflightKey: claimKey }) {
|
|
2803
|
-
const storage = this.ctx.storage;
|
|
2804
|
-
const timeoutMs = this.#deliveryTimeoutMs();
|
|
2805
|
-
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
2806
|
-
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
2807
|
-
if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
|
|
2808
|
-
await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
|
|
2809
|
-
return;
|
|
2810
|
-
}
|
|
2811
|
-
let response = null;
|
|
2812
|
-
let transportError = null;
|
|
2813
|
-
try {
|
|
2814
|
-
response = await this.#fetch()(url, {
|
|
2815
|
-
method: "POST",
|
|
2816
|
-
headers: {
|
|
2817
|
-
"content-type": "application/json",
|
|
2818
|
-
"x-vqs-queue-name": row.queueName,
|
|
2819
|
-
"x-vqs-message-id": row.messageId,
|
|
2820
|
-
"x-vqs-message-attempt": String(row.attempt + 1),
|
|
2821
|
-
...callbackSecret ? { "x-workflow-callback-secret": callbackSecret } : {}
|
|
2822
|
-
},
|
|
2823
|
-
body: row.body,
|
|
2824
|
-
signal: AbortSignal.timeout(timeoutMs)
|
|
2825
|
-
});
|
|
2826
|
-
} catch (error) {
|
|
2827
|
-
transportError = error;
|
|
2828
|
-
}
|
|
2829
|
-
const now = this.#now();
|
|
2830
|
-
if (response?.ok) {
|
|
2831
|
-
await cancelUnusedResponseBody(response);
|
|
2832
|
-
await this.#ack(row, claimKey);
|
|
2833
|
-
} else if (response && response.status === 503) {
|
|
2834
|
-
let timeoutSeconds;
|
|
2835
|
-
try {
|
|
2836
|
-
timeoutSeconds = (await response.json())?.timeoutSeconds;
|
|
2837
|
-
} catch {
|
|
2838
|
-
timeoutSeconds = void 0;
|
|
2839
|
-
}
|
|
2840
|
-
const redeliveryAt = queueDelayDeadline(now, timeoutSeconds, 1, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2841
|
-
if (redeliveryAt !== null) {
|
|
2842
|
-
const reschedule = async (txn) => {
|
|
2843
|
-
if (!await txn.delete(claimKey)) return;
|
|
2844
|
-
const scheduleKey = dueKey(redeliveryAt, row.messageId);
|
|
2845
|
-
await txn.put(scheduleKey, row.messageId);
|
|
2846
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2847
|
-
messageId: row.messageId,
|
|
2848
|
-
dueKey: scheduleKey,
|
|
2849
|
-
idempotencyKey: row.idempotencyKey
|
|
2850
|
-
});
|
|
2851
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2852
|
-
};
|
|
2853
|
-
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
|
|
2854
|
-
else await storage.transaction(reschedule);
|
|
2855
|
-
} else await this.#retry(row, claimKey, "HTTP 503");
|
|
2856
|
-
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
2857
|
-
await cancelUnusedResponseBody(response);
|
|
2858
|
-
await this.#ack(row, claimKey);
|
|
2859
|
-
} else {
|
|
2860
|
-
const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
|
|
2861
|
-
if (response) await cancelUnusedResponseBody(response);
|
|
2862
|
-
await this.#retry(row, claimKey, reason);
|
|
2863
|
-
}
|
|
2864
|
-
}
|
|
2865
|
-
async #ack(row, claimKey) {
|
|
2866
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
2867
|
-
if (!await txn.delete(claimKey)) return;
|
|
2868
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
2869
|
-
if (row.idempotencyKey) {
|
|
2870
|
-
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
2871
|
-
}
|
|
2872
|
-
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
2873
|
-
await this.#scheduleNextAlarm(txn, this.#now());
|
|
2874
|
-
});
|
|
2875
|
-
}
|
|
2876
|
-
async #dropExpiredMessage(txn, row, claimKey, now) {
|
|
2877
|
-
if (!await txn.delete(claimKey)) return;
|
|
2878
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
2879
|
-
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
2880
|
-
if (row.idempotencyKey) {
|
|
2881
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
2882
|
-
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
2883
|
-
}
|
|
2884
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2885
|
-
}
|
|
2886
|
-
async #retry(row, claimKey, reason) {
|
|
2887
|
-
const storage = this.ctx.storage;
|
|
2888
|
-
const now = this.#now();
|
|
2889
|
-
const maxAttempts = this.#maxAttempts();
|
|
2890
|
-
const attempt = row.attempt + 1;
|
|
2891
|
-
const retryAt = checkedQueueTimestampAdd(now, backoffSeconds(attempt) * 1e3, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
2892
|
-
const persistRetry = async (txn) => {
|
|
2893
|
-
if (!await txn.delete(claimKey)) return;
|
|
2894
|
-
if (attempt >= maxAttempts || retryAt === null) {
|
|
2895
|
-
await this.#deadLetter(txn, row, attempt, retryAt === null ? `${reason}; retry exceeded the queue timestamp horizon` : reason, now);
|
|
2896
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2897
|
-
return;
|
|
2898
|
-
}
|
|
2899
|
-
const updated = {
|
|
2900
|
-
...row,
|
|
2901
|
-
attempt,
|
|
2902
|
-
lastError: reason
|
|
2903
|
-
};
|
|
2904
|
-
const scheduleKey = dueKey(retryAt, row.messageId);
|
|
2905
|
-
await txn.put(`msg:${row.messageId}`, updated);
|
|
2906
|
-
await txn.put(scheduleKey, row.messageId);
|
|
2907
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2908
|
-
messageId: row.messageId,
|
|
2909
|
-
dueKey: scheduleKey,
|
|
2910
|
-
idempotencyKey: row.idempotencyKey
|
|
2911
|
-
});
|
|
2912
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
2913
|
-
};
|
|
2914
|
-
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
|
|
2915
|
-
else await storage.transaction(persistRetry);
|
|
2916
|
-
}
|
|
2917
|
-
async #deadLetter(txn, row, attempt, reason, now) {
|
|
2918
|
-
const failedAt = checkedQueueTimestampAdd(now, 0);
|
|
2919
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
2920
|
-
if (row.idempotencyKey) {
|
|
2921
|
-
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
2922
|
-
}
|
|
2923
|
-
if (failedAt === null) {
|
|
2924
|
-
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
2925
|
-
return;
|
|
2926
|
-
}
|
|
2927
|
-
const dead = {
|
|
2928
|
-
...row,
|
|
2929
|
-
attempt,
|
|
2930
|
-
lastError: reason,
|
|
2931
|
-
failedAt
|
|
2932
|
-
};
|
|
2933
|
-
const deadKey = `dlq:${pad$1(failedAt)}:${row.messageId}`;
|
|
2934
|
-
await txn.put(deadKey, dead);
|
|
2935
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
2936
|
-
messageId: row.messageId,
|
|
2937
|
-
dlqKey: deadKey,
|
|
2938
|
-
idempotencyKey: row.idempotencyKey
|
|
2939
|
-
});
|
|
2940
|
-
}
|
|
2941
|
-
/** Return true when another due compaction page must run immediately. */
|
|
2942
|
-
async #compactExpiredRunFences(txn, now) {
|
|
2943
|
-
const due = await txn.list({
|
|
2944
|
-
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
2945
|
-
end: inclusiveTimestampEnd(EXPIRED_RUN_GC_PREFIX, now),
|
|
2946
|
-
limit: 129
|
|
2947
|
-
});
|
|
2948
|
-
const page = Array.from(due).slice(0, 128);
|
|
2949
|
-
if (page.length === 0) return false;
|
|
2950
|
-
const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
|
|
2951
|
-
const markers = await getMany(txn, markerKeys);
|
|
2952
|
-
const deletes = [];
|
|
2953
|
-
for (let index = 0; index < page.length; index++) {
|
|
2954
|
-
const [gcKey, item] = page[index];
|
|
2955
|
-
const markerKey = markerKeys[index];
|
|
2956
|
-
const marker = markers.get(markerKey);
|
|
2957
|
-
deletes.push(gcKey);
|
|
2958
|
-
if (marker?.compactAt !== item.compactAt) continue;
|
|
2959
|
-
deletes.push(markerKey);
|
|
2960
|
-
}
|
|
2961
|
-
await deleteMany(txn, deletes);
|
|
2962
|
-
return due.size > 128;
|
|
2963
|
-
}
|
|
2964
|
-
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
2965
|
-
async #armAlarmAtMost(storage, atMs, now) {
|
|
2966
|
-
const current = await storage.getAlarm();
|
|
2967
|
-
const immediate = checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS);
|
|
2968
|
-
if (current !== null && current <= now) {
|
|
2969
|
-
if (immediate === null) await storage.deleteAlarm();
|
|
2970
|
-
else await storage.setAlarm(immediate);
|
|
2971
|
-
return;
|
|
2972
|
-
}
|
|
2973
|
-
const target = atMs <= now ? immediate : atMs;
|
|
2974
|
-
if (target === null) {
|
|
2975
|
-
await storage.deleteAlarm();
|
|
2976
|
-
return;
|
|
2977
|
-
}
|
|
2978
|
-
if (current === null || target < current) await storage.setAlarm(target);
|
|
2979
|
-
}
|
|
2980
|
-
/** Recompute the alarm from the earliest due entry / inflight deadline. */
|
|
2981
|
-
async #rearm() {
|
|
2982
|
-
await this.#scheduleNextAlarm(this.ctx.storage, this.#now());
|
|
2983
|
-
}
|
|
2984
|
-
/** Keep the alarm index consistent with message state in the same transaction. */
|
|
2985
|
-
async #scheduleNextAlarm(storage, now) {
|
|
2986
|
-
let next = null;
|
|
2987
|
-
const due = await storage.list({
|
|
2988
|
-
prefix: "due:",
|
|
2989
|
-
limit: 1
|
|
2990
|
-
});
|
|
2991
|
-
for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
|
|
2992
|
-
const inflight = await storage.list({
|
|
2993
|
-
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
2994
|
-
limit: 1
|
|
2995
|
-
});
|
|
2996
|
-
for (const key of inflight.keys()) {
|
|
2997
|
-
const deadline = deadlineFromInflightKey(key);
|
|
2998
|
-
if (next === null || deadline < next) next = deadline;
|
|
2999
|
-
}
|
|
3000
|
-
const expiryGc = await storage.list({
|
|
3001
|
-
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
3002
|
-
limit: 1
|
|
3003
|
-
});
|
|
3004
|
-
for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
|
|
3005
|
-
if (next !== null) {
|
|
3006
|
-
const target = next <= now ? checkedQueueTimestampAdd(now, MIN_ALARM_DELAY_MS) : next;
|
|
3007
|
-
if (target === null) await storage.deleteAlarm();
|
|
3008
|
-
else await storage.setAlarm(target);
|
|
3009
|
-
} else await storage.deleteAlarm();
|
|
3010
|
-
}
|
|
3011
|
-
async stats() {
|
|
3012
|
-
const storage = this.ctx.storage;
|
|
3013
|
-
const [due, inflight, dlq, alarmAt] = await Promise.all([
|
|
3014
|
-
storage.list({ prefix: "due:" }),
|
|
3015
|
-
storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
|
|
3016
|
-
storage.list({ prefix: "dlq:" }),
|
|
3017
|
-
storage.getAlarm()
|
|
3018
|
-
]);
|
|
3019
|
-
return {
|
|
3020
|
-
pending: due.size,
|
|
3021
|
-
inflight: inflight.size,
|
|
3022
|
-
deadLetters: dlq.size,
|
|
3023
|
-
alarmAt
|
|
3024
|
-
};
|
|
3025
|
-
}
|
|
3026
|
-
async listDeadLetters(params) {
|
|
3027
|
-
if (params !== void 0 && !isRecord(params)) throw new Error("world-celld queue dead-letter list options must be an object");
|
|
3028
|
-
const limit = boundedIntegerOption("world-celld queue dead-letter limit", params?.limit, 100, 1, MAX_DEAD_LETTER_LIST_SIZE);
|
|
3029
|
-
if (params?.cursor !== void 0 && typeof params.cursor !== "string") throw new Error("world-celld queue dead-letter cursor must be a string");
|
|
3030
|
-
const entries = await this.ctx.storage.list({
|
|
3031
|
-
prefix: "dlq:",
|
|
3032
|
-
startAfter: params?.cursor,
|
|
3033
|
-
limit: limit + 1
|
|
3034
|
-
});
|
|
3035
|
-
const keys = Array.from(entries.keys());
|
|
3036
|
-
const page = keys.slice(0, limit);
|
|
3037
|
-
const hasMore = keys.length > limit;
|
|
3038
|
-
return {
|
|
3039
|
-
data: page.map((k) => entries.get(k)),
|
|
3040
|
-
cursor: hasMore ? page.at(-1) : null,
|
|
3041
|
-
hasMore
|
|
3042
|
-
};
|
|
3043
|
-
}
|
|
3044
|
-
/** Move a dead letter back to the live queue (attempt count reset). */
|
|
3045
|
-
async redriveDeadLetter(messageId) {
|
|
3046
|
-
if (typeof messageId !== "string" || messageId.length === 0) throw new Error("world-celld queue redrive messageId must be a non-empty string");
|
|
3047
|
-
const storage = this.ctx.storage;
|
|
3048
|
-
const now = this.#now();
|
|
3049
|
-
if (checkedQueueTimestampAdd(now, 0, 330001) === null) return { ok: false };
|
|
3050
|
-
const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
|
|
3051
|
-
if (!found) return { ok: false };
|
|
3052
|
-
const [deadKey, candidate] = found;
|
|
3053
|
-
const redrive = async (txn) => {
|
|
3054
|
-
const dead = await txn.get(deadKey);
|
|
3055
|
-
if (!dead || dead.messageId !== messageId) return { ok: false };
|
|
3056
|
-
const { failedAt: _failedAt, ...rest } = dead;
|
|
3057
|
-
const row = {
|
|
3058
|
-
...rest,
|
|
3059
|
-
attempt: 0,
|
|
3060
|
-
lastError: void 0
|
|
3061
|
-
};
|
|
3062
|
-
if (row.idempotencyKey) {
|
|
3063
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
3064
|
-
const holder = await txn.get(dedupKey);
|
|
3065
|
-
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
3066
|
-
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
3067
|
-
await txn.put(dedupKey, row.messageId);
|
|
3068
|
-
}
|
|
3069
|
-
await txn.put(`msg:${row.messageId}`, row);
|
|
3070
|
-
const scheduleKey = dueKey(now, row.messageId);
|
|
3071
|
-
await txn.put(scheduleKey, row.messageId);
|
|
3072
|
-
await txn.delete(deadKey);
|
|
3073
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
3074
|
-
messageId: row.messageId,
|
|
3075
|
-
dueKey: scheduleKey,
|
|
3076
|
-
idempotencyKey: row.idempotencyKey
|
|
3077
|
-
});
|
|
3078
|
-
await this.#armAlarmAtMost(txn, now, now);
|
|
3079
|
-
return { ok: true };
|
|
3080
|
-
};
|
|
3081
|
-
const rejectExpired = async (txn) => {
|
|
3082
|
-
if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
|
|
3083
|
-
return { ok: false };
|
|
3084
|
-
};
|
|
3085
|
-
return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
|
|
3086
|
-
}
|
|
3087
|
-
async purgeDeadLetters() {
|
|
3088
|
-
return await this.ctx.storage.transaction(async (txn) => {
|
|
3089
|
-
const entries = await txn.list({
|
|
3090
|
-
prefix: "dlq:",
|
|
3091
|
-
limit: 129
|
|
3092
|
-
});
|
|
3093
|
-
const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
|
|
3094
|
-
await deleteMany(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
|
|
3095
|
-
return {
|
|
3096
|
-
purged: page.length,
|
|
3097
|
-
hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
|
|
3098
|
-
};
|
|
3099
|
-
});
|
|
3100
|
-
}
|
|
3101
|
-
/**
|
|
3102
|
-
* Recovery hatch for celld's six-failure alarm abandonment: derive the due
|
|
3103
|
-
* time again and arm it.
|
|
3104
|
-
*/
|
|
3105
|
-
async rearmAlarm() {
|
|
3106
|
-
await this.#rearm();
|
|
3107
|
-
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
3108
|
-
}
|
|
3109
|
-
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
3110
|
-
async expireRun(runId, expiredAt, options) {
|
|
3111
|
-
if (typeof runId !== "string" || runId.length === 0) throw new Error("world-celld queue expiry runId must be a non-empty string");
|
|
3112
|
-
if (!isNonNegativeSafeInteger(expiredAt)) throw new Error("world-celld queue expiry expiredAt must be a non-negative safe integer");
|
|
3113
|
-
if (options !== void 0 && !isRecord(options)) throw new Error("world-celld queue expiry options must be an object");
|
|
3114
|
-
const storage = this.ctx.storage;
|
|
3115
|
-
const limit = boundedIntegerOption("world-celld queue expiry limit", options?.limit, EXPIRE_RUN_REFERENCE_BATCH, 1, EXPIRE_RUN_REFERENCE_BATCH);
|
|
3116
|
-
return await storage.transaction(async (txn) => {
|
|
3117
|
-
const existing = await txn.get(expiredRunKey(runId));
|
|
3118
|
-
const references = await txn.list({
|
|
3119
|
-
prefix: `run:${runId}:`,
|
|
3120
|
-
limit: limit + 1
|
|
3121
|
-
});
|
|
3122
|
-
const page = Array.from(references).slice(0, limit);
|
|
3123
|
-
const dedupHolders = await getMany(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
|
|
3124
|
-
const keys = [];
|
|
3125
|
-
let count = existing?.deleted ?? 0;
|
|
3126
|
-
for (const [key, reference] of page) {
|
|
3127
|
-
keys.push(`msg:${reference.messageId}`, key);
|
|
3128
|
-
if (reference.inflightKey) keys.push(reference.inflightKey);
|
|
3129
|
-
if (reference.dueKey) keys.push(reference.dueKey);
|
|
3130
|
-
if (reference.dlqKey) keys.push(reference.dlqKey);
|
|
3131
|
-
if (reference.idempotencyKey) {
|
|
3132
|
-
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
3133
|
-
if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
|
|
3134
|
-
}
|
|
3135
|
-
count++;
|
|
3136
|
-
}
|
|
3137
|
-
await deleteMany(txn, keys);
|
|
3138
|
-
const done = references.size <= limit;
|
|
3139
|
-
const markerExpiredAt = existing?.expiredAt ?? expiredAt;
|
|
3140
|
-
await txn.put(expiredRunKey(runId), {
|
|
3141
|
-
expiredAt: markerExpiredAt,
|
|
3142
|
-
deleted: count,
|
|
3143
|
-
compactAt: existing?.compactAt ?? null
|
|
3144
|
-
});
|
|
3145
|
-
await this.#scheduleNextAlarm(txn, this.#now());
|
|
3146
|
-
return done ? {
|
|
3147
|
-
deleted: count,
|
|
3148
|
-
done: true,
|
|
3149
|
-
receipt: {
|
|
3150
|
-
expiredAt: markerExpiredAt,
|
|
3151
|
-
deleted: count
|
|
3152
|
-
}
|
|
3153
|
-
} : {
|
|
3154
|
-
deleted: count,
|
|
3155
|
-
done: false
|
|
3156
|
-
};
|
|
3157
|
-
});
|
|
3158
|
-
}
|
|
3159
|
-
/** Acknowledge a final cleanup receipt before making its marker compactable. */
|
|
3160
|
-
async acknowledgeExpireRun(runId, receipt) {
|
|
3161
|
-
if (typeof runId !== "string" || runId.length === 0) throw new Error("world-celld queue expiry runId must be a non-empty string");
|
|
3162
|
-
if (!isRecord(receipt) || !isNonNegativeSafeInteger(receipt.expiredAt) || !isNonNegativeSafeInteger(receipt.deleted)) throw new Error("world-celld queue expiry receipt is invalid");
|
|
3163
|
-
return await this.ctx.storage.transaction(async (txn) => {
|
|
3164
|
-
const key = expiredRunKey(runId);
|
|
3165
|
-
const marker = await txn.get(key);
|
|
3166
|
-
if (!marker) return { acknowledged: true };
|
|
3167
|
-
if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
|
|
3168
|
-
if (marker.compactAt === null) {
|
|
3169
|
-
const compactAt = checkedQueueTimestampAdd(this.#now(), QUEUE_FENCE_GRACE_MS);
|
|
3170
|
-
if (compactAt !== null) {
|
|
3171
|
-
await txn.put(key, {
|
|
3172
|
-
...marker,
|
|
3173
|
-
compactAt
|
|
3174
|
-
});
|
|
3175
|
-
await txn.put(expiredRunGcKey(runId, compactAt), {
|
|
3176
|
-
runId,
|
|
3177
|
-
compactAt
|
|
3178
|
-
});
|
|
3179
|
-
}
|
|
3180
|
-
}
|
|
3181
|
-
await this.#scheduleNextAlarm(txn, this.#now());
|
|
3182
|
-
return { acknowledged: true };
|
|
3183
|
-
});
|
|
3184
|
-
}
|
|
3185
|
-
};
|
|
3186
|
-
//#endregion
|
|
3187
2684
|
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
3188
2685
|
const MAX_INDEX_LIST_SIZE = 1001;
|
|
3189
2686
|
const EXPIRY_GC_PREFIX = "expiry-gc:";
|
|
@@ -3761,15 +3258,14 @@ const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
|
3761
3258
|
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
3762
3259
|
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3763
3260
|
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
3764
|
-
const
|
|
3261
|
+
const QUEUE_PAYLOADS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3765
3262
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
3766
|
-
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
3767
|
-
const RUN_QUEUE_SHARDS_KEY = "retention:queue-shards";
|
|
3768
3263
|
const MAX_DATE_MS = 864e13;
|
|
3264
|
+
const QUEUE_ORPHAN_KEY = "queue-orphan";
|
|
3265
|
+
const QUEUE_RESERVATION_KEY = "queue-reservation";
|
|
3769
3266
|
function validateCleanupRequest(request, allowDisabled) {
|
|
3770
|
-
if (!isRecord(request)) throw new Error("world-celld retention request must be an object");
|
|
3267
|
+
if (!isRecord$1(request)) throw new Error("world-celld retention request must be an object");
|
|
3771
3268
|
if (!(allowDisabled ? isNonNegativeSafeInteger(request.retentionMs) : isPositiveSafeInteger(request.retentionMs))) throw new Error(`world-celld retentionMs must be a ${allowDisabled ? "non-negative" : "positive"} safe integer`);
|
|
3772
|
-
if (!isPositiveSafeInteger(request.queueShards)) throw new Error("world-celld retention queueShards must be a positive safe integer");
|
|
3773
3269
|
return request;
|
|
3774
3270
|
}
|
|
3775
3271
|
function retentionDeadline(baseMs, retentionMs, runId) {
|
|
@@ -3777,7 +3273,7 @@ function retentionDeadline(baseMs, retentionMs, runId) {
|
|
|
3777
3273
|
if (!Number.isSafeInteger(dueAtMs) || dueAtMs > MAX_DATE_MS) throw new Error(`world-celld retention deadline is out of range for ${runId}`);
|
|
3778
3274
|
return dueAtMs;
|
|
3779
3275
|
}
|
|
3780
|
-
function cleanupRecord(run, dueAt,
|
|
3276
|
+
function cleanupRecord(run, dueAt, reason) {
|
|
3781
3277
|
const terminal = isTerminalWorkflowRunStatus(run.status) ? run : void 0;
|
|
3782
3278
|
return {
|
|
3783
3279
|
version: 1,
|
|
@@ -3789,13 +3285,12 @@ function cleanupRecord(run, dueAt, queueShards, reason) {
|
|
|
3789
3285
|
terminalStatus: terminal?.status,
|
|
3790
3286
|
reason,
|
|
3791
3287
|
dueAt,
|
|
3792
|
-
queueShards,
|
|
3793
3288
|
phase: "retained",
|
|
3794
3289
|
generation: 0,
|
|
3795
3290
|
attempts: 0,
|
|
3796
3291
|
deletedPayloadKeys: 0,
|
|
3797
3292
|
deletedStreams: 0,
|
|
3798
|
-
|
|
3293
|
+
deletedQueuePayloads: 0
|
|
3799
3294
|
};
|
|
3800
3295
|
}
|
|
3801
3296
|
/**
|
|
@@ -3828,8 +3323,8 @@ async function putStorageEntries(storage, entries) {
|
|
|
3828
3323
|
* storage transaction: guard checks, the event append, and the entity
|
|
3829
3324
|
* mutation are atomic even if the DO is evicted mid-operation.
|
|
3830
3325
|
*
|
|
3831
|
-
* Instances named `claim:<queueName>:<idempotencyKey>`
|
|
3832
|
-
*
|
|
3326
|
+
* Instances named `claim:<queueName>:<idempotencyKey>` reserve one message ID
|
|
3327
|
+
* at publication time and serialize its deliveries until completion.
|
|
3833
3328
|
*/
|
|
3834
3329
|
var WorkflowRunDO = class extends DurableObject {
|
|
3835
3330
|
now() {
|
|
@@ -3896,7 +3391,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3896
3391
|
now
|
|
3897
3392
|
});
|
|
3898
3393
|
if (!outcome.ok) return outcome;
|
|
3899
|
-
if (outcome.runCreated && request.cleanup) await txn.put(RUN_QUEUE_SHARDS_KEY, request.cleanup.queueShards);
|
|
3900
3394
|
await txn.put("event_sequence", eventSequence);
|
|
3901
3395
|
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
3902
3396
|
hookId: outcome.hookToIndex.hookId,
|
|
@@ -3910,7 +3404,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
3910
3404
|
...terminalRun,
|
|
3911
3405
|
expiredAt: dueAt
|
|
3912
3406
|
});
|
|
3913
|
-
const cleanup = cleanupRecord(run, dueAt,
|
|
3407
|
+
const cleanup = cleanupRecord(run, dueAt, "terminal-retention");
|
|
3914
3408
|
await txn.put("run", run);
|
|
3915
3409
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
3916
3410
|
await txn.setAlarm(dueAt);
|
|
@@ -4020,7 +3514,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4020
3514
|
...terminalRun,
|
|
4021
3515
|
expiredAt: dueAt
|
|
4022
3516
|
});
|
|
4023
|
-
const cleanup = cleanupRecord(run, dueAt,
|
|
3517
|
+
const cleanup = cleanupRecord(run, dueAt, "manual");
|
|
4024
3518
|
await txn.put("run", run);
|
|
4025
3519
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
4026
3520
|
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
@@ -4041,7 +3535,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4041
3535
|
const values = await txn.get([
|
|
4042
3536
|
CLEANUP_RECORD_KEY,
|
|
4043
3537
|
TOMBSTONE_KEY,
|
|
4044
|
-
RUN_QUEUE_SHARDS_KEY,
|
|
4045
3538
|
"run"
|
|
4046
3539
|
]);
|
|
4047
3540
|
const tombstone = values.get(TOMBSTONE_KEY);
|
|
@@ -4060,12 +3553,9 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4060
3553
|
cleanup: null
|
|
4061
3554
|
};
|
|
4062
3555
|
const dueAt = new Date(dueAtMs);
|
|
4063
|
-
|
|
4064
|
-
const storedQueueShards = values.get(RUN_QUEUE_SHARDS_KEY);
|
|
4065
|
-
const queueShards = Number.isSafeInteger(storedQueueShards) && storedQueueShards > 0 ? storedQueueShards : request.queueShards;
|
|
4066
|
-
let cleanup = existing;
|
|
3556
|
+
let cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
4067
3557
|
if (!cleanup) {
|
|
4068
|
-
cleanup = cleanupRecord(run, dueAt,
|
|
3558
|
+
cleanup = cleanupRecord(run, dueAt, "maximum-age");
|
|
4069
3559
|
await txn.put("run", WorkflowRunSchema.parse({
|
|
4070
3560
|
...run,
|
|
4071
3561
|
expiredAt: dueAt
|
|
@@ -4168,9 +3658,69 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4168
3658
|
});
|
|
4169
3659
|
}
|
|
4170
3660
|
async executeScheduledCleanupPage() {
|
|
4171
|
-
const values = await this.ctx.storage.get([
|
|
3661
|
+
const values = await this.ctx.storage.get([
|
|
3662
|
+
CLEANUP_RECORD_KEY,
|
|
3663
|
+
TERMINAL_CLEANUP_KEY,
|
|
3664
|
+
"claim",
|
|
3665
|
+
QUEUE_RESERVATION_KEY,
|
|
3666
|
+
QUEUE_ORPHAN_KEY,
|
|
3667
|
+
"run"
|
|
3668
|
+
]);
|
|
4172
3669
|
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
4173
3670
|
const terminalCleanup = values.get(TERMINAL_CLEANUP_KEY);
|
|
3671
|
+
const claim = values.get("claim");
|
|
3672
|
+
const reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
3673
|
+
if (values.get(QUEUE_ORPHAN_KEY)) {
|
|
3674
|
+
const leased = await this.ctx.storage.transaction(async (txn) => {
|
|
3675
|
+
const current = await txn.get(QUEUE_ORPHAN_KEY);
|
|
3676
|
+
if (!current) return null;
|
|
3677
|
+
if (current.expiresAt > this.now()) {
|
|
3678
|
+
await txn.setAlarm(current.expiresAt);
|
|
3679
|
+
return null;
|
|
3680
|
+
}
|
|
3681
|
+
if (current.deleting) return current;
|
|
3682
|
+
const next = {
|
|
3683
|
+
...current,
|
|
3684
|
+
generation: current.generation + 1,
|
|
3685
|
+
deleting: true
|
|
3686
|
+
};
|
|
3687
|
+
await txn.put(QUEUE_ORPHAN_KEY, next);
|
|
3688
|
+
return next;
|
|
3689
|
+
});
|
|
3690
|
+
if (!leased) return;
|
|
3691
|
+
const bucket = this.env?.WORKFLOW_QUEUE_PAYLOADS;
|
|
3692
|
+
const runs = this.env?.WORKFLOW_DB;
|
|
3693
|
+
if (!bucket || !runs) throw new Error("world-celld queue orphan cleanup is missing required bindings");
|
|
3694
|
+
await deleteQueuePayloadObjects(bucket, leased.key);
|
|
3695
|
+
await runs.get(runs.idFromName(leased.runId)).unregisterQueuePayload(leased.messageId);
|
|
3696
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3697
|
+
const current = await txn.get(QUEUE_ORPHAN_KEY);
|
|
3698
|
+
if (current?.messageId !== leased.messageId || current.generation !== leased.generation || !current.deleting) return;
|
|
3699
|
+
await txn.delete(QUEUE_ORPHAN_KEY);
|
|
3700
|
+
await txn.deleteAlarm();
|
|
3701
|
+
});
|
|
3702
|
+
return;
|
|
3703
|
+
}
|
|
3704
|
+
if (claim || reservation) {
|
|
3705
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3706
|
+
const current = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
3707
|
+
let nextClaim = current.get("claim");
|
|
3708
|
+
let nextReservation = current.get(QUEUE_RESERVATION_KEY);
|
|
3709
|
+
const now = this.now();
|
|
3710
|
+
if (nextClaim && nextClaim.expiresAt <= now) {
|
|
3711
|
+
await txn.delete("claim");
|
|
3712
|
+
nextClaim = void 0;
|
|
3713
|
+
}
|
|
3714
|
+
if (nextReservation && nextReservation.expiresAt <= now) {
|
|
3715
|
+
await txn.delete(QUEUE_RESERVATION_KEY);
|
|
3716
|
+
nextReservation = void 0;
|
|
3717
|
+
}
|
|
3718
|
+
const nextAlarm = Math.min(nextClaim?.expiresAt ?? Number.POSITIVE_INFINITY, nextReservation?.expiresAt ?? Number.POSITIVE_INFINITY);
|
|
3719
|
+
if (Number.isFinite(nextAlarm)) await txn.setAlarm(nextAlarm);
|
|
3720
|
+
else await txn.deleteAlarm();
|
|
3721
|
+
});
|
|
3722
|
+
return;
|
|
3723
|
+
}
|
|
4174
3724
|
if (cleanup && cleanup.phase !== "tombstoned" && cleanup.dueAt.getTime() <= this.now()) {
|
|
4175
3725
|
try {
|
|
4176
3726
|
await this.executeCleanup(cleanup);
|
|
@@ -4369,85 +3919,27 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4369
3919
|
});
|
|
4370
3920
|
}
|
|
4371
3921
|
async deleteQueuePage(cleanup) {
|
|
4372
|
-
const
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
4381
|
-
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
4382
|
-
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
4383
|
-
const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
|
|
4384
|
-
if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
|
|
4385
|
-
await txn.put(CLEANUP_RECORD_KEY, {
|
|
4386
|
-
...current,
|
|
4387
|
-
generation: current.generation + 1,
|
|
4388
|
-
attempts: 0,
|
|
4389
|
-
lastError: void 0
|
|
4390
|
-
});
|
|
4391
|
-
await txn.put(CLEANUP_PROGRESS_KEY, {
|
|
4392
|
-
queueShard: queueShard + 1,
|
|
4393
|
-
queueShardDeleted: 0
|
|
4394
|
-
});
|
|
4395
|
-
await txn.setAlarm(this.now() + 1);
|
|
4396
|
-
});
|
|
4397
|
-
return;
|
|
4398
|
-
}
|
|
4399
|
-
if (progress.queueShard < cleanup.queueShards) {
|
|
4400
|
-
const requestedShard = progress.queueShard;
|
|
4401
|
-
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
4402
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
4403
|
-
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
4404
|
-
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
4405
|
-
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
4406
|
-
queueShard: 0,
|
|
4407
|
-
queueShardDeleted: 0
|
|
4408
|
-
};
|
|
4409
|
-
if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
|
|
4410
|
-
if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
|
|
4411
|
-
const delta = result.deleted - persisted.queueShardDeleted;
|
|
4412
|
-
const next = result.done ? {
|
|
4413
|
-
queueShard: requestedShard,
|
|
4414
|
-
queueShardDeleted: result.deleted,
|
|
4415
|
-
pendingAck: {
|
|
4416
|
-
queueShard: requestedShard,
|
|
4417
|
-
receipt: result.receipt
|
|
4418
|
-
}
|
|
4419
|
-
} : {
|
|
4420
|
-
queueShard: requestedShard,
|
|
4421
|
-
queueShardDeleted: result.deleted
|
|
4422
|
-
};
|
|
4423
|
-
await txn.put(CLEANUP_RECORD_KEY, {
|
|
4424
|
-
...current,
|
|
4425
|
-
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
4426
|
-
generation: current.generation + 1,
|
|
4427
|
-
attempts: 0,
|
|
4428
|
-
lastError: void 0
|
|
4429
|
-
});
|
|
4430
|
-
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
4431
|
-
await txn.setAlarm(this.now() + 1);
|
|
4432
|
-
});
|
|
4433
|
-
return;
|
|
4434
|
-
}
|
|
3922
|
+
const entries = await this.ctx.storage.list({
|
|
3923
|
+
prefix: QUEUE_PAYLOAD_REGISTRY_PREFIX,
|
|
3924
|
+
limit: 129
|
|
3925
|
+
});
|
|
3926
|
+
const page = Array.from(entries).slice(0, QUEUE_PAYLOADS_PER_CLEANUP_PAGE);
|
|
3927
|
+
const bucket = this.env?.WORKFLOW_QUEUE_PAYLOADS;
|
|
3928
|
+
if (!bucket) throw new Error("world-celld retention missing binding WORKFLOW_QUEUE_PAYLOADS");
|
|
3929
|
+
if (page.length > 0) await deleteQueuePayloadObjects(bucket, page.map(([, registration]) => registration.key));
|
|
4435
3930
|
await this.ctx.storage.transaction(async (txn) => {
|
|
4436
3931
|
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
4437
3932
|
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
await txn.delete(CLEANUP_PROGRESS_KEY);
|
|
4449
|
-
await txn.setAlarm(this.now() + 1);
|
|
4450
|
-
}
|
|
3933
|
+
if (page.length > 0) await txn.delete(page.map(([key]) => key));
|
|
3934
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3935
|
+
...current,
|
|
3936
|
+
deletedQueuePayloads: current.deletedQueuePayloads + page.length,
|
|
3937
|
+
phase: entries.size <= QUEUE_PAYLOADS_PER_CLEANUP_PAGE ? "payload" : "queues",
|
|
3938
|
+
generation: current.generation + 1,
|
|
3939
|
+
attempts: 0,
|
|
3940
|
+
lastError: void 0
|
|
3941
|
+
});
|
|
3942
|
+
await txn.setAlarm(this.now() + 1);
|
|
4451
3943
|
});
|
|
4452
3944
|
}
|
|
4453
3945
|
async deletePayloadBatch() {
|
|
@@ -4518,20 +4010,163 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
4518
4010
|
await txn.setAlarm(this.now() + delay);
|
|
4519
4011
|
});
|
|
4520
4012
|
}
|
|
4013
|
+
async registerQueuePayload(registration) {
|
|
4014
|
+
if (!registration || typeof registration.messageId !== "string" || registration.messageId.length === 0 || typeof registration.key !== "string" || registration.key.length === 0 || !isNonNegativeSafeInteger(registration.orphanExpiresAt)) throw new TypeError("world-celld queue payload registration is invalid");
|
|
4015
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4016
|
+
const { tombstone } = await this.retentionState(txn);
|
|
4017
|
+
if (tombstone) return {
|
|
4018
|
+
ok: false,
|
|
4019
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
4020
|
+
};
|
|
4021
|
+
await txn.put(queuePayloadRegistryKey(registration.messageId), registration);
|
|
4022
|
+
return { ok: true };
|
|
4023
|
+
});
|
|
4024
|
+
}
|
|
4025
|
+
async finalizeQueuePayload(messageId) {
|
|
4026
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue payload messageId must be a non-empty string");
|
|
4027
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4028
|
+
const { tombstone } = await this.retentionState(txn);
|
|
4029
|
+
if (tombstone) return {
|
|
4030
|
+
ok: false,
|
|
4031
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`
|
|
4032
|
+
};
|
|
4033
|
+
return await txn.get(queuePayloadRegistryKey(messageId)) ? { ok: true } : {
|
|
4034
|
+
ok: false,
|
|
4035
|
+
message: "workflow queue payload registration was removed before publication"
|
|
4036
|
+
};
|
|
4037
|
+
});
|
|
4038
|
+
}
|
|
4039
|
+
async unregisterQueuePayload(messageId) {
|
|
4040
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue payload messageId must be a non-empty string");
|
|
4041
|
+
await this.ctx.storage.delete(queuePayloadRegistryKey(messageId));
|
|
4042
|
+
}
|
|
4043
|
+
async scheduleQueuePayloadOrphan(orphan) {
|
|
4044
|
+
if (!orphan || typeof orphan.messageId !== "string" || orphan.messageId.length === 0 || typeof orphan.runId !== "string" || orphan.runId.length === 0 || typeof orphan.key !== "string" || orphan.key.length === 0 || !isNonNegativeSafeInteger(orphan.expiresAt)) throw new TypeError("world-celld queue payload orphan is invalid");
|
|
4045
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
4046
|
+
const existing = await txn.get(QUEUE_ORPHAN_KEY);
|
|
4047
|
+
if (existing && existing.messageId !== orphan.messageId) throw new Error("world-celld queue payload orphan identity mismatch");
|
|
4048
|
+
if (existing?.deleting) throw new Error("world-celld queue payload orphan deletion is in progress");
|
|
4049
|
+
const next = {
|
|
4050
|
+
...orphan,
|
|
4051
|
+
expiresAt: Math.max(existing?.expiresAt ?? 0, orphan.expiresAt),
|
|
4052
|
+
generation: (existing?.generation ?? 0) + 1,
|
|
4053
|
+
deleting: false
|
|
4054
|
+
};
|
|
4055
|
+
await txn.put(QUEUE_ORPHAN_KEY, next);
|
|
4056
|
+
await txn.setAlarm(next.expiresAt);
|
|
4057
|
+
});
|
|
4058
|
+
}
|
|
4059
|
+
async cancelQueuePayloadOrphan(messageId) {
|
|
4060
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue payload orphan messageId must be a non-empty string");
|
|
4061
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
4062
|
+
const existing = await txn.get(QUEUE_ORPHAN_KEY);
|
|
4063
|
+
if (!existing || existing.messageId !== messageId) return;
|
|
4064
|
+
await txn.delete(QUEUE_ORPHAN_KEY);
|
|
4065
|
+
await txn.deleteAlarm();
|
|
4066
|
+
});
|
|
4067
|
+
}
|
|
4068
|
+
async reserveQueueMessage(params) {
|
|
4069
|
+
if (typeof params?.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.expiresAt)) throw new TypeError("world-celld queue message reservation is invalid");
|
|
4070
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4071
|
+
const values = await txn.get([QUEUE_RESERVATION_KEY, "claim"]);
|
|
4072
|
+
const existing = values.get(QUEUE_RESERVATION_KEY);
|
|
4073
|
+
const claim = values.get("claim");
|
|
4074
|
+
const now = this.now();
|
|
4075
|
+
if (existing && existing.expiresAt > now && existing.messageId !== params.messageId) {
|
|
4076
|
+
await txn.setAlarm(Math.min(existing.expiresAt, claim?.expiresAt ?? existing.expiresAt));
|
|
4077
|
+
return {
|
|
4078
|
+
admitted: false,
|
|
4079
|
+
messageId: existing.messageId
|
|
4080
|
+
};
|
|
4081
|
+
}
|
|
4082
|
+
const reservation = {
|
|
4083
|
+
messageId: params.messageId,
|
|
4084
|
+
expiresAt: Math.max(existing?.expiresAt ?? 0, params.expiresAt)
|
|
4085
|
+
};
|
|
4086
|
+
await txn.put(QUEUE_RESERVATION_KEY, reservation);
|
|
4087
|
+
await txn.setAlarm(Math.min(reservation.expiresAt, claim?.expiresAt ?? reservation.expiresAt));
|
|
4088
|
+
return {
|
|
4089
|
+
admitted: true,
|
|
4090
|
+
messageId: params.messageId
|
|
4091
|
+
};
|
|
4092
|
+
});
|
|
4093
|
+
}
|
|
4094
|
+
async completeQueueMessage(messageId) {
|
|
4095
|
+
if (typeof messageId !== "string" || messageId.length === 0) throw new TypeError("world-celld queue messageId must be a non-empty string");
|
|
4096
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
4097
|
+
const values = await txn.get([QUEUE_RESERVATION_KEY, "claim"]);
|
|
4098
|
+
let reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
4099
|
+
let claim = values.get("claim");
|
|
4100
|
+
if (reservation?.messageId === messageId) {
|
|
4101
|
+
await txn.delete(QUEUE_RESERVATION_KEY);
|
|
4102
|
+
reservation = void 0;
|
|
4103
|
+
}
|
|
4104
|
+
if (claim?.messageId === messageId) {
|
|
4105
|
+
await txn.delete("claim");
|
|
4106
|
+
claim = void 0;
|
|
4107
|
+
}
|
|
4108
|
+
const nextAlarm = Math.min(reservation?.expiresAt ?? Number.POSITIVE_INFINITY, claim?.expiresAt ?? Number.POSITIVE_INFINITY);
|
|
4109
|
+
if (Number.isFinite(nextAlarm)) await txn.setAlarm(nextAlarm);
|
|
4110
|
+
else await txn.deleteAlarm();
|
|
4111
|
+
});
|
|
4112
|
+
}
|
|
4521
4113
|
async claimInflight(params) {
|
|
4114
|
+
if (typeof params?.messageId !== "string" || params.messageId.length === 0 || !isPositiveSafeInteger(params.staleMs)) throw new TypeError("world-celld queue claim is invalid");
|
|
4522
4115
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
4523
|
-
const
|
|
4116
|
+
const values = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
4117
|
+
const existing = values.get("claim");
|
|
4118
|
+
const reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
4524
4119
|
const now = this.now();
|
|
4525
|
-
if (existing && existing.
|
|
4120
|
+
if (existing && existing.expiresAt > now) {
|
|
4121
|
+
if (existing.messageId !== params.messageId) return { claimed: false };
|
|
4122
|
+
if (existing.retryAt === void 0 || existing.retryAt > now) return {
|
|
4123
|
+
claimed: false,
|
|
4124
|
+
retryAt: existing.retryAt ?? existing.expiresAt
|
|
4125
|
+
};
|
|
4126
|
+
}
|
|
4127
|
+
const expiresAt = now + params.staleMs;
|
|
4526
4128
|
await txn.put("claim", {
|
|
4527
4129
|
messageId: params.messageId,
|
|
4528
|
-
|
|
4130
|
+
expiresAt
|
|
4529
4131
|
});
|
|
4132
|
+
await txn.setAlarm(Math.min(expiresAt, reservation?.expiresAt ?? expiresAt));
|
|
4530
4133
|
return { claimed: true };
|
|
4531
4134
|
});
|
|
4532
4135
|
}
|
|
4533
|
-
async
|
|
4534
|
-
|
|
4136
|
+
async holdInflight(params) {
|
|
4137
|
+
if (typeof params?.messageId !== "string" || params.messageId.length === 0 || !isNonNegativeSafeInteger(params.retryAt) || !isNonNegativeSafeInteger(params.expiresAt) || params.reservationExpiresAt !== void 0 && !isNonNegativeSafeInteger(params.reservationExpiresAt)) throw new TypeError("world-celld queue claim hold is invalid");
|
|
4138
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
4139
|
+
const values = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
4140
|
+
const existing = values.get("claim");
|
|
4141
|
+
let reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
4142
|
+
if (!existing || existing.messageId !== params.messageId) return { held: false };
|
|
4143
|
+
const expiresAt = Math.max(existing.expiresAt, params.expiresAt);
|
|
4144
|
+
await txn.put("claim", {
|
|
4145
|
+
...existing,
|
|
4146
|
+
retryAt: params.retryAt,
|
|
4147
|
+
expiresAt
|
|
4148
|
+
});
|
|
4149
|
+
if (reservation?.messageId === params.messageId && params.reservationExpiresAt !== void 0 && params.reservationExpiresAt > reservation.expiresAt) {
|
|
4150
|
+
reservation = {
|
|
4151
|
+
...reservation,
|
|
4152
|
+
expiresAt: params.reservationExpiresAt
|
|
4153
|
+
};
|
|
4154
|
+
await txn.put(QUEUE_RESERVATION_KEY, reservation);
|
|
4155
|
+
}
|
|
4156
|
+
await txn.setAlarm(Math.min(expiresAt, reservation?.expiresAt ?? expiresAt));
|
|
4157
|
+
return { held: true };
|
|
4158
|
+
});
|
|
4159
|
+
}
|
|
4160
|
+
async releaseInflight(messageId) {
|
|
4161
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
4162
|
+
const values = await txn.get(["claim", QUEUE_RESERVATION_KEY]);
|
|
4163
|
+
const existing = values.get("claim");
|
|
4164
|
+
const reservation = values.get(QUEUE_RESERVATION_KEY);
|
|
4165
|
+
if (!existing || messageId !== void 0 && existing.messageId !== messageId) return;
|
|
4166
|
+
await txn.delete("claim");
|
|
4167
|
+
if (reservation) await txn.setAlarm(reservation.expiresAt);
|
|
4168
|
+
else await txn.deleteAlarm();
|
|
4169
|
+
});
|
|
4535
4170
|
}
|
|
4536
4171
|
};
|
|
4537
4172
|
//#endregion
|
|
@@ -4550,11 +4185,23 @@ async function scheduled(controller, env) {
|
|
|
4550
4185
|
}
|
|
4551
4186
|
var worker_default = {
|
|
4552
4187
|
async fetch(request, env) {
|
|
4553
|
-
return createRouter(
|
|
4188
|
+
return createRouter({
|
|
4189
|
+
...env,
|
|
4190
|
+
WORKFLOW_QUEUE_PAYLOADS: env.WORKFLOW_QUEUE_PAYLOADS ? createQueuePayloadStore(env.WORKFLOW_QUEUE_PAYLOADS) : void 0
|
|
4191
|
+
})(request);
|
|
4554
4192
|
},
|
|
4555
4193
|
scheduled
|
|
4556
4194
|
};
|
|
4195
|
+
/** Only the companion consumer binds this internal delivery entrypoint. */
|
|
4196
|
+
var QueueDeliveryRpc = class extends WorkerEntrypoint {
|
|
4197
|
+
deliver(secret, envelope, attempt) {
|
|
4198
|
+
return deliverQueueMessage({
|
|
4199
|
+
...this.env,
|
|
4200
|
+
WORKFLOW_QUEUE_PAYLOADS: this.env.WORKFLOW_QUEUE_PAYLOADS ? createQueuePayloadStore(this.env.WORKFLOW_QUEUE_PAYLOADS) : void 0
|
|
4201
|
+
}, secret, envelope, attempt);
|
|
4202
|
+
}
|
|
4203
|
+
};
|
|
4557
4204
|
//#endregion
|
|
4558
|
-
export { HookIdDO, HookTokenDO,
|
|
4205
|
+
export { HookIdDO, HookTokenDO, QueueDeliveryRpc, RunCatalogDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
|
|
4559
4206
|
|
|
4560
4207
|
//# sourceMappingURL=worker.js.map
|