@ewhauser/world-celld 0.3.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +101 -22
- package/celld-worker/README.md +24 -0
- package/celld-worker/wrangler.jsonc +12 -0
- package/dist/index.d.ts +16 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +270 -52
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts +10 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +3128 -2522
- package/dist/testing.js.map +1 -1
- package/dist/{util-ya3V_FTA.js → util-Cu09mVzK.js} +86 -11
- package/dist/util-Cu09mVzK.js.map +1 -0
- package/dist/worker.d.ts +49 -6
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +3642 -2885
- package/dist/worker.js.map +1 -1
- package/package.json +2 -1
- package/dist/util-ya3V_FTA.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
import { C as
|
|
2
|
-
import { EventSchema, HookSchema, MessageId, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema, isTerminalWorkflowRunStatus, parseQueueName } from "@workflow/world";
|
|
1
|
+
import { A as MAX_STREAM_READ_CHUNKS, C as queueDelayDeadline, D as MAX_STREAM_ERROR_BYTES, E as MAX_STREAM_CHUNK_BYTES, F as decodeStreamWriteResult, L as encodeStreamWriteBatch, M as STREAM_BATCH_CONTENT_TYPE, N as decodeStreamReadResult, O as MAX_STREAM_LONG_POLL_MS, T as MAX_STREAM_BATCH_BYTES, _ as boundedIntegerOption, a as stringify, d as MAX_QUEUE_DELIVERY_TIMEOUT_MS, f as MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS, i as parse, j as MAX_STREAM_WRITE_CHUNKS, k as MAX_STREAM_READ_BYTES, n as rpcParse, o as createWorkflowIndex, r as rpcStringify, t as compact, u as MAX_FLEET_RPC_TIMEOUT_MS, v as checkedQueueTimestampAdd, w as strictIntegerSetting, z as normalizeStreamError } from "./util-Cu09mVzK.js";
|
|
2
|
+
import { CreateEventSchema, EventSchema, HookSchema, MessageId, QueuePayloadSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, isTerminalWorkflowRunStatus, parseQueueName } from "@workflow/world";
|
|
3
3
|
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
4
|
-
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
5
4
|
import { monotonicFactory } from "ulid";
|
|
6
5
|
//#region src/config.ts
|
|
7
6
|
function resolveConfig(config) {
|
|
8
|
-
const
|
|
9
|
-
const runRetentionMs =
|
|
10
|
-
if (!Number.isSafeInteger(runRetentionMs) || runRetentionMs < 0) throw new Error("world-celld: runRetentionMs must be a non-negative safe integer");
|
|
7
|
+
const queueShards = boundedIntegerOption("world-celld: queueShards", config?.queueShards, 1, 1);
|
|
8
|
+
const runRetentionMs = config?.runRetentionMs !== void 0 ? boundedIntegerOption("world-celld: runRetentionMs", config.runRetentionMs, 0, 0) : strictIntegerSetting("world-celld: runRetentionMs", process.env.CELLD_RUN_RETENTION_MS, 0, 0);
|
|
11
9
|
const rpcTimeoutMs = config?.rpcTimeoutMs ?? 3e4;
|
|
12
10
|
if (!Number.isSafeInteger(rpcTimeoutMs) || rpcTimeoutMs < 1 || rpcTimeoutMs > 3e5) throw new Error(`world-celld: rpcTimeoutMs must be between 1 and ${MAX_FLEET_RPC_TIMEOUT_MS}`);
|
|
13
11
|
const streamLongPollMs = config?.streamLongPollMs ?? Math.min(2e4, Math.max(1, rpcTimeoutMs - 1e3));
|
|
@@ -21,7 +19,7 @@ function resolveConfig(config) {
|
|
|
21
19
|
env: config?.env,
|
|
22
20
|
deploymentId: config?.deploymentId ?? process.env.CELLD_DEPLOYMENT_ID ?? "celld-default",
|
|
23
21
|
baseUrl: config?.baseUrl,
|
|
24
|
-
queueShards
|
|
22
|
+
queueShards,
|
|
25
23
|
runRetentionMs,
|
|
26
24
|
streamLongPollMs,
|
|
27
25
|
streamFlushIntervalMs,
|
|
@@ -59,10 +57,15 @@ const RETRYABLE_STATUSES$1 = /* @__PURE__ */ new Set([
|
|
|
59
57
|
503,
|
|
60
58
|
504
|
|
61
59
|
]);
|
|
62
|
-
const DEFAULT_TIMEOUT_MS
|
|
60
|
+
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
63
61
|
function delayMs$1(ms) {
|
|
64
62
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
65
63
|
}
|
|
64
|
+
function resolveFleetTimeoutMs(transport) {
|
|
65
|
+
const timeoutMs = transport.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
66
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 3e5) throw new Error(`world-celld: fleet RPC timeout must be between 1 and ${MAX_FLEET_RPC_TIMEOUT_MS}`);
|
|
67
|
+
return timeoutMs;
|
|
68
|
+
}
|
|
66
69
|
/**
|
|
67
70
|
* Invoke one DO method through the worker router.
|
|
68
71
|
*
|
|
@@ -79,8 +82,7 @@ async function callFleetRoute(transport, path, args, opts) {
|
|
|
79
82
|
const doFetch = transport.fetchImpl ?? fetch;
|
|
80
83
|
const url = `${transport.fleetUrl.replace(/\/$/, "")}${path}`;
|
|
81
84
|
const attempts = opts?.idempotent ? 3 : 1;
|
|
82
|
-
const timeoutMs = transport
|
|
83
|
-
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 3e5) throw new Error(`world-celld: fleet RPC timeout must be between 1 and ${MAX_FLEET_RPC_TIMEOUT_MS}`);
|
|
85
|
+
const timeoutMs = resolveFleetTimeoutMs(transport);
|
|
84
86
|
let lastError;
|
|
85
87
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
86
88
|
if (attempt > 1) await delayMs$1(100 * attempt + Math.random() * 200);
|
|
@@ -129,15 +131,27 @@ const RETRYABLE_STATUSES = /* @__PURE__ */ new Set([
|
|
|
129
131
|
504
|
|
130
132
|
]);
|
|
131
133
|
const READ_ATTEMPTS = 3;
|
|
132
|
-
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
133
134
|
const MAX_WRITE_RESPONSE_BYTES = 64;
|
|
134
135
|
const MAX_READ_RESPONSE_BYTES = MAX_STREAM_READ_BYTES + 65536;
|
|
135
136
|
const MAX_ERROR_RESPONSE_BYTES = 65536;
|
|
136
|
-
function delayMs(ms) {
|
|
137
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
137
|
+
function delayMs(ms, signal) {
|
|
138
|
+
if (!signal) return new Promise((resolve) => setTimeout(resolve, ms));
|
|
139
|
+
if (signal.aborted) return Promise.reject(aborted(signal));
|
|
140
|
+
return new Promise((resolve, reject) => {
|
|
141
|
+
const timer = setTimeout(() => {
|
|
142
|
+
signal.removeEventListener("abort", onAbort);
|
|
143
|
+
resolve();
|
|
144
|
+
}, ms);
|
|
145
|
+
const onAbort = () => {
|
|
146
|
+
clearTimeout(timer);
|
|
147
|
+
signal.removeEventListener("abort", onAbort);
|
|
148
|
+
reject(aborted(signal));
|
|
149
|
+
};
|
|
150
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
151
|
+
});
|
|
138
152
|
}
|
|
139
153
|
function requestSignal(timeoutMs, signal) {
|
|
140
|
-
const timeout = AbortSignal.timeout(
|
|
154
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
141
155
|
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
142
156
|
}
|
|
143
157
|
function aborted(signal) {
|
|
@@ -188,6 +202,7 @@ function streamUrl(transport, name) {
|
|
|
188
202
|
return `${transport.fleetUrl.replace(/\/$/, "")}/v1/streams/${encodeURIComponent(name)}/chunks`;
|
|
189
203
|
}
|
|
190
204
|
async function writeStreamChunks(transport, name, runId, chunks) {
|
|
205
|
+
const timeoutMs = resolveFleetTimeoutMs(transport);
|
|
191
206
|
const body = encodeStreamWriteBatch(chunks);
|
|
192
207
|
if (body.byteLength > 8389632) throw new Error("world-celld: encoded stream batch exceeds configured limit");
|
|
193
208
|
const url = new URL(streamUrl(transport, name));
|
|
@@ -202,7 +217,7 @@ async function writeStreamChunks(transport, name, runId, chunks) {
|
|
|
202
217
|
authorization: `Bearer ${transport.secret}`
|
|
203
218
|
},
|
|
204
219
|
body,
|
|
205
|
-
signal: requestSignal(
|
|
220
|
+
signal: requestSignal(timeoutMs)
|
|
206
221
|
});
|
|
207
222
|
} catch (error) {
|
|
208
223
|
throw new FleetTransportError(`world-celld: fleet unreachable at ${url.href}`, error);
|
|
@@ -216,6 +231,7 @@ async function writeStreamChunks(transport, name, runId, chunks) {
|
|
|
216
231
|
}
|
|
217
232
|
}
|
|
218
233
|
async function readStreamChunks(transport, name, request, signal) {
|
|
234
|
+
const timeoutMs = resolveFleetTimeoutMs(transport);
|
|
219
235
|
const url = new URL(streamUrl(transport, name));
|
|
220
236
|
url.searchParams.set("runId", request.runId);
|
|
221
237
|
url.searchParams.set("startIndex", String(request.startIndex));
|
|
@@ -226,27 +242,38 @@ async function readStreamChunks(transport, name, request, signal) {
|
|
|
226
242
|
let lastError;
|
|
227
243
|
for (let attempt = 1; attempt <= READ_ATTEMPTS; attempt++) {
|
|
228
244
|
if (signal?.aborted) throw aborted(signal);
|
|
229
|
-
if (attempt > 1) await delayMs(100 * attempt + Math.random() * 200);
|
|
245
|
+
if (attempt > 1) await delayMs(100 * attempt + Math.random() * 200, signal);
|
|
230
246
|
let response;
|
|
231
247
|
try {
|
|
232
248
|
response = await doFetch(url, {
|
|
233
249
|
method: "GET",
|
|
234
250
|
headers: { authorization: `Bearer ${transport.secret}` },
|
|
235
|
-
signal: requestSignal(
|
|
251
|
+
signal: requestSignal(timeoutMs, signal)
|
|
236
252
|
});
|
|
237
253
|
} catch (error) {
|
|
238
254
|
if (signal?.aborted) throw aborted(signal);
|
|
239
255
|
lastError = new FleetTransportError(`world-celld: fleet unreachable at ${url.href}`, error);
|
|
240
256
|
continue;
|
|
241
257
|
}
|
|
258
|
+
if (signal?.aborted) {
|
|
259
|
+
await response.body?.cancel(aborted(signal)).catch(() => void 0);
|
|
260
|
+
throw aborted(signal);
|
|
261
|
+
}
|
|
242
262
|
if (response.ok) try {
|
|
243
263
|
return decodeStreamReadResult(await readBoundedResponse(response, MAX_READ_RESPONSE_BYTES));
|
|
244
264
|
} catch (error) {
|
|
245
|
-
|
|
265
|
+
if (signal?.aborted) throw aborted(signal);
|
|
266
|
+
lastError = error instanceof FleetTransportError ? error : new FleetTransportError(`world-celld: malformed stream response from ${url.href}`, error);
|
|
246
267
|
if (attempt < READ_ATTEMPTS) continue;
|
|
247
268
|
throw lastError;
|
|
248
269
|
}
|
|
249
|
-
|
|
270
|
+
let error;
|
|
271
|
+
try {
|
|
272
|
+
error = await errorFromResponse(response);
|
|
273
|
+
} catch (cause) {
|
|
274
|
+
if (signal?.aborted) throw aborted(signal);
|
|
275
|
+
error = cause instanceof FleetTransportError ? cause : new FleetTransportError(`world-celld: malformed stream error response from ${url.href}`, cause);
|
|
276
|
+
}
|
|
250
277
|
if (RETRYABLE_STATUSES.has(response.status) && attempt < READ_ATTEMPTS) {
|
|
251
278
|
lastError = error;
|
|
252
279
|
continue;
|
|
@@ -277,7 +304,8 @@ const RUNS = {
|
|
|
277
304
|
"applyEvent",
|
|
278
305
|
"scheduleCleanup",
|
|
279
306
|
"cleanupNow",
|
|
280
|
-
"rearmCleanup"
|
|
307
|
+
"rearmCleanup",
|
|
308
|
+
"resolveHookTokenClaim"
|
|
281
309
|
])
|
|
282
310
|
};
|
|
283
311
|
const STREAMS = {
|
|
@@ -464,6 +492,19 @@ const debug = createDebugLogger("celld-world");
|
|
|
464
492
|
* HTTP statuses. QueueDO deliveries and the in-process test pump hit the
|
|
465
493
|
* exact same handler path.
|
|
466
494
|
*/
|
|
495
|
+
const MAX_TIMER_DELAY_MS = 2147483647;
|
|
496
|
+
const MAX_TEST_PUMP_BACKOFF_MS = 6e4;
|
|
497
|
+
function delay(milliseconds) {
|
|
498
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
499
|
+
}
|
|
500
|
+
async function delayFor(milliseconds) {
|
|
501
|
+
let remaining = milliseconds;
|
|
502
|
+
while (remaining > MAX_TIMER_DELAY_MS) {
|
|
503
|
+
await delay(MAX_TIMER_DELAY_MS);
|
|
504
|
+
remaining -= MAX_TIMER_DELAY_MS;
|
|
505
|
+
}
|
|
506
|
+
await delay(remaining);
|
|
507
|
+
}
|
|
467
508
|
/**
|
|
468
509
|
* HTTP status codes that indicate permanent (non-retryable) failures.
|
|
469
510
|
* Deliveries with these statuses are dropped immediately instead of retried.
|
|
@@ -502,6 +543,7 @@ function resolveBaseUrl(config) {
|
|
|
502
543
|
* land on the same cell so dedup stays strictly consistent.
|
|
503
544
|
*/
|
|
504
545
|
function shardFor(key, shards) {
|
|
546
|
+
if (!Number.isSafeInteger(shards) || shards < 1) throw new Error("world-celld queueShards must be a positive safe integer");
|
|
505
547
|
if (shards <= 1) return 0;
|
|
506
548
|
let hash = 2166136261;
|
|
507
549
|
for (let i = 0; i < key.length; i++) {
|
|
@@ -523,9 +565,9 @@ function shardFor(key, shards) {
|
|
|
523
565
|
* matching QueueDO's production semantics.
|
|
524
566
|
*/
|
|
525
567
|
function createTestPump(config) {
|
|
526
|
-
const httpTimeoutMs = config.httpTimeoutMs
|
|
527
|
-
const maxAttempts = config.maxAttempts
|
|
528
|
-
const baseBackoffMs = config.backoffDelayMs
|
|
568
|
+
const httpTimeoutMs = boundedIntegerOption("world-celld test pump httpTimeoutMs", config.httpTimeoutMs, 3e5, 1, MAX_QUEUE_DELIVERY_TIMEOUT_MS);
|
|
569
|
+
const maxAttempts = boundedIntegerOption("world-celld test pump maxAttempts", config.maxAttempts, 5, 1);
|
|
570
|
+
const baseBackoffMs = boundedIntegerOption("world-celld test pump backoffDelayMs", config.backoffDelayMs, 1e3, 0, MAX_TEST_PUMP_BACKOFF_MS);
|
|
529
571
|
const queues = { flow: [] };
|
|
530
572
|
const wakers = { flow: [] };
|
|
531
573
|
/** Inflight messageIds by idempotencyKey (world-local queue.js semantics). */
|
|
@@ -574,8 +616,10 @@ function createTestPump(config) {
|
|
|
574
616
|
parsed = null;
|
|
575
617
|
}
|
|
576
618
|
const timeoutSeconds = parsed?.timeoutSeconds;
|
|
577
|
-
|
|
578
|
-
|
|
619
|
+
const now = Date.now();
|
|
620
|
+
const redeliveryAt = queueDelayDeadline(now, timeoutSeconds, 1, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
621
|
+
if (redeliveryAt !== null) {
|
|
622
|
+
delayFor(redeliveryAt - now).then(() => enqueue(pathname, envelope));
|
|
579
623
|
return;
|
|
580
624
|
}
|
|
581
625
|
}
|
|
@@ -589,8 +633,13 @@ function createTestPump(config) {
|
|
|
589
633
|
...envelope,
|
|
590
634
|
attempt: envelope.attempt + 1
|
|
591
635
|
};
|
|
592
|
-
const backoff = baseBackoffMs * 2 ** (next.attempt - 1);
|
|
593
|
-
|
|
636
|
+
const backoff = Math.min(MAX_TEST_PUMP_BACKOFF_MS, baseBackoffMs * 2 ** Math.min(next.attempt - 1, 30));
|
|
637
|
+
if (checkedQueueTimestampAdd(Date.now(), backoff, 330001) === null) {
|
|
638
|
+
release(envelope);
|
|
639
|
+
console.error(`[world-celld test pump] dropping ${envelope.messageId}: retry deadline exceeds the queue timestamp horizon`);
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
delayFor(backoff).then(() => enqueue(pathname, next));
|
|
594
643
|
} else {
|
|
595
644
|
release(envelope);
|
|
596
645
|
console.error(`[world-celld test pump] dropping ${envelope.messageId} after ${envelope.attempt} attempts: HTTP ${response.status}: ${text}`);
|
|
@@ -615,8 +664,9 @@ function createTestPump(config) {
|
|
|
615
664
|
return idempotencyKey ? inflightMessages.get(idempotencyKey) : void 0;
|
|
616
665
|
},
|
|
617
666
|
push(pathname, envelope, delaySeconds) {
|
|
667
|
+
if (queueDelayDeadline(Date.now(), delaySeconds ?? 0, 0, 330001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
|
|
618
668
|
if (envelope.idempotencyKey) inflightMessages.set(envelope.idempotencyKey, MessageId.parse(envelope.messageId));
|
|
619
|
-
if (delaySeconds && delaySeconds > 0)
|
|
669
|
+
if (delaySeconds && delaySeconds > 0) delayFor(delaySeconds * 1e3).then(() => enqueue(pathname, envelope));
|
|
620
670
|
else enqueue(pathname, envelope);
|
|
621
671
|
},
|
|
622
672
|
async start() {
|
|
@@ -632,7 +682,8 @@ function createTestPump(config) {
|
|
|
632
682
|
}
|
|
633
683
|
function createQueue(config) {
|
|
634
684
|
const { env, deploymentId } = config;
|
|
635
|
-
|
|
685
|
+
if (!env?.WORKFLOW_QUEUE) throw new Error("world-celld queue missing WORKFLOW_QUEUE binding");
|
|
686
|
+
const queueShards = boundedIntegerOption("world-celld queueShards", config.queueShards, 1, 1);
|
|
636
687
|
const generateMessageId = monotonicFactory();
|
|
637
688
|
const testPump = createTestPump(config);
|
|
638
689
|
const getQueueStub = (shard) => {
|
|
@@ -642,7 +693,10 @@ function createQueue(config) {
|
|
|
642
693
|
return {
|
|
643
694
|
async queue(queueName, message, opts) {
|
|
644
695
|
parseQueueName(queueName);
|
|
645
|
-
const
|
|
696
|
+
const parsedMessage = QueuePayloadSchema.safeParse(message);
|
|
697
|
+
if (!parsedMessage.success) throw new WorkflowWorldError("world-celld queue payload is invalid", { status: 422 });
|
|
698
|
+
if (queueDelayDeadline(Date.now(), opts?.delaySeconds ?? 0, 0, 330001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
|
|
699
|
+
const runId = "runId" in parsedMessage.data && typeof parsedMessage.data.runId === "string" ? parsedMessage.data.runId : void 0;
|
|
646
700
|
if (isTestMode()) {
|
|
647
701
|
const existing = testPump.inflight(opts?.idempotencyKey);
|
|
648
702
|
if (existing) return { messageId: existing };
|
|
@@ -651,7 +705,7 @@ function createQueue(config) {
|
|
|
651
705
|
messageId,
|
|
652
706
|
queueName,
|
|
653
707
|
attempt: 1,
|
|
654
|
-
message,
|
|
708
|
+
message: parsedMessage.data,
|
|
655
709
|
idempotencyKey: opts?.idempotencyKey
|
|
656
710
|
}, opts?.delaySeconds);
|
|
657
711
|
return { messageId };
|
|
@@ -662,7 +716,7 @@ function createQueue(config) {
|
|
|
662
716
|
messageId,
|
|
663
717
|
queueName,
|
|
664
718
|
pathname: QUEUE_PATHNAME,
|
|
665
|
-
body: stringify(
|
|
719
|
+
body: stringify(parsedMessage.data),
|
|
666
720
|
runId,
|
|
667
721
|
idempotencyKey: opts?.idempotencyKey,
|
|
668
722
|
delaySeconds: opts?.delaySeconds,
|
|
@@ -679,22 +733,40 @@ function createQueue(config) {
|
|
|
679
733
|
},
|
|
680
734
|
createQueueHandler(queueNamePrefix, handler) {
|
|
681
735
|
return async (req) => {
|
|
736
|
+
if (req.method !== "POST") return Response.json({ error: "Expected POST" }, { status: 405 });
|
|
737
|
+
if (req.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json") return Response.json({ error: "Expected application/json" }, { status: 415 });
|
|
682
738
|
const reqQueueName = req.headers.get("x-vqs-queue-name");
|
|
683
739
|
const reqMessageId = req.headers.get("x-vqs-message-id");
|
|
684
740
|
const attemptStr = req.headers.get("x-vqs-message-attempt");
|
|
685
741
|
if (!reqQueueName || !reqMessageId || !attemptStr || !req.body) return Response.json({ error: "Missing required headers or body" }, { status: 400 });
|
|
686
742
|
if (!reqQueueName.startsWith(queueNamePrefix)) return Response.json({ error: "Unhandled queue" }, { status: 400 });
|
|
687
|
-
|
|
743
|
+
let attempt;
|
|
688
744
|
try {
|
|
689
|
-
|
|
745
|
+
attempt = strictIntegerSetting("x-vqs-message-attempt", attemptStr, 1, 1);
|
|
746
|
+
} catch {
|
|
747
|
+
return Response.json({ error: "Invalid x-vqs-message-attempt header" }, { status: 400 });
|
|
748
|
+
}
|
|
749
|
+
try {
|
|
750
|
+
let body;
|
|
751
|
+
try {
|
|
752
|
+
body = parse(await req.text());
|
|
753
|
+
} catch {
|
|
754
|
+
return Response.json({ error: "Malformed tagged JSON body" }, { status: 422 });
|
|
755
|
+
}
|
|
756
|
+
const parsedBody = QueuePayloadSchema.safeParse(body);
|
|
757
|
+
if (!parsedBody.success) return Response.json({ error: "Invalid queue payload" }, { status: 422 });
|
|
758
|
+
const result = await handler(parsedBody.data, {
|
|
690
759
|
attempt,
|
|
691
760
|
queueName: reqQueueName,
|
|
692
761
|
messageId: reqMessageId
|
|
693
762
|
});
|
|
694
|
-
if (result && typeof result.timeoutSeconds === "number")
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
763
|
+
if (result && typeof result.timeoutSeconds === "number") {
|
|
764
|
+
if (queueDelayDeadline(Date.now(), result.timeoutSeconds, 1, 330001) === null) throw new Error("Queue handler timeoutSeconds is out of range");
|
|
765
|
+
return Response.json({ timeoutSeconds: result.timeoutSeconds }, {
|
|
766
|
+
status: 503,
|
|
767
|
+
headers: { "Retry-After": String(result.timeoutSeconds) }
|
|
768
|
+
});
|
|
769
|
+
}
|
|
698
770
|
return new Response(null, { status: 204 });
|
|
699
771
|
} catch (error) {
|
|
700
772
|
if (isPermanentError(error)) {
|
|
@@ -738,6 +810,114 @@ function hookOwner(hook) {
|
|
|
738
810
|
hookId: hook.hookId
|
|
739
811
|
};
|
|
740
812
|
}
|
|
813
|
+
const APPLY_EVENT_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
814
|
+
"RUN_NOT_FOUND",
|
|
815
|
+
"STEP_NOT_FOUND",
|
|
816
|
+
"HOOK_NOT_FOUND",
|
|
817
|
+
"WAIT_NOT_FOUND",
|
|
818
|
+
"ENTITY_CONFLICT",
|
|
819
|
+
"HOOK_CLAIM_CANCELLED",
|
|
820
|
+
"RUN_EXPIRED",
|
|
821
|
+
"TOO_EARLY",
|
|
822
|
+
"RUN_NOT_SUPPORTED"
|
|
823
|
+
]);
|
|
824
|
+
function malformedApplyEventOutcome(reason) {
|
|
825
|
+
throw new FleetTransportError(`world-celld: malformed applyEvent outcome: ${reason}`);
|
|
826
|
+
}
|
|
827
|
+
function isRecord(value) {
|
|
828
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
829
|
+
}
|
|
830
|
+
function validateOptionalNumber(outcome, key, integer) {
|
|
831
|
+
const value = outcome[key];
|
|
832
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isFinite(value) || integer && !Number.isSafeInteger(value))) malformedApplyEventOutcome(`${key} must be ${integer ? "a safe integer" : "a finite number"}`);
|
|
833
|
+
}
|
|
834
|
+
function validateOptionalRetryAfterMs(outcome) {
|
|
835
|
+
const value = outcome.retryAfterMs;
|
|
836
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)) malformedApplyEventOutcome("retryAfterMs must be a non-negative safe integer");
|
|
837
|
+
}
|
|
838
|
+
function parseSuccessEntity(outcome, key, schema) {
|
|
839
|
+
const value = outcome[key];
|
|
840
|
+
if (value === void 0) return void 0;
|
|
841
|
+
if (!isRecord(value)) malformedApplyEventOutcome(`${key} is invalid`);
|
|
842
|
+
const result = schema.safeParse(compact(value));
|
|
843
|
+
if (!result.success) malformedApplyEventOutcome(`${key} is invalid`);
|
|
844
|
+
return result.data;
|
|
845
|
+
}
|
|
846
|
+
/**
|
|
847
|
+
* Validate the untrusted value decoded from the remote applyEvent RPC.
|
|
848
|
+
* Unknown failure codes are intentionally valid and classified separately so
|
|
849
|
+
* an older client can reconstruct their wire metadata without treating them
|
|
850
|
+
* as success. All fields consumed by the success path are checked before any
|
|
851
|
+
* derivative index operation can run.
|
|
852
|
+
*/
|
|
853
|
+
function parseApplyEventOutcome(value) {
|
|
854
|
+
if (!isRecord(value)) malformedApplyEventOutcome("response must be an object");
|
|
855
|
+
if (value.ok === false) {
|
|
856
|
+
if (typeof value.code !== "string" || value.code.length === 0) malformedApplyEventOutcome("failure code must be a non-empty string");
|
|
857
|
+
if (typeof value.message !== "string") malformedApplyEventOutcome("failure message must be a string");
|
|
858
|
+
validateOptionalNumber(value, "status", true);
|
|
859
|
+
validateOptionalNumber(value, "retryAfter", false);
|
|
860
|
+
validateOptionalRetryAfterMs(value);
|
|
861
|
+
validateOptionalNumber(value, "retryAfterSeconds", false);
|
|
862
|
+
validateOptionalNumber(value, "runSpecVersion", true);
|
|
863
|
+
validateOptionalNumber(value, "worldSpecVersion", true);
|
|
864
|
+
const outcome = value;
|
|
865
|
+
return APPLY_EVENT_ERROR_CODES.has(outcome.code) ? {
|
|
866
|
+
kind: "known-failure",
|
|
867
|
+
outcome
|
|
868
|
+
} : {
|
|
869
|
+
kind: "unknown-failure",
|
|
870
|
+
outcome
|
|
871
|
+
};
|
|
872
|
+
}
|
|
873
|
+
if (value.ok !== true) malformedApplyEventOutcome("ok must be true or false");
|
|
874
|
+
if (!Array.isArray(value.releasedHooks)) malformedApplyEventOutcome("releasedHooks must be an array");
|
|
875
|
+
for (const releasedHook of value.releasedHooks) if (!isRecord(releasedHook) || typeof releasedHook.hookId !== "string" || releasedHook.hookId.length === 0 || typeof releasedHook.token !== "string" || releasedHook.token.length === 0) malformedApplyEventOutcome("releasedHooks entries must contain hookId and token strings");
|
|
876
|
+
const event = parseSuccessEntity(value, "event", EventSchema);
|
|
877
|
+
const run = parseSuccessEntity(value, "run", WorkflowRunSchema);
|
|
878
|
+
const step = parseSuccessEntity(value, "step", StepSchema);
|
|
879
|
+
const hook = parseSuccessEntity(value, "hook", HookSchema);
|
|
880
|
+
const wait = parseSuccessEntity(value, "wait", WaitSchema);
|
|
881
|
+
const hookToIndex = parseSuccessEntity(value, "hookToIndex", HookSchema);
|
|
882
|
+
if (value.stepCreated !== void 0 && value.stepCreated !== true) malformedApplyEventOutcome("stepCreated must be true when present");
|
|
883
|
+
if (value.runCreated !== void 0) {
|
|
884
|
+
if (!isRecord(value.runCreated) || typeof value.runCreated.workflowName !== "string" || !(value.runCreated.createdAt instanceof Date) || Number.isNaN(value.runCreated.createdAt.getTime())) malformedApplyEventOutcome("runCreated is invalid");
|
|
885
|
+
}
|
|
886
|
+
let events;
|
|
887
|
+
if (value.events !== void 0) {
|
|
888
|
+
if (!Array.isArray(value.events)) malformedApplyEventOutcome("events must be an array");
|
|
889
|
+
events = value.events.map((candidate) => {
|
|
890
|
+
if (!isRecord(candidate)) malformedApplyEventOutcome("events contains an invalid event");
|
|
891
|
+
const result = EventSchema.safeParse(compact(candidate));
|
|
892
|
+
if (!result.success) malformedApplyEventOutcome("events contains an invalid event");
|
|
893
|
+
return result.data;
|
|
894
|
+
});
|
|
895
|
+
if (value.cursor !== null && typeof value.cursor !== "string") malformedApplyEventOutcome("cursor is required when events are present");
|
|
896
|
+
if (typeof value.hasMore !== "boolean") malformedApplyEventOutcome("hasMore is required when events are present");
|
|
897
|
+
}
|
|
898
|
+
if (value.cursor !== void 0 && value.cursor !== null && typeof value.cursor !== "string") malformedApplyEventOutcome("cursor must be a string or null");
|
|
899
|
+
if (value.hasMore !== void 0 && typeof value.hasMore !== "boolean") malformedApplyEventOutcome("hasMore must be a boolean");
|
|
900
|
+
if (value.maxEvents !== void 0 && (typeof value.maxEvents !== "number" || !Number.isSafeInteger(value.maxEvents) || value.maxEvents < 0)) malformedApplyEventOutcome("maxEvents must be a non-negative safe integer");
|
|
901
|
+
if (run !== void 0 && value.indexPublicationExpiresAt === void 0) malformedApplyEventOutcome("indexPublicationExpiresAt is required when run is present");
|
|
902
|
+
if (value.indexPublicationExpiresAt !== void 0 && (typeof value.indexPublicationExpiresAt !== "number" || !Number.isFinite(value.indexPublicationExpiresAt))) malformedApplyEventOutcome("indexPublicationExpiresAt must be a finite number");
|
|
903
|
+
return {
|
|
904
|
+
kind: "success",
|
|
905
|
+
outcome: {
|
|
906
|
+
...value,
|
|
907
|
+
event,
|
|
908
|
+
run,
|
|
909
|
+
step,
|
|
910
|
+
hook,
|
|
911
|
+
wait,
|
|
912
|
+
hookToIndex,
|
|
913
|
+
events,
|
|
914
|
+
releasedHooks: value.releasedHooks.map((releasedHook) => ({
|
|
915
|
+
hookId: releasedHook.hookId,
|
|
916
|
+
token: releasedHook.token
|
|
917
|
+
}))
|
|
918
|
+
}
|
|
919
|
+
};
|
|
920
|
+
}
|
|
741
921
|
/**
|
|
742
922
|
* Filter data based on ResolveData parameter.
|
|
743
923
|
* When resolveData is 'none', strips specified keys to reduce data transfer.
|
|
@@ -770,16 +950,44 @@ function filterHookData(hook, resolveData) {
|
|
|
770
950
|
* outcome objects instead of throwing.
|
|
771
951
|
*/
|
|
772
952
|
function throwOutcomeError(outcome, runId, data) {
|
|
953
|
+
const withCode = (error) => {
|
|
954
|
+
Object.assign(error, { code: outcome.code });
|
|
955
|
+
return error;
|
|
956
|
+
};
|
|
773
957
|
switch (outcome.code) {
|
|
774
|
-
case "RUN_NOT_FOUND": throw new WorkflowRunNotFoundError(runId);
|
|
775
|
-
case "STEP_NOT_FOUND": throw new WorkflowWorldError(outcome.message, {
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
case "
|
|
780
|
-
case "
|
|
781
|
-
|
|
782
|
-
|
|
958
|
+
case "RUN_NOT_FOUND": throw withCode(new WorkflowRunNotFoundError(runId));
|
|
959
|
+
case "STEP_NOT_FOUND": throw new WorkflowWorldError(outcome.message, {
|
|
960
|
+
status: 404,
|
|
961
|
+
code: outcome.code
|
|
962
|
+
});
|
|
963
|
+
case "HOOK_NOT_FOUND": throw withCode(new HookNotFoundError(data.correlationId ?? runId));
|
|
964
|
+
case "WAIT_NOT_FOUND": throw new WorkflowWorldError(outcome.message, {
|
|
965
|
+
status: 404,
|
|
966
|
+
code: outcome.code
|
|
967
|
+
});
|
|
968
|
+
case "ENTITY_CONFLICT": throw withCode(new EntityConflictError(outcome.message));
|
|
969
|
+
case "HOOK_CLAIM_CANCELLED": throw new WorkflowWorldError(outcome.message, {
|
|
970
|
+
status: 503,
|
|
971
|
+
code: outcome.code
|
|
972
|
+
});
|
|
973
|
+
case "RUN_EXPIRED": throw withCode(new RunExpiredError(outcome.message));
|
|
974
|
+
case "TOO_EARLY": throw withCode(new TooEarlyError(outcome.message, { retryAfter: outcome.retryAfterSeconds }));
|
|
975
|
+
case "RUN_NOT_SUPPORTED": throw withCode(new RunNotSupportedError(outcome.runSpecVersion ?? 0, SPEC_VERSION_CURRENT));
|
|
976
|
+
default: {
|
|
977
|
+
const error = new WorkflowWorldError(outcome.message, {
|
|
978
|
+
status: outcome.status,
|
|
979
|
+
code: outcome.code,
|
|
980
|
+
retryAfter: outcome.retryAfter
|
|
981
|
+
});
|
|
982
|
+
for (const key of [
|
|
983
|
+
"retryAfterMs",
|
|
984
|
+
"retryAfterSeconds",
|
|
985
|
+
"runSpecVersion",
|
|
986
|
+
"worldSpecVersion",
|
|
987
|
+
"details"
|
|
988
|
+
]) if (Object.hasOwn(outcome, key)) Object.assign(error, { [key]: outcome[key] });
|
|
989
|
+
throw error;
|
|
990
|
+
}
|
|
783
991
|
}
|
|
784
992
|
}
|
|
785
993
|
const parseRun = (run) => WorkflowRunSchema.parse(compact(run));
|
|
@@ -881,6 +1089,8 @@ function createStorage(config) {
|
|
|
881
1089
|
},
|
|
882
1090
|
events: {
|
|
883
1091
|
async create(runId, data, params) {
|
|
1092
|
+
data = CreateEventSchema.parse(compact(data));
|
|
1093
|
+
if (runId !== null && typeof runId !== "string") throw new WorkflowWorldError("runId must be a string or null", { status: 400 });
|
|
884
1094
|
let effectiveRunId;
|
|
885
1095
|
if (data.eventType === "run_created" && (!runId || runId === "")) effectiveRunId = `wrun_${ulid()}`;
|
|
886
1096
|
else if (!runId) throw new WorkflowWorldError("runId is required for non-run_created events", { status: 400 });
|
|
@@ -903,16 +1113,16 @@ function createStorage(config) {
|
|
|
903
1113
|
};
|
|
904
1114
|
} else tokenHolder = admission.holder;
|
|
905
1115
|
}
|
|
906
|
-
let
|
|
1116
|
+
let parsedOutcome;
|
|
907
1117
|
try {
|
|
908
|
-
|
|
1118
|
+
parsedOutcome = parseApplyEventOutcome(await stub.applyEvent({
|
|
909
1119
|
runId: effectiveRunId,
|
|
910
1120
|
data,
|
|
911
1121
|
params,
|
|
912
1122
|
tokenHolder,
|
|
913
1123
|
hookClaimId: hookAdmission?.reservation.claimId,
|
|
914
1124
|
cleanup
|
|
915
|
-
});
|
|
1125
|
+
}));
|
|
916
1126
|
} catch (error) {
|
|
917
1127
|
if (hookAdmission) {
|
|
918
1128
|
let resolution;
|
|
@@ -929,10 +1139,11 @@ function createStorage(config) {
|
|
|
929
1139
|
}
|
|
930
1140
|
throw error;
|
|
931
1141
|
}
|
|
932
|
-
if (
|
|
1142
|
+
if (parsedOutcome.kind !== "success") {
|
|
933
1143
|
if (hookAdmission) await env.WORKFLOW_INDEX.releaseHookReservation(hookAdmission.token, hookAdmission.owner, hookAdmission.reservation);
|
|
934
|
-
throwOutcomeError(outcome, effectiveRunId, data);
|
|
1144
|
+
throwOutcomeError(parsedOutcome.outcome, effectiveRunId, data);
|
|
935
1145
|
}
|
|
1146
|
+
const outcome = parsedOutcome.outcome;
|
|
936
1147
|
if (outcome.run) {
|
|
937
1148
|
const meta = JSON.stringify({
|
|
938
1149
|
runId: effectiveRunId,
|
|
@@ -1260,6 +1471,12 @@ function createStreamer(config) {
|
|
|
1260
1471
|
* DevKit. Modeled on vinnymac/worlds packages/world-cloudflare/src/index.ts
|
|
1261
1472
|
* (Apache-2.0, see NOTICE).
|
|
1262
1473
|
*/
|
|
1474
|
+
const REQUIRED_ENV_BINDINGS = [
|
|
1475
|
+
"WORKFLOW_DB",
|
|
1476
|
+
"WORKFLOW_INDEX",
|
|
1477
|
+
"WORKFLOW_QUEUE",
|
|
1478
|
+
"WORKFLOW_STREAMS"
|
|
1479
|
+
];
|
|
1263
1480
|
function createCelldWorld(config) {
|
|
1264
1481
|
const resolved = resolveConfig(config);
|
|
1265
1482
|
let env = resolved.env;
|
|
@@ -1278,6 +1495,7 @@ function createCelldWorld(config) {
|
|
|
1278
1495
|
}
|
|
1279
1496
|
}
|
|
1280
1497
|
if (!env) throw new Error("celld environment not configured. Provide config.fleetUrl + config.secret (or CELLD_FLEET_URL / CELLD_WORLD_SECRET), or pass config.env with WORKFLOW_DB, WORKFLOW_INDEX, WORKFLOW_QUEUE, WORKFLOW_STREAMS");
|
|
1498
|
+
for (const binding of REQUIRED_ENV_BINDINGS) if (!env[binding]) throw new Error(`world-celld: config.env is missing required binding ${binding}`);
|
|
1281
1499
|
const storage = createStorage({
|
|
1282
1500
|
env: {
|
|
1283
1501
|
WORKFLOW_DB: env.WORKFLOW_DB,
|