@ewhauser/world-celld 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +50 -11
- package/celld-worker/README.md +5 -3
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +269 -53
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts +10 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +3007 -2498
- package/dist/testing.js.map +1 -1
- package/dist/{util-ya3V_FTA.js → util-Cu09mVzK.js} +86 -11
- package/dist/util-Cu09mVzK.js.map +1 -0
- package/dist/worker.d.ts +3 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +3472 -2890
- package/dist/worker.js.map +1 -1
- package/package.json +2 -1
- package/dist/util-ya3V_FTA.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
import { C as
|
|
2
|
-
import { EventSchema, HookSchema, MessageId, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema, isTerminalWorkflowRunStatus, parseQueueName } from "@workflow/world";
|
|
1
|
+
import { A as MAX_STREAM_READ_CHUNKS, C as queueDelayDeadline, D as MAX_STREAM_ERROR_BYTES, E as MAX_STREAM_CHUNK_BYTES, F as decodeStreamWriteResult, L as encodeStreamWriteBatch, M as STREAM_BATCH_CONTENT_TYPE, N as decodeStreamReadResult, O as MAX_STREAM_LONG_POLL_MS, T as MAX_STREAM_BATCH_BYTES, _ as boundedIntegerOption, a as stringify, d as MAX_QUEUE_DELIVERY_TIMEOUT_MS, f as MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS, i as parse, j as MAX_STREAM_WRITE_CHUNKS, k as MAX_STREAM_READ_BYTES, n as rpcParse, o as createWorkflowIndex, r as rpcStringify, t as compact, u as MAX_FLEET_RPC_TIMEOUT_MS, v as checkedQueueTimestampAdd, w as strictIntegerSetting, z as normalizeStreamError } from "./util-Cu09mVzK.js";
|
|
2
|
+
import { CreateEventSchema, EventSchema, HookSchema, MessageId, QueuePayloadSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, isTerminalWorkflowRunStatus, parseQueueName } from "@workflow/world";
|
|
3
3
|
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
4
|
-
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
5
4
|
import { monotonicFactory } from "ulid";
|
|
6
5
|
//#region src/config.ts
|
|
7
6
|
function resolveConfig(config) {
|
|
8
|
-
const queueShards = config?.queueShards
|
|
9
|
-
|
|
10
|
-
const retentionRaw = config?.runRetentionMs ?? process.env.CELLD_RUN_RETENTION_MS ?? 0;
|
|
11
|
-
const runRetentionMs = typeof retentionRaw === "number" ? retentionRaw : Number.parseInt(retentionRaw, 10);
|
|
12
|
-
if (!Number.isSafeInteger(runRetentionMs) || runRetentionMs < 0) throw new Error("world-celld: runRetentionMs must be a non-negative safe integer");
|
|
7
|
+
const queueShards = boundedIntegerOption("world-celld: queueShards", config?.queueShards, 1, 1);
|
|
8
|
+
const runRetentionMs = config?.runRetentionMs !== void 0 ? boundedIntegerOption("world-celld: runRetentionMs", config.runRetentionMs, 0, 0) : strictIntegerSetting("world-celld: runRetentionMs", process.env.CELLD_RUN_RETENTION_MS, 0, 0);
|
|
13
9
|
const rpcTimeoutMs = config?.rpcTimeoutMs ?? 3e4;
|
|
14
10
|
if (!Number.isSafeInteger(rpcTimeoutMs) || rpcTimeoutMs < 1 || rpcTimeoutMs > 3e5) throw new Error(`world-celld: rpcTimeoutMs must be between 1 and ${MAX_FLEET_RPC_TIMEOUT_MS}`);
|
|
15
11
|
const streamLongPollMs = config?.streamLongPollMs ?? Math.min(2e4, Math.max(1, rpcTimeoutMs - 1e3));
|
|
@@ -61,10 +57,15 @@ const RETRYABLE_STATUSES$1 = /* @__PURE__ */ new Set([
|
|
|
61
57
|
503,
|
|
62
58
|
504
|
|
63
59
|
]);
|
|
64
|
-
const DEFAULT_TIMEOUT_MS
|
|
60
|
+
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
65
61
|
function delayMs$1(ms) {
|
|
66
62
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
67
63
|
}
|
|
64
|
+
function resolveFleetTimeoutMs(transport) {
|
|
65
|
+
const timeoutMs = transport.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
66
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 3e5) throw new Error(`world-celld: fleet RPC timeout must be between 1 and ${MAX_FLEET_RPC_TIMEOUT_MS}`);
|
|
67
|
+
return timeoutMs;
|
|
68
|
+
}
|
|
68
69
|
/**
|
|
69
70
|
* Invoke one DO method through the worker router.
|
|
70
71
|
*
|
|
@@ -81,8 +82,7 @@ async function callFleetRoute(transport, path, args, opts) {
|
|
|
81
82
|
const doFetch = transport.fetchImpl ?? fetch;
|
|
82
83
|
const url = `${transport.fleetUrl.replace(/\/$/, "")}${path}`;
|
|
83
84
|
const attempts = opts?.idempotent ? 3 : 1;
|
|
84
|
-
const timeoutMs = transport
|
|
85
|
-
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 3e5) throw new Error(`world-celld: fleet RPC timeout must be between 1 and ${MAX_FLEET_RPC_TIMEOUT_MS}`);
|
|
85
|
+
const timeoutMs = resolveFleetTimeoutMs(transport);
|
|
86
86
|
let lastError;
|
|
87
87
|
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
88
88
|
if (attempt > 1) await delayMs$1(100 * attempt + Math.random() * 200);
|
|
@@ -131,15 +131,27 @@ const RETRYABLE_STATUSES = /* @__PURE__ */ new Set([
|
|
|
131
131
|
504
|
|
132
132
|
]);
|
|
133
133
|
const READ_ATTEMPTS = 3;
|
|
134
|
-
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
135
134
|
const MAX_WRITE_RESPONSE_BYTES = 64;
|
|
136
135
|
const MAX_READ_RESPONSE_BYTES = MAX_STREAM_READ_BYTES + 65536;
|
|
137
136
|
const MAX_ERROR_RESPONSE_BYTES = 65536;
|
|
138
|
-
function delayMs(ms) {
|
|
139
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
137
|
+
function delayMs(ms, signal) {
|
|
138
|
+
if (!signal) return new Promise((resolve) => setTimeout(resolve, ms));
|
|
139
|
+
if (signal.aborted) return Promise.reject(aborted(signal));
|
|
140
|
+
return new Promise((resolve, reject) => {
|
|
141
|
+
const timer = setTimeout(() => {
|
|
142
|
+
signal.removeEventListener("abort", onAbort);
|
|
143
|
+
resolve();
|
|
144
|
+
}, ms);
|
|
145
|
+
const onAbort = () => {
|
|
146
|
+
clearTimeout(timer);
|
|
147
|
+
signal.removeEventListener("abort", onAbort);
|
|
148
|
+
reject(aborted(signal));
|
|
149
|
+
};
|
|
150
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
151
|
+
});
|
|
140
152
|
}
|
|
141
153
|
function requestSignal(timeoutMs, signal) {
|
|
142
|
-
const timeout = AbortSignal.timeout(
|
|
154
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
143
155
|
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
144
156
|
}
|
|
145
157
|
function aborted(signal) {
|
|
@@ -190,6 +202,7 @@ function streamUrl(transport, name) {
|
|
|
190
202
|
return `${transport.fleetUrl.replace(/\/$/, "")}/v1/streams/${encodeURIComponent(name)}/chunks`;
|
|
191
203
|
}
|
|
192
204
|
async function writeStreamChunks(transport, name, runId, chunks) {
|
|
205
|
+
const timeoutMs = resolveFleetTimeoutMs(transport);
|
|
193
206
|
const body = encodeStreamWriteBatch(chunks);
|
|
194
207
|
if (body.byteLength > 8389632) throw new Error("world-celld: encoded stream batch exceeds configured limit");
|
|
195
208
|
const url = new URL(streamUrl(transport, name));
|
|
@@ -204,7 +217,7 @@ async function writeStreamChunks(transport, name, runId, chunks) {
|
|
|
204
217
|
authorization: `Bearer ${transport.secret}`
|
|
205
218
|
},
|
|
206
219
|
body,
|
|
207
|
-
signal: requestSignal(
|
|
220
|
+
signal: requestSignal(timeoutMs)
|
|
208
221
|
});
|
|
209
222
|
} catch (error) {
|
|
210
223
|
throw new FleetTransportError(`world-celld: fleet unreachable at ${url.href}`, error);
|
|
@@ -218,6 +231,7 @@ async function writeStreamChunks(transport, name, runId, chunks) {
|
|
|
218
231
|
}
|
|
219
232
|
}
|
|
220
233
|
async function readStreamChunks(transport, name, request, signal) {
|
|
234
|
+
const timeoutMs = resolveFleetTimeoutMs(transport);
|
|
221
235
|
const url = new URL(streamUrl(transport, name));
|
|
222
236
|
url.searchParams.set("runId", request.runId);
|
|
223
237
|
url.searchParams.set("startIndex", String(request.startIndex));
|
|
@@ -228,27 +242,38 @@ async function readStreamChunks(transport, name, request, signal) {
|
|
|
228
242
|
let lastError;
|
|
229
243
|
for (let attempt = 1; attempt <= READ_ATTEMPTS; attempt++) {
|
|
230
244
|
if (signal?.aborted) throw aborted(signal);
|
|
231
|
-
if (attempt > 1) await delayMs(100 * attempt + Math.random() * 200);
|
|
245
|
+
if (attempt > 1) await delayMs(100 * attempt + Math.random() * 200, signal);
|
|
232
246
|
let response;
|
|
233
247
|
try {
|
|
234
248
|
response = await doFetch(url, {
|
|
235
249
|
method: "GET",
|
|
236
250
|
headers: { authorization: `Bearer ${transport.secret}` },
|
|
237
|
-
signal: requestSignal(
|
|
251
|
+
signal: requestSignal(timeoutMs, signal)
|
|
238
252
|
});
|
|
239
253
|
} catch (error) {
|
|
240
254
|
if (signal?.aborted) throw aborted(signal);
|
|
241
255
|
lastError = new FleetTransportError(`world-celld: fleet unreachable at ${url.href}`, error);
|
|
242
256
|
continue;
|
|
243
257
|
}
|
|
258
|
+
if (signal?.aborted) {
|
|
259
|
+
await response.body?.cancel(aborted(signal)).catch(() => void 0);
|
|
260
|
+
throw aborted(signal);
|
|
261
|
+
}
|
|
244
262
|
if (response.ok) try {
|
|
245
263
|
return decodeStreamReadResult(await readBoundedResponse(response, MAX_READ_RESPONSE_BYTES));
|
|
246
264
|
} catch (error) {
|
|
247
|
-
|
|
265
|
+
if (signal?.aborted) throw aborted(signal);
|
|
266
|
+
lastError = error instanceof FleetTransportError ? error : new FleetTransportError(`world-celld: malformed stream response from ${url.href}`, error);
|
|
248
267
|
if (attempt < READ_ATTEMPTS) continue;
|
|
249
268
|
throw lastError;
|
|
250
269
|
}
|
|
251
|
-
|
|
270
|
+
let error;
|
|
271
|
+
try {
|
|
272
|
+
error = await errorFromResponse(response);
|
|
273
|
+
} catch (cause) {
|
|
274
|
+
if (signal?.aborted) throw aborted(signal);
|
|
275
|
+
error = cause instanceof FleetTransportError ? cause : new FleetTransportError(`world-celld: malformed stream error response from ${url.href}`, cause);
|
|
276
|
+
}
|
|
252
277
|
if (RETRYABLE_STATUSES.has(response.status) && attempt < READ_ATTEMPTS) {
|
|
253
278
|
lastError = error;
|
|
254
279
|
continue;
|
|
@@ -279,7 +304,8 @@ const RUNS = {
|
|
|
279
304
|
"applyEvent",
|
|
280
305
|
"scheduleCleanup",
|
|
281
306
|
"cleanupNow",
|
|
282
|
-
"rearmCleanup"
|
|
307
|
+
"rearmCleanup",
|
|
308
|
+
"resolveHookTokenClaim"
|
|
283
309
|
])
|
|
284
310
|
};
|
|
285
311
|
const STREAMS = {
|
|
@@ -466,6 +492,19 @@ const debug = createDebugLogger("celld-world");
|
|
|
466
492
|
* HTTP statuses. QueueDO deliveries and the in-process test pump hit the
|
|
467
493
|
* exact same handler path.
|
|
468
494
|
*/
|
|
495
|
+
const MAX_TIMER_DELAY_MS = 2147483647;
|
|
496
|
+
const MAX_TEST_PUMP_BACKOFF_MS = 6e4;
|
|
497
|
+
function delay(milliseconds) {
|
|
498
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
499
|
+
}
|
|
500
|
+
async function delayFor(milliseconds) {
|
|
501
|
+
let remaining = milliseconds;
|
|
502
|
+
while (remaining > MAX_TIMER_DELAY_MS) {
|
|
503
|
+
await delay(MAX_TIMER_DELAY_MS);
|
|
504
|
+
remaining -= MAX_TIMER_DELAY_MS;
|
|
505
|
+
}
|
|
506
|
+
await delay(remaining);
|
|
507
|
+
}
|
|
469
508
|
/**
|
|
470
509
|
* HTTP status codes that indicate permanent (non-retryable) failures.
|
|
471
510
|
* Deliveries with these statuses are dropped immediately instead of retried.
|
|
@@ -504,6 +543,7 @@ function resolveBaseUrl(config) {
|
|
|
504
543
|
* land on the same cell so dedup stays strictly consistent.
|
|
505
544
|
*/
|
|
506
545
|
function shardFor(key, shards) {
|
|
546
|
+
if (!Number.isSafeInteger(shards) || shards < 1) throw new Error("world-celld queueShards must be a positive safe integer");
|
|
507
547
|
if (shards <= 1) return 0;
|
|
508
548
|
let hash = 2166136261;
|
|
509
549
|
for (let i = 0; i < key.length; i++) {
|
|
@@ -525,9 +565,9 @@ function shardFor(key, shards) {
|
|
|
525
565
|
* matching QueueDO's production semantics.
|
|
526
566
|
*/
|
|
527
567
|
function createTestPump(config) {
|
|
528
|
-
const httpTimeoutMs = config.httpTimeoutMs
|
|
529
|
-
const maxAttempts = config.maxAttempts
|
|
530
|
-
const baseBackoffMs = config.backoffDelayMs
|
|
568
|
+
const httpTimeoutMs = boundedIntegerOption("world-celld test pump httpTimeoutMs", config.httpTimeoutMs, 3e5, 1, MAX_QUEUE_DELIVERY_TIMEOUT_MS);
|
|
569
|
+
const maxAttempts = boundedIntegerOption("world-celld test pump maxAttempts", config.maxAttempts, 5, 1);
|
|
570
|
+
const baseBackoffMs = boundedIntegerOption("world-celld test pump backoffDelayMs", config.backoffDelayMs, 1e3, 0, MAX_TEST_PUMP_BACKOFF_MS);
|
|
531
571
|
const queues = { flow: [] };
|
|
532
572
|
const wakers = { flow: [] };
|
|
533
573
|
/** Inflight messageIds by idempotencyKey (world-local queue.js semantics). */
|
|
@@ -576,8 +616,10 @@ function createTestPump(config) {
|
|
|
576
616
|
parsed = null;
|
|
577
617
|
}
|
|
578
618
|
const timeoutSeconds = parsed?.timeoutSeconds;
|
|
579
|
-
|
|
580
|
-
|
|
619
|
+
const now = Date.now();
|
|
620
|
+
const redeliveryAt = queueDelayDeadline(now, timeoutSeconds, 1, MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS);
|
|
621
|
+
if (redeliveryAt !== null) {
|
|
622
|
+
delayFor(redeliveryAt - now).then(() => enqueue(pathname, envelope));
|
|
581
623
|
return;
|
|
582
624
|
}
|
|
583
625
|
}
|
|
@@ -591,8 +633,13 @@ function createTestPump(config) {
|
|
|
591
633
|
...envelope,
|
|
592
634
|
attempt: envelope.attempt + 1
|
|
593
635
|
};
|
|
594
|
-
const backoff = baseBackoffMs * 2 ** (next.attempt - 1);
|
|
595
|
-
|
|
636
|
+
const backoff = Math.min(MAX_TEST_PUMP_BACKOFF_MS, baseBackoffMs * 2 ** Math.min(next.attempt - 1, 30));
|
|
637
|
+
if (checkedQueueTimestampAdd(Date.now(), backoff, 330001) === null) {
|
|
638
|
+
release(envelope);
|
|
639
|
+
console.error(`[world-celld test pump] dropping ${envelope.messageId}: retry deadline exceeds the queue timestamp horizon`);
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
delayFor(backoff).then(() => enqueue(pathname, next));
|
|
596
643
|
} else {
|
|
597
644
|
release(envelope);
|
|
598
645
|
console.error(`[world-celld test pump] dropping ${envelope.messageId} after ${envelope.attempt} attempts: HTTP ${response.status}: ${text}`);
|
|
@@ -617,8 +664,9 @@ function createTestPump(config) {
|
|
|
617
664
|
return idempotencyKey ? inflightMessages.get(idempotencyKey) : void 0;
|
|
618
665
|
},
|
|
619
666
|
push(pathname, envelope, delaySeconds) {
|
|
667
|
+
if (queueDelayDeadline(Date.now(), delaySeconds ?? 0, 0, 330001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
|
|
620
668
|
if (envelope.idempotencyKey) inflightMessages.set(envelope.idempotencyKey, MessageId.parse(envelope.messageId));
|
|
621
|
-
if (delaySeconds && delaySeconds > 0)
|
|
669
|
+
if (delaySeconds && delaySeconds > 0) delayFor(delaySeconds * 1e3).then(() => enqueue(pathname, envelope));
|
|
622
670
|
else enqueue(pathname, envelope);
|
|
623
671
|
},
|
|
624
672
|
async start() {
|
|
@@ -634,7 +682,8 @@ function createTestPump(config) {
|
|
|
634
682
|
}
|
|
635
683
|
function createQueue(config) {
|
|
636
684
|
const { env, deploymentId } = config;
|
|
637
|
-
|
|
685
|
+
if (!env?.WORKFLOW_QUEUE) throw new Error("world-celld queue missing WORKFLOW_QUEUE binding");
|
|
686
|
+
const queueShards = boundedIntegerOption("world-celld queueShards", config.queueShards, 1, 1);
|
|
638
687
|
const generateMessageId = monotonicFactory();
|
|
639
688
|
const testPump = createTestPump(config);
|
|
640
689
|
const getQueueStub = (shard) => {
|
|
@@ -644,7 +693,10 @@ function createQueue(config) {
|
|
|
644
693
|
return {
|
|
645
694
|
async queue(queueName, message, opts) {
|
|
646
695
|
parseQueueName(queueName);
|
|
647
|
-
const
|
|
696
|
+
const parsedMessage = QueuePayloadSchema.safeParse(message);
|
|
697
|
+
if (!parsedMessage.success) throw new WorkflowWorldError("world-celld queue payload is invalid", { status: 422 });
|
|
698
|
+
if (queueDelayDeadline(Date.now(), opts?.delaySeconds ?? 0, 0, 330001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
|
|
699
|
+
const runId = "runId" in parsedMessage.data && typeof parsedMessage.data.runId === "string" ? parsedMessage.data.runId : void 0;
|
|
648
700
|
if (isTestMode()) {
|
|
649
701
|
const existing = testPump.inflight(opts?.idempotencyKey);
|
|
650
702
|
if (existing) return { messageId: existing };
|
|
@@ -653,7 +705,7 @@ function createQueue(config) {
|
|
|
653
705
|
messageId,
|
|
654
706
|
queueName,
|
|
655
707
|
attempt: 1,
|
|
656
|
-
message,
|
|
708
|
+
message: parsedMessage.data,
|
|
657
709
|
idempotencyKey: opts?.idempotencyKey
|
|
658
710
|
}, opts?.delaySeconds);
|
|
659
711
|
return { messageId };
|
|
@@ -664,7 +716,7 @@ function createQueue(config) {
|
|
|
664
716
|
messageId,
|
|
665
717
|
queueName,
|
|
666
718
|
pathname: QUEUE_PATHNAME,
|
|
667
|
-
body: stringify(
|
|
719
|
+
body: stringify(parsedMessage.data),
|
|
668
720
|
runId,
|
|
669
721
|
idempotencyKey: opts?.idempotencyKey,
|
|
670
722
|
delaySeconds: opts?.delaySeconds,
|
|
@@ -681,22 +733,40 @@ function createQueue(config) {
|
|
|
681
733
|
},
|
|
682
734
|
createQueueHandler(queueNamePrefix, handler) {
|
|
683
735
|
return async (req) => {
|
|
736
|
+
if (req.method !== "POST") return Response.json({ error: "Expected POST" }, { status: 405 });
|
|
737
|
+
if (req.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json") return Response.json({ error: "Expected application/json" }, { status: 415 });
|
|
684
738
|
const reqQueueName = req.headers.get("x-vqs-queue-name");
|
|
685
739
|
const reqMessageId = req.headers.get("x-vqs-message-id");
|
|
686
740
|
const attemptStr = req.headers.get("x-vqs-message-attempt");
|
|
687
741
|
if (!reqQueueName || !reqMessageId || !attemptStr || !req.body) return Response.json({ error: "Missing required headers or body" }, { status: 400 });
|
|
688
742
|
if (!reqQueueName.startsWith(queueNamePrefix)) return Response.json({ error: "Unhandled queue" }, { status: 400 });
|
|
689
|
-
|
|
743
|
+
let attempt;
|
|
690
744
|
try {
|
|
691
|
-
|
|
745
|
+
attempt = strictIntegerSetting("x-vqs-message-attempt", attemptStr, 1, 1);
|
|
746
|
+
} catch {
|
|
747
|
+
return Response.json({ error: "Invalid x-vqs-message-attempt header" }, { status: 400 });
|
|
748
|
+
}
|
|
749
|
+
try {
|
|
750
|
+
let body;
|
|
751
|
+
try {
|
|
752
|
+
body = parse(await req.text());
|
|
753
|
+
} catch {
|
|
754
|
+
return Response.json({ error: "Malformed tagged JSON body" }, { status: 422 });
|
|
755
|
+
}
|
|
756
|
+
const parsedBody = QueuePayloadSchema.safeParse(body);
|
|
757
|
+
if (!parsedBody.success) return Response.json({ error: "Invalid queue payload" }, { status: 422 });
|
|
758
|
+
const result = await handler(parsedBody.data, {
|
|
692
759
|
attempt,
|
|
693
760
|
queueName: reqQueueName,
|
|
694
761
|
messageId: reqMessageId
|
|
695
762
|
});
|
|
696
|
-
if (result && typeof result.timeoutSeconds === "number")
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
763
|
+
if (result && typeof result.timeoutSeconds === "number") {
|
|
764
|
+
if (queueDelayDeadline(Date.now(), result.timeoutSeconds, 1, 330001) === null) throw new Error("Queue handler timeoutSeconds is out of range");
|
|
765
|
+
return Response.json({ timeoutSeconds: result.timeoutSeconds }, {
|
|
766
|
+
status: 503,
|
|
767
|
+
headers: { "Retry-After": String(result.timeoutSeconds) }
|
|
768
|
+
});
|
|
769
|
+
}
|
|
700
770
|
return new Response(null, { status: 204 });
|
|
701
771
|
} catch (error) {
|
|
702
772
|
if (isPermanentError(error)) {
|
|
@@ -740,6 +810,114 @@ function hookOwner(hook) {
|
|
|
740
810
|
hookId: hook.hookId
|
|
741
811
|
};
|
|
742
812
|
}
|
|
813
|
+
const APPLY_EVENT_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
814
|
+
"RUN_NOT_FOUND",
|
|
815
|
+
"STEP_NOT_FOUND",
|
|
816
|
+
"HOOK_NOT_FOUND",
|
|
817
|
+
"WAIT_NOT_FOUND",
|
|
818
|
+
"ENTITY_CONFLICT",
|
|
819
|
+
"HOOK_CLAIM_CANCELLED",
|
|
820
|
+
"RUN_EXPIRED",
|
|
821
|
+
"TOO_EARLY",
|
|
822
|
+
"RUN_NOT_SUPPORTED"
|
|
823
|
+
]);
|
|
824
|
+
function malformedApplyEventOutcome(reason) {
|
|
825
|
+
throw new FleetTransportError(`world-celld: malformed applyEvent outcome: ${reason}`);
|
|
826
|
+
}
|
|
827
|
+
function isRecord(value) {
|
|
828
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
829
|
+
}
|
|
830
|
+
function validateOptionalNumber(outcome, key, integer) {
|
|
831
|
+
const value = outcome[key];
|
|
832
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isFinite(value) || integer && !Number.isSafeInteger(value))) malformedApplyEventOutcome(`${key} must be ${integer ? "a safe integer" : "a finite number"}`);
|
|
833
|
+
}
|
|
834
|
+
function validateOptionalRetryAfterMs(outcome) {
|
|
835
|
+
const value = outcome.retryAfterMs;
|
|
836
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)) malformedApplyEventOutcome("retryAfterMs must be a non-negative safe integer");
|
|
837
|
+
}
|
|
838
|
+
function parseSuccessEntity(outcome, key, schema) {
|
|
839
|
+
const value = outcome[key];
|
|
840
|
+
if (value === void 0) return void 0;
|
|
841
|
+
if (!isRecord(value)) malformedApplyEventOutcome(`${key} is invalid`);
|
|
842
|
+
const result = schema.safeParse(compact(value));
|
|
843
|
+
if (!result.success) malformedApplyEventOutcome(`${key} is invalid`);
|
|
844
|
+
return result.data;
|
|
845
|
+
}
|
|
846
|
+
/**
|
|
847
|
+
* Validate the untrusted value decoded from the remote applyEvent RPC.
|
|
848
|
+
* Unknown failure codes are intentionally valid and classified separately so
|
|
849
|
+
* an older client can reconstruct their wire metadata without treating them
|
|
850
|
+
* as success. All fields consumed by the success path are checked before any
|
|
851
|
+
* derivative index operation can run.
|
|
852
|
+
*/
|
|
853
|
+
function parseApplyEventOutcome(value) {
|
|
854
|
+
if (!isRecord(value)) malformedApplyEventOutcome("response must be an object");
|
|
855
|
+
if (value.ok === false) {
|
|
856
|
+
if (typeof value.code !== "string" || value.code.length === 0) malformedApplyEventOutcome("failure code must be a non-empty string");
|
|
857
|
+
if (typeof value.message !== "string") malformedApplyEventOutcome("failure message must be a string");
|
|
858
|
+
validateOptionalNumber(value, "status", true);
|
|
859
|
+
validateOptionalNumber(value, "retryAfter", false);
|
|
860
|
+
validateOptionalRetryAfterMs(value);
|
|
861
|
+
validateOptionalNumber(value, "retryAfterSeconds", false);
|
|
862
|
+
validateOptionalNumber(value, "runSpecVersion", true);
|
|
863
|
+
validateOptionalNumber(value, "worldSpecVersion", true);
|
|
864
|
+
const outcome = value;
|
|
865
|
+
return APPLY_EVENT_ERROR_CODES.has(outcome.code) ? {
|
|
866
|
+
kind: "known-failure",
|
|
867
|
+
outcome
|
|
868
|
+
} : {
|
|
869
|
+
kind: "unknown-failure",
|
|
870
|
+
outcome
|
|
871
|
+
};
|
|
872
|
+
}
|
|
873
|
+
if (value.ok !== true) malformedApplyEventOutcome("ok must be true or false");
|
|
874
|
+
if (!Array.isArray(value.releasedHooks)) malformedApplyEventOutcome("releasedHooks must be an array");
|
|
875
|
+
for (const releasedHook of value.releasedHooks) if (!isRecord(releasedHook) || typeof releasedHook.hookId !== "string" || releasedHook.hookId.length === 0 || typeof releasedHook.token !== "string" || releasedHook.token.length === 0) malformedApplyEventOutcome("releasedHooks entries must contain hookId and token strings");
|
|
876
|
+
const event = parseSuccessEntity(value, "event", EventSchema);
|
|
877
|
+
const run = parseSuccessEntity(value, "run", WorkflowRunSchema);
|
|
878
|
+
const step = parseSuccessEntity(value, "step", StepSchema);
|
|
879
|
+
const hook = parseSuccessEntity(value, "hook", HookSchema);
|
|
880
|
+
const wait = parseSuccessEntity(value, "wait", WaitSchema);
|
|
881
|
+
const hookToIndex = parseSuccessEntity(value, "hookToIndex", HookSchema);
|
|
882
|
+
if (value.stepCreated !== void 0 && value.stepCreated !== true) malformedApplyEventOutcome("stepCreated must be true when present");
|
|
883
|
+
if (value.runCreated !== void 0) {
|
|
884
|
+
if (!isRecord(value.runCreated) || typeof value.runCreated.workflowName !== "string" || !(value.runCreated.createdAt instanceof Date) || Number.isNaN(value.runCreated.createdAt.getTime())) malformedApplyEventOutcome("runCreated is invalid");
|
|
885
|
+
}
|
|
886
|
+
let events;
|
|
887
|
+
if (value.events !== void 0) {
|
|
888
|
+
if (!Array.isArray(value.events)) malformedApplyEventOutcome("events must be an array");
|
|
889
|
+
events = value.events.map((candidate) => {
|
|
890
|
+
if (!isRecord(candidate)) malformedApplyEventOutcome("events contains an invalid event");
|
|
891
|
+
const result = EventSchema.safeParse(compact(candidate));
|
|
892
|
+
if (!result.success) malformedApplyEventOutcome("events contains an invalid event");
|
|
893
|
+
return result.data;
|
|
894
|
+
});
|
|
895
|
+
if (value.cursor !== null && typeof value.cursor !== "string") malformedApplyEventOutcome("cursor is required when events are present");
|
|
896
|
+
if (typeof value.hasMore !== "boolean") malformedApplyEventOutcome("hasMore is required when events are present");
|
|
897
|
+
}
|
|
898
|
+
if (value.cursor !== void 0 && value.cursor !== null && typeof value.cursor !== "string") malformedApplyEventOutcome("cursor must be a string or null");
|
|
899
|
+
if (value.hasMore !== void 0 && typeof value.hasMore !== "boolean") malformedApplyEventOutcome("hasMore must be a boolean");
|
|
900
|
+
if (value.maxEvents !== void 0 && (typeof value.maxEvents !== "number" || !Number.isSafeInteger(value.maxEvents) || value.maxEvents < 0)) malformedApplyEventOutcome("maxEvents must be a non-negative safe integer");
|
|
901
|
+
if (run !== void 0 && value.indexPublicationExpiresAt === void 0) malformedApplyEventOutcome("indexPublicationExpiresAt is required when run is present");
|
|
902
|
+
if (value.indexPublicationExpiresAt !== void 0 && (typeof value.indexPublicationExpiresAt !== "number" || !Number.isFinite(value.indexPublicationExpiresAt))) malformedApplyEventOutcome("indexPublicationExpiresAt must be a finite number");
|
|
903
|
+
return {
|
|
904
|
+
kind: "success",
|
|
905
|
+
outcome: {
|
|
906
|
+
...value,
|
|
907
|
+
event,
|
|
908
|
+
run,
|
|
909
|
+
step,
|
|
910
|
+
hook,
|
|
911
|
+
wait,
|
|
912
|
+
hookToIndex,
|
|
913
|
+
events,
|
|
914
|
+
releasedHooks: value.releasedHooks.map((releasedHook) => ({
|
|
915
|
+
hookId: releasedHook.hookId,
|
|
916
|
+
token: releasedHook.token
|
|
917
|
+
}))
|
|
918
|
+
}
|
|
919
|
+
};
|
|
920
|
+
}
|
|
743
921
|
/**
|
|
744
922
|
* Filter data based on ResolveData parameter.
|
|
745
923
|
* When resolveData is 'none', strips specified keys to reduce data transfer.
|
|
@@ -772,16 +950,44 @@ function filterHookData(hook, resolveData) {
|
|
|
772
950
|
* outcome objects instead of throwing.
|
|
773
951
|
*/
|
|
774
952
|
function throwOutcomeError(outcome, runId, data) {
|
|
953
|
+
const withCode = (error) => {
|
|
954
|
+
Object.assign(error, { code: outcome.code });
|
|
955
|
+
return error;
|
|
956
|
+
};
|
|
775
957
|
switch (outcome.code) {
|
|
776
|
-
case "RUN_NOT_FOUND": throw new WorkflowRunNotFoundError(runId);
|
|
777
|
-
case "STEP_NOT_FOUND": throw new WorkflowWorldError(outcome.message, {
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
case "
|
|
782
|
-
case "
|
|
783
|
-
|
|
784
|
-
|
|
958
|
+
case "RUN_NOT_FOUND": throw withCode(new WorkflowRunNotFoundError(runId));
|
|
959
|
+
case "STEP_NOT_FOUND": throw new WorkflowWorldError(outcome.message, {
|
|
960
|
+
status: 404,
|
|
961
|
+
code: outcome.code
|
|
962
|
+
});
|
|
963
|
+
case "HOOK_NOT_FOUND": throw withCode(new HookNotFoundError(data.correlationId ?? runId));
|
|
964
|
+
case "WAIT_NOT_FOUND": throw new WorkflowWorldError(outcome.message, {
|
|
965
|
+
status: 404,
|
|
966
|
+
code: outcome.code
|
|
967
|
+
});
|
|
968
|
+
case "ENTITY_CONFLICT": throw withCode(new EntityConflictError(outcome.message));
|
|
969
|
+
case "HOOK_CLAIM_CANCELLED": throw new WorkflowWorldError(outcome.message, {
|
|
970
|
+
status: 503,
|
|
971
|
+
code: outcome.code
|
|
972
|
+
});
|
|
973
|
+
case "RUN_EXPIRED": throw withCode(new RunExpiredError(outcome.message));
|
|
974
|
+
case "TOO_EARLY": throw withCode(new TooEarlyError(outcome.message, { retryAfter: outcome.retryAfterSeconds }));
|
|
975
|
+
case "RUN_NOT_SUPPORTED": throw withCode(new RunNotSupportedError(outcome.runSpecVersion ?? 0, SPEC_VERSION_CURRENT));
|
|
976
|
+
default: {
|
|
977
|
+
const error = new WorkflowWorldError(outcome.message, {
|
|
978
|
+
status: outcome.status,
|
|
979
|
+
code: outcome.code,
|
|
980
|
+
retryAfter: outcome.retryAfter
|
|
981
|
+
});
|
|
982
|
+
for (const key of [
|
|
983
|
+
"retryAfterMs",
|
|
984
|
+
"retryAfterSeconds",
|
|
985
|
+
"runSpecVersion",
|
|
986
|
+
"worldSpecVersion",
|
|
987
|
+
"details"
|
|
988
|
+
]) if (Object.hasOwn(outcome, key)) Object.assign(error, { [key]: outcome[key] });
|
|
989
|
+
throw error;
|
|
990
|
+
}
|
|
785
991
|
}
|
|
786
992
|
}
|
|
787
993
|
const parseRun = (run) => WorkflowRunSchema.parse(compact(run));
|
|
@@ -883,6 +1089,8 @@ function createStorage(config) {
|
|
|
883
1089
|
},
|
|
884
1090
|
events: {
|
|
885
1091
|
async create(runId, data, params) {
|
|
1092
|
+
data = CreateEventSchema.parse(compact(data));
|
|
1093
|
+
if (runId !== null && typeof runId !== "string") throw new WorkflowWorldError("runId must be a string or null", { status: 400 });
|
|
886
1094
|
let effectiveRunId;
|
|
887
1095
|
if (data.eventType === "run_created" && (!runId || runId === "")) effectiveRunId = `wrun_${ulid()}`;
|
|
888
1096
|
else if (!runId) throw new WorkflowWorldError("runId is required for non-run_created events", { status: 400 });
|
|
@@ -905,16 +1113,16 @@ function createStorage(config) {
|
|
|
905
1113
|
};
|
|
906
1114
|
} else tokenHolder = admission.holder;
|
|
907
1115
|
}
|
|
908
|
-
let
|
|
1116
|
+
let parsedOutcome;
|
|
909
1117
|
try {
|
|
910
|
-
|
|
1118
|
+
parsedOutcome = parseApplyEventOutcome(await stub.applyEvent({
|
|
911
1119
|
runId: effectiveRunId,
|
|
912
1120
|
data,
|
|
913
1121
|
params,
|
|
914
1122
|
tokenHolder,
|
|
915
1123
|
hookClaimId: hookAdmission?.reservation.claimId,
|
|
916
1124
|
cleanup
|
|
917
|
-
});
|
|
1125
|
+
}));
|
|
918
1126
|
} catch (error) {
|
|
919
1127
|
if (hookAdmission) {
|
|
920
1128
|
let resolution;
|
|
@@ -931,10 +1139,11 @@ function createStorage(config) {
|
|
|
931
1139
|
}
|
|
932
1140
|
throw error;
|
|
933
1141
|
}
|
|
934
|
-
if (
|
|
1142
|
+
if (parsedOutcome.kind !== "success") {
|
|
935
1143
|
if (hookAdmission) await env.WORKFLOW_INDEX.releaseHookReservation(hookAdmission.token, hookAdmission.owner, hookAdmission.reservation);
|
|
936
|
-
throwOutcomeError(outcome, effectiveRunId, data);
|
|
1144
|
+
throwOutcomeError(parsedOutcome.outcome, effectiveRunId, data);
|
|
937
1145
|
}
|
|
1146
|
+
const outcome = parsedOutcome.outcome;
|
|
938
1147
|
if (outcome.run) {
|
|
939
1148
|
const meta = JSON.stringify({
|
|
940
1149
|
runId: effectiveRunId,
|
|
@@ -1262,6 +1471,12 @@ function createStreamer(config) {
|
|
|
1262
1471
|
* DevKit. Modeled on vinnymac/worlds packages/world-cloudflare/src/index.ts
|
|
1263
1472
|
* (Apache-2.0, see NOTICE).
|
|
1264
1473
|
*/
|
|
1474
|
+
const REQUIRED_ENV_BINDINGS = [
|
|
1475
|
+
"WORKFLOW_DB",
|
|
1476
|
+
"WORKFLOW_INDEX",
|
|
1477
|
+
"WORKFLOW_QUEUE",
|
|
1478
|
+
"WORKFLOW_STREAMS"
|
|
1479
|
+
];
|
|
1265
1480
|
function createCelldWorld(config) {
|
|
1266
1481
|
const resolved = resolveConfig(config);
|
|
1267
1482
|
let env = resolved.env;
|
|
@@ -1280,6 +1495,7 @@ function createCelldWorld(config) {
|
|
|
1280
1495
|
}
|
|
1281
1496
|
}
|
|
1282
1497
|
if (!env) throw new Error("celld environment not configured. Provide config.fleetUrl + config.secret (or CELLD_FLEET_URL / CELLD_WORLD_SECRET), or pass config.env with WORKFLOW_DB, WORKFLOW_INDEX, WORKFLOW_QUEUE, WORKFLOW_STREAMS");
|
|
1498
|
+
for (const binding of REQUIRED_ENV_BINDINGS) if (!env[binding]) throw new Error(`world-celld: config.env is missing required binding ${binding}`);
|
|
1283
1499
|
const storage = createStorage({
|
|
1284
1500
|
env: {
|
|
1285
1501
|
WORKFLOW_DB: env.WORKFLOW_DB,
|