@ewhauser/world-celld 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +61 -23
- package/dist/index.d.ts +126 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +206 -135
- package/dist/index.js.map +1 -1
- package/dist/{util-BvWlkqgV.js → retention-bLY1vcOu.js} +44 -2
- package/dist/retention-bLY1vcOu.js.map +1 -0
- package/dist/testing.d.ts +2 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +802 -174
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +133 -62
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +829 -170
- package/dist/worker.js.map +1 -1
- package/package.json +7 -7
- package/dist/util-BvWlkqgV.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { c as globalRunIndexKey, d as workflowRunIndexKey, f as compact, g as stringify, h as parse, m as rpcStringify, o as correlationIndexKey, p as rpcParse } from "./retention-bLY1vcOu.js";
|
|
2
2
|
import { EventSchema, HookSchema, MessageId, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema, parseQueueName } from "@workflow/world";
|
|
3
3
|
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
4
4
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
5
5
|
import { monotonicFactory } from "ulid";
|
|
6
6
|
//#region src/config.ts
|
|
7
7
|
function resolveConfig(config) {
|
|
8
|
+
const retentionRaw = config?.runRetentionMs ?? process.env.CELLD_RUN_RETENTION_MS ?? 0;
|
|
9
|
+
const runRetentionMs = typeof retentionRaw === "number" ? retentionRaw : Number.parseInt(retentionRaw, 10);
|
|
10
|
+
if (!Number.isSafeInteger(runRetentionMs) || runRetentionMs < 0) throw new Error("world-celld: runRetentionMs must be a non-negative safe integer");
|
|
8
11
|
return {
|
|
9
12
|
fleetUrl: config?.fleetUrl ?? process.env.CELLD_FLEET_URL,
|
|
10
13
|
secret: config?.secret ?? process.env.CELLD_WORLD_SECRET,
|
|
@@ -12,6 +15,7 @@ function resolveConfig(config) {
|
|
|
12
15
|
deploymentId: config?.deploymentId ?? process.env.CELLD_DEPLOYMENT_ID ?? "celld-default",
|
|
13
16
|
baseUrl: config?.baseUrl,
|
|
14
17
|
queueShards: config?.queueShards ?? 1,
|
|
18
|
+
runRetentionMs,
|
|
15
19
|
readPollMs: config?.readPollMs ?? 250,
|
|
16
20
|
rpcTimeoutMs: config?.rpcTimeoutMs ?? 3e4
|
|
17
21
|
};
|
|
@@ -113,9 +117,18 @@ const RUNS = {
|
|
|
113
117
|
"getEvent",
|
|
114
118
|
"listEvents",
|
|
115
119
|
"listSteps",
|
|
116
|
-
"listHooks"
|
|
120
|
+
"listHooks",
|
|
121
|
+
"getCleanupStatus",
|
|
122
|
+
"scheduleCleanup",
|
|
123
|
+
"cleanupNow",
|
|
124
|
+
"rearmCleanup"
|
|
117
125
|
],
|
|
118
|
-
mutating: /* @__PURE__ */ new Set([
|
|
126
|
+
mutating: /* @__PURE__ */ new Set([
|
|
127
|
+
"applyEvent",
|
|
128
|
+
"scheduleCleanup",
|
|
129
|
+
"cleanupNow",
|
|
130
|
+
"rearmCleanup"
|
|
131
|
+
])
|
|
119
132
|
};
|
|
120
133
|
const STREAMS = {
|
|
121
134
|
methods: [
|
|
@@ -124,12 +137,18 @@ const STREAMS = {
|
|
|
124
137
|
"getChunks",
|
|
125
138
|
"getInfo",
|
|
126
139
|
"registerStream",
|
|
127
|
-
"listStreams"
|
|
140
|
+
"listStreams",
|
|
141
|
+
"expireRegistry",
|
|
142
|
+
"finalizeRegistry",
|
|
143
|
+
"expireStream"
|
|
128
144
|
],
|
|
129
145
|
mutating: /* @__PURE__ */ new Set([
|
|
130
146
|
"writeChunk",
|
|
131
147
|
"closeStream",
|
|
132
|
-
"registerStream"
|
|
148
|
+
"registerStream",
|
|
149
|
+
"expireRegistry",
|
|
150
|
+
"finalizeRegistry",
|
|
151
|
+
"expireStream"
|
|
133
152
|
])
|
|
134
153
|
};
|
|
135
154
|
const QUEUE = {
|
|
@@ -139,13 +158,15 @@ const QUEUE = {
|
|
|
139
158
|
"listDeadLetters",
|
|
140
159
|
"redriveDeadLetter",
|
|
141
160
|
"purgeDeadLetters",
|
|
142
|
-
"rearmAlarm"
|
|
161
|
+
"rearmAlarm",
|
|
162
|
+
"expireRun"
|
|
143
163
|
],
|
|
144
164
|
mutating: /* @__PURE__ */ new Set([
|
|
145
165
|
"enqueue",
|
|
146
166
|
"redriveDeadLetter",
|
|
147
167
|
"purgeDeadLetters",
|
|
148
|
-
"rearmAlarm"
|
|
168
|
+
"rearmAlarm",
|
|
169
|
+
"expireRun"
|
|
149
170
|
])
|
|
150
171
|
};
|
|
151
172
|
function makeStub(transport, binding, name, spec) {
|
|
@@ -171,6 +192,12 @@ function makeIndexNamespace(transport) {
|
|
|
171
192
|
get: (key) => call("get", [key], true),
|
|
172
193
|
put: (key, value) => call("put", [key, value], false),
|
|
173
194
|
delete: (key) => call("delete", [key], false),
|
|
195
|
+
putOwned: (runId, key, value) => call("putOwned", [
|
|
196
|
+
runId,
|
|
197
|
+
key,
|
|
198
|
+
value
|
|
199
|
+
], false),
|
|
200
|
+
expireRun: (request) => call("expireRun", [request], false),
|
|
174
201
|
list: (options) => call("list", [options], true),
|
|
175
202
|
reserveHookToken: (token, owner) => call("reserveHookToken", [token, owner], false),
|
|
176
203
|
finalizeHookIndexes: (token, hookId, serializedHook, owner) => call("finalizeHookIndexes", [
|
|
@@ -267,10 +294,7 @@ function isPermanentError(err) {
|
|
|
267
294
|
function computeBackoff(attempt) {
|
|
268
295
|
return Math.min(60, 2 ** attempt);
|
|
269
296
|
}
|
|
270
|
-
const
|
|
271
|
-
workflow: "flow",
|
|
272
|
-
step: "step"
|
|
273
|
-
};
|
|
297
|
+
const QUEUE_PATHNAME = "flow";
|
|
274
298
|
function isTestMode() {
|
|
275
299
|
if (process.env.CELLD_QUEUE_MODE === "cells") return false;
|
|
276
300
|
return process.env.VITEST === "true" || process.env.NODE_ENV === "test";
|
|
@@ -294,9 +318,8 @@ function shardFor(key, shards) {
|
|
|
294
318
|
return (hash >>> 0) % shards;
|
|
295
319
|
}
|
|
296
320
|
/**
|
|
297
|
-
* In-process test pump. Holds
|
|
298
|
-
*
|
|
299
|
-
* `${baseUrl}/.well-known/workflow/v1/{flow|step}`.
|
|
321
|
+
* In-process test pump. Holds an in-memory FIFO and HTTP-dispatches envelopes
|
|
322
|
+
* to the user's server at `${baseUrl}/.well-known/workflow/v1/flow`.
|
|
300
323
|
*
|
|
301
324
|
* Mirrors world-local's idempotency semantics: messages are deduplicated on
|
|
302
325
|
* `idempotencyKey` while a message with the same key is in flight, and the
|
|
@@ -310,14 +333,8 @@ function createTestPump(config) {
|
|
|
310
333
|
const httpTimeoutMs = config.httpTimeoutMs ?? 3e5;
|
|
311
334
|
const maxAttempts = config.maxAttempts ?? 5;
|
|
312
335
|
const baseBackoffMs = config.backoffDelayMs ?? 1e3;
|
|
313
|
-
const queues = {
|
|
314
|
-
|
|
315
|
-
step: []
|
|
316
|
-
};
|
|
317
|
-
const wakers = {
|
|
318
|
-
flow: [],
|
|
319
|
-
step: []
|
|
320
|
-
};
|
|
336
|
+
const queues = { flow: [] };
|
|
337
|
+
const wakers = { flow: [] };
|
|
321
338
|
/** Inflight messageIds by idempotencyKey (world-local queue.js semantics). */
|
|
322
339
|
const inflightMessages = /* @__PURE__ */ new Map();
|
|
323
340
|
let running = false;
|
|
@@ -410,7 +427,6 @@ function createTestPump(config) {
|
|
|
410
427
|
if (running) return;
|
|
411
428
|
running = true;
|
|
412
429
|
loop("flow");
|
|
413
|
-
loop("step");
|
|
414
430
|
},
|
|
415
431
|
stop() {
|
|
416
432
|
running = false;
|
|
@@ -429,12 +445,13 @@ function createQueue(config) {
|
|
|
429
445
|
};
|
|
430
446
|
return {
|
|
431
447
|
async queue(queueName, message, opts) {
|
|
432
|
-
|
|
448
|
+
parseQueueName(queueName);
|
|
449
|
+
const runId = "runId" in message && typeof message.runId === "string" ? message.runId : void 0;
|
|
433
450
|
if (isTestMode()) {
|
|
434
451
|
const existing = testPump.inflight(opts?.idempotencyKey);
|
|
435
452
|
if (existing) return { messageId: existing };
|
|
436
453
|
const messageId = MessageId.parse(`msg_${generateMessageId()}`);
|
|
437
|
-
testPump.push(
|
|
454
|
+
testPump.push(QUEUE_PATHNAME, {
|
|
438
455
|
messageId,
|
|
439
456
|
queueName,
|
|
440
457
|
attempt: 1,
|
|
@@ -448,8 +465,9 @@ function createQueue(config) {
|
|
|
448
465
|
const outcome = await getQueueStub(shard).enqueue({
|
|
449
466
|
messageId,
|
|
450
467
|
queueName,
|
|
451
|
-
pathname:
|
|
468
|
+
pathname: QUEUE_PATHNAME,
|
|
452
469
|
body: stringify(message),
|
|
470
|
+
runId,
|
|
453
471
|
idempotencyKey: opts?.idempotencyKey,
|
|
454
472
|
delaySeconds: opts?.delaySeconds,
|
|
455
473
|
config: {
|
|
@@ -457,7 +475,10 @@ function createQueue(config) {
|
|
|
457
475
|
queueShards
|
|
458
476
|
}
|
|
459
477
|
});
|
|
460
|
-
if (!outcome.ok)
|
|
478
|
+
if (!outcome.ok) {
|
|
479
|
+
if (outcome.code === "RUN_EXPIRED") throw new RunExpiredError(outcome.message);
|
|
480
|
+
throw new WorkflowWorldError(outcome.message, { status: 409 });
|
|
481
|
+
}
|
|
461
482
|
return { messageId: MessageId.parse(outcome.messageId) };
|
|
462
483
|
},
|
|
463
484
|
createQueueHandler(queueNamePrefix, handler) {
|
|
@@ -515,15 +536,6 @@ function createQueue(config) {
|
|
|
515
536
|
}
|
|
516
537
|
//#endregion
|
|
517
538
|
//#region src/storage.ts
|
|
518
|
-
function sortableTimestamp(date) {
|
|
519
|
-
return date.getTime().toString().padStart(13, "0");
|
|
520
|
-
}
|
|
521
|
-
function workflowRunIndexKey(run) {
|
|
522
|
-
return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
523
|
-
}
|
|
524
|
-
function globalRunIndexKey(run) {
|
|
525
|
-
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
526
|
-
}
|
|
527
539
|
function hookOwner(hook) {
|
|
528
540
|
return {
|
|
529
541
|
runId: hook.runId,
|
|
@@ -566,26 +578,34 @@ function throwOutcomeError(outcome, runId, data) {
|
|
|
566
578
|
case "RUN_NOT_FOUND": throw new WorkflowRunNotFoundError(runId);
|
|
567
579
|
case "STEP_NOT_FOUND": throw new WorkflowWorldError(outcome.message, { status: 404 });
|
|
568
580
|
case "HOOK_NOT_FOUND": throw new HookNotFoundError(data.correlationId ?? runId);
|
|
581
|
+
case "WAIT_NOT_FOUND": throw new WorkflowWorldError(outcome.message, { status: 404 });
|
|
569
582
|
case "ENTITY_CONFLICT": throw new EntityConflictError(outcome.message);
|
|
570
583
|
case "RUN_EXPIRED": throw new RunExpiredError(outcome.message);
|
|
571
584
|
case "TOO_EARLY": throw new TooEarlyError(outcome.message, { retryAfter: outcome.retryAfterSeconds });
|
|
572
585
|
case "RUN_NOT_SUPPORTED": throw new RunNotSupportedError(outcome.runSpecVersion ?? 0, SPEC_VERSION_CURRENT);
|
|
573
|
-
case "LEGACY_RUN_NOT_SUPPORTED": throw new WorkflowWorldError(outcome.message, { status: 422 });
|
|
574
586
|
}
|
|
575
587
|
}
|
|
576
588
|
const parseRun = (run) => WorkflowRunSchema.parse(compact(run));
|
|
577
589
|
const parseStep = (step) => StepSchema.parse(compact(step));
|
|
578
590
|
const parseHook = (hook) => HookSchema.parse(compact(hook));
|
|
579
591
|
const parseEvent = (event) => EventSchema.parse(compact(event));
|
|
592
|
+
function unwrapRead(outcome) {
|
|
593
|
+
if (!outcome.ok) throw new RunExpiredError(outcome.message);
|
|
594
|
+
return outcome.value;
|
|
595
|
+
}
|
|
580
596
|
function createStorage(config) {
|
|
581
597
|
const { env } = config;
|
|
598
|
+
const cleanup = {
|
|
599
|
+
retentionMs: config.runRetentionMs ?? 0,
|
|
600
|
+
queueShards: config.queueShards ?? 1
|
|
601
|
+
};
|
|
582
602
|
const ulid = monotonicFactory();
|
|
583
603
|
const getRunDO = (runId) => {
|
|
584
604
|
const id = env.WORKFLOW_DB.idFromName(runId);
|
|
585
605
|
return env.WORKFLOW_DB.get(id);
|
|
586
606
|
};
|
|
587
607
|
const runsGet = async (runId, params) => {
|
|
588
|
-
const run = await getRunDO(runId).getRun();
|
|
608
|
+
const run = unwrapRead(await getRunDO(runId).getRun());
|
|
589
609
|
if (!run) throw new WorkflowRunNotFoundError(runId);
|
|
590
610
|
return filterData(parseRun(run), params?.resolveData, ["input", "output"]);
|
|
591
611
|
};
|
|
@@ -624,7 +644,7 @@ function createStorage(config) {
|
|
|
624
644
|
if (matches.length > limit) break;
|
|
625
645
|
}
|
|
626
646
|
} catch (error) {
|
|
627
|
-
if (!WorkflowRunNotFoundError.is(error)) throw error;
|
|
647
|
+
if (!WorkflowRunNotFoundError.is(error) && !RunExpiredError.is(error)) throw error;
|
|
628
648
|
}
|
|
629
649
|
}
|
|
630
650
|
exhausted = kvList.list_complete;
|
|
@@ -640,7 +660,7 @@ function createStorage(config) {
|
|
|
640
660
|
}
|
|
641
661
|
},
|
|
642
662
|
events: {
|
|
643
|
-
async create(runId, data,
|
|
663
|
+
async create(runId, data, params) {
|
|
644
664
|
let effectiveRunId;
|
|
645
665
|
if (data.eventType === "run_created" && (!runId || runId === "")) effectiveRunId = `wrun_${ulid()}`;
|
|
646
666
|
else if (!runId) throw new WorkflowWorldError("runId is required for non-run_created events", { status: 400 });
|
|
@@ -661,7 +681,9 @@ function createStorage(config) {
|
|
|
661
681
|
outcome = await stub.applyEvent({
|
|
662
682
|
runId: effectiveRunId,
|
|
663
683
|
data,
|
|
664
|
-
|
|
684
|
+
params,
|
|
685
|
+
tokenHolder,
|
|
686
|
+
cleanup
|
|
665
687
|
});
|
|
666
688
|
} catch (error) {
|
|
667
689
|
if (hookReservation && data.eventType === "hook_created") await env.WORKFLOW_INDEX.releaseHookToken(data.eventData.token, hookReservation);
|
|
@@ -677,8 +699,8 @@ function createStorage(config) {
|
|
|
677
699
|
createdAt: outcome.run.createdAt.toISOString(),
|
|
678
700
|
status: outcome.run.status
|
|
679
701
|
});
|
|
680
|
-
await env.WORKFLOW_INDEX.
|
|
681
|
-
await env.WORKFLOW_INDEX.
|
|
702
|
+
await env.WORKFLOW_INDEX.putOwned(effectiveRunId, workflowRunIndexKey(outcome.run), meta);
|
|
703
|
+
await env.WORKFLOW_INDEX.putOwned(effectiveRunId, globalRunIndexKey(outcome.run), meta);
|
|
682
704
|
}
|
|
683
705
|
if (outcome.hookToIndex) {
|
|
684
706
|
const serialized = stringify(outcome.hookToIndex);
|
|
@@ -689,33 +711,41 @@ function createStorage(config) {
|
|
|
689
711
|
hookId: released.hookId
|
|
690
712
|
});
|
|
691
713
|
if (outcome.event?.correlationId) {
|
|
692
|
-
const correlationKey =
|
|
693
|
-
await env.WORKFLOW_INDEX.
|
|
714
|
+
const correlationKey = correlationIndexKey(outcome.event.correlationId, outcome.event.createdAt, outcome.event.eventId, effectiveRunId);
|
|
715
|
+
await env.WORKFLOW_INDEX.putOwned(effectiveRunId, correlationKey, JSON.stringify({
|
|
694
716
|
runId: effectiveRunId,
|
|
695
717
|
eventId: outcome.event.eventId
|
|
696
718
|
}));
|
|
697
719
|
}
|
|
720
|
+
const eventPage = outcome.events === void 0 ? {} : {
|
|
721
|
+
events: outcome.events,
|
|
722
|
+
cursor: outcome.cursor ?? null,
|
|
723
|
+
hasMore: outcome.hasMore ?? false
|
|
724
|
+
};
|
|
698
725
|
return {
|
|
699
726
|
event: outcome.event,
|
|
700
727
|
run: outcome.run,
|
|
701
728
|
step: outcome.step,
|
|
702
729
|
hook: outcome.hook,
|
|
703
|
-
|
|
730
|
+
wait: outcome.wait,
|
|
731
|
+
stepCreated: outcome.stepCreated,
|
|
732
|
+
maxEvents: outcome.maxEvents,
|
|
733
|
+
...eventPage
|
|
704
734
|
};
|
|
705
735
|
},
|
|
706
736
|
async get(runId, eventId, _params) {
|
|
707
|
-
const event = await getRunDO(runId).getEvent(eventId);
|
|
737
|
+
const event = unwrapRead(await getRunDO(runId).getEvent(eventId));
|
|
708
738
|
if (!event) throw new WorkflowWorldError(`Event not found: ${eventId}`, { status: 404 });
|
|
709
739
|
return parseEvent(event);
|
|
710
740
|
},
|
|
711
741
|
async list(params) {
|
|
712
742
|
const { runId } = params;
|
|
713
743
|
const limit = params?.pagination?.limit ?? 100;
|
|
714
|
-
const result = await getRunDO(runId).listEvents({
|
|
744
|
+
const result = unwrapRead(await getRunDO(runId).listEvents({
|
|
715
745
|
limit,
|
|
716
746
|
cursor: params?.pagination?.cursor || void 0,
|
|
717
747
|
sortOrder: params.pagination?.sortOrder || "asc"
|
|
718
|
-
});
|
|
748
|
+
}));
|
|
719
749
|
return {
|
|
720
750
|
data: result.data.map(parseEvent),
|
|
721
751
|
cursor: result.cursor,
|
|
@@ -724,40 +754,48 @@ function createStorage(config) {
|
|
|
724
754
|
},
|
|
725
755
|
async listByCorrelationId(params) {
|
|
726
756
|
const limit = params.pagination?.limit ?? 100;
|
|
727
|
-
const
|
|
728
|
-
const
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
757
|
+
const stub = getRunDO(params.runId);
|
|
758
|
+
const matches = [];
|
|
759
|
+
let scanCursor = params.pagination?.cursor;
|
|
760
|
+
let exhausted = false;
|
|
761
|
+
while (matches.length <= limit && !exhausted) {
|
|
762
|
+
const page = unwrapRead(await stub.listEvents({
|
|
763
|
+
limit: Math.min(1e3, Math.max(50, limit + 1)),
|
|
764
|
+
cursor: scanCursor,
|
|
765
|
+
sortOrder: params.pagination?.sortOrder ?? "asc"
|
|
766
|
+
}));
|
|
767
|
+
for (const event of page.data) if (event.correlationId === params.correlationId) {
|
|
768
|
+
matches.push(event);
|
|
769
|
+
if (matches.length > limit) break;
|
|
770
|
+
}
|
|
771
|
+
exhausted = !page.hasMore;
|
|
772
|
+
scanCursor = page.cursor ?? page.data.at(-1)?.eventId;
|
|
773
|
+
if (page.data.length === 0) exhausted = true;
|
|
774
|
+
}
|
|
775
|
+
const hasMore = matches.length > limit;
|
|
776
|
+
const data = matches.slice(0, limit);
|
|
734
777
|
return {
|
|
735
|
-
data:
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
const { runId, eventId } = JSON.parse(raw);
|
|
739
|
-
return getRunDO(runId).getEvent(eventId);
|
|
740
|
-
}))).filter((event) => event !== null).map(parseEvent),
|
|
741
|
-
cursor: listed.list_complete ? null : listed.cursor ?? null,
|
|
742
|
-
hasMore: !listed.list_complete
|
|
778
|
+
data: data.map(parseEvent),
|
|
779
|
+
cursor: hasMore ? data.at(-1)?.eventId ?? null : null,
|
|
780
|
+
hasMore
|
|
743
781
|
};
|
|
744
782
|
}
|
|
745
783
|
},
|
|
746
784
|
steps: {
|
|
747
785
|
async get(runId, stepId, params) {
|
|
748
786
|
if (!runId) throw new WorkflowWorldError("runId is required for Cloudflare step lookup", { status: 400 });
|
|
749
|
-
const step = await getRunDO(runId).getStep(stepId);
|
|
787
|
+
const step = unwrapRead(await getRunDO(runId).getStep(stepId));
|
|
750
788
|
if (!step) throw new WorkflowWorldError(`Step not found: ${stepId}`, { status: 404 });
|
|
751
789
|
return filterData(parseStep(step), params?.resolveData, ["input", "output"]);
|
|
752
790
|
},
|
|
753
791
|
async list(params) {
|
|
754
792
|
const { runId } = params;
|
|
755
793
|
const limit = params?.pagination?.limit ?? 20;
|
|
756
|
-
const result = await getRunDO(runId).listSteps({
|
|
794
|
+
const result = unwrapRead(await getRunDO(runId).listSteps({
|
|
757
795
|
limit,
|
|
758
796
|
cursor: params?.pagination?.cursor || void 0,
|
|
759
797
|
sortOrder: params?.pagination?.sortOrder ?? "asc"
|
|
760
|
-
});
|
|
798
|
+
}));
|
|
761
799
|
return {
|
|
762
800
|
data: result.data.map((s) => filterData(parseStep(s), params?.resolveData, ["input", "output"])),
|
|
763
801
|
cursor: result.cursor,
|
|
@@ -780,11 +818,11 @@ function createStorage(config) {
|
|
|
780
818
|
if (!params.runId) throw new WorkflowWorldError("runId is required for listing hooks", { status: 400 });
|
|
781
819
|
const runId = params.runId;
|
|
782
820
|
const limit = params?.pagination?.limit ?? 100;
|
|
783
|
-
const result = await getRunDO(runId).listHooks({
|
|
821
|
+
const result = unwrapRead(await getRunDO(runId).listHooks({
|
|
784
822
|
limit,
|
|
785
823
|
cursor: params?.pagination?.cursor || void 0,
|
|
786
824
|
sortOrder: params?.pagination?.sortOrder ?? "asc"
|
|
787
|
-
});
|
|
825
|
+
}));
|
|
788
826
|
return {
|
|
789
827
|
data: result.data.map((h) => filterHookData(parseHook(h), params?.resolveData ?? "all")),
|
|
790
828
|
cursor: result.cursor,
|
|
@@ -822,76 +860,90 @@ function createStreamer(config) {
|
|
|
822
860
|
async function registerStreamForRun(name, runId) {
|
|
823
861
|
const cacheKey = `${runId}\0${name}`;
|
|
824
862
|
if (registeredStreams.has(cacheKey)) return;
|
|
825
|
-
await getRunRegistryDO(runId).registerStream(name);
|
|
863
|
+
await getRunRegistryDO(runId).registerStream(runId, name);
|
|
826
864
|
registeredStreams.add(cacheKey);
|
|
827
865
|
}
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
if (done) {
|
|
859
|
-
controller.close();
|
|
860
|
-
return;
|
|
861
|
-
}
|
|
862
|
-
await new Promise((resolve) => setTimeout(resolve, readPollMs));
|
|
866
|
+
const writeToStream = async (name, runId, chunk) => {
|
|
867
|
+
const resolvedRunId = await runId;
|
|
868
|
+
await registerStreamForRun(name, resolvedRunId);
|
|
869
|
+
await getStreamDO(name).writeChunk(resolvedRunId, toBytes(chunk));
|
|
870
|
+
};
|
|
871
|
+
const closeStream = async (name, runId) => {
|
|
872
|
+
const resolvedRunId = await runId;
|
|
873
|
+
await registerStreamForRun(name, resolvedRunId);
|
|
874
|
+
await getStreamDO(name).closeStream(resolvedRunId);
|
|
875
|
+
};
|
|
876
|
+
const readFromStream = async (name, startIndex = 0) => {
|
|
877
|
+
const stub = getStreamDO(name);
|
|
878
|
+
let nextIndex;
|
|
879
|
+
if (startIndex < 0) {
|
|
880
|
+
const info = await stub.getInfo();
|
|
881
|
+
nextIndex = Math.max(0, info.tailIndex + 1 + startIndex);
|
|
882
|
+
} else nextIndex = startIndex;
|
|
883
|
+
let cancelled = false;
|
|
884
|
+
return new ReadableStream({
|
|
885
|
+
async pull(controller) {
|
|
886
|
+
while (true) {
|
|
887
|
+
if (cancelled) return;
|
|
888
|
+
const { chunks, done } = await stub.getChunks({
|
|
889
|
+
startIndex: nextIndex,
|
|
890
|
+
limit: READ_BATCH_SIZE
|
|
891
|
+
});
|
|
892
|
+
if (chunks.length > 0) {
|
|
893
|
+
for (const chunk of chunks) controller.enqueue(chunk);
|
|
894
|
+
nextIndex += chunks.length;
|
|
895
|
+
return;
|
|
863
896
|
}
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
897
|
+
if (done) {
|
|
898
|
+
controller.close();
|
|
899
|
+
return;
|
|
900
|
+
}
|
|
901
|
+
await new Promise((resolve) => setTimeout(resolve, readPollMs));
|
|
867
902
|
}
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
903
|
+
},
|
|
904
|
+
cancel() {
|
|
905
|
+
cancelled = true;
|
|
906
|
+
}
|
|
907
|
+
});
|
|
908
|
+
};
|
|
909
|
+
const listStreamsByRunId = (runId) => getRunRegistryDO(runId).listStreams();
|
|
910
|
+
const getStreamChunks = async (name, _runId, options) => {
|
|
911
|
+
const limit = Math.min(options?.limit ?? DEFAULT_CHUNK_PAGE_SIZE, MAX_CHUNK_PAGE_SIZE);
|
|
912
|
+
const startIndex = options?.cursor ? Number.parseInt(options.cursor, 10) : 0;
|
|
913
|
+
if (Number.isNaN(startIndex) || startIndex < 0) throw new Error(`Invalid stream cursor: ${options?.cursor}`);
|
|
914
|
+
const result = await getStreamDO(name).getChunks({
|
|
915
|
+
startIndex,
|
|
916
|
+
limit: limit + 1
|
|
917
|
+
});
|
|
918
|
+
const hasMore = result.chunks.length > limit;
|
|
919
|
+
return {
|
|
920
|
+
data: result.chunks.slice(0, limit).map((chunk, offset) => ({
|
|
921
|
+
index: startIndex + offset,
|
|
922
|
+
data: chunk
|
|
923
|
+
})),
|
|
924
|
+
cursor: hasMore ? String(startIndex + limit) : null,
|
|
925
|
+
hasMore,
|
|
926
|
+
done: result.done
|
|
927
|
+
};
|
|
928
|
+
};
|
|
929
|
+
const getStreamInfo = (name, _runId) => getStreamDO(name).getInfo();
|
|
930
|
+
return {
|
|
931
|
+
streams: {
|
|
932
|
+
write: (runId, name, chunk) => writeToStream(name, runId, chunk),
|
|
933
|
+
close: (runId, name) => closeStream(name, runId),
|
|
934
|
+
get: (runId, name, startIndex) => {
|
|
935
|
+
return readFromStream(name, startIndex);
|
|
936
|
+
},
|
|
937
|
+
list: listStreamsByRunId,
|
|
938
|
+
getChunks: (runId, name, options) => getStreamChunks(name, runId, options),
|
|
939
|
+
getInfo: (runId, name) => getStreamInfo(name, runId)
|
|
891
940
|
},
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
941
|
+
writeToStream,
|
|
942
|
+
closeStream,
|
|
943
|
+
readFromStream,
|
|
944
|
+
listStreamsByRunId,
|
|
945
|
+
getStreamChunks,
|
|
946
|
+
getStreamInfo
|
|
895
947
|
};
|
|
896
948
|
}
|
|
897
949
|
//#endregion
|
|
@@ -924,7 +976,9 @@ function createCelldWorld(config) {
|
|
|
924
976
|
WORKFLOW_DB: env.WORKFLOW_DB,
|
|
925
977
|
WORKFLOW_INDEX: env.WORKFLOW_INDEX
|
|
926
978
|
},
|
|
927
|
-
deploymentId: resolved.deploymentId
|
|
979
|
+
deploymentId: resolved.deploymentId,
|
|
980
|
+
runRetentionMs: resolved.runRetentionMs,
|
|
981
|
+
queueShards: resolved.queueShards
|
|
928
982
|
});
|
|
929
983
|
const queue = createQueue({
|
|
930
984
|
env: { WORKFLOW_QUEUE: env.WORKFLOW_QUEUE },
|
|
@@ -936,10 +990,27 @@ function createCelldWorld(config) {
|
|
|
936
990
|
env: { WORKFLOW_STREAMS: env.WORKFLOW_STREAMS },
|
|
937
991
|
readPollMs: resolved.readPollMs
|
|
938
992
|
});
|
|
993
|
+
const runStub = (runId) => env.WORKFLOW_DB.get(env.WORKFLOW_DB.idFromName(runId));
|
|
994
|
+
const retention = {
|
|
995
|
+
getStatus: (runId) => runStub(runId).getCleanupStatus(),
|
|
996
|
+
schedule: (runId) => {
|
|
997
|
+
if (resolved.runRetentionMs === 0) throw new Error("world-celld: runRetentionMs must be greater than zero to schedule cleanup");
|
|
998
|
+
return runStub(runId).scheduleCleanup({
|
|
999
|
+
retentionMs: resolved.runRetentionMs,
|
|
1000
|
+
queueShards: resolved.queueShards
|
|
1001
|
+
});
|
|
1002
|
+
},
|
|
1003
|
+
cleanupNow: (runId) => runStub(runId).cleanupNow({
|
|
1004
|
+
retentionMs: Math.max(1, resolved.runRetentionMs),
|
|
1005
|
+
queueShards: resolved.queueShards
|
|
1006
|
+
}),
|
|
1007
|
+
rearm: (runId) => runStub(runId).rearmCleanup()
|
|
1008
|
+
};
|
|
939
1009
|
return {
|
|
940
1010
|
...storage,
|
|
941
1011
|
...queue,
|
|
942
1012
|
...streamer,
|
|
1013
|
+
retention,
|
|
943
1014
|
specVersion: SPEC_VERSION_CURRENT
|
|
944
1015
|
};
|
|
945
1016
|
}
|