@ewhauser/world-celld 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/NOTICE +1 -1
- package/README.md +117 -106
- package/celld-queue-worker/README.md +26 -0
- package/celld-queue-worker/worker.ts +3 -0
- package/celld-queue-worker/wrangler.jsonc +26 -0
- package/celld-worker/README.md +33 -25
- package/celld-worker/worker.ts +1 -1
- package/celld-worker/wrangler.jsonc +12 -8
- package/dist/index.d.ts +32 -70
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +33 -90
- package/dist/index.js.map +1 -1
- package/dist/queue-consumer.d.ts +33 -0
- package/dist/queue-consumer.d.ts.map +1 -0
- package/dist/queue-consumer.js +75 -0
- package/dist/queue-consumer.js.map +1 -0
- package/dist/queue-delivery-DzNOtCYA.d.ts +49 -0
- package/dist/queue-delivery-DzNOtCYA.d.ts.map +1 -0
- package/dist/{util-Cu09mVzK.js → queue-protocol-CmJNMH_5.js} +60 -12
- package/dist/queue-protocol-CmJNMH_5.js.map +1 -0
- package/dist/queue-protocol-Ds44uuND.js +104 -0
- package/dist/queue-protocol-Ds44uuND.js.map +1 -0
- package/dist/testing.d.ts +3 -1
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +420 -833
- package/dist/testing.js.map +1 -1
- package/dist/worker.d.ts +106 -181
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +602 -955
- package/dist/worker.js.map +1 -1
- package/docs/celld-v0.5-upgrade.md +130 -0
- package/package.json +14 -6
- package/dist/util-Cu09mVzK.js.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CreateEventParams, CreateEventRequest, Event, GetChunksOptions, Hook, RunCreatedEventRequest, Step, StreamChunksResponse, StreamInfoResponse, Streamer, TerminalWorkflowRunStatus,
|
|
1
|
+
import { CreateEventParams, CreateEventRequest, Event, GetChunksOptions, Hook, RunCreatedEventRequest, Step, StreamChunksResponse, StreamInfoResponse, Streamer, TerminalWorkflowRunStatus, Wait, WorkflowRun, WorkflowRunStatus, World } from "@workflow/world";
|
|
2
2
|
//#region src/retention.d.ts
|
|
3
3
|
type CleanupPhase = 'retained' | 'index' | 'streams' | 'queues' | 'payload' | 'tombstoned';
|
|
4
4
|
interface CleanupRecord {
|
|
@@ -14,7 +14,6 @@ interface CleanupRecord {
|
|
|
14
14
|
terminalStatus?: TerminalWorkflowRunStatus;
|
|
15
15
|
reason?: 'terminal-retention' | 'maximum-age' | 'manual';
|
|
16
16
|
dueAt: Date;
|
|
17
|
-
queueShards: number;
|
|
18
17
|
phase: CleanupPhase;
|
|
19
18
|
/** Optimistic-concurrency token for alarm/RPC work that awaits another cell. */
|
|
20
19
|
generation: number;
|
|
@@ -24,7 +23,8 @@ interface CleanupRecord {
|
|
|
24
23
|
tombstonedAt?: Date;
|
|
25
24
|
deletedPayloadKeys: number;
|
|
26
25
|
deletedStreams: number;
|
|
27
|
-
|
|
26
|
+
/** Queue payload objects removed from object storage during run cleanup. */
|
|
27
|
+
deletedQueuePayloads: number;
|
|
28
28
|
}
|
|
29
29
|
interface RunTombstone {
|
|
30
30
|
version: 1;
|
|
@@ -84,26 +84,8 @@ interface ExpireStreamResult {
|
|
|
84
84
|
bytes: number;
|
|
85
85
|
done: boolean;
|
|
86
86
|
}
|
|
87
|
-
interface QueueExpiryReceipt {
|
|
88
|
-
expiredAt: number;
|
|
89
|
-
deleted: number;
|
|
90
|
-
}
|
|
91
|
-
type ExpireQueueRunResult = {
|
|
92
|
-
/** Cumulative messages removed for this run in this queue shard. */
|
|
93
|
-
deleted: number;
|
|
94
|
-
} & ({
|
|
95
|
-
done: false;
|
|
96
|
-
} | {
|
|
97
|
-
done: true;
|
|
98
|
-
/** Durable receipt which the cleanup coordinator must acknowledge. */
|
|
99
|
-
receipt: QueueExpiryReceipt;
|
|
100
|
-
});
|
|
101
|
-
interface AcknowledgeQueueExpiryResult {
|
|
102
|
-
acknowledged: boolean;
|
|
103
|
-
}
|
|
104
87
|
interface ScheduleCleanupRequest {
|
|
105
88
|
retentionMs: number;
|
|
106
|
-
queueShards: number;
|
|
107
89
|
}
|
|
108
90
|
//#endregion
|
|
109
91
|
//#region src/apply-event.d.ts
|
|
@@ -305,58 +287,40 @@ interface StreamDOId {
|
|
|
305
287
|
toString(): string;
|
|
306
288
|
}
|
|
307
289
|
//#endregion
|
|
308
|
-
//#region src/queue.d.ts
|
|
309
|
-
type Pathname = 'flow';
|
|
290
|
+
//#region src/queue-protocol.d.ts
|
|
310
291
|
/**
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
292
|
+
* Small broker message carried by celld's native Queue.
|
|
293
|
+
*
|
|
294
|
+
* Run-bearing payload bytes live in object storage so the broker's 128 kB limit does not
|
|
295
|
+
* narrow the Workflow World contract and run retention can delete those bytes
|
|
296
|
+
* without waiting for the Queue's fixed four-day retention window.
|
|
314
297
|
*/
|
|
315
|
-
interface
|
|
298
|
+
interface NativeQueueEnvelope {
|
|
299
|
+
version: 1;
|
|
316
300
|
messageId: string;
|
|
317
|
-
queueName:
|
|
318
|
-
|
|
319
|
-
/** `stringify(message)` — the exact bytes POSTed to the app on delivery. */
|
|
320
|
-
body: string;
|
|
321
|
-
/** Owning workflow run; absent only for health-check messages. */
|
|
301
|
+
queueName: string;
|
|
302
|
+
targetBaseUrl: string;
|
|
322
303
|
runId?: string;
|
|
323
304
|
idempotencyKey?: string;
|
|
305
|
+
payloadKey?: string;
|
|
306
|
+
/** Inline only for the small, run-less health-check envelope. */
|
|
307
|
+
body?: string;
|
|
308
|
+
/** Absolute workflow redelivery deadline; long waits are chained by the consumer. */
|
|
309
|
+
notBefore?: number;
|
|
310
|
+
/** Failed deliveries carried across suspension re-publishes. */
|
|
311
|
+
deliveryFailures?: number;
|
|
312
|
+
suspensionCount?: number;
|
|
313
|
+
}
|
|
314
|
+
interface NativeQueueSendOptions {
|
|
324
315
|
delaySeconds?: number;
|
|
325
|
-
/**
|
|
326
|
-
* Delivery configuration. The cell pins this from its first enqueue (the
|
|
327
|
-
* eve-ambient pattern); a later mismatch is rejected as CONFIG_MISMATCH.
|
|
328
|
-
*/
|
|
329
|
-
config: QueueCellConfig;
|
|
330
316
|
}
|
|
331
|
-
interface
|
|
332
|
-
/** Base URL of the app; deliveries POST to `${baseUrl}/.well-known/workflow/v1/flow`. */
|
|
333
|
-
targetBaseUrl: string;
|
|
334
|
-
queueShards: number;
|
|
335
|
-
}
|
|
336
|
-
type EnqueueOutcome = {
|
|
337
|
-
ok: true;
|
|
317
|
+
interface NativeQueueSendResult {
|
|
338
318
|
messageId: string;
|
|
339
|
-
deduped: boolean;
|
|
340
|
-
} | {
|
|
341
|
-
ok: false;
|
|
342
|
-
code: 'CONFIG_MISMATCH' | 'RUN_EXPIRED';
|
|
343
|
-
message: string;
|
|
344
|
-
};
|
|
345
|
-
/** RPC surface of QueueDO used by the queue producer. */
|
|
346
|
-
interface QueueCellStub {
|
|
347
|
-
enqueue(request: EnqueueRequest): Promise<EnqueueOutcome>;
|
|
348
|
-
expireRun(runId: string, expiredAt: number, options?: {
|
|
349
|
-
limit?: number;
|
|
350
|
-
}): Promise<ExpireQueueRunResult>;
|
|
351
|
-
acknowledgeExpireRun(runId: string, receipt: QueueExpiryReceipt): Promise<AcknowledgeQueueExpiryResult>;
|
|
352
319
|
}
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
get(id: {
|
|
358
|
-
toString(): string;
|
|
359
|
-
}): QueueCellStub;
|
|
320
|
+
//#endregion
|
|
321
|
+
//#region src/queue.d.ts
|
|
322
|
+
interface CelldQueueProducer {
|
|
323
|
+
send(envelope: NativeQueueEnvelope, options?: NativeQueueSendOptions): Promise<NativeQueueSendResult>;
|
|
360
324
|
}
|
|
361
325
|
//#endregion
|
|
362
326
|
//#region src/indexes.d.ts
|
|
@@ -415,7 +379,7 @@ interface HookTokenOwner {
|
|
|
415
379
|
interface CelldWorldEnv {
|
|
416
380
|
WORKFLOW_DB: WorkflowRunDONamespace;
|
|
417
381
|
WORKFLOW_INDEX: IndexNamespace;
|
|
418
|
-
WORKFLOW_QUEUE:
|
|
382
|
+
WORKFLOW_QUEUE: CelldQueueProducer;
|
|
419
383
|
WORKFLOW_STREAMS: StreamDONamespace;
|
|
420
384
|
}
|
|
421
385
|
interface CelldWorldConfig {
|
|
@@ -439,13 +403,11 @@ interface CelldWorldConfig {
|
|
|
439
403
|
/** Default: process.env.CELLD_DEPLOYMENT_ID || 'celld-default' */
|
|
440
404
|
deploymentId?: string;
|
|
441
405
|
/**
|
|
442
|
-
* Base URL the app's workflow endpoints are mounted on;
|
|
443
|
-
*
|
|
406
|
+
* Base URL the app's workflow endpoints are mounted on; the native Queue
|
|
407
|
+
* consumer delivers to `${baseUrl}/.well-known/workflow/v1/flow`.
|
|
444
408
|
* Default: process.env.WORKFLOW_BASE_URL || `http://localhost:${PORT ?? 3000}`
|
|
445
409
|
*/
|
|
446
410
|
baseUrl?: string;
|
|
447
|
-
/** Number of queue cells to spread enqueues over. Default: 1 */
|
|
448
|
-
queueShards?: number;
|
|
449
411
|
/**
|
|
450
412
|
* Keep terminal run payloads for this many milliseconds before replacing
|
|
451
413
|
* them with metadata-only tombstones. Zero disables automatic cleanup.
|
|
@@ -472,5 +434,5 @@ type CelldWorld = World & CelldStreamer & {
|
|
|
472
434
|
};
|
|
473
435
|
declare function createCelldWorld(config?: CelldWorldConfig): CelldWorld;
|
|
474
436
|
//#endregion
|
|
475
|
-
export { CelldRetentionAdmin, CelldWorld, type CelldWorldConfig, type CelldWorldEnv, type CleanupPhase, type CleanupRecord, type
|
|
437
|
+
export { type CelldQueueProducer, CelldRetentionAdmin, CelldWorld, type CelldWorldConfig, type CelldWorldEnv, type CleanupPhase, type CleanupRecord, type IndexNamespace, MAX_STREAM_BATCH_BYTES, MAX_STREAM_CHUNK_BYTES, MAX_STREAM_ERROR_BYTES, MAX_STREAM_LONG_POLL_MS, MAX_STREAM_READ_BYTES, MAX_STREAM_READ_CHUNKS, MAX_STREAM_WRITE_CHUNKS, type NativeQueueEnvelope, type NativeQueueSendOptions, type NativeQueueSendResult, type RunTombstone, type StreamDONamespace, type StreamDOStub, type StreamErrorData, type StreamReadRequest, type StreamReadResult, type StreamTerminalState, type StreamWriteResult, type WorkflowRunDONamespace, type WorkflowRunDOStub, createCelldWorld, createCelldWorld as createWorld };
|
|
476
438
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../src/retention.ts","../src/apply-event.ts","../src/storage.ts","../src/stream-protocol.ts","../src/streamer.ts","../src/queue.ts","../src/indexes.ts","../src/config.ts","../src/index.ts"],"mappings":";;KAOY;UAEK;EACf;EACA;EACA;EACA,WAAW;;EAEX,SAAS;;EAET,cAAc;;EAEd,iBAAiB;EACjB;EACA,OAAO;EACP
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../src/retention.ts","../src/apply-event.ts","../src/storage.ts","../src/stream-protocol.ts","../src/streamer.ts","../src/queue-protocol.ts","../src/queue.ts","../src/indexes.ts","../src/config.ts","../src/index.ts"],"mappings":";;KAOY;UAEK;EACf;EACA;EACA;EACA,WAAW;;EAEX,SAAS;;EAET,cAAc;;EAEd,iBAAiB;EACjB;EACA,OAAO;EACP,OAAO;;EAEP;EACA;EACA;EACA,gBAAgB;EAChB,eAAe;EACf;EACA;;EAEA;;UAGe;EACf;EACA;;EAEA,SAAS;EACT,cAAc;EACd,iBAAiB;EACjB,WAAW;EACX,cAAc;;EAEd,SAAS;;KAGC;KAEA,eAAe;EACrB;EAAU,OAAO;;EACjB;EAAW;EAAqB;EAAiB,WAAW;;UAEjD;EACf;EACA;;UAae;EACf;EACA,OAAO;;UAGQ;EACf;;UAGe;EACf;EACA,OAAO;EACP;;UAGe;EACf;;UAGe;EACf;;UAGe;;EAEf;EACA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;;UAGe;EACf;;;;UC0Ce;;EAEf;EACA,MAAM,qBAAqB;;EAE3B,SAAS;;;;;EAKT;IAAgB;IAAe;;;EAE/B;;EAEA,UAAU;;KAiFA;UAWK;EACf;EACA,MAAM;EACN;;EAEA;;EAEA;;UAGe;EACf;;EAEA,QAAQ;EACR,MAAM;EACN,OAAO;EACP,OAAO;EACP,OAAO;;EAEP;;EAEA,eAAe;IAAQ;IAAgB;;;EAEvC;IAAe;IAAsB,WAAW;;;EAEhD,cAAc;;EAEd,SAAS;;EAET;EACA;;EAEA;;EAEA;;KAGU,oBAAoB,oBAAoB;;;;;;;UCzOnC;EACf,WAAW,SAAS,oBAAoB,QAAQ;EAChD,sBAAsB;IACpB;IACA;IACA;MACE;IAAU;;EACd,sBAAsB,QADlB;EAEJ,UAAU,QAAQ,eAAe;EACjC,QAAQ,iBAAiB,QAAQ,eAAe;EAChD,SAAS,kBAAkB,QAAQ,eAAe;EAClD,WAAW;IACT;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAS;IAAuB;;EACnE,UAAU;IACR;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAQ;IAAuB;;EAClE,UAAU;IACR;IACA;IACA;MACE,QAAQ;IAAiB,MAAM;IAAQ;IAAuB;;EAClE,oBAAoB,QADhB;EAEJ,gBACE,SAFkB,yBAGjB,QADiC;EAEpC,WACE,SAFC,yBAGA,QADiC;EAEpC,gBAAgB,QADb;;UAIY;EACf,WAAW,eAAe;EAC1B,IAAI,IAAI,sBAAsB;;UAGf;EACf;;;;;cC9FW;;cAEA;;cAEA;cACA;cACA;cACA;;cAEA;KAED;UAEK;EACf;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA,QAAQ;EACR,OAAO;EACP;EACA,QAAQ;;;;UCXO,sBAAsB,KAAK;EAC1C,SAAS;IACP,KAAK,eAAe,cAAc,OAAO,2BAA2B;;EAEtE,cACE,cACA,gBAAgB,iBAChB,gBAAgB,aACf;EACH,oBACE,cACA,gBAAgB,iBAChB,2BAA2B,gBAC1B;EACH,YAAY,cAAc,gBAAgB,kBAAkB;EAC5D,YACE,cACA,gBAAgB,iBAChB,OAAO,2BACN;EACH,eACE,cACA,eACA,sBACC,QAAQ,eAAe;EAC1B,mBAAmB,gBAAgB;EACnC,gBACE,cACA,eACA,UAAU,mBACT,QAAQ;EACX,cAAc,cAAc,gBAAgB,QAAQ;;;UAIrC;EACf,YAAY,eAAe,QAAQ,eAAe,QAAQ;EAC1D,WAAW,SAAS,mBAAmB,SAAS,cAAc,QAAQ;EACtE,YAAY,gBAAgB;EAC5B,WAAW,eAAe,OAAO,2BAA2B;EAC5D,eAAe,eAAe,eAAe;EAC7C,eAAe;EACf,eACE,eACA,mBACA;IAAY;MACX,QAAQ;EACX,iBAAiB,eAAe,oBAAoB,QAAQ;EAC5D,aACE,eACA,mBACA;IAAY;IAAgB;MAC3B,QAAQ;;UAGI;EACf,WAAW,eAAe;EAC1B,IAAI,IAAI,aAAa;;UAGN;EACf;;;;;;;;;;;UCjEe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA;EACA;;UAGe;EACf;;UAGe;EACf;;;;UC2Ce;EACf,KACE,UAAU,qBACV,UAAU,yBACT,QAAQ;;;;UCrFI;EACf;EACA;;EAEA;EACA;EACA;;UAGe;;EAEf,MAAM;IAAQ;IAAc;IAAe;;EAC3C;EACA;;UAuBe;EACf;EACA;EACA;;KAGU;EACN;EAAgB,aAAa;;EAC7B;EAAiB,QAAQ;;;UAgDd;EACf,UACE,KAAK,aACL,4BACA,+BACC;IAAU;;EACb,SAAS,UAAU,mBAAmB,QAAQ;EAC9C,eAAe,gBAAgB;EAC/B,YAAY,iBAAiB;EAC7B,YAAY,eAAe,OAAO,iBAAiB,QAAQ;EAC3D,oBACE,eACA,gBACA,wBACA,OAAO,gBACP,cAAc,kBACb;EACH,uBACE,eACA,OAAO,gBACP,aAAa,kBACZ;EACH,mBAAmB,SAAS,4BAA4B,QAAQ;EAChE,UAAU,SAAS,0BAA0B,QAAQ;;;;;KC1H3C,iBAAiB;UAEZ;EACf;EACA;;UAGe;EACf,aAAa;EACb,gBAAgB;EAChB,gBAAgB;EAChB,kBAAkB;;UAGH;;;;;;;EAOf;;;;;EAKA;;;;;EAKA,MAAM;;EAEN;;;;;;EAMA;;;;;;EAMA;;EAEA;;EAEA;;EAEA;;;;UCpBe;EACf,UAAU,gBAAgB,QAAQ;EAClC,SAAS,gBAAgB,QAAQ;EACjC,WAAW,gBAAgB,QAAQ;EACnC,MAAM,gBAAgB,QAAQ;;KAGpB,aAAa,QAAQ;EAAkB,WAAW;;iBAS9C,iBAAiB,SAAS,mBAAmB"}
|
package/dist/index.js
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as MAX_STREAM_BATCH_BYTES, B as decodeStreamWriteResult, C as boundedIntegerOption, F as MAX_STREAM_READ_CHUNKS, H as encodeStreamWriteBatch, I as MAX_STREAM_WRITE_CHUNKS, L as STREAM_BATCH_CONTENT_TYPE, M as MAX_STREAM_ERROR_BYTES, N as MAX_STREAM_LONG_POLL_MS, O as queueDelayDeadline, P as MAX_STREAM_READ_BYTES, R as decodeStreamReadResult, W as normalizeStreamError, b as MAX_QUEUE_DELIVERY_TIMEOUT_MS, d as rpcParse, f as rpcStringify, h as createWorkflowIndex, j as MAX_STREAM_CHUNK_BYTES, k as strictIntegerSetting, m as stringify, p as parse, r as nativeQueueDelaySeconds, u as compact, w as checkedQueueTimestampAdd, x as MAX_QUEUE_DERIVED_DEADLINE_HEADROOM_MS, y as MAX_FLEET_RPC_TIMEOUT_MS } from "./queue-protocol-CmJNMH_5.js";
|
|
2
2
|
import { CreateEventSchema, EventSchema, HookSchema, MessageId, QueuePayloadSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, isTerminalWorkflowRunStatus, parseQueueName } from "@workflow/world";
|
|
3
3
|
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
4
4
|
import { monotonicFactory } from "ulid";
|
|
5
5
|
//#region src/config.ts
|
|
6
6
|
function resolveConfig(config) {
|
|
7
|
-
const queueShards = boundedIntegerOption("world-celld: queueShards", config?.queueShards, 1, 1);
|
|
8
7
|
const runRetentionMs = config?.runRetentionMs !== void 0 ? boundedIntegerOption("world-celld: runRetentionMs", config.runRetentionMs, 0, 0) : strictIntegerSetting("world-celld: runRetentionMs", process.env.CELLD_RUN_RETENTION_MS, 0, 0);
|
|
9
8
|
const rpcTimeoutMs = config?.rpcTimeoutMs ?? 3e4;
|
|
10
9
|
if (!Number.isSafeInteger(rpcTimeoutMs) || rpcTimeoutMs < 1 || rpcTimeoutMs > 3e5) throw new Error(`world-celld: rpcTimeoutMs must be between 1 and ${MAX_FLEET_RPC_TIMEOUT_MS}`);
|
|
@@ -19,7 +18,6 @@ function resolveConfig(config) {
|
|
|
19
18
|
env: config?.env,
|
|
20
19
|
deploymentId: config?.deploymentId ?? process.env.CELLD_DEPLOYMENT_ID ?? "celld-default",
|
|
21
20
|
baseUrl: config?.baseUrl,
|
|
22
|
-
queueShards,
|
|
23
21
|
runRetentionMs,
|
|
24
22
|
streamLongPollMs,
|
|
25
23
|
streamFlushIntervalMs,
|
|
@@ -341,26 +339,6 @@ function makeStreamNamespace(transport) {
|
|
|
341
339
|
}
|
|
342
340
|
};
|
|
343
341
|
}
|
|
344
|
-
const QUEUE = {
|
|
345
|
-
methods: [
|
|
346
|
-
"enqueue",
|
|
347
|
-
"stats",
|
|
348
|
-
"listDeadLetters",
|
|
349
|
-
"redriveDeadLetter",
|
|
350
|
-
"purgeDeadLetters",
|
|
351
|
-
"rearmAlarm",
|
|
352
|
-
"expireRun",
|
|
353
|
-
"acknowledgeExpireRun"
|
|
354
|
-
],
|
|
355
|
-
mutating: /* @__PURE__ */ new Set([
|
|
356
|
-
"enqueue",
|
|
357
|
-
"redriveDeadLetter",
|
|
358
|
-
"purgeDeadLetters",
|
|
359
|
-
"rearmAlarm",
|
|
360
|
-
"expireRun",
|
|
361
|
-
"acknowledgeExpireRun"
|
|
362
|
-
])
|
|
363
|
-
};
|
|
364
342
|
function makeStub(transport, binding, name, spec) {
|
|
365
343
|
const stub = {};
|
|
366
344
|
for (const method of spec.methods) stub[method] = (...args) => callDO(transport, binding, name, method, args, { idempotent: !spec.mutating.has(method) });
|
|
@@ -440,7 +418,7 @@ function createRemoteEnv(transport) {
|
|
|
440
418
|
], { idempotent: true }),
|
|
441
419
|
releaseHookIndexes: (request) => callFleetRoute(transport, "/v1/index/hooks/release", [request], { idempotent: true })
|
|
442
420
|
},
|
|
443
|
-
WORKFLOW_QUEUE:
|
|
421
|
+
WORKFLOW_QUEUE: { send: (envelope, options) => callFleetRoute(transport, "/v1/queue/send", [envelope, options], { idempotent: false }) }
|
|
444
422
|
};
|
|
445
423
|
}
|
|
446
424
|
//#endregion
|
|
@@ -480,16 +458,15 @@ const debug = createDebugLogger("celld-world");
|
|
|
480
458
|
* Vendored from vinnymac/worlds packages/world-cloudflare/src/queue.ts
|
|
481
459
|
* (Apache-2.0, see NOTICE), modified for celld:
|
|
482
460
|
*
|
|
483
|
-
* -
|
|
484
|
-
*
|
|
485
|
-
*
|
|
486
|
-
*
|
|
487
|
-
*
|
|
488
|
-
*
|
|
489
|
-
* cell (at enqueue and across the delivery state machine).
|
|
461
|
+
* - Production publishes a small pointer through celld v0.5.0 native Queues.
|
|
462
|
+
* The deployed Queue consumer forwards deliveries to the same HTTP handler
|
|
463
|
+
* used by the in-process test pump.
|
|
464
|
+
* - Run-bearing message bodies live in the fleet object store. This preserves
|
|
465
|
+
* the World's larger payload contract and lets run retention delete payload
|
|
466
|
+
* bytes independently of the broker's fixed retention window.
|
|
490
467
|
* - `createQueueHandler` has ONE dialect — the x-vqs wire format the test
|
|
491
468
|
* pump has always used — extended so permanent errors surface as their
|
|
492
|
-
* HTTP statuses.
|
|
469
|
+
* HTTP statuses. Native Queue deliveries and the in-process test pump hit the
|
|
493
470
|
* exact same handler path.
|
|
494
471
|
*/
|
|
495
472
|
const MAX_TIMER_DELAY_MS = 2147483647;
|
|
@@ -530,7 +507,7 @@ function computeBackoff(attempt) {
|
|
|
530
507
|
}
|
|
531
508
|
const QUEUE_PATHNAME = "flow";
|
|
532
509
|
function isTestMode() {
|
|
533
|
-
if (process.env.CELLD_QUEUE_MODE === "
|
|
510
|
+
if (process.env.CELLD_QUEUE_MODE === "native") return false;
|
|
534
511
|
return process.env.VITEST === "true" || process.env.NODE_ENV === "test";
|
|
535
512
|
}
|
|
536
513
|
function resolveBaseUrl(config) {
|
|
@@ -539,20 +516,6 @@ function resolveBaseUrl(config) {
|
|
|
539
516
|
return `http://localhost:${process.env.PORT ?? "3000"}`;
|
|
540
517
|
}
|
|
541
518
|
/**
|
|
542
|
-
* FNV-1a hash for shard selection. All messages sharing an idempotencyKey
|
|
543
|
-
* land on the same cell so dedup stays strictly consistent.
|
|
544
|
-
*/
|
|
545
|
-
function shardFor(key, shards) {
|
|
546
|
-
if (!Number.isSafeInteger(shards) || shards < 1) throw new Error("world-celld queueShards must be a positive safe integer");
|
|
547
|
-
if (shards <= 1) return 0;
|
|
548
|
-
let hash = 2166136261;
|
|
549
|
-
for (let i = 0; i < key.length; i++) {
|
|
550
|
-
hash ^= key.charCodeAt(i);
|
|
551
|
-
hash = Math.imul(hash, 16777619);
|
|
552
|
-
}
|
|
553
|
-
return (hash >>> 0) % shards;
|
|
554
|
-
}
|
|
555
|
-
/**
|
|
556
519
|
* In-process test pump. Holds an in-memory FIFO and HTTP-dispatches envelopes
|
|
557
520
|
* to the user's server at `${baseUrl}/.well-known/workflow/v1/flow`.
|
|
558
521
|
*
|
|
@@ -562,7 +525,7 @@ function shardFor(key, shards) {
|
|
|
562
525
|
*
|
|
563
526
|
* Modified vs upstream: permanent-error statuses (404/409/410/422) from the
|
|
564
527
|
* handler drop the message immediately instead of burning retry budget —
|
|
565
|
-
* matching
|
|
528
|
+
* matching the native Queue consumer's production semantics.
|
|
566
529
|
*/
|
|
567
530
|
function createTestPump(config) {
|
|
568
531
|
const httpTimeoutMs = boundedIntegerOption("world-celld test pump httpTimeoutMs", config.httpTimeoutMs, 3e5, 1, MAX_QUEUE_DELIVERY_TIMEOUT_MS);
|
|
@@ -634,7 +597,7 @@ function createTestPump(config) {
|
|
|
634
597
|
attempt: envelope.attempt + 1
|
|
635
598
|
};
|
|
636
599
|
const backoff = Math.min(MAX_TEST_PUMP_BACKOFF_MS, baseBackoffMs * 2 ** Math.min(next.attempt - 1, 30));
|
|
637
|
-
if (checkedQueueTimestampAdd(Date.now(), backoff,
|
|
600
|
+
if (checkedQueueTimestampAdd(Date.now(), backoff, 900001) === null) {
|
|
638
601
|
release(envelope);
|
|
639
602
|
console.error(`[world-celld test pump] dropping ${envelope.messageId}: retry deadline exceeds the queue timestamp horizon`);
|
|
640
603
|
return;
|
|
@@ -664,7 +627,7 @@ function createTestPump(config) {
|
|
|
664
627
|
return idempotencyKey ? inflightMessages.get(idempotencyKey) : void 0;
|
|
665
628
|
},
|
|
666
629
|
push(pathname, envelope, delaySeconds) {
|
|
667
|
-
if (queueDelayDeadline(Date.now(), delaySeconds ?? 0, 0,
|
|
630
|
+
if (queueDelayDeadline(Date.now(), delaySeconds ?? 0, 0, 900001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
|
|
668
631
|
if (envelope.idempotencyKey) inflightMessages.set(envelope.idempotencyKey, MessageId.parse(envelope.messageId));
|
|
669
632
|
if (delaySeconds && delaySeconds > 0) delayFor(delaySeconds * 1e3).then(() => enqueue(pathname, envelope));
|
|
670
633
|
else enqueue(pathname, envelope);
|
|
@@ -683,19 +646,14 @@ function createTestPump(config) {
|
|
|
683
646
|
function createQueue(config) {
|
|
684
647
|
const { env, deploymentId } = config;
|
|
685
648
|
if (!env?.WORKFLOW_QUEUE) throw new Error("world-celld queue missing WORKFLOW_QUEUE binding");
|
|
686
|
-
const queueShards = boundedIntegerOption("world-celld queueShards", config.queueShards, 1, 1);
|
|
687
649
|
const generateMessageId = monotonicFactory();
|
|
688
650
|
const testPump = createTestPump(config);
|
|
689
|
-
const getQueueStub = (shard) => {
|
|
690
|
-
const id = env.WORKFLOW_QUEUE.idFromName(`q:${shard}`);
|
|
691
|
-
return env.WORKFLOW_QUEUE.get(id);
|
|
692
|
-
};
|
|
693
651
|
return {
|
|
694
652
|
async queue(queueName, message, opts) {
|
|
695
653
|
parseQueueName(queueName);
|
|
696
654
|
const parsedMessage = QueuePayloadSchema.safeParse(message);
|
|
697
655
|
if (!parsedMessage.success) throw new WorkflowWorldError("world-celld queue payload is invalid", { status: 422 });
|
|
698
|
-
if (queueDelayDeadline(Date.now(), opts?.delaySeconds ?? 0, 0,
|
|
656
|
+
if (queueDelayDeadline(Date.now(), opts?.delaySeconds ?? 0, 0, 900001) === null) throw new Error("world-celld queue delaySeconds lacks required delivery headroom");
|
|
699
657
|
const runId = "runId" in parsedMessage.data && typeof parsedMessage.data.runId === "string" ? parsedMessage.data.runId : void 0;
|
|
700
658
|
if (isTestMode()) {
|
|
701
659
|
const existing = testPump.inflight(opts?.idempotencyKey);
|
|
@@ -711,25 +669,20 @@ function createQueue(config) {
|
|
|
711
669
|
return { messageId };
|
|
712
670
|
}
|
|
713
671
|
const messageId = MessageId.parse(`msg_${generateMessageId()}`);
|
|
714
|
-
const
|
|
715
|
-
const
|
|
672
|
+
const body = stringify(parsedMessage.data);
|
|
673
|
+
const notBefore = opts?.delaySeconds && opts.delaySeconds > 0 ? queueDelayDeadline(Date.now(), opts.delaySeconds, 0, 900001) ?? void 0 : void 0;
|
|
674
|
+
const envelope = {
|
|
675
|
+
version: 1,
|
|
716
676
|
messageId,
|
|
717
677
|
queueName,
|
|
718
|
-
|
|
719
|
-
body: stringify(parsedMessage.data),
|
|
678
|
+
targetBaseUrl: resolveBaseUrl(config),
|
|
720
679
|
runId,
|
|
721
680
|
idempotencyKey: opts?.idempotencyKey,
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
});
|
|
728
|
-
if (!outcome.ok) {
|
|
729
|
-
if (outcome.code === "RUN_EXPIRED") throw new RunExpiredError(outcome.message);
|
|
730
|
-
throw new WorkflowWorldError(outcome.message, { status: 409 });
|
|
731
|
-
}
|
|
732
|
-
return { messageId: MessageId.parse(outcome.messageId) };
|
|
681
|
+
body,
|
|
682
|
+
notBefore
|
|
683
|
+
};
|
|
684
|
+
const published = await env.WORKFLOW_QUEUE.send(envelope, { delaySeconds: nativeQueueDelaySeconds(notBefore, Date.now()) });
|
|
685
|
+
return { messageId: MessageId.parse(published.messageId) };
|
|
733
686
|
},
|
|
734
687
|
createQueueHandler(queueNamePrefix, handler) {
|
|
735
688
|
return async (req) => {
|
|
@@ -761,7 +714,7 @@ function createQueue(config) {
|
|
|
761
714
|
messageId: reqMessageId
|
|
762
715
|
});
|
|
763
716
|
if (result && typeof result.timeoutSeconds === "number") {
|
|
764
|
-
if (queueDelayDeadline(Date.now(), result.timeoutSeconds, 1,
|
|
717
|
+
if (queueDelayDeadline(Date.now(), result.timeoutSeconds, 1, 900001) === null) throw new Error("Queue handler timeoutSeconds is out of range");
|
|
765
718
|
return Response.json({ timeoutSeconds: result.timeoutSeconds }, {
|
|
766
719
|
status: 503,
|
|
767
720
|
headers: { "Retry-After": String(result.timeoutSeconds) }
|
|
@@ -1013,10 +966,7 @@ function indexStatusExcludes(indexed, requested) {
|
|
|
1013
966
|
}
|
|
1014
967
|
function createStorage(config) {
|
|
1015
968
|
const { env } = config;
|
|
1016
|
-
const cleanup = {
|
|
1017
|
-
retentionMs: config.runRetentionMs ?? 0,
|
|
1018
|
-
queueShards: config.queueShards ?? 1
|
|
1019
|
-
};
|
|
969
|
+
const cleanup = { retentionMs: config.runRetentionMs ?? 0 };
|
|
1020
970
|
const ulid = monotonicFactory();
|
|
1021
971
|
const getRunDO = (runId) => {
|
|
1022
972
|
const id = env.WORKFLOW_DB.idFromName(runId);
|
|
@@ -1031,6 +981,7 @@ function createStorage(config) {
|
|
|
1031
981
|
runs: {
|
|
1032
982
|
get: runsGet,
|
|
1033
983
|
async list(params) {
|
|
984
|
+
const statuses = params?.status === void 0 ? void 0 : Array.isArray(params.status) ? params.status : [params.status];
|
|
1034
985
|
const limit = params?.pagination?.limit ?? 20;
|
|
1035
986
|
const prefix = params?.workflowName ? `run:${params.workflowName}:` : "runall:";
|
|
1036
987
|
const reverse = params?.pagination?.sortOrder === "desc";
|
|
@@ -1051,7 +1002,7 @@ function createStorage(config) {
|
|
|
1051
1002
|
const candidates = [];
|
|
1052
1003
|
for (const key of kvList.keys) {
|
|
1053
1004
|
const metadata = JSON.parse(key.value);
|
|
1054
|
-
if (indexStatusExcludes(metadata.status,
|
|
1005
|
+
if (statuses?.every((status) => indexStatusExcludes(metadata.status, status))) continue;
|
|
1055
1006
|
candidates.push({
|
|
1056
1007
|
key: key.name,
|
|
1057
1008
|
metadata
|
|
@@ -1063,7 +1014,7 @@ function createStorage(config) {
|
|
|
1063
1014
|
const resolved = await Promise.all(batch.map(async ({ key, metadata }) => {
|
|
1064
1015
|
try {
|
|
1065
1016
|
const run = await runsGet(metadata.runId, { resolveData: params?.resolveData });
|
|
1066
|
-
return
|
|
1017
|
+
return statuses === void 0 || statuses.includes(run.status) ? {
|
|
1067
1018
|
key,
|
|
1068
1019
|
run
|
|
1069
1020
|
} : null;
|
|
@@ -1502,14 +1453,12 @@ function createCelldWorld(config) {
|
|
|
1502
1453
|
WORKFLOW_INDEX: env.WORKFLOW_INDEX
|
|
1503
1454
|
},
|
|
1504
1455
|
deploymentId: resolved.deploymentId,
|
|
1505
|
-
runRetentionMs: resolved.runRetentionMs
|
|
1506
|
-
queueShards: resolved.queueShards
|
|
1456
|
+
runRetentionMs: resolved.runRetentionMs
|
|
1507
1457
|
});
|
|
1508
1458
|
const queue = createQueue({
|
|
1509
1459
|
env: { WORKFLOW_QUEUE: env.WORKFLOW_QUEUE },
|
|
1510
1460
|
deploymentId: resolved.deploymentId,
|
|
1511
|
-
baseUrl: resolved.baseUrl
|
|
1512
|
-
queueShards: resolved.queueShards
|
|
1461
|
+
baseUrl: resolved.baseUrl
|
|
1513
1462
|
});
|
|
1514
1463
|
const streamer = createStreamer({
|
|
1515
1464
|
env: { WORKFLOW_STREAMS: env.WORKFLOW_STREAMS },
|
|
@@ -1521,15 +1470,9 @@ function createCelldWorld(config) {
|
|
|
1521
1470
|
getStatus: (runId) => runStub(runId).getCleanupStatus(),
|
|
1522
1471
|
schedule: (runId) => {
|
|
1523
1472
|
if (resolved.runRetentionMs === 0) throw new Error("world-celld: runRetentionMs must be greater than zero to schedule cleanup");
|
|
1524
|
-
return runStub(runId).scheduleCleanup({
|
|
1525
|
-
retentionMs: resolved.runRetentionMs,
|
|
1526
|
-
queueShards: resolved.queueShards
|
|
1527
|
-
});
|
|
1473
|
+
return runStub(runId).scheduleCleanup({ retentionMs: resolved.runRetentionMs });
|
|
1528
1474
|
},
|
|
1529
|
-
cleanupNow: (runId) => runStub(runId).cleanupNow({
|
|
1530
|
-
retentionMs: Math.max(1, resolved.runRetentionMs),
|
|
1531
|
-
queueShards: resolved.queueShards
|
|
1532
|
-
}),
|
|
1475
|
+
cleanupNow: (runId) => runStub(runId).cleanupNow({ retentionMs: Math.max(1, resolved.runRetentionMs) }),
|
|
1533
1476
|
rearm: (runId) => runStub(runId).rearmCleanup()
|
|
1534
1477
|
};
|
|
1535
1478
|
return {
|