octaflow 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pg-boss/index.d.ts +226 -2
- package/dist/pg-boss/index.js +410 -1
- package/dist/postgres/index.d.ts +194 -2
- package/dist/postgres/index.js +747 -1
- package/package.json +1 -11
- package/dist/dispatcher-pgboss/index.d.ts +0 -2
- package/dist/dispatcher-pgboss/index.js +0 -2
- package/dist/event-sink-DdAm2ZHR.js +0 -748
- package/dist/event-sink-XNXFMGyK.d.ts +0 -195
- package/dist/scheduler-CzQ1bY0V.js +0 -411
- package/dist/scheduler-YCFUqjXf.d.ts +0 -227
- package/dist/store-pg/index.d.ts +0 -2
- package/dist/store-pg/index.js +0 -2
package/dist/pg-boss/index.d.ts
CHANGED
|
@@ -1,2 +1,226 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
1
|
+
import "../index-BURAYZui.js";
|
|
2
|
+
import { n as Logger } from "../logger-BCus3AOa.js";
|
|
3
|
+
import { r as Dispatcher } from "../dispatcher-g7Ct0J6w.js";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { PgBoss } from "pg-boss";
|
|
6
|
+
//#region src/pg-boss/payload.d.ts
|
|
7
|
+
/**
|
|
8
|
+
* The wire payload for a step job. It is the flow-core `DispatchStepPayload`
|
|
9
|
+
* plus the `partitionKey` — the worker needs the partition to reconstruct a
|
|
10
|
+
* partition-scoped engine before handing the job to `engine.handleStepJob`.
|
|
11
|
+
*/
|
|
12
|
+
export declare const WIRE_STEP_PAYLOAD_SCHEMA: z.ZodObject<{
|
|
13
|
+
partitionKey: z.ZodString;
|
|
14
|
+
workflowId: z.ZodNumber;
|
|
15
|
+
stepId: z.ZodNumber;
|
|
16
|
+
stepKey: z.ZodString;
|
|
17
|
+
stepType: z.ZodString;
|
|
18
|
+
kind: z.ZodDefault<z.ZodEnum<{
|
|
19
|
+
execute: "execute";
|
|
20
|
+
timeout: "timeout";
|
|
21
|
+
}>>;
|
|
22
|
+
}, z.core.$strip>;
|
|
23
|
+
export type WireStepPayload = z.infer<typeof WIRE_STEP_PAYLOAD_SCHEMA>;
|
|
24
|
+
export interface StepQueueConfig {
|
|
25
|
+
/** Retries before a job is dead-lettered. Default 2. */
|
|
26
|
+
retryLimit?: number;
|
|
27
|
+
/** Seconds between retries. Default 30. */
|
|
28
|
+
retryDelay?: number;
|
|
29
|
+
/** Seconds before an in-flight job is considered expired. Default 600. */
|
|
30
|
+
expireInSeconds?: number;
|
|
31
|
+
}
|
|
32
|
+
export declare const DEFAULT_STEP_QUEUE_CONFIG: Required<StepQueueConfig>;
|
|
33
|
+
/**
|
|
34
|
+
* The wire payload for a scheduled (or ad-hoc) workflow **start**. Carries the
|
|
35
|
+
* partition plus what `engine.startWorkflow` needs; the host resolves `workflowType`
|
|
36
|
+
* to a definition.
|
|
37
|
+
*
|
|
38
|
+
* Two mutually exclusive ways to ask for start idempotency, because a cron schedule
|
|
39
|
+
* stores its payload **once** and pg-boss redelivers that same payload on every tick:
|
|
40
|
+
*
|
|
41
|
+
* - `idempotencyKeyPrefix` — the usual choice. The start worker appends the job id,
|
|
42
|
+
* so each tick gets a distinct key while a redelivery of the *same* tick reuses it.
|
|
43
|
+
* - `idempotencyKey` — used verbatim. On a schedule this means "start exactly one
|
|
44
|
+
* workflow, ever"; every later tick returns the first one.
|
|
45
|
+
*/
|
|
46
|
+
export declare const WIRE_START_PAYLOAD_SCHEMA: z.ZodObject<{
|
|
47
|
+
partitionKey: z.ZodString;
|
|
48
|
+
workflowType: z.ZodString;
|
|
49
|
+
input: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
50
|
+
entityRef: z.ZodOptional<z.ZodString>;
|
|
51
|
+
idempotencyKey: z.ZodOptional<z.ZodString>;
|
|
52
|
+
idempotencyKeyPrefix: z.ZodOptional<z.ZodString>;
|
|
53
|
+
}, z.core.$strip>;
|
|
54
|
+
export type WireStartPayload = z.infer<typeof WIRE_START_PAYLOAD_SCHEMA>;
|
|
55
|
+
/**
|
|
56
|
+
* What a {@link WireStartPayload} becomes at delivery time: the same fields, plus the
|
|
57
|
+
* pg-boss job id and the **resolved** `idempotencyKey` to hand `engine.startWorkflow`.
|
|
58
|
+
*
|
|
59
|
+
* `idempotencyKey` is verbatim when the payload set one, `${idempotencyKeyPrefix}:${jobId}`
|
|
60
|
+
* when it set a prefix, and `undefined` when it set neither.
|
|
61
|
+
*/
|
|
62
|
+
export interface StartJobContext extends WireStartPayload {
|
|
63
|
+
/** pg-boss job id — unique per cron tick, stable across that job's retries. */
|
|
64
|
+
jobId: string;
|
|
65
|
+
}
|
|
66
|
+
/** Resolve the per-delivery idempotency key. Explicit key wins over a prefix. */
|
|
67
|
+
export declare function resolveStartIdempotencyKey(payload: WireStartPayload, jobId: string): string | undefined;
|
|
68
|
+
//#endregion
|
|
69
|
+
//#region src/pg-boss/queue.d.ts
|
|
70
|
+
/**
|
|
71
|
+
* Create the step queue and its dead-letter queue if they do not exist. Call
|
|
72
|
+
* once at startup; the dispatcher also runs it from `prepare` and lazily on a
|
|
73
|
+
* plain first enqueue.
|
|
74
|
+
*
|
|
75
|
+
* Only what is missing is created: pg-boss's `createQueue` evicts the queue's
|
|
76
|
+
* entry from its per-instance cache, and a `send` that misses that cache reads
|
|
77
|
+
* the queue on pg-boss's *own* connection — inside a store transaction on a
|
|
78
|
+
* single-connection database that is a deadlock. An existing queue is left
|
|
79
|
+
* alone, cache entry and all.
|
|
80
|
+
*/
|
|
81
|
+
export declare function ensureStepQueue(boss: PgBoss, queueName: string, config?: StepQueueConfig): Promise<void>;
|
|
82
|
+
export interface PgBossDispatcherDeps {
|
|
83
|
+
boss: PgBoss;
|
|
84
|
+
queueName: string;
|
|
85
|
+
/** Partition this dispatcher is bound to — stamped into every enqueued job. */
|
|
86
|
+
partitionKey: string;
|
|
87
|
+
config?: StepQueueConfig;
|
|
88
|
+
}
|
|
89
|
+
/** A flow-core `Dispatcher` backed by pg-boss. */
|
|
90
|
+
export declare function createPgBossDispatcher(deps: PgBossDispatcherDeps): Dispatcher;
|
|
91
|
+
export interface PgBossStepWorkerOptions {
|
|
92
|
+
/** Steps fetched per poll. Default 1. */
|
|
93
|
+
batchSize?: number;
|
|
94
|
+
/** Base poll interval in seconds (pg-boss minimum 0.5). Default: pg-boss's own. */
|
|
95
|
+
pollingIntervalSeconds?: number;
|
|
96
|
+
/**
|
|
97
|
+
* Poll interval used while pg-boss's LISTEN/NOTIFY is active for the queue, where
|
|
98
|
+
* polling is only a backstop. Ignored when notify is off or unavailable.
|
|
99
|
+
*/
|
|
100
|
+
notifyPollingIntervalSeconds?: number;
|
|
101
|
+
/**
|
|
102
|
+
* Keep fetching with no delay while every fetch returns a full `batchSize` batch;
|
|
103
|
+
* the first short fetch ends the burst. Ignored when `batchSize` is 1.
|
|
104
|
+
*
|
|
105
|
+
* **Worth enabling for throughput.** Without it a worker spends most of its time
|
|
106
|
+
* waiting out the poll interval: it drains a batch in milliseconds, then sleeps.
|
|
107
|
+
* See the performance table in the README.
|
|
108
|
+
*/
|
|
109
|
+
burstWhenBatchFull?: boolean;
|
|
110
|
+
/** Burst while the queue's cached ready count exceeds this. Reacts on pg-boss's stats cadence. */
|
|
111
|
+
burstWhenReadyExceeds?: number;
|
|
112
|
+
/**
|
|
113
|
+
* How many steps from one fetched batch run at a time. Default 1 (serial).
|
|
114
|
+
*
|
|
115
|
+
* Raising this only helps once the worker is not poll-bound — pair it with
|
|
116
|
+
* `burstWhenBatchFull`. Each in-flight step needs a store connection, so keep
|
|
117
|
+
* `concurrency × workers` within your Postgres pool and `max_connections`.
|
|
118
|
+
*/
|
|
119
|
+
concurrency?: number;
|
|
120
|
+
}
|
|
121
|
+
export interface PgBossStepWorkerDeps {
|
|
122
|
+
boss: PgBoss;
|
|
123
|
+
queueName: string;
|
|
124
|
+
config?: StepQueueConfig;
|
|
125
|
+
logger?: Logger;
|
|
126
|
+
workerOptions?: PgBossStepWorkerOptions;
|
|
127
|
+
}
|
|
128
|
+
/** Handles a single step job. Reconstruct the partition-scoped engine and call
|
|
129
|
+
* `engine.executeStep(workflowId, stepId)`. Throw to trigger a pg-boss retry. */
|
|
130
|
+
export type StepJobProcessor = (payload: WireStepPayload) => Promise<void>;
|
|
131
|
+
/** A pg-boss worker that drives `executeStep` for step jobs. */
|
|
132
|
+
export declare function createPgBossStepWorker(deps: PgBossStepWorkerDeps): {
|
|
133
|
+
start: (process: StepJobProcessor) => Promise<void>;
|
|
134
|
+
stop: () => Promise<void>;
|
|
135
|
+
};
|
|
136
|
+
export interface PgBossDlqWorkerDeps {
|
|
137
|
+
boss: PgBoss;
|
|
138
|
+
queueName: string;
|
|
139
|
+
logger?: Logger;
|
|
140
|
+
pollingIntervalSeconds?: number;
|
|
141
|
+
}
|
|
142
|
+
/** Invoked for a step job that exhausted all retries. Typically calls
|
|
143
|
+
* `engine.handleStepExhausted(workflowId, stepId, reason)`. */
|
|
144
|
+
export type DlqProcessor = (payload: WireStepPayload) => Promise<void>;
|
|
145
|
+
/** A pg-boss worker on the step queue's dead-letter queue. */
|
|
146
|
+
export declare function createPgBossDlqWorker(deps: PgBossDlqWorkerDeps): {
|
|
147
|
+
start: (onDlq: DlqProcessor) => Promise<void>;
|
|
148
|
+
stop: () => Promise<void>;
|
|
149
|
+
};
|
|
150
|
+
//#endregion
|
|
151
|
+
//#region src/pg-boss/scheduler.d.ts
|
|
152
|
+
/** Create the start queue (idempotent). */
|
|
153
|
+
export declare function ensureStartQueue(boss: PgBoss, queueName: string): Promise<void>;
|
|
154
|
+
export interface PgBossSchedulerDeps {
|
|
155
|
+
boss: PgBoss;
|
|
156
|
+
/** Queue scheduled start jobs are sent to (and the start worker reads). */
|
|
157
|
+
queueName: string;
|
|
158
|
+
/** Partition stamped into every scheduled start payload. */
|
|
159
|
+
partitionKey: string;
|
|
160
|
+
logger?: Logger;
|
|
161
|
+
}
|
|
162
|
+
export interface ScheduleStartInput {
|
|
163
|
+
/** Unique key for this schedule within the queue (allows many crons per queue). */
|
|
164
|
+
key: string;
|
|
165
|
+
/** Cron expression (5- or 6-field). */
|
|
166
|
+
cron: string;
|
|
167
|
+
/** Workflow type to start (the host resolves it to a definition). */
|
|
168
|
+
workflowType: string;
|
|
169
|
+
input?: Record<string, unknown>;
|
|
170
|
+
entityRef?: string;
|
|
171
|
+
/**
|
|
172
|
+
* Per-tick start idempotency. The start worker resolves this to
|
|
173
|
+
* `${idempotencyKeyPrefix}:${jobId}`, so each cron tick gets a distinct key while a
|
|
174
|
+
* redelivery of the same tick reuses it — the usual thing you want on a schedule.
|
|
175
|
+
*/
|
|
176
|
+
idempotencyKeyPrefix?: string;
|
|
177
|
+
/**
|
|
178
|
+
* Verbatim start idempotency key.
|
|
179
|
+
*
|
|
180
|
+
* A schedule stores its payload **once**, so this same key rides every tick — and
|
|
181
|
+
* start keys never expire. On a cron schedule that means **exactly one workflow,
|
|
182
|
+
* ever**; every later tick returns the first one. That is occasionally what you
|
|
183
|
+
* want ("run this once and never again"); if it isn't, use `idempotencyKeyPrefix`.
|
|
184
|
+
*/
|
|
185
|
+
idempotencyKey?: string;
|
|
186
|
+
/** Optional IANA time zone (default UTC). */
|
|
187
|
+
tz?: string;
|
|
188
|
+
}
|
|
189
|
+
/** Registers cron schedules that enqueue workflow-start jobs. */
|
|
190
|
+
export declare function createPgBossScheduler(deps: PgBossSchedulerDeps): {
|
|
191
|
+
/** Create or replace a cron schedule that starts a workflow. */
|
|
192
|
+
schedule(input: ScheduleStartInput): Promise<void>;
|
|
193
|
+
/** Remove a schedule by key. */
|
|
194
|
+
unschedule(key: string): Promise<void>;
|
|
195
|
+
/** List the schedules registered on this queue. */
|
|
196
|
+
list(): Promise<import("pg-boss").Schedule[]>;
|
|
197
|
+
};
|
|
198
|
+
/**
|
|
199
|
+
* Invoked for each start job — typically resolves the type and calls `engine.startWorkflow`.
|
|
200
|
+
*
|
|
201
|
+
* The context's `idempotencyKey` is already resolved for **this delivery** (see
|
|
202
|
+
* {@link resolveStartIdempotencyKey}), so forwarding it is the right default:
|
|
203
|
+
*
|
|
204
|
+
* ```ts
|
|
205
|
+
* await starter.start(async (job) => {
|
|
206
|
+
* const wf = workflowsByType[job.workflowType];
|
|
207
|
+
* await engine.startWorkflow(wf.definition, job.input, { idempotencyKey: job.idempotencyKey });
|
|
208
|
+
* });
|
|
209
|
+
* ```
|
|
210
|
+
*/
|
|
211
|
+
export type StartJobProcessor = (payload: StartJobContext) => Promise<void>;
|
|
212
|
+
export interface PgBossStartWorkerDeps {
|
|
213
|
+
boss: PgBoss;
|
|
214
|
+
queueName: string;
|
|
215
|
+
logger?: Logger;
|
|
216
|
+
workerOptions?: {
|
|
217
|
+
batchSize?: number;
|
|
218
|
+
pollingIntervalSeconds?: number;
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
/** A pg-boss worker on the start queue that drives a host start callback. */
|
|
222
|
+
export declare function createPgBossStartWorker(deps: PgBossStartWorkerDeps): {
|
|
223
|
+
start: (process: StartJobProcessor) => Promise<void>;
|
|
224
|
+
stop: () => Promise<void>;
|
|
225
|
+
};
|
|
226
|
+
//#endregion
|
package/dist/pg-boss/index.js
CHANGED
|
@@ -1,2 +1,411 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
//#region src/pg-boss/payload.ts
|
|
3
|
+
/**
|
|
4
|
+
* The wire payload for a step job. It is the flow-core `DispatchStepPayload`
|
|
5
|
+
* plus the `partitionKey` — the worker needs the partition to reconstruct a
|
|
6
|
+
* partition-scoped engine before handing the job to `engine.handleStepJob`.
|
|
7
|
+
*/
|
|
8
|
+
const WIRE_STEP_PAYLOAD_SCHEMA = z.object({
|
|
9
|
+
partitionKey: z.string().min(1),
|
|
10
|
+
workflowId: z.number().int().positive(),
|
|
11
|
+
stepId: z.number().int().positive(),
|
|
12
|
+
stepKey: z.string().min(1),
|
|
13
|
+
stepType: z.string().min(1),
|
|
14
|
+
/**
|
|
15
|
+
* What the job asks for: run the step, or settle it because its wait deadline
|
|
16
|
+
* elapsed. Defaulted rather than required, so a job enqueued before wait
|
|
17
|
+
* deadlines existed still parses — and still means "run the step".
|
|
18
|
+
*/
|
|
19
|
+
kind: z.enum(["execute", "timeout"]).default("execute")
|
|
20
|
+
});
|
|
21
|
+
const DEFAULT_STEP_QUEUE_CONFIG = {
|
|
22
|
+
retryLimit: 2,
|
|
23
|
+
retryDelay: 30,
|
|
24
|
+
expireInSeconds: 600
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* The wire payload for a scheduled (or ad-hoc) workflow **start**. Carries the
|
|
28
|
+
* partition plus what `engine.startWorkflow` needs; the host resolves `workflowType`
|
|
29
|
+
* to a definition.
|
|
30
|
+
*
|
|
31
|
+
* Two mutually exclusive ways to ask for start idempotency, because a cron schedule
|
|
32
|
+
* stores its payload **once** and pg-boss redelivers that same payload on every tick:
|
|
33
|
+
*
|
|
34
|
+
* - `idempotencyKeyPrefix` — the usual choice. The start worker appends the job id,
|
|
35
|
+
* so each tick gets a distinct key while a redelivery of the *same* tick reuses it.
|
|
36
|
+
* - `idempotencyKey` — used verbatim. On a schedule this means "start exactly one
|
|
37
|
+
* workflow, ever"; every later tick returns the first one.
|
|
38
|
+
*/
|
|
39
|
+
const WIRE_START_PAYLOAD_SCHEMA = z.object({
|
|
40
|
+
partitionKey: z.string().min(1),
|
|
41
|
+
workflowType: z.string().min(1),
|
|
42
|
+
input: z.record(z.string(), z.unknown()).default({}),
|
|
43
|
+
entityRef: z.string().optional(),
|
|
44
|
+
/** Verbatim dedup key — collapses *every* delivery, including future cron ticks. */
|
|
45
|
+
idempotencyKey: z.string().optional(),
|
|
46
|
+
/** Per-tick dedup: the start worker resolves this to `${prefix}:${jobId}`. */
|
|
47
|
+
idempotencyKeyPrefix: z.string().optional()
|
|
48
|
+
});
|
|
49
|
+
/** Resolve the per-delivery idempotency key. Explicit key wins over a prefix. */
|
|
50
|
+
function resolveStartIdempotencyKey(payload, jobId) {
|
|
51
|
+
if (payload.idempotencyKey !== void 0) return payload.idempotencyKey;
|
|
52
|
+
if (payload.idempotencyKeyPrefix !== void 0) return `${payload.idempotencyKeyPrefix}:${jobId}`;
|
|
53
|
+
}
|
|
54
|
+
//#endregion
|
|
55
|
+
//#region src/pg-boss/queue.ts
|
|
56
|
+
function dlqName(queueName) {
|
|
57
|
+
return `${queueName}-dlq`;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Create the step queue and its dead-letter queue if they do not exist. Call
|
|
61
|
+
* once at startup; the dispatcher also runs it from `prepare` and lazily on a
|
|
62
|
+
* plain first enqueue.
|
|
63
|
+
*
|
|
64
|
+
* Only what is missing is created: pg-boss's `createQueue` evicts the queue's
|
|
65
|
+
* entry from its per-instance cache, and a `send` that misses that cache reads
|
|
66
|
+
* the queue on pg-boss's *own* connection — inside a store transaction on a
|
|
67
|
+
* single-connection database that is a deadlock. An existing queue is left
|
|
68
|
+
* alone, cache entry and all.
|
|
69
|
+
*/
|
|
70
|
+
async function ensureStepQueue(boss, queueName, config = {}) {
|
|
71
|
+
const cfg = {
|
|
72
|
+
...DEFAULT_STEP_QUEUE_CONFIG,
|
|
73
|
+
...config
|
|
74
|
+
};
|
|
75
|
+
const dlq = dlqName(queueName);
|
|
76
|
+
const swallowExists = (error) => {
|
|
77
|
+
if (error instanceof Error && error.message.includes("already exists")) return;
|
|
78
|
+
throw error;
|
|
79
|
+
};
|
|
80
|
+
if (!await boss.getQueue(dlq)) try {
|
|
81
|
+
await boss.createQueue(dlq, { retryLimit: 0 });
|
|
82
|
+
} catch (e) {
|
|
83
|
+
swallowExists(e);
|
|
84
|
+
}
|
|
85
|
+
if (!await boss.getQueue(queueName)) try {
|
|
86
|
+
await boss.createQueue(queueName, {
|
|
87
|
+
retryLimit: cfg.retryLimit,
|
|
88
|
+
retryDelay: cfg.retryDelay,
|
|
89
|
+
expireInSeconds: cfg.expireInSeconds,
|
|
90
|
+
deadLetter: dlq
|
|
91
|
+
});
|
|
92
|
+
} catch (e) {
|
|
93
|
+
swallowExists(e);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/** A job id that cannot exist — see {@link fillQueueCache}. */
|
|
97
|
+
const NIL_JOB_ID = "00000000-0000-0000-0000-000000000000";
|
|
98
|
+
/**
|
|
99
|
+
* pg-boss keeps a per-instance cache of queue rows and consults it on every
|
|
100
|
+
* `send` — on a miss it reads the queue on its *own* connection, whatever `db`
|
|
101
|
+
* the send was handed. `createQueue` evicts the entry rather than filling it,
|
|
102
|
+
* and the `PgBoss` facade keeps the cache private. So after creating our queue
|
|
103
|
+
* we fill the entry here, outside any transaction, through the one public call
|
|
104
|
+
* that reads the queue via the cache and then changes nothing: cancelling a job
|
|
105
|
+
* id that cannot exist. (A boss that does expose `getQueueCache` — a bare
|
|
106
|
+
* manager — is asked directly.) Without this the first transactional send would
|
|
107
|
+
* read on a second connection — a deadlock on a single-connection database.
|
|
108
|
+
*/
|
|
109
|
+
async function fillQueueCache(boss, queueName) {
|
|
110
|
+
const cache = boss.getQueueCache;
|
|
111
|
+
if (typeof cache === "function") await cache.call(boss, queueName);
|
|
112
|
+
else await boss.cancel(queueName, NIL_JOB_ID);
|
|
113
|
+
}
|
|
114
|
+
/** A flow-core `Dispatcher` backed by pg-boss. */
|
|
115
|
+
function createPgBossDispatcher(deps) {
|
|
116
|
+
const { boss, queueName, partitionKey } = deps;
|
|
117
|
+
const cfg = {
|
|
118
|
+
...DEFAULT_STEP_QUEUE_CONFIG,
|
|
119
|
+
...deps.config
|
|
120
|
+
};
|
|
121
|
+
/**
|
|
122
|
+
* Queue DDL and cache warm, once. Memoised as a promise so concurrent first
|
|
123
|
+
* sends share it; cleared on failure so the next attempt retries. The engine
|
|
124
|
+
* calls this through `prepare` before its first transaction; a direct
|
|
125
|
+
* `enqueueStep` outside any transaction is free to trigger it lazily.
|
|
126
|
+
*/
|
|
127
|
+
let prepared = null;
|
|
128
|
+
function ensurePrepared() {
|
|
129
|
+
prepared ??= (async () => {
|
|
130
|
+
await ensureStepQueue(boss, queueName, cfg);
|
|
131
|
+
await fillQueueCache(boss, queueName);
|
|
132
|
+
})().catch((error) => {
|
|
133
|
+
prepared = null;
|
|
134
|
+
throw error;
|
|
135
|
+
});
|
|
136
|
+
return prepared;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Adapt flow's `SqlExecutor` to the shape pg-boss wants for a caller-supplied
|
|
140
|
+
* connection. The two are the same idea under different names.
|
|
141
|
+
*/
|
|
142
|
+
function asPgBossDb(handle) {
|
|
143
|
+
const exec = handle;
|
|
144
|
+
if (!exec || typeof exec.query !== "function") return void 0;
|
|
145
|
+
return { executeSql: (text, values) => exec.query(text, values) };
|
|
146
|
+
}
|
|
147
|
+
async function send(payload, options, handle) {
|
|
148
|
+
try {
|
|
149
|
+
const wire = {
|
|
150
|
+
kind: "execute",
|
|
151
|
+
...payload,
|
|
152
|
+
partitionKey
|
|
153
|
+
};
|
|
154
|
+
const parsed = WIRE_STEP_PAYLOAD_SCHEMA.safeParse(wire);
|
|
155
|
+
if (!parsed.success) return {
|
|
156
|
+
ok: false,
|
|
157
|
+
error: {
|
|
158
|
+
key: "queue_error",
|
|
159
|
+
message: `Invalid step payload: ${parsed.error.message}`
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
await ensurePrepared();
|
|
163
|
+
const startAfter = options?.startAfterSeconds;
|
|
164
|
+
const db = handle === void 0 ? void 0 : asPgBossDb(handle);
|
|
165
|
+
if (handle !== void 0 && !db) return {
|
|
166
|
+
ok: false,
|
|
167
|
+
error: {
|
|
168
|
+
key: "queue_error",
|
|
169
|
+
message: "Transaction handle is not a SqlExecutor — is the store octaflow/postgres?"
|
|
170
|
+
}
|
|
171
|
+
};
|
|
172
|
+
if (!await boss.send(queueName, parsed.data, {
|
|
173
|
+
retryLimit: cfg.retryLimit,
|
|
174
|
+
retryDelay: cfg.retryDelay,
|
|
175
|
+
expireInSeconds: cfg.expireInSeconds,
|
|
176
|
+
...startAfter != null && startAfter > 0 ? { startAfter } : {},
|
|
177
|
+
...db ? { db } : {}
|
|
178
|
+
})) return {
|
|
179
|
+
ok: false,
|
|
180
|
+
error: {
|
|
181
|
+
key: "queue_error",
|
|
182
|
+
message: `Failed to enqueue job to ${queueName}`
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
return {
|
|
186
|
+
ok: true,
|
|
187
|
+
value: void 0
|
|
188
|
+
};
|
|
189
|
+
} catch (error) {
|
|
190
|
+
return {
|
|
191
|
+
ok: false,
|
|
192
|
+
error: {
|
|
193
|
+
key: "queue_error",
|
|
194
|
+
message: error instanceof Error ? error.message : "Unknown enqueue error"
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return {
|
|
200
|
+
/** Queue DDL + cache fill, outside any transaction. See `Dispatcher.prepare`. */
|
|
201
|
+
async prepare() {
|
|
202
|
+
try {
|
|
203
|
+
await ensurePrepared();
|
|
204
|
+
return {
|
|
205
|
+
ok: true,
|
|
206
|
+
value: void 0
|
|
207
|
+
};
|
|
208
|
+
} catch (error) {
|
|
209
|
+
return {
|
|
210
|
+
ok: false,
|
|
211
|
+
error: {
|
|
212
|
+
key: "queue_error",
|
|
213
|
+
message: error instanceof Error ? error.message : "Failed to prepare step queue"
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
},
|
|
218
|
+
enqueueStep: (payload, options) => send(payload, options),
|
|
219
|
+
/**
|
|
220
|
+
* Enqueue on the store's open transaction, so the job and the state change
|
|
221
|
+
* that produced it commit together. `handle` is the transaction-bound
|
|
222
|
+
* `SqlExecutor` that `createWorkflowStore.runInTransaction` hands out.
|
|
223
|
+
*/
|
|
224
|
+
enqueueStepIn: (handle, payload, options) => send(payload, options, handle)
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
/** A pg-boss worker that drives `executeStep` for step jobs. */
|
|
228
|
+
function createPgBossStepWorker(deps) {
|
|
229
|
+
const { boss, queueName } = deps;
|
|
230
|
+
let workerId = null;
|
|
231
|
+
async function start(process) {
|
|
232
|
+
await ensureStepQueue(boss, queueName, deps.config);
|
|
233
|
+
const opts = deps.workerOptions ?? {};
|
|
234
|
+
const lanes = Math.max(1, opts.concurrency ?? 1);
|
|
235
|
+
const workOpts = {
|
|
236
|
+
batchSize: opts.batchSize ?? 1,
|
|
237
|
+
perJobResults: true,
|
|
238
|
+
...opts.pollingIntervalSeconds != null && { pollingIntervalSeconds: opts.pollingIntervalSeconds },
|
|
239
|
+
...opts.notifyPollingIntervalSeconds != null && { notifyPollingIntervalSeconds: opts.notifyPollingIntervalSeconds },
|
|
240
|
+
...opts.burstWhenBatchFull != null && { burstWhenBatchFull: opts.burstWhenBatchFull },
|
|
241
|
+
...opts.burstWhenReadyExceeds != null && { burstWhenReadyExceeds: opts.burstWhenReadyExceeds }
|
|
242
|
+
};
|
|
243
|
+
workerId = await boss.work(queueName, workOpts, async (jobs) => {
|
|
244
|
+
const results = [];
|
|
245
|
+
const pending = [...jobs];
|
|
246
|
+
await Promise.all(Array.from({ length: Math.min(lanes, pending.length) }, async () => {
|
|
247
|
+
for (;;) {
|
|
248
|
+
const job = pending.shift();
|
|
249
|
+
if (!job) break;
|
|
250
|
+
const parsed = WIRE_STEP_PAYLOAD_SCHEMA.safeParse(job.data);
|
|
251
|
+
if (!parsed.success) {
|
|
252
|
+
results.push({
|
|
253
|
+
id: job.id,
|
|
254
|
+
status: "deadletter",
|
|
255
|
+
output: { message: `Invalid step payload for job ${job.id}: ${parsed.error.message}` }
|
|
256
|
+
});
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
try {
|
|
260
|
+
await process(parsed.data);
|
|
261
|
+
results.push({
|
|
262
|
+
id: job.id,
|
|
263
|
+
status: "completed"
|
|
264
|
+
});
|
|
265
|
+
} catch (error) {
|
|
266
|
+
results.push({
|
|
267
|
+
id: job.id,
|
|
268
|
+
status: "failed",
|
|
269
|
+
output: { message: error instanceof Error ? error.message : String(error) }
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}));
|
|
274
|
+
return results;
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
async function stop() {
|
|
278
|
+
if (workerId) {
|
|
279
|
+
await boss.offWork(workerId);
|
|
280
|
+
workerId = null;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return {
|
|
284
|
+
start,
|
|
285
|
+
stop
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
/** A pg-boss worker on the step queue's dead-letter queue. */
|
|
289
|
+
function createPgBossDlqWorker(deps) {
|
|
290
|
+
const { boss, queueName, logger } = deps;
|
|
291
|
+
const dlq = dlqName(queueName);
|
|
292
|
+
let workerId = null;
|
|
293
|
+
async function start(onDlq) {
|
|
294
|
+
workerId = await boss.work(dlq, { pollingIntervalSeconds: deps.pollingIntervalSeconds ?? 30 }, async (jobs) => {
|
|
295
|
+
for (const job of jobs) {
|
|
296
|
+
const parsed = WIRE_STEP_PAYLOAD_SCHEMA.safeParse(job.data);
|
|
297
|
+
if (!parsed.success) {
|
|
298
|
+
logger?.error("Invalid payload in DLQ", void 0, { jobId: job.id });
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
logger?.error("Step job dead-lettered", void 0, {
|
|
302
|
+
jobId: job.id,
|
|
303
|
+
workflowId: parsed.data.workflowId,
|
|
304
|
+
stepId: parsed.data.stepId,
|
|
305
|
+
stepKey: parsed.data.stepKey
|
|
306
|
+
});
|
|
307
|
+
try {
|
|
308
|
+
await onDlq(parsed.data);
|
|
309
|
+
} catch (error) {
|
|
310
|
+
logger?.error("DLQ handler failed", error instanceof Error ? error : void 0, { jobId: job.id });
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
async function stop() {
|
|
316
|
+
if (workerId) {
|
|
317
|
+
await boss.offWork(workerId);
|
|
318
|
+
workerId = null;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
return {
|
|
322
|
+
start,
|
|
323
|
+
stop
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
//#endregion
|
|
327
|
+
//#region src/pg-boss/scheduler.ts
|
|
328
|
+
/** Create the start queue (idempotent). */
|
|
329
|
+
async function ensureStartQueue(boss, queueName) {
|
|
330
|
+
try {
|
|
331
|
+
await boss.createQueue(queueName);
|
|
332
|
+
} catch (e) {
|
|
333
|
+
if (e instanceof Error && e.message.includes("already exists")) return;
|
|
334
|
+
throw e;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
/** Registers cron schedules that enqueue workflow-start jobs. */
|
|
338
|
+
function createPgBossScheduler(deps) {
|
|
339
|
+
const { boss, queueName, partitionKey } = deps;
|
|
340
|
+
let ensured = false;
|
|
341
|
+
async function ensure() {
|
|
342
|
+
if (!ensured) {
|
|
343
|
+
await ensureStartQueue(boss, queueName);
|
|
344
|
+
ensured = true;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
return {
|
|
348
|
+
/** Create or replace a cron schedule that starts a workflow. */
|
|
349
|
+
async schedule(input) {
|
|
350
|
+
await ensure();
|
|
351
|
+
const payload = {
|
|
352
|
+
partitionKey,
|
|
353
|
+
workflowType: input.workflowType,
|
|
354
|
+
input: input.input ?? {},
|
|
355
|
+
entityRef: input.entityRef,
|
|
356
|
+
idempotencyKey: input.idempotencyKey,
|
|
357
|
+
idempotencyKeyPrefix: input.idempotencyKeyPrefix
|
|
358
|
+
};
|
|
359
|
+
await boss.schedule(queueName, input.cron, payload, {
|
|
360
|
+
key: input.key,
|
|
361
|
+
...input.tz ? { tz: input.tz } : {}
|
|
362
|
+
});
|
|
363
|
+
},
|
|
364
|
+
/** Remove a schedule by key. */
|
|
365
|
+
async unschedule(key) {
|
|
366
|
+
await boss.unschedule(queueName, key);
|
|
367
|
+
},
|
|
368
|
+
/** List the schedules registered on this queue. */
|
|
369
|
+
async list() {
|
|
370
|
+
return (await boss.getSchedules()).filter((s) => s.name === queueName);
|
|
371
|
+
}
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
/** A pg-boss worker on the start queue that drives a host start callback. */
|
|
375
|
+
function createPgBossStartWorker(deps) {
|
|
376
|
+
const { boss, queueName, logger } = deps;
|
|
377
|
+
let workerId = null;
|
|
378
|
+
async function start(process) {
|
|
379
|
+
await ensureStartQueue(boss, queueName);
|
|
380
|
+
const workOpts = {
|
|
381
|
+
batchSize: deps.workerOptions?.batchSize ?? 1,
|
|
382
|
+
...deps.workerOptions?.pollingIntervalSeconds != null && { pollingIntervalSeconds: deps.workerOptions.pollingIntervalSeconds }
|
|
383
|
+
};
|
|
384
|
+
workerId = await boss.work(queueName, workOpts, async (jobs) => {
|
|
385
|
+
for (const job of jobs) {
|
|
386
|
+
const parsed = WIRE_START_PAYLOAD_SCHEMA.safeParse(job.data);
|
|
387
|
+
if (!parsed.success) {
|
|
388
|
+
logger?.error("Invalid start payload", void 0, { jobId: job.id });
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
await process({
|
|
392
|
+
...parsed.data,
|
|
393
|
+
jobId: job.id,
|
|
394
|
+
idempotencyKey: resolveStartIdempotencyKey(parsed.data, job.id)
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
async function stop() {
|
|
400
|
+
if (workerId) {
|
|
401
|
+
await boss.offWork(workerId);
|
|
402
|
+
workerId = null;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
return {
|
|
406
|
+
start,
|
|
407
|
+
stop
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
//#endregion
|
|
2
411
|
export { DEFAULT_STEP_QUEUE_CONFIG, WIRE_START_PAYLOAD_SCHEMA, WIRE_STEP_PAYLOAD_SCHEMA, createPgBossDispatcher, createPgBossDlqWorker, createPgBossScheduler, createPgBossStartWorker, createPgBossStepWorker, ensureStartQueue, ensureStepQueue, resolveStartIdempotencyKey };
|