bunqueue 2.8.49 → 2.8.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -9
- package/dist/application/contextFactory.d.ts +4 -2
- package/dist/application/contextFactory.js +3 -0
- package/dist/application/flowFailureRecovery.d.ts +17 -0
- package/dist/application/flowFailureRecovery.js +92 -0
- package/dist/application/operations/ack.d.ts +3 -2
- package/dist/application/operations/ack.js +34 -8
- package/dist/application/operations/flowPush.d.ts +9 -0
- package/dist/application/operations/flowPush.js +112 -0
- package/dist/application/operations/flowTopologyValidation.d.ts +3 -0
- package/dist/application/operations/flowTopologyValidation.js +101 -0
- package/dist/application/operations/flowValidation.d.ts +3 -0
- package/dist/application/operations/flowValidation.js +166 -0
- package/dist/application/operations/push.d.ts +3 -0
- package/dist/application/operations/push.js +2 -2
- package/dist/application/operations/pushInsert.d.ts +7 -1
- package/dist/application/operations/pushInsert.js +18 -8
- package/dist/application/operations/pushLocks.d.ts +1 -0
- package/dist/application/operations/pushLocks.js +5 -1
- package/dist/application/operations/queryOperations.js +7 -6
- package/dist/application/queueManager.d.ts +5 -0
- package/dist/application/queueManager.js +195 -60
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/client/flow.d.ts +3 -6
- package/dist/client/flow.js +71 -210
- package/dist/client/flowAtomic.d.ts +9 -0
- package/dist/client/flowAtomic.js +24 -0
- package/dist/client/flowJobCoreMethods.d.ts +28 -0
- package/dist/client/flowJobCoreMethods.js +151 -0
- package/dist/client/flowJobDependencies.d.ts +7 -0
- package/dist/client/flowJobDependencies.js +64 -0
- package/dist/client/flowJobFactory.d.ts +11 -24
- package/dist/client/flowJobFactory.js +91 -360
- package/dist/client/flowJobMoveMethods.d.ts +16 -0
- package/dist/client/flowJobMoveMethods.js +112 -0
- package/dist/client/flowJobTypes.d.ts +29 -0
- package/dist/client/flowJobTypes.js +6 -0
- package/dist/client/flowLegacyPlan.d.ts +16 -0
- package/dist/client/flowLegacyPlan.js +118 -0
- package/dist/client/flowOptions.d.ts +10 -0
- package/dist/client/flowOptions.js +44 -0
- package/dist/client/flowPlan.d.ts +16 -0
- package/dist/client/flowPlan.js +93 -0
- package/dist/client/flowPush.js +4 -1
- package/dist/client/flowReader.d.ts +10 -0
- package/dist/client/flowReader.js +147 -0
- package/dist/client/jobHelpers.js +2 -0
- package/dist/client/workflow/clock.d.ts +3 -1
- package/dist/client/workflow/clock.js +18 -1
- package/dist/client/workflow/compensationChild.d.ts +30 -0
- package/dist/client/workflow/compensationChild.js +65 -0
- package/dist/client/workflow/compensationPass.d.ts +16 -0
- package/dist/client/workflow/compensationPass.js +96 -0
- package/dist/client/workflow/compensationSupport.d.ts +24 -0
- package/dist/client/workflow/compensationSupport.js +57 -0
- package/dist/client/workflow/compensator.d.ts +11 -50
- package/dist/client/workflow/compensator.js +22 -374
- package/dist/client/workflow/definitionGuard.d.ts +11 -0
- package/dist/client/workflow/definitionGuard.js +22 -0
- package/dist/client/workflow/engine.d.ts +2 -2
- package/dist/client/workflow/engine.js +8 -8
- package/dist/client/workflow/eventTypes.d.ts +44 -0
- package/dist/client/workflow/eventTypes.js +2 -0
- package/dist/client/workflow/executionTypes.d.ts +133 -0
- package/dist/client/workflow/executionTypes.js +2 -0
- package/dist/client/workflow/executor.d.ts +4 -17
- package/dist/client/workflow/executor.js +50 -208
- package/dist/client/workflow/executorLifecycle.d.ts +18 -0
- package/dist/client/workflow/executorLifecycle.js +69 -0
- package/dist/client/workflow/executorNodes.d.ts +17 -0
- package/dist/client/workflow/executorNodes.js +138 -0
- package/dist/client/workflow/identity.d.ts +2 -0
- package/dist/client/workflow/identity.js +8 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.d.ts +3 -5
- package/dist/client/workflow/loops.js +25 -23
- package/dist/client/workflow/mapRunner.d.ts +4 -0
- package/dist/client/workflow/mapRunner.js +45 -0
- package/dist/client/workflow/recovery.js +10 -3
- package/dist/client/workflow/runner.d.ts +2 -21
- package/dist/client/workflow/runner.js +39 -104
- package/dist/client/workflow/runnerTiming.d.ts +14 -0
- package/dist/client/workflow/runnerTiming.js +66 -0
- package/dist/client/workflow/stepTypes.d.ts +171 -0
- package/dist/client/workflow/stepTypes.js +3 -0
- package/dist/client/workflow/store.d.ts +17 -17
- package/dist/client/workflow/store.js +65 -100
- package/dist/client/workflow/storeExecutionCodec.d.ts +11 -0
- package/dist/client/workflow/storeExecutionCodec.js +34 -0
- package/dist/client/workflow/storeListing.d.ts +11 -0
- package/dist/client/workflow/storeListing.js +45 -0
- package/dist/client/workflow/storeMaintenance.d.ts +4 -0
- package/dist/client/workflow/storeMaintenance.js +40 -0
- package/dist/client/workflow/storeSignals.d.ts +9 -0
- package/dist/client/workflow/storeSignals.js +38 -2
- package/dist/client/workflow/subWorkflowRunner.d.ts +13 -0
- package/dist/client/workflow/subWorkflowRunner.js +40 -0
- package/dist/client/workflow/types.d.ts +4 -356
- package/dist/client/workflow/types.js +1 -3
- package/dist/client/workflow/waitFor.js +39 -26
- package/dist/client/workflow/workflow.d.ts +16 -59
- package/dist/client/workflow/workflow.js +53 -179
- package/dist/client/workflow/workflowDecisions.d.ts +11 -0
- package/dist/client/workflow/workflowDecisions.js +27 -0
- package/dist/client/workflow/workflowDefinition.d.ts +16 -0
- package/dist/client/workflow/workflowDefinition.js +123 -0
- package/dist/client/workflow/workflowIntrospection.d.ts +5 -0
- package/dist/client/workflow/workflowIntrospection.js +46 -0
- package/dist/client/workflow/workflowValidation.d.ts +44 -0
- package/dist/client/workflow/workflowValidation.js +143 -0
- package/dist/domain/types/command.d.ts +7 -1
- package/dist/domain/types/flow.d.ts +25 -0
- package/dist/domain/types/flow.js +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +52 -3
- package/dist/infrastructure/persistence/sqlite.d.ts +16 -0
- package/dist/infrastructure/persistence/sqlite.js +129 -7
- package/dist/infrastructure/persistence/sqliteBatch.js +10 -4
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +2 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +18 -5
- package/dist/infrastructure/persistence/statements.d.ts +4 -0
- package/dist/infrastructure/persistence/statements.js +8 -2
- package/dist/infrastructure/server/handlerRoutes.js +3 -0
- package/dist/infrastructure/server/handlers/advanced.js +2 -2
- package/dist/infrastructure/server/handlers/flow.d.ts +7 -0
- package/dist/infrastructure/server/handlers/flow.js +11 -0
- package/dist/infrastructure/server/handlers/index.d.ts +1 -0
- package/dist/infrastructure/server/handlers/index.js +1 -0
- package/package.json +4 -2
package/dist/client/flow.js
CHANGED
|
@@ -6,9 +6,13 @@ import { EventEmitter } from 'events';
|
|
|
6
6
|
import { getSharedManager } from './manager';
|
|
7
7
|
import { TcpConnectionPool, getSharedPool, releaseSharedPool } from './tcpPool';
|
|
8
8
|
import { jobId } from '../domain/types/job';
|
|
9
|
-
import { createFlowJobObject
|
|
10
|
-
import { pushJob, pushJobWithParent, cleanupJobs } from './flowPush';
|
|
9
|
+
import { createFlowJobObject } from './flowJobFactory';
|
|
11
10
|
import * as managementOps from './queue/operations/management';
|
|
11
|
+
import { commitFlow } from './flowAtomic';
|
|
12
|
+
import { assertFlowTcpOk } from './flowJobTypes';
|
|
13
|
+
import { planFlows } from './flowPlan';
|
|
14
|
+
import { planBulkThen, planChain, planTree } from './flowLegacyPlan';
|
|
15
|
+
import { readFlow } from './flowReader';
|
|
12
16
|
const FORCE_EMBEDDED = Bun.env.BUNQUEUE_EMBEDDED === '1';
|
|
13
17
|
/**
|
|
14
18
|
* FlowProducer creates job flows with automatic dependencies.
|
|
@@ -39,6 +43,7 @@ export class FlowProducer extends EventEmitter {
|
|
|
39
43
|
embedded;
|
|
40
44
|
tcp;
|
|
41
45
|
useSharedPool;
|
|
46
|
+
closed = false;
|
|
42
47
|
constructor(opts = {}) {
|
|
43
48
|
super();
|
|
44
49
|
this.embedded = opts.embedded ?? FORCE_EMBEDDED;
|
|
@@ -56,6 +61,8 @@ export class FlowProducer extends EventEmitter {
|
|
|
56
61
|
poolSize,
|
|
57
62
|
pingInterval: connOpts.pingInterval,
|
|
58
63
|
commandTimeout: connOpts.commandTimeout,
|
|
64
|
+
maxCommandTimeouts: connOpts.maxCommandTimeouts,
|
|
65
|
+
tls: connOpts.tls,
|
|
59
66
|
pipelining: connOpts.pipelining,
|
|
60
67
|
maxInFlight: connOpts.maxInFlight,
|
|
61
68
|
});
|
|
@@ -69,6 +76,8 @@ export class FlowProducer extends EventEmitter {
|
|
|
69
76
|
poolSize,
|
|
70
77
|
pingInterval: connOpts.pingInterval,
|
|
71
78
|
commandTimeout: connOpts.commandTimeout,
|
|
79
|
+
maxCommandTimeouts: connOpts.maxCommandTimeouts,
|
|
80
|
+
tls: connOpts.tls,
|
|
72
81
|
pipelining: connOpts.pipelining,
|
|
73
82
|
maxInFlight: connOpts.maxInFlight,
|
|
74
83
|
});
|
|
@@ -82,6 +91,9 @@ export class FlowProducer extends EventEmitter {
|
|
|
82
91
|
}
|
|
83
92
|
/** Close the connection pool (only if using dedicated pool) */
|
|
84
93
|
close() {
|
|
94
|
+
if (this.closed)
|
|
95
|
+
return this.closing;
|
|
96
|
+
this.closed = true;
|
|
85
97
|
if (this.tcp && !this.useSharedPool) {
|
|
86
98
|
this.tcp.close();
|
|
87
99
|
}
|
|
@@ -100,49 +112,31 @@ export class FlowProducer extends EventEmitter {
|
|
|
100
112
|
if (this.embedded)
|
|
101
113
|
return;
|
|
102
114
|
if (this.tcp)
|
|
103
|
-
await this.tcp.send({ cmd: 'Ping' });
|
|
115
|
+
assertFlowTcpOk(await this.tcp.send({ cmd: 'Ping' }), 'Ping');
|
|
104
116
|
}
|
|
105
117
|
// ============================================================================
|
|
106
118
|
// BullMQ v5 Compatible Methods
|
|
107
119
|
// ============================================================================
|
|
108
120
|
/** Add a flow (BullMQ v5 compatible). Children are processed BEFORE their parent. */
|
|
109
121
|
async add(flow, opts) {
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
}
|
|
114
|
-
catch (error) {
|
|
115
|
-
// Atomic rollback: clean up all created jobs
|
|
116
|
-
await cleanupJobs(this.pushCtx, createdJobIds);
|
|
117
|
-
throw error;
|
|
118
|
-
}
|
|
122
|
+
const plan = planFlows([flow], opts);
|
|
123
|
+
const snapshots = await commitFlow(this.pushCtx, plan.batch);
|
|
124
|
+
return this.buildPlannedNode(plan.roots[0], this.indexSnapshots(snapshots));
|
|
119
125
|
}
|
|
120
126
|
/** Add multiple flows (BullMQ v5 compatible). */
|
|
121
127
|
async addBulk(flows) {
|
|
122
|
-
const
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
const createdJobIds = [];
|
|
127
|
-
const result = await this.addFlowNode(flow, null, createdJobIds);
|
|
128
|
-
allCreatedJobIds.push(...createdJobIds);
|
|
129
|
-
results.push(result);
|
|
130
|
-
}
|
|
131
|
-
return results;
|
|
132
|
-
}
|
|
133
|
-
catch (error) {
|
|
134
|
-
// Atomic rollback: clean up all created jobs from all flows
|
|
135
|
-
await cleanupJobs(this.pushCtx, allCreatedJobIds);
|
|
136
|
-
throw error;
|
|
137
|
-
}
|
|
128
|
+
const plan = planFlows(flows);
|
|
129
|
+
const snapshots = await commitFlow(this.pushCtx, plan.batch);
|
|
130
|
+
const snapshotsById = this.indexSnapshots(snapshots);
|
|
131
|
+
return plan.roots.map((root) => this.buildPlannedNode(root, snapshotsById));
|
|
138
132
|
}
|
|
139
133
|
/** Get a flow tree starting from a job (BullMQ v5 compatible). */
|
|
140
134
|
async getFlow(opts) {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
135
|
+
return readFlow({
|
|
136
|
+
embedded: this.embedded,
|
|
137
|
+
tcp: this.tcp,
|
|
138
|
+
buildCallbacks: (queueName) => this.buildCallbacks(queueName),
|
|
139
|
+
}, opts);
|
|
146
140
|
}
|
|
147
141
|
// ============================================================================
|
|
148
142
|
// Legacy bunqueue API Methods
|
|
@@ -151,80 +145,24 @@ export class FlowProducer extends EventEmitter {
|
|
|
151
145
|
async addChain(steps) {
|
|
152
146
|
if (steps.length === 0)
|
|
153
147
|
return { jobIds: [] };
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
for (const step of steps) {
|
|
158
|
-
const data = { name: step.name, __flowParentId: prevId, ...step.data };
|
|
159
|
-
const id = await pushJob(this.pushCtx, step.queueName, data, step.opts ?? {}, prevId ? [prevId] : undefined);
|
|
160
|
-
jobIds.push(id);
|
|
161
|
-
prevId = id;
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
catch (error) {
|
|
165
|
-
await cleanupJobs(this.pushCtx, jobIds);
|
|
166
|
-
throw error;
|
|
167
|
-
}
|
|
168
|
-
return { jobIds };
|
|
148
|
+
const plan = planChain(steps);
|
|
149
|
+
await commitFlow(this.pushCtx, plan.batch);
|
|
150
|
+
return { jobIds: plan.ids.map(String) };
|
|
169
151
|
}
|
|
170
152
|
/** Add parallel jobs that converge to a final job. */
|
|
171
153
|
async addBulkThen(parallel, final) {
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
}));
|
|
179
|
-
// Collect successful IDs and check for failures
|
|
180
|
-
const errors = [];
|
|
181
|
-
for (const r of results) {
|
|
182
|
-
if (r.status === 'fulfilled') {
|
|
183
|
-
parallelIds.push(r.value);
|
|
184
|
-
}
|
|
185
|
-
else {
|
|
186
|
-
errors.push(r.reason);
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
if (errors.length > 0) {
|
|
190
|
-
// Clean up successfully created jobs before throwing
|
|
191
|
-
await cleanupJobs(this.pushCtx, parallelIds);
|
|
192
|
-
throw errors[0] instanceof Error ? errors[0] : new Error(String(errors[0]));
|
|
193
|
-
}
|
|
194
|
-
const finalData = {
|
|
195
|
-
name: final.name,
|
|
196
|
-
__flowParentIds: parallelIds,
|
|
197
|
-
...final.data,
|
|
198
|
-
};
|
|
199
|
-
// pushJobWithParent (not pushJob) so the final job records childrenIds =
|
|
200
|
-
// parallelIds: this keeps the same dependsOn ordering (waits for all
|
|
201
|
-
// parallel jobs) AND lets the merge step read their results via
|
|
202
|
-
// getChildrenValues()/getDependencies(), like a BullMQ fan-in parent.
|
|
203
|
-
const finalId = await pushJobWithParent(this.pushCtx, {
|
|
204
|
-
queueName: final.queueName,
|
|
205
|
-
data: finalData,
|
|
206
|
-
opts: final.opts ?? {},
|
|
207
|
-
parentRef: null,
|
|
208
|
-
childIds: parallelIds,
|
|
209
|
-
});
|
|
210
|
-
return { parallelIds, finalId };
|
|
211
|
-
}
|
|
212
|
-
catch (error) {
|
|
213
|
-
await cleanupJobs(this.pushCtx, parallelIds);
|
|
214
|
-
throw error;
|
|
215
|
-
}
|
|
154
|
+
const plan = planBulkThen(parallel, final);
|
|
155
|
+
await commitFlow(this.pushCtx, plan.batch);
|
|
156
|
+
return {
|
|
157
|
+
parallelIds: plan.parallelIds.map(String),
|
|
158
|
+
finalId: String(plan.finalId),
|
|
159
|
+
};
|
|
216
160
|
}
|
|
217
161
|
/** Add a tree of jobs where children depend on parent. */
|
|
218
162
|
async addTree(root) {
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
}
|
|
223
|
-
catch (error) {
|
|
224
|
-
await cleanupJobs(this.pushCtx, jobIds);
|
|
225
|
-
throw error;
|
|
226
|
-
}
|
|
227
|
-
return { jobIds };
|
|
163
|
+
const plan = planTree(root);
|
|
164
|
+
await commitFlow(this.pushCtx, plan.batch);
|
|
165
|
+
return { jobIds: plan.ids.map(String) };
|
|
228
166
|
}
|
|
229
167
|
/**
|
|
230
168
|
* Get the result of a completed parent job (embedded only).
|
|
@@ -258,9 +196,24 @@ export class FlowProducer extends EventEmitter {
|
|
|
258
196
|
/** Build callbacks that wire flow job methods to actual management operations */
|
|
259
197
|
buildCallbacks(queueName) {
|
|
260
198
|
const ctx = { name: queueName, embedded: this.embedded, tcp: this.tcp };
|
|
199
|
+
if (!this.embedded) {
|
|
200
|
+
return {
|
|
201
|
+
embedded: false,
|
|
202
|
+
tcp: this.tcp,
|
|
203
|
+
getState: (id) => {
|
|
204
|
+
if (!this.tcp)
|
|
205
|
+
return Promise.resolve('unknown');
|
|
206
|
+
return this.tcp.send({ cmd: 'GetState', id }).then((response) => {
|
|
207
|
+
if (response.ok !== true) {
|
|
208
|
+
throw new Error(typeof response.error === 'string' ? response.error : 'GetState failed');
|
|
209
|
+
}
|
|
210
|
+
return typeof response.state === 'string' ? response.state : 'unknown';
|
|
211
|
+
});
|
|
212
|
+
},
|
|
213
|
+
};
|
|
214
|
+
}
|
|
261
215
|
return {
|
|
262
|
-
embedded:
|
|
263
|
-
tcp: this.tcp,
|
|
216
|
+
embedded: true,
|
|
264
217
|
updateData: (id, data) => managementOps.updateJobData(ctx, id, data),
|
|
265
218
|
updateProgress: (id, progress) => managementOps.updateJobProgress(ctx, id, progress),
|
|
266
219
|
log: (id, msg) => managementOps.addJobLog(ctx, id, msg).then(() => { }),
|
|
@@ -268,117 +221,25 @@ export class FlowProducer extends EventEmitter {
|
|
|
268
221
|
remove: (id) => managementOps.removeAsync(ctx, id),
|
|
269
222
|
changePriority: (id, opts) => managementOps.changeJobPriority(ctx, id, opts),
|
|
270
223
|
changeDelay: (id, d) => managementOps.changeJobDelay(ctx, id, d),
|
|
271
|
-
clearLogs: (id) => managementOps.clearJobLogs(ctx, id),
|
|
224
|
+
clearLogs: (id, keepLogs) => managementOps.clearJobLogs(ctx, id, keepLogs),
|
|
272
225
|
retry: (id) => managementOps.retryJob(ctx, id),
|
|
273
|
-
getState: (id) =>
|
|
274
|
-
if (this.embedded) {
|
|
275
|
-
return getSharedManager().getJobState(jobId(id));
|
|
276
|
-
}
|
|
277
|
-
if (!this.tcp)
|
|
278
|
-
return Promise.resolve('unknown');
|
|
279
|
-
return this.tcp
|
|
280
|
-
.send({ cmd: 'GetState', id })
|
|
281
|
-
.then((r) => (typeof r.state === 'string' ? r.state : 'unknown'));
|
|
282
|
-
},
|
|
226
|
+
getState: (id) => getSharedManager().getJobState(jobId(id)),
|
|
283
227
|
};
|
|
284
228
|
}
|
|
285
|
-
|
|
286
|
-
const
|
|
287
|
-
if (
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
return null;
|
|
297
|
-
const jobData = response.job;
|
|
298
|
-
if (jobData.queue !== queueName)
|
|
299
|
-
return null;
|
|
300
|
-
return this.buildJobNodeFromTcp(jobData, depth, maxChildren);
|
|
301
|
-
}
|
|
302
|
-
async buildJobNode(job, depth, maxChildren) {
|
|
303
|
-
const data = job.data;
|
|
304
|
-
const name = typeof data.name === 'string' ? data.name : 'default';
|
|
305
|
-
const userData = extractUserDataFromInternal(data);
|
|
306
|
-
const jobObj = createFlowJobObject(String(job.id), name, userData, job.queue, this.buildCallbacks(job.queue));
|
|
307
|
-
if (depth <= 0 || job.childrenIds.length === 0)
|
|
308
|
-
return { job: jobObj };
|
|
309
|
-
const childNodes = [];
|
|
310
|
-
const childrenToFetch = maxChildren ? job.childrenIds.slice(0, maxChildren) : job.childrenIds;
|
|
311
|
-
for (const childId of childrenToFetch) {
|
|
312
|
-
const childJob = await getSharedManager().getJob(childId);
|
|
313
|
-
if (childJob)
|
|
314
|
-
childNodes.push(await this.buildJobNode(childJob, depth - 1, maxChildren));
|
|
315
|
-
}
|
|
316
|
-
return { job: jobObj, children: childNodes.length > 0 ? childNodes : undefined };
|
|
317
|
-
}
|
|
318
|
-
async buildJobNodeFromTcp(jobData, depth, maxChildren) {
|
|
319
|
-
const id = String(jobData.id);
|
|
320
|
-
const queueName = String(jobData.queue);
|
|
321
|
-
const data = jobData.data;
|
|
322
|
-
const name = typeof data?.name === 'string' ? data.name : 'default';
|
|
323
|
-
const userData = extractUserDataFromInternal(data ?? {});
|
|
324
|
-
const rawChildrenIds = data?.__childrenIds;
|
|
325
|
-
const childrenIds = Array.isArray(rawChildrenIds) ? rawChildrenIds : [];
|
|
326
|
-
const jobObj = createFlowJobObject(id, name, userData, queueName, this.buildCallbacks(queueName));
|
|
327
|
-
if (depth <= 0 || childrenIds.length === 0 || !this.tcp)
|
|
328
|
-
return { job: jobObj };
|
|
329
|
-
const childNodes = [];
|
|
330
|
-
const childrenToFetch = maxChildren ? childrenIds.slice(0, maxChildren) : childrenIds;
|
|
331
|
-
for (const childId of childrenToFetch) {
|
|
332
|
-
const response = await this.tcp.send({ cmd: 'GetJob', id: childId });
|
|
333
|
-
if (response.ok && response.job) {
|
|
334
|
-
childNodes.push(await this.buildJobNodeFromTcp(response.job, depth - 1, maxChildren));
|
|
335
|
-
}
|
|
336
|
-
}
|
|
337
|
-
return { job: jobObj, children: childNodes.length > 0 ? childNodes : undefined };
|
|
338
|
-
}
|
|
339
|
-
async addFlowNode(node, parentRef, createdJobIds, flowOpts) {
|
|
340
|
-
const childNodes = [];
|
|
341
|
-
const childIds = [];
|
|
342
|
-
if (node.children && node.children.length > 0) {
|
|
343
|
-
const tempParentRef = { id: 'pending', queue: node.queueName };
|
|
344
|
-
// Siblings are independent — create them concurrently
|
|
345
|
-
const results = await Promise.all(node.children.map((child) => this.addFlowNode(child, tempParentRef, createdJobIds, flowOpts)));
|
|
346
|
-
for (const childNode of results) {
|
|
347
|
-
childNodes.push(childNode);
|
|
348
|
-
childIds.push(childNode.job.id);
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
// Merge node opts with per-queue defaults from FlowOpts
|
|
352
|
-
const queueDefaults = flowOpts?.queuesOptions?.[node.queueName];
|
|
353
|
-
const mergedOpts = queueDefaults ? { ...queueDefaults, ...node.opts } : (node.opts ?? {});
|
|
354
|
-
const jobData = {
|
|
355
|
-
name: node.name,
|
|
356
|
-
...node.data,
|
|
229
|
+
buildPlannedNode(node, snapshots) {
|
|
230
|
+
const snapshot = snapshots.get(String(node.id));
|
|
231
|
+
if (!snapshot)
|
|
232
|
+
throw new Error(`Committed flow snapshot missing for ${String(node.id)}`);
|
|
233
|
+
const children = node.children?.map((child) => this.buildPlannedNode(child, snapshots));
|
|
234
|
+
return {
|
|
235
|
+
job: createFlowJobObject(String(node.id), node.name, node.data, node.queueName, {
|
|
236
|
+
callbacks: this.buildCallbacks(node.queueName),
|
|
237
|
+
snapshot,
|
|
238
|
+
}),
|
|
239
|
+
children: children && children.length > 0 ? children : undefined,
|
|
357
240
|
};
|
|
358
|
-
if (parentRef) {
|
|
359
|
-
jobData.__parentId = parentRef.id;
|
|
360
|
-
jobData.__parentQueue = parentRef.queue;
|
|
361
|
-
}
|
|
362
|
-
if (childIds.length > 0)
|
|
363
|
-
jobData.__childrenIds = childIds;
|
|
364
|
-
const jobIdStr = await pushJobWithParent(this.pushCtx, {
|
|
365
|
-
queueName: node.queueName,
|
|
366
|
-
data: jobData,
|
|
367
|
-
opts: mergedOpts,
|
|
368
|
-
parentRef,
|
|
369
|
-
childIds,
|
|
370
|
-
});
|
|
371
|
-
createdJobIds.push(jobIdStr);
|
|
372
|
-
const job = createFlowJobObject(jobIdStr, node.name, node.data, node.queueName, this.buildCallbacks(node.queueName));
|
|
373
|
-
return { job, children: childNodes.length > 0 ? childNodes : undefined };
|
|
374
241
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
const id = await pushJob(this.pushCtx, step.queueName, data, step.opts ?? {}, parentId ? [parentId] : undefined);
|
|
378
|
-
jobIds.push(id);
|
|
379
|
-
if (step.children) {
|
|
380
|
-
await Promise.all(step.children.map((child) => this.addTreeNode(child, id, jobIds)));
|
|
381
|
-
}
|
|
382
|
-
return id;
|
|
242
|
+
indexSnapshots(snapshots) {
|
|
243
|
+
return new Map(snapshots.map((snapshot) => [String(snapshot.id), snapshot]));
|
|
383
244
|
}
|
|
384
245
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Job as DomainJob } from '../domain/types/job';
|
|
2
|
+
import type { AtomicFlowBatchInput } from '../domain/types/flow';
|
|
3
|
+
import type { TcpConnectionPool } from './tcpPool';
|
|
4
|
+
export interface AtomicFlowContext {
|
|
5
|
+
readonly embedded: boolean;
|
|
6
|
+
readonly tcp: TcpConnectionPool | null;
|
|
7
|
+
}
|
|
8
|
+
/** Execute the same one-command atomic primitive in embedded and TCP modes. */
|
|
9
|
+
export declare function commitFlow(ctx: AtomicFlowContext, batch: AtomicFlowBatchInput): Promise<DomainJob[]>;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { getSharedManager } from './manager';
|
|
2
|
+
/** Execute the same one-command atomic primitive in embedded and TCP modes. */
|
|
3
|
+
export async function commitFlow(ctx, batch) {
|
|
4
|
+
if (batch.jobs.length === 0)
|
|
5
|
+
return [];
|
|
6
|
+
if (ctx.embedded) {
|
|
7
|
+
return (await getSharedManager().pushFlow(batch)).jobs;
|
|
8
|
+
}
|
|
9
|
+
if (!ctx.tcp)
|
|
10
|
+
throw new Error('TCP connection not initialized');
|
|
11
|
+
const response = await ctx.tcp.send({ cmd: 'PUSHF', jobs: batch.jobs });
|
|
12
|
+
if (response.ok !== true) {
|
|
13
|
+
throw new Error(typeof response.error === 'string' ? response.error : 'Failed to add flow');
|
|
14
|
+
}
|
|
15
|
+
const result = response.data;
|
|
16
|
+
if (!result || !Array.isArray(result.jobs) || result.jobs.length !== batch.jobs.length) {
|
|
17
|
+
throw new Error('Invalid PUSHF response: committed job snapshots are missing');
|
|
18
|
+
}
|
|
19
|
+
const expected = new Set(batch.jobs.map((job) => String(job.id)));
|
|
20
|
+
if (result.jobs.some((job) => !expected.delete(String(job.id))) || expected.size > 0) {
|
|
21
|
+
throw new Error('Invalid PUSHF response: committed job IDs do not match the request');
|
|
22
|
+
}
|
|
23
|
+
return result.jobs;
|
|
24
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { ChangePriorityOpts } from './types';
|
|
2
|
+
import { type FlowJobRuntime } from './flowJobTypes';
|
|
3
|
+
/** Core state, mutation, and dependency methods exposed by a FlowProducer Job. */
|
|
4
|
+
export declare function buildFlowJobCoreMethods(runtime: FlowJobRuntime): {
|
|
5
|
+
updateProgress: (progress: number | object) => Promise<void>;
|
|
6
|
+
log: (message: string) => Promise<void>;
|
|
7
|
+
getState: () => Promise<import("./types").JobStateType>;
|
|
8
|
+
remove: () => Promise<void>;
|
|
9
|
+
retry: () => Promise<void>;
|
|
10
|
+
getChildrenValues: <R = unknown>() => Promise<Record<string, R>>;
|
|
11
|
+
isWaiting: () => Promise<boolean>;
|
|
12
|
+
isActive: () => Promise<boolean>;
|
|
13
|
+
isDelayed: () => Promise<boolean>;
|
|
14
|
+
isCompleted: () => Promise<boolean>;
|
|
15
|
+
isFailed: () => Promise<boolean>;
|
|
16
|
+
isWaitingChildren: () => Promise<boolean>;
|
|
17
|
+
updateData: (data: unknown) => Promise<void>;
|
|
18
|
+
promote: () => Promise<void>;
|
|
19
|
+
changeDelay: (delay: number) => Promise<void>;
|
|
20
|
+
changePriority: (opts: ChangePriorityOpts) => Promise<void>;
|
|
21
|
+
extendLock: (token: string, duration: number) => Promise<number>;
|
|
22
|
+
clearLogs: (keepLogs?: number) => Promise<void>;
|
|
23
|
+
getDependencies: () => Promise<import("./flowJobDependencies").FlowDependencies>;
|
|
24
|
+
getDependenciesCount: () => Promise<{
|
|
25
|
+
processed: number;
|
|
26
|
+
unprocessed: number;
|
|
27
|
+
}>;
|
|
28
|
+
};
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { jobId } from '../domain/types/job';
|
|
2
|
+
import { getSharedManager } from './manager';
|
|
3
|
+
import { getFlowDependencies } from './flowJobDependencies';
|
|
4
|
+
import { assertFlowTcpOk } from './flowJobTypes';
|
|
5
|
+
/** Core state, mutation, and dependency methods exposed by a FlowProducer Job. */
|
|
6
|
+
export function buildFlowJobCoreMethods(runtime) {
|
|
7
|
+
const { id, queueName, callbacks, embedded, tcp, getState } = runtime;
|
|
8
|
+
return {
|
|
9
|
+
updateProgress: async (progress) => {
|
|
10
|
+
if (callbacks?.updateProgress)
|
|
11
|
+
return callbacks.updateProgress(id, progress);
|
|
12
|
+
if (embedded) {
|
|
13
|
+
const value = typeof progress === 'number' ? progress : 0;
|
|
14
|
+
const message = typeof progress === 'object' ? JSON.stringify(progress) : undefined;
|
|
15
|
+
return void (await getSharedManager().updateProgress(jobId(id), value, message));
|
|
16
|
+
}
|
|
17
|
+
if (!tcp)
|
|
18
|
+
return;
|
|
19
|
+
const value = typeof progress === 'number' ? progress : 0;
|
|
20
|
+
const message = typeof progress === 'object' ? JSON.stringify(progress) : undefined;
|
|
21
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'Progress', id, progress: value, message }), 'Progress');
|
|
22
|
+
},
|
|
23
|
+
log: async (message) => {
|
|
24
|
+
if (callbacks?.log)
|
|
25
|
+
return callbacks.log(id, message);
|
|
26
|
+
if (embedded)
|
|
27
|
+
return void getSharedManager().addLog(jobId(id), message);
|
|
28
|
+
if (!tcp)
|
|
29
|
+
return;
|
|
30
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'AddLog', id, message }), 'AddLog');
|
|
31
|
+
},
|
|
32
|
+
getState,
|
|
33
|
+
remove: async () => {
|
|
34
|
+
if (callbacks?.remove)
|
|
35
|
+
return callbacks.remove(id);
|
|
36
|
+
if (embedded)
|
|
37
|
+
return void (await getSharedManager().cancel(jobId(id)));
|
|
38
|
+
if (!tcp)
|
|
39
|
+
return;
|
|
40
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'Cancel', id }), 'Cancel');
|
|
41
|
+
},
|
|
42
|
+
retry: async () => {
|
|
43
|
+
if (callbacks?.retry)
|
|
44
|
+
return callbacks.retry(id);
|
|
45
|
+
if (embedded) {
|
|
46
|
+
const manager = getSharedManager();
|
|
47
|
+
const state = await manager.getJobState(jobId(id));
|
|
48
|
+
if (state === 'failed') {
|
|
49
|
+
if (manager.retryDlq(queueName, jobId(id)) === 0) {
|
|
50
|
+
throw new Error(`Job ${id} is failed but not present in DLQ`);
|
|
51
|
+
}
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
if (state === 'active') {
|
|
55
|
+
if (!(await manager.moveActiveToWait(jobId(id)))) {
|
|
56
|
+
throw new Error(`Failed to retry active job ${id}`);
|
|
57
|
+
}
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
if (state === 'waiting' || state === 'prioritized' || state === 'delayed')
|
|
61
|
+
return;
|
|
62
|
+
throw new Error(`Cannot retry job ${id} from state '${state}'`);
|
|
63
|
+
}
|
|
64
|
+
if (!tcp)
|
|
65
|
+
return;
|
|
66
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'MoveToWait', id }), 'MoveToWait');
|
|
67
|
+
},
|
|
68
|
+
getChildrenValues: async () => {
|
|
69
|
+
if (embedded) {
|
|
70
|
+
return (await getSharedManager().getChildrenValues(jobId(id)));
|
|
71
|
+
}
|
|
72
|
+
if (!tcp)
|
|
73
|
+
return {};
|
|
74
|
+
const response = await tcp.send({ cmd: 'GetChildrenValues', id });
|
|
75
|
+
assertFlowTcpOk(response, 'GetChildrenValues');
|
|
76
|
+
const values = response.data?.values ?? {};
|
|
77
|
+
return values;
|
|
78
|
+
},
|
|
79
|
+
isWaiting: async () => (await getState()) === 'waiting',
|
|
80
|
+
isActive: async () => (await getState()) === 'active',
|
|
81
|
+
isDelayed: async () => (await getState()) === 'delayed',
|
|
82
|
+
isCompleted: async () => (await getState()) === 'completed',
|
|
83
|
+
isFailed: async () => (await getState()) === 'failed',
|
|
84
|
+
isWaitingChildren: async () => (await getState()) === 'waiting-children',
|
|
85
|
+
updateData: async (data) => {
|
|
86
|
+
if (callbacks?.updateData)
|
|
87
|
+
return callbacks.updateData(id, data);
|
|
88
|
+
if (embedded)
|
|
89
|
+
return void (await getSharedManager().updateJobData(jobId(id), data));
|
|
90
|
+
if (!tcp)
|
|
91
|
+
return;
|
|
92
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'Update', id, data }), 'Update');
|
|
93
|
+
},
|
|
94
|
+
promote: async () => {
|
|
95
|
+
if (callbacks?.promote)
|
|
96
|
+
return callbacks.promote(id);
|
|
97
|
+
if (embedded)
|
|
98
|
+
return void (await getSharedManager().promote(jobId(id)));
|
|
99
|
+
if (!tcp)
|
|
100
|
+
return;
|
|
101
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'Promote', id }), 'Promote');
|
|
102
|
+
},
|
|
103
|
+
changeDelay: async (delay) => {
|
|
104
|
+
if (callbacks?.changeDelay)
|
|
105
|
+
return callbacks.changeDelay(id, delay);
|
|
106
|
+
if (embedded)
|
|
107
|
+
return void (await getSharedManager().changeDelay(jobId(id), delay));
|
|
108
|
+
if (!tcp)
|
|
109
|
+
return;
|
|
110
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'ChangeDelay', id, delay }), 'ChangeDelay');
|
|
111
|
+
},
|
|
112
|
+
changePriority: async (opts) => {
|
|
113
|
+
if (callbacks?.changePriority) {
|
|
114
|
+
return callbacks.changePriority(id, { priority: opts.priority, lifo: opts.lifo });
|
|
115
|
+
}
|
|
116
|
+
if (embedded) {
|
|
117
|
+
return void (await getSharedManager().changePriority(jobId(id), opts.priority, opts.lifo));
|
|
118
|
+
}
|
|
119
|
+
if (!tcp)
|
|
120
|
+
return;
|
|
121
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'ChangePriority', id, priority: opts.priority, lifo: opts.lifo }), 'ChangePriority');
|
|
122
|
+
},
|
|
123
|
+
extendLock: async (token, duration) => {
|
|
124
|
+
if (embedded) {
|
|
125
|
+
return (await getSharedManager().extendLock(jobId(id), token, duration)) ? duration : 0;
|
|
126
|
+
}
|
|
127
|
+
if (!tcp)
|
|
128
|
+
return 0;
|
|
129
|
+
const response = await tcp.send({ cmd: 'ExtendLock', id, token, duration });
|
|
130
|
+
assertFlowTcpOk(response, 'ExtendLock');
|
|
131
|
+
return duration;
|
|
132
|
+
},
|
|
133
|
+
clearLogs: async (keepLogs) => {
|
|
134
|
+
if (callbacks?.clearLogs)
|
|
135
|
+
return callbacks.clearLogs(id, keepLogs);
|
|
136
|
+
if (embedded)
|
|
137
|
+
return void getSharedManager().clearLogs(jobId(id), keepLogs);
|
|
138
|
+
if (!tcp)
|
|
139
|
+
return;
|
|
140
|
+
assertFlowTcpOk(await tcp.send({ cmd: 'ClearLogs', id, keepLogs }), 'ClearLogs');
|
|
141
|
+
},
|
|
142
|
+
getDependencies: () => getFlowDependencies(id, queueName, embedded, tcp),
|
|
143
|
+
getDependenciesCount: async () => {
|
|
144
|
+
const dependencies = await getFlowDependencies(id, queueName, embedded, tcp);
|
|
145
|
+
return {
|
|
146
|
+
processed: Object.keys(dependencies.processed).length,
|
|
147
|
+
unprocessed: dependencies.unprocessed.length,
|
|
148
|
+
};
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { TcpConnectionPool } from './tcpPool';
|
|
2
|
+
export interface FlowDependencies {
|
|
3
|
+
processed: Record<string, unknown>;
|
|
4
|
+
unprocessed: string[];
|
|
5
|
+
}
|
|
6
|
+
/** Resolve dependency keys using the actual queue of every child. */
|
|
7
|
+
export declare function getFlowDependencies(id: string, queueName: string, embedded: boolean, tcp: TcpConnectionPool | null): Promise<FlowDependencies>;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { jobId } from '../domain/types/job';
|
|
2
|
+
import { getSharedManager } from './manager';
|
|
3
|
+
import { assertFlowTcpOk } from './flowJobTypes';
|
|
4
|
+
async function fetchChildIds(id, embedded, tcp) {
|
|
5
|
+
if (embedded) {
|
|
6
|
+
const job = await getSharedManager().getJob(jobId(id));
|
|
7
|
+
if (!job)
|
|
8
|
+
throw new Error(`Flow job ${id} not found`);
|
|
9
|
+
return job.childrenIds.map(String);
|
|
10
|
+
}
|
|
11
|
+
if (!tcp)
|
|
12
|
+
return [];
|
|
13
|
+
const response = await tcp.send({ cmd: 'GetJob', id });
|
|
14
|
+
assertFlowTcpOk(response, 'GetJob');
|
|
15
|
+
const parent = response.job;
|
|
16
|
+
return (parent?.childrenIds ?? []).map(String);
|
|
17
|
+
}
|
|
18
|
+
async function embeddedDependency(fallbackQueue, childId) {
|
|
19
|
+
const manager = getSharedManager();
|
|
20
|
+
const child = await manager.getJob(jobId(childId));
|
|
21
|
+
if (!child)
|
|
22
|
+
throw new Error(`Flow child job ${childId} not found`);
|
|
23
|
+
const state = await manager.getJobState(jobId(childId));
|
|
24
|
+
return {
|
|
25
|
+
key: `${child.queue || fallbackQueue}:${childId}`,
|
|
26
|
+
state,
|
|
27
|
+
result: state === 'completed' ? manager.getResult(jobId(childId)) : undefined,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
async function tcpDependency(fallbackQueue, childId, tcp) {
|
|
31
|
+
const response = await tcp.send({ cmd: 'GetJob', id: childId });
|
|
32
|
+
assertFlowTcpOk(response, 'GetJob');
|
|
33
|
+
const child = response.job;
|
|
34
|
+
if (!child)
|
|
35
|
+
throw new Error(`Flow child job ${childId} not found`);
|
|
36
|
+
const state = child.state ?? 'unknown';
|
|
37
|
+
let result;
|
|
38
|
+
if (state === 'completed') {
|
|
39
|
+
const resultResponse = await tcp.send({ cmd: 'GetResult', id: childId });
|
|
40
|
+
assertFlowTcpOk(resultResponse, 'GetResult');
|
|
41
|
+
result = resultResponse.result;
|
|
42
|
+
}
|
|
43
|
+
return { key: `${child.queue ?? fallbackQueue}:${childId}`, state, result };
|
|
44
|
+
}
|
|
45
|
+
/** Resolve dependency keys using the actual queue of every child. */
|
|
46
|
+
export async function getFlowDependencies(id, queueName, embedded, tcp) {
|
|
47
|
+
const childIds = await fetchChildIds(id, embedded, tcp);
|
|
48
|
+
const processed = {};
|
|
49
|
+
const unprocessed = [];
|
|
50
|
+
for (const childId of childIds) {
|
|
51
|
+
const dependency = embedded
|
|
52
|
+
? await embeddedDependency(queueName, childId)
|
|
53
|
+
: tcp
|
|
54
|
+
? await tcpDependency(queueName, childId, tcp)
|
|
55
|
+
: { key: `${queueName}:${childId}`, state: 'unknown', result: undefined };
|
|
56
|
+
if (dependency.state === 'completed' || dependency.state === 'failed') {
|
|
57
|
+
processed[dependency.key] = dependency.result ?? null;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
unprocessed.push(dependency.key);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { processed, unprocessed };
|
|
64
|
+
}
|