bunqueue 2.0.3 → 2.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/operations/push.d.ts.map +1 -1
- package/dist/application/operations/push.js +3 -3
- package/dist/application/operations/push.js.map +1 -1
- package/dist/application/webhookManager.js +3 -3
- package/dist/application/webhookManager.js.map +1 -1
- package/dist/application/workerManager.js +2 -2
- package/dist/application/workerManager.js.map +1 -1
- package/dist/cli/commands/backup.js +1 -1
- package/dist/cli/commands/backup.js.map +1 -1
- package/dist/cli/commands/server.js +10 -10
- package/dist/cli/commands/server.js.map +1 -1
- package/dist/cli/output.d.ts.map +1 -1
- package/dist/cli/output.js +15 -9
- package/dist/cli/output.js.map +1 -1
- package/dist/client/flow.d.ts +20 -175
- package/dist/client/flow.d.ts.map +1 -1
- package/dist/client/flow.js +122 -475
- package/dist/client/flow.js.map +1 -1
- package/dist/client/flowJobFactory.d.ts +10 -0
- package/dist/client/flowJobFactory.d.ts.map +1 -0
- package/dist/client/flowJobFactory.js +104 -0
- package/dist/client/flowJobFactory.js.map +1 -0
- package/dist/client/flowPush.d.ts +27 -0
- package/dist/client/flowPush.d.ts.map +1 -0
- package/dist/client/flowPush.js +129 -0
- package/dist/client/flowPush.js.map +1 -0
- package/dist/client/flowTypes.d.ts +70 -0
- package/dist/client/flowTypes.d.ts.map +1 -0
- package/dist/client/flowTypes.js +6 -0
- package/dist/client/flowTypes.js.map +1 -0
- package/dist/client/jobConversion.d.ts +88 -0
- package/dist/client/jobConversion.d.ts.map +1 -0
- package/dist/client/jobConversion.js +235 -0
- package/dist/client/jobConversion.js.map +1 -0
- package/dist/client/jobHelpers.d.ts +27 -0
- package/dist/client/jobHelpers.d.ts.map +1 -0
- package/dist/client/jobHelpers.js +113 -0
- package/dist/client/jobHelpers.js.map +1 -0
- package/dist/client/manager.js +1 -1
- package/dist/client/manager.js.map +1 -1
- package/dist/client/queue/helpers.d.ts.map +1 -1
- package/dist/client/queue/helpers.js +1 -1
- package/dist/client/queue/helpers.js.map +1 -1
- package/dist/client/sandboxed/worker.d.ts +1 -1
- package/dist/client/sandboxed/worker.d.ts.map +1 -1
- package/dist/client/sandboxed/worker.js +3 -3
- package/dist/client/sandboxed/worker.js.map +1 -1
- package/dist/client/sandboxed/wrapper.d.ts +2 -2
- package/dist/client/sandboxed/wrapper.d.ts.map +1 -1
- package/dist/client/sandboxed/wrapper.js +20 -17
- package/dist/client/sandboxed/wrapper.js.map +1 -1
- package/dist/client/types.d.ts +3 -82
- package/dist/client/types.d.ts.map +1 -1
- package/dist/client/types.js +3 -401
- package/dist/client/types.js.map +1 -1
- package/dist/client/worker/ackBatcher.js +1 -1
- package/dist/client/worker/ackBatcher.js.map +1 -1
- package/dist/client/worker/types.d.ts.map +1 -1
- package/dist/client/worker/types.js +1 -1
- package/dist/client/worker/types.js.map +1 -1
- package/dist/client/worker/worker.js +2 -2
- package/dist/client/worker/worker.js.map +1 -1
- package/dist/infrastructure/backup/s3BackupConfig.d.ts +5 -0
- package/dist/infrastructure/backup/s3BackupConfig.d.ts.map +1 -1
- package/dist/infrastructure/backup/s3BackupConfig.js +9 -9
- package/dist/infrastructure/backup/s3BackupConfig.js.map +1 -1
- package/dist/infrastructure/backup/s3BackupOperations.d.ts.map +1 -1
- package/dist/infrastructure/backup/s3BackupOperations.js +35 -13
- package/dist/infrastructure/backup/s3BackupOperations.js.map +1 -1
- package/dist/infrastructure/server/rateLimiter.js +4 -4
- package/dist/infrastructure/server/rateLimiter.js.map +1 -1
- package/dist/infrastructure/server/tcp.js +1 -1
- package/dist/infrastructure/server/tcp.js.map +1 -1
- package/dist/main.js +15 -16
- package/dist/main.js.map +1 -1
- package/dist/mcp/index.js +1 -1
- package/dist/mcp/index.js.map +1 -1
- package/dist/shared/hash.d.ts.map +1 -1
- package/dist/shared/hash.js +1 -2
- package/dist/shared/hash.js.map +1 -1
- package/dist/shared/lock.js +1 -1
- package/dist/shared/lock.js.map +1 -1
- package/package.json +1 -1
package/dist/client/flow.js
CHANGED
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
import { getSharedManager } from './manager';
|
|
6
6
|
import { TcpConnectionPool, getSharedPool, releaseSharedPool } from './tcpPool';
|
|
7
7
|
import { jobId } from '../domain/types/job';
|
|
8
|
-
|
|
8
|
+
import { createFlowJobObject, extractUserDataFromInternal } from './flowJobFactory';
|
|
9
|
+
import { pushJob, pushJobWithParent, cleanupJobs } from './flowPush';
|
|
10
|
+
const FORCE_EMBEDDED = Bun.env.BUNQUEUE_EMBEDDED === '1';
|
|
9
11
|
/**
|
|
10
12
|
* FlowProducer creates job flows with automatic dependencies.
|
|
11
13
|
*
|
|
@@ -58,6 +60,10 @@ export class FlowProducer {
|
|
|
58
60
|
}
|
|
59
61
|
}
|
|
60
62
|
}
|
|
63
|
+
/** Get push context for helper functions */
|
|
64
|
+
get pushCtx() {
|
|
65
|
+
return { embedded: this.embedded, tcp: this.tcp };
|
|
66
|
+
}
|
|
61
67
|
/** Close the connection pool (only if using dedicated pool) */
|
|
62
68
|
close() {
|
|
63
69
|
if (this.tcp && !this.useSharedPool) {
|
|
@@ -67,60 +73,26 @@ export class FlowProducer {
|
|
|
67
73
|
releaseSharedPool(this.tcp);
|
|
68
74
|
}
|
|
69
75
|
}
|
|
70
|
-
/**
|
|
71
|
-
* Disconnect from the server (BullMQ v5 compatible).
|
|
72
|
-
* Alias for close().
|
|
73
|
-
*/
|
|
76
|
+
/** Disconnect from the server (BullMQ v5 compatible). Alias for close(). */
|
|
74
77
|
disconnect() {
|
|
75
78
|
this.close();
|
|
76
79
|
return Promise.resolve();
|
|
77
80
|
}
|
|
78
|
-
/**
|
|
79
|
-
* Wait until the FlowProducer is ready (BullMQ v5 compatible).
|
|
80
|
-
* In embedded mode, resolves immediately. In TCP mode, ensures connection.
|
|
81
|
-
*/
|
|
81
|
+
/** Wait until the FlowProducer is ready (BullMQ v5 compatible). */
|
|
82
82
|
async waitUntilReady() {
|
|
83
|
-
if (this.embedded)
|
|
84
|
-
// Embedded mode is always ready
|
|
83
|
+
if (this.embedded)
|
|
85
84
|
return;
|
|
86
|
-
|
|
87
|
-
// TCP mode - ensure we can send a ping
|
|
88
|
-
if (this.tcp) {
|
|
85
|
+
if (this.tcp)
|
|
89
86
|
await this.tcp.send({ cmd: 'Ping' });
|
|
90
|
-
}
|
|
91
87
|
}
|
|
92
88
|
// ============================================================================
|
|
93
89
|
// BullMQ v5 Compatible Methods
|
|
94
90
|
// ============================================================================
|
|
95
|
-
/**
|
|
96
|
-
* Add a flow (BullMQ v5 compatible).
|
|
97
|
-
*
|
|
98
|
-
* Children are processed BEFORE their parent. When all children complete,
|
|
99
|
-
* the parent becomes processable and can access children results via
|
|
100
|
-
* `job.getChildrenValues()`.
|
|
101
|
-
*
|
|
102
|
-
* @example
|
|
103
|
-
* ```typescript
|
|
104
|
-
* const flow = new FlowProducer();
|
|
105
|
-
*
|
|
106
|
-
* // Children execute first, then parent
|
|
107
|
-
* const { job, children } = await flow.add({
|
|
108
|
-
* name: 'parent',
|
|
109
|
-
* queueName: 'main',
|
|
110
|
-
* data: { type: 'aggregate' },
|
|
111
|
-
* children: [
|
|
112
|
-
* { name: 'child1', queueName: 'main', data: { id: 1 } },
|
|
113
|
-
* { name: 'child2', queueName: 'main', data: { id: 2 } },
|
|
114
|
-
* ],
|
|
115
|
-
* });
|
|
116
|
-
* ```
|
|
117
|
-
*/
|
|
91
|
+
/** Add a flow (BullMQ v5 compatible). Children are processed BEFORE their parent. */
|
|
118
92
|
async add(flow) {
|
|
119
93
|
return this.addFlowNode(flow, null);
|
|
120
94
|
}
|
|
121
|
-
/**
|
|
122
|
-
* Add multiple flows (BullMQ v5 compatible).
|
|
123
|
-
*/
|
|
95
|
+
/** Add multiple flows (BullMQ v5 compatible). */
|
|
124
96
|
async addBulk(flows) {
|
|
125
97
|
const results = [];
|
|
126
98
|
for (const flow of flows) {
|
|
@@ -128,27 +100,7 @@ export class FlowProducer {
|
|
|
128
100
|
}
|
|
129
101
|
return results;
|
|
130
102
|
}
|
|
131
|
-
/**
|
|
132
|
-
* Get a flow tree starting from a job (BullMQ v5 compatible).
|
|
133
|
-
*
|
|
134
|
-
* Retrieves the job and all its children recursively, building a JobNode tree.
|
|
135
|
-
*
|
|
136
|
-
* @example
|
|
137
|
-
* ```typescript
|
|
138
|
-
* const flow = new FlowProducer();
|
|
139
|
-
*
|
|
140
|
-
* // Get a previously created flow
|
|
141
|
-
* const node = await flow.getFlow({
|
|
142
|
-
* id: 'job-id',
|
|
143
|
-
* queueName: 'my-queue',
|
|
144
|
-
* });
|
|
145
|
-
*
|
|
146
|
-
* if (node) {
|
|
147
|
-
* console.log('Parent:', node.job.name);
|
|
148
|
-
* console.log('Children:', node.children?.length ?? 0);
|
|
149
|
-
* }
|
|
150
|
-
* ```
|
|
151
|
-
*/
|
|
103
|
+
/** Get a flow tree starting from a job (BullMQ v5 compatible). */
|
|
152
104
|
async getFlow(opts) {
|
|
153
105
|
const { id, queueName, depth, maxChildren } = opts;
|
|
154
106
|
if (this.embedded) {
|
|
@@ -156,114 +108,143 @@ export class FlowProducer {
|
|
|
156
108
|
}
|
|
157
109
|
return this.getFlowTcp(id, queueName, depth ?? Infinity, maxChildren);
|
|
158
110
|
}
|
|
159
|
-
|
|
160
|
-
|
|
111
|
+
// ============================================================================
|
|
112
|
+
// Legacy bunqueue API Methods
|
|
113
|
+
// ============================================================================
|
|
114
|
+
/** Add a chain of jobs. Jobs execute sequentially: step[0] → step[1] → ... */
|
|
115
|
+
async addChain(steps) {
|
|
116
|
+
if (steps.length === 0)
|
|
117
|
+
return { jobIds: [] };
|
|
118
|
+
const jobIds = [];
|
|
119
|
+
let prevId = null;
|
|
120
|
+
try {
|
|
121
|
+
for (const step of steps) {
|
|
122
|
+
const data = { name: step.name, __flowParentId: prevId, ...step.data };
|
|
123
|
+
const id = await pushJob(this.pushCtx, step.queueName, data, step.opts ?? {}, prevId ? [prevId] : undefined);
|
|
124
|
+
jobIds.push(id);
|
|
125
|
+
prevId = id;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
await cleanupJobs(this.pushCtx, jobIds);
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
132
|
+
return { jobIds };
|
|
133
|
+
}
|
|
134
|
+
/** Add parallel jobs that converge to a final job. */
|
|
135
|
+
async addBulkThen(parallel, final) {
|
|
136
|
+
const parallelIds = [];
|
|
137
|
+
try {
|
|
138
|
+
for (const step of parallel) {
|
|
139
|
+
const data = { name: step.name, ...step.data };
|
|
140
|
+
const id = await pushJob(this.pushCtx, step.queueName, data, step.opts ?? {});
|
|
141
|
+
parallelIds.push(id);
|
|
142
|
+
}
|
|
143
|
+
const finalData = {
|
|
144
|
+
name: final.name,
|
|
145
|
+
__flowParentIds: parallelIds,
|
|
146
|
+
...final.data,
|
|
147
|
+
};
|
|
148
|
+
const finalId = await pushJob(this.pushCtx, final.queueName, finalData, final.opts ?? {}, parallelIds);
|
|
149
|
+
return { parallelIds, finalId };
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
await cleanupJobs(this.pushCtx, parallelIds);
|
|
153
|
+
throw error;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** Add a tree of jobs where children depend on parent. */
|
|
157
|
+
async addTree(root) {
|
|
158
|
+
const jobIds = [];
|
|
159
|
+
try {
|
|
160
|
+
await this.addTreeNode(root, null, jobIds);
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
await cleanupJobs(this.pushCtx, jobIds);
|
|
164
|
+
throw error;
|
|
165
|
+
}
|
|
166
|
+
return { jobIds };
|
|
167
|
+
}
|
|
168
|
+
/** Get the result of a completed parent job (embedded only). */
|
|
169
|
+
getParentResult(parentId) {
|
|
170
|
+
if (!this.embedded)
|
|
171
|
+
throw new Error('getParentResult is only available in embedded mode');
|
|
172
|
+
return getSharedManager().getResult(jobId(parentId));
|
|
173
|
+
}
|
|
174
|
+
/** Get results from multiple parent jobs (embedded only). */
|
|
175
|
+
getParentResults(parentIds) {
|
|
176
|
+
if (!this.embedded)
|
|
177
|
+
throw new Error('getParentResults is only available in embedded mode');
|
|
161
178
|
const manager = getSharedManager();
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
179
|
+
const results = new Map();
|
|
180
|
+
for (const id of parentIds) {
|
|
181
|
+
const result = manager.getResult(jobId(id));
|
|
182
|
+
if (result !== undefined)
|
|
183
|
+
results.set(id, result);
|
|
165
184
|
}
|
|
185
|
+
return results;
|
|
186
|
+
}
|
|
187
|
+
// ============================================================================
|
|
188
|
+
// Private Methods
|
|
189
|
+
// ============================================================================
|
|
190
|
+
async getFlowEmbedded(id, queueName, depth, maxChildren) {
|
|
191
|
+
const job = await getSharedManager().getJob(jobId(id));
|
|
192
|
+
if (job?.queue !== queueName)
|
|
193
|
+
return null;
|
|
166
194
|
return this.buildJobNode(job, depth, maxChildren);
|
|
167
195
|
}
|
|
168
|
-
/** Get flow in TCP mode */
|
|
169
196
|
async getFlowTcp(id, queueName, depth, maxChildren) {
|
|
170
197
|
if (!this.tcp)
|
|
171
198
|
throw new Error('TCP connection not initialized');
|
|
172
|
-
const response = await this.tcp.send({
|
|
173
|
-
|
|
174
|
-
id,
|
|
175
|
-
});
|
|
176
|
-
if (!response.ok || !response.job) {
|
|
199
|
+
const response = await this.tcp.send({ cmd: 'GetJob', id });
|
|
200
|
+
if (!response.ok || !response.job)
|
|
177
201
|
return null;
|
|
178
|
-
}
|
|
179
202
|
const jobData = response.job;
|
|
180
|
-
if (jobData.queue !== queueName)
|
|
203
|
+
if (jobData.queue !== queueName)
|
|
181
204
|
return null;
|
|
182
|
-
}
|
|
183
|
-
// Build job node from TCP response
|
|
184
205
|
return this.buildJobNodeFromTcp(jobData, depth, maxChildren);
|
|
185
206
|
}
|
|
186
|
-
/** Build JobNode recursively from internal job */
|
|
187
207
|
async buildJobNode(job, depth, maxChildren) {
|
|
188
208
|
const data = job.data;
|
|
189
|
-
const
|
|
190
|
-
const
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
// If depth exhausted or no children, return without children
|
|
194
|
-
if (depth <= 0 || job.childrenIds.length === 0) {
|
|
209
|
+
const name = typeof data.name === 'string' ? data.name : 'default';
|
|
210
|
+
const userData = extractUserDataFromInternal(data);
|
|
211
|
+
const jobObj = createFlowJobObject(String(job.id), name, userData, job.queue);
|
|
212
|
+
if (depth <= 0 || job.childrenIds.length === 0)
|
|
195
213
|
return { job: jobObj };
|
|
196
|
-
}
|
|
197
|
-
// Fetch children recursively
|
|
198
|
-
const manager = getSharedManager();
|
|
199
214
|
const childNodes = [];
|
|
200
215
|
const childrenToFetch = maxChildren ? job.childrenIds.slice(0, maxChildren) : job.childrenIds;
|
|
201
216
|
for (const childId of childrenToFetch) {
|
|
202
|
-
const childJob = await
|
|
203
|
-
if (childJob)
|
|
204
|
-
|
|
205
|
-
childNodes.push(childNode);
|
|
206
|
-
}
|
|
217
|
+
const childJob = await getSharedManager().getJob(childId);
|
|
218
|
+
if (childJob)
|
|
219
|
+
childNodes.push(await this.buildJobNode(childJob, depth - 1, maxChildren));
|
|
207
220
|
}
|
|
208
|
-
return {
|
|
209
|
-
job: jobObj,
|
|
210
|
-
children: childNodes.length > 0 ? childNodes : undefined,
|
|
211
|
-
};
|
|
221
|
+
return { job: jobObj, children: childNodes.length > 0 ? childNodes : undefined };
|
|
212
222
|
}
|
|
213
|
-
/** Build JobNode from TCP response */
|
|
214
223
|
async buildJobNodeFromTcp(jobData, depth, maxChildren) {
|
|
215
224
|
const id = String(jobData.id);
|
|
216
225
|
const queueName = String(jobData.queue);
|
|
217
226
|
const data = jobData.data;
|
|
218
|
-
const
|
|
219
|
-
const
|
|
220
|
-
const userData = this.extractUserDataFromInternal(data ?? {});
|
|
227
|
+
const name = typeof data?.name === 'string' ? data.name : 'default';
|
|
228
|
+
const userData = extractUserDataFromInternal(data ?? {});
|
|
221
229
|
const rawChildrenIds = data?.__childrenIds;
|
|
222
230
|
const childrenIds = Array.isArray(rawChildrenIds) ? rawChildrenIds : [];
|
|
223
|
-
const jobObj =
|
|
224
|
-
|
|
225
|
-
if (depth <= 0 || childrenIds.length === 0) {
|
|
231
|
+
const jobObj = createFlowJobObject(id, name, userData, queueName);
|
|
232
|
+
if (depth <= 0 || childrenIds.length === 0 || !this.tcp)
|
|
226
233
|
return { job: jobObj };
|
|
227
|
-
}
|
|
228
|
-
// Fetch children recursively via TCP
|
|
229
|
-
if (!this.tcp) {
|
|
230
|
-
return { job: jobObj };
|
|
231
|
-
}
|
|
232
234
|
const childNodes = [];
|
|
233
235
|
const childrenToFetch = maxChildren ? childrenIds.slice(0, maxChildren) : childrenIds;
|
|
234
236
|
for (const childId of childrenToFetch) {
|
|
235
237
|
const response = await this.tcp.send({ cmd: 'GetJob', id: childId });
|
|
236
238
|
if (response.ok && response.job) {
|
|
237
|
-
|
|
238
|
-
childNodes.push(childNode);
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
return {
|
|
242
|
-
job: jobObj,
|
|
243
|
-
children: childNodes.length > 0 ? childNodes : undefined,
|
|
244
|
-
};
|
|
245
|
-
}
|
|
246
|
-
/** Extract user data (remove internal fields) */
|
|
247
|
-
extractUserDataFromInternal(data) {
|
|
248
|
-
const result = {};
|
|
249
|
-
for (const [key, value] of Object.entries(data)) {
|
|
250
|
-
if (!key.startsWith('__') && key !== 'name') {
|
|
251
|
-
result[key] = value;
|
|
239
|
+
childNodes.push(await this.buildJobNodeFromTcp(response.job, depth - 1, maxChildren));
|
|
252
240
|
}
|
|
253
241
|
}
|
|
254
|
-
return
|
|
242
|
+
return { job: jobObj, children: childNodes.length > 0 ? childNodes : undefined };
|
|
255
243
|
}
|
|
256
|
-
/**
|
|
257
|
-
* Internal: Recursively add a flow node and its children.
|
|
258
|
-
* Children are created first with parent reference, then parent is created.
|
|
259
|
-
*/
|
|
260
244
|
async addFlowNode(node, parentRef) {
|
|
261
|
-
// First, create all children recursively
|
|
262
245
|
const childNodes = [];
|
|
263
246
|
const childIds = [];
|
|
264
247
|
if (node.children && node.children.length > 0) {
|
|
265
|
-
// Create a placeholder parent ID for children to reference
|
|
266
|
-
// In BullMQ, children reference the parent, not the other way around
|
|
267
248
|
const tempParentRef = { id: 'pending', queue: node.queueName };
|
|
268
249
|
for (const child of node.children) {
|
|
269
250
|
const childNode = await this.addFlowNode(child, tempParentRef);
|
|
@@ -271,369 +252,35 @@ export class FlowProducer {
|
|
|
271
252
|
childIds.push(childNode.job.id);
|
|
272
253
|
}
|
|
273
254
|
}
|
|
274
|
-
// Create the job data with parent info if this is a child
|
|
275
255
|
const jobData = {
|
|
276
256
|
name: node.name,
|
|
277
257
|
...node.data,
|
|
278
258
|
};
|
|
279
|
-
// Add parent reference if this job has a parent
|
|
280
259
|
if (parentRef) {
|
|
281
260
|
jobData.__parentId = parentRef.id;
|
|
282
261
|
jobData.__parentQueue = parentRef.queue;
|
|
283
262
|
}
|
|
284
|
-
|
|
285
|
-
if (childIds.length > 0) {
|
|
263
|
+
if (childIds.length > 0)
|
|
286
264
|
jobData.__childrenIds = childIds;
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
return {
|
|
293
|
-
job,
|
|
294
|
-
children: childNodes.length > 0 ? childNodes : undefined,
|
|
295
|
-
};
|
|
296
|
-
}
|
|
297
|
-
/** Push a job with parent/children tracking */
|
|
298
|
-
async pushJobWithParent(queueName, data, opts, parentRef, childIds) {
|
|
299
|
-
if (this.embedded) {
|
|
300
|
-
const manager = getSharedManager();
|
|
301
|
-
// Parse removeOnComplete/removeOnFail (can be boolean, number, or KeepJobs)
|
|
302
|
-
const removeOnComplete = typeof opts.removeOnComplete === 'boolean' ? opts.removeOnComplete : false;
|
|
303
|
-
const removeOnFail = typeof opts.removeOnFail === 'boolean' ? opts.removeOnFail : false;
|
|
304
|
-
const job = await manager.push(queueName, {
|
|
305
|
-
data,
|
|
306
|
-
priority: opts.priority,
|
|
307
|
-
delay: opts.delay,
|
|
308
|
-
maxAttempts: opts.attempts,
|
|
309
|
-
backoff: opts.backoff,
|
|
310
|
-
timeout: opts.timeout,
|
|
311
|
-
customId: opts.jobId,
|
|
312
|
-
removeOnComplete,
|
|
313
|
-
removeOnFail,
|
|
314
|
-
parentId: parentRef ? jobId(parentRef.id) : undefined,
|
|
315
|
-
// Note: childrenIds tracking happens via job.childrenIds field
|
|
316
|
-
});
|
|
317
|
-
// If this job has children, update the children's parent references
|
|
318
|
-
// with the actual parent ID (they were created with 'pending')
|
|
319
|
-
if (childIds.length > 0) {
|
|
320
|
-
for (const childIdStr of childIds) {
|
|
321
|
-
await manager.updateJobParent(jobId(childIdStr), job.id);
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
return String(job.id);
|
|
325
|
-
}
|
|
326
|
-
// TCP mode
|
|
327
|
-
if (!this.tcp)
|
|
328
|
-
throw new Error('TCP connection not initialized');
|
|
329
|
-
const response = await this.tcp.send({
|
|
330
|
-
cmd: 'PUSH',
|
|
331
|
-
queue: queueName,
|
|
332
|
-
data,
|
|
333
|
-
priority: opts.priority,
|
|
334
|
-
delay: opts.delay,
|
|
335
|
-
maxAttempts: opts.attempts,
|
|
336
|
-
backoff: opts.backoff,
|
|
337
|
-
timeout: opts.timeout,
|
|
338
|
-
jobId: opts.jobId,
|
|
339
|
-
removeOnComplete: opts.removeOnComplete,
|
|
340
|
-
removeOnFail: opts.removeOnFail,
|
|
341
|
-
parentId: parentRef?.id,
|
|
265
|
+
const jobIdStr = await pushJobWithParent(this.pushCtx, {
|
|
266
|
+
queueName: node.queueName,
|
|
267
|
+
data: jobData,
|
|
268
|
+
opts: node.opts ?? {},
|
|
269
|
+
parentRef,
|
|
342
270
|
childIds,
|
|
343
271
|
});
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
}
|
|
347
|
-
return response.id;
|
|
348
|
-
}
|
|
349
|
-
/** Create a simple Job object */
|
|
350
|
-
createJobObject(id, name, data, queueName) {
|
|
351
|
-
const ts = Date.now();
|
|
352
|
-
return {
|
|
353
|
-
id,
|
|
354
|
-
name,
|
|
355
|
-
data,
|
|
356
|
-
queueName,
|
|
357
|
-
attemptsMade: 0,
|
|
358
|
-
timestamp: ts,
|
|
359
|
-
progress: 0,
|
|
360
|
-
// BullMQ v5 properties
|
|
361
|
-
delay: 0,
|
|
362
|
-
processedOn: undefined,
|
|
363
|
-
finishedOn: undefined,
|
|
364
|
-
stacktrace: null,
|
|
365
|
-
stalledCounter: 0,
|
|
366
|
-
priority: 0,
|
|
367
|
-
parentKey: undefined,
|
|
368
|
-
opts: {},
|
|
369
|
-
token: undefined,
|
|
370
|
-
processedBy: undefined,
|
|
371
|
-
deduplicationId: undefined,
|
|
372
|
-
repeatJobKey: undefined,
|
|
373
|
-
attemptsStarted: 0,
|
|
374
|
-
// Methods
|
|
375
|
-
updateProgress: () => Promise.resolve(),
|
|
376
|
-
log: () => Promise.resolve(),
|
|
377
|
-
getState: () => Promise.resolve('waiting'),
|
|
378
|
-
remove: () => Promise.resolve(),
|
|
379
|
-
retry: () => Promise.resolve(),
|
|
380
|
-
getChildrenValues: () => Promise.resolve({}),
|
|
381
|
-
// BullMQ v5 state check methods
|
|
382
|
-
isWaiting: () => Promise.resolve(true),
|
|
383
|
-
isActive: () => Promise.resolve(false),
|
|
384
|
-
isDelayed: () => Promise.resolve(false),
|
|
385
|
-
isCompleted: () => Promise.resolve(false),
|
|
386
|
-
isFailed: () => Promise.resolve(false),
|
|
387
|
-
isWaitingChildren: () => Promise.resolve(false),
|
|
388
|
-
// BullMQ v5 mutation methods
|
|
389
|
-
updateData: () => Promise.resolve(),
|
|
390
|
-
promote: () => Promise.resolve(),
|
|
391
|
-
changeDelay: () => Promise.resolve(),
|
|
392
|
-
changePriority: () => Promise.resolve(),
|
|
393
|
-
extendLock: () => Promise.resolve(0),
|
|
394
|
-
clearLogs: () => Promise.resolve(),
|
|
395
|
-
// BullMQ v5 dependency methods
|
|
396
|
-
getDependencies: () => Promise.resolve({ processed: {}, unprocessed: [] }),
|
|
397
|
-
getDependenciesCount: () => Promise.resolve({ processed: 0, unprocessed: 0 }),
|
|
398
|
-
// BullMQ v5 serialization methods
|
|
399
|
-
toJSON: () => ({
|
|
400
|
-
id,
|
|
401
|
-
name,
|
|
402
|
-
data,
|
|
403
|
-
opts: {},
|
|
404
|
-
progress: 0,
|
|
405
|
-
delay: 0,
|
|
406
|
-
timestamp: ts,
|
|
407
|
-
attemptsMade: 0,
|
|
408
|
-
stacktrace: null,
|
|
409
|
-
queueQualifiedName: `bull:${queueName}`,
|
|
410
|
-
}),
|
|
411
|
-
asJSON: () => ({
|
|
412
|
-
id,
|
|
413
|
-
name,
|
|
414
|
-
data: JSON.stringify(data),
|
|
415
|
-
opts: '{}',
|
|
416
|
-
progress: '0',
|
|
417
|
-
delay: '0',
|
|
418
|
-
timestamp: String(ts),
|
|
419
|
-
attemptsMade: '0',
|
|
420
|
-
stacktrace: null,
|
|
421
|
-
}),
|
|
422
|
-
// BullMQ v5 move methods
|
|
423
|
-
moveToCompleted: () => Promise.resolve(null),
|
|
424
|
-
moveToFailed: () => Promise.resolve(),
|
|
425
|
-
moveToWait: () => Promise.resolve(false),
|
|
426
|
-
moveToDelayed: () => Promise.resolve(),
|
|
427
|
-
moveToWaitingChildren: () => Promise.resolve(false),
|
|
428
|
-
waitUntilFinished: () => Promise.resolve(undefined),
|
|
429
|
-
// BullMQ v5 additional methods
|
|
430
|
-
discard: () => { },
|
|
431
|
-
getFailedChildrenValues: () => Promise.resolve({}),
|
|
432
|
-
getIgnoredChildrenFailures: () => Promise.resolve({}),
|
|
433
|
-
removeChildDependency: () => Promise.resolve(false),
|
|
434
|
-
removeDeduplicationKey: () => Promise.resolve(false),
|
|
435
|
-
removeUnprocessedChildren: () => Promise.resolve(),
|
|
436
|
-
};
|
|
437
|
-
}
|
|
438
|
-
/** Push a job via embedded manager or TCP */
|
|
439
|
-
async pushJob(queueName, data, opts = {}, dependsOn) {
|
|
440
|
-
if (this.embedded) {
|
|
441
|
-
const manager = getSharedManager();
|
|
442
|
-
// Parse removeOnComplete/removeOnFail (can be boolean, number, or KeepJobs)
|
|
443
|
-
const removeOnComplete = typeof opts.removeOnComplete === 'boolean' ? opts.removeOnComplete : false;
|
|
444
|
-
const removeOnFail = typeof opts.removeOnFail === 'boolean' ? opts.removeOnFail : false;
|
|
445
|
-
const job = await manager.push(queueName, {
|
|
446
|
-
data,
|
|
447
|
-
priority: opts.priority,
|
|
448
|
-
delay: opts.delay,
|
|
449
|
-
maxAttempts: opts.attempts,
|
|
450
|
-
backoff: opts.backoff,
|
|
451
|
-
timeout: opts.timeout,
|
|
452
|
-
customId: opts.jobId,
|
|
453
|
-
removeOnComplete,
|
|
454
|
-
removeOnFail,
|
|
455
|
-
dependsOn: dependsOn?.map((id) => jobId(id)),
|
|
456
|
-
});
|
|
457
|
-
return String(job.id);
|
|
458
|
-
}
|
|
459
|
-
// TCP mode - tcp is guaranteed to exist when not embedded
|
|
460
|
-
if (!this.tcp)
|
|
461
|
-
throw new Error('TCP connection not initialized');
|
|
462
|
-
const response = await this.tcp.send({
|
|
463
|
-
cmd: 'PUSH',
|
|
464
|
-
queue: queueName,
|
|
465
|
-
data,
|
|
466
|
-
priority: opts.priority,
|
|
467
|
-
delay: opts.delay,
|
|
468
|
-
maxAttempts: opts.attempts,
|
|
469
|
-
backoff: opts.backoff,
|
|
470
|
-
timeout: opts.timeout,
|
|
471
|
-
jobId: opts.jobId,
|
|
472
|
-
removeOnComplete: opts.removeOnComplete,
|
|
473
|
-
removeOnFail: opts.removeOnFail,
|
|
474
|
-
dependsOn,
|
|
475
|
-
});
|
|
476
|
-
if (!response.ok) {
|
|
477
|
-
throw new Error(response.error ?? 'Failed to add job');
|
|
478
|
-
}
|
|
479
|
-
return response.id;
|
|
480
|
-
}
|
|
481
|
-
/**
|
|
482
|
-
* Add a chain of jobs where each depends on the previous.
|
|
483
|
-
* Jobs execute sequentially: step[0] → step[1] → step[2] → ...
|
|
484
|
-
*/
|
|
485
|
-
async addChain(steps) {
|
|
486
|
-
if (steps.length === 0) {
|
|
487
|
-
return { jobIds: [] };
|
|
488
|
-
}
|
|
489
|
-
const jobIds = [];
|
|
490
|
-
let prevId = null;
|
|
491
|
-
try {
|
|
492
|
-
for (const step of steps) {
|
|
493
|
-
const merged = step.opts ?? {};
|
|
494
|
-
const data = { name: step.name, __flowParentId: prevId, ...step.data };
|
|
495
|
-
const id = await this.pushJob(step.queueName, data, merged, prevId ? [prevId] : undefined);
|
|
496
|
-
jobIds.push(id);
|
|
497
|
-
prevId = id;
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
catch (error) {
|
|
501
|
-
// Cleanup already-created jobs on failure
|
|
502
|
-
await this.cleanupJobs(jobIds);
|
|
503
|
-
throw error;
|
|
504
|
-
}
|
|
505
|
-
return { jobIds };
|
|
506
|
-
}
|
|
507
|
-
/**
|
|
508
|
-
* Add parallel jobs that all converge to a final job.
|
|
509
|
-
* Parallel jobs run concurrently, final job runs after all complete.
|
|
510
|
-
*
|
|
511
|
-
* @example
|
|
512
|
-
* ```
|
|
513
|
-
* parallel[0] ──┐
|
|
514
|
-
* parallel[1] ──┼──→ final
|
|
515
|
-
* parallel[2] ──┘
|
|
516
|
-
* ```
|
|
517
|
-
*/
|
|
518
|
-
async addBulkThen(parallel, final) {
|
|
519
|
-
// Create parallel jobs (no dependencies)
|
|
520
|
-
const parallelIds = [];
|
|
521
|
-
try {
|
|
522
|
-
for (const step of parallel) {
|
|
523
|
-
const merged = step.opts ?? {};
|
|
524
|
-
const data = { name: step.name, ...step.data };
|
|
525
|
-
const id = await this.pushJob(step.queueName, data, merged);
|
|
526
|
-
parallelIds.push(id);
|
|
527
|
-
}
|
|
528
|
-
// Create final job with dependencies on all parallel jobs
|
|
529
|
-
const finalMerged = final.opts ?? {};
|
|
530
|
-
const finalData = {
|
|
531
|
-
name: final.name,
|
|
532
|
-
__flowParentIds: parallelIds,
|
|
533
|
-
...final.data,
|
|
534
|
-
};
|
|
535
|
-
const finalId = await this.pushJob(final.queueName, finalData, finalMerged, parallelIds);
|
|
536
|
-
return {
|
|
537
|
-
parallelIds,
|
|
538
|
-
finalId,
|
|
539
|
-
};
|
|
540
|
-
}
|
|
541
|
-
catch (error) {
|
|
542
|
-
// Cleanup already-created parallel jobs on failure
|
|
543
|
-
await this.cleanupJobs(parallelIds);
|
|
544
|
-
throw error;
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
/**
|
|
548
|
-
* Add a tree of jobs where children depend on parent.
|
|
549
|
-
* Recursively creates nested dependencies.
|
|
550
|
-
*/
|
|
551
|
-
async addTree(root) {
|
|
552
|
-
const jobIds = [];
|
|
553
|
-
try {
|
|
554
|
-
await this.addTreeNode(root, null, jobIds);
|
|
555
|
-
}
|
|
556
|
-
catch (error) {
|
|
557
|
-
// Cleanup already-created jobs on failure
|
|
558
|
-
await this.cleanupJobs(jobIds);
|
|
559
|
-
throw error;
|
|
560
|
-
}
|
|
561
|
-
return { jobIds };
|
|
272
|
+
const job = createFlowJobObject(jobIdStr, node.name, node.data, node.queueName);
|
|
273
|
+
return { job, children: childNodes.length > 0 ? childNodes : undefined };
|
|
562
274
|
}
|
|
563
275
|
async addTreeNode(step, parentId, jobIds) {
|
|
564
|
-
const merged = step.opts ?? {};
|
|
565
276
|
const data = { name: step.name, __flowParentId: parentId, ...step.data };
|
|
566
|
-
const id = await this.
|
|
277
|
+
const id = await pushJob(this.pushCtx, step.queueName, data, step.opts ?? {}, parentId ? [parentId] : undefined);
|
|
567
278
|
jobIds.push(id);
|
|
568
|
-
// Create children with this job as parent
|
|
569
279
|
if (step.children) {
|
|
570
|
-
for (const child of step.children)
|
|
280
|
+
for (const child of step.children)
|
|
571
281
|
await this.addTreeNode(child, id, jobIds);
|
|
572
|
-
}
|
|
573
282
|
}
|
|
574
283
|
return id;
|
|
575
284
|
}
|
|
576
|
-
/**
|
|
577
|
-
* Cleanup jobs that were created before a failure occurred.
|
|
578
|
-
* Cancels each job to prevent orphaned jobs in the queue.
|
|
579
|
-
*/
|
|
580
|
-
async cleanupJobs(jobIds) {
|
|
581
|
-
if (jobIds.length === 0)
|
|
582
|
-
return;
|
|
583
|
-
if (this.embedded) {
|
|
584
|
-
const manager = getSharedManager();
|
|
585
|
-
const cleanupPromises = jobIds.map(async (id) => {
|
|
586
|
-
try {
|
|
587
|
-
await manager.cancel(jobId(id));
|
|
588
|
-
}
|
|
589
|
-
catch {
|
|
590
|
-
// Ignore errors during cleanup - job may already be processed or removed
|
|
591
|
-
}
|
|
592
|
-
});
|
|
593
|
-
await Promise.all(cleanupPromises);
|
|
594
|
-
}
|
|
595
|
-
else if (this.tcp) {
|
|
596
|
-
// TCP mode cleanup
|
|
597
|
-
const cleanupPromises = jobIds.map(async (id) => {
|
|
598
|
-
try {
|
|
599
|
-
await this.tcp?.send({ cmd: 'Cancel', id });
|
|
600
|
-
}
|
|
601
|
-
catch {
|
|
602
|
-
// Ignore errors during cleanup
|
|
603
|
-
}
|
|
604
|
-
});
|
|
605
|
-
await Promise.all(cleanupPromises);
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
|
-
/**
|
|
609
|
-
* Get the result of a completed parent job.
|
|
610
|
-
* Call this from within a worker to access the previous step's result.
|
|
611
|
-
* Note: Only works in embedded mode.
|
|
612
|
-
*/
|
|
613
|
-
getParentResult(parentId) {
|
|
614
|
-
if (!this.embedded) {
|
|
615
|
-
throw new Error('getParentResult is only available in embedded mode');
|
|
616
|
-
}
|
|
617
|
-
const manager = getSharedManager();
|
|
618
|
-
return manager.getResult(jobId(parentId));
|
|
619
|
-
}
|
|
620
|
-
/**
|
|
621
|
-
* Get results from multiple parent jobs (for merge scenarios).
|
|
622
|
-
* Note: Only works in embedded mode.
|
|
623
|
-
*/
|
|
624
|
-
getParentResults(parentIds) {
|
|
625
|
-
if (!this.embedded) {
|
|
626
|
-
throw new Error('getParentResults is only available in embedded mode');
|
|
627
|
-
}
|
|
628
|
-
const manager = getSharedManager();
|
|
629
|
-
const results = new Map();
|
|
630
|
-
for (const id of parentIds) {
|
|
631
|
-
const result = manager.getResult(jobId(id));
|
|
632
|
-
if (result !== undefined) {
|
|
633
|
-
results.set(id, result);
|
|
634
|
-
}
|
|
635
|
-
}
|
|
636
|
-
return results;
|
|
637
|
-
}
|
|
638
285
|
}
|
|
639
286
|
//# sourceMappingURL=flow.js.map
|