bunqueue 2.8.49 → 2.8.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +19 -9
  2. package/dist/application/contextFactory.d.ts +4 -2
  3. package/dist/application/contextFactory.js +3 -0
  4. package/dist/application/flowFailureRecovery.d.ts +17 -0
  5. package/dist/application/flowFailureRecovery.js +92 -0
  6. package/dist/application/operations/ack.d.ts +3 -2
  7. package/dist/application/operations/ack.js +34 -8
  8. package/dist/application/operations/flowPush.d.ts +9 -0
  9. package/dist/application/operations/flowPush.js +112 -0
  10. package/dist/application/operations/flowTopologyValidation.d.ts +3 -0
  11. package/dist/application/operations/flowTopologyValidation.js +101 -0
  12. package/dist/application/operations/flowValidation.d.ts +3 -0
  13. package/dist/application/operations/flowValidation.js +166 -0
  14. package/dist/application/operations/push.d.ts +3 -0
  15. package/dist/application/operations/push.js +2 -2
  16. package/dist/application/operations/pushInsert.d.ts +7 -1
  17. package/dist/application/operations/pushInsert.js +18 -8
  18. package/dist/application/operations/pushLocks.d.ts +1 -0
  19. package/dist/application/operations/pushLocks.js +5 -1
  20. package/dist/application/operations/queryOperations.js +7 -6
  21. package/dist/application/queueManager.d.ts +5 -0
  22. package/dist/application/queueManager.js +195 -60
  23. package/dist/cli/commandRouter.d.ts +1 -1
  24. package/dist/client/flow.d.ts +3 -6
  25. package/dist/client/flow.js +71 -210
  26. package/dist/client/flowAtomic.d.ts +9 -0
  27. package/dist/client/flowAtomic.js +24 -0
  28. package/dist/client/flowJobCoreMethods.d.ts +28 -0
  29. package/dist/client/flowJobCoreMethods.js +151 -0
  30. package/dist/client/flowJobDependencies.d.ts +7 -0
  31. package/dist/client/flowJobDependencies.js +64 -0
  32. package/dist/client/flowJobFactory.d.ts +11 -24
  33. package/dist/client/flowJobFactory.js +91 -360
  34. package/dist/client/flowJobMoveMethods.d.ts +16 -0
  35. package/dist/client/flowJobMoveMethods.js +112 -0
  36. package/dist/client/flowJobTypes.d.ts +29 -0
  37. package/dist/client/flowJobTypes.js +6 -0
  38. package/dist/client/flowLegacyPlan.d.ts +16 -0
  39. package/dist/client/flowLegacyPlan.js +118 -0
  40. package/dist/client/flowOptions.d.ts +10 -0
  41. package/dist/client/flowOptions.js +44 -0
  42. package/dist/client/flowPlan.d.ts +16 -0
  43. package/dist/client/flowPlan.js +93 -0
  44. package/dist/client/flowPush.js +4 -1
  45. package/dist/client/flowReader.d.ts +10 -0
  46. package/dist/client/flowReader.js +147 -0
  47. package/dist/client/jobHelpers.js +2 -0
  48. package/dist/client/workflow/clock.d.ts +3 -1
  49. package/dist/client/workflow/clock.js +18 -1
  50. package/dist/client/workflow/compensationChild.d.ts +30 -0
  51. package/dist/client/workflow/compensationChild.js +65 -0
  52. package/dist/client/workflow/compensationPass.d.ts +16 -0
  53. package/dist/client/workflow/compensationPass.js +96 -0
  54. package/dist/client/workflow/compensationSupport.d.ts +24 -0
  55. package/dist/client/workflow/compensationSupport.js +57 -0
  56. package/dist/client/workflow/compensator.d.ts +11 -50
  57. package/dist/client/workflow/compensator.js +22 -374
  58. package/dist/client/workflow/definitionGuard.d.ts +11 -0
  59. package/dist/client/workflow/definitionGuard.js +22 -0
  60. package/dist/client/workflow/engine.d.ts +2 -2
  61. package/dist/client/workflow/engine.js +8 -8
  62. package/dist/client/workflow/eventTypes.d.ts +44 -0
  63. package/dist/client/workflow/eventTypes.js +2 -0
  64. package/dist/client/workflow/executionTypes.d.ts +133 -0
  65. package/dist/client/workflow/executionTypes.js +2 -0
  66. package/dist/client/workflow/executor.d.ts +4 -17
  67. package/dist/client/workflow/executor.js +50 -208
  68. package/dist/client/workflow/executorLifecycle.d.ts +18 -0
  69. package/dist/client/workflow/executorLifecycle.js +69 -0
  70. package/dist/client/workflow/executorNodes.d.ts +17 -0
  71. package/dist/client/workflow/executorNodes.js +138 -0
  72. package/dist/client/workflow/identity.d.ts +2 -0
  73. package/dist/client/workflow/identity.js +8 -0
  74. package/dist/client/workflow/index.d.ts +1 -1
  75. package/dist/client/workflow/loops.d.ts +3 -5
  76. package/dist/client/workflow/loops.js +25 -23
  77. package/dist/client/workflow/mapRunner.d.ts +4 -0
  78. package/dist/client/workflow/mapRunner.js +45 -0
  79. package/dist/client/workflow/recovery.js +10 -3
  80. package/dist/client/workflow/runner.d.ts +2 -21
  81. package/dist/client/workflow/runner.js +39 -104
  82. package/dist/client/workflow/runnerTiming.d.ts +14 -0
  83. package/dist/client/workflow/runnerTiming.js +66 -0
  84. package/dist/client/workflow/stepTypes.d.ts +171 -0
  85. package/dist/client/workflow/stepTypes.js +3 -0
  86. package/dist/client/workflow/store.d.ts +17 -17
  87. package/dist/client/workflow/store.js +65 -100
  88. package/dist/client/workflow/storeExecutionCodec.d.ts +11 -0
  89. package/dist/client/workflow/storeExecutionCodec.js +34 -0
  90. package/dist/client/workflow/storeListing.d.ts +11 -0
  91. package/dist/client/workflow/storeListing.js +45 -0
  92. package/dist/client/workflow/storeMaintenance.d.ts +4 -0
  93. package/dist/client/workflow/storeMaintenance.js +40 -0
  94. package/dist/client/workflow/storeSignals.d.ts +9 -0
  95. package/dist/client/workflow/storeSignals.js +38 -2
  96. package/dist/client/workflow/subWorkflowRunner.d.ts +13 -0
  97. package/dist/client/workflow/subWorkflowRunner.js +40 -0
  98. package/dist/client/workflow/types.d.ts +4 -356
  99. package/dist/client/workflow/types.js +1 -3
  100. package/dist/client/workflow/waitFor.js +39 -26
  101. package/dist/client/workflow/workflow.d.ts +16 -59
  102. package/dist/client/workflow/workflow.js +53 -179
  103. package/dist/client/workflow/workflowDecisions.d.ts +11 -0
  104. package/dist/client/workflow/workflowDecisions.js +27 -0
  105. package/dist/client/workflow/workflowDefinition.d.ts +16 -0
  106. package/dist/client/workflow/workflowDefinition.js +123 -0
  107. package/dist/client/workflow/workflowIntrospection.d.ts +5 -0
  108. package/dist/client/workflow/workflowIntrospection.js +46 -0
  109. package/dist/client/workflow/workflowValidation.d.ts +44 -0
  110. package/dist/client/workflow/workflowValidation.js +143 -0
  111. package/dist/domain/types/command.d.ts +7 -1
  112. package/dist/domain/types/flow.d.ts +25 -0
  113. package/dist/domain/types/flow.js +1 -0
  114. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  115. package/dist/infrastructure/persistence/schema.js +52 -3
  116. package/dist/infrastructure/persistence/sqlite.d.ts +16 -0
  117. package/dist/infrastructure/persistence/sqlite.js +129 -7
  118. package/dist/infrastructure/persistence/sqliteBatch.js +10 -4
  119. package/dist/infrastructure/persistence/sqliteSerializer.d.ts +2 -0
  120. package/dist/infrastructure/persistence/sqliteSerializer.js +18 -5
  121. package/dist/infrastructure/persistence/statements.d.ts +4 -0
  122. package/dist/infrastructure/persistence/statements.js +8 -2
  123. package/dist/infrastructure/server/handlerRoutes.js +3 -0
  124. package/dist/infrastructure/server/handlers/advanced.js +2 -2
  125. package/dist/infrastructure/server/handlers/flow.d.ts +7 -0
  126. package/dist/infrastructure/server/handlers/flow.js +11 -0
  127. package/dist/infrastructure/server/handlers/index.d.ts +1 -0
  128. package/dist/infrastructure/server/handlers/index.js +1 -0
  129. package/package.json +4 -2
@@ -6,9 +6,13 @@ import { EventEmitter } from 'events';
6
6
  import { getSharedManager } from './manager';
7
7
  import { TcpConnectionPool, getSharedPool, releaseSharedPool } from './tcpPool';
8
8
  import { jobId } from '../domain/types/job';
9
- import { createFlowJobObject, extractUserDataFromInternal, } from './flowJobFactory';
10
- import { pushJob, pushJobWithParent, cleanupJobs } from './flowPush';
9
+ import { createFlowJobObject } from './flowJobFactory';
11
10
  import * as managementOps from './queue/operations/management';
11
+ import { commitFlow } from './flowAtomic';
12
+ import { assertFlowTcpOk } from './flowJobTypes';
13
+ import { planFlows } from './flowPlan';
14
+ import { planBulkThen, planChain, planTree } from './flowLegacyPlan';
15
+ import { readFlow } from './flowReader';
12
16
  const FORCE_EMBEDDED = Bun.env.BUNQUEUE_EMBEDDED === '1';
13
17
  /**
14
18
  * FlowProducer creates job flows with automatic dependencies.
@@ -39,6 +43,7 @@ export class FlowProducer extends EventEmitter {
39
43
  embedded;
40
44
  tcp;
41
45
  useSharedPool;
46
+ closed = false;
42
47
  constructor(opts = {}) {
43
48
  super();
44
49
  this.embedded = opts.embedded ?? FORCE_EMBEDDED;
@@ -56,6 +61,8 @@ export class FlowProducer extends EventEmitter {
56
61
  poolSize,
57
62
  pingInterval: connOpts.pingInterval,
58
63
  commandTimeout: connOpts.commandTimeout,
64
+ maxCommandTimeouts: connOpts.maxCommandTimeouts,
65
+ tls: connOpts.tls,
59
66
  pipelining: connOpts.pipelining,
60
67
  maxInFlight: connOpts.maxInFlight,
61
68
  });
@@ -69,6 +76,8 @@ export class FlowProducer extends EventEmitter {
69
76
  poolSize,
70
77
  pingInterval: connOpts.pingInterval,
71
78
  commandTimeout: connOpts.commandTimeout,
79
+ maxCommandTimeouts: connOpts.maxCommandTimeouts,
80
+ tls: connOpts.tls,
72
81
  pipelining: connOpts.pipelining,
73
82
  maxInFlight: connOpts.maxInFlight,
74
83
  });
@@ -82,6 +91,9 @@ export class FlowProducer extends EventEmitter {
82
91
  }
83
92
  /** Close the connection pool (only if using dedicated pool) */
84
93
  close() {
94
+ if (this.closed)
95
+ return this.closing;
96
+ this.closed = true;
85
97
  if (this.tcp && !this.useSharedPool) {
86
98
  this.tcp.close();
87
99
  }
@@ -100,49 +112,31 @@ export class FlowProducer extends EventEmitter {
100
112
  if (this.embedded)
101
113
  return;
102
114
  if (this.tcp)
103
- await this.tcp.send({ cmd: 'Ping' });
115
+ assertFlowTcpOk(await this.tcp.send({ cmd: 'Ping' }), 'Ping');
104
116
  }
105
117
  // ============================================================================
106
118
  // BullMQ v5 Compatible Methods
107
119
  // ============================================================================
108
120
  /** Add a flow (BullMQ v5 compatible). Children are processed BEFORE their parent. */
109
121
  async add(flow, opts) {
110
- const createdJobIds = [];
111
- try {
112
- return await this.addFlowNode(flow, null, createdJobIds, opts);
113
- }
114
- catch (error) {
115
- // Atomic rollback: clean up all created jobs
116
- await cleanupJobs(this.pushCtx, createdJobIds);
117
- throw error;
118
- }
122
+ const plan = planFlows([flow], opts);
123
+ const snapshots = await commitFlow(this.pushCtx, plan.batch);
124
+ return this.buildPlannedNode(plan.roots[0], this.indexSnapshots(snapshots));
119
125
  }
120
126
  /** Add multiple flows (BullMQ v5 compatible). */
121
127
  async addBulk(flows) {
122
- const results = [];
123
- const allCreatedJobIds = [];
124
- try {
125
- for (const flow of flows) {
126
- const createdJobIds = [];
127
- const result = await this.addFlowNode(flow, null, createdJobIds);
128
- allCreatedJobIds.push(...createdJobIds);
129
- results.push(result);
130
- }
131
- return results;
132
- }
133
- catch (error) {
134
- // Atomic rollback: clean up all created jobs from all flows
135
- await cleanupJobs(this.pushCtx, allCreatedJobIds);
136
- throw error;
137
- }
128
+ const plan = planFlows(flows);
129
+ const snapshots = await commitFlow(this.pushCtx, plan.batch);
130
+ const snapshotsById = this.indexSnapshots(snapshots);
131
+ return plan.roots.map((root) => this.buildPlannedNode(root, snapshotsById));
138
132
  }
139
133
  /** Get a flow tree starting from a job (BullMQ v5 compatible). */
140
134
  async getFlow(opts) {
141
- const { id, queueName, depth, maxChildren } = opts;
142
- if (this.embedded) {
143
- return this.getFlowEmbedded(id, queueName, depth ?? Infinity, maxChildren);
144
- }
145
- return this.getFlowTcp(id, queueName, depth ?? Infinity, maxChildren);
135
+ return readFlow({
136
+ embedded: this.embedded,
137
+ tcp: this.tcp,
138
+ buildCallbacks: (queueName) => this.buildCallbacks(queueName),
139
+ }, opts);
146
140
  }
147
141
  // ============================================================================
148
142
  // Legacy bunqueue API Methods
@@ -151,80 +145,24 @@ export class FlowProducer extends EventEmitter {
151
145
  async addChain(steps) {
152
146
  if (steps.length === 0)
153
147
  return { jobIds: [] };
154
- const jobIds = [];
155
- let prevId = null;
156
- try {
157
- for (const step of steps) {
158
- const data = { name: step.name, __flowParentId: prevId, ...step.data };
159
- const id = await pushJob(this.pushCtx, step.queueName, data, step.opts ?? {}, prevId ? [prevId] : undefined);
160
- jobIds.push(id);
161
- prevId = id;
162
- }
163
- }
164
- catch (error) {
165
- await cleanupJobs(this.pushCtx, jobIds);
166
- throw error;
167
- }
168
- return { jobIds };
148
+ const plan = planChain(steps);
149
+ await commitFlow(this.pushCtx, plan.batch);
150
+ return { jobIds: plan.ids.map(String) };
169
151
  }
170
152
  /** Add parallel jobs that converge to a final job. */
171
153
  async addBulkThen(parallel, final) {
172
- const parallelIds = [];
173
- try {
174
- // Push parallel jobs concurrently — they are independent by design
175
- const results = await Promise.allSettled(parallel.map(async (step) => {
176
- const data = { name: step.name, ...step.data };
177
- return pushJob(this.pushCtx, step.queueName, data, step.opts ?? {});
178
- }));
179
- // Collect successful IDs and check for failures
180
- const errors = [];
181
- for (const r of results) {
182
- if (r.status === 'fulfilled') {
183
- parallelIds.push(r.value);
184
- }
185
- else {
186
- errors.push(r.reason);
187
- }
188
- }
189
- if (errors.length > 0) {
190
- // Clean up successfully created jobs before throwing
191
- await cleanupJobs(this.pushCtx, parallelIds);
192
- throw errors[0] instanceof Error ? errors[0] : new Error(String(errors[0]));
193
- }
194
- const finalData = {
195
- name: final.name,
196
- __flowParentIds: parallelIds,
197
- ...final.data,
198
- };
199
- // pushJobWithParent (not pushJob) so the final job records childrenIds =
200
- // parallelIds: this keeps the same dependsOn ordering (waits for all
201
- // parallel jobs) AND lets the merge step read their results via
202
- // getChildrenValues()/getDependencies(), like a BullMQ fan-in parent.
203
- const finalId = await pushJobWithParent(this.pushCtx, {
204
- queueName: final.queueName,
205
- data: finalData,
206
- opts: final.opts ?? {},
207
- parentRef: null,
208
- childIds: parallelIds,
209
- });
210
- return { parallelIds, finalId };
211
- }
212
- catch (error) {
213
- await cleanupJobs(this.pushCtx, parallelIds);
214
- throw error;
215
- }
154
+ const plan = planBulkThen(parallel, final);
155
+ await commitFlow(this.pushCtx, plan.batch);
156
+ return {
157
+ parallelIds: plan.parallelIds.map(String),
158
+ finalId: String(plan.finalId),
159
+ };
216
160
  }
217
161
  /** Add a tree of jobs where children depend on parent. */
218
162
  async addTree(root) {
219
- const jobIds = [];
220
- try {
221
- await this.addTreeNode(root, null, jobIds);
222
- }
223
- catch (error) {
224
- await cleanupJobs(this.pushCtx, jobIds);
225
- throw error;
226
- }
227
- return { jobIds };
163
+ const plan = planTree(root);
164
+ await commitFlow(this.pushCtx, plan.batch);
165
+ return { jobIds: plan.ids.map(String) };
228
166
  }
229
167
  /**
230
168
  * Get the result of a completed parent job (embedded only).
@@ -258,9 +196,24 @@ export class FlowProducer extends EventEmitter {
258
196
  /** Build callbacks that wire flow job methods to actual management operations */
259
197
  buildCallbacks(queueName) {
260
198
  const ctx = { name: queueName, embedded: this.embedded, tcp: this.tcp };
199
+ if (!this.embedded) {
200
+ return {
201
+ embedded: false,
202
+ tcp: this.tcp,
203
+ getState: (id) => {
204
+ if (!this.tcp)
205
+ return Promise.resolve('unknown');
206
+ return this.tcp.send({ cmd: 'GetState', id }).then((response) => {
207
+ if (response.ok !== true) {
208
+ throw new Error(typeof response.error === 'string' ? response.error : 'GetState failed');
209
+ }
210
+ return typeof response.state === 'string' ? response.state : 'unknown';
211
+ });
212
+ },
213
+ };
214
+ }
261
215
  return {
262
- embedded: this.embedded,
263
- tcp: this.tcp,
216
+ embedded: true,
264
217
  updateData: (id, data) => managementOps.updateJobData(ctx, id, data),
265
218
  updateProgress: (id, progress) => managementOps.updateJobProgress(ctx, id, progress),
266
219
  log: (id, msg) => managementOps.addJobLog(ctx, id, msg).then(() => { }),
@@ -268,117 +221,25 @@ export class FlowProducer extends EventEmitter {
268
221
  remove: (id) => managementOps.removeAsync(ctx, id),
269
222
  changePriority: (id, opts) => managementOps.changeJobPriority(ctx, id, opts),
270
223
  changeDelay: (id, d) => managementOps.changeJobDelay(ctx, id, d),
271
- clearLogs: (id) => managementOps.clearJobLogs(ctx, id),
224
+ clearLogs: (id, keepLogs) => managementOps.clearJobLogs(ctx, id, keepLogs),
272
225
  retry: (id) => managementOps.retryJob(ctx, id),
273
- getState: (id) => {
274
- if (this.embedded) {
275
- return getSharedManager().getJobState(jobId(id));
276
- }
277
- if (!this.tcp)
278
- return Promise.resolve('unknown');
279
- return this.tcp
280
- .send({ cmd: 'GetState', id })
281
- .then((r) => (typeof r.state === 'string' ? r.state : 'unknown'));
282
- },
226
+ getState: (id) => getSharedManager().getJobState(jobId(id)),
283
227
  };
284
228
  }
285
- async getFlowEmbedded(id, queueName, depth, maxChildren) {
286
- const job = await getSharedManager().getJob(jobId(id));
287
- if (job?.queue !== queueName)
288
- return null;
289
- return this.buildJobNode(job, depth, maxChildren);
290
- }
291
- async getFlowTcp(id, queueName, depth, maxChildren) {
292
- if (!this.tcp)
293
- throw new Error('TCP connection not initialized');
294
- const response = await this.tcp.send({ cmd: 'GetJob', id });
295
- if (!response.ok || !response.job)
296
- return null;
297
- const jobData = response.job;
298
- if (jobData.queue !== queueName)
299
- return null;
300
- return this.buildJobNodeFromTcp(jobData, depth, maxChildren);
301
- }
302
- async buildJobNode(job, depth, maxChildren) {
303
- const data = job.data;
304
- const name = typeof data.name === 'string' ? data.name : 'default';
305
- const userData = extractUserDataFromInternal(data);
306
- const jobObj = createFlowJobObject(String(job.id), name, userData, job.queue, this.buildCallbacks(job.queue));
307
- if (depth <= 0 || job.childrenIds.length === 0)
308
- return { job: jobObj };
309
- const childNodes = [];
310
- const childrenToFetch = maxChildren ? job.childrenIds.slice(0, maxChildren) : job.childrenIds;
311
- for (const childId of childrenToFetch) {
312
- const childJob = await getSharedManager().getJob(childId);
313
- if (childJob)
314
- childNodes.push(await this.buildJobNode(childJob, depth - 1, maxChildren));
315
- }
316
- return { job: jobObj, children: childNodes.length > 0 ? childNodes : undefined };
317
- }
318
- async buildJobNodeFromTcp(jobData, depth, maxChildren) {
319
- const id = String(jobData.id);
320
- const queueName = String(jobData.queue);
321
- const data = jobData.data;
322
- const name = typeof data?.name === 'string' ? data.name : 'default';
323
- const userData = extractUserDataFromInternal(data ?? {});
324
- const rawChildrenIds = data?.__childrenIds;
325
- const childrenIds = Array.isArray(rawChildrenIds) ? rawChildrenIds : [];
326
- const jobObj = createFlowJobObject(id, name, userData, queueName, this.buildCallbacks(queueName));
327
- if (depth <= 0 || childrenIds.length === 0 || !this.tcp)
328
- return { job: jobObj };
329
- const childNodes = [];
330
- const childrenToFetch = maxChildren ? childrenIds.slice(0, maxChildren) : childrenIds;
331
- for (const childId of childrenToFetch) {
332
- const response = await this.tcp.send({ cmd: 'GetJob', id: childId });
333
- if (response.ok && response.job) {
334
- childNodes.push(await this.buildJobNodeFromTcp(response.job, depth - 1, maxChildren));
335
- }
336
- }
337
- return { job: jobObj, children: childNodes.length > 0 ? childNodes : undefined };
338
- }
339
- async addFlowNode(node, parentRef, createdJobIds, flowOpts) {
340
- const childNodes = [];
341
- const childIds = [];
342
- if (node.children && node.children.length > 0) {
343
- const tempParentRef = { id: 'pending', queue: node.queueName };
344
- // Siblings are independent — create them concurrently
345
- const results = await Promise.all(node.children.map((child) => this.addFlowNode(child, tempParentRef, createdJobIds, flowOpts)));
346
- for (const childNode of results) {
347
- childNodes.push(childNode);
348
- childIds.push(childNode.job.id);
349
- }
350
- }
351
- // Merge node opts with per-queue defaults from FlowOpts
352
- const queueDefaults = flowOpts?.queuesOptions?.[node.queueName];
353
- const mergedOpts = queueDefaults ? { ...queueDefaults, ...node.opts } : (node.opts ?? {});
354
- const jobData = {
355
- name: node.name,
356
- ...node.data,
229
+ buildPlannedNode(node, snapshots) {
230
+ const snapshot = snapshots.get(String(node.id));
231
+ if (!snapshot)
232
+ throw new Error(`Committed flow snapshot missing for ${String(node.id)}`);
233
+ const children = node.children?.map((child) => this.buildPlannedNode(child, snapshots));
234
+ return {
235
+ job: createFlowJobObject(String(node.id), node.name, node.data, node.queueName, {
236
+ callbacks: this.buildCallbacks(node.queueName),
237
+ snapshot,
238
+ }),
239
+ children: children && children.length > 0 ? children : undefined,
357
240
  };
358
- if (parentRef) {
359
- jobData.__parentId = parentRef.id;
360
- jobData.__parentQueue = parentRef.queue;
361
- }
362
- if (childIds.length > 0)
363
- jobData.__childrenIds = childIds;
364
- const jobIdStr = await pushJobWithParent(this.pushCtx, {
365
- queueName: node.queueName,
366
- data: jobData,
367
- opts: mergedOpts,
368
- parentRef,
369
- childIds,
370
- });
371
- createdJobIds.push(jobIdStr);
372
- const job = createFlowJobObject(jobIdStr, node.name, node.data, node.queueName, this.buildCallbacks(node.queueName));
373
- return { job, children: childNodes.length > 0 ? childNodes : undefined };
374
241
  }
375
- async addTreeNode(step, parentId, jobIds) {
376
- const data = { name: step.name, __flowParentId: parentId, ...step.data };
377
- const id = await pushJob(this.pushCtx, step.queueName, data, step.opts ?? {}, parentId ? [parentId] : undefined);
378
- jobIds.push(id);
379
- if (step.children) {
380
- await Promise.all(step.children.map((child) => this.addTreeNode(child, id, jobIds)));
381
- }
382
- return id;
242
+ indexSnapshots(snapshots) {
243
+ return new Map(snapshots.map((snapshot) => [String(snapshot.id), snapshot]));
383
244
  }
384
245
  }
@@ -0,0 +1,9 @@
1
+ import type { Job as DomainJob } from '../domain/types/job';
2
+ import type { AtomicFlowBatchInput } from '../domain/types/flow';
3
+ import type { TcpConnectionPool } from './tcpPool';
4
+ export interface AtomicFlowContext {
5
+ readonly embedded: boolean;
6
+ readonly tcp: TcpConnectionPool | null;
7
+ }
8
+ /** Execute the same one-command atomic primitive in embedded and TCP modes. */
9
+ export declare function commitFlow(ctx: AtomicFlowContext, batch: AtomicFlowBatchInput): Promise<DomainJob[]>;
@@ -0,0 +1,24 @@
1
+ import { getSharedManager } from './manager';
2
+ /** Execute the same one-command atomic primitive in embedded and TCP modes. */
3
+ export async function commitFlow(ctx, batch) {
4
+ if (batch.jobs.length === 0)
5
+ return [];
6
+ if (ctx.embedded) {
7
+ return (await getSharedManager().pushFlow(batch)).jobs;
8
+ }
9
+ if (!ctx.tcp)
10
+ throw new Error('TCP connection not initialized');
11
+ const response = await ctx.tcp.send({ cmd: 'PUSHF', jobs: batch.jobs });
12
+ if (response.ok !== true) {
13
+ throw new Error(typeof response.error === 'string' ? response.error : 'Failed to add flow');
14
+ }
15
+ const result = response.data;
16
+ if (!result || !Array.isArray(result.jobs) || result.jobs.length !== batch.jobs.length) {
17
+ throw new Error('Invalid PUSHF response: committed job snapshots are missing');
18
+ }
19
+ const expected = new Set(batch.jobs.map((job) => String(job.id)));
20
+ if (result.jobs.some((job) => !expected.delete(String(job.id))) || expected.size > 0) {
21
+ throw new Error('Invalid PUSHF response: committed job IDs do not match the request');
22
+ }
23
+ return result.jobs;
24
+ }
@@ -0,0 +1,28 @@
1
+ import type { ChangePriorityOpts } from './types';
2
+ import { type FlowJobRuntime } from './flowJobTypes';
3
+ /** Core state, mutation, and dependency methods exposed by a FlowProducer Job. */
4
+ export declare function buildFlowJobCoreMethods(runtime: FlowJobRuntime): {
5
+ updateProgress: (progress: number | object) => Promise<void>;
6
+ log: (message: string) => Promise<void>;
7
+ getState: () => Promise<import("./types").JobStateType>;
8
+ remove: () => Promise<void>;
9
+ retry: () => Promise<void>;
10
+ getChildrenValues: <R = unknown>() => Promise<Record<string, R>>;
11
+ isWaiting: () => Promise<boolean>;
12
+ isActive: () => Promise<boolean>;
13
+ isDelayed: () => Promise<boolean>;
14
+ isCompleted: () => Promise<boolean>;
15
+ isFailed: () => Promise<boolean>;
16
+ isWaitingChildren: () => Promise<boolean>;
17
+ updateData: (data: unknown) => Promise<void>;
18
+ promote: () => Promise<void>;
19
+ changeDelay: (delay: number) => Promise<void>;
20
+ changePriority: (opts: ChangePriorityOpts) => Promise<void>;
21
+ extendLock: (token: string, duration: number) => Promise<number>;
22
+ clearLogs: (keepLogs?: number) => Promise<void>;
23
+ getDependencies: () => Promise<import("./flowJobDependencies").FlowDependencies>;
24
+ getDependenciesCount: () => Promise<{
25
+ processed: number;
26
+ unprocessed: number;
27
+ }>;
28
+ };
@@ -0,0 +1,151 @@
1
+ import { jobId } from '../domain/types/job';
2
+ import { getSharedManager } from './manager';
3
+ import { getFlowDependencies } from './flowJobDependencies';
4
+ import { assertFlowTcpOk } from './flowJobTypes';
5
+ /** Core state, mutation, and dependency methods exposed by a FlowProducer Job. */
6
+ export function buildFlowJobCoreMethods(runtime) {
7
+ const { id, queueName, callbacks, embedded, tcp, getState } = runtime;
8
+ return {
9
+ updateProgress: async (progress) => {
10
+ if (callbacks?.updateProgress)
11
+ return callbacks.updateProgress(id, progress);
12
+ if (embedded) {
13
+ const value = typeof progress === 'number' ? progress : 0;
14
+ const message = typeof progress === 'object' ? JSON.stringify(progress) : undefined;
15
+ return void (await getSharedManager().updateProgress(jobId(id), value, message));
16
+ }
17
+ if (!tcp)
18
+ return;
19
+ const value = typeof progress === 'number' ? progress : 0;
20
+ const message = typeof progress === 'object' ? JSON.stringify(progress) : undefined;
21
+ assertFlowTcpOk(await tcp.send({ cmd: 'Progress', id, progress: value, message }), 'Progress');
22
+ },
23
+ log: async (message) => {
24
+ if (callbacks?.log)
25
+ return callbacks.log(id, message);
26
+ if (embedded)
27
+ return void getSharedManager().addLog(jobId(id), message);
28
+ if (!tcp)
29
+ return;
30
+ assertFlowTcpOk(await tcp.send({ cmd: 'AddLog', id, message }), 'AddLog');
31
+ },
32
+ getState,
33
+ remove: async () => {
34
+ if (callbacks?.remove)
35
+ return callbacks.remove(id);
36
+ if (embedded)
37
+ return void (await getSharedManager().cancel(jobId(id)));
38
+ if (!tcp)
39
+ return;
40
+ assertFlowTcpOk(await tcp.send({ cmd: 'Cancel', id }), 'Cancel');
41
+ },
42
+ retry: async () => {
43
+ if (callbacks?.retry)
44
+ return callbacks.retry(id);
45
+ if (embedded) {
46
+ const manager = getSharedManager();
47
+ const state = await manager.getJobState(jobId(id));
48
+ if (state === 'failed') {
49
+ if (manager.retryDlq(queueName, jobId(id)) === 0) {
50
+ throw new Error(`Job ${id} is failed but not present in DLQ`);
51
+ }
52
+ return;
53
+ }
54
+ if (state === 'active') {
55
+ if (!(await manager.moveActiveToWait(jobId(id)))) {
56
+ throw new Error(`Failed to retry active job ${id}`);
57
+ }
58
+ return;
59
+ }
60
+ if (state === 'waiting' || state === 'prioritized' || state === 'delayed')
61
+ return;
62
+ throw new Error(`Cannot retry job ${id} from state '${state}'`);
63
+ }
64
+ if (!tcp)
65
+ return;
66
+ assertFlowTcpOk(await tcp.send({ cmd: 'MoveToWait', id }), 'MoveToWait');
67
+ },
68
+ getChildrenValues: async () => {
69
+ if (embedded) {
70
+ return (await getSharedManager().getChildrenValues(jobId(id)));
71
+ }
72
+ if (!tcp)
73
+ return {};
74
+ const response = await tcp.send({ cmd: 'GetChildrenValues', id });
75
+ assertFlowTcpOk(response, 'GetChildrenValues');
76
+ const values = response.data?.values ?? {};
77
+ return values;
78
+ },
79
+ isWaiting: async () => (await getState()) === 'waiting',
80
+ isActive: async () => (await getState()) === 'active',
81
+ isDelayed: async () => (await getState()) === 'delayed',
82
+ isCompleted: async () => (await getState()) === 'completed',
83
+ isFailed: async () => (await getState()) === 'failed',
84
+ isWaitingChildren: async () => (await getState()) === 'waiting-children',
85
+ updateData: async (data) => {
86
+ if (callbacks?.updateData)
87
+ return callbacks.updateData(id, data);
88
+ if (embedded)
89
+ return void (await getSharedManager().updateJobData(jobId(id), data));
90
+ if (!tcp)
91
+ return;
92
+ assertFlowTcpOk(await tcp.send({ cmd: 'Update', id, data }), 'Update');
93
+ },
94
+ promote: async () => {
95
+ if (callbacks?.promote)
96
+ return callbacks.promote(id);
97
+ if (embedded)
98
+ return void (await getSharedManager().promote(jobId(id)));
99
+ if (!tcp)
100
+ return;
101
+ assertFlowTcpOk(await tcp.send({ cmd: 'Promote', id }), 'Promote');
102
+ },
103
+ changeDelay: async (delay) => {
104
+ if (callbacks?.changeDelay)
105
+ return callbacks.changeDelay(id, delay);
106
+ if (embedded)
107
+ return void (await getSharedManager().changeDelay(jobId(id), delay));
108
+ if (!tcp)
109
+ return;
110
+ assertFlowTcpOk(await tcp.send({ cmd: 'ChangeDelay', id, delay }), 'ChangeDelay');
111
+ },
112
+ changePriority: async (opts) => {
113
+ if (callbacks?.changePriority) {
114
+ return callbacks.changePriority(id, { priority: opts.priority, lifo: opts.lifo });
115
+ }
116
+ if (embedded) {
117
+ return void (await getSharedManager().changePriority(jobId(id), opts.priority, opts.lifo));
118
+ }
119
+ if (!tcp)
120
+ return;
121
+ assertFlowTcpOk(await tcp.send({ cmd: 'ChangePriority', id, priority: opts.priority, lifo: opts.lifo }), 'ChangePriority');
122
+ },
123
+ extendLock: async (token, duration) => {
124
+ if (embedded) {
125
+ return (await getSharedManager().extendLock(jobId(id), token, duration)) ? duration : 0;
126
+ }
127
+ if (!tcp)
128
+ return 0;
129
+ const response = await tcp.send({ cmd: 'ExtendLock', id, token, duration });
130
+ assertFlowTcpOk(response, 'ExtendLock');
131
+ return duration;
132
+ },
133
+ clearLogs: async (keepLogs) => {
134
+ if (callbacks?.clearLogs)
135
+ return callbacks.clearLogs(id, keepLogs);
136
+ if (embedded)
137
+ return void getSharedManager().clearLogs(jobId(id), keepLogs);
138
+ if (!tcp)
139
+ return;
140
+ assertFlowTcpOk(await tcp.send({ cmd: 'ClearLogs', id, keepLogs }), 'ClearLogs');
141
+ },
142
+ getDependencies: () => getFlowDependencies(id, queueName, embedded, tcp),
143
+ getDependenciesCount: async () => {
144
+ const dependencies = await getFlowDependencies(id, queueName, embedded, tcp);
145
+ return {
146
+ processed: Object.keys(dependencies.processed).length,
147
+ unprocessed: dependencies.unprocessed.length,
148
+ };
149
+ },
150
+ };
151
+ }
@@ -0,0 +1,7 @@
1
+ import type { TcpConnectionPool } from './tcpPool';
2
+ export interface FlowDependencies {
3
+ processed: Record<string, unknown>;
4
+ unprocessed: string[];
5
+ }
6
+ /** Resolve dependency keys using the actual queue of every child. */
7
+ export declare function getFlowDependencies(id: string, queueName: string, embedded: boolean, tcp: TcpConnectionPool | null): Promise<FlowDependencies>;
@@ -0,0 +1,64 @@
1
+ import { jobId } from '../domain/types/job';
2
+ import { getSharedManager } from './manager';
3
+ import { assertFlowTcpOk } from './flowJobTypes';
4
+ async function fetchChildIds(id, embedded, tcp) {
5
+ if (embedded) {
6
+ const job = await getSharedManager().getJob(jobId(id));
7
+ if (!job)
8
+ throw new Error(`Flow job ${id} not found`);
9
+ return job.childrenIds.map(String);
10
+ }
11
+ if (!tcp)
12
+ return [];
13
+ const response = await tcp.send({ cmd: 'GetJob', id });
14
+ assertFlowTcpOk(response, 'GetJob');
15
+ const parent = response.job;
16
+ return (parent?.childrenIds ?? []).map(String);
17
+ }
18
+ async function embeddedDependency(fallbackQueue, childId) {
19
+ const manager = getSharedManager();
20
+ const child = await manager.getJob(jobId(childId));
21
+ if (!child)
22
+ throw new Error(`Flow child job ${childId} not found`);
23
+ const state = await manager.getJobState(jobId(childId));
24
+ return {
25
+ key: `${child.queue || fallbackQueue}:${childId}`,
26
+ state,
27
+ result: state === 'completed' ? manager.getResult(jobId(childId)) : undefined,
28
+ };
29
+ }
30
+ async function tcpDependency(fallbackQueue, childId, tcp) {
31
+ const response = await tcp.send({ cmd: 'GetJob', id: childId });
32
+ assertFlowTcpOk(response, 'GetJob');
33
+ const child = response.job;
34
+ if (!child)
35
+ throw new Error(`Flow child job ${childId} not found`);
36
+ const state = child.state ?? 'unknown';
37
+ let result;
38
+ if (state === 'completed') {
39
+ const resultResponse = await tcp.send({ cmd: 'GetResult', id: childId });
40
+ assertFlowTcpOk(resultResponse, 'GetResult');
41
+ result = resultResponse.result;
42
+ }
43
+ return { key: `${child.queue ?? fallbackQueue}:${childId}`, state, result };
44
+ }
45
+ /** Resolve dependency keys using the actual queue of every child. */
46
+ export async function getFlowDependencies(id, queueName, embedded, tcp) {
47
+ const childIds = await fetchChildIds(id, embedded, tcp);
48
+ const processed = {};
49
+ const unprocessed = [];
50
+ for (const childId of childIds) {
51
+ const dependency = embedded
52
+ ? await embeddedDependency(queueName, childId)
53
+ : tcp
54
+ ? await tcpDependency(queueName, childId, tcp)
55
+ : { key: `${queueName}:${childId}`, state: 'unknown', result: undefined };
56
+ if (dependency.state === 'completed' || dependency.state === 'failed') {
57
+ processed[dependency.key] = dependency.result ?? null;
58
+ }
59
+ else {
60
+ unprocessed.push(dependency.key);
61
+ }
62
+ }
63
+ return { processed, unprocessed };
64
+ }