bunqueue 2.8.49 → 2.8.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -9
- package/dist/application/contextFactory.d.ts +4 -2
- package/dist/application/contextFactory.js +3 -0
- package/dist/application/flowFailureRecovery.d.ts +17 -0
- package/dist/application/flowFailureRecovery.js +92 -0
- package/dist/application/operations/ack.d.ts +3 -2
- package/dist/application/operations/ack.js +34 -8
- package/dist/application/operations/flowPush.d.ts +9 -0
- package/dist/application/operations/flowPush.js +112 -0
- package/dist/application/operations/flowTopologyValidation.d.ts +3 -0
- package/dist/application/operations/flowTopologyValidation.js +101 -0
- package/dist/application/operations/flowValidation.d.ts +3 -0
- package/dist/application/operations/flowValidation.js +166 -0
- package/dist/application/operations/push.d.ts +3 -0
- package/dist/application/operations/push.js +2 -2
- package/dist/application/operations/pushInsert.d.ts +7 -1
- package/dist/application/operations/pushInsert.js +18 -8
- package/dist/application/operations/pushLocks.d.ts +1 -0
- package/dist/application/operations/pushLocks.js +5 -1
- package/dist/application/operations/queryOperations.js +7 -6
- package/dist/application/queueManager.d.ts +5 -0
- package/dist/application/queueManager.js +195 -60
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/client/flow.d.ts +3 -6
- package/dist/client/flow.js +71 -210
- package/dist/client/flowAtomic.d.ts +9 -0
- package/dist/client/flowAtomic.js +24 -0
- package/dist/client/flowJobCoreMethods.d.ts +28 -0
- package/dist/client/flowJobCoreMethods.js +151 -0
- package/dist/client/flowJobDependencies.d.ts +7 -0
- package/dist/client/flowJobDependencies.js +64 -0
- package/dist/client/flowJobFactory.d.ts +11 -24
- package/dist/client/flowJobFactory.js +91 -360
- package/dist/client/flowJobMoveMethods.d.ts +16 -0
- package/dist/client/flowJobMoveMethods.js +112 -0
- package/dist/client/flowJobTypes.d.ts +29 -0
- package/dist/client/flowJobTypes.js +6 -0
- package/dist/client/flowLegacyPlan.d.ts +16 -0
- package/dist/client/flowLegacyPlan.js +118 -0
- package/dist/client/flowOptions.d.ts +10 -0
- package/dist/client/flowOptions.js +44 -0
- package/dist/client/flowPlan.d.ts +16 -0
- package/dist/client/flowPlan.js +93 -0
- package/dist/client/flowPush.js +4 -1
- package/dist/client/flowReader.d.ts +10 -0
- package/dist/client/flowReader.js +147 -0
- package/dist/client/jobHelpers.js +2 -0
- package/dist/client/workflow/clock.d.ts +3 -1
- package/dist/client/workflow/clock.js +18 -1
- package/dist/client/workflow/compensationChild.d.ts +30 -0
- package/dist/client/workflow/compensationChild.js +65 -0
- package/dist/client/workflow/compensationPass.d.ts +16 -0
- package/dist/client/workflow/compensationPass.js +96 -0
- package/dist/client/workflow/compensationSupport.d.ts +24 -0
- package/dist/client/workflow/compensationSupport.js +57 -0
- package/dist/client/workflow/compensator.d.ts +11 -50
- package/dist/client/workflow/compensator.js +22 -374
- package/dist/client/workflow/definitionGuard.d.ts +11 -0
- package/dist/client/workflow/definitionGuard.js +22 -0
- package/dist/client/workflow/engine.d.ts +2 -2
- package/dist/client/workflow/engine.js +8 -8
- package/dist/client/workflow/eventTypes.d.ts +44 -0
- package/dist/client/workflow/eventTypes.js +2 -0
- package/dist/client/workflow/executionTypes.d.ts +133 -0
- package/dist/client/workflow/executionTypes.js +2 -0
- package/dist/client/workflow/executor.d.ts +4 -17
- package/dist/client/workflow/executor.js +50 -208
- package/dist/client/workflow/executorLifecycle.d.ts +18 -0
- package/dist/client/workflow/executorLifecycle.js +69 -0
- package/dist/client/workflow/executorNodes.d.ts +17 -0
- package/dist/client/workflow/executorNodes.js +138 -0
- package/dist/client/workflow/identity.d.ts +2 -0
- package/dist/client/workflow/identity.js +8 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.d.ts +3 -5
- package/dist/client/workflow/loops.js +25 -23
- package/dist/client/workflow/mapRunner.d.ts +4 -0
- package/dist/client/workflow/mapRunner.js +45 -0
- package/dist/client/workflow/recovery.js +10 -3
- package/dist/client/workflow/runner.d.ts +2 -21
- package/dist/client/workflow/runner.js +39 -104
- package/dist/client/workflow/runnerTiming.d.ts +14 -0
- package/dist/client/workflow/runnerTiming.js +66 -0
- package/dist/client/workflow/stepTypes.d.ts +171 -0
- package/dist/client/workflow/stepTypes.js +3 -0
- package/dist/client/workflow/store.d.ts +17 -17
- package/dist/client/workflow/store.js +65 -100
- package/dist/client/workflow/storeExecutionCodec.d.ts +11 -0
- package/dist/client/workflow/storeExecutionCodec.js +34 -0
- package/dist/client/workflow/storeListing.d.ts +11 -0
- package/dist/client/workflow/storeListing.js +45 -0
- package/dist/client/workflow/storeMaintenance.d.ts +4 -0
- package/dist/client/workflow/storeMaintenance.js +40 -0
- package/dist/client/workflow/storeSignals.d.ts +9 -0
- package/dist/client/workflow/storeSignals.js +38 -2
- package/dist/client/workflow/subWorkflowRunner.d.ts +13 -0
- package/dist/client/workflow/subWorkflowRunner.js +40 -0
- package/dist/client/workflow/types.d.ts +4 -356
- package/dist/client/workflow/types.js +1 -3
- package/dist/client/workflow/waitFor.js +39 -26
- package/dist/client/workflow/workflow.d.ts +16 -59
- package/dist/client/workflow/workflow.js +53 -179
- package/dist/client/workflow/workflowDecisions.d.ts +11 -0
- package/dist/client/workflow/workflowDecisions.js +27 -0
- package/dist/client/workflow/workflowDefinition.d.ts +16 -0
- package/dist/client/workflow/workflowDefinition.js +123 -0
- package/dist/client/workflow/workflowIntrospection.d.ts +5 -0
- package/dist/client/workflow/workflowIntrospection.js +46 -0
- package/dist/client/workflow/workflowValidation.d.ts +44 -0
- package/dist/client/workflow/workflowValidation.js +143 -0
- package/dist/domain/types/command.d.ts +7 -1
- package/dist/domain/types/flow.d.ts +25 -0
- package/dist/domain/types/flow.js +1 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +52 -3
- package/dist/infrastructure/persistence/sqlite.d.ts +16 -0
- package/dist/infrastructure/persistence/sqlite.js +129 -7
- package/dist/infrastructure/persistence/sqliteBatch.js +10 -4
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +2 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +18 -5
- package/dist/infrastructure/persistence/statements.d.ts +4 -0
- package/dist/infrastructure/persistence/statements.js +8 -2
- package/dist/infrastructure/server/handlerRoutes.js +3 -0
- package/dist/infrastructure/server/handlers/advanced.js +2 -2
- package/dist/infrastructure/server/handlers/flow.d.ts +7 -0
- package/dist/infrastructure/server/handlers/flow.js +11 -0
- package/dist/infrastructure/server/handlers/index.d.ts +1 -0
- package/dist/infrastructure/server/handlers/index.js +1 -0
- package/package.json +4 -2
package/README.md
CHANGED
|
@@ -103,7 +103,9 @@ Python, PHP, Go, Rust and Elixir clients speak the same protocol — see
|
|
|
103
103
|
- **BullMQ-compatible API** — same `Queue`, `Worker`, `QueueEvents`; [migrating takes minutes](https://bunqueue.dev/guide/migration/)
|
|
104
104
|
- **MCP server included** — 73 tools; AI agents get full queue control out of the box
|
|
105
105
|
- **Everything server-side** — retries with backoff, priorities, cron, rate limits, dead letter queue
|
|
106
|
-
- **
|
|
106
|
+
- **Measured, operation-specific performance** — 729K jobs/sec internal
|
|
107
|
+
in-memory batch push, 186K jobs/sec public on-disk Embedded `addBulk`, and
|
|
108
|
+
159K jobs/sec TCP `PUSHB`; [methodology and distributions](https://bunqueue.dev/guide/benchmarks/)
|
|
107
109
|
|
|
108
110
|
**Great for:** single-server deployments, AI agents that need a scheduler,
|
|
109
111
|
prototypes and MVPs, embedded use cases (CLI tools, edge, serverless), teams
|
|
@@ -120,7 +122,7 @@ failover today. If you already run Redis and BullMQ works for you, keep it.
|
|
|
120
122
|
| ---------------- | ------------------------------------- | -------------------------------------------- |
|
|
121
123
|
| **How it works** | Queue runs inside your process | Standalone server, clients connect via TCP |
|
|
122
124
|
| **Setup** | `bun add bunqueue` | `docker run` or `bunqueue start` |
|
|
123
|
-
| **Performance** |
|
|
125
|
+
| **Performance** | 186K jobs/sec on-disk `addBulk`; 729K internal in-memory batch | 159K jobs/sec TCP `PUSHB`; 17K jobs/sec worker drain |
|
|
124
126
|
| **Best for** | Single-process apps, CLIs, serverless | Multiple workers, separate producer/consumer |
|
|
125
127
|
| **Scaling** | Same process only | Multiple clients across machines |
|
|
126
128
|
|
|
@@ -343,13 +345,21 @@ https://github.com/user-attachments/assets/e8a8d38e-b4a6-4dc8-8360-876c0f24d116
|
|
|
343
345
|
|
|
344
346
|
## Performance
|
|
345
347
|
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
|
349
|
-
|
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
348
|
+
Native Ryzen 9 9950X3D, Bun 1.3.14; medians from repeated fresh processes:
|
|
349
|
+
|
|
350
|
+
| Workload | Mode | Median | Persistence |
|
|
351
|
+
| --- | --- | ---: | --- |
|
|
352
|
+
| Internal batched push, 1M jobs | Embedded | 729,395 jobs/sec | In-memory, no `dataPath` |
|
|
353
|
+
| Public sustained `addBulk`, 50K cell | Embedded | 186,384 jobs/sec | On-disk buffered SQLite |
|
|
354
|
+
| `PUSHB`, fresh 50K sample | TCP | 158,779 jobs/sec | On-disk buffered SQLite |
|
|
355
|
+
| No-work worker drain, concurrency 50 | TCP | 17,256 jobs/sec | Full pull/process/ACK |
|
|
356
|
+
| Linear Workflow Engine | Embedded / TCP | 2,700 / 3,187 workflows/sec | Workflow SQLite + 3 queue nodes |
|
|
357
|
+
|
|
358
|
+
These operations do different work; the internal in-memory result is not an
|
|
359
|
+
SQLite or public-API claim. Run `bun run bench`, `bun run bench:tcp`, or
|
|
360
|
+
`bun run bench:workflow` on your hardware.
|
|
361
|
+
[Benchmark methodology →](https://bunqueue.dev/guide/benchmarks/) ·
|
|
362
|
+
[full engineering report](docs/benchmarks/native-engineering-2026-07-30.md)
|
|
353
363
|
|
|
354
364
|
## Documentation
|
|
355
365
|
|
|
@@ -29,6 +29,7 @@ export interface ContextDependencies {
|
|
|
29
29
|
storage: SqliteStorage | null;
|
|
30
30
|
shards: Shard[];
|
|
31
31
|
shardLocks: RWLock[];
|
|
32
|
+
customIdLock: RWLock;
|
|
32
33
|
processingShards: Map<JobId, Job>[];
|
|
33
34
|
processingLocks: RWLock[];
|
|
34
35
|
jobIndex: Map<JobId, JobLocation>;
|
|
@@ -77,12 +78,13 @@ export interface ContextCallbacks {
|
|
|
77
78
|
registerQueueName: (queue: string) => void;
|
|
78
79
|
unregisterQueueName: (queue: string) => void;
|
|
79
80
|
onJobCompleted: (completedId: JobId) => void;
|
|
81
|
+
onJobFailed?: (failedId: JobId) => void;
|
|
80
82
|
onJobsCompleted: (completedIds: JobId[]) => void;
|
|
81
83
|
hasPendingDeps: () => boolean;
|
|
82
84
|
onRepeat: (job: Job) => void;
|
|
83
85
|
emitDashboardEvent?: (event: string, data: Record<string, unknown>) => void;
|
|
84
|
-
onChildTerminalFailure?: (childJob: Job, error: string | undefined) => void
|
|
85
|
-
onChildDependencyOption?: (childJob: Job, error: string | undefined) => void
|
|
86
|
+
onChildTerminalFailure?: (childJob: Job, error: string | undefined) => Promise<void>;
|
|
87
|
+
onChildDependencyOption?: (childJob: Job, error: string | undefined) => Promise<void>;
|
|
86
88
|
}
|
|
87
89
|
/**
|
|
88
90
|
* Factory for building context objects
|
|
@@ -82,6 +82,7 @@ export class ContextFactory {
|
|
|
82
82
|
storage: this.deps.storage,
|
|
83
83
|
shards: this.deps.shards,
|
|
84
84
|
shardLocks: this.deps.shardLocks,
|
|
85
|
+
customIdLock: this.deps.customIdLock,
|
|
85
86
|
completedJobs: this.deps.completedJobs,
|
|
86
87
|
completedJobsData: this.deps.completedJobsData,
|
|
87
88
|
depCompletions: this.deps.depCompletions,
|
|
@@ -93,6 +94,7 @@ export class ContextFactory {
|
|
|
93
94
|
totalPushed: this.deps.metrics.totalPushed,
|
|
94
95
|
broadcast: this.deps.eventsManager.broadcast.bind(this.deps.eventsManager),
|
|
95
96
|
dashboardEmit: this.callbacks.emitDashboardEvent,
|
|
97
|
+
registerQueueName: this.callbacks.registerQueueName,
|
|
96
98
|
};
|
|
97
99
|
}
|
|
98
100
|
getPullContext() {
|
|
@@ -127,6 +129,7 @@ export class ContextFactory {
|
|
|
127
129
|
perQueueMetrics: this.deps.perQueueMetrics,
|
|
128
130
|
broadcast: this.deps.eventsManager.broadcast.bind(this.deps.eventsManager),
|
|
129
131
|
onJobCompleted: this.callbacks.onJobCompleted,
|
|
132
|
+
onJobFailed: this.callbacks.onJobFailed,
|
|
130
133
|
onJobsCompleted: this.callbacks.onJobsCompleted,
|
|
131
134
|
needsBroadcast: this.deps.eventsManager.needsBroadcast.bind(this.deps.eventsManager),
|
|
132
135
|
hasPendingDeps: this.callbacks.hasPendingDeps,
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { JobId } from '../domain/types/job';
|
|
2
|
+
import type { JobLocation } from '../domain/types/queue';
|
|
3
|
+
import type { Shard } from '../domain/queue/shard';
|
|
4
|
+
import type { SqliteStorage } from '../infrastructure/persistence/sqlite';
|
|
5
|
+
import type { SetLike } from '../shared/lru';
|
|
6
|
+
import type { DependencyResultTracker } from './dependencyResultTracker';
|
|
7
|
+
export interface FlowFailureRecoveryContext {
|
|
8
|
+
readonly storage: SqliteStorage;
|
|
9
|
+
readonly shards: Shard[];
|
|
10
|
+
readonly jobIndex: Map<JobId, JobLocation>;
|
|
11
|
+
readonly completedJobs: SetLike<JobId>;
|
|
12
|
+
readonly dependencyResults: DependencyResultTracker;
|
|
13
|
+
readonly failedChildrenValues: Map<JobId, Record<string, string>>;
|
|
14
|
+
readonly ignoredChildrenFailures: Map<JobId, Record<string, string>>;
|
|
15
|
+
}
|
|
16
|
+
/** Replay the durable flow-failure outbox before workers can observe recovered jobs. */
|
|
17
|
+
export declare function recoverFlowFailures(ctx: FlowFailureRecoveryContext): void;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { shardIndex } from '../shared/hash';
|
|
2
|
+
function findQueuedJob(ctx, id) {
|
|
3
|
+
const location = ctx.jobIndex.get(id);
|
|
4
|
+
if (location?.type !== 'queue')
|
|
5
|
+
return null;
|
|
6
|
+
const shard = ctx.shards[location.shardIdx];
|
|
7
|
+
return (shard.getQueue(location.queueName).find(id) ??
|
|
8
|
+
shard.waitingDeps.get(id) ??
|
|
9
|
+
shard.waitingChildren.get(id) ??
|
|
10
|
+
null);
|
|
11
|
+
}
|
|
12
|
+
function promote(parent, ctx) {
|
|
13
|
+
const idx = shardIndex(parent.queue);
|
|
14
|
+
const shard = ctx.shards[idx];
|
|
15
|
+
if (shard.waitingDeps.delete(parent.id)) {
|
|
16
|
+
shard.unregisterDependencies(parent.id, parent.dependsOn);
|
|
17
|
+
}
|
|
18
|
+
shard.waitingChildren.delete(parent.id);
|
|
19
|
+
const queue = shard.getQueue(parent.queue);
|
|
20
|
+
if (!queue.find(parent.id)) {
|
|
21
|
+
parent.runAt = Date.now();
|
|
22
|
+
queue.push(parent);
|
|
23
|
+
shard.incrementQueued(parent.id, false, parent.createdAt, parent.queue, parent.runAt);
|
|
24
|
+
shard.notify(parent.queue);
|
|
25
|
+
}
|
|
26
|
+
ctx.storage.updateFlowParentResolution(parent);
|
|
27
|
+
}
|
|
28
|
+
function failParent(parent, childId, error, ctx) {
|
|
29
|
+
const shard = ctx.shards[shardIndex(parent.queue)];
|
|
30
|
+
if (shard.waitingDeps.delete(parent.id)) {
|
|
31
|
+
shard.unregisterDependencies(parent.id, parent.dependsOn);
|
|
32
|
+
}
|
|
33
|
+
shard.waitingChildren.delete(parent.id);
|
|
34
|
+
const queued = shard.getQueue(parent.queue).remove(parent.id);
|
|
35
|
+
if (queued)
|
|
36
|
+
shard.decrementQueued(parent.id);
|
|
37
|
+
const message = `Child job ${String(childId)} failed: ${error}`;
|
|
38
|
+
const entry = shard.addToDlq(parent, "unknown" /* FailureReason.Unknown */, message);
|
|
39
|
+
ctx.storage.commitFailedJob(parent.id, entry, null);
|
|
40
|
+
ctx.jobIndex.set(parent.id, { type: 'dlq', queueName: parent.queue });
|
|
41
|
+
ctx.dependencyResults.releaseConsumer(parent.id);
|
|
42
|
+
}
|
|
43
|
+
/** Replay the durable flow-failure outbox before workers can observe recovered jobs. */
|
|
44
|
+
export function recoverFlowFailures(ctx) {
|
|
45
|
+
for (const record of ctx.storage.loadFlowFailures()) {
|
|
46
|
+
const parent = findQueuedJob(ctx, record.parentId);
|
|
47
|
+
if (!parent) {
|
|
48
|
+
if (ctx.jobIndex.get(record.parentId)?.type !== 'queue') {
|
|
49
|
+
ctx.storage.deleteFlowFailure(record.parentId, record.childId);
|
|
50
|
+
}
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const childKey = `${record.childQueue}:${String(record.childId)}`;
|
|
54
|
+
if (record.mode === 'fail') {
|
|
55
|
+
failParent(parent, record.childId, record.error, ctx);
|
|
56
|
+
ctx.storage.deleteFlowFailure(record.parentId, record.childId);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (record.mode === 'continue') {
|
|
60
|
+
const values = ctx.failedChildrenValues.get(parent.id) ?? {};
|
|
61
|
+
values[childKey] = record.error;
|
|
62
|
+
ctx.failedChildrenValues.set(parent.id, values);
|
|
63
|
+
const dependencies = [...parent.dependsOn];
|
|
64
|
+
ctx.shards[shardIndex(parent.queue)].unregisterDependencies(parent.id, dependencies);
|
|
65
|
+
for (const dependency of dependencies) {
|
|
66
|
+
ctx.dependencyResults.releaseDependency(parent.id, dependency);
|
|
67
|
+
}
|
|
68
|
+
parent.dependsOn = [];
|
|
69
|
+
promote(parent, ctx);
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (record.mode === 'ignore') {
|
|
73
|
+
const values = ctx.ignoredChildrenFailures.get(parent.id) ?? {};
|
|
74
|
+
values[childKey] = record.error;
|
|
75
|
+
ctx.ignoredChildrenFailures.set(parent.id, values);
|
|
76
|
+
}
|
|
77
|
+
const dependencyIndex = parent.dependsOn.indexOf(record.childId);
|
|
78
|
+
if (dependencyIndex !== -1) {
|
|
79
|
+
parent.dependsOn.splice(dependencyIndex, 1);
|
|
80
|
+
ctx.shards[shardIndex(parent.queue)].unregisterDependencies(parent.id, [record.childId]);
|
|
81
|
+
ctx.dependencyResults.releaseDependency(parent.id, record.childId);
|
|
82
|
+
ctx.storage.updateFlowParentResolution(parent);
|
|
83
|
+
}
|
|
84
|
+
const ready = parent.dependsOn.length === 0 ||
|
|
85
|
+
parent.dependsOn.every((dependency) => ctx.completedJobs.has(dependency));
|
|
86
|
+
if (ready)
|
|
87
|
+
promote(parent, ctx);
|
|
88
|
+
if (record.mode === 'remove') {
|
|
89
|
+
ctx.storage.deleteFlowFailure(record.parentId, record.childId);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -44,15 +44,16 @@ export interface AckContext {
|
|
|
44
44
|
prev?: string;
|
|
45
45
|
}) => void;
|
|
46
46
|
onJobCompleted: (jobId: JobId) => void;
|
|
47
|
+
onJobFailed?: (jobId: JobId) => void;
|
|
47
48
|
onJobsCompleted?: (jobIds: JobId[]) => void;
|
|
48
49
|
needsBroadcast?: () => boolean;
|
|
49
50
|
emitDashboardEvent?: (event: string, data: Record<string, unknown>) => void;
|
|
50
51
|
hasPendingDeps?: () => boolean;
|
|
51
52
|
onRepeat?: (job: Job) => void;
|
|
52
53
|
/** Called when a child job with failParentOnFailure terminally fails */
|
|
53
|
-
onChildTerminalFailure?: (childJob: Job, error: string | undefined) => void
|
|
54
|
+
onChildTerminalFailure?: (childJob: Job, error: string | undefined) => Promise<void>;
|
|
54
55
|
/** Called when a child job with removeDependencyOnFailure/ignoreDependencyOnFailure/continueParentOnFailure terminally fails */
|
|
55
|
-
onChildDependencyOption?: (childJob: Job, error: string | undefined) => void
|
|
56
|
+
onChildDependencyOption?: (childJob: Job, error: string | undefined) => Promise<void>;
|
|
56
57
|
}
|
|
57
58
|
/**
|
|
58
59
|
* Acknowledge job completion
|
|
@@ -91,11 +91,11 @@ export async function ackJob(jobId, result, ctx) {
|
|
|
91
91
|
latencyTracker.ack.observe((Bun.nanoseconds() - startNs) / 1e6);
|
|
92
92
|
}
|
|
93
93
|
/** Move a permanently-failed job to DLQ (terminal path in failJob). */
|
|
94
|
-
function moveFailedJobToDlq(
|
|
94
|
+
function moveFailedJobToDlq(input) {
|
|
95
|
+
const { job, jobId, error, shard, ctx, flowFailure } = input;
|
|
95
96
|
const entry = shard.addToDlq(job, "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */, error ?? null);
|
|
96
97
|
ctx.jobIndex.set(jobId, { type: 'dlq', queueName: job.queue });
|
|
97
|
-
ctx.storage?.
|
|
98
|
-
ctx.storage?.deleteJob(jobId);
|
|
98
|
+
ctx.storage?.commitFailedJob(jobId, entry, flowFailure);
|
|
99
99
|
ctx.totalFailed.value++;
|
|
100
100
|
if (ctx.perQueueMetrics) {
|
|
101
101
|
const pq = ctx.perQueueMetrics.get(job.queue);
|
|
@@ -121,6 +121,29 @@ function moveFailedJobToDlq(job, jobId, error, shard, ctx) {
|
|
|
121
121
|
});
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
|
+
function flowFailureRecord(job, error) {
|
|
125
|
+
if (!job.parentId)
|
|
126
|
+
return null;
|
|
127
|
+
let mode = null;
|
|
128
|
+
if (job.failParentOnFailure)
|
|
129
|
+
mode = 'fail';
|
|
130
|
+
else if (job.continueParentOnFailure)
|
|
131
|
+
mode = 'continue';
|
|
132
|
+
else if (job.ignoreDependencyOnFailure)
|
|
133
|
+
mode = 'ignore';
|
|
134
|
+
else if (job.removeDependencyOnFailure)
|
|
135
|
+
mode = 'remove';
|
|
136
|
+
if (!mode)
|
|
137
|
+
return null;
|
|
138
|
+
return {
|
|
139
|
+
parentId: job.parentId,
|
|
140
|
+
childId: job.id,
|
|
141
|
+
childQueue: job.queue,
|
|
142
|
+
mode,
|
|
143
|
+
error: error ?? 'unknown error',
|
|
144
|
+
createdAt: Date.now(),
|
|
145
|
+
};
|
|
146
|
+
}
|
|
124
147
|
/**
|
|
125
148
|
* Mark job as failed
|
|
126
149
|
*/
|
|
@@ -149,10 +172,12 @@ export async function failJob(jobId, error, ctx, unrecoverable = false, stack) {
|
|
|
149
172
|
}
|
|
150
173
|
const idx = shardIndex(job.queue);
|
|
151
174
|
let wasRetried = false;
|
|
175
|
+
const willRetry = !unrecoverable && canRetry(job);
|
|
176
|
+
const flowFailure = willRetry ? null : flowFailureRecord(job, error);
|
|
152
177
|
await withWriteLock(ctx.shardLocks[idx], () => {
|
|
153
178
|
const shard = ctx.shards[idx];
|
|
154
179
|
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
|
|
155
|
-
if (
|
|
180
|
+
if (willRetry) {
|
|
156
181
|
const now = Date.now();
|
|
157
182
|
job.runAt = now + calculateBackoff(job);
|
|
158
183
|
shard.getQueue(job.queue).push(job);
|
|
@@ -166,7 +191,7 @@ export async function failJob(jobId, error, ctx, unrecoverable = false, stack) {
|
|
|
166
191
|
}
|
|
167
192
|
else if (job.removeOnFail) {
|
|
168
193
|
ctx.jobIndex.delete(jobId);
|
|
169
|
-
ctx.storage?.
|
|
194
|
+
ctx.storage?.commitFailedJob(jobId, null, flowFailure);
|
|
170
195
|
ctx.totalFailed.value++;
|
|
171
196
|
if (ctx.perQueueMetrics) {
|
|
172
197
|
const pq = ctx.perQueueMetrics.get(job.queue);
|
|
@@ -184,7 +209,7 @@ export async function failJob(jobId, error, ctx, unrecoverable = false, stack) {
|
|
|
184
209
|
}
|
|
185
210
|
}
|
|
186
211
|
else {
|
|
187
|
-
moveFailedJobToDlq(job, jobId, error, shard, ctx);
|
|
212
|
+
moveFailedJobToDlq({ job, jobId, error, shard, ctx, flowFailure });
|
|
188
213
|
}
|
|
189
214
|
// Terminal failure and retry both release queue concurrency (and possibly
|
|
190
215
|
// an active FIFO group), so another job may now be eligible.
|
|
@@ -210,17 +235,18 @@ export async function failJob(jobId, error, ctx, unrecoverable = false, stack) {
|
|
|
210
235
|
}
|
|
211
236
|
else {
|
|
212
237
|
ctx.dependencyResults.releaseConsumer(jobId);
|
|
238
|
+
ctx.onJobFailed?.(jobId);
|
|
213
239
|
}
|
|
214
240
|
// BullMQ v5: failParentOnFailure — propagate terminal failure to parent
|
|
215
241
|
if (!wasRetried && job.failParentOnFailure && job.parentId && ctx.onChildTerminalFailure) {
|
|
216
|
-
ctx.onChildTerminalFailure(job, error);
|
|
242
|
+
await ctx.onChildTerminalFailure(job, error);
|
|
217
243
|
}
|
|
218
244
|
// removeDependencyOnFailure / ignoreDependencyOnFailure / continueParentOnFailure
|
|
219
245
|
if (!wasRetried &&
|
|
220
246
|
job.parentId &&
|
|
221
247
|
ctx.onChildDependencyOption &&
|
|
222
248
|
(job.removeDependencyOnFailure || job.ignoreDependencyOnFailure || job.continueParentOnFailure)) {
|
|
223
|
-
ctx.onChildDependencyOption(job, error);
|
|
249
|
+
await ctx.onChildDependencyOption(job, error);
|
|
224
250
|
}
|
|
225
251
|
}
|
|
226
252
|
/**
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { AtomicFlowBatchInput, AtomicFlowBatchResult } from '../../domain/types/flow';
|
|
2
|
+
import type { PushContext } from './push';
|
|
3
|
+
/**
|
|
4
|
+
* Commit a complete flow graph as one broker-side operation.
|
|
5
|
+
*
|
|
6
|
+
* SQLite is committed before the graph is published in memory. Workers acquire
|
|
7
|
+
* the same shard locks, so the first visible leaf always observes every edge.
|
|
8
|
+
*/
|
|
9
|
+
export declare function pushFlowBatch(batch: AtomicFlowBatchInput, ctx: PushContext): Promise<AtomicFlowBatchResult>;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { createJob } from '../../domain/types/job';
|
|
2
|
+
import { shardIndex } from '../../shared/hash';
|
|
3
|
+
import { latencyTracker } from '../latencyTracker';
|
|
4
|
+
import { throughputTracker } from '../throughputTracker';
|
|
5
|
+
import { initialJobState, insertJobToShard } from './pushInsert';
|
|
6
|
+
import { validateAtomicFlowBatch } from './flowValidation';
|
|
7
|
+
function existingJobError(id) {
|
|
8
|
+
return new Error(`Flow job ${String(id)} already exists`);
|
|
9
|
+
}
|
|
10
|
+
function assertIdsAvailable(batch, ctx) {
|
|
11
|
+
for (const planned of batch.jobs) {
|
|
12
|
+
if (ctx.jobIndex.has(planned.id) ||
|
|
13
|
+
ctx.completedJobs.has(planned.id) ||
|
|
14
|
+
(ctx.depCompletions?.has(planned.id) ?? false) ||
|
|
15
|
+
ctx.jobResults.has(planned.id) ||
|
|
16
|
+
(ctx.timedOutJobs?.has(planned.id) ?? false) ||
|
|
17
|
+
ctx.shards.some((shard) => (shard.getJobsWaitingFor(planned.id)?.size ?? 0) > 0) ||
|
|
18
|
+
ctx.storage?.getJob(planned.id) ||
|
|
19
|
+
ctx.storage?.hasDlqEntry(planned.id)) {
|
|
20
|
+
throw existingJobError(planned.id);
|
|
21
|
+
}
|
|
22
|
+
if (planned.input.customId) {
|
|
23
|
+
const owner = ctx.customIdMap.get(planned.input.customId);
|
|
24
|
+
if (owner !== undefined)
|
|
25
|
+
throw existingJobError(owner);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function prepareJobs(batch, ctx, now) {
|
|
30
|
+
return batch.jobs.map((planned) => {
|
|
31
|
+
const job = createJob(planned.id, planned.queue, planned.input, now);
|
|
32
|
+
job.timeline.push({ state: initialJobState(job, ctx, now), timestamp: now });
|
|
33
|
+
return job;
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
function publishJobs(jobs, ctx) {
|
|
37
|
+
const notifications = new Map();
|
|
38
|
+
for (const job of jobs) {
|
|
39
|
+
const idx = shardIndex(job.queue);
|
|
40
|
+
insertJobToShard(job, { queue: job.queue, shard: ctx.shards[idx], shardIdx: idx }, ctx, false);
|
|
41
|
+
if (job.customId)
|
|
42
|
+
ctx.customIdMap.set(job.customId, job.id);
|
|
43
|
+
notifications.set(job.queue, (notifications.get(job.queue) ?? 0) + 1);
|
|
44
|
+
}
|
|
45
|
+
return notifications;
|
|
46
|
+
}
|
|
47
|
+
async function acquireFlowLocks(batch, ctx) {
|
|
48
|
+
const guards = [];
|
|
49
|
+
try {
|
|
50
|
+
if (batch.jobs.some((job) => job.input.customId)) {
|
|
51
|
+
guards.push(await ctx.customIdLock.acquireWrite());
|
|
52
|
+
}
|
|
53
|
+
const indexes = [...new Set(batch.jobs.map((job) => shardIndex(job.queue)))].sort((a, b) => a - b);
|
|
54
|
+
for (const index of indexes)
|
|
55
|
+
guards.push(await ctx.shardLocks[index].acquireWrite());
|
|
56
|
+
return guards;
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
for (let index = guards.length - 1; index >= 0; index--)
|
|
60
|
+
guards[index].release();
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Commit a complete flow graph as one broker-side operation.
|
|
66
|
+
*
|
|
67
|
+
* SQLite is committed before the graph is published in memory. Workers acquire
|
|
68
|
+
* the same shard locks, so the first visible leaf always observes every edge.
|
|
69
|
+
*/
|
|
70
|
+
export async function pushFlowBatch(batch, ctx) {
|
|
71
|
+
const startNs = Bun.nanoseconds();
|
|
72
|
+
validateAtomicFlowBatch(batch);
|
|
73
|
+
if (batch.jobs.length === 0)
|
|
74
|
+
return { jobs: [] };
|
|
75
|
+
for (const queue of new Set(batch.jobs.map((job) => job.queue))) {
|
|
76
|
+
ctx.registerQueueName?.(queue);
|
|
77
|
+
}
|
|
78
|
+
const guards = await acquireFlowLocks(batch, ctx);
|
|
79
|
+
let jobs;
|
|
80
|
+
let notifications;
|
|
81
|
+
try {
|
|
82
|
+
assertIdsAvailable(batch, ctx);
|
|
83
|
+
const now = Date.now();
|
|
84
|
+
jobs = prepareJobs(batch, ctx, now);
|
|
85
|
+
ctx.storage?.insertJobsBatch(jobs, true);
|
|
86
|
+
notifications = publishJobs(jobs, ctx);
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
for (let index = guards.length - 1; index >= 0; index--)
|
|
90
|
+
guards[index].release();
|
|
91
|
+
}
|
|
92
|
+
for (const [queue, count] of notifications) {
|
|
93
|
+
ctx.shards[shardIndex(queue)].notifyBatch(queue, count);
|
|
94
|
+
}
|
|
95
|
+
ctx.totalPushed.value += BigInt(jobs.length);
|
|
96
|
+
throughputTracker.pushRate.increment(jobs.length);
|
|
97
|
+
const timestamp = Date.now();
|
|
98
|
+
for (const job of jobs) {
|
|
99
|
+
ctx.broadcast({
|
|
100
|
+
eventType: "pushed" /* EventType.Pushed */,
|
|
101
|
+
queue: job.queue,
|
|
102
|
+
jobId: job.id,
|
|
103
|
+
timestamp,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
ctx.dashboardEmit?.('flow:pushed', {
|
|
107
|
+
jobs: jobs.length,
|
|
108
|
+
queues: notifications.size,
|
|
109
|
+
});
|
|
110
|
+
latencyTracker.push.observe((Bun.nanoseconds() - startNs) / 1e6);
|
|
111
|
+
return { jobs };
|
|
112
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { AtomicFlowBatchInput } from '../../domain/types/flow';
|
|
2
|
+
/** Validate references, symmetric parent edges, metadata, and acyclicity in O(V+E). */
|
|
3
|
+
export declare function validateFlowTopology(batch: AtomicFlowBatchInput, dataById: ReadonlyMap<string, Record<string, unknown>>): void;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
function assertMatchingIds(actual, expected, name) {
|
|
2
|
+
if (!Array.isArray(actual) ||
|
|
3
|
+
actual.length !== expected.length ||
|
|
4
|
+
actual.some((id, index) => id !== expected[index])) {
|
|
5
|
+
throw new Error(`${name} does not match the canonical flow topology`);
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
function assertAcyclic(topology) {
|
|
9
|
+
const remaining = new Map([...topology].map(([id, links]) => [id, links.dependencies.length]));
|
|
10
|
+
const dependents = new Map();
|
|
11
|
+
for (const [id, links] of topology) {
|
|
12
|
+
for (const dependency of links.dependencies) {
|
|
13
|
+
const list = dependents.get(dependency) ?? [];
|
|
14
|
+
list.push(id);
|
|
15
|
+
dependents.set(dependency, list);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
const ready = [...remaining].filter(([, count]) => count === 0).map(([id]) => id);
|
|
19
|
+
let visited = 0;
|
|
20
|
+
while (ready.length > 0) {
|
|
21
|
+
const id = ready.pop();
|
|
22
|
+
if (id === undefined)
|
|
23
|
+
throw new Error('flow topology traversal failed');
|
|
24
|
+
visited++;
|
|
25
|
+
for (const dependent of dependents.get(id) ?? []) {
|
|
26
|
+
const count = remaining.get(dependent);
|
|
27
|
+
if (count === undefined)
|
|
28
|
+
throw new Error(`flow dependency not found: ${dependent}`);
|
|
29
|
+
const next = count - 1;
|
|
30
|
+
remaining.set(dependent, next);
|
|
31
|
+
if (next === 0)
|
|
32
|
+
ready.push(dependent);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (visited !== topology.size)
|
|
36
|
+
throw new Error('flow contains a dependency cycle');
|
|
37
|
+
}
|
|
38
|
+
function buildTopology(job) {
|
|
39
|
+
const dependencies = (job.input.dependsOn ?? []).map(String);
|
|
40
|
+
const children = (job.input.childrenIds ?? []).map(String);
|
|
41
|
+
const dependencySet = new Set(dependencies);
|
|
42
|
+
const childSet = new Set(children);
|
|
43
|
+
if (dependencySet.size !== dependencies.length) {
|
|
44
|
+
throw new Error(`duplicate dependency in flow job: ${String(job.id)}`);
|
|
45
|
+
}
|
|
46
|
+
if (childSet.size !== children.length) {
|
|
47
|
+
throw new Error(`duplicate child in flow job: ${String(job.id)}`);
|
|
48
|
+
}
|
|
49
|
+
return { dependencies, dependencySet, children, childSet };
|
|
50
|
+
}
|
|
51
|
+
/** Validate references, symmetric parent edges, metadata, and acyclicity in O(V+E). */
|
|
52
|
+
export function validateFlowTopology(batch, dataById) {
|
|
53
|
+
const jobsById = new Map(batch.jobs.map((job) => [String(job.id), job]));
|
|
54
|
+
const topology = new Map(batch.jobs.map((job) => [String(job.id), buildTopology(job)]));
|
|
55
|
+
for (const job of batch.jobs) {
|
|
56
|
+
const id = String(job.id);
|
|
57
|
+
const links = topology.get(id);
|
|
58
|
+
if (!links)
|
|
59
|
+
throw new Error(`flow topology missing job: ${id}`);
|
|
60
|
+
for (const dependency of links.dependencies) {
|
|
61
|
+
if (!jobsById.has(dependency))
|
|
62
|
+
throw new Error(`flow dependency not found: ${dependency}`);
|
|
63
|
+
if (dependency === id)
|
|
64
|
+
throw new Error(`flow job cannot depend on itself: ${id}`);
|
|
65
|
+
}
|
|
66
|
+
for (const child of links.children) {
|
|
67
|
+
if (!links.dependencySet.has(child)) {
|
|
68
|
+
throw new Error(`flow child is missing from dependencies: ${child}`);
|
|
69
|
+
}
|
|
70
|
+
const childJob = jobsById.get(child);
|
|
71
|
+
if (!childJob)
|
|
72
|
+
throw new Error(`flow child not found: ${child}`);
|
|
73
|
+
if (String(childJob.input.parentId) !== id) {
|
|
74
|
+
throw new Error(`flow child ${child} does not point back to parent ${id}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const parentId = job.input.parentId ? String(job.input.parentId) : null;
|
|
78
|
+
if (parentId) {
|
|
79
|
+
const parent = jobsById.get(parentId);
|
|
80
|
+
if (!parent)
|
|
81
|
+
throw new Error(`flow parent not found: ${parentId}`);
|
|
82
|
+
const parentLinks = topology.get(parentId);
|
|
83
|
+
if (!parentLinks?.childSet.has(id)) {
|
|
84
|
+
throw new Error(`flow parent ${parentId} does not own child ${id}`);
|
|
85
|
+
}
|
|
86
|
+
const data = dataById.get(id);
|
|
87
|
+
if (!data)
|
|
88
|
+
throw new Error(`flow metadata missing for child ${id}`);
|
|
89
|
+
if (data.__parentId !== parentId || data.__parentQueue !== parent.queue) {
|
|
90
|
+
throw new Error(`flow child ${id} metadata does not match parent ${parentId}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (links.children.length > 0) {
|
|
94
|
+
const data = dataById.get(id);
|
|
95
|
+
if (!data)
|
|
96
|
+
throw new Error(`flow metadata missing for parent ${id}`);
|
|
97
|
+
assertMatchingIds(data.__childrenIds, links.children, `flow parent ${id} children`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
assertAcyclic(topology);
|
|
101
|
+
}
|