bunqueue 2.8.38 → 2.8.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/backgroundTasks.js +6 -0
- package/dist/application/cleanupTasks.js +31 -11
- package/dist/application/contextFactory.d.ts +3 -0
- package/dist/application/contextFactory.js +6 -0
- package/dist/application/dependencyResultTracker.d.ts +17 -0
- package/dist/application/dependencyResultTracker.js +71 -0
- package/dist/application/operations/ack.d.ts +2 -0
- package/dist/application/operations/ack.js +6 -0
- package/dist/application/operations/ackHelpers.d.ts +2 -1
- package/dist/application/operations/ackHelpers.js +6 -0
- package/dist/application/operations/jobManagement.d.ts +2 -0
- package/dist/application/operations/jobManagement.js +3 -0
- package/dist/application/operations/push.d.ts +2 -0
- package/dist/application/operations/pushInsert.d.ts +4 -0
- package/dist/application/operations/pushInsert.js +6 -0
- package/dist/application/operations/queryOperations.d.ts +2 -0
- package/dist/application/operations/queryOperations.js +5 -1
- package/dist/application/operations/queueControl.d.ts +2 -0
- package/dist/application/operations/queueControl.js +4 -0
- package/dist/application/queueManager.d.ts +1 -0
- package/dist/application/queueManager.js +11 -0
- package/dist/application/types.d.ts +2 -0
- package/dist/infrastructure/server/tcp.js +14 -7
- package/package.json +1 -1
|
@@ -324,6 +324,12 @@ export function recover(ctx) {
|
|
|
324
324
|
shard.incrementQueued(job.id, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
325
325
|
}
|
|
326
326
|
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: job.queue });
|
|
327
|
+
ctx.dependencyResults.registerConsumer(job.id, job.dependsOn);
|
|
328
|
+
for (const dependencyId of job.dependsOn) {
|
|
329
|
+
if (ctx.jobResults.has(dependencyId)) {
|
|
330
|
+
ctx.dependencyResults.retain(dependencyId, ctx.jobResults.get(dependencyId));
|
|
331
|
+
}
|
|
332
|
+
}
|
|
327
333
|
// Restore customId mapping for deduplication (fixes idempotency on restart)
|
|
328
334
|
if (job.customId) {
|
|
329
335
|
ctx.customIdMap.set(job.customId, job.id);
|
|
@@ -24,7 +24,7 @@ export async function cleanup(ctx) {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
await cleanOrphanedProcessingEntries(ctx, now, stallTimeout);
|
|
27
|
-
cleanStaleWaitingDependencies(ctx, now);
|
|
27
|
+
await cleanStaleWaitingDependencies(ctx, now);
|
|
28
28
|
cleanUniqueKeysAndGroups(ctx);
|
|
29
29
|
cleanStalledCandidates(ctx);
|
|
30
30
|
await cleanOrphanedJobIndex(ctx);
|
|
@@ -58,23 +58,43 @@ async function cleanOrphanedProcessingEntries(ctx, now, stallTimeout) {
|
|
|
58
58
|
});
|
|
59
59
|
}
|
|
60
60
|
}
|
|
61
|
-
function cleanStaleWaitingDependencies(ctx, now) {
|
|
61
|
+
async function cleanStaleWaitingDependencies(ctx, now) {
|
|
62
62
|
const depTimeout = 60 * 60 * 1000; // 1 hour
|
|
63
63
|
for (let i = 0; i < SHARD_COUNT; i++) {
|
|
64
64
|
const shard = ctx.shards[i];
|
|
65
|
-
const
|
|
65
|
+
const staleIds = [];
|
|
66
66
|
for (const [_id, job] of shard.waitingDeps) {
|
|
67
67
|
if (now - job.createdAt > depTimeout) {
|
|
68
|
-
|
|
68
|
+
staleIds.push(job.id);
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
71
|
+
if (staleIds.length === 0)
|
|
72
|
+
continue;
|
|
73
|
+
const removed = await withWriteLock(ctx.shardLocks[i], () => {
|
|
74
|
+
let count = 0;
|
|
75
|
+
for (const id of staleIds) {
|
|
76
|
+
const job = shard.waitingDeps.get(id);
|
|
77
|
+
if (!job || now - job.createdAt <= depTimeout)
|
|
78
|
+
continue;
|
|
79
|
+
// Persistence first: deleteJob also removes a pending buffered insert.
|
|
80
|
+
// If SQLite throws, the in-memory lifecycle remains live and coherent.
|
|
81
|
+
ctx.storage?.deleteJob(job.id);
|
|
82
|
+
shard.waitingDeps.delete(job.id);
|
|
83
|
+
shard.unregisterDependencies(job.id, job.dependsOn);
|
|
84
|
+
if (job.uniqueKey && shard.getUniqueKeyEntry(job.queue, job.uniqueKey)?.jobId === job.id) {
|
|
85
|
+
shard.releaseUniqueKey(job.queue, job.uniqueKey);
|
|
86
|
+
}
|
|
87
|
+
if (job.customId && ctx.customIdMap.get(job.customId) === job.id) {
|
|
88
|
+
ctx.customIdMap.delete(job.customId);
|
|
89
|
+
}
|
|
90
|
+
ctx.dependencyResults?.releaseConsumer(job.id);
|
|
91
|
+
ctx.jobIndex.delete(job.id);
|
|
92
|
+
count++;
|
|
93
|
+
}
|
|
94
|
+
return count;
|
|
95
|
+
});
|
|
96
|
+
if (removed > 0) {
|
|
97
|
+
ctx.dashboardEmit?.('cleanup:stale-deps-removed', { count: removed });
|
|
78
98
|
}
|
|
79
99
|
}
|
|
80
100
|
}
|
|
@@ -12,6 +12,7 @@ import type { WebhookManager } from './webhookManager';
|
|
|
12
12
|
import type { WorkerManager } from './workerManager';
|
|
13
13
|
import type { EventsManager } from './eventsManager';
|
|
14
14
|
import type { MonitoringState } from './monitoringChecks';
|
|
15
|
+
import type { DependencyResultTracker } from './dependencyResultTracker';
|
|
15
16
|
import type { LockContext, BackgroundContext, StatsContext } from './types';
|
|
16
17
|
import type { PushContext } from './operations/push';
|
|
17
18
|
import type { PullContext } from './operations/pull';
|
|
@@ -36,6 +37,7 @@ export interface ContextDependencies {
|
|
|
36
37
|
depCompletions?: BoundedSet<JobId>;
|
|
37
38
|
timedOutJobs?: BoundedSet<JobId>;
|
|
38
39
|
jobResults: LRUMap<JobId, unknown>;
|
|
40
|
+
dependencyResults: DependencyResultTracker;
|
|
39
41
|
customIdMap: LRUMap<string, JobId>;
|
|
40
42
|
jobLogs: LRUMap<JobId, JobLogEntry[]>;
|
|
41
43
|
jobLocks: Map<JobId, JobLock>;
|
|
@@ -111,6 +113,7 @@ export declare class ContextFactory {
|
|
|
111
113
|
completedJobs: BoundedSet<JobId>;
|
|
112
114
|
completedJobsData: BoundedMap<JobId, Job>;
|
|
113
115
|
jobResults: LRUMap<JobId, unknown>;
|
|
116
|
+
dependencyResults: DependencyResultTracker;
|
|
114
117
|
jobLogs: LRUMap<JobId, JobLogEntry[]>;
|
|
115
118
|
storage: SqliteStorage | null;
|
|
116
119
|
};
|
|
@@ -37,6 +37,7 @@ export class ContextFactory {
|
|
|
37
37
|
depCompletions: this.deps.depCompletions,
|
|
38
38
|
timedOutJobs: this.deps.timedOutJobs,
|
|
39
39
|
jobResults: this.deps.jobResults,
|
|
40
|
+
dependencyResults: this.deps.dependencyResults,
|
|
40
41
|
customIdMap: this.deps.customIdMap,
|
|
41
42
|
jobLogs: this.deps.jobLogs,
|
|
42
43
|
jobLocks: this.deps.jobLocks,
|
|
@@ -86,6 +87,7 @@ export class ContextFactory {
|
|
|
86
87
|
depCompletions: this.deps.depCompletions,
|
|
87
88
|
timedOutJobs: this.deps.timedOutJobs,
|
|
88
89
|
jobResults: this.deps.jobResults,
|
|
90
|
+
dependencyResults: this.deps.dependencyResults,
|
|
89
91
|
customIdMap: this.deps.customIdMap,
|
|
90
92
|
jobIndex: this.deps.jobIndex,
|
|
91
93
|
totalPushed: this.deps.metrics.totalPushed,
|
|
@@ -117,6 +119,7 @@ export class ContextFactory {
|
|
|
117
119
|
completedJobsData: this.deps.completedJobsData,
|
|
118
120
|
depCompletions: this.deps.depCompletions,
|
|
119
121
|
jobResults: this.deps.jobResults,
|
|
122
|
+
dependencyResults: this.deps.dependencyResults,
|
|
120
123
|
jobIndex: this.deps.jobIndex,
|
|
121
124
|
customIdMap: this.deps.customIdMap,
|
|
122
125
|
totalCompleted: this.deps.metrics.totalCompleted,
|
|
@@ -143,6 +146,7 @@ export class ContextFactory {
|
|
|
143
146
|
jobIndex: this.deps.jobIndex,
|
|
144
147
|
jobLocks: this.deps.jobLocks,
|
|
145
148
|
clientJobs: this.deps.clientJobs,
|
|
149
|
+
dependencyResults: this.deps.dependencyResults,
|
|
146
150
|
webhookManager: this.deps.webhookManager,
|
|
147
151
|
eventsManager: this.deps.eventsManager,
|
|
148
152
|
repeatChain: this.deps.repeatChain,
|
|
@@ -159,6 +163,7 @@ export class ContextFactory {
|
|
|
159
163
|
completedJobs: this.deps.completedJobs,
|
|
160
164
|
completedJobsData: this.deps.completedJobsData,
|
|
161
165
|
jobResults: this.deps.jobResults,
|
|
166
|
+
dependencyResults: this.deps.dependencyResults,
|
|
162
167
|
customIdMap: this.deps.customIdMap,
|
|
163
168
|
};
|
|
164
169
|
}
|
|
@@ -196,6 +201,7 @@ export class ContextFactory {
|
|
|
196
201
|
completedJobs: this.deps.completedJobs,
|
|
197
202
|
completedJobsData: this.deps.completedJobsData,
|
|
198
203
|
jobResults: this.deps.jobResults,
|
|
204
|
+
dependencyResults: this.deps.dependencyResults,
|
|
199
205
|
jobLogs: this.deps.jobLogs,
|
|
200
206
|
storage: this.deps.storage,
|
|
201
207
|
};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { JobId } from '../domain/types/job';
|
|
2
|
+
/**
|
|
3
|
+
* Retains results while at least one live dependency consumer may still read
|
|
4
|
+
* them. Ordinary completed-job results remain governed by the configured LRU.
|
|
5
|
+
*/
|
|
6
|
+
export declare class DependencyResultTracker {
|
|
7
|
+
private readonly consumersByDependency;
|
|
8
|
+
private readonly dependenciesByConsumer;
|
|
9
|
+
private readonly protectedResults;
|
|
10
|
+
registerConsumer(consumerId: JobId, dependencyIds: readonly JobId[]): void;
|
|
11
|
+
retain(dependencyId: JobId, result: unknown): void;
|
|
12
|
+
has(dependencyId: JobId): boolean;
|
|
13
|
+
get(dependencyId: JobId): unknown;
|
|
14
|
+
releaseConsumer(consumerId: JobId): void;
|
|
15
|
+
releaseDependency(consumerId: JobId, dependencyId: JobId): void;
|
|
16
|
+
clear(): void;
|
|
17
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retains results while at least one live dependency consumer may still read
|
|
3
|
+
* them. Ordinary completed-job results remain governed by the configured LRU.
|
|
4
|
+
*/
|
|
5
|
+
export class DependencyResultTracker {
|
|
6
|
+
consumersByDependency = new Map();
|
|
7
|
+
dependenciesByConsumer = new Map();
|
|
8
|
+
protectedResults = new Map();
|
|
9
|
+
registerConsumer(consumerId, dependencyIds) {
|
|
10
|
+
if (dependencyIds.length === 0)
|
|
11
|
+
return;
|
|
12
|
+
let dependencies = this.dependenciesByConsumer.get(consumerId);
|
|
13
|
+
if (!dependencies) {
|
|
14
|
+
dependencies = new Set();
|
|
15
|
+
this.dependenciesByConsumer.set(consumerId, dependencies);
|
|
16
|
+
}
|
|
17
|
+
for (const dependencyId of dependencyIds) {
|
|
18
|
+
dependencies.add(dependencyId);
|
|
19
|
+
let consumers = this.consumersByDependency.get(dependencyId);
|
|
20
|
+
if (!consumers) {
|
|
21
|
+
consumers = new Set();
|
|
22
|
+
this.consumersByDependency.set(dependencyId, consumers);
|
|
23
|
+
}
|
|
24
|
+
consumers.add(consumerId);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
retain(dependencyId, result) {
|
|
28
|
+
if (this.consumersByDependency.has(dependencyId)) {
|
|
29
|
+
this.protectedResults.set(dependencyId, result);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
has(dependencyId) {
|
|
33
|
+
return this.protectedResults.has(dependencyId);
|
|
34
|
+
}
|
|
35
|
+
get(dependencyId) {
|
|
36
|
+
return this.protectedResults.get(dependencyId);
|
|
37
|
+
}
|
|
38
|
+
releaseConsumer(consumerId) {
|
|
39
|
+
const dependencies = this.dependenciesByConsumer.get(consumerId);
|
|
40
|
+
if (!dependencies)
|
|
41
|
+
return;
|
|
42
|
+
for (const dependencyId of dependencies) {
|
|
43
|
+
const consumers = this.consumersByDependency.get(dependencyId);
|
|
44
|
+
if (!consumers)
|
|
45
|
+
continue;
|
|
46
|
+
consumers.delete(consumerId);
|
|
47
|
+
if (consumers.size === 0) {
|
|
48
|
+
this.consumersByDependency.delete(dependencyId);
|
|
49
|
+
this.protectedResults.delete(dependencyId);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
this.dependenciesByConsumer.delete(consumerId);
|
|
53
|
+
}
|
|
54
|
+
releaseDependency(consumerId, dependencyId) {
|
|
55
|
+
const dependencies = this.dependenciesByConsumer.get(consumerId);
|
|
56
|
+
dependencies?.delete(dependencyId);
|
|
57
|
+
if (dependencies?.size === 0)
|
|
58
|
+
this.dependenciesByConsumer.delete(consumerId);
|
|
59
|
+
const consumers = this.consumersByDependency.get(dependencyId);
|
|
60
|
+
consumers?.delete(consumerId);
|
|
61
|
+
if (consumers?.size === 0) {
|
|
62
|
+
this.consumersByDependency.delete(dependencyId);
|
|
63
|
+
this.protectedResults.delete(dependencyId);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
clear() {
|
|
67
|
+
this.consumersByDependency.clear();
|
|
68
|
+
this.dependenciesByConsumer.clear();
|
|
69
|
+
this.protectedResults.clear();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -8,6 +8,7 @@ import type { Shard } from '../../domain/queue/shard';
|
|
|
8
8
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
9
9
|
import type { RWLock } from '../../shared/lock';
|
|
10
10
|
import type { SetLike, MapLike } from '../../shared/lru';
|
|
11
|
+
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
11
12
|
/** Ack operation context */
|
|
12
13
|
export interface AckContext {
|
|
13
14
|
storage: SqliteStorage | null;
|
|
@@ -20,6 +21,7 @@ export interface AckContext {
|
|
|
20
21
|
/** Bare completion ids for removeOnComplete jobs so dependents can unblock */
|
|
21
22
|
depCompletions?: SetLike<JobId>;
|
|
22
23
|
jobResults: MapLike<JobId, unknown>;
|
|
24
|
+
dependencyResults: DependencyResultTracker;
|
|
23
25
|
jobIndex: Map<JobId, JobLocation>;
|
|
24
26
|
customIdMap?: MapLike<string, JobId>;
|
|
25
27
|
totalCompleted: {
|
|
@@ -60,6 +60,9 @@ export async function ackJob(jobId, result, ctx) {
|
|
|
60
60
|
// without making the job appear in state/stats queries.
|
|
61
61
|
ctx.depCompletions?.add(jobId);
|
|
62
62
|
}
|
|
63
|
+
if (result !== undefined)
|
|
64
|
+
ctx.dependencyResults.retain(jobId, result);
|
|
65
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
63
66
|
ctx.totalCompleted.value++;
|
|
64
67
|
if (ctx.perQueueMetrics) {
|
|
65
68
|
const pq = ctx.perQueueMetrics.get(job.queue);
|
|
@@ -205,6 +208,9 @@ export async function failJob(jobId, error, ctx, unrecoverable = false, stack) {
|
|
|
205
208
|
prev: 'failed',
|
|
206
209
|
});
|
|
207
210
|
}
|
|
211
|
+
else {
|
|
212
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
213
|
+
}
|
|
208
214
|
// BullMQ v5: failParentOnFailure — propagate terminal failure to parent
|
|
209
215
|
if (!wasRetried && job.failParentOnFailure && job.parentId && ctx.onChildTerminalFailure) {
|
|
210
216
|
ctx.onChildTerminalFailure(job, error);
|
|
@@ -7,7 +7,7 @@ import type { Shard } from '../../domain/queue/shard';
|
|
|
7
7
|
import type { RWLock } from '../../shared/lock';
|
|
8
8
|
import type { SetLike, MapLike } from '../../shared/lru';
|
|
9
9
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
10
|
-
|
|
10
|
+
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
11
11
|
export interface ExtractedJob<T = unknown> {
|
|
12
12
|
id: JobId;
|
|
13
13
|
job: Job;
|
|
@@ -58,6 +58,7 @@ export interface FinalizeContext {
|
|
|
58
58
|
/** Bare completion ids for removeOnComplete jobs so dependents can unblock */
|
|
59
59
|
depCompletions?: SetLike<JobId>;
|
|
60
60
|
jobResults: MapLike<JobId, unknown>;
|
|
61
|
+
dependencyResults?: DependencyResultTracker;
|
|
61
62
|
jobIndex: Map<JobId, JobLocation>;
|
|
62
63
|
customIdMap?: MapLike<string, JobId>;
|
|
63
64
|
totalCompleted: {
|
|
@@ -171,6 +171,12 @@ export function finalizeBatchAck(extractedJobs, ctx, includeResults) {
|
|
|
171
171
|
ctx.depCompletions?.add(jobId);
|
|
172
172
|
}
|
|
173
173
|
}
|
|
174
|
+
for (let i = 0; i < jobCount; i++) {
|
|
175
|
+
const { id, result } = extractedJobs[i];
|
|
176
|
+
if (includeResults && result !== undefined)
|
|
177
|
+
ctx.dependencyResults?.retain(id, result);
|
|
178
|
+
ctx.dependencyResults?.releaseConsumer(id);
|
|
179
|
+
}
|
|
174
180
|
// Broadcast events
|
|
175
181
|
if (needsBroadcast) {
|
|
176
182
|
for (let i = 0; i < jobCount; i++) {
|
|
@@ -9,6 +9,7 @@ import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
|
9
9
|
import type { WebhookManager } from '../webhookManager';
|
|
10
10
|
import type { EventsManager } from '../eventsManager';
|
|
11
11
|
import { type RWLock } from '../../shared/lock';
|
|
12
|
+
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
12
13
|
export { discardJob, moveJobToDelayed } from './jobMoveOperations';
|
|
13
14
|
/** Context for job management operations */
|
|
14
15
|
export interface JobManagementContext {
|
|
@@ -23,6 +24,7 @@ export interface JobManagementContext {
|
|
|
23
24
|
webhookManager: WebhookManager;
|
|
24
25
|
eventsManager: EventsManager;
|
|
25
26
|
repeatChain?: Map<JobId, JobId>;
|
|
27
|
+
dependencyResults: DependencyResultTracker;
|
|
26
28
|
}
|
|
27
29
|
/** Cancel a job (remove from queue) */
|
|
28
30
|
export declare function cancelJob(jobId: JobId, ctx: JobManagementContext): Promise<boolean>;
|
|
@@ -22,6 +22,7 @@ export async function cancelJob(jobId, ctx) {
|
|
|
22
22
|
shard.releaseUniqueKey(location.queueName, job.uniqueKey);
|
|
23
23
|
ctx.jobIndex.delete(jobId);
|
|
24
24
|
ctx.storage?.deleteJob(jobId);
|
|
25
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
25
26
|
return { success: true, queueName: location.queueName };
|
|
26
27
|
}
|
|
27
28
|
// Not in the run queue — it may be parked in waitingChildren (moved via
|
|
@@ -32,6 +33,7 @@ export async function cancelJob(jobId, ctx) {
|
|
|
32
33
|
shard.waitingChildren.delete(jobId);
|
|
33
34
|
ctx.jobIndex.delete(jobId);
|
|
34
35
|
ctx.storage?.deleteJob(jobId);
|
|
36
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
35
37
|
return { success: true, queueName: location.queueName };
|
|
36
38
|
}
|
|
37
39
|
// Or parked in waitingDeps (flow-chain dependent inserted by push.ts while
|
|
@@ -47,6 +49,7 @@ export async function cancelJob(jobId, ctx) {
|
|
|
47
49
|
shard.releaseUniqueKey(location.queueName, waiting.uniqueKey);
|
|
48
50
|
ctx.jobIndex.delete(jobId);
|
|
49
51
|
ctx.storage?.deleteJob(jobId);
|
|
52
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
50
53
|
return { success: true, queueName: location.queueName };
|
|
51
54
|
}
|
|
52
55
|
return { success: false, queueName: location.queueName };
|
|
@@ -8,6 +8,7 @@ import type { Shard } from '../../domain/queue/shard';
|
|
|
8
8
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
9
9
|
import type { RWLock } from '../../shared/lock';
|
|
10
10
|
import type { SetLike, MapLike } from '../../shared/lru';
|
|
11
|
+
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
11
12
|
/** Push operation context */
|
|
12
13
|
export interface PushContext {
|
|
13
14
|
storage: SqliteStorage | null;
|
|
@@ -20,6 +21,7 @@ export interface PushContext {
|
|
|
20
21
|
/** Timeout markers — cleared on custom-id reuse so a recycled id starts clean */
|
|
21
22
|
timedOutJobs?: SetLike<JobId>;
|
|
22
23
|
jobResults: MapLike<JobId, unknown>;
|
|
24
|
+
dependencyResults: DependencyResultTracker;
|
|
23
25
|
customIdMap: MapLike<string, JobId>;
|
|
24
26
|
jobIndex: Map<JobId, JobLocation>;
|
|
25
27
|
totalPushed: {
|
|
@@ -2,9 +2,13 @@ import type { Shard } from '../../domain/queue/shard';
|
|
|
2
2
|
import type { Job, JobId } from '../../domain/types/job';
|
|
3
3
|
import type { JobLocation } from '../../domain/types/queue';
|
|
4
4
|
import type { SetLike } from '../../shared/lru';
|
|
5
|
+
import type { MapLike } from '../../shared/lru';
|
|
6
|
+
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
5
7
|
export interface PushInsertContext {
|
|
6
8
|
completedJobs: SetLike<JobId>;
|
|
7
9
|
depCompletions?: SetLike<JobId>;
|
|
10
|
+
jobResults: MapLike<JobId, unknown>;
|
|
11
|
+
dependencyResults: DependencyResultTracker;
|
|
8
12
|
jobIndex: Map<JobId, JobLocation>;
|
|
9
13
|
dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
|
|
10
14
|
}
|
|
@@ -22,4 +22,10 @@ export function insertJobToShard(job, queue, shard, shardIdx, ctx) {
|
|
|
22
22
|
job.timeline.push({ state, timestamp: now });
|
|
23
23
|
}
|
|
24
24
|
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
|
|
25
|
+
ctx.dependencyResults.registerConsumer(job.id, job.dependsOn);
|
|
26
|
+
for (const dependencyId of job.dependsOn) {
|
|
27
|
+
if (ctx.jobResults.has(dependencyId)) {
|
|
28
|
+
ctx.dependencyResults.retain(dependencyId, ctx.jobResults.get(dependencyId));
|
|
29
|
+
}
|
|
30
|
+
}
|
|
25
31
|
}
|
|
@@ -9,6 +9,7 @@ import type { Shard } from '../../domain/queue/shard';
|
|
|
9
9
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
10
10
|
import { type RWLock } from '../../shared/lock';
|
|
11
11
|
import type { SetLike, MapLike } from '../../shared/lru';
|
|
12
|
+
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
12
13
|
/** Context for query operations */
|
|
13
14
|
export interface QueryContext {
|
|
14
15
|
storage: SqliteStorage | null;
|
|
@@ -20,6 +21,7 @@ export interface QueryContext {
|
|
|
20
21
|
completedJobs: SetLike<JobId>;
|
|
21
22
|
completedJobsData: MapLike<JobId, Job>;
|
|
22
23
|
jobResults: MapLike<JobId, unknown>;
|
|
24
|
+
dependencyResults: DependencyResultTracker;
|
|
23
25
|
customIdMap: MapLike<string, JobId>;
|
|
24
26
|
}
|
|
25
27
|
/** Get job by ID */
|
|
@@ -78,7 +78,11 @@ export async function getJob(jobId, ctx) {
|
|
|
78
78
|
}
|
|
79
79
|
/** Get job result */
|
|
80
80
|
export function getJobResult(jobId, ctx) {
|
|
81
|
-
|
|
81
|
+
if (ctx.jobResults.has(jobId))
|
|
82
|
+
return ctx.jobResults.get(jobId);
|
|
83
|
+
if (ctx.dependencyResults.has(jobId))
|
|
84
|
+
return ctx.dependencyResults.get(jobId);
|
|
85
|
+
return ctx.storage?.getResult(jobId);
|
|
82
86
|
}
|
|
83
87
|
/** Get job by custom ID */
|
|
84
88
|
export function getJobByCustomId(customId, ctx) {
|
|
@@ -6,6 +6,7 @@ import type { Job, JobId } from '../../domain/types/job';
|
|
|
6
6
|
import type { Shard } from '../../domain/queue/shard';
|
|
7
7
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
8
8
|
import type { MapLike, SetLike } from '../../shared/lru';
|
|
9
|
+
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
9
10
|
/** Context for queue control operations */
|
|
10
11
|
export interface QueueControlContext {
|
|
11
12
|
shards: Shard[];
|
|
@@ -19,6 +20,7 @@ export interface QueueControlContext {
|
|
|
19
20
|
completedJobsData?: MapLike<JobId, Job>;
|
|
20
21
|
jobResults?: MapLike<JobId, unknown>;
|
|
21
22
|
jobLogs?: MapLike<JobId, unknown>;
|
|
23
|
+
dependencyResults: DependencyResultTracker;
|
|
22
24
|
storage: SqliteStorage | null;
|
|
23
25
|
}
|
|
24
26
|
/** Pause a queue */
|
|
@@ -28,6 +28,7 @@ export function drainQueue(queue, ctx) {
|
|
|
28
28
|
// (not-yet-flushed) add cannot land on disk after the drain.
|
|
29
29
|
for (const jobId of jobIds) {
|
|
30
30
|
ctx.jobIndex.delete(jobId);
|
|
31
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
31
32
|
safeDeleteJob(ctx, jobId);
|
|
32
33
|
}
|
|
33
34
|
return count;
|
|
@@ -84,6 +85,7 @@ function cleanWaitingLike(queue, graceMs, ctx, maxJobs) {
|
|
|
84
85
|
shard.decrementQueued(jobId);
|
|
85
86
|
shard.removeFromTemporalIndex(jobId);
|
|
86
87
|
ctx.jobIndex.delete(jobId);
|
|
88
|
+
ctx.dependencyResults.releaseConsumer(jobId);
|
|
87
89
|
safeDeleteJob(ctx, jobId);
|
|
88
90
|
removed.push(jobId);
|
|
89
91
|
}
|
|
@@ -112,6 +114,7 @@ function cleanCompleted(queue, graceMs, ctx, maxJobs) {
|
|
|
112
114
|
ctx.jobResults?.delete(jid);
|
|
113
115
|
ctx.jobLogs?.delete(jid);
|
|
114
116
|
ctx.jobIndex.delete(jid);
|
|
117
|
+
ctx.dependencyResults.releaseConsumer(jid);
|
|
115
118
|
safeDeleteJob(ctx, jid);
|
|
116
119
|
}
|
|
117
120
|
return toRemove;
|
|
@@ -133,6 +136,7 @@ function cleanFailed(queue, graceMs, ctx, maxJobs) {
|
|
|
133
136
|
for (const jid of toRemove) {
|
|
134
137
|
shard.removeFromDlq(queue, jid);
|
|
135
138
|
ctx.jobIndex.delete(jid);
|
|
139
|
+
ctx.dependencyResults.releaseConsumer(jid);
|
|
136
140
|
ctx.jobResults?.delete(jid);
|
|
137
141
|
ctx.jobLogs?.delete(jid);
|
|
138
142
|
safeDeleteDlqEntry(ctx, jid);
|
|
@@ -31,6 +31,7 @@ export declare class QueueManager {
|
|
|
31
31
|
private readonly depCompletions;
|
|
32
32
|
private readonly timedOutJobs;
|
|
33
33
|
private readonly jobResults;
|
|
34
|
+
private readonly dependencyResults;
|
|
34
35
|
private readonly customIdMap;
|
|
35
36
|
private readonly jobLogs;
|
|
36
37
|
private readonly pendingDepChecks;
|
|
@@ -32,6 +32,7 @@ import * as statsMgr from './statsManager';
|
|
|
32
32
|
import { ContextFactory } from './contextFactory';
|
|
33
33
|
import { processPendingDependencies } from './dependencyProcessor';
|
|
34
34
|
import { handleTaskError, handleTaskSuccess } from './taskErrorTracking';
|
|
35
|
+
import { DependencyResultTracker } from './dependencyResultTracker';
|
|
35
36
|
/**
|
|
36
37
|
* QueueManager - Central coordinator
|
|
37
38
|
*/
|
|
@@ -58,6 +59,7 @@ export class QueueManager {
|
|
|
58
59
|
// current token and bypasses the stale-token recovery path entirely.
|
|
59
60
|
timedOutJobs;
|
|
60
61
|
jobResults;
|
|
62
|
+
dependencyResults = new DependencyResultTracker();
|
|
61
63
|
customIdMap;
|
|
62
64
|
jobLogs;
|
|
63
65
|
// Deferred dependency resolution queue
|
|
@@ -178,6 +180,7 @@ export class QueueManager {
|
|
|
178
180
|
depCompletions: this.depCompletions,
|
|
179
181
|
timedOutJobs: this.timedOutJobs,
|
|
180
182
|
jobResults: this.jobResults,
|
|
183
|
+
dependencyResults: this.dependencyResults,
|
|
181
184
|
customIdMap: this.customIdMap,
|
|
182
185
|
jobLogs: this.jobLogs,
|
|
183
186
|
jobLocks: this.jobLocks,
|
|
@@ -541,6 +544,9 @@ export class QueueManager {
|
|
|
541
544
|
ctx.jobIndex.delete(jId);
|
|
542
545
|
ctx.storage?.deleteJob(jId);
|
|
543
546
|
}
|
|
547
|
+
if (result !== undefined)
|
|
548
|
+
ctx.dependencyResults.retain(jId, result);
|
|
549
|
+
ctx.dependencyResults.releaseConsumer(jId);
|
|
544
550
|
ctx.totalCompleted.value++;
|
|
545
551
|
ctx.broadcast({
|
|
546
552
|
eventType: 'completed',
|
|
@@ -769,6 +775,7 @@ export class QueueManager {
|
|
|
769
775
|
this.jobResults.delete(jid);
|
|
770
776
|
this.jobLogs.delete(jid);
|
|
771
777
|
this.jobLocks.delete(jid);
|
|
778
|
+
this.dependencyResults.releaseConsumer(jid);
|
|
772
779
|
this.failedChildrenValues.delete(jid);
|
|
773
780
|
this.ignoredChildrenFailures.delete(jid);
|
|
774
781
|
this.pendingDepChecks.delete(jid);
|
|
@@ -1259,6 +1266,7 @@ export class QueueManager {
|
|
|
1259
1266
|
// Parent reached a terminal (DLQ) state — release its flow-failure tracking.
|
|
1260
1267
|
this.failedChildrenValues.delete(parentId);
|
|
1261
1268
|
this.ignoredChildrenFailures.delete(parentId);
|
|
1269
|
+
this.dependencyResults.releaseConsumer(parentId);
|
|
1262
1270
|
// Broadcast failed event for parent
|
|
1263
1271
|
this.eventsManager.broadcast({
|
|
1264
1272
|
eventType: 'failed',
|
|
@@ -1396,6 +1404,7 @@ export class QueueManager {
|
|
|
1396
1404
|
if (depIndex !== -1) {
|
|
1397
1405
|
parentJob.dependsOn.splice(depIndex, 1);
|
|
1398
1406
|
shard.unregisterDependencies(parentId, [childJob.id]);
|
|
1407
|
+
this.dependencyResults.releaseDependency(parentId, childJob.id);
|
|
1399
1408
|
}
|
|
1400
1409
|
// If no more pending deps, the parent is ready to be promoted
|
|
1401
1410
|
readyToPromote =
|
|
@@ -1449,6 +1458,7 @@ export class QueueManager {
|
|
|
1449
1458
|
if (depIndex !== -1) {
|
|
1450
1459
|
parentJob.dependsOn.splice(depIndex, 1);
|
|
1451
1460
|
shard.unregisterDependencies(parentId, [childJobId]);
|
|
1461
|
+
this.dependencyResults.releaseDependency(parentId, childJobId);
|
|
1452
1462
|
}
|
|
1453
1463
|
const allDone = parentJob.dependsOn.length === 0 ||
|
|
1454
1464
|
parentJob.dependsOn.every((dep) => this.completedJobs.has(dep));
|
|
@@ -1615,6 +1625,7 @@ export class QueueManager {
|
|
|
1615
1625
|
this.depCompletions.clear();
|
|
1616
1626
|
this.timedOutJobs.clear();
|
|
1617
1627
|
this.jobResults.clear();
|
|
1628
|
+
this.dependencyResults.clear();
|
|
1618
1629
|
this.jobLogs.clear();
|
|
1619
1630
|
this.customIdMap.clear();
|
|
1620
1631
|
this.pendingDepChecks.clear();
|
|
@@ -12,6 +12,7 @@ import type { EventsManager } from './eventsManager';
|
|
|
12
12
|
import type { WebhookManager } from './webhookManager';
|
|
13
13
|
import type { WorkerManager } from './workerManager';
|
|
14
14
|
import type { MonitoringState } from './monitoringChecks';
|
|
15
|
+
import type { DependencyResultTracker } from './dependencyResultTracker';
|
|
15
16
|
/** Queue Manager configuration */
|
|
16
17
|
export interface QueueManagerConfig {
|
|
17
18
|
dataPath?: string;
|
|
@@ -52,6 +53,7 @@ export interface QueueManagerState {
|
|
|
52
53
|
readonly jobIndex: Map<JobId, JobLocation>;
|
|
53
54
|
readonly completedJobs: BoundedSet<JobId>;
|
|
54
55
|
readonly jobResults: LRUMap<JobId, unknown>;
|
|
56
|
+
readonly dependencyResults: DependencyResultTracker;
|
|
55
57
|
readonly customIdMap: LRUMap<string, JobId>;
|
|
56
58
|
readonly jobLogs: LRUMap<JobId, JobLogEntry[]>;
|
|
57
59
|
readonly jobLocks: Map<JobId, JobLock>;
|
|
@@ -144,12 +144,6 @@ export function createTcpServer(queueManager, config) {
|
|
|
144
144
|
initConnection(socket);
|
|
145
145
|
const { frameParser, ctx, state, semaphore, writeQueue } = socket.data;
|
|
146
146
|
const rateLimiter = getRateLimiter();
|
|
147
|
-
// Check rate limit
|
|
148
|
-
if (!rateLimiter.isAllowed(state.clientId)) {
|
|
149
|
-
ctx.queueManager.emitDashboardEvent('ratelimit:hit', { clientId: state.clientId });
|
|
150
|
-
writeQueue.write(socket, errorResponse('Rate limit exceeded'));
|
|
151
|
-
return;
|
|
152
|
-
}
|
|
153
147
|
let frames;
|
|
154
148
|
try {
|
|
155
149
|
// `data` is a Bun Buffer (a Uint8Array). addData copies it into its own
|
|
@@ -194,11 +188,24 @@ export function createTcpServer(queueManager, config) {
|
|
|
194
188
|
cmd = unpack(frame);
|
|
195
189
|
}
|
|
196
190
|
catch {
|
|
191
|
+
// Invalid complete frames still consume protocol quota. They cannot
|
|
192
|
+
// carry a trustworthy reqId because msgpack decoding failed.
|
|
193
|
+
}
|
|
194
|
+
const reqId = typeof cmd?.reqId === 'string' ? cmd.reqId : undefined;
|
|
195
|
+
if (!rateLimiter.isAllowed(state.clientId)) {
|
|
196
|
+
ctx.queueManager.emitDashboardEvent('ratelimit:hit', { clientId: state.clientId });
|
|
197
|
+
writeQueue.write(socket, errorResponse('Rate limit exceeded', reqId));
|
|
198
|
+
dropForWriteOverflow();
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
if (!cmd) {
|
|
197
202
|
writeQueue.write(socket, errorResponse('Invalid command format'));
|
|
203
|
+
dropForWriteOverflow();
|
|
198
204
|
return;
|
|
199
205
|
}
|
|
200
206
|
if (!cmd?.cmd) {
|
|
201
|
-
writeQueue.write(socket, errorResponse('Invalid command'));
|
|
207
|
+
writeQueue.write(socket, errorResponse('Invalid command', reqId));
|
|
208
|
+
dropForWriteOverflow();
|
|
202
209
|
return;
|
|
203
210
|
}
|
|
204
211
|
// Process with concurrency limit
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "bunqueue",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.39",
|
|
4
4
|
"description": "High-performance job queue for Bun & AI agents. SQLite persistence, cron scheduling, priorities, retries, DLQ, webhooks, native MCP server. Zero external infrastructure.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/main.js",
|