bunqueue 2.8.50 → 2.8.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -0
- package/dist/application/backgroundTasks.js +21 -4
- package/dist/application/cleanupTasks.js +7 -3
- package/dist/application/contextFactory.d.ts +2 -1
- package/dist/application/contextFactory.js +5 -0
- package/dist/application/dependencyCompletions.d.ts +53 -0
- package/dist/application/dependencyCompletions.js +123 -0
- package/dist/application/dependencyProcessor.d.ts +5 -0
- package/dist/application/dependencyProcessor.js +25 -13
- package/dist/application/flowFailureRecovery.d.ts +3 -0
- package/dist/application/flowFailureRecovery.js +3 -1
- package/dist/application/flowParentBackpatch.d.ts +27 -0
- package/dist/application/flowParentBackpatch.js +120 -0
- package/dist/application/operations/ack.d.ts +3 -1
- package/dist/application/operations/ack.js +2 -2
- package/dist/application/operations/ackHelpers.d.ts +5 -4
- package/dist/application/operations/ackHelpers.js +4 -10
- package/dist/application/operations/customId.d.ts +3 -0
- package/dist/application/operations/customId.js +10 -0
- package/dist/application/operations/jobManagement.d.ts +3 -0
- package/dist/application/operations/jobManagement.js +10 -4
- package/dist/application/operations/push.d.ts +3 -1
- package/dist/application/operations/pushInsert.d.ts +4 -1
- package/dist/application/operations/pushInsert.js +2 -0
- package/dist/application/queueManager.d.ts +2 -0
- package/dist/application/queueManager.js +134 -75
- package/dist/application/types.d.ts +3 -1
- package/dist/client/flowPlan.js +8 -0
- package/dist/client/flowTypes.d.ts +2 -2
- package/dist/infrastructure/persistence/dependencyCompletionSchema.d.ts +6 -0
- package/dist/infrastructure/persistence/dependencyCompletionSchema.js +16 -0
- package/dist/infrastructure/persistence/dependencyCompletionStore.d.ts +38 -0
- package/dist/infrastructure/persistence/dependencyCompletionStore.js +105 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -5
- package/dist/infrastructure/persistence/schema.js +6 -1
- package/dist/infrastructure/persistence/sqlite.d.ts +20 -1
- package/dist/infrastructure/persistence/sqlite.js +119 -15
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +3 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +10 -0
- package/package.json +1 -1
|
@@ -9,6 +9,7 @@ import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
|
9
9
|
import type { RWLock } from '../../shared/lock';
|
|
10
10
|
import type { SetLike, MapLike } from '../../shared/lru';
|
|
11
11
|
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
12
|
+
import { type DependencyCompletionTracker } from '../dependencyCompletions';
|
|
12
13
|
/** Ack operation context */
|
|
13
14
|
export interface AckContext {
|
|
14
15
|
storage: SqliteStorage | null;
|
|
@@ -19,7 +20,8 @@ export interface AckContext {
|
|
|
19
20
|
completedJobs: SetLike<JobId>;
|
|
20
21
|
completedJobsData: MapLike<JobId, Job>;
|
|
21
22
|
/** Bare completion ids for removeOnComplete jobs so dependents can unblock */
|
|
22
|
-
depCompletions?:
|
|
23
|
+
depCompletions?: DependencyCompletionTracker;
|
|
24
|
+
maxDependencyCompletions: number;
|
|
23
25
|
jobResults: MapLike<JobId, unknown>;
|
|
24
26
|
dependencyResults: DependencyResultTracker;
|
|
25
27
|
jobIndex: Map<JobId, JobLocation>;
|
|
@@ -8,6 +8,7 @@ import { shardIndex, processingShardIndex } from '../../shared/hash';
|
|
|
8
8
|
import { latencyTracker } from '../latencyTracker';
|
|
9
9
|
import { throughputTracker } from '../throughputTracker';
|
|
10
10
|
import { groupByProcShard, groupItemsByProcShard, extractJobs, extractJobsWithResults, groupByQueueShard, releaseResources, finalizeBatchAck, } from './ackHelpers';
|
|
11
|
+
import { commitRemovedCompletion, } from '../dependencyCompletions';
|
|
11
12
|
/**
|
|
12
13
|
* Acknowledge job completion
|
|
13
14
|
*/
|
|
@@ -52,13 +53,12 @@ export async function ackJob(jobId, result, ctx) {
|
|
|
52
53
|
ctx.storage?.markCompleted(jobId, now, job.timeline);
|
|
53
54
|
}
|
|
54
55
|
else {
|
|
56
|
+
commitRemovedCompletion(job, ctx);
|
|
55
57
|
ctx.jobIndex.delete(jobId);
|
|
56
|
-
ctx.storage?.deleteJob(jobId);
|
|
57
58
|
// removeOnComplete drops the full job (index + data + persisted row) to bound
|
|
58
59
|
// memory, but dependent jobs gate readiness on completedJobs.has(parentId).
|
|
59
60
|
// Record the bare completion id (no payload) so dependents still unblock,
|
|
60
61
|
// without making the job appear in state/stats queries.
|
|
61
|
-
ctx.depCompletions?.add(jobId);
|
|
62
62
|
}
|
|
63
63
|
if (result !== undefined)
|
|
64
64
|
ctx.dependencyResults.retain(jobId, result);
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Ack Helpers - Shared batch processing utilities
|
|
3
|
-
*/
|
|
1
|
+
/** Shared batch acknowledgement utilities. */
|
|
4
2
|
import type { Job, JobId } from '../../domain/types/job';
|
|
5
3
|
import type { JobLocation, EventType } from '../../domain/types/queue';
|
|
6
4
|
import type { Shard } from '../../domain/queue/shard';
|
|
@@ -8,6 +6,7 @@ import type { RWLock } from '../../shared/lock';
|
|
|
8
6
|
import type { SetLike, MapLike } from '../../shared/lru';
|
|
9
7
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
10
8
|
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
9
|
+
import { type DependencyCompletionTracker } from '../dependencyCompletions';
|
|
11
10
|
export interface ExtractedJob<T = unknown> {
|
|
12
11
|
id: JobId;
|
|
13
12
|
job: Job;
|
|
@@ -53,10 +52,12 @@ export declare function releaseResources(byQueueShard: Map<number, Job[]>, ctx:
|
|
|
53
52
|
/** Context for finalize operations */
|
|
54
53
|
export interface FinalizeContext {
|
|
55
54
|
storage: SqliteStorage | null;
|
|
55
|
+
shards: Shard[];
|
|
56
56
|
completedJobs: SetLike<JobId>;
|
|
57
57
|
completedJobsData: MapLike<JobId, Job>;
|
|
58
58
|
/** Bare completion ids for removeOnComplete jobs so dependents can unblock */
|
|
59
|
-
depCompletions?:
|
|
59
|
+
depCompletions?: DependencyCompletionTracker;
|
|
60
|
+
maxDependencyCompletions: number;
|
|
60
61
|
jobResults: MapLike<JobId, unknown>;
|
|
61
62
|
dependencyResults?: DependencyResultTracker;
|
|
62
63
|
jobIndex: Map<JobId, JobLocation>;
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Ack Helpers - Shared batch processing utilities
|
|
3
|
-
*/
|
|
4
1
|
import { MAX_TIMELINE_ENTRIES } from '../../domain/types/job';
|
|
5
2
|
import { withWriteLock } from '../../shared/lock';
|
|
6
3
|
import { shardIndex, processingShardIndex } from '../../shared/hash';
|
|
7
4
|
import { throughputTracker } from '../throughputTracker';
|
|
5
|
+
import { commitRemovedCompletion, } from '../dependencyCompletions';
|
|
8
6
|
/**
|
|
9
7
|
* Group job IDs by processing shard
|
|
10
8
|
* Returns Map<shardIndex, jobIds[]>
|
|
@@ -161,14 +159,10 @@ export function finalizeBatchAck(extractedJobs, ctx, includeResults) {
|
|
|
161
159
|
ctx.completedJobs.add(jobId);
|
|
162
160
|
}
|
|
163
161
|
else {
|
|
162
|
+
commitRemovedCompletion(job, ctx, now);
|
|
164
163
|
ctx.jobIndex.delete(jobId);
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
// removeOnComplete drops the full job to bound memory, but dependents gate
|
|
168
|
-
// readiness on completedJobs.has(parentId). Record the bare completion id
|
|
169
|
-
// (no payload) so dependent jobs still unblock, without surfacing the job
|
|
170
|
-
// in state/stats queries.
|
|
171
|
-
ctx.depCompletions?.add(jobId);
|
|
164
|
+
// The helper publishes payload-free dependency evidence only after its
|
|
165
|
+
// durable job-removal transaction commits.
|
|
172
166
|
}
|
|
173
167
|
}
|
|
174
168
|
for (let i = 0; i < jobCount; i++) {
|
|
@@ -3,11 +3,14 @@ import { type Job, type JobId, type JobInput } from '../../domain/types/job';
|
|
|
3
3
|
import type { JobLocation } from '../../domain/types/queue';
|
|
4
4
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
5
5
|
import type { MapLike, SetLike } from '../../shared/lru';
|
|
6
|
+
import { type DependencyCompletionTracker } from '../dependencyCompletions';
|
|
6
7
|
export interface CustomIdContext {
|
|
7
8
|
storage: SqliteStorage | null;
|
|
8
9
|
shards: Shard[];
|
|
9
10
|
completedJobs: SetLike<JobId>;
|
|
10
11
|
completedJobsData: MapLike<JobId, Job>;
|
|
12
|
+
depCompletions?: DependencyCompletionTracker;
|
|
13
|
+
maxDependencyCompletions: number;
|
|
11
14
|
timedOutJobs?: SetLike<JobId>;
|
|
12
15
|
jobResults: MapLike<JobId, unknown>;
|
|
13
16
|
customIdMap: MapLike<string, JobId>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { generateJobId, jobId } from '../../domain/types/job';
|
|
2
2
|
import { shardIndex } from '../../shared/hash';
|
|
3
|
+
import { releaseDependencyCompletionPins, } from '../dependencyCompletions';
|
|
3
4
|
/**
|
|
4
5
|
* Enforce custom-ID idempotency while the caller holds the target shard lock.
|
|
5
6
|
* Live generations are returned unchanged; terminal generations are retired
|
|
@@ -50,6 +51,15 @@ export function handleCustomId(input, ctx, lockedShardIndexes) {
|
|
|
50
51
|
// id. The fresh jobs row is inserted only after this retirement completes.
|
|
51
52
|
ctx.storage?.deleteDlqEntry(id);
|
|
52
53
|
}
|
|
54
|
+
if (ctx.depCompletions?.has(id)) {
|
|
55
|
+
const hasUnresolvedConsumers = ctx.shards.some((shard) => (shard.getJobsWaitingFor(id)?.size ?? 0) > 0);
|
|
56
|
+
if (hasUnresolvedConsumers) {
|
|
57
|
+
throw new Error(`Custom ID ${String(id)} still has unresolved dependency consumers`);
|
|
58
|
+
}
|
|
59
|
+
releaseDependencyCompletionPins([id], ctx);
|
|
60
|
+
ctx.storage?.deleteDependencyCompletion(id);
|
|
61
|
+
ctx.depCompletions.delete(id);
|
|
62
|
+
}
|
|
53
63
|
// A durable jobs row may outlive its in-memory tracking after an interrupted
|
|
54
64
|
// cleanup. The storage insert upserts that orphan without adding a DELETE to
|
|
55
65
|
// every custom-ID push.
|
|
@@ -10,6 +10,7 @@ import type { WebhookManager } from '../webhookManager';
|
|
|
10
10
|
import type { EventsManager } from '../eventsManager';
|
|
11
11
|
import { type RWLock } from '../../shared/lock';
|
|
12
12
|
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
13
|
+
import { type DependencyCompletionTracker } from '../dependencyCompletions';
|
|
13
14
|
export { discardJob, moveJobToDelayed } from './jobMoveOperations';
|
|
14
15
|
/** Context for job management operations */
|
|
15
16
|
export interface JobManagementContext {
|
|
@@ -25,6 +26,8 @@ export interface JobManagementContext {
|
|
|
25
26
|
eventsManager: EventsManager;
|
|
26
27
|
repeatChain?: Map<JobId, JobId>;
|
|
27
28
|
dependencyResults: DependencyResultTracker;
|
|
29
|
+
depCompletions?: DependencyCompletionTracker;
|
|
30
|
+
maxDependencyCompletions: number;
|
|
28
31
|
}
|
|
29
32
|
/** Cancel a job (remove from queue) */
|
|
30
33
|
export declare function cancelJob(jobId: JobId, ctx: JobManagementContext): Promise<boolean>;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { processingShardIndex } from '../../shared/hash';
|
|
6
6
|
import { webhookLog } from '../../shared/logger';
|
|
7
7
|
import { withWriteLock } from '../../shared/lock';
|
|
8
|
+
import { releaseDependencyCompletionPins, } from '../dependencyCompletions';
|
|
8
9
|
export { discardJob, moveJobToDelayed } from './jobMoveOperations';
|
|
9
10
|
/** Cancel a job (remove from queue) */
|
|
10
11
|
export async function cancelJob(jobId, ctx) {
|
|
@@ -23,7 +24,7 @@ export async function cancelJob(jobId, ctx) {
|
|
|
23
24
|
ctx.jobIndex.delete(jobId);
|
|
24
25
|
ctx.storage?.deleteJob(jobId);
|
|
25
26
|
ctx.dependencyResults.releaseConsumer(jobId);
|
|
26
|
-
return { success: true, queueName: location.queueName };
|
|
27
|
+
return { success: true, queueName: location.queueName, released: [] };
|
|
27
28
|
}
|
|
28
29
|
// Not in the run queue — it may be parked in waitingChildren (moved via
|
|
29
30
|
// moveToWaitingChildren, which already released its resources and does not
|
|
@@ -34,7 +35,7 @@ export async function cancelJob(jobId, ctx) {
|
|
|
34
35
|
ctx.jobIndex.delete(jobId);
|
|
35
36
|
ctx.storage?.deleteJob(jobId);
|
|
36
37
|
ctx.dependencyResults.releaseConsumer(jobId);
|
|
37
|
-
return { success: true, queueName: location.queueName };
|
|
38
|
+
return { success: true, queueName: location.queueName, released: [] };
|
|
38
39
|
}
|
|
39
40
|
// Or parked in waitingDeps (flow-chain dependent inserted by push.ts while
|
|
40
41
|
// its predecessors are unresolved). These are NOT counted in the queued
|
|
@@ -50,11 +51,16 @@ export async function cancelJob(jobId, ctx) {
|
|
|
50
51
|
ctx.jobIndex.delete(jobId);
|
|
51
52
|
ctx.storage?.deleteJob(jobId);
|
|
52
53
|
ctx.dependencyResults.releaseConsumer(jobId);
|
|
53
|
-
return {
|
|
54
|
+
return {
|
|
55
|
+
success: true,
|
|
56
|
+
queueName: location.queueName,
|
|
57
|
+
released: waiting.dependsOn,
|
|
58
|
+
};
|
|
54
59
|
}
|
|
55
|
-
return { success: false, queueName: location.queueName };
|
|
60
|
+
return { success: false, queueName: location.queueName, released: [] };
|
|
56
61
|
});
|
|
57
62
|
if (result.success) {
|
|
63
|
+
releaseDependencyCompletionPins(result.released, ctx);
|
|
58
64
|
// Emit removed event (BullMQ v5)
|
|
59
65
|
ctx.eventsManager.broadcast({
|
|
60
66
|
eventType: "removed" /* EventType.Removed */,
|
|
@@ -9,6 +9,7 @@ import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
|
9
9
|
import type { RWLock } from '../../shared/lock';
|
|
10
10
|
import type { SetLike, MapLike } from '../../shared/lru';
|
|
11
11
|
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
12
|
+
import type { DependencyCompletionTracker } from '../dependencyCompletions';
|
|
12
13
|
/** Push operation context */
|
|
13
14
|
export interface PushContext {
|
|
14
15
|
storage: SqliteStorage | null;
|
|
@@ -19,7 +20,8 @@ export interface PushContext {
|
|
|
19
20
|
completedJobs: SetLike<JobId>;
|
|
20
21
|
completedJobsData: MapLike<JobId, Job>;
|
|
21
22
|
/** Bare completion ids for removeOnComplete jobs so dependents start ready */
|
|
22
|
-
depCompletions?:
|
|
23
|
+
depCompletions?: DependencyCompletionTracker;
|
|
24
|
+
maxDependencyCompletions: number;
|
|
23
25
|
/** Timeout markers — cleared on custom-id reuse so a recycled id starts clean */
|
|
24
26
|
timedOutJobs?: SetLike<JobId>;
|
|
25
27
|
jobResults: MapLike<JobId, unknown>;
|
|
@@ -4,9 +4,12 @@ import type { JobLocation } from '../../domain/types/queue';
|
|
|
4
4
|
import type { SetLike } from '../../shared/lru';
|
|
5
5
|
import type { MapLike } from '../../shared/lru';
|
|
6
6
|
import type { DependencyResultTracker } from '../dependencyResultTracker';
|
|
7
|
+
import { type DependencyCompletionTracker } from '../dependencyCompletions';
|
|
8
|
+
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
7
9
|
export interface PushInsertContext {
|
|
10
|
+
storage: SqliteStorage | null;
|
|
8
11
|
completedJobs: SetLike<JobId>;
|
|
9
|
-
depCompletions?:
|
|
12
|
+
depCompletions?: DependencyCompletionTracker;
|
|
10
13
|
jobResults: MapLike<JobId, unknown>;
|
|
11
14
|
dependencyResults: DependencyResultTracker;
|
|
12
15
|
jobIndex: Map<JobId, JobLocation>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { pinReferencedCompletions, } from '../dependencyCompletions';
|
|
1
2
|
/** Resolve the first externally visible state without mutating queue structures. */
|
|
2
3
|
export function initialJobState(job, ctx, now = Date.now()) {
|
|
3
4
|
const needsWaiting = job.dependsOn.length > 0 &&
|
|
@@ -14,6 +15,7 @@ export function insertJobToShard(job, target, ctx, recordTimeline = true) {
|
|
|
14
15
|
const now = Date.now();
|
|
15
16
|
const state = initialJobState(job, ctx, now);
|
|
16
17
|
if (state === 'waiting-children') {
|
|
18
|
+
pinReferencedCompletions(job.dependsOn, ctx);
|
|
17
19
|
shard.waitingDeps.set(job.id, job);
|
|
18
20
|
shard.registerDependencies(job.id, job.dependsOn);
|
|
19
21
|
if (recordTimeline)
|
|
@@ -187,6 +187,8 @@ export declare class QueueManager {
|
|
|
187
187
|
listQueues(): string[];
|
|
188
188
|
private registerQueueName;
|
|
189
189
|
private unregisterQueueName;
|
|
190
|
+
private releaseCompletionPins;
|
|
191
|
+
private reconcileCompletionPins;
|
|
190
192
|
clean(queue: string, graceMs: number, state?: string, limit?: number): JobId[];
|
|
191
193
|
getCountsPerPriority(queue: string): Record<number, number>;
|
|
192
194
|
getJobs(queue: string, options?: {
|
|
@@ -37,6 +37,8 @@ import { processPendingDependencies } from './dependencyProcessor';
|
|
|
37
37
|
import { handleTaskError, handleTaskSuccess } from './taskErrorTracking';
|
|
38
38
|
import { DependencyResultTracker } from './dependencyResultTracker';
|
|
39
39
|
import { recoverFlowFailures } from './flowFailureRecovery';
|
|
40
|
+
import { assertFlowParentOwnership, backpatchDeclaredFlowChild, canAcceptRemovedFlowChild, flowChildFailureError, isDeclaredFlowChild, } from './flowParentBackpatch';
|
|
41
|
+
import { commitRemovedCompletion, DependencyCompletionTracker, reconcileDependencyCompletionPins, releaseDependencyCompletionPins, } from './dependencyCompletions';
|
|
40
42
|
/**
|
|
41
43
|
* QueueManager - Central coordinator
|
|
42
44
|
*/
|
|
@@ -53,9 +55,8 @@ export class QueueManager {
|
|
|
53
55
|
jobIndex = new Map();
|
|
54
56
|
completedJobs;
|
|
55
57
|
completedJobsData;
|
|
56
|
-
// Bare
|
|
57
|
-
//
|
|
58
|
-
// completedJobs; entries are pruned by the dependency processor once consumed.
|
|
58
|
+
// Bare removeOnComplete evidence: bounded recent IDs plus IDs pinned by live
|
|
59
|
+
// dependency edges. It is never surfaced as completed job data or statistics.
|
|
59
60
|
depCompletions;
|
|
60
61
|
// Ids of jobs failed by the timeout sweep. A late ACK whose lock token no
|
|
61
62
|
// longer matches (the job was requeued for retry) is discarded for these,
|
|
@@ -128,7 +129,9 @@ export class QueueManager {
|
|
|
128
129
|
this.jobIndex.delete(jobId);
|
|
129
130
|
this.completedJobsData.delete(jobId);
|
|
130
131
|
});
|
|
131
|
-
this.depCompletions = new
|
|
132
|
+
this.depCompletions = new DependencyCompletionTracker(this.config.maxCompletedJobs, (jobId) => {
|
|
133
|
+
this.storage?.deleteDependencyCompletion(jobId);
|
|
134
|
+
});
|
|
132
135
|
this.timedOutJobs = new BoundedSet(this.config.maxCompletedJobs);
|
|
133
136
|
this.perQueueMetrics = new LRUMap(this.config.maxCustomIds);
|
|
134
137
|
this.jobResults = new LRUMap(this.config.maxJobResults);
|
|
@@ -171,6 +174,8 @@ export class QueueManager {
|
|
|
171
174
|
shards: this.shards,
|
|
172
175
|
jobIndex: this.jobIndex,
|
|
173
176
|
completedJobs: this.completedJobs,
|
|
177
|
+
depCompletions: this.depCompletions,
|
|
178
|
+
maxDependencyCompletions: this.config.maxCompletedJobs,
|
|
174
179
|
dependencyResults: this.dependencyResults,
|
|
175
180
|
failedChildrenValues: this.failedChildrenValues,
|
|
176
181
|
ignoredChildrenFailures: this.ignoredChildrenFailures,
|
|
@@ -564,8 +569,8 @@ export class QueueManager {
|
|
|
564
569
|
ctx.storage?.markCompleted(jId, Date.now(), job.timeline);
|
|
565
570
|
}
|
|
566
571
|
else {
|
|
572
|
+
commitRemovedCompletion(job, ctx);
|
|
567
573
|
ctx.jobIndex.delete(jId);
|
|
568
|
-
ctx.storage?.deleteJob(jId);
|
|
569
574
|
}
|
|
570
575
|
if (result !== undefined)
|
|
571
576
|
ctx.dependencyResults.retain(jId, result);
|
|
@@ -686,94 +691,110 @@ export class QueueManager {
|
|
|
686
691
|
async updateJobParent(childJobId, parentJobId) {
|
|
687
692
|
if (childJobId === parentJobId)
|
|
688
693
|
throw new Error('A flow job cannot be its own parent');
|
|
689
|
-
|
|
690
|
-
if (!childJob)
|
|
691
|
-
throw new Error(`Child job not found: ${String(childJobId)}`);
|
|
694
|
+
let childJob = await this.getJob(childJobId);
|
|
692
695
|
const parentJob = await this.getJob(parentJobId);
|
|
696
|
+
if (!childJob) {
|
|
697
|
+
if (canAcceptRemovedFlowChild(parentJob, childJobId, this.depCompletions))
|
|
698
|
+
return;
|
|
699
|
+
throw new Error(`Child job not found: ${String(childJobId)}`);
|
|
700
|
+
}
|
|
693
701
|
if (!parentJob)
|
|
694
702
|
throw new Error(`Parent job not found: ${String(parentJobId)}`);
|
|
695
|
-
|
|
696
|
-
if (
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
703
|
+
assertFlowParentOwnership(childJob, parentJobId);
|
|
704
|
+
if (isDeclaredFlowChild(parentJob, childJobId)) {
|
|
705
|
+
childJob = await backpatchDeclaredFlowChild(childJob, parentJob, {
|
|
706
|
+
storage: this.storage,
|
|
707
|
+
customIdLock: this.customIdLock,
|
|
708
|
+
shards: this.shards,
|
|
709
|
+
shardLocks: this.shardLocks,
|
|
710
|
+
processingShards: this.processingShards,
|
|
711
|
+
processingLocks: this.processingLocks,
|
|
712
|
+
jobIndex: this.jobIndex,
|
|
713
|
+
completedJobsData: this.completedJobsData,
|
|
714
|
+
depCompletions: this.depCompletions,
|
|
715
|
+
});
|
|
702
716
|
}
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
const processingIndexes = childLocation?.type === 'processing' ? [processingShardIndex(childJobId)] : [];
|
|
708
|
-
const guards = [await this.customIdLock.acquireWrite()];
|
|
709
|
-
try {
|
|
710
|
-
for (const index of shardIndexes)
|
|
711
|
-
guards.push(await this.shardLocks[index].acquireWrite());
|
|
712
|
-
for (const index of processingIndexes) {
|
|
713
|
-
guards.push(await this.processingLocks[index].acquireWrite());
|
|
714
|
-
}
|
|
715
|
-
if (this.jobIndex.get(parentJobId)?.type !== 'queue') {
|
|
716
|
-
throw new Error(`Parent job ${String(parentJobId)} changed state while linking`);
|
|
717
|
+
else {
|
|
718
|
+
const parentLocation = this.jobIndex.get(parentJobId);
|
|
719
|
+
if (parentLocation?.type !== 'queue') {
|
|
720
|
+
throw new Error(`Parent job ${String(parentJobId)} is not linkable`);
|
|
717
721
|
}
|
|
718
|
-
const
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
const
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
722
|
+
const childLocation = this.jobIndex.get(childJobId);
|
|
723
|
+
const shardIndexes = [
|
|
724
|
+
...new Set([shardIndex(childJob.queue), shardIndex(parentJob.queue)]),
|
|
725
|
+
].sort((a, b) => a - b);
|
|
726
|
+
const processingIndexes = childLocation?.type === 'processing' ? [processingShardIndex(childJobId)] : [];
|
|
727
|
+
const guards = [await this.customIdLock.acquireWrite()];
|
|
728
|
+
try {
|
|
729
|
+
for (const index of shardIndexes)
|
|
730
|
+
guards.push(await this.shardLocks[index].acquireWrite());
|
|
731
|
+
for (const index of processingIndexes) {
|
|
732
|
+
guards.push(await this.processingLocks[index].acquireWrite());
|
|
733
|
+
}
|
|
734
|
+
if (this.jobIndex.get(parentJobId)?.type !== 'queue') {
|
|
735
|
+
throw new Error(`Parent job ${String(parentJobId)} changed state while linking`);
|
|
736
|
+
}
|
|
737
|
+
const childData = {
|
|
738
|
+
...childJob.data,
|
|
739
|
+
__parentId: String(parentJobId),
|
|
740
|
+
__parentQueue: parentJob.queue,
|
|
741
|
+
};
|
|
742
|
+
const childrenIds = [...parentJob.childrenIds, childJobId];
|
|
743
|
+
const dependsOn = parentJob.dependsOn.includes(childJobId)
|
|
744
|
+
? [...parentJob.dependsOn]
|
|
745
|
+
: [...parentJob.dependsOn, childJobId];
|
|
746
|
+
const parentData = {
|
|
747
|
+
...parentJob.data,
|
|
748
|
+
__childrenIds: childrenIds.map(String),
|
|
749
|
+
};
|
|
750
|
+
const linkedChild = { ...childJob, data: childData, parentId: parentJobId };
|
|
751
|
+
const linkedParent = { ...parentJob, data: parentData, childrenIds, dependsOn };
|
|
752
|
+
const childFinished = this.completedJobs.has(childJobId) || this.depCompletions.has(childJobId);
|
|
753
|
+
const parentState = childFinished
|
|
754
|
+
? parentJob.runAt > Date.now()
|
|
755
|
+
? 'delayed'
|
|
756
|
+
: parentJob.priority > 0
|
|
757
|
+
? 'prioritized'
|
|
758
|
+
: 'waiting'
|
|
759
|
+
: 'waiting-children';
|
|
760
|
+
this.storage?.updateFlowLink(linkedChild, linkedParent, parentState);
|
|
761
|
+
childJob.parentId = parentJobId;
|
|
762
|
+
childJob.data = childData;
|
|
763
|
+
parentJob.childrenIds = childrenIds;
|
|
764
|
+
parentJob.dependsOn = dependsOn;
|
|
765
|
+
parentJob.data = parentData;
|
|
766
|
+
if (!childFinished) {
|
|
767
|
+
const shard = this.shards[shardIndex(parentJob.queue)];
|
|
768
|
+
if (!shard.waitingDeps.has(parentJobId)) {
|
|
769
|
+
const removed = shard.getQueue(parentJob.queue).remove(parentJobId);
|
|
770
|
+
if (removed)
|
|
771
|
+
shard.decrementQueued(parentJobId);
|
|
772
|
+
shard.waitingDeps.set(parentJobId, parentJob);
|
|
773
|
+
}
|
|
774
|
+
shard.registerDependencies(parentJobId, [childJobId]);
|
|
775
|
+
this.dependencyResults.registerConsumer(parentJobId, dependsOn);
|
|
756
776
|
}
|
|
757
|
-
shard.registerDependencies(parentJobId, [childJobId]);
|
|
758
|
-
this.dependencyResults.registerConsumer(parentJobId, dependsOn);
|
|
759
777
|
}
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
778
|
+
finally {
|
|
779
|
+
for (let index = guards.length - 1; index >= 0; index--)
|
|
780
|
+
guards[index].release();
|
|
781
|
+
}
|
|
764
782
|
}
|
|
765
783
|
// Handle race condition: child may have already terminally failed
|
|
766
784
|
// before parent linkage was established (parentId was 'pending').
|
|
767
785
|
// If so, propagate failParentOnFailure now with the real parent ID.
|
|
768
786
|
const childLoc = this.jobIndex.get(childJobId);
|
|
787
|
+
const childFailureError = childLoc?.type === 'dlq'
|
|
788
|
+
? (flowChildFailureError(childJob, this.shards) ?? 'Child job failed')
|
|
789
|
+
: 'Child job failed';
|
|
769
790
|
if (childLoc?.type === 'dlq' && childJob.failParentOnFailure) {
|
|
770
|
-
await this.moveParentToFailed(parentJobId, childJob,
|
|
791
|
+
await this.moveParentToFailed(parentJobId, childJob, childFailureError);
|
|
771
792
|
}
|
|
772
793
|
if (childLoc?.type === 'dlq' &&
|
|
773
794
|
(childJob.removeDependencyOnFailure ||
|
|
774
795
|
childJob.ignoreDependencyOnFailure ||
|
|
775
796
|
childJob.continueParentOnFailure)) {
|
|
776
|
-
await this.onChildDependencyOption(childJob,
|
|
797
|
+
await this.onChildDependencyOption(childJob, childFailureError);
|
|
777
798
|
}
|
|
778
799
|
}
|
|
779
800
|
getJobByCustomId(customId) {
|
|
@@ -869,6 +890,12 @@ export class QueueManager {
|
|
|
869
890
|
}
|
|
870
891
|
for (const cid of customIdsToDelete)
|
|
871
892
|
this.customIdMap.delete(cid);
|
|
893
|
+
// removeOnComplete jobs have no jobIndex entry, but their bounded durable
|
|
894
|
+
// dependency proofs still belong to the obliterated queue.
|
|
895
|
+
const removedCompletions = this.storage?.deleteDependencyCompletionsForQueue(queue) ?? [];
|
|
896
|
+
for (const jobId of removedCompletions)
|
|
897
|
+
this.depCompletions.delete(jobId);
|
|
898
|
+
this.reconcileCompletionPins();
|
|
872
899
|
// Per-queue cumulative counters are keyed by queue name and never expire on
|
|
873
900
|
// their own; obliterate is the documented way to reclaim ALL state for a
|
|
874
901
|
// queue, so drop its metrics entry too (prevents unbounded growth for
|
|
@@ -900,6 +927,22 @@ export class QueueManager {
|
|
|
900
927
|
unregisterQueueName(queue) {
|
|
901
928
|
this.queueNamesCache.delete(queue);
|
|
902
929
|
}
|
|
930
|
+
releaseCompletionPins(dependencyIds) {
|
|
931
|
+
releaseDependencyCompletionPins(dependencyIds, {
|
|
932
|
+
storage: this.storage,
|
|
933
|
+
shards: this.shards,
|
|
934
|
+
depCompletions: this.depCompletions,
|
|
935
|
+
maxDependencyCompletions: this.config.maxCompletedJobs,
|
|
936
|
+
});
|
|
937
|
+
}
|
|
938
|
+
reconcileCompletionPins() {
|
|
939
|
+
reconcileDependencyCompletionPins({
|
|
940
|
+
storage: this.storage,
|
|
941
|
+
shards: this.shards,
|
|
942
|
+
depCompletions: this.depCompletions,
|
|
943
|
+
maxDependencyCompletions: this.config.maxCompletedJobs,
|
|
944
|
+
});
|
|
945
|
+
}
|
|
903
946
|
clean(queue, graceMs, state, limit) {
|
|
904
947
|
return queueControl.cleanQueue(queue, graceMs, this.contextFactory.getQueueControlContext(), state, limit);
|
|
905
948
|
}
|
|
@@ -1337,6 +1380,7 @@ export class QueueManager {
|
|
|
1337
1380
|
if (parentLoc.type !== 'queue')
|
|
1338
1381
|
return;
|
|
1339
1382
|
const idx = shardIndex(parentJob.queue);
|
|
1383
|
+
let releasedDependencies = [];
|
|
1340
1384
|
await withWriteLock(this.shardLocks[idx], () => {
|
|
1341
1385
|
// Re-check inside lock to prevent duplicate DLQ entries (TOCTOU guard)
|
|
1342
1386
|
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
@@ -1344,6 +1388,7 @@ export class QueueManager {
|
|
|
1344
1388
|
const shard = this.shards[idx];
|
|
1345
1389
|
// Remove from waitingDeps if present
|
|
1346
1390
|
if (shard.waitingDeps.has(parentId)) {
|
|
1391
|
+
releasedDependencies = [...parentJob.dependsOn];
|
|
1347
1392
|
shard.waitingDeps.delete(parentId);
|
|
1348
1393
|
shard.unregisterDependencies(parentId, parentJob.dependsOn);
|
|
1349
1394
|
}
|
|
@@ -1365,6 +1410,7 @@ export class QueueManager {
|
|
|
1365
1410
|
this.storage?.deleteJob(parentId);
|
|
1366
1411
|
this.storage?.deleteFlowFailure(parentId, childJob.id);
|
|
1367
1412
|
});
|
|
1413
|
+
this.releaseCompletionPins(releasedDependencies);
|
|
1368
1414
|
// Parent reached a terminal (DLQ) state — release its flow-failure tracking.
|
|
1369
1415
|
this.failedChildrenValues.delete(parentId);
|
|
1370
1416
|
this.ignoredChildrenFailures.delete(parentId);
|
|
@@ -1419,11 +1465,13 @@ export class QueueManager {
|
|
|
1419
1465
|
existing[childKey] = error ?? 'unknown error';
|
|
1420
1466
|
this.failedChildrenValues.set(parentId, existing);
|
|
1421
1467
|
const idx = shardIndex(parentJob.queue);
|
|
1468
|
+
let releasedDependencies = [];
|
|
1422
1469
|
await withWriteLock(this.shardLocks[idx], () => {
|
|
1423
1470
|
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
1424
1471
|
return;
|
|
1425
1472
|
const shard = this.shards[idx];
|
|
1426
1473
|
const dependencies = [...parentJob.dependsOn];
|
|
1474
|
+
releasedDependencies = dependencies;
|
|
1427
1475
|
shard.unregisterDependencies(parentId, dependencies);
|
|
1428
1476
|
for (const dependency of dependencies) {
|
|
1429
1477
|
this.dependencyResults.releaseDependency(parentId, dependency);
|
|
@@ -1431,6 +1479,7 @@ export class QueueManager {
|
|
|
1431
1479
|
parentJob.dependsOn = [];
|
|
1432
1480
|
this.storage?.updateFlowParentResolution(parentJob);
|
|
1433
1481
|
});
|
|
1482
|
+
this.releaseCompletionPins(releasedDependencies);
|
|
1434
1483
|
await this.promoteParentAfterChildFailure(parentId, parentJob, idx);
|
|
1435
1484
|
}
|
|
1436
1485
|
/**
|
|
@@ -1445,6 +1494,7 @@ export class QueueManager {
|
|
|
1445
1494
|
timer.unref?.();
|
|
1446
1495
|
});
|
|
1447
1496
|
let promoted = false;
|
|
1497
|
+
let releasedDependencies = [];
|
|
1448
1498
|
await withWriteLock(this.shardLocks[idx], () => {
|
|
1449
1499
|
// TOCTOU guard
|
|
1450
1500
|
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
@@ -1452,6 +1502,7 @@ export class QueueManager {
|
|
|
1452
1502
|
const shard = this.shards[idx];
|
|
1453
1503
|
// Remove from waitingDeps
|
|
1454
1504
|
if (shard.waitingDeps.has(parentId)) {
|
|
1505
|
+
releasedDependencies = [...parentJob.dependsOn];
|
|
1455
1506
|
shard.waitingDeps.delete(parentId);
|
|
1456
1507
|
shard.unregisterDependencies(parentId, parentJob.dependsOn);
|
|
1457
1508
|
}
|
|
@@ -1472,6 +1523,7 @@ export class QueueManager {
|
|
|
1472
1523
|
promoted = true;
|
|
1473
1524
|
}
|
|
1474
1525
|
});
|
|
1526
|
+
this.releaseCompletionPins(releasedDependencies);
|
|
1475
1527
|
if (promoted) {
|
|
1476
1528
|
this.eventsManager.broadcast({
|
|
1477
1529
|
eventType: 'waiting',
|
|
@@ -1507,6 +1559,7 @@ export class QueueManager {
|
|
|
1507
1559
|
// Remove the failed child from the parent's pending deps synchronously so
|
|
1508
1560
|
// dependency tracking stays consistent; defer only the promotion decision.
|
|
1509
1561
|
let readyToPromote = false;
|
|
1562
|
+
let releasedDependency = null;
|
|
1510
1563
|
await withWriteLock(this.shardLocks[idx], () => {
|
|
1511
1564
|
if (this.jobIndex.get(parentId)?.type !== 'queue')
|
|
1512
1565
|
return;
|
|
@@ -1519,6 +1572,7 @@ export class QueueManager {
|
|
|
1519
1572
|
if (depIndex !== -1) {
|
|
1520
1573
|
parentJob.dependsOn.splice(depIndex, 1);
|
|
1521
1574
|
shard.unregisterDependencies(parentId, [childJob.id]);
|
|
1575
|
+
releasedDependency = childJob.id;
|
|
1522
1576
|
this.dependencyResults.releaseDependency(parentId, childJob.id);
|
|
1523
1577
|
this.storage?.updateFlowParentResolution(parentJob);
|
|
1524
1578
|
}
|
|
@@ -1527,6 +1581,8 @@ export class QueueManager {
|
|
|
1527
1581
|
parentJob.dependsOn.length === 0 ||
|
|
1528
1582
|
parentJob.dependsOn.every((dep) => this.completedJobs.has(dep));
|
|
1529
1583
|
});
|
|
1584
|
+
if (releasedDependency)
|
|
1585
|
+
this.releaseCompletionPins([releasedDependency]);
|
|
1530
1586
|
if (readyToPromote) {
|
|
1531
1587
|
await this.promoteParentAfterChildFailure(parentId, parentJob, idx);
|
|
1532
1588
|
}
|
|
@@ -1572,6 +1628,7 @@ export class QueueManager {
|
|
|
1572
1628
|
const guards = [];
|
|
1573
1629
|
let removed = false;
|
|
1574
1630
|
let promoted = false;
|
|
1631
|
+
let releasedDependencies = [];
|
|
1575
1632
|
try {
|
|
1576
1633
|
for (const index of shardIndexes)
|
|
1577
1634
|
guards.push(await this.shardLocks[index].acquireWrite());
|
|
@@ -1590,6 +1647,7 @@ export class QueueManager {
|
|
|
1590
1647
|
!this.completedJobs.has(dependency) &&
|
|
1591
1648
|
!this.depCompletions.has(dependency));
|
|
1592
1649
|
const released = parentJob.dependsOn.filter((dependency) => !unresolved.includes(dependency));
|
|
1650
|
+
releasedDependencies = released;
|
|
1593
1651
|
const childrenIds = parentJob.childrenIds.filter((childId) => childId !== childJobId);
|
|
1594
1652
|
const childData = { ...childJob.data };
|
|
1595
1653
|
delete childData.__parentId;
|
|
@@ -1642,6 +1700,7 @@ export class QueueManager {
|
|
|
1642
1700
|
for (let index = guards.length - 1; index >= 0; index--)
|
|
1643
1701
|
guards[index].release();
|
|
1644
1702
|
}
|
|
1703
|
+
this.releaseCompletionPins(releasedDependencies);
|
|
1645
1704
|
if (promoted) {
|
|
1646
1705
|
this.eventsManager.broadcast({
|
|
1647
1706
|
eventType: 'waiting',
|