bunqueue 2.8.35 → 2.8.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/application/backgroundTasks.js +28 -5
- package/dist/application/contextFactory.js +5 -0
- package/dist/application/dlqManager.d.ts +6 -0
- package/dist/application/dlqManager.js +35 -14
- package/dist/application/lockManager.js +22 -8
- package/dist/application/operations/customId.d.ts +31 -0
- package/dist/application/operations/customId.js +59 -0
- package/dist/application/operations/jobClaim.d.ts +17 -0
- package/dist/application/operations/jobClaim.js +20 -0
- package/dist/application/operations/jobManagement.d.ts +4 -5
- package/dist/application/operations/jobManagement.js +15 -116
- package/dist/application/operations/jobMoveOperations.d.ts +9 -0
- package/dist/application/operations/jobMoveOperations.js +94 -0
- package/dist/application/operations/jobStateTransitions.js +3 -0
- package/dist/application/operations/pull.js +5 -0
- package/dist/application/operations/push.js +8 -103
- package/dist/application/operations/pushInsert.d.ts +12 -0
- package/dist/application/operations/pushInsert.js +25 -0
- package/dist/application/operations/pushLocks.d.ts +14 -0
- package/dist/application/operations/pushLocks.js +43 -0
- package/dist/application/operations/queueControl.d.ts +1 -1
- package/dist/application/operations/queueControl.js +1 -1
- package/dist/application/queueManager.d.ts +1 -1
- package/dist/application/queueManager.js +17 -6
- package/dist/application/stallDetection.js +17 -8
- package/dist/client/queue/helpers.js +2 -0
- package/dist/client/queue/operations/counts.d.ts +1 -0
- package/dist/client/queue/operations/counts.js +3 -0
- package/dist/domain/queue/dependencyTracker.d.ts +5 -0
- package/dist/domain/queue/dependencyTracker.js +20 -0
- package/dist/domain/queue/shard.d.ts +1 -1
- package/dist/domain/queue/shard.js +6 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +25 -2
- package/dist/infrastructure/persistence/sqlite.d.ts +11 -0
- package/dist/infrastructure/persistence/sqlite.js +59 -5
- package/dist/infrastructure/persistence/sqliteBatch.js +8 -6
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +9 -1
- package/dist/infrastructure/persistence/statements.d.ts +6 -1
- package/dist/infrastructure/persistence/statements.js +7 -5
- package/package.json +4 -1
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job move operations that claim active jobs for delayed or DLQ state.
|
|
3
|
+
*/
|
|
4
|
+
import { processingShardIndex, shardIndex } from '../../shared/hash';
|
|
5
|
+
import { withWriteLock } from '../../shared/lock';
|
|
6
|
+
import { releaseClaimedJobOwnership } from './jobClaim';
|
|
7
|
+
/** Move active job back to delayed. */
|
|
8
|
+
export async function moveJobToDelayed(jobId, delay, ctx) {
|
|
9
|
+
const location = ctx.jobIndex.get(jobId);
|
|
10
|
+
if (location?.type !== 'processing')
|
|
11
|
+
return false;
|
|
12
|
+
const procIdx = processingShardIndex(jobId);
|
|
13
|
+
const job = await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
14
|
+
const claimed = ctx.processingShards[procIdx].get(jobId);
|
|
15
|
+
if (claimed) {
|
|
16
|
+
ctx.processingShards[procIdx].delete(jobId);
|
|
17
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
18
|
+
}
|
|
19
|
+
return claimed;
|
|
20
|
+
});
|
|
21
|
+
if (!job)
|
|
22
|
+
return false;
|
|
23
|
+
const now = Date.now();
|
|
24
|
+
job.runAt = now + delay;
|
|
25
|
+
job.startedAt = null;
|
|
26
|
+
const idx = shardIndex(job.queue);
|
|
27
|
+
const queueName = job.queue;
|
|
28
|
+
await withWriteLock(ctx.shardLocks[idx], () => {
|
|
29
|
+
const shard = ctx.shards[idx];
|
|
30
|
+
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
|
|
31
|
+
shard.getQueue(job.queue).push(job);
|
|
32
|
+
const isDelayed = job.runAt > now;
|
|
33
|
+
shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
34
|
+
ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
|
|
35
|
+
shard.notify(job.queue);
|
|
36
|
+
});
|
|
37
|
+
ctx.storage?.updateRunAt(jobId, job.runAt);
|
|
38
|
+
ctx.eventsManager.broadcast({
|
|
39
|
+
eventType: "delayed" /* EventType.Delayed */,
|
|
40
|
+
jobId,
|
|
41
|
+
queue: queueName,
|
|
42
|
+
timestamp: Date.now(),
|
|
43
|
+
delay,
|
|
44
|
+
});
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
/** Discard a waiting or active job to the DLQ. */
|
|
48
|
+
export async function discardJob(jobId, ctx) {
|
|
49
|
+
const location = ctx.jobIndex.get(jobId);
|
|
50
|
+
if (!location)
|
|
51
|
+
return false;
|
|
52
|
+
let job = null;
|
|
53
|
+
if (location.type === 'queue') {
|
|
54
|
+
job = await withWriteLock(ctx.shardLocks[location.shardIdx], () => {
|
|
55
|
+
const shard = ctx.shards[location.shardIdx];
|
|
56
|
+
const removed = shard.getQueue(location.queueName).remove(jobId);
|
|
57
|
+
if (removed)
|
|
58
|
+
shard.decrementQueued(jobId);
|
|
59
|
+
return removed;
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
else if (location.type === 'processing') {
|
|
63
|
+
const procIdx = processingShardIndex(jobId);
|
|
64
|
+
job = await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
65
|
+
const claimed = ctx.processingShards[procIdx].get(jobId) ?? null;
|
|
66
|
+
if (claimed) {
|
|
67
|
+
ctx.processingShards[procIdx].delete(jobId);
|
|
68
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
69
|
+
}
|
|
70
|
+
return claimed;
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
if (!job)
|
|
74
|
+
return false;
|
|
75
|
+
const validJob = job;
|
|
76
|
+
const idx = shardIndex(validJob.queue);
|
|
77
|
+
const fromProcessing = location.type === 'processing';
|
|
78
|
+
const entry = await withWriteLock(ctx.shardLocks[idx], () => {
|
|
79
|
+
const shard = ctx.shards[idx];
|
|
80
|
+
if (fromProcessing) {
|
|
81
|
+
shard.releaseJobResources(validJob.queue, validJob.uniqueKey, validJob.groupId);
|
|
82
|
+
shard.notify(validJob.queue);
|
|
83
|
+
}
|
|
84
|
+
else if (validJob.uniqueKey) {
|
|
85
|
+
shard.releaseUniqueKey(validJob.queue, validJob.uniqueKey);
|
|
86
|
+
}
|
|
87
|
+
const dlqEntry = shard.addToDlq(validJob);
|
|
88
|
+
ctx.jobIndex.set(jobId, { type: 'dlq', queueName: validJob.queue });
|
|
89
|
+
return dlqEntry;
|
|
90
|
+
});
|
|
91
|
+
ctx.storage?.saveDlqEntry(entry);
|
|
92
|
+
ctx.storage?.deleteJob(jobId);
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { shardIndex, processingShardIndex } from '../../shared/hash';
|
|
9
9
|
import { withWriteLock } from '../../shared/lock';
|
|
10
|
+
import { releaseClaimedJobOwnership } from './jobClaim';
|
|
10
11
|
/** Move active job back to waiting */
|
|
11
12
|
export async function moveActiveToWait(jobId, ctx) {
|
|
12
13
|
const location = ctx.jobIndex.get(jobId);
|
|
@@ -17,6 +18,7 @@ export async function moveActiveToWait(jobId, ctx) {
|
|
|
17
18
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
18
19
|
if (job) {
|
|
19
20
|
ctx.processingShards[procIdx].delete(jobId);
|
|
21
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
20
22
|
}
|
|
21
23
|
return job;
|
|
22
24
|
});
|
|
@@ -73,6 +75,7 @@ export async function moveToWaitingChildren(jobId, ctx) {
|
|
|
73
75
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
74
76
|
if (job) {
|
|
75
77
|
ctx.processingShards[procIdx].delete(jobId);
|
|
78
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
76
79
|
}
|
|
77
80
|
return job;
|
|
78
81
|
});
|
|
@@ -35,6 +35,11 @@ function tryDequeueNextJob(shard, queue, now, ctx) {
|
|
|
35
35
|
// Expired entries are not parked: they are genuinely removed from the
|
|
36
36
|
// logical queue and therefore update counters/index exactly once.
|
|
37
37
|
if (isExpired(job, now)) {
|
|
38
|
+
// Delete persistence before mutating the in-memory indexes. deleteJob
|
|
39
|
+
// also removes a pending buffered INSERT, so the expired job cannot be
|
|
40
|
+
// flushed after this pull or resurrect during crash recovery. This is
|
|
41
|
+
// synchronous and therefore keeps the shard critical section await-free.
|
|
42
|
+
ctx.storage?.deleteJob(job.id);
|
|
38
43
|
q.pop();
|
|
39
44
|
shard.decrementQueued(job.id);
|
|
40
45
|
ctx.jobIndex.delete(job.id);
|
|
@@ -2,80 +2,13 @@
|
|
|
2
2
|
* Push Operations
|
|
3
3
|
* Job push and batch push logic
|
|
4
4
|
*/
|
|
5
|
-
import { createJob
|
|
6
|
-
import { withWriteLock } from '../../shared/lock';
|
|
5
|
+
import { createJob } from '../../domain/types/job';
|
|
7
6
|
import { shardIndex } from '../../shared/hash';
|
|
8
7
|
import { latencyTracker } from '../latencyTracker';
|
|
9
8
|
import { throughputTracker } from '../throughputTracker';
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
*/
|
|
14
|
-
function handleCustomId(input, shard, ctx) {
|
|
15
|
-
if (!input.customId) {
|
|
16
|
-
return { skip: false, id: generateJobId() };
|
|
17
|
-
}
|
|
18
|
-
const id = jobId(input.customId);
|
|
19
|
-
const existing = ctx.customIdMap.get(input.customId);
|
|
20
|
-
// Re-adding an existing custom id while its job is UNFINISHED is an idempotent
|
|
21
|
-
// no-op (BullMQ parity): return the existing job, never a second insert of the
|
|
22
|
-
// same deterministic id (which would collide on the jobs.id PRIMARY KEY). Three
|
|
23
|
-
// unfinished states are covered:
|
|
24
|
-
// • waiting/delayed/prioritized — live in the priority queue → return the job.
|
|
25
|
-
// • waiting-children — tracked under 'queue' but held in shard.waitingDeps (its
|
|
26
|
-
// row is already persisted) → idempotent skip via the existing id.
|
|
27
|
-
// • active (processing) — popped from the queue, still running on a worker; its
|
|
28
|
-
// row survives on disk → idempotent skip via the existing id.
|
|
29
|
-
// PushContext has no processingShards, so for the latter two we cannot fetch the
|
|
30
|
-
// live Job here; pushJob rebuilds a placeholder { ...job, id } after createJob,
|
|
31
|
-
// exactly like handleDeduplication's active path. Completed (#92) and DLQ jobs
|
|
32
|
-
// are terminal and fall through to the reuse path below.
|
|
33
|
-
if (existing && !ctx.completedJobs.has(id)) {
|
|
34
|
-
const location = ctx.jobIndex.get(existing);
|
|
35
|
-
if (location?.type === 'queue') {
|
|
36
|
-
const existingJob = shard.getQueue(location.queueName).find(existing);
|
|
37
|
-
if (existingJob) {
|
|
38
|
-
return { skip: true, existingJob };
|
|
39
|
-
}
|
|
40
|
-
if (shard.waitingDeps.has(existing)) {
|
|
41
|
-
return { skip: true, existingId: id };
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
else if (location?.type === 'processing') {
|
|
45
|
-
return { skip: true, existingId: id };
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
// Reuse path: the id is free in the queue (no mapping, or the prior job is
|
|
49
|
-
// processing/completed). If the prior job COMPLETED, its row survives on disk
|
|
50
|
-
// (markCompleted does an UPDATE, not a DELETE) and it is still in completedJobs.
|
|
51
|
-
// Reusing the same deterministic id would then (a) make getJobState return
|
|
52
|
-
// 'completed' for the brand-new job and (b) collide on the `jobs.id` PRIMARY KEY
|
|
53
|
-
// at flush time. Evict the stale completed job so the reused id starts fresh as
|
|
54
|
-
// 'waiting' (#92). Checked regardless of customIdMap state — the mapping may have
|
|
55
|
-
// been cleared on completion, which would otherwise skip this path entirely.
|
|
56
|
-
if (ctx.completedJobs.has(id)) {
|
|
57
|
-
ctx.completedJobs.delete(id);
|
|
58
|
-
ctx.completedJobsData.delete(id);
|
|
59
|
-
ctx.jobResults.delete(id);
|
|
60
|
-
ctx.jobIndex.delete(id);
|
|
61
|
-
ctx.storage?.deleteJob(id); // removes the surviving row + result + any buffered insert
|
|
62
|
-
}
|
|
63
|
-
// NOTE on ORPHAN rows: a durable jobs row can outlive its in-memory tracking when
|
|
64
|
-
// obliterate() (fire-and-forget void over TCP) or a buffer flush races an in-flight
|
|
65
|
-
// durable insert. Reaching the reuse path with such a stale row no longer throws —
|
|
66
|
-
// the insert statements use ON CONFLICT(id) DO UPDATE (upsert), so the orphan is
|
|
67
|
-
// overwritten in place at insert time with ZERO per-push cost. We deliberately do
|
|
68
|
-
// NOT pre-delete here: deleteJob() is a synchronous DELETE + O(buffer) scan inside
|
|
69
|
-
// the shard lock and would halve customId push/addBulk throughput.
|
|
70
|
-
// A recycled custom id may carry a stale timeout marker from a prior job (which
|
|
71
|
-
// may have DLQ'd, so it is NOT in completedJobs above). Clear it so the new
|
|
72
|
-
// job's stall-retry recovery is not wrongly discarded — otherwise the
|
|
73
|
-
// timeout-resurrection guard would reintroduce #33/#75 duplicate execution for
|
|
74
|
-
// reused ids.
|
|
75
|
-
ctx.timedOutJobs?.delete(id);
|
|
76
|
-
ctx.customIdMap.set(input.customId, id);
|
|
77
|
-
return { skip: false, id };
|
|
78
|
-
}
|
|
9
|
+
import { handleCustomId } from './customId';
|
|
10
|
+
import { insertJobToShard } from './pushInsert';
|
|
11
|
+
import { withPushWriteLocks } from './pushLocks';
|
|
79
12
|
/**
|
|
80
13
|
* Handle unique key deduplication
|
|
81
14
|
* Returns existing job ID if duplicate found and should skip, or allows insert
|
|
@@ -143,34 +76,6 @@ function handleDeduplication(job, input, queue, shard, ctx) {
|
|
|
143
76
|
shard.registerUniqueKeyWithTtl(queue, job.uniqueKey, job.id, input.dedup?.ttl);
|
|
144
77
|
return { skip: false };
|
|
145
78
|
}
|
|
146
|
-
/**
|
|
147
|
-
* Insert job into shard (queue or waitingDeps)
|
|
148
|
-
*/
|
|
149
|
-
function insertJobToShard(job, queue, shard, shardIdx, ctx) {
|
|
150
|
-
const hasDeps = job.dependsOn.length > 0;
|
|
151
|
-
const needsWaiting = hasDeps &&
|
|
152
|
-
!job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
|
|
153
|
-
const now = Date.now();
|
|
154
|
-
if (needsWaiting) {
|
|
155
|
-
shard.waitingDeps.set(job.id, job);
|
|
156
|
-
shard.registerDependencies(job.id, job.dependsOn);
|
|
157
|
-
job.timeline.push({ state: 'waiting-children', timestamp: now });
|
|
158
|
-
ctx.dashboardEmit?.('job:waiting-children', {
|
|
159
|
-
jobId: String(job.id),
|
|
160
|
-
queue,
|
|
161
|
-
dependsOn: job.dependsOn.map(String),
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
else {
|
|
165
|
-
shard.getQueue(queue).push(job);
|
|
166
|
-
const isDelayed = job.runAt > now;
|
|
167
|
-
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
168
|
-
// Timeline: initial state based on scheduling and priority
|
|
169
|
-
const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
|
|
170
|
-
job.timeline.push({ state, timestamp: now });
|
|
171
|
-
}
|
|
172
|
-
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
|
|
173
|
-
}
|
|
174
79
|
/**
|
|
175
80
|
* Push a single job to queue
|
|
176
81
|
* NOTE: customId check happens INSIDE lock to prevent race conditions
|
|
@@ -180,10 +85,10 @@ export async function pushJob(queue, input, ctx) {
|
|
|
180
85
|
const idx = shardIndex(queue);
|
|
181
86
|
const now = Date.now();
|
|
182
87
|
let result;
|
|
183
|
-
await
|
|
88
|
+
await withPushWriteLocks(queue, [input], ctx, (lockedShardIndexes) => {
|
|
184
89
|
const shard = ctx.shards[idx];
|
|
185
90
|
// Check custom ID idempotency INSIDE lock to prevent race conditions
|
|
186
|
-
const customIdResult = handleCustomId(input,
|
|
91
|
+
const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
|
|
187
92
|
if (customIdResult.skip) {
|
|
188
93
|
// Idempotent re-add: return the live queued job if we have it, otherwise
|
|
189
94
|
// (active / waiting-children) a placeholder carrying the existing id so the
|
|
@@ -247,11 +152,11 @@ export async function pushJobBatch(queue, inputs, ctx) {
|
|
|
247
152
|
// path), exactly like a single durable push — otherwise addBulk silently
|
|
248
153
|
// downgrades the documented "durable" guarantee.
|
|
249
154
|
const durableJobs = [];
|
|
250
|
-
await
|
|
155
|
+
await withPushWriteLocks(queue, inputs, ctx, (lockedShardIndexes) => {
|
|
251
156
|
const shard = ctx.shards[idx];
|
|
252
157
|
for (const input of inputs) {
|
|
253
158
|
// Check custom ID idempotency
|
|
254
|
-
const customIdResult = handleCustomId(input,
|
|
159
|
+
const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
|
|
255
160
|
if (customIdResult.skip) {
|
|
256
161
|
// Idempotent re-add (queued, active, or waiting-children) — no insert.
|
|
257
162
|
resultIds.push('existingJob' in customIdResult
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Shard } from '../../domain/queue/shard';
|
|
2
|
+
import type { Job, JobId } from '../../domain/types/job';
|
|
3
|
+
import type { JobLocation } from '../../domain/types/queue';
|
|
4
|
+
import type { SetLike } from '../../shared/lru';
|
|
5
|
+
export interface PushInsertContext {
|
|
6
|
+
completedJobs: SetLike<JobId>;
|
|
7
|
+
depCompletions?: SetLike<JobId>;
|
|
8
|
+
jobIndex: Map<JobId, JobLocation>;
|
|
9
|
+
dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
|
|
10
|
+
}
|
|
11
|
+
/** Insert a new job into the runnable heap or dependency wait set. */
|
|
12
|
+
export declare function insertJobToShard(job: Job, queue: string, shard: Shard, shardIdx: number, ctx: PushInsertContext): void;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Insert a new job into the runnable heap or dependency wait set. */
|
|
2
|
+
export function insertJobToShard(job, queue, shard, shardIdx, ctx) {
|
|
3
|
+
const hasDeps = job.dependsOn.length > 0;
|
|
4
|
+
const needsWaiting = hasDeps &&
|
|
5
|
+
!job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
|
|
6
|
+
const now = Date.now();
|
|
7
|
+
if (needsWaiting) {
|
|
8
|
+
shard.waitingDeps.set(job.id, job);
|
|
9
|
+
shard.registerDependencies(job.id, job.dependsOn);
|
|
10
|
+
job.timeline.push({ state: 'waiting-children', timestamp: now });
|
|
11
|
+
ctx.dashboardEmit?.('job:waiting-children', {
|
|
12
|
+
jobId: String(job.id),
|
|
13
|
+
queue,
|
|
14
|
+
dependsOn: job.dependsOn.map(String),
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
shard.getQueue(queue).push(job);
|
|
19
|
+
const isDelayed = job.runAt > now;
|
|
20
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
21
|
+
const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
|
|
22
|
+
job.timeline.push({ state, timestamp: now });
|
|
23
|
+
}
|
|
24
|
+
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
|
|
25
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { JobId, JobInput } from '../../domain/types/job';
|
|
2
|
+
import type { JobLocation } from '../../domain/types/queue';
|
|
3
|
+
import type { RWLock } from '../../shared/lock';
|
|
4
|
+
interface PushLockContext {
|
|
5
|
+
shardLocks: RWLock[];
|
|
6
|
+
jobIndex: Map<JobId, JobLocation>;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Lock the target shard plus any shard owning a terminal custom-ID generation.
|
|
10
|
+
* The required set is rechecked after acquisition because jobIndex may change
|
|
11
|
+
* while an earlier lock is awaited.
|
|
12
|
+
*/
|
|
13
|
+
export declare function withPushWriteLocks<T>(queue: string, inputs: readonly JobInput[], ctx: PushLockContext, fn: (lockedShardIndexes: ReadonlySet<number>) => T): Promise<T>;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { jobId } from '../../domain/types/job';
|
|
2
|
+
import { shardIndex } from '../../shared/hash';
|
|
3
|
+
function requiredShardIndexes(queue, inputs, jobIndex) {
|
|
4
|
+
const indexes = new Set([shardIndex(queue)]);
|
|
5
|
+
for (const input of inputs) {
|
|
6
|
+
if (!input.customId)
|
|
7
|
+
continue;
|
|
8
|
+
const location = jobIndex.get(jobId(input.customId));
|
|
9
|
+
if (location?.type === 'queue') {
|
|
10
|
+
indexes.add(location.shardIdx);
|
|
11
|
+
}
|
|
12
|
+
else if (location?.type === 'dlq') {
|
|
13
|
+
indexes.add(shardIndex(location.queueName));
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return [...indexes].sort((a, b) => a - b);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Lock the target shard plus any shard owning a terminal custom-ID generation.
|
|
20
|
+
* The required set is rechecked after acquisition because jobIndex may change
|
|
21
|
+
* while an earlier lock is awaited.
|
|
22
|
+
*/
|
|
23
|
+
export async function withPushWriteLocks(queue, inputs, ctx, fn) {
|
|
24
|
+
while (true) {
|
|
25
|
+
const indexes = requiredShardIndexes(queue, inputs, ctx.jobIndex);
|
|
26
|
+
const guards = [];
|
|
27
|
+
try {
|
|
28
|
+
for (const index of indexes) {
|
|
29
|
+
guards.push(await ctx.shardLocks[index].acquireWrite());
|
|
30
|
+
}
|
|
31
|
+
const locked = new Set(indexes);
|
|
32
|
+
const current = requiredShardIndexes(queue, inputs, ctx.jobIndex);
|
|
33
|
+
if (current.some((index) => !locked.has(index)))
|
|
34
|
+
continue;
|
|
35
|
+
return fn(locked);
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
for (let i = guards.length - 1; i >= 0; i--) {
|
|
39
|
+
guards[i].release();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -30,7 +30,7 @@ export declare function isQueuePaused(queue: string, ctx: QueueControlContext):
|
|
|
30
30
|
/** Drain all waiting jobs from queue */
|
|
31
31
|
export declare function drainQueue(queue: string, ctx: QueueControlContext): number;
|
|
32
32
|
/** Remove all queue data */
|
|
33
|
-
export declare function obliterateQueue(queue: string, ctx: QueueControlContext):
|
|
33
|
+
export declare function obliterateQueue(queue: string, ctx: QueueControlContext): JobId[];
|
|
34
34
|
/** List all queue names */
|
|
35
35
|
export declare function listAllQueues(ctx: QueueControlContext): string[];
|
|
36
36
|
/**
|
|
@@ -35,7 +35,7 @@ export function drainQueue(queue, ctx) {
|
|
|
35
35
|
/** Remove all queue data */
|
|
36
36
|
export function obliterateQueue(queue, ctx) {
|
|
37
37
|
const idx = shardIndex(queue);
|
|
38
|
-
ctx.shards[idx].obliterate(queue);
|
|
38
|
+
return ctx.shards[idx].obliterate(queue);
|
|
39
39
|
}
|
|
40
40
|
/** List all queue names */
|
|
41
41
|
export function listAllQueues(ctx) {
|
|
@@ -6,7 +6,7 @@ import type { Job, JobId, JobInput, JobLock, LockToken } from '../domain/types/j
|
|
|
6
6
|
import type { JobLocation, JobEvent } from '../domain/types/queue';
|
|
7
7
|
import type { CronJob, CronJobInput } from '../domain/types/cron';
|
|
8
8
|
import type { JobLogEntry, CreateWorkerOptions } from '../domain/types/worker';
|
|
9
|
-
import type
|
|
9
|
+
import { type StallConfig } from '../domain/types/stall';
|
|
10
10
|
import type { DlqConfig, DlqEntry, DlqFilter, DlqStats } from '../domain/types/dlq';
|
|
11
11
|
import { Shard } from '../domain/queue/shard';
|
|
12
12
|
import { WebhookManager } from './webhookManager';
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Core orchestrator for all queue operations
|
|
4
4
|
*/
|
|
5
5
|
import { DEFAULT_LOCK_TTL } from '../domain/types/job';
|
|
6
|
+
import { DEFAULT_STALL_CONFIG } from '../domain/types/stall';
|
|
6
7
|
import { Shard } from '../domain/queue/shard';
|
|
7
8
|
import { SqliteStorage } from '../infrastructure/persistence/sqlite';
|
|
8
9
|
import { CronScheduler } from '../infrastructure/scheduler/cronScheduler';
|
|
@@ -741,12 +742,12 @@ export class QueueManager {
|
|
|
741
742
|
return count;
|
|
742
743
|
}
|
|
743
744
|
obliterate(queue) {
|
|
744
|
-
queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
|
|
745
|
+
const shardJobs = queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
|
|
745
746
|
dlqOps.purgeDlqJobs(queue, this.contextFactory.getDlqContext());
|
|
746
|
-
// obliterateQueue()
|
|
747
|
-
// processingShards, plus completed/result/log/lock
|
|
748
|
-
//
|
|
749
|
-
const toDrop = new Set();
|
|
747
|
+
// obliterateQueue() returns every queued, DLQ, and dependency-gated job it
|
|
748
|
+
// removed. Active jobs in processingShards, plus completed/result/log/lock
|
|
749
|
+
// state in global indexes, still need to be discovered and purged here.
|
|
750
|
+
const toDrop = new Set(shardJobs);
|
|
750
751
|
for (const [jid, loc] of this.jobIndex) {
|
|
751
752
|
if (loc.type === 'processing') {
|
|
752
753
|
const job = this.processingShards[loc.shardIdx]?.get(jid);
|
|
@@ -885,11 +886,19 @@ export class QueueManager {
|
|
|
885
886
|
if (!this.storage)
|
|
886
887
|
return;
|
|
887
888
|
const state = this.shards[shardIndex(queue)].getState(queue);
|
|
889
|
+
const stallConfig = this.shards[shardIndex(queue)].getStallConfig(queue);
|
|
890
|
+
const hasCustomStallConfig = stallConfig.enabled !== DEFAULT_STALL_CONFIG.enabled ||
|
|
891
|
+
stallConfig.stallInterval !== DEFAULT_STALL_CONFIG.stallInterval ||
|
|
892
|
+
stallConfig.maxStalls !== DEFAULT_STALL_CONFIG.maxStalls ||
|
|
893
|
+
stallConfig.gracePeriod !== DEFAULT_STALL_CONFIG.gracePeriod;
|
|
888
894
|
// When control-state returns fully to default (not paused, no limits), drop
|
|
889
895
|
// the row instead of persisting an all-default placeholder. Keeps the table
|
|
890
896
|
// free of noise rows for ephemeral queues that only ever call resume/clear*,
|
|
891
897
|
// and recovers identically (absent row → default state).
|
|
892
|
-
if (!state.paused &&
|
|
898
|
+
if (!state.paused &&
|
|
899
|
+
state.rateLimit === null &&
|
|
900
|
+
state.concurrencyLimit === null &&
|
|
901
|
+
!hasCustomStallConfig) {
|
|
893
902
|
this.storage.deleteQueueState(queue);
|
|
894
903
|
return;
|
|
895
904
|
}
|
|
@@ -899,6 +908,7 @@ export class QueueManager {
|
|
|
899
908
|
concurrencyLimit: state.concurrencyLimit,
|
|
900
909
|
rateLimitDuration: state.rateLimitDuration,
|
|
901
910
|
rateLimitExpiresAt: state.rateLimitExpiresAt,
|
|
911
|
+
stallConfig,
|
|
902
912
|
});
|
|
903
913
|
}
|
|
904
914
|
/** Get rate limit and concurrency limit for a queue */
|
|
@@ -930,6 +940,7 @@ export class QueueManager {
|
|
|
930
940
|
// ============ Stall & DLQ Config ============
|
|
931
941
|
setStallConfig(queue, config) {
|
|
932
942
|
this.shards[shardIndex(queue)].setStallConfig(queue, config);
|
|
943
|
+
this.persistQueueState(queue);
|
|
933
944
|
}
|
|
934
945
|
getStallConfig(queue) {
|
|
935
946
|
return this.shards[shardIndex(queue)].getStallConfig(queue);
|
|
@@ -66,6 +66,7 @@ async function handleStalledJob(job, action, ctx) {
|
|
|
66
66
|
// Lock order: shardLocks BEFORE processingLocks (per lock hierarchy in CLAUDE.md)
|
|
67
67
|
// Broadcast events AFTER verifying job is still stalled to avoid false positives
|
|
68
68
|
let handled = false;
|
|
69
|
+
let handledAction = action;
|
|
69
70
|
await withWriteLock(ctx.shardLocks[idx], async () => {
|
|
70
71
|
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
71
72
|
// Verify job is still in processing (might have been handled already)
|
|
@@ -73,8 +74,10 @@ async function handleStalledJob(job, action, ctx) {
|
|
|
73
74
|
return; // Job already completed, don't broadcast stalled event
|
|
74
75
|
}
|
|
75
76
|
const shard = ctx.shards[idx];
|
|
76
|
-
|
|
77
|
-
|
|
77
|
+
const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
|
|
78
|
+
if (action === "move_to_dlq" /* StallAction.MoveToDlq */ || attemptsExhausted) {
|
|
79
|
+
handledAction = "move_to_dlq" /* StallAction.MoveToDlq */;
|
|
80
|
+
moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
|
|
78
81
|
}
|
|
79
82
|
else {
|
|
80
83
|
retryStalliedJob(job, ctx, shard, procIdx, idx);
|
|
@@ -87,23 +90,23 @@ async function handleStalledJob(job, action, ctx) {
|
|
|
87
90
|
ctx.dashboardEmit?.('job:stalled', {
|
|
88
91
|
jobId: String(job.id),
|
|
89
92
|
queue: job.queue,
|
|
90
|
-
stallCount: job.stallCount
|
|
91
|
-
action,
|
|
93
|
+
stallCount: job.stallCount,
|
|
94
|
+
action: handledAction,
|
|
92
95
|
});
|
|
93
96
|
ctx.eventsManager.broadcast({
|
|
94
97
|
eventType: "stalled" /* EventType.Stalled */,
|
|
95
98
|
queue: job.queue,
|
|
96
99
|
jobId: job.id,
|
|
97
100
|
timestamp: Date.now(),
|
|
98
|
-
data: { stallCount: job.stallCount
|
|
101
|
+
data: { stallCount: job.stallCount, action: handledAction },
|
|
99
102
|
});
|
|
100
103
|
void ctx.webhookManager.trigger('stalled', String(job.id), job.queue, {
|
|
101
|
-
data: { stallCount: job.stallCount
|
|
104
|
+
data: { stallCount: job.stallCount, action: handledAction },
|
|
102
105
|
});
|
|
103
106
|
}
|
|
104
107
|
}
|
|
105
108
|
/** Move stalled job to DLQ */
|
|
106
|
-
function moveStalliedJobToDlq(job, ctx, shard, procIdx,
|
|
109
|
+
function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
|
|
107
110
|
ctx.processingShards[procIdx].delete(job.id);
|
|
108
111
|
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
|
|
109
112
|
// Discard cron jobs with preventOverlap instead of sending to DLQ (#73).
|
|
@@ -112,7 +115,13 @@ function moveStalliedJobToDlq(job, ctx, shard, procIdx, _idx) {
|
|
|
112
115
|
ctx.storage?.deleteJob(job.id);
|
|
113
116
|
return;
|
|
114
117
|
}
|
|
115
|
-
|
|
118
|
+
incrementStallCount(job);
|
|
119
|
+
job.attempts++;
|
|
120
|
+
job.startedAt = null;
|
|
121
|
+
job.lastHeartbeat = Date.now();
|
|
122
|
+
const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */ : "stalled" /* FailureReason.Stalled */, attemptsExhausted
|
|
123
|
+
? `Job stalled at max attempts (${job.maxAttempts})`
|
|
124
|
+
: `Job stalled ${job.stallCount} times`);
|
|
116
125
|
ctx.jobIndex.set(job.id, { type: 'dlq', queueName: job.queue });
|
|
117
126
|
ctx.storage?.saveDlqEntry(entry);
|
|
118
127
|
ctx.storage?.deleteJob(job.id);
|
|
@@ -19,6 +19,8 @@ export function getDlqContext(manager) {
|
|
|
19
19
|
return {
|
|
20
20
|
shards: getShards(manager),
|
|
21
21
|
jobIndex: manager.getJobIndex(),
|
|
22
|
+
jobResults: manager.jobResults,
|
|
23
|
+
jobLogs: manager.jobLogs,
|
|
22
24
|
// #110-class: without storage, retryDlqByFilter's deleteDlqEntry/insertJob
|
|
23
25
|
// silently no-op — filtered retries were never persisted in embedded mode
|
|
24
26
|
// (dlq rows resurrected the jobs into the DLQ on restart).
|
|
@@ -30,6 +30,7 @@ export function getJobCounts(ctx) {
|
|
|
30
30
|
failed: counts.failed,
|
|
31
31
|
delayed: counts.delayed,
|
|
32
32
|
paused: pv.paused,
|
|
33
|
+
'waiting-children': counts['waiting-children'],
|
|
33
34
|
};
|
|
34
35
|
}
|
|
35
36
|
/** Get job counts (async, works with TCP) */
|
|
@@ -47,6 +48,7 @@ export async function getJobCountsAsync(ctx) {
|
|
|
47
48
|
failed: 0,
|
|
48
49
|
delayed: 0,
|
|
49
50
|
paused: 0,
|
|
51
|
+
'waiting-children': 0,
|
|
50
52
|
};
|
|
51
53
|
}
|
|
52
54
|
const counts = response.counts;
|
|
@@ -58,6 +60,7 @@ export async function getJobCountsAsync(ctx) {
|
|
|
58
60
|
failed: counts?.failed ?? 0,
|
|
59
61
|
delayed: counts?.delayed ?? 0,
|
|
60
62
|
paused: counts?.paused ?? 0,
|
|
63
|
+
'waiting-children': counts?.['waiting-children'] ?? 0,
|
|
61
64
|
};
|
|
62
65
|
}
|
|
63
66
|
/** Get waiting job count */
|
|
@@ -54,6 +54,11 @@ export declare class DependencyTracker {
|
|
|
54
54
|
* Remove a parent job from waiting
|
|
55
55
|
*/
|
|
56
56
|
removeWaitingParent(jobId: JobId): Job | undefined;
|
|
57
|
+
/**
|
|
58
|
+
* Remove every dependency-gated job owned by a queue.
|
|
59
|
+
* Returns the removed ids so the caller can purge global indexes and storage.
|
|
60
|
+
*/
|
|
61
|
+
removeQueue(queue: string): JobId[];
|
|
57
62
|
/**
|
|
58
63
|
* Get a parent job waiting for children
|
|
59
64
|
*/
|
|
@@ -100,6 +100,26 @@ export class DependencyTracker {
|
|
|
100
100
|
}
|
|
101
101
|
return job;
|
|
102
102
|
}
|
|
103
|
+
/**
|
|
104
|
+
* Remove every dependency-gated job owned by a queue.
|
|
105
|
+
* Returns the removed ids so the caller can purge global indexes and storage.
|
|
106
|
+
*/
|
|
107
|
+
removeQueue(queue) {
|
|
108
|
+
const removed = [];
|
|
109
|
+
for (const [jobId, job] of this.waitingDeps) {
|
|
110
|
+
if (job.queue !== queue)
|
|
111
|
+
continue;
|
|
112
|
+
this.removeWaitingJob(jobId);
|
|
113
|
+
removed.push(jobId);
|
|
114
|
+
}
|
|
115
|
+
for (const [jobId, job] of this.waitingChildren) {
|
|
116
|
+
if (job.queue !== queue)
|
|
117
|
+
continue;
|
|
118
|
+
this.waitingChildren.delete(jobId);
|
|
119
|
+
removed.push(jobId);
|
|
120
|
+
}
|
|
121
|
+
return removed;
|
|
122
|
+
}
|
|
103
123
|
/**
|
|
104
124
|
* Get a parent job waiting for children
|
|
105
125
|
*/
|
|
@@ -364,13 +364,18 @@ export class Shard {
|
|
|
364
364
|
return { count, jobIds };
|
|
365
365
|
}
|
|
366
366
|
obliterate(queue) {
|
|
367
|
+
const removed = new Set(this.dependencyTracker.removeQueue(queue));
|
|
367
368
|
const q = this.queues.get(queue);
|
|
368
369
|
if (q) {
|
|
369
370
|
for (const job of q.values()) {
|
|
371
|
+
removed.add(job.id);
|
|
370
372
|
this.temporalManager.removeDelayed(job.id);
|
|
371
373
|
}
|
|
372
374
|
this.counters.adjustQueued(-q.size);
|
|
373
375
|
}
|
|
376
|
+
for (const entry of this.dlqManager.getEntries(queue)) {
|
|
377
|
+
removed.add(entry.job.id);
|
|
378
|
+
}
|
|
374
379
|
const dlqCount = this.dlqManager.deleteQueue(queue);
|
|
375
380
|
if (dlqCount > 0) {
|
|
376
381
|
this.counters.adjustDlq(-dlqCount);
|
|
@@ -381,5 +386,6 @@ export class Shard {
|
|
|
381
386
|
this.uniqueKeyManager.clearQueue(queue);
|
|
382
387
|
this.limiterManager.deleteQueue(queue);
|
|
383
388
|
this.activeGroups.delete(queue);
|
|
389
|
+
return Array.from(removed);
|
|
384
390
|
}
|
|
385
391
|
}
|