bunqueue 2.8.35 → 2.8.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +14 -2
  2. package/dist/application/backgroundTasks.js +28 -5
  3. package/dist/application/contextFactory.js +5 -0
  4. package/dist/application/dlqManager.d.ts +6 -0
  5. package/dist/application/dlqManager.js +35 -14
  6. package/dist/application/lockManager.js +22 -8
  7. package/dist/application/operations/customId.d.ts +31 -0
  8. package/dist/application/operations/customId.js +59 -0
  9. package/dist/application/operations/jobClaim.d.ts +17 -0
  10. package/dist/application/operations/jobClaim.js +20 -0
  11. package/dist/application/operations/jobManagement.d.ts +4 -5
  12. package/dist/application/operations/jobManagement.js +15 -116
  13. package/dist/application/operations/jobMoveOperations.d.ts +9 -0
  14. package/dist/application/operations/jobMoveOperations.js +94 -0
  15. package/dist/application/operations/jobStateTransitions.js +3 -0
  16. package/dist/application/operations/pull.js +5 -0
  17. package/dist/application/operations/push.js +8 -103
  18. package/dist/application/operations/pushInsert.d.ts +12 -0
  19. package/dist/application/operations/pushInsert.js +25 -0
  20. package/dist/application/operations/pushLocks.d.ts +14 -0
  21. package/dist/application/operations/pushLocks.js +43 -0
  22. package/dist/application/operations/queueControl.d.ts +1 -1
  23. package/dist/application/operations/queueControl.js +1 -1
  24. package/dist/application/queueManager.d.ts +1 -1
  25. package/dist/application/queueManager.js +17 -6
  26. package/dist/application/stallDetection.js +17 -8
  27. package/dist/client/queue/helpers.js +2 -0
  28. package/dist/client/queue/operations/counts.d.ts +1 -0
  29. package/dist/client/queue/operations/counts.js +3 -0
  30. package/dist/domain/queue/dependencyTracker.d.ts +5 -0
  31. package/dist/domain/queue/dependencyTracker.js +20 -0
  32. package/dist/domain/queue/shard.d.ts +1 -1
  33. package/dist/domain/queue/shard.js +6 -0
  34. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  35. package/dist/infrastructure/persistence/schema.js +25 -2
  36. package/dist/infrastructure/persistence/sqlite.d.ts +11 -0
  37. package/dist/infrastructure/persistence/sqlite.js +59 -5
  38. package/dist/infrastructure/persistence/sqliteBatch.js +8 -6
  39. package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -0
  40. package/dist/infrastructure/persistence/sqliteSerializer.js +9 -1
  41. package/dist/infrastructure/persistence/statements.d.ts +6 -1
  42. package/dist/infrastructure/persistence/statements.js +7 -5
  43. package/package.json +4 -1
@@ -0,0 +1,94 @@
1
+ /**
2
+ * Job move operations that claim active jobs for delayed or DLQ state.
3
+ */
4
+ import { processingShardIndex, shardIndex } from '../../shared/hash';
5
+ import { withWriteLock } from '../../shared/lock';
6
+ import { releaseClaimedJobOwnership } from './jobClaim';
7
+ /** Move active job back to delayed. */
8
+ export async function moveJobToDelayed(jobId, delay, ctx) {
9
+ const location = ctx.jobIndex.get(jobId);
10
+ if (location?.type !== 'processing')
11
+ return false;
12
+ const procIdx = processingShardIndex(jobId);
13
+ const job = await withWriteLock(ctx.processingLocks[procIdx], () => {
14
+ const claimed = ctx.processingShards[procIdx].get(jobId);
15
+ if (claimed) {
16
+ ctx.processingShards[procIdx].delete(jobId);
17
+ releaseClaimedJobOwnership(jobId, ctx);
18
+ }
19
+ return claimed;
20
+ });
21
+ if (!job)
22
+ return false;
23
+ const now = Date.now();
24
+ job.runAt = now + delay;
25
+ job.startedAt = null;
26
+ const idx = shardIndex(job.queue);
27
+ const queueName = job.queue;
28
+ await withWriteLock(ctx.shardLocks[idx], () => {
29
+ const shard = ctx.shards[idx];
30
+ shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
31
+ shard.getQueue(job.queue).push(job);
32
+ const isDelayed = job.runAt > now;
33
+ shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
34
+ ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
35
+ shard.notify(job.queue);
36
+ });
37
+ ctx.storage?.updateRunAt(jobId, job.runAt);
38
+ ctx.eventsManager.broadcast({
39
+ eventType: "delayed" /* EventType.Delayed */,
40
+ jobId,
41
+ queue: queueName,
42
+ timestamp: Date.now(),
43
+ delay,
44
+ });
45
+ return true;
46
+ }
47
+ /** Discard a waiting or active job to the DLQ. */
48
+ export async function discardJob(jobId, ctx) {
49
+ const location = ctx.jobIndex.get(jobId);
50
+ if (!location)
51
+ return false;
52
+ let job = null;
53
+ if (location.type === 'queue') {
54
+ job = await withWriteLock(ctx.shardLocks[location.shardIdx], () => {
55
+ const shard = ctx.shards[location.shardIdx];
56
+ const removed = shard.getQueue(location.queueName).remove(jobId);
57
+ if (removed)
58
+ shard.decrementQueued(jobId);
59
+ return removed;
60
+ });
61
+ }
62
+ else if (location.type === 'processing') {
63
+ const procIdx = processingShardIndex(jobId);
64
+ job = await withWriteLock(ctx.processingLocks[procIdx], () => {
65
+ const claimed = ctx.processingShards[procIdx].get(jobId) ?? null;
66
+ if (claimed) {
67
+ ctx.processingShards[procIdx].delete(jobId);
68
+ releaseClaimedJobOwnership(jobId, ctx);
69
+ }
70
+ return claimed;
71
+ });
72
+ }
73
+ if (!job)
74
+ return false;
75
+ const validJob = job;
76
+ const idx = shardIndex(validJob.queue);
77
+ const fromProcessing = location.type === 'processing';
78
+ const entry = await withWriteLock(ctx.shardLocks[idx], () => {
79
+ const shard = ctx.shards[idx];
80
+ if (fromProcessing) {
81
+ shard.releaseJobResources(validJob.queue, validJob.uniqueKey, validJob.groupId);
82
+ shard.notify(validJob.queue);
83
+ }
84
+ else if (validJob.uniqueKey) {
85
+ shard.releaseUniqueKey(validJob.queue, validJob.uniqueKey);
86
+ }
87
+ const dlqEntry = shard.addToDlq(validJob);
88
+ ctx.jobIndex.set(jobId, { type: 'dlq', queueName: validJob.queue });
89
+ return dlqEntry;
90
+ });
91
+ ctx.storage?.saveDlqEntry(entry);
92
+ ctx.storage?.deleteJob(jobId);
93
+ return true;
94
+ }
@@ -7,6 +7,7 @@
7
7
  */
8
8
  import { shardIndex, processingShardIndex } from '../../shared/hash';
9
9
  import { withWriteLock } from '../../shared/lock';
10
+ import { releaseClaimedJobOwnership } from './jobClaim';
10
11
  /** Move active job back to waiting */
11
12
  export async function moveActiveToWait(jobId, ctx) {
12
13
  const location = ctx.jobIndex.get(jobId);
@@ -17,6 +18,7 @@ export async function moveActiveToWait(jobId, ctx) {
17
18
  const job = ctx.processingShards[procIdx].get(jobId);
18
19
  if (job) {
19
20
  ctx.processingShards[procIdx].delete(jobId);
21
+ releaseClaimedJobOwnership(jobId, ctx);
20
22
  }
21
23
  return job;
22
24
  });
@@ -73,6 +75,7 @@ export async function moveToWaitingChildren(jobId, ctx) {
73
75
  const job = ctx.processingShards[procIdx].get(jobId);
74
76
  if (job) {
75
77
  ctx.processingShards[procIdx].delete(jobId);
78
+ releaseClaimedJobOwnership(jobId, ctx);
76
79
  }
77
80
  return job;
78
81
  });
@@ -35,6 +35,11 @@ function tryDequeueNextJob(shard, queue, now, ctx) {
35
35
  // Expired entries are not parked: they are genuinely removed from the
36
36
  // logical queue and therefore update counters/index exactly once.
37
37
  if (isExpired(job, now)) {
38
+ // Delete persistence before mutating the in-memory indexes. deleteJob
39
+ // also removes a pending buffered INSERT, so the expired job cannot be
40
+ // flushed after this pull or resurrect during crash recovery. This is
41
+ // synchronous and therefore keeps the shard critical section await-free.
42
+ ctx.storage?.deleteJob(job.id);
38
43
  q.pop();
39
44
  shard.decrementQueued(job.id);
40
45
  ctx.jobIndex.delete(job.id);
@@ -2,80 +2,13 @@
2
2
  * Push Operations
3
3
  * Job push and batch push logic
4
4
  */
5
- import { createJob, generateJobId, jobId, } from '../../domain/types/job';
6
- import { withWriteLock } from '../../shared/lock';
5
+ import { createJob } from '../../domain/types/job';
7
6
  import { shardIndex } from '../../shared/hash';
8
7
  import { latencyTracker } from '../latencyTracker';
9
8
  import { throughputTracker } from '../throughputTracker';
10
- /**
11
- * Handle custom ID idempotency check
12
- * Returns existing job if found, or new ID to use
13
- */
14
- function handleCustomId(input, shard, ctx) {
15
- if (!input.customId) {
16
- return { skip: false, id: generateJobId() };
17
- }
18
- const id = jobId(input.customId);
19
- const existing = ctx.customIdMap.get(input.customId);
20
- // Re-adding an existing custom id while its job is UNFINISHED is an idempotent
21
- // no-op (BullMQ parity): return the existing job, never a second insert of the
22
- // same deterministic id (which would collide on the jobs.id PRIMARY KEY). Three
23
- // unfinished states are covered:
24
- // • waiting/delayed/prioritized — live in the priority queue → return the job.
25
- // • waiting-children — tracked under 'queue' but held in shard.waitingDeps (its
26
- // row is already persisted) → idempotent skip via the existing id.
27
- // • active (processing) — popped from the queue, still running on a worker; its
28
- // row survives on disk → idempotent skip via the existing id.
29
- // PushContext has no processingShards, so for the latter two we cannot fetch the
30
- // live Job here; pushJob rebuilds a placeholder { ...job, id } after createJob,
31
- // exactly like handleDeduplication's active path. Completed (#92) and DLQ jobs
32
- // are terminal and fall through to the reuse path below.
33
- if (existing && !ctx.completedJobs.has(id)) {
34
- const location = ctx.jobIndex.get(existing);
35
- if (location?.type === 'queue') {
36
- const existingJob = shard.getQueue(location.queueName).find(existing);
37
- if (existingJob) {
38
- return { skip: true, existingJob };
39
- }
40
- if (shard.waitingDeps.has(existing)) {
41
- return { skip: true, existingId: id };
42
- }
43
- }
44
- else if (location?.type === 'processing') {
45
- return { skip: true, existingId: id };
46
- }
47
- }
48
- // Reuse path: the id is free in the queue (no mapping, or the prior job is
49
- // processing/completed). If the prior job COMPLETED, its row survives on disk
50
- // (markCompleted does an UPDATE, not a DELETE) and it is still in completedJobs.
51
- // Reusing the same deterministic id would then (a) make getJobState return
52
- // 'completed' for the brand-new job and (b) collide on the `jobs.id` PRIMARY KEY
53
- // at flush time. Evict the stale completed job so the reused id starts fresh as
54
- // 'waiting' (#92). Checked regardless of customIdMap state — the mapping may have
55
- // been cleared on completion, which would otherwise skip this path entirely.
56
- if (ctx.completedJobs.has(id)) {
57
- ctx.completedJobs.delete(id);
58
- ctx.completedJobsData.delete(id);
59
- ctx.jobResults.delete(id);
60
- ctx.jobIndex.delete(id);
61
- ctx.storage?.deleteJob(id); // removes the surviving row + result + any buffered insert
62
- }
63
- // NOTE on ORPHAN rows: a durable jobs row can outlive its in-memory tracking when
64
- // obliterate() (fire-and-forget void over TCP) or a buffer flush races an in-flight
65
- // durable insert. Reaching the reuse path with such a stale row no longer throws —
66
- // the insert statements use ON CONFLICT(id) DO UPDATE (upsert), so the orphan is
67
- // overwritten in place at insert time with ZERO per-push cost. We deliberately do
68
- // NOT pre-delete here: deleteJob() is a synchronous DELETE + O(buffer) scan inside
69
- // the shard lock and would halve customId push/addBulk throughput.
70
- // A recycled custom id may carry a stale timeout marker from a prior job (which
71
- // may have DLQ'd, so it is NOT in completedJobs above). Clear it so the new
72
- // job's stall-retry recovery is not wrongly discarded — otherwise the
73
- // timeout-resurrection guard would reintroduce #33/#75 duplicate execution for
74
- // reused ids.
75
- ctx.timedOutJobs?.delete(id);
76
- ctx.customIdMap.set(input.customId, id);
77
- return { skip: false, id };
78
- }
9
+ import { handleCustomId } from './customId';
10
+ import { insertJobToShard } from './pushInsert';
11
+ import { withPushWriteLocks } from './pushLocks';
79
12
  /**
80
13
  * Handle unique key deduplication
81
14
  * Returns existing job ID if duplicate found and should skip, or allows insert
@@ -143,34 +76,6 @@ function handleDeduplication(job, input, queue, shard, ctx) {
143
76
  shard.registerUniqueKeyWithTtl(queue, job.uniqueKey, job.id, input.dedup?.ttl);
144
77
  return { skip: false };
145
78
  }
146
- /**
147
- * Insert job into shard (queue or waitingDeps)
148
- */
149
- function insertJobToShard(job, queue, shard, shardIdx, ctx) {
150
- const hasDeps = job.dependsOn.length > 0;
151
- const needsWaiting = hasDeps &&
152
- !job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
153
- const now = Date.now();
154
- if (needsWaiting) {
155
- shard.waitingDeps.set(job.id, job);
156
- shard.registerDependencies(job.id, job.dependsOn);
157
- job.timeline.push({ state: 'waiting-children', timestamp: now });
158
- ctx.dashboardEmit?.('job:waiting-children', {
159
- jobId: String(job.id),
160
- queue,
161
- dependsOn: job.dependsOn.map(String),
162
- });
163
- }
164
- else {
165
- shard.getQueue(queue).push(job);
166
- const isDelayed = job.runAt > now;
167
- shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
168
- // Timeline: initial state based on scheduling and priority
169
- const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
170
- job.timeline.push({ state, timestamp: now });
171
- }
172
- ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
173
- }
174
79
  /**
175
80
  * Push a single job to queue
176
81
  * NOTE: customId check happens INSIDE lock to prevent race conditions
@@ -180,10 +85,10 @@ export async function pushJob(queue, input, ctx) {
180
85
  const idx = shardIndex(queue);
181
86
  const now = Date.now();
182
87
  let result;
183
- await withWriteLock(ctx.shardLocks[idx], () => {
88
+ await withPushWriteLocks(queue, [input], ctx, (lockedShardIndexes) => {
184
89
  const shard = ctx.shards[idx];
185
90
  // Check custom ID idempotency INSIDE lock to prevent race conditions
186
- const customIdResult = handleCustomId(input, shard, ctx);
91
+ const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
187
92
  if (customIdResult.skip) {
188
93
  // Idempotent re-add: return the live queued job if we have it, otherwise
189
94
  // (active / waiting-children) a placeholder carrying the existing id so the
@@ -247,11 +152,11 @@ export async function pushJobBatch(queue, inputs, ctx) {
247
152
  // path), exactly like a single durable push — otherwise addBulk silently
248
153
  // downgrades the documented "durable" guarantee.
249
154
  const durableJobs = [];
250
- await withWriteLock(ctx.shardLocks[idx], () => {
155
+ await withPushWriteLocks(queue, inputs, ctx, (lockedShardIndexes) => {
251
156
  const shard = ctx.shards[idx];
252
157
  for (const input of inputs) {
253
158
  // Check custom ID idempotency
254
- const customIdResult = handleCustomId(input, shard, ctx);
159
+ const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
255
160
  if (customIdResult.skip) {
256
161
  // Idempotent re-add (queued, active, or waiting-children) — no insert.
257
162
  resultIds.push('existingJob' in customIdResult
@@ -0,0 +1,12 @@
1
+ import type { Shard } from '../../domain/queue/shard';
2
+ import type { Job, JobId } from '../../domain/types/job';
3
+ import type { JobLocation } from '../../domain/types/queue';
4
+ import type { SetLike } from '../../shared/lru';
5
+ export interface PushInsertContext {
6
+ completedJobs: SetLike<JobId>;
7
+ depCompletions?: SetLike<JobId>;
8
+ jobIndex: Map<JobId, JobLocation>;
9
+ dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
10
+ }
11
+ /** Insert a new job into the runnable heap or dependency wait set. */
12
+ export declare function insertJobToShard(job: Job, queue: string, shard: Shard, shardIdx: number, ctx: PushInsertContext): void;
@@ -0,0 +1,25 @@
1
+ /** Insert a new job into the runnable heap or dependency wait set. */
2
+ export function insertJobToShard(job, queue, shard, shardIdx, ctx) {
3
+ const hasDeps = job.dependsOn.length > 0;
4
+ const needsWaiting = hasDeps &&
5
+ !job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
6
+ const now = Date.now();
7
+ if (needsWaiting) {
8
+ shard.waitingDeps.set(job.id, job);
9
+ shard.registerDependencies(job.id, job.dependsOn);
10
+ job.timeline.push({ state: 'waiting-children', timestamp: now });
11
+ ctx.dashboardEmit?.('job:waiting-children', {
12
+ jobId: String(job.id),
13
+ queue,
14
+ dependsOn: job.dependsOn.map(String),
15
+ });
16
+ }
17
+ else {
18
+ shard.getQueue(queue).push(job);
19
+ const isDelayed = job.runAt > now;
20
+ shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
21
+ const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
22
+ job.timeline.push({ state, timestamp: now });
23
+ }
24
+ ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
25
+ }
@@ -0,0 +1,14 @@
1
+ import type { JobId, JobInput } from '../../domain/types/job';
2
+ import type { JobLocation } from '../../domain/types/queue';
3
+ import type { RWLock } from '../../shared/lock';
4
+ interface PushLockContext {
5
+ shardLocks: RWLock[];
6
+ jobIndex: Map<JobId, JobLocation>;
7
+ }
8
+ /**
9
+ * Lock the target shard plus any shard owning a terminal custom-ID generation.
10
+ * The required set is rechecked after acquisition because jobIndex may change
11
+ * while an earlier lock is awaited.
12
+ */
13
+ export declare function withPushWriteLocks<T>(queue: string, inputs: readonly JobInput[], ctx: PushLockContext, fn: (lockedShardIndexes: ReadonlySet<number>) => T): Promise<T>;
14
+ export {};
@@ -0,0 +1,43 @@
1
+ import { jobId } from '../../domain/types/job';
2
+ import { shardIndex } from '../../shared/hash';
3
+ function requiredShardIndexes(queue, inputs, jobIndex) {
4
+ const indexes = new Set([shardIndex(queue)]);
5
+ for (const input of inputs) {
6
+ if (!input.customId)
7
+ continue;
8
+ const location = jobIndex.get(jobId(input.customId));
9
+ if (location?.type === 'queue') {
10
+ indexes.add(location.shardIdx);
11
+ }
12
+ else if (location?.type === 'dlq') {
13
+ indexes.add(shardIndex(location.queueName));
14
+ }
15
+ }
16
+ return [...indexes].sort((a, b) => a - b);
17
+ }
18
+ /**
19
+ * Lock the target shard plus any shard owning a terminal custom-ID generation.
20
+ * The required set is rechecked after acquisition because jobIndex may change
21
+ * while an earlier lock is awaited.
22
+ */
23
+ export async function withPushWriteLocks(queue, inputs, ctx, fn) {
24
+ while (true) {
25
+ const indexes = requiredShardIndexes(queue, inputs, ctx.jobIndex);
26
+ const guards = [];
27
+ try {
28
+ for (const index of indexes) {
29
+ guards.push(await ctx.shardLocks[index].acquireWrite());
30
+ }
31
+ const locked = new Set(indexes);
32
+ const current = requiredShardIndexes(queue, inputs, ctx.jobIndex);
33
+ if (current.some((index) => !locked.has(index)))
34
+ continue;
35
+ return fn(locked);
36
+ }
37
+ finally {
38
+ for (let i = guards.length - 1; i >= 0; i--) {
39
+ guards[i].release();
40
+ }
41
+ }
42
+ }
43
+ }
@@ -30,7 +30,7 @@ export declare function isQueuePaused(queue: string, ctx: QueueControlContext):
30
30
  /** Drain all waiting jobs from queue */
31
31
  export declare function drainQueue(queue: string, ctx: QueueControlContext): number;
32
32
  /** Remove all queue data */
33
- export declare function obliterateQueue(queue: string, ctx: QueueControlContext): void;
33
+ export declare function obliterateQueue(queue: string, ctx: QueueControlContext): JobId[];
34
34
  /** List all queue names */
35
35
  export declare function listAllQueues(ctx: QueueControlContext): string[];
36
36
  /**
@@ -35,7 +35,7 @@ export function drainQueue(queue, ctx) {
35
35
  /** Remove all queue data */
36
36
  export function obliterateQueue(queue, ctx) {
37
37
  const idx = shardIndex(queue);
38
- ctx.shards[idx].obliterate(queue);
38
+ return ctx.shards[idx].obliterate(queue);
39
39
  }
40
40
  /** List all queue names */
41
41
  export function listAllQueues(ctx) {
@@ -6,7 +6,7 @@ import type { Job, JobId, JobInput, JobLock, LockToken } from '../domain/types/j
6
6
  import type { JobLocation, JobEvent } from '../domain/types/queue';
7
7
  import type { CronJob, CronJobInput } from '../domain/types/cron';
8
8
  import type { JobLogEntry, CreateWorkerOptions } from '../domain/types/worker';
9
- import type { StallConfig } from '../domain/types/stall';
9
+ import { type StallConfig } from '../domain/types/stall';
10
10
  import type { DlqConfig, DlqEntry, DlqFilter, DlqStats } from '../domain/types/dlq';
11
11
  import { Shard } from '../domain/queue/shard';
12
12
  import { WebhookManager } from './webhookManager';
@@ -3,6 +3,7 @@
3
3
  * Core orchestrator for all queue operations
4
4
  */
5
5
  import { DEFAULT_LOCK_TTL } from '../domain/types/job';
6
+ import { DEFAULT_STALL_CONFIG } from '../domain/types/stall';
6
7
  import { Shard } from '../domain/queue/shard';
7
8
  import { SqliteStorage } from '../infrastructure/persistence/sqlite';
8
9
  import { CronScheduler } from '../infrastructure/scheduler/cronScheduler';
@@ -741,12 +742,12 @@ export class QueueManager {
741
742
  return count;
742
743
  }
743
744
  obliterate(queue) {
744
- queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
745
+ const shardJobs = queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
745
746
  dlqOps.purgeDlqJobs(queue, this.contextFactory.getDlqContext());
746
- // obliterateQueue() clears the waiting/delayed shard only. Active jobs in
747
- // processingShards, plus completed/result/log/lock state in global indexes,
748
- // plus SQLite rows, all survive unless we purge them here.
749
- const toDrop = new Set();
747
+ // obliterateQueue() returns every queued, DLQ, and dependency-gated job it
748
+ // removed. Active jobs in processingShards, plus completed/result/log/lock
749
+ // state in global indexes, still need to be discovered and purged here.
750
+ const toDrop = new Set(shardJobs);
750
751
  for (const [jid, loc] of this.jobIndex) {
751
752
  if (loc.type === 'processing') {
752
753
  const job = this.processingShards[loc.shardIdx]?.get(jid);
@@ -885,11 +886,19 @@ export class QueueManager {
885
886
  if (!this.storage)
886
887
  return;
887
888
  const state = this.shards[shardIndex(queue)].getState(queue);
889
+ const stallConfig = this.shards[shardIndex(queue)].getStallConfig(queue);
890
+ const hasCustomStallConfig = stallConfig.enabled !== DEFAULT_STALL_CONFIG.enabled ||
891
+ stallConfig.stallInterval !== DEFAULT_STALL_CONFIG.stallInterval ||
892
+ stallConfig.maxStalls !== DEFAULT_STALL_CONFIG.maxStalls ||
893
+ stallConfig.gracePeriod !== DEFAULT_STALL_CONFIG.gracePeriod;
888
894
  // When control-state returns fully to default (not paused, no limits), drop
889
895
  // the row instead of persisting an all-default placeholder. Keeps the table
890
896
  // free of noise rows for ephemeral queues that only ever call resume/clear*,
891
897
  // and recovers identically (absent row → default state).
892
- if (!state.paused && state.rateLimit === null && state.concurrencyLimit === null) {
898
+ if (!state.paused &&
899
+ state.rateLimit === null &&
900
+ state.concurrencyLimit === null &&
901
+ !hasCustomStallConfig) {
893
902
  this.storage.deleteQueueState(queue);
894
903
  return;
895
904
  }
@@ -899,6 +908,7 @@ export class QueueManager {
899
908
  concurrencyLimit: state.concurrencyLimit,
900
909
  rateLimitDuration: state.rateLimitDuration,
901
910
  rateLimitExpiresAt: state.rateLimitExpiresAt,
911
+ stallConfig,
902
912
  });
903
913
  }
904
914
  /** Get rate limit and concurrency limit for a queue */
@@ -930,6 +940,7 @@ export class QueueManager {
930
940
  // ============ Stall & DLQ Config ============
931
941
  setStallConfig(queue, config) {
932
942
  this.shards[shardIndex(queue)].setStallConfig(queue, config);
943
+ this.persistQueueState(queue);
933
944
  }
934
945
  getStallConfig(queue) {
935
946
  return this.shards[shardIndex(queue)].getStallConfig(queue);
@@ -66,6 +66,7 @@ async function handleStalledJob(job, action, ctx) {
66
66
  // Lock order: shardLocks BEFORE processingLocks (per lock hierarchy in CLAUDE.md)
67
67
  // Broadcast events AFTER verifying job is still stalled to avoid false positives
68
68
  let handled = false;
69
+ let handledAction = action;
69
70
  await withWriteLock(ctx.shardLocks[idx], async () => {
70
71
  await withWriteLock(ctx.processingLocks[procIdx], () => {
71
72
  // Verify job is still in processing (might have been handled already)
@@ -73,8 +74,10 @@ async function handleStalledJob(job, action, ctx) {
73
74
  return; // Job already completed, don't broadcast stalled event
74
75
  }
75
76
  const shard = ctx.shards[idx];
76
- if (action === "move_to_dlq" /* StallAction.MoveToDlq */) {
77
- moveStalliedJobToDlq(job, ctx, shard, procIdx, idx);
77
+ const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
78
+ if (action === "move_to_dlq" /* StallAction.MoveToDlq */ || attemptsExhausted) {
79
+ handledAction = "move_to_dlq" /* StallAction.MoveToDlq */;
80
+ moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
78
81
  }
79
82
  else {
80
83
  retryStalliedJob(job, ctx, shard, procIdx, idx);
@@ -87,23 +90,23 @@ async function handleStalledJob(job, action, ctx) {
87
90
  ctx.dashboardEmit?.('job:stalled', {
88
91
  jobId: String(job.id),
89
92
  queue: job.queue,
90
- stallCount: job.stallCount + 1,
91
- action,
93
+ stallCount: job.stallCount,
94
+ action: handledAction,
92
95
  });
93
96
  ctx.eventsManager.broadcast({
94
97
  eventType: "stalled" /* EventType.Stalled */,
95
98
  queue: job.queue,
96
99
  jobId: job.id,
97
100
  timestamp: Date.now(),
98
- data: { stallCount: job.stallCount + 1, action },
101
+ data: { stallCount: job.stallCount, action: handledAction },
99
102
  });
100
103
  void ctx.webhookManager.trigger('stalled', String(job.id), job.queue, {
101
- data: { stallCount: job.stallCount + 1, action },
104
+ data: { stallCount: job.stallCount, action: handledAction },
102
105
  });
103
106
  }
104
107
  }
105
108
  /** Move stalled job to DLQ */
106
- function moveStalliedJobToDlq(job, ctx, shard, procIdx, _idx) {
109
+ function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
107
110
  ctx.processingShards[procIdx].delete(job.id);
108
111
  shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
109
112
  // Discard cron jobs with preventOverlap instead of sending to DLQ (#73).
@@ -112,7 +115,13 @@ function moveStalliedJobToDlq(job, ctx, shard, procIdx, _idx) {
112
115
  ctx.storage?.deleteJob(job.id);
113
116
  return;
114
117
  }
115
- const entry = shard.addToDlq(job, "stalled" /* FailureReason.Stalled */, `Job stalled ${job.stallCount + 1} times`);
118
+ incrementStallCount(job);
119
+ job.attempts++;
120
+ job.startedAt = null;
121
+ job.lastHeartbeat = Date.now();
122
+ const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */ : "stalled" /* FailureReason.Stalled */, attemptsExhausted
123
+ ? `Job stalled at max attempts (${job.maxAttempts})`
124
+ : `Job stalled ${job.stallCount} times`);
116
125
  ctx.jobIndex.set(job.id, { type: 'dlq', queueName: job.queue });
117
126
  ctx.storage?.saveDlqEntry(entry);
118
127
  ctx.storage?.deleteJob(job.id);
@@ -19,6 +19,8 @@ export function getDlqContext(manager) {
19
19
  return {
20
20
  shards: getShards(manager),
21
21
  jobIndex: manager.getJobIndex(),
22
+ jobResults: manager.jobResults,
23
+ jobLogs: manager.jobLogs,
22
24
  // #110-class: without storage, retryDlqByFilter's deleteDlqEntry/insertJob
23
25
  // silently no-op — filtered retries were never persisted in embedded mode
24
26
  // (dlq rows resurrected the jobs into the DLQ on restart).
@@ -16,6 +16,7 @@ export interface JobCounts {
16
16
  failed: number;
17
17
  delayed: number;
18
18
  paused: number;
19
+ 'waiting-children': number;
19
20
  }
20
21
  /**
21
22
  * Get job counts.
@@ -30,6 +30,7 @@ export function getJobCounts(ctx) {
30
30
  failed: counts.failed,
31
31
  delayed: counts.delayed,
32
32
  paused: pv.paused,
33
+ 'waiting-children': counts['waiting-children'],
33
34
  };
34
35
  }
35
36
  /** Get job counts (async, works with TCP) */
@@ -47,6 +48,7 @@ export async function getJobCountsAsync(ctx) {
47
48
  failed: 0,
48
49
  delayed: 0,
49
50
  paused: 0,
51
+ 'waiting-children': 0,
50
52
  };
51
53
  }
52
54
  const counts = response.counts;
@@ -58,6 +60,7 @@ export async function getJobCountsAsync(ctx) {
58
60
  failed: counts?.failed ?? 0,
59
61
  delayed: counts?.delayed ?? 0,
60
62
  paused: counts?.paused ?? 0,
63
+ 'waiting-children': counts?.['waiting-children'] ?? 0,
61
64
  };
62
65
  }
63
66
  /** Get waiting job count */
@@ -54,6 +54,11 @@ export declare class DependencyTracker {
54
54
  * Remove a parent job from waiting
55
55
  */
56
56
  removeWaitingParent(jobId: JobId): Job | undefined;
57
+ /**
58
+ * Remove every dependency-gated job owned by a queue.
59
+ * Returns the removed ids so the caller can purge global indexes and storage.
60
+ */
61
+ removeQueue(queue: string): JobId[];
57
62
  /**
58
63
  * Get a parent job waiting for children
59
64
  */
@@ -100,6 +100,26 @@ export class DependencyTracker {
100
100
  }
101
101
  return job;
102
102
  }
103
+ /**
104
+ * Remove every dependency-gated job owned by a queue.
105
+ * Returns the removed ids so the caller can purge global indexes and storage.
106
+ */
107
+ removeQueue(queue) {
108
+ const removed = [];
109
+ for (const [jobId, job] of this.waitingDeps) {
110
+ if (job.queue !== queue)
111
+ continue;
112
+ this.removeWaitingJob(jobId);
113
+ removed.push(jobId);
114
+ }
115
+ for (const [jobId, job] of this.waitingChildren) {
116
+ if (job.queue !== queue)
117
+ continue;
118
+ this.waitingChildren.delete(jobId);
119
+ removed.push(jobId);
120
+ }
121
+ return removed;
122
+ }
103
123
  /**
104
124
  * Get a parent job waiting for children
105
125
  */
@@ -135,5 +135,5 @@ export declare class Shard {
135
135
  count: number;
136
136
  jobIds: JobId[];
137
137
  };
138
- obliterate(queue: string): void;
138
+ obliterate(queue: string): JobId[];
139
139
  }
@@ -364,13 +364,18 @@ export class Shard {
364
364
  return { count, jobIds };
365
365
  }
366
366
  obliterate(queue) {
367
+ const removed = new Set(this.dependencyTracker.removeQueue(queue));
367
368
  const q = this.queues.get(queue);
368
369
  if (q) {
369
370
  for (const job of q.values()) {
371
+ removed.add(job.id);
370
372
  this.temporalManager.removeDelayed(job.id);
371
373
  }
372
374
  this.counters.adjustQueued(-q.size);
373
375
  }
376
+ for (const entry of this.dlqManager.getEntries(queue)) {
377
+ removed.add(entry.job.id);
378
+ }
374
379
  const dlqCount = this.dlqManager.deleteQueue(queue);
375
380
  if (dlqCount > 0) {
376
381
  this.counters.adjustDlq(-dlqCount);
@@ -381,5 +386,6 @@ export class Shard {
381
386
  this.uniqueKeyManager.clearQueue(queue);
382
387
  this.limiterManager.deleteQueue(queue);
383
388
  this.activeGroups.delete(queue);
389
+ return Array.from(removed);
384
390
  }
385
391
  }