bunqueue 2.8.34 → 2.8.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +14 -2
  2. package/dist/application/backgroundTasks.js +37 -7
  3. package/dist/application/contextFactory.js +5 -0
  4. package/dist/application/dlqManager.d.ts +6 -0
  5. package/dist/application/dlqManager.js +35 -14
  6. package/dist/application/lockManager.js +22 -8
  7. package/dist/application/operations/customId.d.ts +31 -0
  8. package/dist/application/operations/customId.js +59 -0
  9. package/dist/application/operations/jobClaim.d.ts +17 -0
  10. package/dist/application/operations/jobClaim.js +20 -0
  11. package/dist/application/operations/jobManagement.d.ts +4 -5
  12. package/dist/application/operations/jobManagement.js +15 -116
  13. package/dist/application/operations/jobMoveOperations.d.ts +9 -0
  14. package/dist/application/operations/jobMoveOperations.js +94 -0
  15. package/dist/application/operations/jobStateTransitions.js +3 -0
  16. package/dist/application/operations/pull.js +5 -0
  17. package/dist/application/operations/push.js +8 -103
  18. package/dist/application/operations/pushInsert.d.ts +12 -0
  19. package/dist/application/operations/pushInsert.js +25 -0
  20. package/dist/application/operations/pushLocks.d.ts +14 -0
  21. package/dist/application/operations/pushLocks.js +43 -0
  22. package/dist/application/operations/queueControl.d.ts +1 -1
  23. package/dist/application/operations/queueControl.js +1 -1
  24. package/dist/application/queueManager.d.ts +2 -2
  25. package/dist/application/queueManager.js +30 -10
  26. package/dist/application/stallDetection.js +17 -8
  27. package/dist/client/queue/dlq.d.ts +23 -1
  28. package/dist/client/queue/dlq.js +75 -0
  29. package/dist/client/queue/helpers.js +2 -0
  30. package/dist/client/queue/jobProxy.d.ts +1 -1
  31. package/dist/client/queue/operations/control.d.ts +17 -0
  32. package/dist/client/queue/operations/control.js +41 -0
  33. package/dist/client/queue/operations/counts.d.ts +1 -0
  34. package/dist/client/queue/operations/counts.js +3 -0
  35. package/dist/client/queue/queue.d.ts +13 -0
  36. package/dist/client/queue/queue.js +39 -0
  37. package/dist/client/queue/rateLimit.d.ts +18 -3
  38. package/dist/client/queue/rateLimit.js +45 -10
  39. package/dist/client/queue/stall.d.ts +2 -0
  40. package/dist/client/queue/stall.js +12 -0
  41. package/dist/domain/queue/dependencyTracker.d.ts +5 -0
  42. package/dist/domain/queue/dependencyTracker.js +20 -0
  43. package/dist/domain/queue/limiterManager.d.ts +14 -2
  44. package/dist/domain/queue/limiterManager.js +32 -5
  45. package/dist/domain/queue/shard.d.ts +3 -2
  46. package/dist/domain/queue/shard.js +11 -2
  47. package/dist/domain/types/command.d.ts +4 -0
  48. package/dist/domain/types/queue.d.ts +4 -0
  49. package/dist/domain/types/queue.js +2 -0
  50. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  51. package/dist/infrastructure/persistence/schema.js +38 -2
  52. package/dist/infrastructure/persistence/sqlite.d.ts +20 -1
  53. package/dist/infrastructure/persistence/sqlite.js +62 -6
  54. package/dist/infrastructure/persistence/sqliteBatch.js +8 -6
  55. package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -0
  56. package/dist/infrastructure/persistence/sqliteSerializer.js +9 -1
  57. package/dist/infrastructure/persistence/statements.d.ts +8 -1
  58. package/dist/infrastructure/persistence/statements.js +7 -5
  59. package/dist/infrastructure/scheduler/cronScheduler.d.ts +5 -0
  60. package/dist/infrastructure/scheduler/cronScheduler.js +39 -13
  61. package/dist/infrastructure/server/handlers/advanced.js +11 -2
  62. package/dist/infrastructure/server/httpRouteQueueConfig.js +2 -0
  63. package/package.json +4 -1
@@ -0,0 +1,94 @@
1
+ /**
2
+ * Job move operations that claim active jobs for delayed or DLQ state.
3
+ */
4
+ import { processingShardIndex, shardIndex } from '../../shared/hash';
5
+ import { withWriteLock } from '../../shared/lock';
6
+ import { releaseClaimedJobOwnership } from './jobClaim';
7
+ /** Move active job back to delayed. */
8
+ export async function moveJobToDelayed(jobId, delay, ctx) {
9
+ const location = ctx.jobIndex.get(jobId);
10
+ if (location?.type !== 'processing')
11
+ return false;
12
+ const procIdx = processingShardIndex(jobId);
13
+ const job = await withWriteLock(ctx.processingLocks[procIdx], () => {
14
+ const claimed = ctx.processingShards[procIdx].get(jobId);
15
+ if (claimed) {
16
+ ctx.processingShards[procIdx].delete(jobId);
17
+ releaseClaimedJobOwnership(jobId, ctx);
18
+ }
19
+ return claimed;
20
+ });
21
+ if (!job)
22
+ return false;
23
+ const now = Date.now();
24
+ job.runAt = now + delay;
25
+ job.startedAt = null;
26
+ const idx = shardIndex(job.queue);
27
+ const queueName = job.queue;
28
+ await withWriteLock(ctx.shardLocks[idx], () => {
29
+ const shard = ctx.shards[idx];
30
+ shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
31
+ shard.getQueue(job.queue).push(job);
32
+ const isDelayed = job.runAt > now;
33
+ shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
34
+ ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
35
+ shard.notify(job.queue);
36
+ });
37
+ ctx.storage?.updateRunAt(jobId, job.runAt);
38
+ ctx.eventsManager.broadcast({
39
+ eventType: "delayed" /* EventType.Delayed */,
40
+ jobId,
41
+ queue: queueName,
42
+ timestamp: Date.now(),
43
+ delay,
44
+ });
45
+ return true;
46
+ }
47
+ /** Discard a waiting or active job to the DLQ. */
48
+ export async function discardJob(jobId, ctx) {
49
+ const location = ctx.jobIndex.get(jobId);
50
+ if (!location)
51
+ return false;
52
+ let job = null;
53
+ if (location.type === 'queue') {
54
+ job = await withWriteLock(ctx.shardLocks[location.shardIdx], () => {
55
+ const shard = ctx.shards[location.shardIdx];
56
+ const removed = shard.getQueue(location.queueName).remove(jobId);
57
+ if (removed)
58
+ shard.decrementQueued(jobId);
59
+ return removed;
60
+ });
61
+ }
62
+ else if (location.type === 'processing') {
63
+ const procIdx = processingShardIndex(jobId);
64
+ job = await withWriteLock(ctx.processingLocks[procIdx], () => {
65
+ const claimed = ctx.processingShards[procIdx].get(jobId) ?? null;
66
+ if (claimed) {
67
+ ctx.processingShards[procIdx].delete(jobId);
68
+ releaseClaimedJobOwnership(jobId, ctx);
69
+ }
70
+ return claimed;
71
+ });
72
+ }
73
+ if (!job)
74
+ return false;
75
+ const validJob = job;
76
+ const idx = shardIndex(validJob.queue);
77
+ const fromProcessing = location.type === 'processing';
78
+ const entry = await withWriteLock(ctx.shardLocks[idx], () => {
79
+ const shard = ctx.shards[idx];
80
+ if (fromProcessing) {
81
+ shard.releaseJobResources(validJob.queue, validJob.uniqueKey, validJob.groupId);
82
+ shard.notify(validJob.queue);
83
+ }
84
+ else if (validJob.uniqueKey) {
85
+ shard.releaseUniqueKey(validJob.queue, validJob.uniqueKey);
86
+ }
87
+ const dlqEntry = shard.addToDlq(validJob);
88
+ ctx.jobIndex.set(jobId, { type: 'dlq', queueName: validJob.queue });
89
+ return dlqEntry;
90
+ });
91
+ ctx.storage?.saveDlqEntry(entry);
92
+ ctx.storage?.deleteJob(jobId);
93
+ return true;
94
+ }
@@ -7,6 +7,7 @@
7
7
  */
8
8
  import { shardIndex, processingShardIndex } from '../../shared/hash';
9
9
  import { withWriteLock } from '../../shared/lock';
10
+ import { releaseClaimedJobOwnership } from './jobClaim';
10
11
  /** Move active job back to waiting */
11
12
  export async function moveActiveToWait(jobId, ctx) {
12
13
  const location = ctx.jobIndex.get(jobId);
@@ -17,6 +18,7 @@ export async function moveActiveToWait(jobId, ctx) {
17
18
  const job = ctx.processingShards[procIdx].get(jobId);
18
19
  if (job) {
19
20
  ctx.processingShards[procIdx].delete(jobId);
21
+ releaseClaimedJobOwnership(jobId, ctx);
20
22
  }
21
23
  return job;
22
24
  });
@@ -73,6 +75,7 @@ export async function moveToWaitingChildren(jobId, ctx) {
73
75
  const job = ctx.processingShards[procIdx].get(jobId);
74
76
  if (job) {
75
77
  ctx.processingShards[procIdx].delete(jobId);
78
+ releaseClaimedJobOwnership(jobId, ctx);
76
79
  }
77
80
  return job;
78
81
  });
@@ -35,6 +35,11 @@ function tryDequeueNextJob(shard, queue, now, ctx) {
35
35
  // Expired entries are not parked: they are genuinely removed from the
36
36
  // logical queue and therefore update counters/index exactly once.
37
37
  if (isExpired(job, now)) {
38
+ // Delete persistence before mutating the in-memory indexes. deleteJob
39
+ // also removes a pending buffered INSERT, so the expired job cannot be
40
+ // flushed after this pull or resurrect during crash recovery. This is
41
+ // synchronous and therefore keeps the shard critical section await-free.
42
+ ctx.storage?.deleteJob(job.id);
38
43
  q.pop();
39
44
  shard.decrementQueued(job.id);
40
45
  ctx.jobIndex.delete(job.id);
@@ -2,80 +2,13 @@
2
2
  * Push Operations
3
3
  * Job push and batch push logic
4
4
  */
5
- import { createJob, generateJobId, jobId, } from '../../domain/types/job';
6
- import { withWriteLock } from '../../shared/lock';
5
+ import { createJob } from '../../domain/types/job';
7
6
  import { shardIndex } from '../../shared/hash';
8
7
  import { latencyTracker } from '../latencyTracker';
9
8
  import { throughputTracker } from '../throughputTracker';
10
- /**
11
- * Handle custom ID idempotency check
12
- * Returns existing job if found, or new ID to use
13
- */
14
- function handleCustomId(input, shard, ctx) {
15
- if (!input.customId) {
16
- return { skip: false, id: generateJobId() };
17
- }
18
- const id = jobId(input.customId);
19
- const existing = ctx.customIdMap.get(input.customId);
20
- // Re-adding an existing custom id while its job is UNFINISHED is an idempotent
21
- // no-op (BullMQ parity): return the existing job, never a second insert of the
22
- // same deterministic id (which would collide on the jobs.id PRIMARY KEY). Three
23
- // unfinished states are covered:
24
- // • waiting/delayed/prioritized — live in the priority queue → return the job.
25
- // • waiting-children — tracked under 'queue' but held in shard.waitingDeps (its
26
- // row is already persisted) → idempotent skip via the existing id.
27
- // • active (processing) — popped from the queue, still running on a worker; its
28
- // row survives on disk → idempotent skip via the existing id.
29
- // PushContext has no processingShards, so for the latter two we cannot fetch the
30
- // live Job here; pushJob rebuilds a placeholder { ...job, id } after createJob,
31
- // exactly like handleDeduplication's active path. Completed (#92) and DLQ jobs
32
- // are terminal and fall through to the reuse path below.
33
- if (existing && !ctx.completedJobs.has(id)) {
34
- const location = ctx.jobIndex.get(existing);
35
- if (location?.type === 'queue') {
36
- const existingJob = shard.getQueue(location.queueName).find(existing);
37
- if (existingJob) {
38
- return { skip: true, existingJob };
39
- }
40
- if (shard.waitingDeps.has(existing)) {
41
- return { skip: true, existingId: id };
42
- }
43
- }
44
- else if (location?.type === 'processing') {
45
- return { skip: true, existingId: id };
46
- }
47
- }
48
- // Reuse path: the id is free in the queue (no mapping, or the prior job is
49
- // processing/completed). If the prior job COMPLETED, its row survives on disk
50
- // (markCompleted does an UPDATE, not a DELETE) and it is still in completedJobs.
51
- // Reusing the same deterministic id would then (a) make getJobState return
52
- // 'completed' for the brand-new job and (b) collide on the `jobs.id` PRIMARY KEY
53
- // at flush time. Evict the stale completed job so the reused id starts fresh as
54
- // 'waiting' (#92). Checked regardless of customIdMap state — the mapping may have
55
- // been cleared on completion, which would otherwise skip this path entirely.
56
- if (ctx.completedJobs.has(id)) {
57
- ctx.completedJobs.delete(id);
58
- ctx.completedJobsData.delete(id);
59
- ctx.jobResults.delete(id);
60
- ctx.jobIndex.delete(id);
61
- ctx.storage?.deleteJob(id); // removes the surviving row + result + any buffered insert
62
- }
63
- // NOTE on ORPHAN rows: a durable jobs row can outlive its in-memory tracking when
64
- // obliterate() (fire-and-forget void over TCP) or a buffer flush races an in-flight
65
- // durable insert. Reaching the reuse path with such a stale row no longer throws —
66
- // the insert statements use ON CONFLICT(id) DO UPDATE (upsert), so the orphan is
67
- // overwritten in place at insert time with ZERO per-push cost. We deliberately do
68
- // NOT pre-delete here: deleteJob() is a synchronous DELETE + O(buffer) scan inside
69
- // the shard lock and would halve customId push/addBulk throughput.
70
- // A recycled custom id may carry a stale timeout marker from a prior job (which
71
- // may have DLQ'd, so it is NOT in completedJobs above). Clear it so the new
72
- // job's stall-retry recovery is not wrongly discarded — otherwise the
73
- // timeout-resurrection guard would reintroduce #33/#75 duplicate execution for
74
- // reused ids.
75
- ctx.timedOutJobs?.delete(id);
76
- ctx.customIdMap.set(input.customId, id);
77
- return { skip: false, id };
78
- }
9
+ import { handleCustomId } from './customId';
10
+ import { insertJobToShard } from './pushInsert';
11
+ import { withPushWriteLocks } from './pushLocks';
79
12
  /**
80
13
  * Handle unique key deduplication
81
14
  * Returns existing job ID if duplicate found and should skip, or allows insert
@@ -143,34 +76,6 @@ function handleDeduplication(job, input, queue, shard, ctx) {
143
76
  shard.registerUniqueKeyWithTtl(queue, job.uniqueKey, job.id, input.dedup?.ttl);
144
77
  return { skip: false };
145
78
  }
146
- /**
147
- * Insert job into shard (queue or waitingDeps)
148
- */
149
- function insertJobToShard(job, queue, shard, shardIdx, ctx) {
150
- const hasDeps = job.dependsOn.length > 0;
151
- const needsWaiting = hasDeps &&
152
- !job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
153
- const now = Date.now();
154
- if (needsWaiting) {
155
- shard.waitingDeps.set(job.id, job);
156
- shard.registerDependencies(job.id, job.dependsOn);
157
- job.timeline.push({ state: 'waiting-children', timestamp: now });
158
- ctx.dashboardEmit?.('job:waiting-children', {
159
- jobId: String(job.id),
160
- queue,
161
- dependsOn: job.dependsOn.map(String),
162
- });
163
- }
164
- else {
165
- shard.getQueue(queue).push(job);
166
- const isDelayed = job.runAt > now;
167
- shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
168
- // Timeline: initial state based on scheduling and priority
169
- const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
170
- job.timeline.push({ state, timestamp: now });
171
- }
172
- ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
173
- }
174
79
  /**
175
80
  * Push a single job to queue
176
81
  * NOTE: customId check happens INSIDE lock to prevent race conditions
@@ -180,10 +85,10 @@ export async function pushJob(queue, input, ctx) {
180
85
  const idx = shardIndex(queue);
181
86
  const now = Date.now();
182
87
  let result;
183
- await withWriteLock(ctx.shardLocks[idx], () => {
88
+ await withPushWriteLocks(queue, [input], ctx, (lockedShardIndexes) => {
184
89
  const shard = ctx.shards[idx];
185
90
  // Check custom ID idempotency INSIDE lock to prevent race conditions
186
- const customIdResult = handleCustomId(input, shard, ctx);
91
+ const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
187
92
  if (customIdResult.skip) {
188
93
  // Idempotent re-add: return the live queued job if we have it, otherwise
189
94
  // (active / waiting-children) a placeholder carrying the existing id so the
@@ -247,11 +152,11 @@ export async function pushJobBatch(queue, inputs, ctx) {
247
152
  // path), exactly like a single durable push — otherwise addBulk silently
248
153
  // downgrades the documented "durable" guarantee.
249
154
  const durableJobs = [];
250
- await withWriteLock(ctx.shardLocks[idx], () => {
155
+ await withPushWriteLocks(queue, inputs, ctx, (lockedShardIndexes) => {
251
156
  const shard = ctx.shards[idx];
252
157
  for (const input of inputs) {
253
158
  // Check custom ID idempotency
254
- const customIdResult = handleCustomId(input, shard, ctx);
159
+ const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
255
160
  if (customIdResult.skip) {
256
161
  // Idempotent re-add (queued, active, or waiting-children) — no insert.
257
162
  resultIds.push('existingJob' in customIdResult
@@ -0,0 +1,12 @@
1
+ import type { Shard } from '../../domain/queue/shard';
2
+ import type { Job, JobId } from '../../domain/types/job';
3
+ import type { JobLocation } from '../../domain/types/queue';
4
+ import type { SetLike } from '../../shared/lru';
5
+ export interface PushInsertContext {
6
+ completedJobs: SetLike<JobId>;
7
+ depCompletions?: SetLike<JobId>;
8
+ jobIndex: Map<JobId, JobLocation>;
9
+ dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
10
+ }
11
+ /** Insert a new job into the runnable heap or dependency wait set. */
12
+ export declare function insertJobToShard(job: Job, queue: string, shard: Shard, shardIdx: number, ctx: PushInsertContext): void;
@@ -0,0 +1,25 @@
1
+ /** Insert a new job into the runnable heap or dependency wait set. */
2
+ export function insertJobToShard(job, queue, shard, shardIdx, ctx) {
3
+ const hasDeps = job.dependsOn.length > 0;
4
+ const needsWaiting = hasDeps &&
5
+ !job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
6
+ const now = Date.now();
7
+ if (needsWaiting) {
8
+ shard.waitingDeps.set(job.id, job);
9
+ shard.registerDependencies(job.id, job.dependsOn);
10
+ job.timeline.push({ state: 'waiting-children', timestamp: now });
11
+ ctx.dashboardEmit?.('job:waiting-children', {
12
+ jobId: String(job.id),
13
+ queue,
14
+ dependsOn: job.dependsOn.map(String),
15
+ });
16
+ }
17
+ else {
18
+ shard.getQueue(queue).push(job);
19
+ const isDelayed = job.runAt > now;
20
+ shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
21
+ const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
22
+ job.timeline.push({ state, timestamp: now });
23
+ }
24
+ ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
25
+ }
@@ -0,0 +1,14 @@
1
+ import type { JobId, JobInput } from '../../domain/types/job';
2
+ import type { JobLocation } from '../../domain/types/queue';
3
+ import type { RWLock } from '../../shared/lock';
4
+ interface PushLockContext {
5
+ shardLocks: RWLock[];
6
+ jobIndex: Map<JobId, JobLocation>;
7
+ }
8
+ /**
9
+ * Lock the target shard plus any shard owning a terminal custom-ID generation.
10
+ * The required set is rechecked after acquisition because jobIndex may change
11
+ * while an earlier lock is awaited.
12
+ */
13
+ export declare function withPushWriteLocks<T>(queue: string, inputs: readonly JobInput[], ctx: PushLockContext, fn: (lockedShardIndexes: ReadonlySet<number>) => T): Promise<T>;
14
+ export {};
@@ -0,0 +1,43 @@
1
+ import { jobId } from '../../domain/types/job';
2
+ import { shardIndex } from '../../shared/hash';
3
+ function requiredShardIndexes(queue, inputs, jobIndex) {
4
+ const indexes = new Set([shardIndex(queue)]);
5
+ for (const input of inputs) {
6
+ if (!input.customId)
7
+ continue;
8
+ const location = jobIndex.get(jobId(input.customId));
9
+ if (location?.type === 'queue') {
10
+ indexes.add(location.shardIdx);
11
+ }
12
+ else if (location?.type === 'dlq') {
13
+ indexes.add(shardIndex(location.queueName));
14
+ }
15
+ }
16
+ return [...indexes].sort((a, b) => a - b);
17
+ }
18
+ /**
19
+ * Lock the target shard plus any shard owning a terminal custom-ID generation.
20
+ * The required set is rechecked after acquisition because jobIndex may change
21
+ * while an earlier lock is awaited.
22
+ */
23
+ export async function withPushWriteLocks(queue, inputs, ctx, fn) {
24
+ while (true) {
25
+ const indexes = requiredShardIndexes(queue, inputs, ctx.jobIndex);
26
+ const guards = [];
27
+ try {
28
+ for (const index of indexes) {
29
+ guards.push(await ctx.shardLocks[index].acquireWrite());
30
+ }
31
+ const locked = new Set(indexes);
32
+ const current = requiredShardIndexes(queue, inputs, ctx.jobIndex);
33
+ if (current.some((index) => !locked.has(index)))
34
+ continue;
35
+ return fn(locked);
36
+ }
37
+ finally {
38
+ for (let i = guards.length - 1; i >= 0; i--) {
39
+ guards[i].release();
40
+ }
41
+ }
42
+ }
43
+ }
@@ -30,7 +30,7 @@ export declare function isQueuePaused(queue: string, ctx: QueueControlContext):
30
30
  /** Drain all waiting jobs from queue */
31
31
  export declare function drainQueue(queue: string, ctx: QueueControlContext): number;
32
32
  /** Remove all queue data */
33
- export declare function obliterateQueue(queue: string, ctx: QueueControlContext): void;
33
+ export declare function obliterateQueue(queue: string, ctx: QueueControlContext): JobId[];
34
34
  /** List all queue names */
35
35
  export declare function listAllQueues(ctx: QueueControlContext): string[];
36
36
  /**
@@ -35,7 +35,7 @@ export function drainQueue(queue, ctx) {
35
35
  /** Remove all queue data */
36
36
  export function obliterateQueue(queue, ctx) {
37
37
  const idx = shardIndex(queue);
38
- ctx.shards[idx].obliterate(queue);
38
+ return ctx.shards[idx].obliterate(queue);
39
39
  }
40
40
  /** List all queue names */
41
41
  export function listAllQueues(ctx) {
@@ -6,7 +6,7 @@ import type { Job, JobId, JobInput, JobLock, LockToken } from '../domain/types/j
6
6
  import type { JobLocation, JobEvent } from '../domain/types/queue';
7
7
  import type { CronJob, CronJobInput } from '../domain/types/cron';
8
8
  import type { JobLogEntry, CreateWorkerOptions } from '../domain/types/worker';
9
- import type { StallConfig } from '../domain/types/stall';
9
+ import { type StallConfig } from '../domain/types/stall';
10
10
  import type { DlqConfig, DlqEntry, DlqFilter, DlqStats } from '../domain/types/dlq';
11
11
  import { Shard } from '../domain/queue/shard';
12
12
  import { WebhookManager } from './webhookManager';
@@ -196,7 +196,7 @@ export declare class QueueManager {
196
196
  retryDlq(queue: string, jobId?: JobId, limit?: number): number;
197
197
  purgeDlq(queue: string): number;
198
198
  retryCompleted(queue: string, jobId?: JobId): number;
199
- setRateLimit(queue: string, limit: number): void;
199
+ setRateLimit(queue: string, limit: number, durationMs?: number, ttlMs?: number): void;
200
200
  clearRateLimit(queue: string): void;
201
201
  setConcurrency(queue: string, limit: number): void;
202
202
  clearConcurrency(queue: string): void;
@@ -3,6 +3,7 @@
3
3
  * Core orchestrator for all queue operations
4
4
  */
5
5
  import { DEFAULT_LOCK_TTL } from '../domain/types/job';
6
+ import { DEFAULT_STALL_CONFIG } from '../domain/types/stall';
6
7
  import { Shard } from '../domain/queue/shard';
7
8
  import { SqliteStorage } from '../infrastructure/persistence/sqlite';
8
9
  import { CronScheduler } from '../infrastructure/scheduler/cronScheduler';
@@ -741,12 +742,12 @@ export class QueueManager {
741
742
  return count;
742
743
  }
743
744
  obliterate(queue) {
744
- queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
745
+ const shardJobs = queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
745
746
  dlqOps.purgeDlqJobs(queue, this.contextFactory.getDlqContext());
746
- // obliterateQueue() clears the waiting/delayed shard only. Active jobs in
747
- // processingShards, plus completed/result/log/lock state in global indexes,
748
- // plus SQLite rows, all survive unless we purge them here.
749
- const toDrop = new Set();
747
+ // obliterateQueue() returns every queued, DLQ, and dependency-gated job it
748
+ // removed. Active jobs in processingShards, plus completed/result/log/lock
749
+ // state in global indexes, still need to be discovered and purged here.
750
+ const toDrop = new Set(shardJobs);
750
751
  for (const [jid, loc] of this.jobIndex) {
751
752
  if (loc.type === 'processing') {
752
753
  const job = this.processingShards[loc.shardIdx]?.get(jid);
@@ -860,8 +861,8 @@ export class QueueManager {
860
861
  return dlqOps.retryCompletedJobs(queue, this.contextFactory.getRetryCompletedContext(), jobId);
861
862
  }
862
863
  // ============ Rate Limiting ============
863
- setRateLimit(queue, limit) {
864
- this.shards[shardIndex(queue)].setRateLimit(queue, limit);
864
+ setRateLimit(queue, limit, durationMs, ttlMs) {
865
+ this.shards[shardIndex(queue)].setRateLimit(queue, limit, durationMs, ttlMs);
865
866
  this.persistQueueState(queue);
866
867
  }
867
868
  clearRateLimit(queue) {
@@ -885,19 +886,37 @@ export class QueueManager {
885
886
  if (!this.storage)
886
887
  return;
887
888
  const state = this.shards[shardIndex(queue)].getState(queue);
889
+ const stallConfig = this.shards[shardIndex(queue)].getStallConfig(queue);
890
+ const hasCustomStallConfig = stallConfig.enabled !== DEFAULT_STALL_CONFIG.enabled ||
891
+ stallConfig.stallInterval !== DEFAULT_STALL_CONFIG.stallInterval ||
892
+ stallConfig.maxStalls !== DEFAULT_STALL_CONFIG.maxStalls ||
893
+ stallConfig.gracePeriod !== DEFAULT_STALL_CONFIG.gracePeriod;
888
894
  // When control-state returns fully to default (not paused, no limits), drop
889
895
  // the row instead of persisting an all-default placeholder. Keeps the table
890
896
  // free of noise rows for ephemeral queues that only ever call resume/clear*,
891
897
  // and recovers identically (absent row → default state).
892
- if (!state.paused && state.rateLimit === null && state.concurrencyLimit === null) {
898
+ if (!state.paused &&
899
+ state.rateLimit === null &&
900
+ state.concurrencyLimit === null &&
901
+ !hasCustomStallConfig) {
893
902
  this.storage.deleteQueueState(queue);
894
903
  return;
895
904
  }
896
- this.storage.saveQueueState(queue, state.paused, state.rateLimit, state.concurrencyLimit);
905
+ this.storage.saveQueueState(queue, {
906
+ paused: state.paused,
907
+ rateLimit: state.rateLimit,
908
+ concurrencyLimit: state.concurrencyLimit,
909
+ rateLimitDuration: state.rateLimitDuration,
910
+ rateLimitExpiresAt: state.rateLimitExpiresAt,
911
+ stallConfig,
912
+ });
897
913
  }
898
914
  /** Get rate limit and concurrency limit for a queue */
899
915
  getQueueLimits(queue) {
900
- const state = this.shards[shardIndex(queue)].getState(queue);
916
+ const shard = this.shards[shardIndex(queue)];
917
+ // Lazy TTL expiry so reads never report a limit that no longer throttles.
918
+ shard.expireRateLimitIfNeeded(queue);
919
+ const state = shard.getState(queue);
901
920
  return { rateLimit: state.rateLimit, concurrencyLimit: state.concurrencyLimit };
902
921
  }
903
922
  /** Get all job results (for cloud telemetry) */
@@ -921,6 +940,7 @@ export class QueueManager {
921
940
  // ============ Stall & DLQ Config ============
922
941
  setStallConfig(queue, config) {
923
942
  this.shards[shardIndex(queue)].setStallConfig(queue, config);
943
+ this.persistQueueState(queue);
924
944
  }
925
945
  getStallConfig(queue) {
926
946
  return this.shards[shardIndex(queue)].getStallConfig(queue);
@@ -66,6 +66,7 @@ async function handleStalledJob(job, action, ctx) {
66
66
  // Lock order: shardLocks BEFORE processingLocks (per lock hierarchy in CLAUDE.md)
67
67
  // Broadcast events AFTER verifying job is still stalled to avoid false positives
68
68
  let handled = false;
69
+ let handledAction = action;
69
70
  await withWriteLock(ctx.shardLocks[idx], async () => {
70
71
  await withWriteLock(ctx.processingLocks[procIdx], () => {
71
72
  // Verify job is still in processing (might have been handled already)
@@ -73,8 +74,10 @@ async function handleStalledJob(job, action, ctx) {
73
74
  return; // Job already completed, don't broadcast stalled event
74
75
  }
75
76
  const shard = ctx.shards[idx];
76
- if (action === "move_to_dlq" /* StallAction.MoveToDlq */) {
77
- moveStalliedJobToDlq(job, ctx, shard, procIdx, idx);
77
+ const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
78
+ if (action === "move_to_dlq" /* StallAction.MoveToDlq */ || attemptsExhausted) {
79
+ handledAction = "move_to_dlq" /* StallAction.MoveToDlq */;
80
+ moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
78
81
  }
79
82
  else {
80
83
  retryStalliedJob(job, ctx, shard, procIdx, idx);
@@ -87,23 +90,23 @@ async function handleStalledJob(job, action, ctx) {
87
90
  ctx.dashboardEmit?.('job:stalled', {
88
91
  jobId: String(job.id),
89
92
  queue: job.queue,
90
- stallCount: job.stallCount + 1,
91
- action,
93
+ stallCount: job.stallCount,
94
+ action: handledAction,
92
95
  });
93
96
  ctx.eventsManager.broadcast({
94
97
  eventType: "stalled" /* EventType.Stalled */,
95
98
  queue: job.queue,
96
99
  jobId: job.id,
97
100
  timestamp: Date.now(),
98
- data: { stallCount: job.stallCount + 1, action },
101
+ data: { stallCount: job.stallCount, action: handledAction },
99
102
  });
100
103
  void ctx.webhookManager.trigger('stalled', String(job.id), job.queue, {
101
- data: { stallCount: job.stallCount + 1, action },
104
+ data: { stallCount: job.stallCount, action: handledAction },
102
105
  });
103
106
  }
104
107
  }
105
108
  /** Move stalled job to DLQ */
106
- function moveStalliedJobToDlq(job, ctx, shard, procIdx, _idx) {
109
+ function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
107
110
  ctx.processingShards[procIdx].delete(job.id);
108
111
  shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
109
112
  // Discard cron jobs with preventOverlap instead of sending to DLQ (#73).
@@ -112,7 +115,13 @@ function moveStalliedJobToDlq(job, ctx, shard, procIdx, _idx) {
112
115
  ctx.storage?.deleteJob(job.id);
113
116
  return;
114
117
  }
115
- const entry = shard.addToDlq(job, "stalled" /* FailureReason.Stalled */, `Job stalled ${job.stallCount + 1} times`);
118
+ incrementStallCount(job);
119
+ job.attempts++;
120
+ job.startedAt = null;
121
+ job.lastHeartbeat = Date.now();
122
+ const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */ : "stalled" /* FailureReason.Stalled */, attemptsExhausted
123
+ ? `Job stalled at max attempts (${job.maxAttempts})`
124
+ : `Job stalled ${job.stallCount} times`);
116
125
  ctx.jobIndex.set(job.id, { type: 'dlq', queueName: job.queue });
117
126
  ctx.storage?.saveDlqEntry(entry);
118
127
  ctx.storage?.deleteJob(job.id);
@@ -3,7 +3,8 @@
3
3
  * Wraps dlqOps for Queue class usage
4
4
  */
5
5
  import type { TcpConnectionPool } from '../tcpPool';
6
- import type { DlqConfig, DlqEntry, DlqStats, DlqFilter } from '../types';
6
+ import type { Job, DlqConfig, DlqEntry, DlqStats, DlqFilter } from '../types';
7
+ import { type SimpleJobContext } from './jobProxy';
7
8
  interface DlqContext {
8
9
  name: string;
9
10
  embedded: boolean;
@@ -11,20 +12,41 @@ interface DlqContext {
11
12
  }
12
13
  /** Set DLQ configuration */
13
14
  export declare function setDlqConfig(ctx: DlqContext, config: Partial<DlqConfig>): void;
15
+ /** Set DLQ configuration and resolve once the server has applied it. */
16
+ export declare function setDlqConfigAsync(ctx: DlqContext, config: Partial<DlqConfig>): Promise<void>;
14
17
  /** Get DLQ configuration */
15
18
  export declare function getDlqConfig(ctx: DlqContext): DlqConfig;
16
19
  /** Get DLQ configuration (async, works in TCP mode) */
17
20
  export declare function getDlqConfigAsync(ctx: DlqContext): Promise<DlqConfig>;
18
21
  /** Get DLQ entries */
19
22
  export declare function getDlq<T>(ctx: DlqContext, filter?: DlqFilter): DlqEntry<T>[];
23
+ type DlqQueryContext = DlqContext & Pick<SimpleJobContext, 'getJobState' | 'removeAsync' | 'retryJob' | 'getChildrenValues'>;
24
+ /**
25
+ * Get the dead jobs in the DLQ, working over TCP too (wire command `Dlq`).
26
+ * Returns public Job objects; the wire does not carry DlqEntry metadata
27
+ * (reason, enteredAt, attempts) — that stays embedded-only via getDlq().
28
+ */
29
+ export declare function getDlqJobsAsync<T>(ctx: DlqQueryContext, count?: number): Promise<Job<T>[]>;
20
30
  /** Get DLQ stats */
21
31
  export declare function getDlqStats(ctx: DlqContext): DlqStats;
22
32
  /** Retry DLQ entries */
23
33
  export declare function retryDlq(ctx: DlqContext, id?: string): number;
34
+ /**
35
+ * Retry DLQ entries and resolve with the retried count once the server has
36
+ * processed it. The fire-and-forget retryDlq() always returns 0 over TCP and
37
+ * discards the server's count.
38
+ */
39
+ export declare function retryDlqAsync(ctx: DlqContext, id?: string): Promise<number>;
24
40
  /** Retry DLQ entries by filter */
25
41
  export declare function retryDlqByFilter(ctx: DlqContext, filter: DlqFilter): number;
26
42
  /** Purge DLQ */
27
43
  export declare function purgeDlq(ctx: DlqContext): number;
44
+ /**
45
+ * Purge the DLQ and resolve with the purged count once the server has
46
+ * processed it. The fire-and-forget purgeDlq() always returns 0 over TCP;
47
+ * purge-then-assert-empty patterns need this variant.
48
+ */
49
+ export declare function purgeDlqAsync(ctx: DlqContext): Promise<number>;
28
50
  /** Retry completed job */
29
51
  export declare function retryCompleted(ctx: DlqContext, id?: string): number;
30
52
  /** Retry completed job (async) */