bunqueue 2.8.34 → 2.8.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/application/backgroundTasks.js +37 -7
- package/dist/application/contextFactory.js +5 -0
- package/dist/application/dlqManager.d.ts +6 -0
- package/dist/application/dlqManager.js +35 -14
- package/dist/application/lockManager.js +22 -8
- package/dist/application/operations/customId.d.ts +31 -0
- package/dist/application/operations/customId.js +59 -0
- package/dist/application/operations/jobClaim.d.ts +17 -0
- package/dist/application/operations/jobClaim.js +20 -0
- package/dist/application/operations/jobManagement.d.ts +4 -5
- package/dist/application/operations/jobManagement.js +15 -116
- package/dist/application/operations/jobMoveOperations.d.ts +9 -0
- package/dist/application/operations/jobMoveOperations.js +94 -0
- package/dist/application/operations/jobStateTransitions.js +3 -0
- package/dist/application/operations/pull.js +5 -0
- package/dist/application/operations/push.js +8 -103
- package/dist/application/operations/pushInsert.d.ts +12 -0
- package/dist/application/operations/pushInsert.js +25 -0
- package/dist/application/operations/pushLocks.d.ts +14 -0
- package/dist/application/operations/pushLocks.js +43 -0
- package/dist/application/operations/queueControl.d.ts +1 -1
- package/dist/application/operations/queueControl.js +1 -1
- package/dist/application/queueManager.d.ts +2 -2
- package/dist/application/queueManager.js +30 -10
- package/dist/application/stallDetection.js +17 -8
- package/dist/client/queue/dlq.d.ts +23 -1
- package/dist/client/queue/dlq.js +75 -0
- package/dist/client/queue/helpers.js +2 -0
- package/dist/client/queue/jobProxy.d.ts +1 -1
- package/dist/client/queue/operations/control.d.ts +17 -0
- package/dist/client/queue/operations/control.js +41 -0
- package/dist/client/queue/operations/counts.d.ts +1 -0
- package/dist/client/queue/operations/counts.js +3 -0
- package/dist/client/queue/queue.d.ts +13 -0
- package/dist/client/queue/queue.js +39 -0
- package/dist/client/queue/rateLimit.d.ts +18 -3
- package/dist/client/queue/rateLimit.js +45 -10
- package/dist/client/queue/stall.d.ts +2 -0
- package/dist/client/queue/stall.js +12 -0
- package/dist/domain/queue/dependencyTracker.d.ts +5 -0
- package/dist/domain/queue/dependencyTracker.js +20 -0
- package/dist/domain/queue/limiterManager.d.ts +14 -2
- package/dist/domain/queue/limiterManager.js +32 -5
- package/dist/domain/queue/shard.d.ts +3 -2
- package/dist/domain/queue/shard.js +11 -2
- package/dist/domain/types/command.d.ts +4 -0
- package/dist/domain/types/queue.d.ts +4 -0
- package/dist/domain/types/queue.js +2 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +38 -2
- package/dist/infrastructure/persistence/sqlite.d.ts +20 -1
- package/dist/infrastructure/persistence/sqlite.js +62 -6
- package/dist/infrastructure/persistence/sqliteBatch.js +8 -6
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +9 -1
- package/dist/infrastructure/persistence/statements.d.ts +8 -1
- package/dist/infrastructure/persistence/statements.js +7 -5
- package/dist/infrastructure/scheduler/cronScheduler.d.ts +5 -0
- package/dist/infrastructure/scheduler/cronScheduler.js +39 -13
- package/dist/infrastructure/server/handlers/advanced.js +11 -2
- package/dist/infrastructure/server/httpRouteQueueConfig.js +2 -0
- package/package.json +4 -1
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job move operations that claim active jobs for delayed or DLQ state.
|
|
3
|
+
*/
|
|
4
|
+
import { processingShardIndex, shardIndex } from '../../shared/hash';
|
|
5
|
+
import { withWriteLock } from '../../shared/lock';
|
|
6
|
+
import { releaseClaimedJobOwnership } from './jobClaim';
|
|
7
|
+
/** Move active job back to delayed. */
|
|
8
|
+
export async function moveJobToDelayed(jobId, delay, ctx) {
|
|
9
|
+
const location = ctx.jobIndex.get(jobId);
|
|
10
|
+
if (location?.type !== 'processing')
|
|
11
|
+
return false;
|
|
12
|
+
const procIdx = processingShardIndex(jobId);
|
|
13
|
+
const job = await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
14
|
+
const claimed = ctx.processingShards[procIdx].get(jobId);
|
|
15
|
+
if (claimed) {
|
|
16
|
+
ctx.processingShards[procIdx].delete(jobId);
|
|
17
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
18
|
+
}
|
|
19
|
+
return claimed;
|
|
20
|
+
});
|
|
21
|
+
if (!job)
|
|
22
|
+
return false;
|
|
23
|
+
const now = Date.now();
|
|
24
|
+
job.runAt = now + delay;
|
|
25
|
+
job.startedAt = null;
|
|
26
|
+
const idx = shardIndex(job.queue);
|
|
27
|
+
const queueName = job.queue;
|
|
28
|
+
await withWriteLock(ctx.shardLocks[idx], () => {
|
|
29
|
+
const shard = ctx.shards[idx];
|
|
30
|
+
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
|
|
31
|
+
shard.getQueue(job.queue).push(job);
|
|
32
|
+
const isDelayed = job.runAt > now;
|
|
33
|
+
shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
34
|
+
ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
|
|
35
|
+
shard.notify(job.queue);
|
|
36
|
+
});
|
|
37
|
+
ctx.storage?.updateRunAt(jobId, job.runAt);
|
|
38
|
+
ctx.eventsManager.broadcast({
|
|
39
|
+
eventType: "delayed" /* EventType.Delayed */,
|
|
40
|
+
jobId,
|
|
41
|
+
queue: queueName,
|
|
42
|
+
timestamp: Date.now(),
|
|
43
|
+
delay,
|
|
44
|
+
});
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
/** Discard a waiting or active job to the DLQ. */
|
|
48
|
+
export async function discardJob(jobId, ctx) {
|
|
49
|
+
const location = ctx.jobIndex.get(jobId);
|
|
50
|
+
if (!location)
|
|
51
|
+
return false;
|
|
52
|
+
let job = null;
|
|
53
|
+
if (location.type === 'queue') {
|
|
54
|
+
job = await withWriteLock(ctx.shardLocks[location.shardIdx], () => {
|
|
55
|
+
const shard = ctx.shards[location.shardIdx];
|
|
56
|
+
const removed = shard.getQueue(location.queueName).remove(jobId);
|
|
57
|
+
if (removed)
|
|
58
|
+
shard.decrementQueued(jobId);
|
|
59
|
+
return removed;
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
else if (location.type === 'processing') {
|
|
63
|
+
const procIdx = processingShardIndex(jobId);
|
|
64
|
+
job = await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
65
|
+
const claimed = ctx.processingShards[procIdx].get(jobId) ?? null;
|
|
66
|
+
if (claimed) {
|
|
67
|
+
ctx.processingShards[procIdx].delete(jobId);
|
|
68
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
69
|
+
}
|
|
70
|
+
return claimed;
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
if (!job)
|
|
74
|
+
return false;
|
|
75
|
+
const validJob = job;
|
|
76
|
+
const idx = shardIndex(validJob.queue);
|
|
77
|
+
const fromProcessing = location.type === 'processing';
|
|
78
|
+
const entry = await withWriteLock(ctx.shardLocks[idx], () => {
|
|
79
|
+
const shard = ctx.shards[idx];
|
|
80
|
+
if (fromProcessing) {
|
|
81
|
+
shard.releaseJobResources(validJob.queue, validJob.uniqueKey, validJob.groupId);
|
|
82
|
+
shard.notify(validJob.queue);
|
|
83
|
+
}
|
|
84
|
+
else if (validJob.uniqueKey) {
|
|
85
|
+
shard.releaseUniqueKey(validJob.queue, validJob.uniqueKey);
|
|
86
|
+
}
|
|
87
|
+
const dlqEntry = shard.addToDlq(validJob);
|
|
88
|
+
ctx.jobIndex.set(jobId, { type: 'dlq', queueName: validJob.queue });
|
|
89
|
+
return dlqEntry;
|
|
90
|
+
});
|
|
91
|
+
ctx.storage?.saveDlqEntry(entry);
|
|
92
|
+
ctx.storage?.deleteJob(jobId);
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { shardIndex, processingShardIndex } from '../../shared/hash';
|
|
9
9
|
import { withWriteLock } from '../../shared/lock';
|
|
10
|
+
import { releaseClaimedJobOwnership } from './jobClaim';
|
|
10
11
|
/** Move active job back to waiting */
|
|
11
12
|
export async function moveActiveToWait(jobId, ctx) {
|
|
12
13
|
const location = ctx.jobIndex.get(jobId);
|
|
@@ -17,6 +18,7 @@ export async function moveActiveToWait(jobId, ctx) {
|
|
|
17
18
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
18
19
|
if (job) {
|
|
19
20
|
ctx.processingShards[procIdx].delete(jobId);
|
|
21
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
20
22
|
}
|
|
21
23
|
return job;
|
|
22
24
|
});
|
|
@@ -73,6 +75,7 @@ export async function moveToWaitingChildren(jobId, ctx) {
|
|
|
73
75
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
74
76
|
if (job) {
|
|
75
77
|
ctx.processingShards[procIdx].delete(jobId);
|
|
78
|
+
releaseClaimedJobOwnership(jobId, ctx);
|
|
76
79
|
}
|
|
77
80
|
return job;
|
|
78
81
|
});
|
|
@@ -35,6 +35,11 @@ function tryDequeueNextJob(shard, queue, now, ctx) {
|
|
|
35
35
|
// Expired entries are not parked: they are genuinely removed from the
|
|
36
36
|
// logical queue and therefore update counters/index exactly once.
|
|
37
37
|
if (isExpired(job, now)) {
|
|
38
|
+
// Delete persistence before mutating the in-memory indexes. deleteJob
|
|
39
|
+
// also removes a pending buffered INSERT, so the expired job cannot be
|
|
40
|
+
// flushed after this pull or resurrect during crash recovery. This is
|
|
41
|
+
// synchronous and therefore keeps the shard critical section await-free.
|
|
42
|
+
ctx.storage?.deleteJob(job.id);
|
|
38
43
|
q.pop();
|
|
39
44
|
shard.decrementQueued(job.id);
|
|
40
45
|
ctx.jobIndex.delete(job.id);
|
|
@@ -2,80 +2,13 @@
|
|
|
2
2
|
* Push Operations
|
|
3
3
|
* Job push and batch push logic
|
|
4
4
|
*/
|
|
5
|
-
import { createJob
|
|
6
|
-
import { withWriteLock } from '../../shared/lock';
|
|
5
|
+
import { createJob } from '../../domain/types/job';
|
|
7
6
|
import { shardIndex } from '../../shared/hash';
|
|
8
7
|
import { latencyTracker } from '../latencyTracker';
|
|
9
8
|
import { throughputTracker } from '../throughputTracker';
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
*/
|
|
14
|
-
function handleCustomId(input, shard, ctx) {
|
|
15
|
-
if (!input.customId) {
|
|
16
|
-
return { skip: false, id: generateJobId() };
|
|
17
|
-
}
|
|
18
|
-
const id = jobId(input.customId);
|
|
19
|
-
const existing = ctx.customIdMap.get(input.customId);
|
|
20
|
-
// Re-adding an existing custom id while its job is UNFINISHED is an idempotent
|
|
21
|
-
// no-op (BullMQ parity): return the existing job, never a second insert of the
|
|
22
|
-
// same deterministic id (which would collide on the jobs.id PRIMARY KEY). Three
|
|
23
|
-
// unfinished states are covered:
|
|
24
|
-
// • waiting/delayed/prioritized — live in the priority queue → return the job.
|
|
25
|
-
// • waiting-children — tracked under 'queue' but held in shard.waitingDeps (its
|
|
26
|
-
// row is already persisted) → idempotent skip via the existing id.
|
|
27
|
-
// • active (processing) — popped from the queue, still running on a worker; its
|
|
28
|
-
// row survives on disk → idempotent skip via the existing id.
|
|
29
|
-
// PushContext has no processingShards, so for the latter two we cannot fetch the
|
|
30
|
-
// live Job here; pushJob rebuilds a placeholder { ...job, id } after createJob,
|
|
31
|
-
// exactly like handleDeduplication's active path. Completed (#92) and DLQ jobs
|
|
32
|
-
// are terminal and fall through to the reuse path below.
|
|
33
|
-
if (existing && !ctx.completedJobs.has(id)) {
|
|
34
|
-
const location = ctx.jobIndex.get(existing);
|
|
35
|
-
if (location?.type === 'queue') {
|
|
36
|
-
const existingJob = shard.getQueue(location.queueName).find(existing);
|
|
37
|
-
if (existingJob) {
|
|
38
|
-
return { skip: true, existingJob };
|
|
39
|
-
}
|
|
40
|
-
if (shard.waitingDeps.has(existing)) {
|
|
41
|
-
return { skip: true, existingId: id };
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
else if (location?.type === 'processing') {
|
|
45
|
-
return { skip: true, existingId: id };
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
// Reuse path: the id is free in the queue (no mapping, or the prior job is
|
|
49
|
-
// processing/completed). If the prior job COMPLETED, its row survives on disk
|
|
50
|
-
// (markCompleted does an UPDATE, not a DELETE) and it is still in completedJobs.
|
|
51
|
-
// Reusing the same deterministic id would then (a) make getJobState return
|
|
52
|
-
// 'completed' for the brand-new job and (b) collide on the `jobs.id` PRIMARY KEY
|
|
53
|
-
// at flush time. Evict the stale completed job so the reused id starts fresh as
|
|
54
|
-
// 'waiting' (#92). Checked regardless of customIdMap state — the mapping may have
|
|
55
|
-
// been cleared on completion, which would otherwise skip this path entirely.
|
|
56
|
-
if (ctx.completedJobs.has(id)) {
|
|
57
|
-
ctx.completedJobs.delete(id);
|
|
58
|
-
ctx.completedJobsData.delete(id);
|
|
59
|
-
ctx.jobResults.delete(id);
|
|
60
|
-
ctx.jobIndex.delete(id);
|
|
61
|
-
ctx.storage?.deleteJob(id); // removes the surviving row + result + any buffered insert
|
|
62
|
-
}
|
|
63
|
-
// NOTE on ORPHAN rows: a durable jobs row can outlive its in-memory tracking when
|
|
64
|
-
// obliterate() (fire-and-forget void over TCP) or a buffer flush races an in-flight
|
|
65
|
-
// durable insert. Reaching the reuse path with such a stale row no longer throws —
|
|
66
|
-
// the insert statements use ON CONFLICT(id) DO UPDATE (upsert), so the orphan is
|
|
67
|
-
// overwritten in place at insert time with ZERO per-push cost. We deliberately do
|
|
68
|
-
// NOT pre-delete here: deleteJob() is a synchronous DELETE + O(buffer) scan inside
|
|
69
|
-
// the shard lock and would halve customId push/addBulk throughput.
|
|
70
|
-
// A recycled custom id may carry a stale timeout marker from a prior job (which
|
|
71
|
-
// may have DLQ'd, so it is NOT in completedJobs above). Clear it so the new
|
|
72
|
-
// job's stall-retry recovery is not wrongly discarded — otherwise the
|
|
73
|
-
// timeout-resurrection guard would reintroduce #33/#75 duplicate execution for
|
|
74
|
-
// reused ids.
|
|
75
|
-
ctx.timedOutJobs?.delete(id);
|
|
76
|
-
ctx.customIdMap.set(input.customId, id);
|
|
77
|
-
return { skip: false, id };
|
|
78
|
-
}
|
|
9
|
+
import { handleCustomId } from './customId';
|
|
10
|
+
import { insertJobToShard } from './pushInsert';
|
|
11
|
+
import { withPushWriteLocks } from './pushLocks';
|
|
79
12
|
/**
|
|
80
13
|
* Handle unique key deduplication
|
|
81
14
|
* Returns existing job ID if duplicate found and should skip, or allows insert
|
|
@@ -143,34 +76,6 @@ function handleDeduplication(job, input, queue, shard, ctx) {
|
|
|
143
76
|
shard.registerUniqueKeyWithTtl(queue, job.uniqueKey, job.id, input.dedup?.ttl);
|
|
144
77
|
return { skip: false };
|
|
145
78
|
}
|
|
146
|
-
/**
|
|
147
|
-
* Insert job into shard (queue or waitingDeps)
|
|
148
|
-
*/
|
|
149
|
-
function insertJobToShard(job, queue, shard, shardIdx, ctx) {
|
|
150
|
-
const hasDeps = job.dependsOn.length > 0;
|
|
151
|
-
const needsWaiting = hasDeps &&
|
|
152
|
-
!job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
|
|
153
|
-
const now = Date.now();
|
|
154
|
-
if (needsWaiting) {
|
|
155
|
-
shard.waitingDeps.set(job.id, job);
|
|
156
|
-
shard.registerDependencies(job.id, job.dependsOn);
|
|
157
|
-
job.timeline.push({ state: 'waiting-children', timestamp: now });
|
|
158
|
-
ctx.dashboardEmit?.('job:waiting-children', {
|
|
159
|
-
jobId: String(job.id),
|
|
160
|
-
queue,
|
|
161
|
-
dependsOn: job.dependsOn.map(String),
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
else {
|
|
165
|
-
shard.getQueue(queue).push(job);
|
|
166
|
-
const isDelayed = job.runAt > now;
|
|
167
|
-
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
168
|
-
// Timeline: initial state based on scheduling and priority
|
|
169
|
-
const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
|
|
170
|
-
job.timeline.push({ state, timestamp: now });
|
|
171
|
-
}
|
|
172
|
-
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
|
|
173
|
-
}
|
|
174
79
|
/**
|
|
175
80
|
* Push a single job to queue
|
|
176
81
|
* NOTE: customId check happens INSIDE lock to prevent race conditions
|
|
@@ -180,10 +85,10 @@ export async function pushJob(queue, input, ctx) {
|
|
|
180
85
|
const idx = shardIndex(queue);
|
|
181
86
|
const now = Date.now();
|
|
182
87
|
let result;
|
|
183
|
-
await
|
|
88
|
+
await withPushWriteLocks(queue, [input], ctx, (lockedShardIndexes) => {
|
|
184
89
|
const shard = ctx.shards[idx];
|
|
185
90
|
// Check custom ID idempotency INSIDE lock to prevent race conditions
|
|
186
|
-
const customIdResult = handleCustomId(input,
|
|
91
|
+
const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
|
|
187
92
|
if (customIdResult.skip) {
|
|
188
93
|
// Idempotent re-add: return the live queued job if we have it, otherwise
|
|
189
94
|
// (active / waiting-children) a placeholder carrying the existing id so the
|
|
@@ -247,11 +152,11 @@ export async function pushJobBatch(queue, inputs, ctx) {
|
|
|
247
152
|
// path), exactly like a single durable push — otherwise addBulk silently
|
|
248
153
|
// downgrades the documented "durable" guarantee.
|
|
249
154
|
const durableJobs = [];
|
|
250
|
-
await
|
|
155
|
+
await withPushWriteLocks(queue, inputs, ctx, (lockedShardIndexes) => {
|
|
251
156
|
const shard = ctx.shards[idx];
|
|
252
157
|
for (const input of inputs) {
|
|
253
158
|
// Check custom ID idempotency
|
|
254
|
-
const customIdResult = handleCustomId(input,
|
|
159
|
+
const customIdResult = handleCustomId(input, ctx, lockedShardIndexes);
|
|
255
160
|
if (customIdResult.skip) {
|
|
256
161
|
// Idempotent re-add (queued, active, or waiting-children) — no insert.
|
|
257
162
|
resultIds.push('existingJob' in customIdResult
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Shard } from '../../domain/queue/shard';
|
|
2
|
+
import type { Job, JobId } from '../../domain/types/job';
|
|
3
|
+
import type { JobLocation } from '../../domain/types/queue';
|
|
4
|
+
import type { SetLike } from '../../shared/lru';
|
|
5
|
+
export interface PushInsertContext {
|
|
6
|
+
completedJobs: SetLike<JobId>;
|
|
7
|
+
depCompletions?: SetLike<JobId>;
|
|
8
|
+
jobIndex: Map<JobId, JobLocation>;
|
|
9
|
+
dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
|
|
10
|
+
}
|
|
11
|
+
/** Insert a new job into the runnable heap or dependency wait set. */
|
|
12
|
+
export declare function insertJobToShard(job: Job, queue: string, shard: Shard, shardIdx: number, ctx: PushInsertContext): void;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Insert a new job into the runnable heap or dependency wait set. */
|
|
2
|
+
export function insertJobToShard(job, queue, shard, shardIdx, ctx) {
|
|
3
|
+
const hasDeps = job.dependsOn.length > 0;
|
|
4
|
+
const needsWaiting = hasDeps &&
|
|
5
|
+
!job.dependsOn.every((depId) => ctx.completedJobs.has(depId) || (ctx.depCompletions?.has(depId) ?? false));
|
|
6
|
+
const now = Date.now();
|
|
7
|
+
if (needsWaiting) {
|
|
8
|
+
shard.waitingDeps.set(job.id, job);
|
|
9
|
+
shard.registerDependencies(job.id, job.dependsOn);
|
|
10
|
+
job.timeline.push({ state: 'waiting-children', timestamp: now });
|
|
11
|
+
ctx.dashboardEmit?.('job:waiting-children', {
|
|
12
|
+
jobId: String(job.id),
|
|
13
|
+
queue,
|
|
14
|
+
dependsOn: job.dependsOn.map(String),
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
shard.getQueue(queue).push(job);
|
|
19
|
+
const isDelayed = job.runAt > now;
|
|
20
|
+
shard.incrementQueued(job.id, isDelayed, job.createdAt, queue, job.runAt);
|
|
21
|
+
const state = isDelayed ? 'delayed' : job.priority > 0 ? 'prioritized' : 'waiting';
|
|
22
|
+
job.timeline.push({ state, timestamp: now });
|
|
23
|
+
}
|
|
24
|
+
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx, queueName: queue });
|
|
25
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { JobId, JobInput } from '../../domain/types/job';
|
|
2
|
+
import type { JobLocation } from '../../domain/types/queue';
|
|
3
|
+
import type { RWLock } from '../../shared/lock';
|
|
4
|
+
interface PushLockContext {
|
|
5
|
+
shardLocks: RWLock[];
|
|
6
|
+
jobIndex: Map<JobId, JobLocation>;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Lock the target shard plus any shard owning a terminal custom-ID generation.
|
|
10
|
+
* The required set is rechecked after acquisition because jobIndex may change
|
|
11
|
+
* while an earlier lock is awaited.
|
|
12
|
+
*/
|
|
13
|
+
export declare function withPushWriteLocks<T>(queue: string, inputs: readonly JobInput[], ctx: PushLockContext, fn: (lockedShardIndexes: ReadonlySet<number>) => T): Promise<T>;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { jobId } from '../../domain/types/job';
|
|
2
|
+
import { shardIndex } from '../../shared/hash';
|
|
3
|
+
function requiredShardIndexes(queue, inputs, jobIndex) {
|
|
4
|
+
const indexes = new Set([shardIndex(queue)]);
|
|
5
|
+
for (const input of inputs) {
|
|
6
|
+
if (!input.customId)
|
|
7
|
+
continue;
|
|
8
|
+
const location = jobIndex.get(jobId(input.customId));
|
|
9
|
+
if (location?.type === 'queue') {
|
|
10
|
+
indexes.add(location.shardIdx);
|
|
11
|
+
}
|
|
12
|
+
else if (location?.type === 'dlq') {
|
|
13
|
+
indexes.add(shardIndex(location.queueName));
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return [...indexes].sort((a, b) => a - b);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Lock the target shard plus any shard owning a terminal custom-ID generation.
|
|
20
|
+
* The required set is rechecked after acquisition because jobIndex may change
|
|
21
|
+
* while an earlier lock is awaited.
|
|
22
|
+
*/
|
|
23
|
+
export async function withPushWriteLocks(queue, inputs, ctx, fn) {
|
|
24
|
+
while (true) {
|
|
25
|
+
const indexes = requiredShardIndexes(queue, inputs, ctx.jobIndex);
|
|
26
|
+
const guards = [];
|
|
27
|
+
try {
|
|
28
|
+
for (const index of indexes) {
|
|
29
|
+
guards.push(await ctx.shardLocks[index].acquireWrite());
|
|
30
|
+
}
|
|
31
|
+
const locked = new Set(indexes);
|
|
32
|
+
const current = requiredShardIndexes(queue, inputs, ctx.jobIndex);
|
|
33
|
+
if (current.some((index) => !locked.has(index)))
|
|
34
|
+
continue;
|
|
35
|
+
return fn(locked);
|
|
36
|
+
}
|
|
37
|
+
finally {
|
|
38
|
+
for (let i = guards.length - 1; i >= 0; i--) {
|
|
39
|
+
guards[i].release();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -30,7 +30,7 @@ export declare function isQueuePaused(queue: string, ctx: QueueControlContext):
|
|
|
30
30
|
/** Drain all waiting jobs from queue */
|
|
31
31
|
export declare function drainQueue(queue: string, ctx: QueueControlContext): number;
|
|
32
32
|
/** Remove all queue data */
|
|
33
|
-
export declare function obliterateQueue(queue: string, ctx: QueueControlContext):
|
|
33
|
+
export declare function obliterateQueue(queue: string, ctx: QueueControlContext): JobId[];
|
|
34
34
|
/** List all queue names */
|
|
35
35
|
export declare function listAllQueues(ctx: QueueControlContext): string[];
|
|
36
36
|
/**
|
|
@@ -35,7 +35,7 @@ export function drainQueue(queue, ctx) {
|
|
|
35
35
|
/** Remove all queue data */
|
|
36
36
|
export function obliterateQueue(queue, ctx) {
|
|
37
37
|
const idx = shardIndex(queue);
|
|
38
|
-
ctx.shards[idx].obliterate(queue);
|
|
38
|
+
return ctx.shards[idx].obliterate(queue);
|
|
39
39
|
}
|
|
40
40
|
/** List all queue names */
|
|
41
41
|
export function listAllQueues(ctx) {
|
|
@@ -6,7 +6,7 @@ import type { Job, JobId, JobInput, JobLock, LockToken } from '../domain/types/j
|
|
|
6
6
|
import type { JobLocation, JobEvent } from '../domain/types/queue';
|
|
7
7
|
import type { CronJob, CronJobInput } from '../domain/types/cron';
|
|
8
8
|
import type { JobLogEntry, CreateWorkerOptions } from '../domain/types/worker';
|
|
9
|
-
import type
|
|
9
|
+
import { type StallConfig } from '../domain/types/stall';
|
|
10
10
|
import type { DlqConfig, DlqEntry, DlqFilter, DlqStats } from '../domain/types/dlq';
|
|
11
11
|
import { Shard } from '../domain/queue/shard';
|
|
12
12
|
import { WebhookManager } from './webhookManager';
|
|
@@ -196,7 +196,7 @@ export declare class QueueManager {
|
|
|
196
196
|
retryDlq(queue: string, jobId?: JobId, limit?: number): number;
|
|
197
197
|
purgeDlq(queue: string): number;
|
|
198
198
|
retryCompleted(queue: string, jobId?: JobId): number;
|
|
199
|
-
setRateLimit(queue: string, limit: number): void;
|
|
199
|
+
setRateLimit(queue: string, limit: number, durationMs?: number, ttlMs?: number): void;
|
|
200
200
|
clearRateLimit(queue: string): void;
|
|
201
201
|
setConcurrency(queue: string, limit: number): void;
|
|
202
202
|
clearConcurrency(queue: string): void;
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Core orchestrator for all queue operations
|
|
4
4
|
*/
|
|
5
5
|
import { DEFAULT_LOCK_TTL } from '../domain/types/job';
|
|
6
|
+
import { DEFAULT_STALL_CONFIG } from '../domain/types/stall';
|
|
6
7
|
import { Shard } from '../domain/queue/shard';
|
|
7
8
|
import { SqliteStorage } from '../infrastructure/persistence/sqlite';
|
|
8
9
|
import { CronScheduler } from '../infrastructure/scheduler/cronScheduler';
|
|
@@ -741,12 +742,12 @@ export class QueueManager {
|
|
|
741
742
|
return count;
|
|
742
743
|
}
|
|
743
744
|
obliterate(queue) {
|
|
744
|
-
queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
|
|
745
|
+
const shardJobs = queueControl.obliterateQueue(queue, this.contextFactory.getQueueControlContext());
|
|
745
746
|
dlqOps.purgeDlqJobs(queue, this.contextFactory.getDlqContext());
|
|
746
|
-
// obliterateQueue()
|
|
747
|
-
// processingShards, plus completed/result/log/lock
|
|
748
|
-
//
|
|
749
|
-
const toDrop = new Set();
|
|
747
|
+
// obliterateQueue() returns every queued, DLQ, and dependency-gated job it
|
|
748
|
+
// removed. Active jobs in processingShards, plus completed/result/log/lock
|
|
749
|
+
// state in global indexes, still need to be discovered and purged here.
|
|
750
|
+
const toDrop = new Set(shardJobs);
|
|
750
751
|
for (const [jid, loc] of this.jobIndex) {
|
|
751
752
|
if (loc.type === 'processing') {
|
|
752
753
|
const job = this.processingShards[loc.shardIdx]?.get(jid);
|
|
@@ -860,8 +861,8 @@ export class QueueManager {
|
|
|
860
861
|
return dlqOps.retryCompletedJobs(queue, this.contextFactory.getRetryCompletedContext(), jobId);
|
|
861
862
|
}
|
|
862
863
|
// ============ Rate Limiting ============
|
|
863
|
-
setRateLimit(queue, limit) {
|
|
864
|
-
this.shards[shardIndex(queue)].setRateLimit(queue, limit);
|
|
864
|
+
setRateLimit(queue, limit, durationMs, ttlMs) {
|
|
865
|
+
this.shards[shardIndex(queue)].setRateLimit(queue, limit, durationMs, ttlMs);
|
|
865
866
|
this.persistQueueState(queue);
|
|
866
867
|
}
|
|
867
868
|
clearRateLimit(queue) {
|
|
@@ -885,19 +886,37 @@ export class QueueManager {
|
|
|
885
886
|
if (!this.storage)
|
|
886
887
|
return;
|
|
887
888
|
const state = this.shards[shardIndex(queue)].getState(queue);
|
|
889
|
+
const stallConfig = this.shards[shardIndex(queue)].getStallConfig(queue);
|
|
890
|
+
const hasCustomStallConfig = stallConfig.enabled !== DEFAULT_STALL_CONFIG.enabled ||
|
|
891
|
+
stallConfig.stallInterval !== DEFAULT_STALL_CONFIG.stallInterval ||
|
|
892
|
+
stallConfig.maxStalls !== DEFAULT_STALL_CONFIG.maxStalls ||
|
|
893
|
+
stallConfig.gracePeriod !== DEFAULT_STALL_CONFIG.gracePeriod;
|
|
888
894
|
// When control-state returns fully to default (not paused, no limits), drop
|
|
889
895
|
// the row instead of persisting an all-default placeholder. Keeps the table
|
|
890
896
|
// free of noise rows for ephemeral queues that only ever call resume/clear*,
|
|
891
897
|
// and recovers identically (absent row → default state).
|
|
892
|
-
if (!state.paused &&
|
|
898
|
+
if (!state.paused &&
|
|
899
|
+
state.rateLimit === null &&
|
|
900
|
+
state.concurrencyLimit === null &&
|
|
901
|
+
!hasCustomStallConfig) {
|
|
893
902
|
this.storage.deleteQueueState(queue);
|
|
894
903
|
return;
|
|
895
904
|
}
|
|
896
|
-
this.storage.saveQueueState(queue,
|
|
905
|
+
this.storage.saveQueueState(queue, {
|
|
906
|
+
paused: state.paused,
|
|
907
|
+
rateLimit: state.rateLimit,
|
|
908
|
+
concurrencyLimit: state.concurrencyLimit,
|
|
909
|
+
rateLimitDuration: state.rateLimitDuration,
|
|
910
|
+
rateLimitExpiresAt: state.rateLimitExpiresAt,
|
|
911
|
+
stallConfig,
|
|
912
|
+
});
|
|
897
913
|
}
|
|
898
914
|
/** Get rate limit and concurrency limit for a queue */
|
|
899
915
|
getQueueLimits(queue) {
|
|
900
|
-
const
|
|
916
|
+
const shard = this.shards[shardIndex(queue)];
|
|
917
|
+
// Lazy TTL expiry so reads never report a limit that no longer throttles.
|
|
918
|
+
shard.expireRateLimitIfNeeded(queue);
|
|
919
|
+
const state = shard.getState(queue);
|
|
901
920
|
return { rateLimit: state.rateLimit, concurrencyLimit: state.concurrencyLimit };
|
|
902
921
|
}
|
|
903
922
|
/** Get all job results (for cloud telemetry) */
|
|
@@ -921,6 +940,7 @@ export class QueueManager {
|
|
|
921
940
|
// ============ Stall & DLQ Config ============
|
|
922
941
|
setStallConfig(queue, config) {
|
|
923
942
|
this.shards[shardIndex(queue)].setStallConfig(queue, config);
|
|
943
|
+
this.persistQueueState(queue);
|
|
924
944
|
}
|
|
925
945
|
getStallConfig(queue) {
|
|
926
946
|
return this.shards[shardIndex(queue)].getStallConfig(queue);
|
|
@@ -66,6 +66,7 @@ async function handleStalledJob(job, action, ctx) {
|
|
|
66
66
|
// Lock order: shardLocks BEFORE processingLocks (per lock hierarchy in CLAUDE.md)
|
|
67
67
|
// Broadcast events AFTER verifying job is still stalled to avoid false positives
|
|
68
68
|
let handled = false;
|
|
69
|
+
let handledAction = action;
|
|
69
70
|
await withWriteLock(ctx.shardLocks[idx], async () => {
|
|
70
71
|
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
71
72
|
// Verify job is still in processing (might have been handled already)
|
|
@@ -73,8 +74,10 @@ async function handleStalledJob(job, action, ctx) {
|
|
|
73
74
|
return; // Job already completed, don't broadcast stalled event
|
|
74
75
|
}
|
|
75
76
|
const shard = ctx.shards[idx];
|
|
76
|
-
|
|
77
|
-
|
|
77
|
+
const attemptsExhausted = job.attempts + 1 >= job.maxAttempts;
|
|
78
|
+
if (action === "move_to_dlq" /* StallAction.MoveToDlq */ || attemptsExhausted) {
|
|
79
|
+
handledAction = "move_to_dlq" /* StallAction.MoveToDlq */;
|
|
80
|
+
moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted);
|
|
78
81
|
}
|
|
79
82
|
else {
|
|
80
83
|
retryStalliedJob(job, ctx, shard, procIdx, idx);
|
|
@@ -87,23 +90,23 @@ async function handleStalledJob(job, action, ctx) {
|
|
|
87
90
|
ctx.dashboardEmit?.('job:stalled', {
|
|
88
91
|
jobId: String(job.id),
|
|
89
92
|
queue: job.queue,
|
|
90
|
-
stallCount: job.stallCount
|
|
91
|
-
action,
|
|
93
|
+
stallCount: job.stallCount,
|
|
94
|
+
action: handledAction,
|
|
92
95
|
});
|
|
93
96
|
ctx.eventsManager.broadcast({
|
|
94
97
|
eventType: "stalled" /* EventType.Stalled */,
|
|
95
98
|
queue: job.queue,
|
|
96
99
|
jobId: job.id,
|
|
97
100
|
timestamp: Date.now(),
|
|
98
|
-
data: { stallCount: job.stallCount
|
|
101
|
+
data: { stallCount: job.stallCount, action: handledAction },
|
|
99
102
|
});
|
|
100
103
|
void ctx.webhookManager.trigger('stalled', String(job.id), job.queue, {
|
|
101
|
-
data: { stallCount: job.stallCount
|
|
104
|
+
data: { stallCount: job.stallCount, action: handledAction },
|
|
102
105
|
});
|
|
103
106
|
}
|
|
104
107
|
}
|
|
105
108
|
/** Move stalled job to DLQ */
|
|
106
|
-
function moveStalliedJobToDlq(job, ctx, shard, procIdx,
|
|
109
|
+
function moveStalliedJobToDlq(job, ctx, shard, procIdx, attemptsExhausted) {
|
|
107
110
|
ctx.processingShards[procIdx].delete(job.id);
|
|
108
111
|
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
|
|
109
112
|
// Discard cron jobs with preventOverlap instead of sending to DLQ (#73).
|
|
@@ -112,7 +115,13 @@ function moveStalliedJobToDlq(job, ctx, shard, procIdx, _idx) {
|
|
|
112
115
|
ctx.storage?.deleteJob(job.id);
|
|
113
116
|
return;
|
|
114
117
|
}
|
|
115
|
-
|
|
118
|
+
incrementStallCount(job);
|
|
119
|
+
job.attempts++;
|
|
120
|
+
job.startedAt = null;
|
|
121
|
+
job.lastHeartbeat = Date.now();
|
|
122
|
+
const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */ : "stalled" /* FailureReason.Stalled */, attemptsExhausted
|
|
123
|
+
? `Job stalled at max attempts (${job.maxAttempts})`
|
|
124
|
+
: `Job stalled ${job.stallCount} times`);
|
|
116
125
|
ctx.jobIndex.set(job.id, { type: 'dlq', queueName: job.queue });
|
|
117
126
|
ctx.storage?.saveDlqEntry(entry);
|
|
118
127
|
ctx.storage?.deleteJob(job.id);
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
* Wraps dlqOps for Queue class usage
|
|
4
4
|
*/
|
|
5
5
|
import type { TcpConnectionPool } from '../tcpPool';
|
|
6
|
-
import type { DlqConfig, DlqEntry, DlqStats, DlqFilter } from '../types';
|
|
6
|
+
import type { Job, DlqConfig, DlqEntry, DlqStats, DlqFilter } from '../types';
|
|
7
|
+
import { type SimpleJobContext } from './jobProxy';
|
|
7
8
|
interface DlqContext {
|
|
8
9
|
name: string;
|
|
9
10
|
embedded: boolean;
|
|
@@ -11,20 +12,41 @@ interface DlqContext {
|
|
|
11
12
|
}
|
|
12
13
|
/** Set DLQ configuration */
|
|
13
14
|
export declare function setDlqConfig(ctx: DlqContext, config: Partial<DlqConfig>): void;
|
|
15
|
+
/** Set DLQ configuration and resolve once the server has applied it. */
|
|
16
|
+
export declare function setDlqConfigAsync(ctx: DlqContext, config: Partial<DlqConfig>): Promise<void>;
|
|
14
17
|
/** Get DLQ configuration */
|
|
15
18
|
export declare function getDlqConfig(ctx: DlqContext): DlqConfig;
|
|
16
19
|
/** Get DLQ configuration (async, works in TCP mode) */
|
|
17
20
|
export declare function getDlqConfigAsync(ctx: DlqContext): Promise<DlqConfig>;
|
|
18
21
|
/** Get DLQ entries */
|
|
19
22
|
export declare function getDlq<T>(ctx: DlqContext, filter?: DlqFilter): DlqEntry<T>[];
|
|
23
|
+
type DlqQueryContext = DlqContext & Pick<SimpleJobContext, 'getJobState' | 'removeAsync' | 'retryJob' | 'getChildrenValues'>;
|
|
24
|
+
/**
|
|
25
|
+
* Get the dead jobs in the DLQ, working over TCP too (wire command `Dlq`).
|
|
26
|
+
* Returns public Job objects; the wire does not carry DlqEntry metadata
|
|
27
|
+
* (reason, enteredAt, attempts) — that stays embedded-only via getDlq().
|
|
28
|
+
*/
|
|
29
|
+
export declare function getDlqJobsAsync<T>(ctx: DlqQueryContext, count?: number): Promise<Job<T>[]>;
|
|
20
30
|
/** Get DLQ stats */
|
|
21
31
|
export declare function getDlqStats(ctx: DlqContext): DlqStats;
|
|
22
32
|
/** Retry DLQ entries */
|
|
23
33
|
export declare function retryDlq(ctx: DlqContext, id?: string): number;
|
|
34
|
+
/**
|
|
35
|
+
* Retry DLQ entries and resolve with the retried count once the server has
|
|
36
|
+
* processed it. The fire-and-forget retryDlq() always returns 0 over TCP and
|
|
37
|
+
* discards the server's count.
|
|
38
|
+
*/
|
|
39
|
+
export declare function retryDlqAsync(ctx: DlqContext, id?: string): Promise<number>;
|
|
24
40
|
/** Retry DLQ entries by filter */
|
|
25
41
|
export declare function retryDlqByFilter(ctx: DlqContext, filter: DlqFilter): number;
|
|
26
42
|
/** Purge DLQ */
|
|
27
43
|
export declare function purgeDlq(ctx: DlqContext): number;
|
|
44
|
+
/**
|
|
45
|
+
* Purge the DLQ and resolve with the purged count once the server has
|
|
46
|
+
* processed it. The fire-and-forget purgeDlq() always returns 0 over TCP;
|
|
47
|
+
* purge-then-assert-empty patterns need this variant.
|
|
48
|
+
*/
|
|
49
|
+
export declare function purgeDlqAsync(ctx: DlqContext): Promise<number>;
|
|
28
50
|
/** Retry completed job */
|
|
29
51
|
export declare function retryCompleted(ctx: DlqContext, id?: string): number;
|
|
30
52
|
/** Retry completed job (async) */
|