bunqueue 2.8.39 → 2.8.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/backgroundTasks.js +3 -0
- package/dist/application/operations/jobManagement.js +7 -4
- package/dist/application/operations/jobStateTransitions.js +1 -0
- package/dist/application/operations/pull.d.ts +5 -26
- package/dist/application/operations/pull.js +68 -179
- package/dist/application/operations/pullStateTransition.d.ts +50 -0
- package/dist/application/operations/pullStateTransition.js +114 -0
- package/dist/application/queueManager.d.ts +4 -4
- package/dist/application/queueManager.js +26 -11
- package/dist/cli/client.js +6 -47
- package/dist/cli/commandRegistry.d.ts +27 -0
- package/dist/cli/commandRegistry.js +40 -0
- package/dist/cli/commandRouter.d.ts +3 -0
- package/dist/cli/commandRouter.js +44 -0
- package/dist/cli/commands/backup.js +1 -1
- package/dist/cli/commands/core.js +54 -2
- package/dist/cli/commands/doctor.d.ts +50 -5
- package/dist/cli/commands/doctor.js +125 -108
- package/dist/cli/commands/types.js +5 -1
- package/dist/cli/globalOptions.d.ts +16 -0
- package/dist/cli/globalOptions.js +192 -0
- package/dist/cli/help.d.ts +9 -0
- package/dist/cli/help.js +42 -21
- package/dist/cli/index.d.ts +1 -25
- package/dist/cli/index.js +84 -348
- package/dist/cli/localOutput.d.ts +22 -0
- package/dist/cli/localOutput.js +45 -0
- package/dist/client/queue/dlqOps.js +1 -3
- package/dist/client/tcp/client.js +10 -8
- package/dist/domain/queue/shard.d.ts +2 -2
- package/dist/domain/queue/shard.js +3 -3
- package/dist/domain/queue/waiterManager.d.ts +2 -2
- package/dist/domain/queue/waiterManager.js +26 -5
- package/dist/infrastructure/cloud/wsSender.js +4 -4
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +7 -2
- package/dist/infrastructure/persistence/sqlite.d.ts +5 -1
- package/dist/infrastructure/persistence/sqlite.js +10 -2
- package/dist/infrastructure/persistence/sqliteSerializer.js +1 -1
- package/dist/infrastructure/persistence/statements.d.ts +2 -1
- package/dist/infrastructure/persistence/statements.js +3 -2
- package/dist/infrastructure/server/handlers/core.js +4 -4
- package/dist/infrastructure/server/tcp.d.ts +1 -0
- package/dist/infrastructure/server/tcp.js +9 -5
- package/dist/infrastructure/server/types.d.ts +2 -0
- package/dist/shared/msgpack.d.ts +3 -0
- package/dist/shared/msgpack.js +70 -0
- package/package.json +1 -1
|
@@ -208,6 +208,9 @@ export function recover(ctx) {
|
|
|
208
208
|
if (qs.stallConfig) {
|
|
209
209
|
ctx.shards[shardIndex(qs.name)].setStallConfig(qs.name, qs.stallConfig);
|
|
210
210
|
}
|
|
211
|
+
if (qs.dlqConfig) {
|
|
212
|
+
ctx.shards[shardIndex(qs.name)].setDlqConfig(qs.name, qs.dlqConfig);
|
|
213
|
+
}
|
|
211
214
|
}
|
|
212
215
|
const now = Date.now();
|
|
213
216
|
// === PHASE 1: Recover active jobs (were processing when server stopped) ===
|
|
@@ -78,10 +78,13 @@ export async function updateJobProgress(jobId, progress, ctx, message) {
|
|
|
78
78
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
79
79
|
if (!job)
|
|
80
80
|
return false;
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
const clampedProgress = Math.max(0, Math.min(100, progress));
|
|
82
|
+
const effectiveMessage = message === undefined ? job.progressMessage : message;
|
|
83
|
+
const heartbeat = Date.now();
|
|
84
|
+
ctx.storage?.updateJobProgress(jobId, clampedProgress, effectiveMessage, heartbeat);
|
|
85
|
+
job.progress = clampedProgress;
|
|
86
|
+
job.progressMessage = effectiveMessage;
|
|
87
|
+
job.lastHeartbeat = heartbeat;
|
|
85
88
|
// Broadcast progress event to internal subscribers
|
|
86
89
|
ctx.eventsManager.broadcast({
|
|
87
90
|
eventType: 'progress',
|
|
@@ -37,6 +37,7 @@ export async function moveActiveToWait(jobId, ctx) {
|
|
|
37
37
|
ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
|
|
38
38
|
shard.notify(job.queue);
|
|
39
39
|
});
|
|
40
|
+
ctx.storage?.updateRunAt(jobId, job.runAt);
|
|
40
41
|
ctx.eventsManager.broadcast({
|
|
41
42
|
eventType: 'waiting',
|
|
42
43
|
jobId,
|
|
@@ -2,35 +2,14 @@
|
|
|
2
2
|
* Pull Operations
|
|
3
3
|
* Job pull logic with timeout support
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
6
|
-
import
|
|
7
|
-
|
|
8
|
-
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
9
|
-
import type { RWLock } from '../../shared/lock';
|
|
10
|
-
/** Pull operation context */
|
|
11
|
-
export interface PullContext {
|
|
12
|
-
storage: SqliteStorage | null;
|
|
13
|
-
shards: Shard[];
|
|
14
|
-
shardLocks: RWLock[];
|
|
15
|
-
processingShards: Map<JobId, Job>[];
|
|
16
|
-
processingLocks: RWLock[];
|
|
17
|
-
jobIndex: Map<JobId, JobLocation>;
|
|
18
|
-
totalPulled: {
|
|
19
|
-
value: bigint;
|
|
20
|
-
};
|
|
21
|
-
broadcast: (event: {
|
|
22
|
-
eventType: EventType;
|
|
23
|
-
queue: string;
|
|
24
|
-
jobId: JobId;
|
|
25
|
-
timestamp: number;
|
|
26
|
-
}) => void;
|
|
27
|
-
dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
|
|
28
|
-
}
|
|
5
|
+
import type { Job } from '../../domain/types/job';
|
|
6
|
+
import { type PullContext } from './pullStateTransition';
|
|
7
|
+
export type { PullContext } from './pullStateTransition';
|
|
29
8
|
/**
|
|
30
9
|
* Pull next job from queue
|
|
31
10
|
*/
|
|
32
|
-
export declare function pullJob(queue: string, timeoutMs: number, ctx: PullContext): Promise<Job | null>;
|
|
11
|
+
export declare function pullJob(queue: string, timeoutMs: number, ctx: PullContext, signal?: AbortSignal): Promise<Job | null>;
|
|
33
12
|
/**
|
|
34
13
|
* Pull multiple jobs from queue
|
|
35
14
|
*/
|
|
36
|
-
export declare function pullJobBatch(queue: string, count: number, timeoutMs: number, ctx: PullContext): Promise<Job[]>;
|
|
15
|
+
export declare function pullJobBatch(queue: string, count: number, timeoutMs: number, ctx: PullContext, signal?: AbortSignal): Promise<Job[]>;
|
|
@@ -2,130 +2,17 @@
|
|
|
2
2
|
* Pull Operations
|
|
3
3
|
* Job pull logic with timeout support
|
|
4
4
|
*/
|
|
5
|
-
import { isExpired, isReady, MAX_TIMELINE_ENTRIES, } from '../../domain/types/job';
|
|
6
5
|
import { withWriteLock } from '../../shared/lock';
|
|
7
6
|
import { shardIndex, processingShardIndex } from '../../shared/hash';
|
|
8
7
|
import { latencyTracker } from '../latencyTracker';
|
|
9
8
|
import { throughputTracker } from '../throughputTracker';
|
|
10
|
-
|
|
11
|
-
* Dequeue the highest-ranked eligible job without letting a delayed or
|
|
12
|
-
* group-blocked heap root stall unrelated work.
|
|
13
|
-
*
|
|
14
|
-
* The priority heap cannot encode time-varying readiness or active-group
|
|
15
|
-
* membership in its comparator. Temporarily park ineligible roots, select the
|
|
16
|
-
* first eligible entry (therefore the best eligible entry in heap order), and
|
|
17
|
-
* restore every parked job before leaving the shard critical section. Parked
|
|
18
|
-
* jobs remain logically queued throughout: their shard counters and jobIndex
|
|
19
|
-
* entries are deliberately untouched.
|
|
20
|
-
*
|
|
21
|
-
* Complexity is O((k + 1) log n) time and O(k) scratch space, where k is the
|
|
22
|
-
* number of higher-ranked ineligible jobs skipped. A separate delayed heap and
|
|
23
|
-
* per-group eligible-head heap can reduce this to O(log n), but this scan keeps
|
|
24
|
-
* the existing queue representation and state transitions intact.
|
|
25
|
-
*/
|
|
26
|
-
function tryDequeueNextJob(shard, queue, now, ctx) {
|
|
27
|
-
const q = shard.getQueue(queue);
|
|
28
|
-
const parked = [];
|
|
29
|
-
let nextRunAt = null;
|
|
30
|
-
try {
|
|
31
|
-
while (true) {
|
|
32
|
-
const job = q.peek();
|
|
33
|
-
if (!job)
|
|
34
|
-
return { status: 'stop', nextRunAt };
|
|
35
|
-
// Expired entries are not parked: they are genuinely removed from the
|
|
36
|
-
// logical queue and therefore update counters/index exactly once.
|
|
37
|
-
if (isExpired(job, now)) {
|
|
38
|
-
// Delete persistence before mutating the in-memory indexes. deleteJob
|
|
39
|
-
// also removes a pending buffered INSERT, so the expired job cannot be
|
|
40
|
-
// flushed after this pull or resurrect during crash recovery. This is
|
|
41
|
-
// synchronous and therefore keeps the shard critical section await-free.
|
|
42
|
-
ctx.storage?.deleteJob(job.id);
|
|
43
|
-
q.pop();
|
|
44
|
-
shard.decrementQueued(job.id);
|
|
45
|
-
ctx.jobIndex.delete(job.id);
|
|
46
|
-
ctx.dashboardEmit?.('job:expired', {
|
|
47
|
-
queue,
|
|
48
|
-
jobId: String(job.id),
|
|
49
|
-
ttl: job.ttl,
|
|
50
|
-
age: now - job.createdAt,
|
|
51
|
-
});
|
|
52
|
-
continue;
|
|
53
|
-
}
|
|
54
|
-
if (!isReady(job, now)) {
|
|
55
|
-
const delayed = q.pop();
|
|
56
|
-
if (delayed)
|
|
57
|
-
parked.push(delayed);
|
|
58
|
-
nextRunAt = nextRunAt === null ? job.runAt : Math.min(nextRunAt, job.runAt);
|
|
59
|
-
continue;
|
|
60
|
-
}
|
|
61
|
-
if (job.groupId && shard.isGroupActive(queue, job.groupId)) {
|
|
62
|
-
const blocked = q.pop();
|
|
63
|
-
if (blocked)
|
|
64
|
-
parked.push(blocked);
|
|
65
|
-
continue;
|
|
66
|
-
}
|
|
67
|
-
// Check capacity only after proving that an eligible job exists. Check
|
|
68
|
-
// concurrency first because that slot can be returned if rate limiting
|
|
69
|
-
// rejects; rate-limit tokens have no rollback operation.
|
|
70
|
-
if (!shard.tryAcquireConcurrency(queue)) {
|
|
71
|
-
ctx.dashboardEmit?.('concurrency:rejected', { queue });
|
|
72
|
-
return { status: 'stop', nextRunAt };
|
|
73
|
-
}
|
|
74
|
-
if (!shard.tryAcquireRateLimit(queue)) {
|
|
75
|
-
shard.releaseConcurrency(queue);
|
|
76
|
-
ctx.dashboardEmit?.('ratelimit:rejected', { queue });
|
|
77
|
-
return { status: 'stop', nextRunAt };
|
|
78
|
-
}
|
|
79
|
-
// Dequeue and update. q.peek() and q.pop() are synchronous under the
|
|
80
|
-
// shard write lock, so pop returns the same eligible entry selected above.
|
|
81
|
-
const dequeued = q.pop();
|
|
82
|
-
if (!dequeued) {
|
|
83
|
-
shard.releaseConcurrency(queue);
|
|
84
|
-
return { status: 'stop', nextRunAt };
|
|
85
|
-
}
|
|
86
|
-
shard.decrementQueued(dequeued.id);
|
|
87
|
-
if (dequeued.groupId) {
|
|
88
|
-
shard.activateGroup(queue, dequeued.groupId);
|
|
89
|
-
}
|
|
90
|
-
dequeued.startedAt = now;
|
|
91
|
-
dequeued.lastHeartbeat = now;
|
|
92
|
-
if (dequeued.timeline.length < MAX_TIMELINE_ENTRIES) {
|
|
93
|
-
dequeued.timeline.push({ state: 'active', timestamp: now });
|
|
94
|
-
}
|
|
95
|
-
// Make the queue -> processing transition atomic for observers. From the
|
|
96
|
-
// pop above, the job is no longer findable in the shard queue, so a
|
|
97
|
-
// jobIndex entry still saying 'queue' would make getJob/getJobState return
|
|
98
|
-
// a false null/'unknown' for an existing job (JOB -> null -> JOB(active)
|
|
99
|
-
// flicker), and cleanOrphanedJobIndex would even drop its index entry.
|
|
100
|
-
// Insert into processingShards and flip the index HERE, in the same
|
|
101
|
-
// synchronous critical section as the pop (the caller holds the shard
|
|
102
|
-
// write lock; JS is single-threaded, so no other task can interleave).
|
|
103
|
-
// Writing processingShards without its RWLock is safe: until this
|
|
104
|
-
// statement the index said 'queue', so no id-targeted critical section
|
|
105
|
-
// (ack/fail/discard operate only on ids they find as 'processing') can be
|
|
106
|
-
// mid-operation on this id; a sync Map.set cannot corrupt a paused
|
|
107
|
-
// critical section; and lock-free sync access to processingShards is
|
|
108
|
-
// established practice (stallDetection phases 1/2, collectActiveJobs,
|
|
109
|
-
// getJobProgress). Acquiring the async RWLock here would instead hold the
|
|
110
|
-
// hot shard write lock across an await.
|
|
111
|
-
const procIdx = processingShardIndex(dequeued.id);
|
|
112
|
-
ctx.processingShards[procIdx].set(dequeued.id, dequeued);
|
|
113
|
-
ctx.jobIndex.set(dequeued.id, { type: 'processing', shardIdx: procIdx });
|
|
114
|
-
return { status: 'job', job: dequeued };
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
finally {
|
|
118
|
-
// Restore only the priority-queue membership. The logical queue counters,
|
|
119
|
-
// temporal index and global jobIndex never changed for parked entries.
|
|
120
|
-
for (const parkedJob of parked)
|
|
121
|
-
q.push(parkedJob);
|
|
122
|
-
}
|
|
123
|
-
}
|
|
9
|
+
import { createDequeueScan, requeueJob, restoreParkedJobs, tryDequeueNextJob, } from './pullStateTransition';
|
|
124
10
|
/** Wait until either a notification arrives, the next delay matures, or the pull deadline. */
|
|
125
|
-
async function waitForNextCandidate(
|
|
11
|
+
async function waitForNextCandidate(options) {
|
|
12
|
+
const { shard, queue, deadline, now, nextRunAt, signal } = options;
|
|
126
13
|
const remaining = deadline - now;
|
|
127
14
|
const untilNextRun = nextRunAt === null ? remaining : Math.max(1, nextRunAt - now);
|
|
128
|
-
await shard.waitForJob(queue, Math.min(remaining, untilNextRun));
|
|
15
|
+
await shard.waitForJob(queue, Math.min(remaining, untilNextRun), signal);
|
|
129
16
|
}
|
|
130
17
|
/**
|
|
131
18
|
* Finish the handoff of a dequeued job: persist active state and broadcast.
|
|
@@ -174,57 +61,23 @@ function finalizeProcessingBatch(jobs, queue, ctx) {
|
|
|
174
61
|
}
|
|
175
62
|
return delivered;
|
|
176
63
|
}
|
|
177
|
-
/**
|
|
178
|
-
* Requeue a job whose handoff to the worker failed after dequeue.
|
|
179
|
-
* The dequeue critical section moved the job to processingShards and flipped
|
|
180
|
-
* jobIndex to 'processing'; this restores both, keeping the same atomicity
|
|
181
|
-
* rule: the job becomes findable in the queue (push + index flip in ONE
|
|
182
|
-
* shard-lock section) BEFORE the processing entry is dropped, so a reader
|
|
183
|
-
* never sees a location that misses. Lock order shard -> processing matches
|
|
184
|
-
* the documented hierarchy.
|
|
185
|
-
*/
|
|
186
|
-
async function requeueJob(job, queue, idx, ctx) {
|
|
187
|
-
const procIdx = processingShardIndex(job.id);
|
|
188
|
-
let requeued = false;
|
|
189
|
-
await withWriteLock(ctx.shardLocks[idx], () => {
|
|
190
|
-
// A management op (discardJob, moveJobToDelayed) may have claimed the job
|
|
191
|
-
// from processingShards after the failed handoff; it owns the job now.
|
|
192
|
-
if (!ctx.processingShards[procIdx].has(job.id))
|
|
193
|
-
return;
|
|
194
|
-
const shard = ctx.shards[idx];
|
|
195
|
-
if (job.groupId)
|
|
196
|
-
shard.releaseGroup(queue, job.groupId);
|
|
197
|
-
shard.releaseConcurrency(queue);
|
|
198
|
-
// Reset job state (was modified in tryDequeueNextJob)
|
|
199
|
-
job.startedAt = null;
|
|
200
|
-
shard.getQueue(queue).push(job);
|
|
201
|
-
shard.incrementQueued(job.id, false, job.createdAt, queue, job.runAt);
|
|
202
|
-
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
|
|
203
|
-
shard.notify(queue);
|
|
204
|
-
requeued = true;
|
|
205
|
-
});
|
|
206
|
-
if (!requeued)
|
|
207
|
-
return;
|
|
208
|
-
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
209
|
-
// A concurrent pull may have re-popped the job between the two lock
|
|
210
|
-
// sections and flipped the index back to 'processing'; do not delete the
|
|
211
|
-
// (re-inserted, same-object) entry out from under it.
|
|
212
|
-
if (ctx.jobIndex.get(job.id)?.type !== 'processing') {
|
|
213
|
-
ctx.processingShards[procIdx].delete(job.id);
|
|
214
|
-
}
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
64
|
/**
|
|
218
65
|
* Pull next job from queue
|
|
219
66
|
*/
|
|
220
|
-
export async function pullJob(queue, timeoutMs, ctx) {
|
|
67
|
+
export async function pullJob(queue, timeoutMs, ctx, signal) {
|
|
221
68
|
const startNs = Bun.nanoseconds();
|
|
222
69
|
const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : 0;
|
|
223
70
|
const idx = shardIndex(queue);
|
|
224
71
|
while (true) {
|
|
225
|
-
|
|
72
|
+
if (signal?.aborted)
|
|
73
|
+
return null;
|
|
74
|
+
const attempt = await tryPullFromShard(queue, idx, ctx, signal);
|
|
226
75
|
const job = attempt.job;
|
|
227
76
|
if (job) {
|
|
77
|
+
if (signal?.aborted) {
|
|
78
|
+
await requeueJob(job, queue, idx, ctx);
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
228
81
|
let delivered = false;
|
|
229
82
|
try {
|
|
230
83
|
delivered = finalizeProcessing(job, queue, ctx);
|
|
@@ -246,37 +99,60 @@ export async function pullJob(queue, timeoutMs, ctx) {
|
|
|
246
99
|
if (deadline === 0 || now >= deadline) {
|
|
247
100
|
return null;
|
|
248
101
|
}
|
|
249
|
-
await waitForNextCandidate(
|
|
102
|
+
await waitForNextCandidate({
|
|
103
|
+
shard: ctx.shards[idx],
|
|
104
|
+
queue,
|
|
105
|
+
deadline,
|
|
106
|
+
now,
|
|
107
|
+
nextRunAt: attempt.nextRunAt,
|
|
108
|
+
signal,
|
|
109
|
+
});
|
|
250
110
|
}
|
|
251
111
|
}
|
|
252
112
|
/**
|
|
253
113
|
* Try to pull a job from a specific shard
|
|
254
114
|
*/
|
|
255
|
-
async function tryPullFromShard(queue, idx, ctx) {
|
|
115
|
+
async function tryPullFromShard(queue, idx, ctx, signal) {
|
|
256
116
|
return await withWriteLock(ctx.shardLocks[idx], () => {
|
|
117
|
+
if (signal?.aborted)
|
|
118
|
+
return { job: null, nextRunAt: null };
|
|
257
119
|
const shard = ctx.shards[idx];
|
|
258
120
|
const state = shard.getState(queue);
|
|
259
121
|
if (state.paused) {
|
|
260
122
|
return { job: null, nextRunAt: null };
|
|
261
123
|
}
|
|
262
124
|
const now = Date.now();
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
125
|
+
const scan = createDequeueScan();
|
|
126
|
+
try {
|
|
127
|
+
const result = tryDequeueNextJob(shard, queue, now, ctx, scan);
|
|
128
|
+
return result.status === 'job'
|
|
129
|
+
? { job: result.job, nextRunAt: null }
|
|
130
|
+
: { job: null, nextRunAt: scan.nextRunAt };
|
|
131
|
+
}
|
|
132
|
+
finally {
|
|
133
|
+
restoreParkedJobs(shard, queue, scan);
|
|
134
|
+
}
|
|
267
135
|
});
|
|
268
136
|
}
|
|
269
137
|
/**
|
|
270
138
|
* Pull multiple jobs from queue
|
|
271
139
|
*/
|
|
272
|
-
export async function pullJobBatch(queue, count, timeoutMs, ctx) {
|
|
140
|
+
export async function pullJobBatch(queue, count, timeoutMs, ctx, signal) {
|
|
273
141
|
const startNs = Bun.nanoseconds();
|
|
274
142
|
const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : 0;
|
|
275
143
|
const idx = shardIndex(queue);
|
|
276
144
|
while (true) {
|
|
277
|
-
|
|
145
|
+
if (signal?.aborted)
|
|
146
|
+
return [];
|
|
147
|
+
const attempt = await tryPullBatchFromShard(queue, idx, count, ctx, signal);
|
|
278
148
|
const jobs = attempt.jobs;
|
|
279
149
|
if (jobs.length > 0) {
|
|
150
|
+
if (signal?.aborted) {
|
|
151
|
+
for (const job of jobs) {
|
|
152
|
+
await requeueJob(job, queue, idx, ctx);
|
|
153
|
+
}
|
|
154
|
+
return [];
|
|
155
|
+
}
|
|
280
156
|
let delivered;
|
|
281
157
|
try {
|
|
282
158
|
delivered = finalizeProcessingBatch(jobs, queue, ctx);
|
|
@@ -305,31 +181,44 @@ export async function pullJobBatch(queue, count, timeoutMs, ctx) {
|
|
|
305
181
|
if (deadline === 0 || now >= deadline) {
|
|
306
182
|
return [];
|
|
307
183
|
}
|
|
308
|
-
await waitForNextCandidate(
|
|
184
|
+
await waitForNextCandidate({
|
|
185
|
+
shard: ctx.shards[idx],
|
|
186
|
+
queue,
|
|
187
|
+
deadline,
|
|
188
|
+
now,
|
|
189
|
+
nextRunAt: attempt.nextRunAt,
|
|
190
|
+
signal,
|
|
191
|
+
});
|
|
309
192
|
}
|
|
310
193
|
}
|
|
311
194
|
/**
|
|
312
195
|
* Try to pull multiple jobs from a shard
|
|
313
196
|
*/
|
|
314
|
-
async function tryPullBatchFromShard(queue, idx, count, ctx) {
|
|
197
|
+
async function tryPullBatchFromShard(queue, idx, count, ctx, signal) {
|
|
315
198
|
return await withWriteLock(ctx.shardLocks[idx], () => {
|
|
199
|
+
if (signal?.aborted)
|
|
200
|
+
return { jobs: [], nextRunAt: null };
|
|
316
201
|
const shard = ctx.shards[idx];
|
|
317
202
|
const state = shard.getState(queue);
|
|
318
203
|
const jobs = [];
|
|
319
204
|
if (state.paused)
|
|
320
205
|
return { jobs, nextRunAt: null };
|
|
321
206
|
const now = Date.now();
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
207
|
+
const scan = createDequeueScan();
|
|
208
|
+
try {
|
|
209
|
+
while (jobs.length < count) {
|
|
210
|
+
const result = tryDequeueNextJob(shard, queue, now, ctx, scan);
|
|
211
|
+
if (result.status === 'job') {
|
|
212
|
+
jobs.push(result.job);
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
break;
|
|
216
|
+
}
|
|
331
217
|
}
|
|
218
|
+
return { jobs, nextRunAt: scan.nextRunAt };
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
restoreParkedJobs(shard, queue, scan);
|
|
332
222
|
}
|
|
333
|
-
return { jobs, nextRunAt };
|
|
334
223
|
});
|
|
335
224
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic queue-state transitions used by single and batch pull orchestration.
|
|
3
|
+
*/
|
|
4
|
+
import type { Shard } from '../../domain/queue/shard';
|
|
5
|
+
import { type Job, type JobId } from '../../domain/types/job';
|
|
6
|
+
import type { EventType, JobLocation } from '../../domain/types/queue';
|
|
7
|
+
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
8
|
+
import type { RWLock } from '../../shared/lock';
|
|
9
|
+
/** Dependencies shared by the single and batch pull paths. */
|
|
10
|
+
export interface PullContext {
|
|
11
|
+
storage: SqliteStorage | null;
|
|
12
|
+
shards: Shard[];
|
|
13
|
+
shardLocks: RWLock[];
|
|
14
|
+
processingShards: Map<JobId, Job>[];
|
|
15
|
+
processingLocks: RWLock[];
|
|
16
|
+
jobIndex: Map<JobId, JobLocation>;
|
|
17
|
+
totalPulled: {
|
|
18
|
+
value: bigint;
|
|
19
|
+
};
|
|
20
|
+
broadcast: (event: {
|
|
21
|
+
eventType: EventType;
|
|
22
|
+
queue: string;
|
|
23
|
+
jobId: JobId;
|
|
24
|
+
timestamp: number;
|
|
25
|
+
}) => void;
|
|
26
|
+
dashboardEmit?: (event: string, data: Record<string, unknown>) => void;
|
|
27
|
+
}
|
|
28
|
+
export interface DequeueScan {
|
|
29
|
+
parked: Job[];
|
|
30
|
+
nextRunAt: number | null;
|
|
31
|
+
}
|
|
32
|
+
export type DequeueResult = {
|
|
33
|
+
status: 'job';
|
|
34
|
+
job: Job;
|
|
35
|
+
} | {
|
|
36
|
+
status: 'stop';
|
|
37
|
+
};
|
|
38
|
+
export declare function createDequeueScan(): DequeueScan;
|
|
39
|
+
/** Restore physical heap membership without changing logical queue state. */
|
|
40
|
+
export declare function restoreParkedJobs(shard: Shard, queue: string, scan: DequeueScan): void;
|
|
41
|
+
/**
|
|
42
|
+
* Remove the best eligible job while retaining ineligible roots in `scan`.
|
|
43
|
+
*
|
|
44
|
+
* A batch reuses one scan, because readiness cannot change while the synchronous
|
|
45
|
+
* shard critical section is held: time is fixed and active FIFO groups only
|
|
46
|
+
* grow. Its parked jobs therefore need to be restored only once after the batch.
|
|
47
|
+
*/
|
|
48
|
+
export declare function tryDequeueNextJob(shard: Shard, queue: string, now: number, ctx: PullContext, scan: DequeueScan): DequeueResult;
|
|
49
|
+
/** Restore a dequeued job when its handoff to the worker cannot complete. */
|
|
50
|
+
export declare function requeueJob(job: Job, queue: string, idx: number, ctx: PullContext): Promise<void>;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic queue-state transitions used by single and batch pull orchestration.
|
|
3
|
+
*/
|
|
4
|
+
import { isExpired, isReady, MAX_TIMELINE_ENTRIES, } from '../../domain/types/job';
|
|
5
|
+
import { processingShardIndex } from '../../shared/hash';
|
|
6
|
+
import { withWriteLock } from '../../shared/lock';
|
|
7
|
+
export function createDequeueScan() {
|
|
8
|
+
return { parked: [], nextRunAt: null };
|
|
9
|
+
}
|
|
10
|
+
/** Restore physical heap membership without changing logical queue state. */
|
|
11
|
+
export function restoreParkedJobs(shard, queue, scan) {
|
|
12
|
+
const priorityQueue = shard.getQueue(queue);
|
|
13
|
+
for (const job of scan.parked)
|
|
14
|
+
priorityQueue.push(job);
|
|
15
|
+
scan.parked.length = 0;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Remove the best eligible job while retaining ineligible roots in `scan`.
|
|
19
|
+
*
|
|
20
|
+
* A batch reuses one scan, because readiness cannot change while the synchronous
|
|
21
|
+
* shard critical section is held: time is fixed and active FIFO groups only
|
|
22
|
+
* grow. Its parked jobs therefore need to be restored only once after the batch.
|
|
23
|
+
*/
|
|
24
|
+
export function tryDequeueNextJob(shard, queue, now, ctx, scan) {
|
|
25
|
+
const priorityQueue = shard.getQueue(queue);
|
|
26
|
+
while (true) {
|
|
27
|
+
const job = priorityQueue.peek();
|
|
28
|
+
if (!job)
|
|
29
|
+
return { status: 'stop' };
|
|
30
|
+
if (isExpired(job, now)) {
|
|
31
|
+
// Persistence is removed first so a failed durable delete leaves the
|
|
32
|
+
// in-memory job available for a later attempt.
|
|
33
|
+
ctx.storage?.deleteJob(job.id);
|
|
34
|
+
priorityQueue.pop();
|
|
35
|
+
shard.decrementQueued(job.id);
|
|
36
|
+
ctx.jobIndex.delete(job.id);
|
|
37
|
+
ctx.dashboardEmit?.('job:expired', {
|
|
38
|
+
queue,
|
|
39
|
+
jobId: String(job.id),
|
|
40
|
+
ttl: job.ttl,
|
|
41
|
+
age: now - job.createdAt,
|
|
42
|
+
});
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
if (!isReady(job, now)) {
|
|
46
|
+
const delayed = priorityQueue.pop();
|
|
47
|
+
if (delayed)
|
|
48
|
+
scan.parked.push(delayed);
|
|
49
|
+
scan.nextRunAt = scan.nextRunAt === null ? job.runAt : Math.min(scan.nextRunAt, job.runAt);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (job.groupId && shard.isGroupActive(queue, job.groupId)) {
|
|
53
|
+
const blocked = priorityQueue.pop();
|
|
54
|
+
if (blocked)
|
|
55
|
+
scan.parked.push(blocked);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
// Capacity is consumed only after an eligible job has been found.
|
|
59
|
+
if (!shard.tryAcquireConcurrency(queue)) {
|
|
60
|
+
ctx.dashboardEmit?.('concurrency:rejected', { queue });
|
|
61
|
+
return { status: 'stop' };
|
|
62
|
+
}
|
|
63
|
+
if (!shard.tryAcquireRateLimit(queue)) {
|
|
64
|
+
shard.releaseConcurrency(queue);
|
|
65
|
+
ctx.dashboardEmit?.('ratelimit:rejected', { queue });
|
|
66
|
+
return { status: 'stop' };
|
|
67
|
+
}
|
|
68
|
+
const dequeued = priorityQueue.pop();
|
|
69
|
+
if (!dequeued) {
|
|
70
|
+
shard.releaseConcurrency(queue);
|
|
71
|
+
return { status: 'stop' };
|
|
72
|
+
}
|
|
73
|
+
shard.decrementQueued(dequeued.id);
|
|
74
|
+
if (dequeued.groupId)
|
|
75
|
+
shard.activateGroup(queue, dequeued.groupId);
|
|
76
|
+
dequeued.startedAt = now;
|
|
77
|
+
dequeued.lastHeartbeat = now;
|
|
78
|
+
if (dequeued.timeline.length < MAX_TIMELINE_ENTRIES) {
|
|
79
|
+
dequeued.timeline.push({ state: 'active', timestamp: now });
|
|
80
|
+
}
|
|
81
|
+
// Queue -> processing is atomic for observers while the shard lock is held.
|
|
82
|
+
const procIdx = processingShardIndex(dequeued.id);
|
|
83
|
+
ctx.processingShards[procIdx].set(dequeued.id, dequeued);
|
|
84
|
+
ctx.jobIndex.set(dequeued.id, { type: 'processing', shardIdx: procIdx });
|
|
85
|
+
return { status: 'job', job: dequeued };
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/** Restore a dequeued job when its handoff to the worker cannot complete. */
|
|
89
|
+
export async function requeueJob(job, queue, idx, ctx) {
|
|
90
|
+
const procIdx = processingShardIndex(job.id);
|
|
91
|
+
let requeued = false;
|
|
92
|
+
await withWriteLock(ctx.shardLocks[idx], () => {
|
|
93
|
+
if (!ctx.processingShards[procIdx].has(job.id))
|
|
94
|
+
return;
|
|
95
|
+
const shard = ctx.shards[idx];
|
|
96
|
+
if (job.groupId)
|
|
97
|
+
shard.releaseGroup(queue, job.groupId);
|
|
98
|
+
shard.releaseConcurrency(queue);
|
|
99
|
+
job.startedAt = null;
|
|
100
|
+
shard.getQueue(queue).push(job);
|
|
101
|
+
shard.incrementQueued(job.id, false, job.createdAt, queue, job.runAt);
|
|
102
|
+
ctx.jobIndex.set(job.id, { type: 'queue', shardIdx: idx, queueName: queue });
|
|
103
|
+
shard.notify(queue);
|
|
104
|
+
requeued = true;
|
|
105
|
+
});
|
|
106
|
+
if (!requeued)
|
|
107
|
+
return;
|
|
108
|
+
await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
109
|
+
// A concurrent pull may already have moved the same object back to active.
|
|
110
|
+
if (ctx.jobIndex.get(job.id)?.type !== 'processing') {
|
|
111
|
+
ctx.processingShards[procIdx].delete(job.id);
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
}
|
|
@@ -65,13 +65,13 @@ export declare class QueueManager {
|
|
|
65
65
|
private handleRepeat;
|
|
66
66
|
push(queue: string, input: JobInput): Promise<Job>;
|
|
67
67
|
pushBatch(queue: string, inputs: JobInput[]): Promise<JobId[]>;
|
|
68
|
-
pull(queue: string, timeoutMs?: number): Promise<Job | null>;
|
|
69
|
-
pullWithLock(queue: string, owner: string, timeoutMs?: number, lockTtl?: number): Promise<{
|
|
68
|
+
pull(queue: string, timeoutMs?: number, signal?: AbortSignal): Promise<Job | null>;
|
|
69
|
+
pullWithLock(queue: string, owner: string, timeoutMs?: number, lockTtl?: number, signal?: AbortSignal): Promise<{
|
|
70
70
|
job: Job | null;
|
|
71
71
|
token: string | null;
|
|
72
72
|
}>;
|
|
73
|
-
pullBatch(queue: string, count: number, timeoutMs?: number): Promise<Job[]>;
|
|
74
|
-
pullBatchWithLock(queue: string, count: number, owner: string, timeoutMs?: number, lockTtl?: number): Promise<{
|
|
73
|
+
pullBatch(queue: string, count: number, timeoutMs?: number, signal?: AbortSignal): Promise<Job[]>;
|
|
74
|
+
pullBatchWithLock(queue: string, count: number, owner: string, timeoutMs?: number, lockTtl?: number, signal?: AbortSignal): Promise<{
|
|
75
75
|
jobs: Job[];
|
|
76
76
|
tokens: string[];
|
|
77
77
|
}>;
|