bunqueue 2.8.35 → 2.8.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/application/backgroundTasks.js +28 -5
- package/dist/application/contextFactory.js +5 -0
- package/dist/application/dlqManager.d.ts +6 -0
- package/dist/application/dlqManager.js +35 -14
- package/dist/application/lockManager.js +22 -8
- package/dist/application/operations/customId.d.ts +31 -0
- package/dist/application/operations/customId.js +59 -0
- package/dist/application/operations/jobClaim.d.ts +17 -0
- package/dist/application/operations/jobClaim.js +20 -0
- package/dist/application/operations/jobManagement.d.ts +4 -5
- package/dist/application/operations/jobManagement.js +15 -116
- package/dist/application/operations/jobMoveOperations.d.ts +9 -0
- package/dist/application/operations/jobMoveOperations.js +94 -0
- package/dist/application/operations/jobStateTransitions.js +3 -0
- package/dist/application/operations/pull.js +5 -0
- package/dist/application/operations/push.js +8 -103
- package/dist/application/operations/pushInsert.d.ts +12 -0
- package/dist/application/operations/pushInsert.js +25 -0
- package/dist/application/operations/pushLocks.d.ts +14 -0
- package/dist/application/operations/pushLocks.js +43 -0
- package/dist/application/operations/queueControl.d.ts +1 -1
- package/dist/application/operations/queueControl.js +1 -1
- package/dist/application/queueManager.d.ts +1 -1
- package/dist/application/queueManager.js +17 -6
- package/dist/application/stallDetection.js +17 -8
- package/dist/client/queue/helpers.js +2 -0
- package/dist/client/queue/operations/counts.d.ts +1 -0
- package/dist/client/queue/operations/counts.js +3 -0
- package/dist/domain/queue/dependencyTracker.d.ts +5 -0
- package/dist/domain/queue/dependencyTracker.js +20 -0
- package/dist/domain/queue/shard.d.ts +1 -1
- package/dist/domain/queue/shard.js +6 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -2
- package/dist/infrastructure/persistence/schema.js +25 -2
- package/dist/infrastructure/persistence/sqlite.d.ts +11 -0
- package/dist/infrastructure/persistence/sqlite.js +59 -5
- package/dist/infrastructure/persistence/sqliteBatch.js +8 -6
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +9 -1
- package/dist/infrastructure/persistence/statements.d.ts +6 -1
- package/dist/infrastructure/persistence/statements.js +7 -5
- package/package.json +4 -1
package/README.md
CHANGED
|
@@ -830,8 +830,20 @@ full feature parity, so producers and workers can live anywhere in your stack:
|
|
|
830
830
|
|
|
831
831
|
| Where your code runs | Install | Status |
|
|
832
832
|
| -------------------- | ------- | ------ |
|
|
833
|
-
| **Node.js ≥ 20, Deno ≥ 2, Bun, Cloudflare Workers** | [`npm install bunqueue-client`](https://www.npmjs.com/package/bunqueue-client) |
|
|
834
|
-
| **Python ≥ 3.9** | PyPI coming soon — today: `pip install "git+https://github.com/egeominotti/bunqueue.git#subdirectory=sdk/python"` |
|
|
833
|
+
| **Node.js ≥ 20, Deno ≥ 2, Bun, Cloudflare Workers** | [`npm install bunqueue-client`](https://www.npmjs.com/package/bunqueue-client) | Native integration/e2e + protocol conformance |
|
|
834
|
+
| **Python ≥ 3.9** | PyPI coming soon — today: `pip install "git+https://github.com/egeominotti/bunqueue.git#subdirectory=sdk/python"` | Native integration/e2e + protocol conformance |
|
|
835
|
+
| **PHP ≥ 8.1** | Packagist coming soon — today: use [`sdk/php`](./sdk/php) | Native e2e + protocol conformance |
|
|
836
|
+
| **Go 1.26.5** | `go get github.com/egeominotti/bunqueue/sdk/go` | Native tests + protocol conformance |
|
|
837
|
+
| **Rust ≥ 1.85** | crates.io coming soon — today: use [`sdk/rust`](./sdk/rust) as a path dependency | Native unit/integration + protocol conformance |
|
|
838
|
+
| **Elixir ≥ 1.15** | Hex coming soon — today: use [`sdk/elixir`](./sdk/elixir) as a path dependency | ExUnit integration + protocol conformance |
|
|
839
|
+
|
|
840
|
+
All six SDKs run the same production hardening matrix: independent-connection
|
|
841
|
+
idempotency and single-lease races, generated payload invariants,
|
|
842
|
+
malformed-input fuzz corpora, broker SIGKILL/reconnect durability, bounded
|
|
843
|
+
spikes, structured telemetry, and 17 shared protocol checks. Weekly CI adds
|
|
844
|
+
runtime compatibility, live dependency advisories, native Go race/fuzz checks,
|
|
845
|
+
and a 15-minute sustained profile per SDK. See
|
|
846
|
+
[`docs/testing.md`](./docs/testing.md#sdk-hardening-matrix).
|
|
835
847
|
|
|
836
848
|
```typescript
|
|
837
849
|
// Node.js / Deno / Cloudflare Workers
|
|
@@ -144,6 +144,8 @@ function performDlqMaintenance(ctx) {
|
|
|
144
144
|
const dlqCtx = {
|
|
145
145
|
shards: ctx.shards,
|
|
146
146
|
jobIndex: ctx.jobIndex,
|
|
147
|
+
jobResults: ctx.jobResults,
|
|
148
|
+
jobLogs: ctx.jobLogs,
|
|
147
149
|
storage: ctx.storage,
|
|
148
150
|
};
|
|
149
151
|
for (const queueName of ctx.queueNamesCache) {
|
|
@@ -198,6 +200,15 @@ export function recover(ctx) {
|
|
|
198
200
|
// Load DLQ job IDs so Phase 1 can skip stale active rows for DLQ'd jobs
|
|
199
201
|
// (legacy DBs predate the DLQ-row cleanup fix in failJob).
|
|
200
202
|
const dlqJobIds = ctx.storage.loadDlqJobIds();
|
|
203
|
+
const queueStates = ctx.storage.loadQueueState();
|
|
204
|
+
// Recovery bounds are needed before Phase 1 classifies interrupted active
|
|
205
|
+
// jobs. Restore the stall policy first; paused/limiter state can wait until
|
|
206
|
+
// the queues themselves have been rebuilt below.
|
|
207
|
+
for (const qs of queueStates) {
|
|
208
|
+
if (qs.stallConfig) {
|
|
209
|
+
ctx.shards[shardIndex(qs.name)].setStallConfig(qs.name, qs.stallConfig);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
201
212
|
const now = Date.now();
|
|
202
213
|
// === PHASE 1: Recover active jobs (were processing when server stopped) ===
|
|
203
214
|
// These jobs are considered "stalled" and need to be retried or moved to DLQ
|
|
@@ -240,12 +251,24 @@ export function recover(ctx) {
|
|
|
240
251
|
job.attempts++;
|
|
241
252
|
job.startedAt = null;
|
|
242
253
|
job.lastHeartbeat = now;
|
|
243
|
-
//
|
|
254
|
+
// A crash consumes one processing attempt and one stall allowance. Both
|
|
255
|
+
// bounds are terminal: requeueing after either reaches its configured
|
|
256
|
+
// maximum would make attempts/stalls unbounded across restart loops.
|
|
244
257
|
const maxStalls = stallConfig.maxStalls ?? 3;
|
|
245
|
-
|
|
258
|
+
const attemptsExhausted = job.attempts >= job.maxAttempts;
|
|
259
|
+
const stallsExhausted = maxStalls > 0 && job.stallCount >= maxStalls;
|
|
260
|
+
if (attemptsExhausted || stallsExhausted) {
|
|
246
261
|
// Move to DLQ
|
|
247
|
-
const
|
|
248
|
-
|
|
262
|
+
const reason = attemptsExhausted
|
|
263
|
+
? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */
|
|
264
|
+
: "stalled" /* FailureReason.Stalled */;
|
|
265
|
+
const error = attemptsExhausted
|
|
266
|
+
? `Job reached max attempts (${job.maxAttempts}) during startup recovery`
|
|
267
|
+
: `Job stalled ${job.stallCount} times (recovered at startup)`;
|
|
268
|
+
// Persist only here; the later loadDlq() phase is the single in-memory
|
|
269
|
+
// restore path. Adding to the shard now and restoring the same row below
|
|
270
|
+
// would duplicate the DLQ entry after every terminal crash recovery.
|
|
271
|
+
const entry = createDlqEntry(job, reason, error, shard.getDlqConfig(job.queue));
|
|
249
272
|
ctx.storage.saveDlqEntry(entry);
|
|
250
273
|
ctx.storage.deleteJob(job.id);
|
|
251
274
|
}
|
|
@@ -338,7 +361,7 @@ export function recover(ctx) {
|
|
|
338
361
|
// Map; without this load every queue silently un-pauses and loses its limits
|
|
339
362
|
// on restart. Applied directly to the owning shard (in-memory only — these
|
|
340
363
|
// setters do not re-persist, so there is no write-back loop).
|
|
341
|
-
for (const qs of
|
|
364
|
+
for (const qs of queueStates) {
|
|
342
365
|
const shard = ctx.shards[shardIndex(qs.name)];
|
|
343
366
|
if (qs.paused)
|
|
344
367
|
shard.pause(qs.name);
|
|
@@ -141,6 +141,8 @@ export class ContextFactory {
|
|
|
141
141
|
processingShards: this.deps.processingShards,
|
|
142
142
|
processingLocks: this.deps.processingLocks,
|
|
143
143
|
jobIndex: this.deps.jobIndex,
|
|
144
|
+
jobLocks: this.deps.jobLocks,
|
|
145
|
+
clientJobs: this.deps.clientJobs,
|
|
144
146
|
webhookManager: this.deps.webhookManager,
|
|
145
147
|
eventsManager: this.deps.eventsManager,
|
|
146
148
|
repeatChain: this.deps.repeatChain,
|
|
@@ -164,6 +166,8 @@ export class ContextFactory {
|
|
|
164
166
|
return {
|
|
165
167
|
shards: this.deps.shards,
|
|
166
168
|
jobIndex: this.deps.jobIndex,
|
|
169
|
+
jobResults: this.deps.jobResults,
|
|
170
|
+
jobLogs: this.deps.jobLogs,
|
|
167
171
|
storage: this.deps.storage,
|
|
168
172
|
};
|
|
169
173
|
}
|
|
@@ -171,6 +175,7 @@ export class ContextFactory {
|
|
|
171
175
|
return {
|
|
172
176
|
shards: this.deps.shards,
|
|
173
177
|
jobIndex: this.deps.jobIndex,
|
|
178
|
+
jobLogs: this.deps.jobLogs,
|
|
174
179
|
storage: this.deps.storage,
|
|
175
180
|
completedJobs: this.deps.completedJobs,
|
|
176
181
|
jobResults: this.deps.jobResults,
|
|
@@ -11,6 +11,12 @@ import type { SqliteStorage } from '../infrastructure/persistence/sqlite';
|
|
|
11
11
|
export interface DlqContext {
|
|
12
12
|
shards: Shard[];
|
|
13
13
|
jobIndex: Map<JobId, JobLocation>;
|
|
14
|
+
jobResults: {
|
|
15
|
+
delete(id: JobId): boolean;
|
|
16
|
+
};
|
|
17
|
+
jobLogs: {
|
|
18
|
+
delete(id: JobId): boolean;
|
|
19
|
+
};
|
|
14
20
|
storage: SqliteStorage | null;
|
|
15
21
|
}
|
|
16
22
|
/** Get jobs from DLQ (backward compatible) */
|
|
@@ -209,24 +209,45 @@ export function processAutoRetry(queue, ctx) {
|
|
|
209
209
|
export function purgeExpiredDlq(queue, ctx) {
|
|
210
210
|
const idx = shardIndex(queue);
|
|
211
211
|
const shard = ctx.shards[idx];
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
if (
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
return count;
|
|
212
|
+
const now = Date.now();
|
|
213
|
+
// Use one timestamp for the snapshot and mutation so every purged entry is
|
|
214
|
+
// represented in the cleanup set.
|
|
215
|
+
const expiredEntries = shard.getExpiredEntries(queue, now);
|
|
216
|
+
if (expiredEntries.length === 0)
|
|
217
|
+
return 0;
|
|
218
|
+
cleanupPurgedEntries(queue, expiredEntries, ctx, false);
|
|
219
|
+
shard.purgeExpired(queue, now);
|
|
220
|
+
return expiredEntries.length;
|
|
222
221
|
}
|
|
223
222
|
/** Purge all jobs from DLQ */
|
|
224
223
|
export function purgeDlqJobs(queue, ctx) {
|
|
225
224
|
const idx = shardIndex(queue);
|
|
226
|
-
const
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
225
|
+
const shard = ctx.shards[idx];
|
|
226
|
+
const entries = shard.getDlqEntries(queue);
|
|
227
|
+
cleanupPurgedEntries(queue, entries, ctx, true);
|
|
228
|
+
shard.clearDlq(queue);
|
|
229
|
+
return entries.length;
|
|
230
|
+
}
|
|
231
|
+
/** Remove every durable and global reference owned by terminal DLQ entries. */
|
|
232
|
+
function cleanupPurgedEntries(queue, entries, ctx, clearQueue) {
|
|
233
|
+
const jobIds = [...new Set(entries.map((entry) => entry.job.id))];
|
|
234
|
+
const terminalJobIds = jobIds.filter((jobId) => {
|
|
235
|
+
const location = ctx.jobIndex.get(jobId);
|
|
236
|
+
return location === undefined || (location.type === 'dlq' && location.queueName === queue);
|
|
237
|
+
});
|
|
238
|
+
// One SQLite transaction removes DLQ rows, any orphan jobs/result rows, and
|
|
239
|
+
// pending buffered inserts before in-memory references become unobservable.
|
|
240
|
+
// A newer live generation can reuse the same custom id; its jobs row and
|
|
241
|
+
// auxiliary state must survive cleanup of the stale terminal generation.
|
|
242
|
+
ctx.storage?.purgeDlqEntries(queue, jobIds, terminalJobIds, clearQueue);
|
|
243
|
+
for (const jobId of terminalJobIds) {
|
|
244
|
+
const location = ctx.jobIndex.get(jobId);
|
|
245
|
+
if (location?.type === 'dlq' && location.queueName === queue) {
|
|
246
|
+
ctx.jobIndex.delete(jobId);
|
|
247
|
+
}
|
|
248
|
+
ctx.jobResults.delete(jobId);
|
|
249
|
+
ctx.jobLogs.delete(jobId);
|
|
250
|
+
}
|
|
230
251
|
}
|
|
231
252
|
/** Configure DLQ for a queue */
|
|
232
253
|
export function configureDlq(queue, ctx, config) {
|
|
@@ -92,10 +92,20 @@ function processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now)
|
|
|
92
92
|
job.startedAt = null;
|
|
93
93
|
job.lastHeartbeat = now;
|
|
94
94
|
job.stallCount++;
|
|
95
|
-
//
|
|
95
|
+
// A lost lock consumes both retry budgets. Never requeue beyond either bound.
|
|
96
96
|
const stallConfig = shard.getStallConfig(job.queue);
|
|
97
|
-
|
|
98
|
-
|
|
97
|
+
const attemptsExhausted = job.attempts >= job.maxAttempts;
|
|
98
|
+
const stallsExhausted = stallConfig.maxStalls > 0 && job.stallCount >= stallConfig.maxStalls;
|
|
99
|
+
if (attemptsExhausted || stallsExhausted) {
|
|
100
|
+
handleRecoveryBoundExceeded({
|
|
101
|
+
jobId,
|
|
102
|
+
job,
|
|
103
|
+
lock,
|
|
104
|
+
shard,
|
|
105
|
+
ctx,
|
|
106
|
+
now,
|
|
107
|
+
attemptsExhausted,
|
|
108
|
+
});
|
|
99
109
|
}
|
|
100
110
|
else {
|
|
101
111
|
requeueExpiredJob({ jobId, job, lock, queue, idx: shardIdx, ctx, now });
|
|
@@ -108,14 +118,16 @@ function processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now)
|
|
|
108
118
|
renewalCount: lock.renewalCount,
|
|
109
119
|
});
|
|
110
120
|
}
|
|
111
|
-
/** Move job to DLQ when
|
|
112
|
-
function
|
|
113
|
-
const { jobId, job, lock, shard, ctx, now } = opts;
|
|
121
|
+
/** Move job to DLQ when either crash-recovery budget is exhausted. */
|
|
122
|
+
function handleRecoveryBoundExceeded(opts) {
|
|
123
|
+
const { jobId, job, lock, shard, ctx, now, attemptsExhausted } = opts;
|
|
114
124
|
// Release the concurrency slot (+group+uniqueKey) acquired at pull before
|
|
115
125
|
// moving to DLQ — otherwise the slot leaks (mirrors
|
|
116
126
|
// stallDetection.moveStalliedJobToDlq).
|
|
117
127
|
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
|
|
118
|
-
const entry = shard.addToDlq(job, "
|
|
128
|
+
const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */ : "stalled" /* FailureReason.Stalled */, attemptsExhausted
|
|
129
|
+
? `Lock expired at max attempts (${job.maxAttempts})`
|
|
130
|
+
: `Lock expired after ${lock.renewalCount} renewals`);
|
|
119
131
|
ctx.jobIndex.set(jobId, { type: 'dlq', queueName: job.queue });
|
|
120
132
|
// Persist the DLQ move like the sibling paths (ack.moveFailedJobToDlq,
|
|
121
133
|
// stallDetection.moveStalliedJobToDlq, backgroundTasks startup-recovery).
|
|
@@ -129,7 +141,9 @@ function handleMaxStallsExceeded(opts) {
|
|
|
129
141
|
jobId,
|
|
130
142
|
queue: job.queue,
|
|
131
143
|
timestamp: now,
|
|
132
|
-
error:
|
|
144
|
+
error: attemptsExhausted
|
|
145
|
+
? 'Lock expired (max attempts reached)'
|
|
146
|
+
: 'Lock expired (max stalls reached)',
|
|
133
147
|
});
|
|
134
148
|
}
|
|
135
149
|
/** Requeue job for retry */
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Shard } from '../../domain/queue/shard';
|
|
2
|
+
import { type Job, type JobId, type JobInput } from '../../domain/types/job';
|
|
3
|
+
import type { JobLocation } from '../../domain/types/queue';
|
|
4
|
+
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
5
|
+
import type { MapLike, SetLike } from '../../shared/lru';
|
|
6
|
+
export interface CustomIdContext {
|
|
7
|
+
storage: SqliteStorage | null;
|
|
8
|
+
shards: Shard[];
|
|
9
|
+
completedJobs: SetLike<JobId>;
|
|
10
|
+
completedJobsData: MapLike<JobId, Job>;
|
|
11
|
+
timedOutJobs?: SetLike<JobId>;
|
|
12
|
+
jobResults: MapLike<JobId, unknown>;
|
|
13
|
+
customIdMap: MapLike<string, JobId>;
|
|
14
|
+
jobIndex: Map<JobId, JobLocation>;
|
|
15
|
+
}
|
|
16
|
+
export type CustomIdResult = {
|
|
17
|
+
skip: true;
|
|
18
|
+
existingJob: Job;
|
|
19
|
+
} | {
|
|
20
|
+
skip: true;
|
|
21
|
+
existingId: JobId;
|
|
22
|
+
} | {
|
|
23
|
+
skip: false;
|
|
24
|
+
id: JobId;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Enforce custom-ID idempotency while the caller holds the target shard lock.
|
|
28
|
+
* Live generations are returned unchanged; terminal generations are retired
|
|
29
|
+
* before the deterministic ID is admitted again.
|
|
30
|
+
*/
|
|
31
|
+
export declare function handleCustomId(input: JobInput, ctx: CustomIdContext, lockedShardIndexes: ReadonlySet<number>): CustomIdResult;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { generateJobId, jobId } from '../../domain/types/job';
|
|
2
|
+
import { shardIndex } from '../../shared/hash';
|
|
3
|
+
/**
|
|
4
|
+
* Enforce custom-ID idempotency while the caller holds the target shard lock.
|
|
5
|
+
* Live generations are returned unchanged; terminal generations are retired
|
|
6
|
+
* before the deterministic ID is admitted again.
|
|
7
|
+
*/
|
|
8
|
+
export function handleCustomId(input, ctx, lockedShardIndexes) {
|
|
9
|
+
if (!input.customId) {
|
|
10
|
+
return { skip: false, id: generateJobId() };
|
|
11
|
+
}
|
|
12
|
+
const id = jobId(input.customId);
|
|
13
|
+
const existing = ctx.customIdMap.get(input.customId);
|
|
14
|
+
if (existing && !ctx.completedJobs.has(id)) {
|
|
15
|
+
const location = ctx.jobIndex.get(existing);
|
|
16
|
+
if (location?.type === 'queue') {
|
|
17
|
+
if (!lockedShardIndexes.has(location.shardIdx)) {
|
|
18
|
+
throw new Error('Live custom ID shard must be locked before lookup');
|
|
19
|
+
}
|
|
20
|
+
const existingShard = ctx.shards[location.shardIdx];
|
|
21
|
+
const existingJob = existingShard.getQueue(location.queueName).find(existing);
|
|
22
|
+
if (existingJob) {
|
|
23
|
+
return { skip: true, existingJob };
|
|
24
|
+
}
|
|
25
|
+
if (existingShard.waitingDeps.has(existing)) {
|
|
26
|
+
return { skip: true, existingId: id };
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
else if (location?.type === 'processing') {
|
|
30
|
+
return { skip: true, existingId: id };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (ctx.completedJobs.has(id)) {
|
|
34
|
+
ctx.completedJobs.delete(id);
|
|
35
|
+
ctx.completedJobsData.delete(id);
|
|
36
|
+
ctx.jobResults.delete(id);
|
|
37
|
+
ctx.jobIndex.delete(id);
|
|
38
|
+
ctx.storage?.deleteJob(id);
|
|
39
|
+
}
|
|
40
|
+
const terminalLocation = ctx.jobIndex.get(id);
|
|
41
|
+
if (terminalLocation?.type === 'dlq') {
|
|
42
|
+
const terminalShardIdx = shardIndex(terminalLocation.queueName);
|
|
43
|
+
if (!lockedShardIndexes.has(terminalShardIdx)) {
|
|
44
|
+
throw new Error('Terminal custom ID shard must be locked before reuse');
|
|
45
|
+
}
|
|
46
|
+
const terminalShard = ctx.shards[terminalShardIdx];
|
|
47
|
+
terminalShard.removeFromDlq(terminalLocation.queueName, id);
|
|
48
|
+
ctx.jobIndex.delete(id);
|
|
49
|
+
// deleteDlqEntry removes every persisted generation for this deterministic
|
|
50
|
+
// id. The fresh jobs row is inserted only after this retirement completes.
|
|
51
|
+
ctx.storage?.deleteDlqEntry(id);
|
|
52
|
+
}
|
|
53
|
+
// A durable jobs row may outlive its in-memory tracking after an interrupted
|
|
54
|
+
// cleanup. The storage insert upserts that orphan without adding a DELETE to
|
|
55
|
+
// every custom-ID push.
|
|
56
|
+
ctx.timedOutJobs?.delete(id);
|
|
57
|
+
ctx.customIdMap.set(input.customId, id);
|
|
58
|
+
return { skip: false, id };
|
|
59
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cleanup shared ownership state when a management operation claims an active
|
|
3
|
+
* job before its worker can ACK or FAIL it.
|
|
4
|
+
*/
|
|
5
|
+
import type { JobId, JobLock } from '../../domain/types/job';
|
|
6
|
+
export interface JobClaimContext {
|
|
7
|
+
jobLocks: Map<JobId, JobLock>;
|
|
8
|
+
clientJobs: Map<string, Set<JobId>>;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Release the live lease and detach the job from its TCP client owner.
|
|
12
|
+
*
|
|
13
|
+
* Map/Set deletion is intentionally idempotent: management commands can race
|
|
14
|
+
* with disconnect cleanup, but ownership must be removed exactly once from the
|
|
15
|
+
* observable state.
|
|
16
|
+
*/
|
|
17
|
+
export declare function releaseClaimedJobOwnership(jobId: JobId, ctx: JobClaimContext): void;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cleanup shared ownership state when a management operation claims an active
|
|
3
|
+
* job before its worker can ACK or FAIL it.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Release the live lease and detach the job from its TCP client owner.
|
|
7
|
+
*
|
|
8
|
+
* Map/Set deletion is intentionally idempotent: management commands can race
|
|
9
|
+
* with disconnect cleanup, but ownership must be removed exactly once from the
|
|
10
|
+
* observable state.
|
|
11
|
+
*/
|
|
12
|
+
export function releaseClaimedJobOwnership(jobId, ctx) {
|
|
13
|
+
ctx.jobLocks.delete(jobId);
|
|
14
|
+
for (const [clientId, jobs] of ctx.clientJobs) {
|
|
15
|
+
if (!jobs.delete(jobId))
|
|
16
|
+
continue;
|
|
17
|
+
if (jobs.size === 0)
|
|
18
|
+
ctx.clientJobs.delete(clientId);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
* Job Management Operations
|
|
3
3
|
* Cancel, update progress, change priority, promote, move to delayed, discard
|
|
4
4
|
*/
|
|
5
|
-
import type { Job, JobId } from '../../domain/types/job';
|
|
5
|
+
import type { Job, JobId, JobLock } from '../../domain/types/job';
|
|
6
6
|
import { type JobLocation } from '../../domain/types/queue';
|
|
7
7
|
import type { Shard } from '../../domain/queue/shard';
|
|
8
8
|
import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
|
|
9
9
|
import type { WebhookManager } from '../webhookManager';
|
|
10
10
|
import type { EventsManager } from '../eventsManager';
|
|
11
11
|
import { type RWLock } from '../../shared/lock';
|
|
12
|
+
export { discardJob, moveJobToDelayed } from './jobMoveOperations';
|
|
12
13
|
/** Context for job management operations */
|
|
13
14
|
export interface JobManagementContext {
|
|
14
15
|
storage: SqliteStorage | null;
|
|
@@ -17,6 +18,8 @@ export interface JobManagementContext {
|
|
|
17
18
|
processingShards: Map<JobId, Job>[];
|
|
18
19
|
processingLocks: RWLock[];
|
|
19
20
|
jobIndex: Map<JobId, JobLocation>;
|
|
21
|
+
jobLocks: Map<JobId, JobLock>;
|
|
22
|
+
clientJobs: Map<string, Set<JobId>>;
|
|
20
23
|
webhookManager: WebhookManager;
|
|
21
24
|
eventsManager: EventsManager;
|
|
22
25
|
repeatChain?: Map<JobId, JobId>;
|
|
@@ -29,7 +32,3 @@ export declare function updateJobProgress(jobId: JobId, progress: number, ctx: J
|
|
|
29
32
|
export declare function updateJobData(jobId: JobId, data: unknown, ctx: JobManagementContext): Promise<boolean>;
|
|
30
33
|
/** Change job priority */
|
|
31
34
|
export declare function changeJobPriority(jobId: JobId, priority: number, ctx: JobManagementContext, lifo?: boolean): Promise<boolean>;
|
|
32
|
-
/** Move active job back to delayed */
|
|
33
|
-
export declare function moveJobToDelayed(jobId: JobId, delay: number, ctx: JobManagementContext): Promise<boolean>;
|
|
34
|
-
/** Discard job to DLQ */
|
|
35
|
-
export declare function discardJob(jobId: JobId, ctx: JobManagementContext): Promise<boolean>;
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
* Job Management Operations
|
|
3
3
|
* Cancel, update progress, change priority, promote, move to delayed, discard
|
|
4
4
|
*/
|
|
5
|
-
import {
|
|
5
|
+
import { processingShardIndex } from '../../shared/hash';
|
|
6
6
|
import { webhookLog } from '../../shared/logger';
|
|
7
7
|
import { withWriteLock } from '../../shared/lock';
|
|
8
|
+
export { discardJob, moveJobToDelayed } from './jobMoveOperations';
|
|
8
9
|
/** Cancel a job (remove from queue) */
|
|
9
10
|
export async function cancelJob(jobId, ctx) {
|
|
10
11
|
const location = ctx.jobIndex.get(jobId);
|
|
@@ -108,6 +109,7 @@ export async function updateJobData(jobId, data, ctx) {
|
|
|
108
109
|
const job = shard.getQueue(location.queueName).find(jobId) ?? shard.waitingDeps.get(jobId);
|
|
109
110
|
if (job) {
|
|
110
111
|
job.data = data;
|
|
112
|
+
ctx.storage?.updateJobData(jobId, data);
|
|
111
113
|
return true;
|
|
112
114
|
}
|
|
113
115
|
return false;
|
|
@@ -119,6 +121,7 @@ export async function updateJobData(jobId, data, ctx) {
|
|
|
119
121
|
const job = ctx.processingShards[procIdx].get(jobId);
|
|
120
122
|
if (job) {
|
|
121
123
|
job.data = data;
|
|
124
|
+
ctx.storage?.updateJobData(jobId, data);
|
|
122
125
|
return true;
|
|
123
126
|
}
|
|
124
127
|
return false;
|
|
@@ -147,6 +150,7 @@ async function updateRepeatSuccessor(originalId, data, ctx) {
|
|
|
147
150
|
.find(successorId);
|
|
148
151
|
if (job) {
|
|
149
152
|
job.data = data;
|
|
153
|
+
ctx.storage?.updateJobData(successorId, data);
|
|
150
154
|
return true;
|
|
151
155
|
}
|
|
152
156
|
return false;
|
|
@@ -159,120 +163,15 @@ export async function changeJobPriority(jobId, priority, ctx, lifo) {
|
|
|
159
163
|
return false;
|
|
160
164
|
return withWriteLock(ctx.shardLocks[location.shardIdx], () => {
|
|
161
165
|
const q = ctx.shards[location.shardIdx].getQueue(location.queueName);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
// First remove from processing with lock
|
|
172
|
-
const job = await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
173
|
-
const job = ctx.processingShards[procIdx].get(jobId);
|
|
174
|
-
if (job) {
|
|
175
|
-
ctx.processingShards[procIdx].delete(jobId);
|
|
176
|
-
}
|
|
177
|
-
return job;
|
|
178
|
-
});
|
|
179
|
-
if (!job)
|
|
180
|
-
return false;
|
|
181
|
-
// Then add back to queue with lock
|
|
182
|
-
const now = Date.now();
|
|
183
|
-
job.runAt = now + delay;
|
|
184
|
-
job.startedAt = null;
|
|
185
|
-
const idx = shardIndex(job.queue);
|
|
186
|
-
const queueName = job.queue;
|
|
187
|
-
await withWriteLock(ctx.shardLocks[idx], () => {
|
|
188
|
-
const shard = ctx.shards[idx];
|
|
189
|
-
// Release the concurrency slot (+group+uniqueKey) acquired at pull before
|
|
190
|
-
// re-queueing, mirroring moveActiveToWait (jobStateTransitions.ts) --
|
|
191
|
-
// otherwise the slot leaks and setConcurrency(N) wedges after N moves.
|
|
192
|
-
// Dropping the uniqueKey reservation on requeue matches the sibling
|
|
193
|
-
// requeue paths (failJob retry, moveActiveToWait, requeueExpiredJob).
|
|
194
|
-
shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
|
|
195
|
-
shard.getQueue(job.queue).push(job);
|
|
196
|
-
// Update running counters for O(1) stats and temporal index (job is delayed since delay > 0)
|
|
197
|
-
const isDelayed = job.runAt > now;
|
|
198
|
-
shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
|
|
199
|
-
ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
|
|
200
|
-
// The freed slot may unblock waiting jobs for long-pollers.
|
|
201
|
-
shard.notify(job.queue);
|
|
202
|
-
});
|
|
203
|
-
// Persist the move so the delay survives a restart: the on-disk row was
|
|
204
|
-
// `state='active'` with the old run_at, which recovery would otherwise treat as a
|
|
205
|
-
// stalled active job. updateRunAt re-derives 'delayed' from the future run_at and
|
|
206
|
-
// clears started_at.
|
|
207
|
-
ctx.storage?.updateRunAt(jobId, job.runAt);
|
|
208
|
-
// Emit delayed event (BullMQ v5)
|
|
209
|
-
ctx.eventsManager.broadcast({
|
|
210
|
-
eventType: "delayed" /* EventType.Delayed */,
|
|
211
|
-
jobId,
|
|
212
|
-
queue: queueName,
|
|
213
|
-
timestamp: Date.now(),
|
|
214
|
-
delay,
|
|
215
|
-
});
|
|
216
|
-
return true;
|
|
217
|
-
}
|
|
218
|
-
/** Discard job to DLQ */
|
|
219
|
-
export async function discardJob(jobId, ctx) {
|
|
220
|
-
const location = ctx.jobIndex.get(jobId);
|
|
221
|
-
if (!location)
|
|
222
|
-
return false;
|
|
223
|
-
let job = null;
|
|
224
|
-
if (location.type === 'queue') {
|
|
225
|
-
job = await withWriteLock(ctx.shardLocks[location.shardIdx], () => {
|
|
226
|
-
const shard = ctx.shards[location.shardIdx];
|
|
227
|
-
const removed = shard.getQueue(location.queueName).remove(jobId);
|
|
228
|
-
if (removed) {
|
|
229
|
-
// Update running counters for O(1) stats
|
|
230
|
-
shard.decrementQueued(jobId);
|
|
231
|
-
}
|
|
232
|
-
return removed;
|
|
233
|
-
});
|
|
234
|
-
}
|
|
235
|
-
else if (location.type === 'processing') {
|
|
236
|
-
const procIdx = processingShardIndex(jobId);
|
|
237
|
-
job = await withWriteLock(ctx.processingLocks[procIdx], () => {
|
|
238
|
-
const j = ctx.processingShards[procIdx].get(jobId) ?? null;
|
|
239
|
-
if (j)
|
|
240
|
-
ctx.processingShards[procIdx].delete(jobId);
|
|
241
|
-
return j;
|
|
242
|
-
});
|
|
243
|
-
}
|
|
244
|
-
if (job) {
|
|
245
|
-
const validJob = job; // Local reference for closure
|
|
246
|
-
const idx = shardIndex(validJob.queue);
|
|
247
|
-
const fromProcessing = location.type === 'processing';
|
|
248
|
-
const entry = await withWriteLock(ctx.shardLocks[idx], () => {
|
|
249
|
-
const shard = ctx.shards[idx];
|
|
250
|
-
if (fromProcessing) {
|
|
251
|
-
// The pull that activated this job acquired a concurrency slot (and
|
|
252
|
-
// possibly a FIFO group); return them before the DLQ move or the slot
|
|
253
|
-
// leaks and setConcurrency(N) wedges after N discards. The uniqueKey
|
|
254
|
-
// is freed too: every sibling terminal DLQ path releases the full
|
|
255
|
-
// reservation on entry (failJob, ack.ts; handleMaxStallsExceeded,
|
|
256
|
-
// lockManager.ts), so the DLQ entry never keeps it.
|
|
257
|
-
shard.releaseJobResources(validJob.queue, validJob.uniqueKey, validJob.groupId);
|
|
258
|
-
// The freed slot may unblock waiting jobs for long-pollers.
|
|
259
|
-
shard.notify(validJob.queue);
|
|
260
|
-
}
|
|
261
|
-
else if (validJob.uniqueKey) {
|
|
262
|
-
// Queue branch: the waiting job never acquired a slot or group at
|
|
263
|
-
// pull, so a full releaseJobResources would free a slot legitimately
|
|
264
|
-
// held by ANOTHER active job. Release only the uniqueKey reservation,
|
|
265
|
-
// matching cancelJob and the DLQ-entry semantics above.
|
|
266
|
-
shard.releaseUniqueKey(validJob.queue, validJob.uniqueKey);
|
|
267
|
-
}
|
|
268
|
-
// addToDlq already updates dlq counter and returns entry
|
|
269
|
-
const dlqEntry = shard.addToDlq(validJob);
|
|
270
|
-
ctx.jobIndex.set(jobId, { type: 'dlq', queueName: validJob.queue });
|
|
271
|
-
return dlqEntry;
|
|
272
|
-
});
|
|
273
|
-
ctx.storage?.saveDlqEntry(entry);
|
|
274
|
-
ctx.storage?.deleteJob(jobId);
|
|
166
|
+
if (!q.updatePriority(jobId, priority, lifo))
|
|
167
|
+
return false;
|
|
168
|
+
// updatePriority replaces the immutable Job held by the indexed heap.
|
|
169
|
+
// Persist the effective LIFO value from that replacement so an omitted
|
|
170
|
+
// optional argument preserves the existing tie-break across recovery.
|
|
171
|
+
const updatedJob = q.find(jobId);
|
|
172
|
+
if (!updatedJob)
|
|
173
|
+
return false;
|
|
174
|
+
ctx.storage?.updateJobPriority(jobId, updatedJob.priority, updatedJob.lifo);
|
|
275
175
|
return true;
|
|
276
|
-
}
|
|
277
|
-
return false;
|
|
176
|
+
});
|
|
278
177
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Job move operations that claim active jobs for delayed or DLQ state.
|
|
3
|
+
*/
|
|
4
|
+
import type { JobId } from '../../domain/types/job';
|
|
5
|
+
import type { JobManagementContext } from './jobManagement';
|
|
6
|
+
/** Move active job back to delayed. */
|
|
7
|
+
export declare function moveJobToDelayed(jobId: JobId, delay: number, ctx: JobManagementContext): Promise<boolean>;
|
|
8
|
+
/** Discard a waiting or active job to the DLQ. */
|
|
9
|
+
export declare function discardJob(jobId: JobId, ctx: JobManagementContext): Promise<boolean>;
|