bunqueue 2.8.35 → 2.8.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +14 -2
  2. package/dist/application/backgroundTasks.js +28 -5
  3. package/dist/application/contextFactory.js +5 -0
  4. package/dist/application/dlqManager.d.ts +6 -0
  5. package/dist/application/dlqManager.js +35 -14
  6. package/dist/application/lockManager.js +22 -8
  7. package/dist/application/operations/customId.d.ts +31 -0
  8. package/dist/application/operations/customId.js +59 -0
  9. package/dist/application/operations/jobClaim.d.ts +17 -0
  10. package/dist/application/operations/jobClaim.js +20 -0
  11. package/dist/application/operations/jobManagement.d.ts +4 -5
  12. package/dist/application/operations/jobManagement.js +15 -116
  13. package/dist/application/operations/jobMoveOperations.d.ts +9 -0
  14. package/dist/application/operations/jobMoveOperations.js +94 -0
  15. package/dist/application/operations/jobStateTransitions.js +3 -0
  16. package/dist/application/operations/pull.js +5 -0
  17. package/dist/application/operations/push.js +8 -103
  18. package/dist/application/operations/pushInsert.d.ts +12 -0
  19. package/dist/application/operations/pushInsert.js +25 -0
  20. package/dist/application/operations/pushLocks.d.ts +14 -0
  21. package/dist/application/operations/pushLocks.js +43 -0
  22. package/dist/application/operations/queueControl.d.ts +1 -1
  23. package/dist/application/operations/queueControl.js +1 -1
  24. package/dist/application/queueManager.d.ts +1 -1
  25. package/dist/application/queueManager.js +17 -6
  26. package/dist/application/stallDetection.js +17 -8
  27. package/dist/client/queue/helpers.js +2 -0
  28. package/dist/client/queue/operations/counts.d.ts +1 -0
  29. package/dist/client/queue/operations/counts.js +3 -0
  30. package/dist/domain/queue/dependencyTracker.d.ts +5 -0
  31. package/dist/domain/queue/dependencyTracker.js +20 -0
  32. package/dist/domain/queue/shard.d.ts +1 -1
  33. package/dist/domain/queue/shard.js +6 -0
  34. package/dist/infrastructure/persistence/schema.d.ts +2 -2
  35. package/dist/infrastructure/persistence/schema.js +25 -2
  36. package/dist/infrastructure/persistence/sqlite.d.ts +11 -0
  37. package/dist/infrastructure/persistence/sqlite.js +59 -5
  38. package/dist/infrastructure/persistence/sqliteBatch.js +8 -6
  39. package/dist/infrastructure/persistence/sqliteSerializer.d.ts +6 -0
  40. package/dist/infrastructure/persistence/sqliteSerializer.js +9 -1
  41. package/dist/infrastructure/persistence/statements.d.ts +6 -1
  42. package/dist/infrastructure/persistence/statements.js +7 -5
  43. package/package.json +4 -1
package/README.md CHANGED
@@ -830,8 +830,20 @@ full feature parity, so producers and workers can live anywhere in your stack:
830
830
 
831
831
  | Where your code runs | Install | Status |
832
832
  | -------------------- | ------- | ------ |
833
- | **Node.js ≥ 20, Deno ≥ 2, Bun, Cloudflare Workers** | [`npm install bunqueue-client`](https://www.npmjs.com/package/bunqueue-client) | 100/100 e2e per runtime + 16/16 inside workerd |
834
- | **Python ≥ 3.9** | PyPI coming soon — today: `pip install "git+https://github.com/egeominotti/bunqueue.git#subdirectory=sdk/python"` | 78/78 e2e |
833
+ | **Node.js ≥ 20, Deno ≥ 2, Bun, Cloudflare Workers** | [`npm install bunqueue-client`](https://www.npmjs.com/package/bunqueue-client) | Native integration/e2e + protocol conformance |
834
+ | **Python ≥ 3.9** | PyPI coming soon — today: `pip install "git+https://github.com/egeominotti/bunqueue.git#subdirectory=sdk/python"` | Native integration/e2e + protocol conformance |
835
+ | **PHP ≥ 8.1** | Packagist coming soon — today: use [`sdk/php`](./sdk/php) | Native e2e + protocol conformance |
836
+ | **Go 1.26.5** | `go get github.com/egeominotti/bunqueue/sdk/go` | Native tests + protocol conformance |
837
+ | **Rust ≥ 1.85** | crates.io coming soon — today: use [`sdk/rust`](./sdk/rust) as a path dependency | Native unit/integration + protocol conformance |
838
+ | **Elixir ≥ 1.15** | Hex coming soon — today: use [`sdk/elixir`](./sdk/elixir) as a path dependency | ExUnit integration + protocol conformance |
839
+
840
+ All six SDKs run the same production hardening matrix: independent-connection
841
+ idempotency and single-lease races, generated payload invariants,
842
+ malformed-input fuzz corpora, broker SIGKILL/reconnect durability, bounded
843
+ spikes, structured telemetry, and 17 shared protocol checks. Weekly CI adds
844
+ runtime compatibility, live dependency advisories, native Go race/fuzz checks,
845
+ and a 15-minute sustained profile per SDK. See
846
+ [`docs/testing.md`](./docs/testing.md#sdk-hardening-matrix).
835
847
 
836
848
  ```typescript
837
849
  // Node.js / Deno / Cloudflare Workers
@@ -144,6 +144,8 @@ function performDlqMaintenance(ctx) {
144
144
  const dlqCtx = {
145
145
  shards: ctx.shards,
146
146
  jobIndex: ctx.jobIndex,
147
+ jobResults: ctx.jobResults,
148
+ jobLogs: ctx.jobLogs,
147
149
  storage: ctx.storage,
148
150
  };
149
151
  for (const queueName of ctx.queueNamesCache) {
@@ -198,6 +200,15 @@ export function recover(ctx) {
198
200
  // Load DLQ job IDs so Phase 1 can skip stale active rows for DLQ'd jobs
199
201
  // (legacy DBs predate the DLQ-row cleanup fix in failJob).
200
202
  const dlqJobIds = ctx.storage.loadDlqJobIds();
203
+ const queueStates = ctx.storage.loadQueueState();
204
+ // Recovery bounds are needed before Phase 1 classifies interrupted active
205
+ // jobs. Restore the stall policy first; paused/limiter state can wait until
206
+ // the queues themselves have been rebuilt below.
207
+ for (const qs of queueStates) {
208
+ if (qs.stallConfig) {
209
+ ctx.shards[shardIndex(qs.name)].setStallConfig(qs.name, qs.stallConfig);
210
+ }
211
+ }
201
212
  const now = Date.now();
202
213
  // === PHASE 1: Recover active jobs (were processing when server stopped) ===
203
214
  // These jobs are considered "stalled" and need to be retried or moved to DLQ
@@ -240,12 +251,24 @@ export function recover(ctx) {
240
251
  job.attempts++;
241
252
  job.startedAt = null;
242
253
  job.lastHeartbeat = now;
243
- // Check if exceeded max stalls
254
+ // A crash consumes one processing attempt and one stall allowance. Both
255
+ // bounds are terminal: requeueing after either reaches its configured
256
+ // maximum would make attempts/stalls unbounded across restart loops.
244
257
  const maxStalls = stallConfig.maxStalls ?? 3;
245
- if (job.stallCount >= maxStalls) {
258
+ const attemptsExhausted = job.attempts >= job.maxAttempts;
259
+ const stallsExhausted = maxStalls > 0 && job.stallCount >= maxStalls;
260
+ if (attemptsExhausted || stallsExhausted) {
246
261
  // Move to DLQ
247
- const entry = shard.addToDlq(job, "stalled" /* FailureReason.Stalled */, `Job stalled ${job.stallCount} times (recovered at startup)`);
248
- ctx.jobIndex.set(job.id, { type: 'dlq', queueName: job.queue });
262
+ const reason = attemptsExhausted
263
+ ? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */
264
+ : "stalled" /* FailureReason.Stalled */;
265
+ const error = attemptsExhausted
266
+ ? `Job reached max attempts (${job.maxAttempts}) during startup recovery`
267
+ : `Job stalled ${job.stallCount} times (recovered at startup)`;
268
+ // Persist only here; the later loadDlq() phase is the single in-memory
269
+ // restore path. Adding to the shard now and restoring the same row below
270
+ // would duplicate the DLQ entry after every terminal crash recovery.
271
+ const entry = createDlqEntry(job, reason, error, shard.getDlqConfig(job.queue));
249
272
  ctx.storage.saveDlqEntry(entry);
250
273
  ctx.storage.deleteJob(job.id);
251
274
  }
@@ -338,7 +361,7 @@ export function recover(ctx) {
338
361
  // Map; without this load every queue silently un-pauses and loses its limits
339
362
  // on restart. Applied directly to the owning shard (in-memory only — these
340
363
  // setters do not re-persist, so there is no write-back loop).
341
- for (const qs of ctx.storage.loadQueueState()) {
364
+ for (const qs of queueStates) {
342
365
  const shard = ctx.shards[shardIndex(qs.name)];
343
366
  if (qs.paused)
344
367
  shard.pause(qs.name);
@@ -141,6 +141,8 @@ export class ContextFactory {
141
141
  processingShards: this.deps.processingShards,
142
142
  processingLocks: this.deps.processingLocks,
143
143
  jobIndex: this.deps.jobIndex,
144
+ jobLocks: this.deps.jobLocks,
145
+ clientJobs: this.deps.clientJobs,
144
146
  webhookManager: this.deps.webhookManager,
145
147
  eventsManager: this.deps.eventsManager,
146
148
  repeatChain: this.deps.repeatChain,
@@ -164,6 +166,8 @@ export class ContextFactory {
164
166
  return {
165
167
  shards: this.deps.shards,
166
168
  jobIndex: this.deps.jobIndex,
169
+ jobResults: this.deps.jobResults,
170
+ jobLogs: this.deps.jobLogs,
167
171
  storage: this.deps.storage,
168
172
  };
169
173
  }
@@ -171,6 +175,7 @@ export class ContextFactory {
171
175
  return {
172
176
  shards: this.deps.shards,
173
177
  jobIndex: this.deps.jobIndex,
178
+ jobLogs: this.deps.jobLogs,
174
179
  storage: this.deps.storage,
175
180
  completedJobs: this.deps.completedJobs,
176
181
  jobResults: this.deps.jobResults,
@@ -11,6 +11,12 @@ import type { SqliteStorage } from '../infrastructure/persistence/sqlite';
11
11
  export interface DlqContext {
12
12
  shards: Shard[];
13
13
  jobIndex: Map<JobId, JobLocation>;
14
+ jobResults: {
15
+ delete(id: JobId): boolean;
16
+ };
17
+ jobLogs: {
18
+ delete(id: JobId): boolean;
19
+ };
14
20
  storage: SqliteStorage | null;
15
21
  }
16
22
  /** Get jobs from DLQ (backward compatible) */
@@ -209,24 +209,45 @@ export function processAutoRetry(queue, ctx) {
209
209
  export function purgeExpiredDlq(queue, ctx) {
210
210
  const idx = shardIndex(queue);
211
211
  const shard = ctx.shards[idx];
212
- // Get expired entries before purging (to delete from SQLite)
213
- const expiredEntries = shard.getExpiredEntries(queue);
214
- const count = shard.purgeExpired(queue);
215
- // Delete from SQLite
216
- if (ctx.storage && expiredEntries.length > 0) {
217
- for (const entry of expiredEntries) {
218
- ctx.storage.deleteDlqEntry(entry.job.id);
219
- }
220
- }
221
- return count;
212
+ const now = Date.now();
213
+ // Use one timestamp for the snapshot and mutation so every purged entry is
214
+ // represented in the cleanup set.
215
+ const expiredEntries = shard.getExpiredEntries(queue, now);
216
+ if (expiredEntries.length === 0)
217
+ return 0;
218
+ cleanupPurgedEntries(queue, expiredEntries, ctx, false);
219
+ shard.purgeExpired(queue, now);
220
+ return expiredEntries.length;
222
221
  }
223
222
  /** Purge all jobs from DLQ */
224
223
  export function purgeDlqJobs(queue, ctx) {
225
224
  const idx = shardIndex(queue);
226
- const count = ctx.shards[idx].clearDlq(queue);
227
- // Clear from SQLite
228
- ctx.storage?.clearDlqQueue(queue);
229
- return count;
225
+ const shard = ctx.shards[idx];
226
+ const entries = shard.getDlqEntries(queue);
227
+ cleanupPurgedEntries(queue, entries, ctx, true);
228
+ shard.clearDlq(queue);
229
+ return entries.length;
230
+ }
231
+ /** Remove every durable and global reference owned by terminal DLQ entries. */
232
+ function cleanupPurgedEntries(queue, entries, ctx, clearQueue) {
233
+ const jobIds = [...new Set(entries.map((entry) => entry.job.id))];
234
+ const terminalJobIds = jobIds.filter((jobId) => {
235
+ const location = ctx.jobIndex.get(jobId);
236
+ return location === undefined || (location.type === 'dlq' && location.queueName === queue);
237
+ });
238
+ // One SQLite transaction removes DLQ rows, any orphan jobs/result rows, and
239
+ // pending buffered inserts before in-memory references become unobservable.
240
+ // A newer live generation can reuse the same custom id; its jobs row and
241
+ // auxiliary state must survive cleanup of the stale terminal generation.
242
+ ctx.storage?.purgeDlqEntries(queue, jobIds, terminalJobIds, clearQueue);
243
+ for (const jobId of terminalJobIds) {
244
+ const location = ctx.jobIndex.get(jobId);
245
+ if (location?.type === 'dlq' && location.queueName === queue) {
246
+ ctx.jobIndex.delete(jobId);
247
+ }
248
+ ctx.jobResults.delete(jobId);
249
+ ctx.jobLogs.delete(jobId);
250
+ }
230
251
  }
231
252
  /** Configure DLQ for a queue */
232
253
  export function configureDlq(queue, ctx, config) {
@@ -92,10 +92,20 @@ function processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now)
92
92
  job.startedAt = null;
93
93
  job.lastHeartbeat = now;
94
94
  job.stallCount++;
95
- // Check if max stalls exceeded
95
+ // A lost lock consumes both retry budgets. Never requeue beyond either bound.
96
96
  const stallConfig = shard.getStallConfig(job.queue);
97
- if (stallConfig.maxStalls > 0 && job.stallCount >= stallConfig.maxStalls) {
98
- handleMaxStallsExceeded({ jobId, job, lock, shard, ctx, now });
97
+ const attemptsExhausted = job.attempts >= job.maxAttempts;
98
+ const stallsExhausted = stallConfig.maxStalls > 0 && job.stallCount >= stallConfig.maxStalls;
99
+ if (attemptsExhausted || stallsExhausted) {
100
+ handleRecoveryBoundExceeded({
101
+ jobId,
102
+ job,
103
+ lock,
104
+ shard,
105
+ ctx,
106
+ now,
107
+ attemptsExhausted,
108
+ });
99
109
  }
100
110
  else {
101
111
  requeueExpiredJob({ jobId, job, lock, queue, idx: shardIdx, ctx, now });
@@ -108,14 +118,16 @@ function processExpiredLockInner(jobId, lock, job, shardIdx, procIdx, ctx, now)
108
118
  renewalCount: lock.renewalCount,
109
119
  });
110
120
  }
111
- /** Move job to DLQ when max stalls exceeded */
112
- function handleMaxStallsExceeded(opts) {
113
- const { jobId, job, lock, shard, ctx, now } = opts;
121
+ /** Move job to DLQ when either crash-recovery budget is exhausted. */
122
+ function handleRecoveryBoundExceeded(opts) {
123
+ const { jobId, job, lock, shard, ctx, now, attemptsExhausted } = opts;
114
124
  // Release the concurrency slot (+group+uniqueKey) acquired at pull before
115
125
  // moving to DLQ — otherwise the slot leaks (mirrors
116
126
  // stallDetection.moveStalliedJobToDlq).
117
127
  shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
118
- const entry = shard.addToDlq(job, "stalled" /* FailureReason.Stalled */, `Lock expired after ${lock.renewalCount} renewals`);
128
+ const entry = shard.addToDlq(job, attemptsExhausted ? "max_attempts_exceeded" /* FailureReason.MaxAttemptsExceeded */ : "stalled" /* FailureReason.Stalled */, attemptsExhausted
129
+ ? `Lock expired at max attempts (${job.maxAttempts})`
130
+ : `Lock expired after ${lock.renewalCount} renewals`);
119
131
  ctx.jobIndex.set(jobId, { type: 'dlq', queueName: job.queue });
120
132
  // Persist the DLQ move like the sibling paths (ack.moveFailedJobToDlq,
121
133
  // stallDetection.moveStalliedJobToDlq, backgroundTasks startup-recovery).
@@ -129,7 +141,9 @@ function handleMaxStallsExceeded(opts) {
129
141
  jobId,
130
142
  queue: job.queue,
131
143
  timestamp: now,
132
- error: 'Lock expired (max stalls reached)',
144
+ error: attemptsExhausted
145
+ ? 'Lock expired (max attempts reached)'
146
+ : 'Lock expired (max stalls reached)',
133
147
  });
134
148
  }
135
149
  /** Requeue job for retry */
@@ -0,0 +1,31 @@
1
+ import type { Shard } from '../../domain/queue/shard';
2
+ import { type Job, type JobId, type JobInput } from '../../domain/types/job';
3
+ import type { JobLocation } from '../../domain/types/queue';
4
+ import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
5
+ import type { MapLike, SetLike } from '../../shared/lru';
6
+ export interface CustomIdContext {
7
+ storage: SqliteStorage | null;
8
+ shards: Shard[];
9
+ completedJobs: SetLike<JobId>;
10
+ completedJobsData: MapLike<JobId, Job>;
11
+ timedOutJobs?: SetLike<JobId>;
12
+ jobResults: MapLike<JobId, unknown>;
13
+ customIdMap: MapLike<string, JobId>;
14
+ jobIndex: Map<JobId, JobLocation>;
15
+ }
16
+ export type CustomIdResult = {
17
+ skip: true;
18
+ existingJob: Job;
19
+ } | {
20
+ skip: true;
21
+ existingId: JobId;
22
+ } | {
23
+ skip: false;
24
+ id: JobId;
25
+ };
26
+ /**
27
+ * Enforce custom-ID idempotency while the caller holds the target shard lock.
28
+ * Live generations are returned unchanged; terminal generations are retired
29
+ * before the deterministic ID is admitted again.
30
+ */
31
+ export declare function handleCustomId(input: JobInput, ctx: CustomIdContext, lockedShardIndexes: ReadonlySet<number>): CustomIdResult;
@@ -0,0 +1,59 @@
1
+ import { generateJobId, jobId } from '../../domain/types/job';
2
+ import { shardIndex } from '../../shared/hash';
3
+ /**
4
+ * Enforce custom-ID idempotency while the caller holds the target shard lock.
5
+ * Live generations are returned unchanged; terminal generations are retired
6
+ * before the deterministic ID is admitted again.
7
+ */
8
+ export function handleCustomId(input, ctx, lockedShardIndexes) {
9
+ if (!input.customId) {
10
+ return { skip: false, id: generateJobId() };
11
+ }
12
+ const id = jobId(input.customId);
13
+ const existing = ctx.customIdMap.get(input.customId);
14
+ if (existing && !ctx.completedJobs.has(id)) {
15
+ const location = ctx.jobIndex.get(existing);
16
+ if (location?.type === 'queue') {
17
+ if (!lockedShardIndexes.has(location.shardIdx)) {
18
+ throw new Error('Live custom ID shard must be locked before lookup');
19
+ }
20
+ const existingShard = ctx.shards[location.shardIdx];
21
+ const existingJob = existingShard.getQueue(location.queueName).find(existing);
22
+ if (existingJob) {
23
+ return { skip: true, existingJob };
24
+ }
25
+ if (existingShard.waitingDeps.has(existing)) {
26
+ return { skip: true, existingId: id };
27
+ }
28
+ }
29
+ else if (location?.type === 'processing') {
30
+ return { skip: true, existingId: id };
31
+ }
32
+ }
33
+ if (ctx.completedJobs.has(id)) {
34
+ ctx.completedJobs.delete(id);
35
+ ctx.completedJobsData.delete(id);
36
+ ctx.jobResults.delete(id);
37
+ ctx.jobIndex.delete(id);
38
+ ctx.storage?.deleteJob(id);
39
+ }
40
+ const terminalLocation = ctx.jobIndex.get(id);
41
+ if (terminalLocation?.type === 'dlq') {
42
+ const terminalShardIdx = shardIndex(terminalLocation.queueName);
43
+ if (!lockedShardIndexes.has(terminalShardIdx)) {
44
+ throw new Error('Terminal custom ID shard must be locked before reuse');
45
+ }
46
+ const terminalShard = ctx.shards[terminalShardIdx];
47
+ terminalShard.removeFromDlq(terminalLocation.queueName, id);
48
+ ctx.jobIndex.delete(id);
49
+ // deleteDlqEntry removes every persisted generation for this deterministic
50
+ // id. The fresh jobs row is inserted only after this retirement completes.
51
+ ctx.storage?.deleteDlqEntry(id);
52
+ }
53
+ // A durable jobs row may outlive its in-memory tracking after an interrupted
54
+ // cleanup. The storage insert upserts that orphan without adding a DELETE to
55
+ // every custom-ID push.
56
+ ctx.timedOutJobs?.delete(id);
57
+ ctx.customIdMap.set(input.customId, id);
58
+ return { skip: false, id };
59
+ }
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Cleanup shared ownership state when a management operation claims an active
3
+ * job before its worker can ACK or FAIL it.
4
+ */
5
+ import type { JobId, JobLock } from '../../domain/types/job';
6
+ export interface JobClaimContext {
7
+ jobLocks: Map<JobId, JobLock>;
8
+ clientJobs: Map<string, Set<JobId>>;
9
+ }
10
+ /**
11
+ * Release the live lease and detach the job from its TCP client owner.
12
+ *
13
+ * Map/Set deletion is intentionally idempotent: management commands can race
14
+ * with disconnect cleanup, but ownership must be removed exactly once from the
15
+ * observable state.
16
+ */
17
+ export declare function releaseClaimedJobOwnership(jobId: JobId, ctx: JobClaimContext): void;
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Cleanup shared ownership state when a management operation claims an active
3
+ * job before its worker can ACK or FAIL it.
4
+ */
5
+ /**
6
+ * Release the live lease and detach the job from its TCP client owner.
7
+ *
8
+ * Map/Set deletion is intentionally idempotent: management commands can race
9
+ * with disconnect cleanup, but ownership must be removed exactly once from the
10
+ * observable state.
11
+ */
12
+ export function releaseClaimedJobOwnership(jobId, ctx) {
13
+ ctx.jobLocks.delete(jobId);
14
+ for (const [clientId, jobs] of ctx.clientJobs) {
15
+ if (!jobs.delete(jobId))
16
+ continue;
17
+ if (jobs.size === 0)
18
+ ctx.clientJobs.delete(clientId);
19
+ }
20
+ }
@@ -2,13 +2,14 @@
2
2
  * Job Management Operations
3
3
  * Cancel, update progress, change priority, promote, move to delayed, discard
4
4
  */
5
- import type { Job, JobId } from '../../domain/types/job';
5
+ import type { Job, JobId, JobLock } from '../../domain/types/job';
6
6
  import { type JobLocation } from '../../domain/types/queue';
7
7
  import type { Shard } from '../../domain/queue/shard';
8
8
  import type { SqliteStorage } from '../../infrastructure/persistence/sqlite';
9
9
  import type { WebhookManager } from '../webhookManager';
10
10
  import type { EventsManager } from '../eventsManager';
11
11
  import { type RWLock } from '../../shared/lock';
12
+ export { discardJob, moveJobToDelayed } from './jobMoveOperations';
12
13
  /** Context for job management operations */
13
14
  export interface JobManagementContext {
14
15
  storage: SqliteStorage | null;
@@ -17,6 +18,8 @@ export interface JobManagementContext {
17
18
  processingShards: Map<JobId, Job>[];
18
19
  processingLocks: RWLock[];
19
20
  jobIndex: Map<JobId, JobLocation>;
21
+ jobLocks: Map<JobId, JobLock>;
22
+ clientJobs: Map<string, Set<JobId>>;
20
23
  webhookManager: WebhookManager;
21
24
  eventsManager: EventsManager;
22
25
  repeatChain?: Map<JobId, JobId>;
@@ -29,7 +32,3 @@ export declare function updateJobProgress(jobId: JobId, progress: number, ctx: J
29
32
  export declare function updateJobData(jobId: JobId, data: unknown, ctx: JobManagementContext): Promise<boolean>;
30
33
  /** Change job priority */
31
34
  export declare function changeJobPriority(jobId: JobId, priority: number, ctx: JobManagementContext, lifo?: boolean): Promise<boolean>;
32
- /** Move active job back to delayed */
33
- export declare function moveJobToDelayed(jobId: JobId, delay: number, ctx: JobManagementContext): Promise<boolean>;
34
- /** Discard job to DLQ */
35
- export declare function discardJob(jobId: JobId, ctx: JobManagementContext): Promise<boolean>;
@@ -2,9 +2,10 @@
2
2
  * Job Management Operations
3
3
  * Cancel, update progress, change priority, promote, move to delayed, discard
4
4
  */
5
- import { shardIndex, processingShardIndex } from '../../shared/hash';
5
+ import { processingShardIndex } from '../../shared/hash';
6
6
  import { webhookLog } from '../../shared/logger';
7
7
  import { withWriteLock } from '../../shared/lock';
8
+ export { discardJob, moveJobToDelayed } from './jobMoveOperations';
8
9
  /** Cancel a job (remove from queue) */
9
10
  export async function cancelJob(jobId, ctx) {
10
11
  const location = ctx.jobIndex.get(jobId);
@@ -108,6 +109,7 @@ export async function updateJobData(jobId, data, ctx) {
108
109
  const job = shard.getQueue(location.queueName).find(jobId) ?? shard.waitingDeps.get(jobId);
109
110
  if (job) {
110
111
  job.data = data;
112
+ ctx.storage?.updateJobData(jobId, data);
111
113
  return true;
112
114
  }
113
115
  return false;
@@ -119,6 +121,7 @@ export async function updateJobData(jobId, data, ctx) {
119
121
  const job = ctx.processingShards[procIdx].get(jobId);
120
122
  if (job) {
121
123
  job.data = data;
124
+ ctx.storage?.updateJobData(jobId, data);
122
125
  return true;
123
126
  }
124
127
  return false;
@@ -147,6 +150,7 @@ async function updateRepeatSuccessor(originalId, data, ctx) {
147
150
  .find(successorId);
148
151
  if (job) {
149
152
  job.data = data;
153
+ ctx.storage?.updateJobData(successorId, data);
150
154
  return true;
151
155
  }
152
156
  return false;
@@ -159,120 +163,15 @@ export async function changeJobPriority(jobId, priority, ctx, lifo) {
159
163
  return false;
160
164
  return withWriteLock(ctx.shardLocks[location.shardIdx], () => {
161
165
  const q = ctx.shards[location.shardIdx].getQueue(location.queueName);
162
- return q.updatePriority(jobId, priority, lifo);
163
- });
164
- }
165
- /** Move active job back to delayed */
166
- export async function moveJobToDelayed(jobId, delay, ctx) {
167
- const location = ctx.jobIndex.get(jobId);
168
- if (location?.type !== 'processing')
169
- return false;
170
- const procIdx = processingShardIndex(jobId);
171
- // First remove from processing with lock
172
- const job = await withWriteLock(ctx.processingLocks[procIdx], () => {
173
- const job = ctx.processingShards[procIdx].get(jobId);
174
- if (job) {
175
- ctx.processingShards[procIdx].delete(jobId);
176
- }
177
- return job;
178
- });
179
- if (!job)
180
- return false;
181
- // Then add back to queue with lock
182
- const now = Date.now();
183
- job.runAt = now + delay;
184
- job.startedAt = null;
185
- const idx = shardIndex(job.queue);
186
- const queueName = job.queue;
187
- await withWriteLock(ctx.shardLocks[idx], () => {
188
- const shard = ctx.shards[idx];
189
- // Release the concurrency slot (+group+uniqueKey) acquired at pull before
190
- // re-queueing, mirroring moveActiveToWait (jobStateTransitions.ts) --
191
- // otherwise the slot leaks and setConcurrency(N) wedges after N moves.
192
- // Dropping the uniqueKey reservation on requeue matches the sibling
193
- // requeue paths (failJob retry, moveActiveToWait, requeueExpiredJob).
194
- shard.releaseJobResources(job.queue, job.uniqueKey, job.groupId);
195
- shard.getQueue(job.queue).push(job);
196
- // Update running counters for O(1) stats and temporal index (job is delayed since delay > 0)
197
- const isDelayed = job.runAt > now;
198
- shard.incrementQueued(jobId, isDelayed, job.createdAt, job.queue, job.runAt);
199
- ctx.jobIndex.set(jobId, { type: 'queue', shardIdx: idx, queueName: job.queue });
200
- // The freed slot may unblock waiting jobs for long-pollers.
201
- shard.notify(job.queue);
202
- });
203
- // Persist the move so the delay survives a restart: the on-disk row was
204
- // `state='active'` with the old run_at, which recovery would otherwise treat as a
205
- // stalled active job. updateRunAt re-derives 'delayed' from the future run_at and
206
- // clears started_at.
207
- ctx.storage?.updateRunAt(jobId, job.runAt);
208
- // Emit delayed event (BullMQ v5)
209
- ctx.eventsManager.broadcast({
210
- eventType: "delayed" /* EventType.Delayed */,
211
- jobId,
212
- queue: queueName,
213
- timestamp: Date.now(),
214
- delay,
215
- });
216
- return true;
217
- }
218
- /** Discard job to DLQ */
219
- export async function discardJob(jobId, ctx) {
220
- const location = ctx.jobIndex.get(jobId);
221
- if (!location)
222
- return false;
223
- let job = null;
224
- if (location.type === 'queue') {
225
- job = await withWriteLock(ctx.shardLocks[location.shardIdx], () => {
226
- const shard = ctx.shards[location.shardIdx];
227
- const removed = shard.getQueue(location.queueName).remove(jobId);
228
- if (removed) {
229
- // Update running counters for O(1) stats
230
- shard.decrementQueued(jobId);
231
- }
232
- return removed;
233
- });
234
- }
235
- else if (location.type === 'processing') {
236
- const procIdx = processingShardIndex(jobId);
237
- job = await withWriteLock(ctx.processingLocks[procIdx], () => {
238
- const j = ctx.processingShards[procIdx].get(jobId) ?? null;
239
- if (j)
240
- ctx.processingShards[procIdx].delete(jobId);
241
- return j;
242
- });
243
- }
244
- if (job) {
245
- const validJob = job; // Local reference for closure
246
- const idx = shardIndex(validJob.queue);
247
- const fromProcessing = location.type === 'processing';
248
- const entry = await withWriteLock(ctx.shardLocks[idx], () => {
249
- const shard = ctx.shards[idx];
250
- if (fromProcessing) {
251
- // The pull that activated this job acquired a concurrency slot (and
252
- // possibly a FIFO group); return them before the DLQ move or the slot
253
- // leaks and setConcurrency(N) wedges after N discards. The uniqueKey
254
- // is freed too: every sibling terminal DLQ path releases the full
255
- // reservation on entry (failJob, ack.ts; handleMaxStallsExceeded,
256
- // lockManager.ts), so the DLQ entry never keeps it.
257
- shard.releaseJobResources(validJob.queue, validJob.uniqueKey, validJob.groupId);
258
- // The freed slot may unblock waiting jobs for long-pollers.
259
- shard.notify(validJob.queue);
260
- }
261
- else if (validJob.uniqueKey) {
262
- // Queue branch: the waiting job never acquired a slot or group at
263
- // pull, so a full releaseJobResources would free a slot legitimately
264
- // held by ANOTHER active job. Release only the uniqueKey reservation,
265
- // matching cancelJob and the DLQ-entry semantics above.
266
- shard.releaseUniqueKey(validJob.queue, validJob.uniqueKey);
267
- }
268
- // addToDlq already updates dlq counter and returns entry
269
- const dlqEntry = shard.addToDlq(validJob);
270
- ctx.jobIndex.set(jobId, { type: 'dlq', queueName: validJob.queue });
271
- return dlqEntry;
272
- });
273
- ctx.storage?.saveDlqEntry(entry);
274
- ctx.storage?.deleteJob(jobId);
166
+ if (!q.updatePriority(jobId, priority, lifo))
167
+ return false;
168
+ // updatePriority replaces the immutable Job held by the indexed heap.
169
+ // Persist the effective LIFO value from that replacement so an omitted
170
+ // optional argument preserves the existing tie-break across recovery.
171
+ const updatedJob = q.find(jobId);
172
+ if (!updatedJob)
173
+ return false;
174
+ ctx.storage?.updateJobPriority(jobId, updatedJob.priority, updatedJob.lifo);
275
175
  return true;
276
- }
277
- return false;
176
+ });
278
177
  }
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Job move operations that claim active jobs for delayed or DLQ state.
3
+ */
4
+ import type { JobId } from '../../domain/types/job';
5
+ import type { JobManagementContext } from './jobManagement';
6
+ /** Move active job back to delayed. */
7
+ export declare function moveJobToDelayed(jobId: JobId, delay: number, ctx: JobManagementContext): Promise<boolean>;
8
+ /** Discard a waiting or active job to the DLQ. */
9
+ export declare function discardJob(jobId: JobId, ctx: JobManagementContext): Promise<boolean>;