@zudojs/queue 0.1.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +40 -8
  3. package/dist/deadLetter/deadLetter.core.js +1 -1
  4. package/dist/inMemoryQueue/inMemoryQueue.core.d.ts +95 -4
  5. package/dist/inMemoryQueue/inMemoryQueue.core.js +398 -73
  6. package/dist/inMemoryQueue/inMemoryQueue.processing.d.ts +53 -8
  7. package/dist/inMemoryQueue/inMemoryQueue.processing.js +83 -36
  8. package/dist/inMemoryQueue/inMemoryQueue.scheduling.d.ts +17 -1
  9. package/dist/inMemoryQueue/inMemoryQueue.scheduling.js +43 -11
  10. package/dist/index.d.ts +0 -1
  11. package/dist/index.js +0 -2
  12. package/dist/job/job.core.js +20 -12
  13. package/dist/job/job.type.d.ts +3 -2
  14. package/dist/jobOptions/jobOptions.type.d.ts +12 -3
  15. package/dist/jobTypes/index.d.ts +1 -1
  16. package/dist/jobTypes/jobTypes.type.d.ts +9 -0
  17. package/dist/middleware/middleware.core.d.ts +10 -1
  18. package/dist/middleware/middleware.core.js +40 -17
  19. package/dist/processor/index.d.ts +1 -1
  20. package/dist/processor/index.js +1 -1
  21. package/dist/processor/processor.core.js +2 -0
  22. package/dist/processor/processor.type.d.ts +18 -1
  23. package/dist/processor/processor.type.js +15 -0
  24. package/dist/queue/index.d.ts +1 -1
  25. package/dist/queue/queue.core.js +2 -2
  26. package/dist/queue/queue.type.d.ts +108 -2
  27. package/dist/queueEmitter/queueEmitter.core.d.ts +22 -2
  28. package/dist/queueEmitter/queueEmitter.core.js +47 -7
  29. package/dist/queueManager/queueManager.core.js +24 -5
  30. package/dist/queueRegistry/queueRegistry.core.js +15 -0
  31. package/dist/queueRegistry/queueRegistry.type.d.ts +2 -0
  32. package/dist/retryPolicy/retryPolicy.core.d.ts +42 -3
  33. package/dist/retryPolicy/retryPolicy.core.js +82 -13
  34. package/dist/serializer/serializer.core.d.ts +2 -1
  35. package/dist/worker/worker.core.d.ts +4 -0
  36. package/dist/worker/worker.core.js +87 -23
  37. package/dist/worker/worker.type.d.ts +17 -3
  38. package/package.json +21 -14
  39. package/dist/.tsbuildinfo +0 -1
  40. package/dist/deadLetter/deadLetter.core.d.ts.map +0 -1
  41. package/dist/deadLetter/deadLetter.core.js.map +0 -1
  42. package/dist/deadLetter/deadLetter.type.d.ts.map +0 -1
  43. package/dist/deadLetter/deadLetter.type.js.map +0 -1
  44. package/dist/deadLetter/index.d.ts.map +0 -1
  45. package/dist/deadLetter/index.js.map +0 -1
  46. package/dist/inMemoryQueue/inMemoryQueue.core.d.ts.map +0 -1
  47. package/dist/inMemoryQueue/inMemoryQueue.core.js.map +0 -1
  48. package/dist/inMemoryQueue/inMemoryQueue.factory.d.ts.map +0 -1
  49. package/dist/inMemoryQueue/inMemoryQueue.factory.js.map +0 -1
  50. package/dist/inMemoryQueue/inMemoryQueue.processing.d.ts.map +0 -1
  51. package/dist/inMemoryQueue/inMemoryQueue.processing.js.map +0 -1
  52. package/dist/inMemoryQueue/inMemoryQueue.scheduling.d.ts.map +0 -1
  53. package/dist/inMemoryQueue/inMemoryQueue.scheduling.js.map +0 -1
  54. package/dist/inMemoryQueue/index.d.ts.map +0 -1
  55. package/dist/inMemoryQueue/index.js.map +0 -1
  56. package/dist/index.d.ts.map +0 -1
  57. package/dist/index.js.map +0 -1
  58. package/dist/job/index.d.ts.map +0 -1
  59. package/dist/job/index.js.map +0 -1
  60. package/dist/job/job.core.d.ts.map +0 -1
  61. package/dist/job/job.core.js.map +0 -1
  62. package/dist/job/job.type.d.ts.map +0 -1
  63. package/dist/job/job.type.js.map +0 -1
  64. package/dist/jobContext/index.d.ts.map +0 -1
  65. package/dist/jobContext/index.js.map +0 -1
  66. package/dist/jobContext/jobContext.core.d.ts.map +0 -1
  67. package/dist/jobContext/jobContext.core.js.map +0 -1
  68. package/dist/jobContext/jobContext.type.d.ts.map +0 -1
  69. package/dist/jobContext/jobContext.type.js.map +0 -1
  70. package/dist/jobOptions/index.d.ts.map +0 -1
  71. package/dist/jobOptions/index.js.map +0 -1
  72. package/dist/jobOptions/jobOptions.core.d.ts.map +0 -1
  73. package/dist/jobOptions/jobOptions.core.js.map +0 -1
  74. package/dist/jobOptions/jobOptions.type.d.ts.map +0 -1
  75. package/dist/jobOptions/jobOptions.type.js.map +0 -1
  76. package/dist/jobResult/index.d.ts.map +0 -1
  77. package/dist/jobResult/index.js.map +0 -1
  78. package/dist/jobResult/jobResult.core.d.ts.map +0 -1
  79. package/dist/jobResult/jobResult.core.js.map +0 -1
  80. package/dist/jobResult/jobResult.type.d.ts.map +0 -1
  81. package/dist/jobResult/jobResult.type.js.map +0 -1
  82. package/dist/jobTypes/index.d.ts.map +0 -1
  83. package/dist/jobTypes/index.js.map +0 -1
  84. package/dist/jobTypes/jobTypes.type.d.ts.map +0 -1
  85. package/dist/jobTypes/jobTypes.type.js.map +0 -1
  86. package/dist/middleware/index.d.ts.map +0 -1
  87. package/dist/middleware/index.js.map +0 -1
  88. package/dist/middleware/middleware.core.d.ts.map +0 -1
  89. package/dist/middleware/middleware.core.js.map +0 -1
  90. package/dist/middleware/middleware.type.d.ts.map +0 -1
  91. package/dist/middleware/middleware.type.js.map +0 -1
  92. package/dist/processor/index.d.ts.map +0 -1
  93. package/dist/processor/index.js.map +0 -1
  94. package/dist/processor/processor.core.d.ts.map +0 -1
  95. package/dist/processor/processor.core.js.map +0 -1
  96. package/dist/processor/processor.type.d.ts.map +0 -1
  97. package/dist/processor/processor.type.js.map +0 -1
  98. package/dist/queue/index.d.ts.map +0 -1
  99. package/dist/queue/index.js.map +0 -1
  100. package/dist/queue/queue.core.d.ts.map +0 -1
  101. package/dist/queue/queue.core.js.map +0 -1
  102. package/dist/queue/queue.type.d.ts.map +0 -1
  103. package/dist/queue/queue.type.js.map +0 -1
  104. package/dist/queueEmitter/index.d.ts.map +0 -1
  105. package/dist/queueEmitter/index.js.map +0 -1
  106. package/dist/queueEmitter/queueEmitter.core.d.ts.map +0 -1
  107. package/dist/queueEmitter/queueEmitter.core.js.map +0 -1
  108. package/dist/queueEmitter/queueEmitter.type.d.ts.map +0 -1
  109. package/dist/queueEmitter/queueEmitter.type.js.map +0 -1
  110. package/dist/queueEvents/index.d.ts +0 -8
  111. package/dist/queueEvents/index.d.ts.map +0 -1
  112. package/dist/queueEvents/index.js +0 -7
  113. package/dist/queueEvents/index.js.map +0 -1
  114. package/dist/queueEvents/queueEvents.core.d.ts +0 -6
  115. package/dist/queueEvents/queueEvents.core.d.ts.map +0 -1
  116. package/dist/queueEvents/queueEvents.core.js +0 -12
  117. package/dist/queueEvents/queueEvents.core.js.map +0 -1
  118. package/dist/queueEvents/queueEvents.type.d.ts +0 -97
  119. package/dist/queueEvents/queueEvents.type.d.ts.map +0 -1
  120. package/dist/queueEvents/queueEvents.type.js +0 -2
  121. package/dist/queueEvents/queueEvents.type.js.map +0 -1
  122. package/dist/queueManager/index.d.ts.map +0 -1
  123. package/dist/queueManager/index.js.map +0 -1
  124. package/dist/queueManager/queueManager.core.d.ts.map +0 -1
  125. package/dist/queueManager/queueManager.core.js.map +0 -1
  126. package/dist/queueManager/queueManager.type.d.ts.map +0 -1
  127. package/dist/queueManager/queueManager.type.js.map +0 -1
  128. package/dist/queueRegistry/index.d.ts.map +0 -1
  129. package/dist/queueRegistry/index.js.map +0 -1
  130. package/dist/queueRegistry/queueRegistry.core.d.ts.map +0 -1
  131. package/dist/queueRegistry/queueRegistry.core.js.map +0 -1
  132. package/dist/queueRegistry/queueRegistry.type.d.ts.map +0 -1
  133. package/dist/queueRegistry/queueRegistry.type.js.map +0 -1
  134. package/dist/retryPolicy/index.d.ts.map +0 -1
  135. package/dist/retryPolicy/index.js.map +0 -1
  136. package/dist/retryPolicy/retryPolicy.core.d.ts.map +0 -1
  137. package/dist/retryPolicy/retryPolicy.core.js.map +0 -1
  138. package/dist/serializer/index.d.ts.map +0 -1
  139. package/dist/serializer/index.js.map +0 -1
  140. package/dist/serializer/serializer.core.d.ts.map +0 -1
  141. package/dist/serializer/serializer.core.js.map +0 -1
  142. package/dist/serializer/serializer.type.d.ts.map +0 -1
  143. package/dist/serializer/serializer.type.js.map +0 -1
  144. package/dist/worker/index.d.ts.map +0 -1
  145. package/dist/worker/index.js.map +0 -1
  146. package/dist/worker/worker.core.d.ts.map +0 -1
  147. package/dist/worker/worker.core.js.map +0 -1
  148. package/dist/worker/worker.type.d.ts.map +0 -1
  149. package/dist/worker/worker.type.js.map +0 -1
@@ -1,12 +1,24 @@
1
- import { randomBytes } from "node:crypto";
2
- import { QueueError, QueueClosedError, QueueDisposedError, JobDuplicateError, } from "@zudojs/errors";
3
- import { createJob } from "../job/job.core.js";
1
+ import { randomUUID } from "node:crypto";
2
+ import { QueueError, QueueDisposedError, JobDuplicateError, JobSerializationError, JobStalledError, } from "@zudojs/errors";
3
+ import { assertProcessor } from "../processor/processor.type.js";
4
+ import { createJob, updateJobState } from "../job/job.core.js";
4
5
  import { JobState as JobStateEnum, createJobName, } from "../jobTypes/jobTypes.type.js";
5
6
  import { JsonSerializer } from "../serializer/serializer.core.js";
6
7
  import { createInMemoryDeadLetterStore } from "../deadLetter/deadLetter.core.js";
7
8
  import { createNoopQueueEventEmitter } from "../queueEmitter/queueEmitter.core.js";
8
9
  import { processJob } from "./inMemoryQueue.processing.js";
9
- import { scheduleJob, scheduleDelayedJobs, } from "./inMemoryQueue.scheduling.js";
10
+ import { scheduleJob, promoteDueScheduledJobs, } from "./inMemoryQueue.scheduling.js";
11
+ /** Terminal states a job never leaves. */
12
+ const TERMINAL_STATES = new Set([
13
+ JobStateEnum.COMPLETED,
14
+ JobStateEnum.FAILED,
15
+ JobStateEnum.DEAD_LETTER,
16
+ JobStateEnum.CANCELLED,
17
+ ]);
18
+ /** Retained terminal jobs per outcome when no retention is configured. */
19
+ const DEFAULT_RETAINED_JOBS = 1_000;
20
+ /** How long `close()` waits for in-flight jobs before aborting them. */
21
+ const DEFAULT_CLOSE_TIMEOUT_MS = 30_000;
10
22
  /**
11
23
  * In-memory queue implementation.
12
24
  *
@@ -25,12 +37,21 @@ export class InMemoryQueue {
25
37
  activeCount = 0;
26
38
  pollTimer = null;
27
39
  scheduledTimers = new Map();
40
+ retryTimers = new Map();
41
+ inFlight = new Map();
42
+ settledOrder = [];
43
+ /** How many times each job has been reclaimed after stalling. */
44
+ stalledCounts = new Map();
28
45
  deduplicationIndex = new Map();
29
- deadLetterStore = createInMemoryDeadLetterStore();
30
- processedCount = 0;
31
- succeededCount = 0;
32
- failedCount = 0;
46
+ deadLetterStore;
33
47
  emitter;
48
+ counters = {
49
+ processedCount: 0,
50
+ succeededCount: 0,
51
+ failedCount: 0,
52
+ retriedCount: 0,
53
+ deadLetteredCount: 0,
54
+ };
34
55
  emptySince = 0;
35
56
  backoffMs = 50;
36
57
  constructor(name, options) {
@@ -39,14 +60,20 @@ export class InMemoryQueue {
39
60
  this.serializer = this.options.serializer ?? JsonSerializer;
40
61
  this.middleware = this.options.middleware ?? [];
41
62
  this.emitter = options?.eventEmitter ?? createNoopQueueEventEmitter();
63
+ this.deadLetterStore =
64
+ this.options.deadLetterStore ?? createInMemoryDeadLetterStore();
42
65
  }
43
66
  async add(jobName, data, options) {
44
67
  if (this.disposed)
45
68
  throw new QueueDisposedError(this.name);
46
- if (this.paused)
69
+ // Pausing conventionally stops consumption; whether it also stops
70
+ // production is a policy choice, so it is configurable and defaults
71
+ // to rejecting adds for backwards compatibility.
72
+ if (this.paused && (this.options.pauseRejectsAdd ?? true)) {
47
73
  throw new QueueError(`Queue "${this.name}" is paused.`, {
48
74
  queueName: this.name,
49
75
  });
76
+ }
50
77
  const mergedOptions = { ...this.options.defaultJobOptions, ...options };
51
78
  if (mergedOptions.deduplicationKey) {
52
79
  const existing = this.deduplicationIndex.get(mergedOptions.deduplicationKey);
@@ -56,11 +83,16 @@ export class InMemoryQueue {
56
83
  });
57
84
  }
58
85
  }
59
- const jobId = `job_${Date.now()}_${randomBytes(6).toString("hex")}`;
86
+ const jobId = this.createJobId();
87
+ // Round-tripping through the configured serializer keeps the option
88
+ // honest, isolates the stored payload from later caller mutation, and
89
+ // surfaces a non-serializable payload here rather than after a broker
90
+ // adapter is swapped in.
91
+ const payload = this.encodePayload(jobId, data);
60
92
  const job = createJob({
61
93
  name: createJobName(jobName),
62
94
  queueName: this.name,
63
- data,
95
+ data: payload,
64
96
  options: mergedOptions,
65
97
  }, jobId);
66
98
  this.jobs.set(jobId, job);
@@ -78,6 +110,7 @@ export class InMemoryQueue {
78
110
  process(name, processor) {
79
111
  if (this.disposed)
80
112
  throw new QueueDisposedError(this.name);
113
+ assertProcessor(processor, name);
81
114
  this.processors.set(name, processor);
82
115
  if (!this.pollTimer)
83
116
  this.startPolling();
@@ -85,86 +118,239 @@ export class InMemoryQueue {
85
118
  async getJob(jobId) {
86
119
  return this.jobs.get(jobId) ?? null;
87
120
  }
121
+ /**
122
+ * Returns the job that would be processed next without claiming it.
123
+ *
124
+ * This is a read-only peek. Consumers that intend to run the job must
125
+ * use {@link claimNextJob}, which transitions it to `active` so no
126
+ * other consumer can pick up the same job.
127
+ */
88
128
  async getNextJob() {
89
129
  if (this.paused || this.disposed)
90
130
  return null;
91
- const now = Date.now();
92
- let nextJob = null;
93
- let nextPriority = -1;
94
- for (const job of this.jobs.values()) {
95
- if (job.state !== JobStateEnum.WAITING)
96
- continue;
97
- if (job.scheduledAt && new Date(job.scheduledAt).getTime() > now)
98
- continue;
99
- if (job.priority > nextPriority) {
100
- nextPriority = job.priority;
101
- nextJob = job;
102
- }
103
- else if (job.priority === nextPriority && nextJob) {
104
- const currentTime = new Date(job.createdAt).getTime();
105
- const nextTime = new Date(nextJob.createdAt).getTime();
106
- if (currentTime < nextTime)
107
- nextJob = job;
108
- }
131
+ return this.selectJob();
132
+ }
133
+ /**
134
+ * Atomically selects the next runnable job and marks it `active`.
135
+ *
136
+ * Claiming is what prevents two consumers — or one consumer polling in
137
+ * a loop — from processing the same job repeatedly.
138
+ */
139
+ async claimNextJob() {
140
+ if (this.paused || this.disposed)
141
+ return null;
142
+ const job = this.selectJob((candidate) => this.processors.has(candidate.name));
143
+ if (!job)
144
+ return null;
145
+ const claimed = updateJobState(job, JobStateEnum.ACTIVE, {
146
+ startedAt: new Date().toISOString(),
147
+ });
148
+ this.jobs.set(claimed.id, claimed);
149
+ return claimed;
150
+ }
151
+ /**
152
+ * Returns a claimed job to the waiting pool.
153
+ */
154
+ async releaseJob(jobId) {
155
+ const job = this.jobs.get(jobId);
156
+ if (!job || job.state !== JobStateEnum.ACTIVE) {
157
+ return false;
109
158
  }
110
- return nextJob;
159
+ this.jobs.set(jobId, updateJobState(job, JobStateEnum.WAITING, { startedAt: undefined }));
160
+ return true;
111
161
  }
112
162
  getProcessor(name) {
113
163
  return this.processors.get(name);
114
164
  }
115
165
  async getStats() {
116
- const allJobs = Array.from(this.jobs.values());
166
+ let waiting = 0;
167
+ let active = 0;
168
+ let completed = 0;
169
+ let failed = 0;
170
+ let delayed = 0;
171
+ let retrying = 0;
172
+ for (const job of this.jobs.values()) {
173
+ switch (job.state) {
174
+ case JobStateEnum.WAITING:
175
+ waiting++;
176
+ break;
177
+ case JobStateEnum.ACTIVE:
178
+ active++;
179
+ break;
180
+ case JobStateEnum.COMPLETED:
181
+ completed++;
182
+ break;
183
+ case JobStateEnum.FAILED:
184
+ case JobStateEnum.DEAD_LETTER:
185
+ failed++;
186
+ break;
187
+ case JobStateEnum.SCHEDULED:
188
+ delayed++;
189
+ break;
190
+ case JobStateEnum.RETRYING:
191
+ retrying++;
192
+ break;
193
+ default:
194
+ break;
195
+ }
196
+ }
117
197
  return {
118
- waiting: allJobs.filter((j) => j.state === JobStateEnum.WAITING).length,
119
- active: allJobs.filter((j) => j.state === JobStateEnum.ACTIVE).length,
120
- completed: allJobs.filter((j) => j.state === JobStateEnum.COMPLETED)
121
- .length,
122
- failed: allJobs.filter((j) => j.state === JobStateEnum.FAILED ||
123
- j.state === JobStateEnum.DEAD_LETTER).length,
124
- delayed: allJobs.filter((j) => j.state === JobStateEnum.SCHEDULED).length,
125
- retrying: allJobs.filter((j) => j.state === JobStateEnum.RETRYING).length,
198
+ waiting,
199
+ active,
200
+ completed,
201
+ failed,
202
+ delayed,
203
+ retrying,
204
+ processed: this.counters.processedCount,
205
+ succeeded: this.counters.succeededCount,
206
+ errored: this.counters.failedCount,
207
+ retried: this.counters.retriedCount,
208
+ deadLettered: this.counters.deadLetteredCount,
126
209
  };
127
210
  }
211
+ /**
212
+ * Returns the jobs that exhausted their attempts.
213
+ */
214
+ async getDeadLetterJobs() {
215
+ return this.deadLetterStore.getAll();
216
+ }
128
217
  async pause() {
129
218
  this.paused = true;
130
219
  }
131
220
  async resume() {
221
+ if (this.disposed)
222
+ throw new QueueDisposedError(this.name);
132
223
  this.paused = false;
224
+ this.backoffMs = 50;
225
+ this.emptySince = 0;
226
+ if (!this.pollTimer && this.processors.size > 0)
227
+ this.startPolling();
133
228
  }
134
229
  isPaused() {
135
230
  return this.paused;
136
231
  }
232
+ isDisposed() {
233
+ return this.disposed;
234
+ }
235
+ /**
236
+ * Closes the queue, draining in-flight jobs first.
237
+ *
238
+ * Nothing is torn down until running jobs have settled, so a job can
239
+ * never write back into a cleared queue. Jobs that outlast
240
+ * `closeTimeout` have their `AbortSignal` aborted and are then
241
+ * abandoned so shutdown cannot hang indefinitely.
242
+ */
137
243
  async close() {
244
+ if (this.disposed)
245
+ return;
246
+ // Stop accepting and dispatching work before draining, so the set of
247
+ // in-flight jobs cannot grow while we wait for it.
248
+ this.disposed = true;
138
249
  this.stopPolling();
139
250
  for (const timer of this.scheduledTimers.values())
140
251
  clearTimeout(timer);
141
252
  this.scheduledTimers.clear();
253
+ for (const timer of this.retryTimers.values())
254
+ clearTimeout(timer);
255
+ this.retryTimers.clear();
256
+ await this.drain(this.options.closeTimeout ?? DEFAULT_CLOSE_TIMEOUT_MS);
142
257
  this.jobs.clear();
143
258
  this.processors.clear();
144
259
  this.deduplicationIndex.clear();
260
+ this.settledOrder.length = 0;
261
+ this.stalledCounts.clear();
262
+ this.inFlight.clear();
145
263
  this.activeCount = 0;
146
- this.processedCount = 0;
147
- this.succeededCount = 0;
148
- this.failedCount = 0;
149
264
  this.paused = false;
150
- this.disposed = true;
151
265
  this.emptySince = 0;
152
266
  this.backoffMs = 50;
153
267
  }
268
+ /**
269
+ * Waits for in-flight jobs to settle, aborting them past the timeout.
270
+ */
271
+ async drain(timeoutMs) {
272
+ if (this.activeCount === 0)
273
+ return;
274
+ const deadline = Date.now() + Math.max(0, timeoutMs);
275
+ while (this.activeCount > 0 && Date.now() < deadline) {
276
+ await new Promise((resolve) => setTimeout(resolve, 10));
277
+ }
278
+ if (this.activeCount === 0)
279
+ return;
280
+ for (const controller of this.inFlight.values()) {
281
+ if (!controller.signal.aborted) {
282
+ controller.abort(new QueueDisposedError(this.name));
283
+ }
284
+ }
285
+ // Give aborted processors a brief window to unwind before returning.
286
+ const abortDeadline = Date.now() + 1_000;
287
+ while (this.activeCount > 0 && Date.now() < abortDeadline) {
288
+ await new Promise((resolve) => setTimeout(resolve, 10));
289
+ }
290
+ }
291
+ /**
292
+ * Generates a collision-resistant job id.
293
+ */
294
+ createJobId() {
295
+ return `job_${Date.now()}_${randomUUID().replace(/-/g, "")}`;
296
+ }
297
+ /**
298
+ * Round-trips a payload through the configured serializer.
299
+ */
300
+ encodePayload(jobId, data) {
301
+ if (this.options.serializePayloads === false) {
302
+ return data;
303
+ }
304
+ try {
305
+ return this.serializer.deserialize(this.serializer.serialize(data));
306
+ }
307
+ catch (error) {
308
+ throw new JobSerializationError(jobId, `Job payload for queue "${this.name}" is not serializable.`, { queueName: this.name, cause: error });
309
+ }
310
+ }
311
+ /**
312
+ * Selects the highest-priority job that is due and runnable.
313
+ *
314
+ * Ties on priority are broken by creation time, oldest first. The
315
+ * incumbent is tracked by reference rather than by a sentinel priority,
316
+ * so jobs with negative priorities are selectable like any other.
317
+ */
318
+ selectJob(predicate) {
319
+ const now = Date.now();
320
+ let nextJob = null;
321
+ for (const job of this.jobs.values()) {
322
+ if (job.state !== JobStateEnum.WAITING)
323
+ continue;
324
+ if (job.scheduledAt && new Date(job.scheduledAt).getTime() > now)
325
+ continue;
326
+ if (predicate && !predicate(job))
327
+ continue;
328
+ if (nextJob === null) {
329
+ nextJob = job;
330
+ continue;
331
+ }
332
+ if (job.priority > nextJob.priority) {
333
+ nextJob = job;
334
+ continue;
335
+ }
336
+ if (job.priority === nextJob.priority) {
337
+ const candidateTime = new Date(job.createdAt).getTime();
338
+ const incumbentTime = new Date(nextJob.createdAt).getTime();
339
+ if (candidateTime < incumbentTime)
340
+ nextJob = job;
341
+ }
342
+ }
343
+ return nextJob;
344
+ }
154
345
  startPolling() {
155
- const pollInterval = this.options.concurrency ? 50 : 100;
156
- this.pollTimer = setTimeout(() => {
157
- this.pollTimer = null;
158
- this.processTick().finally(() => {
159
- if (!this.disposed && this.processors.size > 0)
160
- this.scheduleNextTick();
161
- });
162
- }, pollInterval);
346
+ this.scheduleTick(this.options.pollInterval ?? 50);
163
347
  }
164
348
  scheduleNextTick() {
349
+ this.scheduleTick(this.backoffMs);
350
+ }
351
+ scheduleTick(interval) {
165
352
  if (this.disposed)
166
353
  return;
167
- const interval = this.backoffMs;
168
354
  this.pollTimer = setTimeout(() => {
169
355
  this.pollTimer = null;
170
356
  this.processTick().finally(() => {
@@ -172,6 +358,8 @@ export class InMemoryQueue {
172
358
  this.scheduleNextTick();
173
359
  });
174
360
  }, interval);
361
+ // The poll timer must not be the reason a process stays alive.
362
+ this.pollTimer.unref?.();
175
363
  }
176
364
  stopPolling() {
177
365
  if (this.pollTimer) {
@@ -179,34 +367,87 @@ export class InMemoryQueue {
179
367
  this.pollTimer = null;
180
368
  }
181
369
  }
370
+ /**
371
+ * Runs an already-claimed job through this queue's processing pipeline.
372
+ *
373
+ * The queue owns job state, so every consumer — the internal poller and
374
+ * any external `Worker` — must run jobs through here. A consumer that
375
+ * invokes a processor directly leaves the job stuck in `active` and
376
+ * skips retry, dead-lettering and middleware entirely.
377
+ */
378
+ async runJob(job, options) {
379
+ const processor = this.processors.get(job.name);
380
+ if (!processor) {
381
+ await this.releaseJob(job.id);
382
+ return;
383
+ }
384
+ const abortController = new AbortController();
385
+ // A consumer's own signal (a worker draining, say) must reach the
386
+ // job it dispatched.
387
+ if (options?.signal) {
388
+ if (options.signal.aborted) {
389
+ abortController.abort(options.signal.reason);
390
+ }
391
+ else {
392
+ options.signal.addEventListener("abort", () => {
393
+ if (!abortController.signal.aborted) {
394
+ abortController.abort(options.signal?.reason);
395
+ }
396
+ }, { once: true });
397
+ }
398
+ }
399
+ this.inFlight.set(job.id, abortController);
400
+ this.activeCount++;
401
+ try {
402
+ await processJob(job, processor, {
403
+ timeoutMs: job.timeoutMs ?? options?.timeoutMs,
404
+ abortController,
405
+ }, {
406
+ jobs: this.jobs,
407
+ emitter: this.emitter,
408
+ deadLetterStore: this.deadLetterStore,
409
+ counters: this.counters,
410
+ middleware: options?.middleware
411
+ ? [...this.middleware, ...options.middleware]
412
+ : this.middleware,
413
+ registerRetryTimer: (jobId, timer) => {
414
+ this.retryTimers.set(jobId, timer);
415
+ },
416
+ onSettled: (settled) => this.recordSettled(settled),
417
+ isDisposed: () => this.disposed,
418
+ ...(this.options.logger ? { logger: this.options.logger } : {}),
419
+ });
420
+ }
421
+ catch (error) {
422
+ this.emitter.emit("job:failed", {
423
+ job,
424
+ error: error instanceof Error ? error : new Error(String(error)),
425
+ });
426
+ }
427
+ finally {
428
+ this.activeCount--;
429
+ this.inFlight.delete(job.id);
430
+ this.retryTimers.delete(job.id);
431
+ }
432
+ }
182
433
  async processTick() {
183
434
  if (this.paused || this.disposed)
184
435
  return;
185
436
  const concurrency = Math.max(1, this.options.concurrency ?? 1);
186
437
  let processed = 0;
187
438
  while (this.activeCount < concurrency) {
188
- const job = await this.getNextJob();
439
+ // `claimNextJob` only returns jobs that have a registered
440
+ // processor and moves them out of `waiting`, so this loop always
441
+ // terminates. Returning an unrunnable job here is what previously
442
+ // spun the event loop forever.
443
+ const job = await this.claimNextJob();
189
444
  if (!job)
190
445
  break;
191
- const processor = this.processors.get(job.name);
192
- if (!processor)
193
- continue;
194
- this.activeCount++;
195
446
  processed++;
196
- processJob(job, processor, this.middleware, this.jobs, { timeoutMs: job.timeoutMs }, this.emitter, this.deadLetterStore, {
197
- processedCount: this.processedCount,
198
- succeededCount: this.succeededCount,
199
- failedCount: this.failedCount,
200
- })
201
- .catch((error) => {
202
- this.emitter.emit("job:failed", {
203
- job,
204
- error: error instanceof Error ? error : new Error(String(error)),
205
- });
206
- })
207
- .finally(() => {
208
- this.activeCount--;
209
- });
447
+ // Not awaited: jobs run concurrently up to the limit. `runJob`
448
+ // increments `activeCount` synchronously, so the loop condition
449
+ // sees the dispatch immediately.
450
+ void this.runJob(job);
210
451
  }
211
452
  if (processed > 0) {
212
453
  this.backoffMs = 50;
@@ -222,8 +463,92 @@ export class InMemoryQueue {
222
463
  this.backoffMs = Math.min(this.backoffMs * 2, 2000);
223
464
  }
224
465
  }
225
- if (this.scheduledTimers.size > 0) {
226
- scheduleDelayedJobs(this.jobs);
466
+ promoteDueScheduledJobs(this.jobs, this.scheduledTimers);
467
+ this.reclaimStalledJobs();
468
+ }
469
+ /**
470
+ * Returns jobs stuck in `active` to the waiting pool.
471
+ *
472
+ * A consumer can claim a job and then die, or be killed mid-run.
473
+ * Without this the job stays `active` forever and no one picks it up
474
+ * again. A job that stalls repeatedly is dead-lettered rather than
475
+ * cycled indefinitely.
476
+ */
477
+ reclaimStalledJobs() {
478
+ const stalledAfter = this.options.stalledAfter ?? 0;
479
+ if (stalledAfter <= 0) {
480
+ return;
481
+ }
482
+ const now = Date.now();
483
+ const maxStalled = Math.max(1, this.options.maxStalledCount ?? 3);
484
+ for (const job of this.jobs.values()) {
485
+ if (job.state !== JobStateEnum.ACTIVE || !job.startedAt)
486
+ continue;
487
+ // A job this queue is currently running is not stalled.
488
+ if (this.inFlight.has(job.id))
489
+ continue;
490
+ const startedAt = new Date(job.startedAt).getTime();
491
+ if (Number.isNaN(startedAt) || now - startedAt < stalledAfter)
492
+ continue;
493
+ const count = (this.stalledCounts.get(job.id) ?? 0) + 1;
494
+ this.stalledCounts.set(job.id, count);
495
+ const error = new JobStalledError(job.id, { queueName: this.name });
496
+ if (count >= maxStalled) {
497
+ this.stalledCounts.delete(job.id);
498
+ this.counters.failedCount++;
499
+ this.counters.deadLetteredCount++;
500
+ const deadLettered = updateJobState(job, JobStateEnum.DEAD_LETTER, {
501
+ error: error.message,
502
+ failedAt: new Date().toISOString(),
503
+ });
504
+ this.jobs.set(job.id, deadLettered);
505
+ void this.deadLetterStore
506
+ .add({
507
+ job: deadLettered,
508
+ deadLetterAt: new Date(),
509
+ error,
510
+ attempts: deadLettered.attempt,
511
+ reason: `Stalled ${count} time(s).`,
512
+ })
513
+ .catch(() => { });
514
+ this.emitter.emit("job:failed", { job: deadLettered, error });
515
+ this.recordSettled(deadLettered);
516
+ continue;
517
+ }
518
+ this.jobs.set(job.id, updateJobState(job, JobStateEnum.WAITING, { startedAt: undefined }));
519
+ this.emitter.emit("job:failed", { job, error });
520
+ }
521
+ }
522
+ /**
523
+ * Records a job that reached a terminal state and prunes history.
524
+ *
525
+ * Without this the queue retains every job it has ever run, and every
526
+ * deduplication key it has ever seen, for the life of the process.
527
+ */
528
+ recordSettled(job) {
529
+ if (job.deduplicationKey) {
530
+ const owner = this.deduplicationIndex.get(job.deduplicationKey);
531
+ if (owner === job.id) {
532
+ this.deduplicationIndex.delete(job.deduplicationKey);
533
+ }
534
+ }
535
+ this.settledOrder.push(job.id);
536
+ this.pruneSettled();
537
+ }
538
+ /**
539
+ * Drops the oldest terminal jobs beyond the retention limit.
540
+ */
541
+ pruneSettled() {
542
+ const limit = Math.max(0, this.options.retainSettledJobs ?? DEFAULT_RETAINED_JOBS);
543
+ while (this.settledOrder.length > limit) {
544
+ const jobId = this.settledOrder.shift();
545
+ if (jobId === undefined)
546
+ break;
547
+ const job = this.jobs.get(jobId);
548
+ if (job && TERMINAL_STATES.has(job.state)) {
549
+ this.jobs.delete(jobId);
550
+ this.stalledCounts.delete(jobId);
551
+ }
227
552
  }
228
553
  }
229
554
  }
@@ -1,19 +1,64 @@
1
1
  import type { Job } from "../job/job.type.js";
2
2
  import type { Processor } from "../processor/processor.type.js";
3
+ import type { JobId } from "../jobTypes/jobTypes.type.js";
4
+ import type { QueueMiddleware } from "../middleware/middleware.type.js";
5
+ import type { QueueEventEmitter } from "../queueEmitter/queueEmitter.type.js";
6
+ import type { DeadLetterStore } from "../deadLetter/deadLetter.type.js";
7
+ import type { QueueLogger } from "../queue/queue.type.js";
3
8
  /**
4
- * Process a single job with middleware, retry, and failure handling.
9
+ * Mutable throughput counters.
10
+ *
11
+ * Passed by reference so increments are visible to the queue that owns
12
+ * them; a copied object literal would discard every update.
5
13
  */
6
- export declare function processJob<TData>(job: Job<TData>, processor: Processor<TData>, middleware: import("../middleware/middleware.type.js").QueueMiddleware[], jobs: Map<string, Job<TData>>, options: {
7
- timeoutMs?: number;
8
- }, emitter: import("../queueEmitter/queueEmitter.type.js").QueueEventEmitter, deadLetterStore: import("../deadLetter/deadLetter.type.js").DeadLetterStore<TData>, counters: {
14
+ export interface QueueCounters {
9
15
  processedCount: number;
10
16
  succeededCount: number;
11
17
  failedCount: number;
12
- }): Promise<void>;
18
+ retriedCount: number;
19
+ deadLetteredCount: number;
20
+ }
21
+ /**
22
+ * Collaborators a job needs in order to run.
23
+ */
24
+ export interface ProcessJobDependencies<TData> {
25
+ readonly jobs: Map<string, Job<TData>>;
26
+ readonly emitter: QueueEventEmitter;
27
+ readonly deadLetterStore: DeadLetterStore<TData>;
28
+ readonly counters: QueueCounters;
29
+ readonly middleware: readonly QueueMiddleware[];
30
+ /**
31
+ * Registers the timer that returns a retrying job to `waiting`, so the
32
+ * queue can clear it on close instead of leaking it.
33
+ */
34
+ readonly registerRetryTimer: (jobId: JobId, timer: ReturnType<typeof setTimeout>) => void;
35
+ /** Invoked whenever a job reaches a terminal state. */
36
+ readonly onSettled?: (job: Job<TData>) => void;
37
+ /** Whether the owning queue has been disposed. */
38
+ readonly isDisposed: () => boolean;
39
+ /**
40
+ * Destination for lines a processor writes with `context.log()`. Absent
41
+ * when the queue was configured without a logger.
42
+ */
43
+ readonly logger?: QueueLogger;
44
+ }
45
+ /**
46
+ * Process a single job with middleware, retry, and failure handling.
47
+ *
48
+ * The job is expected to already be claimed (`active`); this records the
49
+ * start time, runs the middleware chain, and routes the outcome to
50
+ * completion, retry, or the dead letter store.
51
+ */
52
+ export declare function processJob<TData>(job: Job<TData>, processor: Processor<TData>, options: {
53
+ timeoutMs?: number;
54
+ abortController?: AbortController;
55
+ }, deps: ProcessJobDependencies<TData>): Promise<void>;
13
56
  /**
14
57
  * Handle job failure with retry logic.
58
+ *
59
+ * A failure that will be retried is reported as `job:retrying` and counted
60
+ * separately; only a terminal failure increments `failedCount`, so the
61
+ * counters describe outcomes rather than attempts.
15
62
  */
16
- export declare function handleJobFailure<TData>(job: Job<TData>, errorMessage: string, processor: Processor<TData>, abortController: AbortController, jobs: Map<string, Job<TData>>, emitter: import("../queueEmitter/queueEmitter.type.js").QueueEventEmitter, deadLetterStore: import("../deadLetter/deadLetter.type.js").DeadLetterStore<TData>, counters: {
17
- failedCount: number;
18
- }): Promise<void>;
63
+ export declare function handleJobFailure<TData>(job: Job<TData>, errorMessage: string, deps: ProcessJobDependencies<TData>): Promise<void>;
19
64
  //# sourceMappingURL=inMemoryQueue.processing.d.ts.map