@c9up/bay 0.1.12 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +172 -1
  2. package/dist/BayProvider.d.ts +55 -33
  3. package/dist/BayProvider.d.ts.map +1 -1
  4. package/dist/BayProvider.js +81 -10
  5. package/dist/BayProvider.js.map +1 -1
  6. package/dist/Job.d.ts +99 -0
  7. package/dist/Job.d.ts.map +1 -0
  8. package/dist/Job.js +78 -0
  9. package/dist/Job.js.map +1 -0
  10. package/dist/QueueManager.d.ts +147 -22
  11. package/dist/QueueManager.d.ts.map +1 -1
  12. package/dist/QueueManager.js +290 -52
  13. package/dist/QueueManager.js.map +1 -1
  14. package/dist/adapters.d.ts +68 -0
  15. package/dist/adapters.d.ts.map +1 -0
  16. package/dist/adapters.js +56 -0
  17. package/dist/adapters.js.map +1 -0
  18. package/dist/augmentations.d.ts +28 -0
  19. package/dist/augmentations.d.ts.map +1 -0
  20. package/dist/augmentations.js +17 -0
  21. package/dist/augmentations.js.map +1 -0
  22. package/dist/configure.d.ts +19 -0
  23. package/dist/configure.d.ts.map +1 -0
  24. package/dist/configure.js +48 -0
  25. package/dist/configure.js.map +1 -0
  26. package/dist/console/contract.d.ts +60 -0
  27. package/dist/console/contract.d.ts.map +1 -0
  28. package/dist/console/contract.js +36 -0
  29. package/dist/console/contract.js.map +1 -0
  30. package/dist/console/index.d.ts +29 -0
  31. package/dist/console/index.d.ts.map +1 -0
  32. package/dist/console/index.js +45 -0
  33. package/dist/console/index.js.map +1 -0
  34. package/dist/console/makeJob.d.ts +32 -0
  35. package/dist/console/makeJob.d.ts.map +1 -0
  36. package/dist/console/makeJob.js +118 -0
  37. package/dist/console/makeJob.js.map +1 -0
  38. package/dist/console/queueWork.d.ts +18 -0
  39. package/dist/console/queueWork.d.ts.map +1 -0
  40. package/dist/console/queueWork.js +58 -0
  41. package/dist/console/queueWork.js.map +1 -0
  42. package/dist/drivers/MemoryDriver.d.ts +14 -8
  43. package/dist/drivers/MemoryDriver.d.ts.map +1 -1
  44. package/dist/drivers/MemoryDriver.js +61 -7
  45. package/dist/drivers/MemoryDriver.js.map +1 -1
  46. package/dist/drivers/RedisDriver.d.ts +66 -8
  47. package/dist/drivers/RedisDriver.d.ts.map +1 -1
  48. package/dist/drivers/RedisDriver.js +257 -45
  49. package/dist/drivers/RedisDriver.js.map +1 -1
  50. package/dist/index.d.ts +10 -5
  51. package/dist/index.d.ts.map +1 -1
  52. package/dist/index.js +8 -3
  53. package/dist/index.js.map +1 -1
  54. package/dist/jobs.d.ts +43 -0
  55. package/dist/jobs.d.ts.map +1 -0
  56. package/dist/jobs.js +105 -0
  57. package/dist/jobs.js.map +1 -0
  58. package/dist/nodeEnv.d.ts +16 -0
  59. package/dist/nodeEnv.d.ts.map +1 -0
  60. package/dist/nodeEnv.js +32 -0
  61. package/dist/nodeEnv.js.map +1 -0
  62. package/dist/quasar.d.ts +1 -1
  63. package/dist/quasar.js +1 -1
  64. package/dist/services/main.d.ts +5 -0
  65. package/dist/services/main.d.ts.map +1 -1
  66. package/dist/services/main.js +7 -0
  67. package/dist/services/main.js.map +1 -1
  68. package/dist/testing/FakeQueue.d.ts +15 -9
  69. package/dist/testing/FakeQueue.d.ts.map +1 -1
  70. package/dist/testing/FakeQueue.js +13 -3
  71. package/dist/testing/FakeQueue.js.map +1 -1
  72. package/package.json +9 -3
  73. package/src/BayProvider.ts +143 -25
  74. package/src/Job.ts +137 -0
  75. package/src/QueueManager.ts +454 -56
  76. package/src/adapters.ts +75 -0
  77. package/src/augmentations.ts +31 -0
  78. package/src/configure.ts +63 -0
  79. package/src/console/contract.ts +94 -0
  80. package/src/console/index.ts +68 -0
  81. package/src/console/makeJob.ts +139 -0
  82. package/src/console/queueWork.ts +70 -0
  83. package/src/drivers/MemoryDriver.ts +66 -14
  84. package/src/drivers/RedisDriver.ts +366 -63
  85. package/src/index.ts +35 -5
  86. package/src/jobs.ts +111 -0
  87. package/src/nodeEnv.ts +30 -0
  88. package/src/quasar.ts +1 -1
  89. package/src/services/main.ts +8 -0
  90. package/src/testing/FakeQueue.ts +25 -15
@@ -18,7 +18,9 @@
18
18
  * needs a thin adapter.
19
19
  */
20
20
 
21
- import type { Job, QueueDriver } from "../QueueManager.js";
21
+ import { DEFAULT_QUEUE } from "../Job.js";
22
+ import { inProduction } from "../nodeEnv.js";
23
+ import { type JobRecord, type QueueDriver, queueOf } from "../QueueManager.js";
22
24
 
23
25
  export interface RedisClient {
24
26
  rpush(key: string, ...values: string[]): Promise<number>;
@@ -30,25 +32,80 @@ export interface RedisClient {
30
32
  to: "LEFT" | "RIGHT",
31
33
  ): Promise<string | null>;
32
34
  lrem(key: string, count: number, element: string): Promise<number>;
35
+ /**
36
+ * Optional, like `lmove`. Present on ioredis; without it the failed list
37
+ * simply keeps its entries, which is what this driver did before.
38
+ */
39
+ ltrim?(key: string, start: number, stop: number): Promise<string>;
33
40
  llen(key: string): Promise<number>;
34
41
  lrange(key: string, start: number, stop: number): Promise<string[]>;
35
42
  del(key: string): Promise<number>;
36
43
  set(key: string, value: string, ...args: string[]): Promise<string | null>;
37
44
  get(key: string): Promise<string | null>;
45
+ /**
46
+ * Sorted-set commands, for delayed jobs. Optional like `lmove`: a client
47
+ * without them can still run a queue, and `push` refuses a job carrying a
48
+ * `delay` rather than running it early — which is the one thing a delay
49
+ * must not do.
50
+ */
51
+ zadd?(key: string, score: number, member: string): Promise<number | string>;
52
+ zrangebyscore?(
53
+ key: string,
54
+ min: number | string,
55
+ max: number | string,
56
+ ...args: string[]
57
+ ): Promise<string[]>;
58
+ zrem?(key: string, ...members: string[]): Promise<number>;
59
+ zcard?(key: string): Promise<number>;
38
60
  }
39
61
 
40
- function isValidJob(obj: unknown): obj is Job {
62
+ function isValidJob(obj: unknown): obj is JobRecord {
41
63
  if (typeof obj !== "object" || obj === null) return false;
42
- const j = obj as Record<string, unknown>;
43
64
  return (
44
- typeof j.id === "string" &&
45
- typeof j.name === "string" &&
46
- typeof j.attempts === "number" &&
47
- typeof j.maxAttempts === "number" &&
48
- typeof j.status === "string"
65
+ typeof Reflect.get(obj, "id") === "string" &&
66
+ typeof Reflect.get(obj, "name") === "string" &&
67
+ typeof Reflect.get(obj, "attempts") === "number" &&
68
+ typeof Reflect.get(obj, "maxAttempts") === "number" &&
69
+ typeof Reflect.get(obj, "status") === "string"
49
70
  );
50
71
  }
51
72
 
73
+ /**
74
+ * What a lease holds: the exact string pop() moved into `processing`, and the
75
+ * worker that moved it.
76
+ *
77
+ * The owner is what makes renewal safe. Without it a worker whose lease had
78
+ * already expired — its job recovered, re-popped by somebody else — would go on
79
+ * extending the deadline of a job it no longer had any claim on. Upstream draws
80
+ * the same line inside its renewal script: "Only the worker that currently owns
81
+ * the lease may renew it."
82
+ */
83
+ interface Lease {
84
+ owner: string;
85
+ raw: string;
86
+ }
87
+
88
+ /**
89
+ * Read a lease back. A value written by an older version of this driver is the
90
+ * raw job string on its own, with no owner — still usable for the one thing
91
+ * `#removeFromProcessing` needs it for.
92
+ */
93
+ function readLease(stored: string): { owner?: string; raw: string } {
94
+ let parsed: unknown;
95
+ try {
96
+ parsed = JSON.parse(stored);
97
+ } catch {
98
+ return { raw: stored };
99
+ }
100
+ if (typeof parsed !== "object" || parsed === null) return { raw: stored };
101
+ const raw = Reflect.get(parsed, "raw");
102
+ const owner = Reflect.get(parsed, "owner");
103
+ if (typeof raw !== "string" || typeof owner !== "string") {
104
+ return { raw: stored };
105
+ }
106
+ return { owner, raw };
107
+ }
108
+
52
109
  /**
53
110
  * Where the client comes from. A resolver is what lets a queue name its
54
111
  * connection (`quasarConnection("jobs")`) instead of being handed a client:
@@ -65,13 +122,35 @@ export type RedisClientSource =
65
122
  * has no client to inspect yet.
66
123
  */
67
124
  const warned = new WeakSet<object>();
68
- function warnWithoutLmove(client: RedisClient): void {
69
- if (typeof client.lmove === "function" || warned.has(client)) return;
125
+ function checkLmove(client: RedisClient, allowNonAtomicPop: boolean): void {
126
+ if (typeof client.lmove === "function") return;
127
+
128
+ // A queue's whole promise is that a job it accepted gets run. Without LMOVE
129
+ // the pop is `lpop` then `rpush`, and a crash between the two deletes the
130
+ // job from pending before it reaches processing: nothing recovers it,
131
+ // because nothing knows it existed. That is a different product, and in
132
+ // production it must be asked for rather than fallen into.
133
+ if (inProduction() && !allowNonAtomicPop) {
134
+ throw new Error(
135
+ "[bay] this Redis client has no LMOVE (Redis < 6.2), so pop() would be a non-atomic lpop+rpush — " +
136
+ "a crash between the two loses the in-flight job, turning at-least-once delivery into at-most-once.\n" +
137
+ " Upgrade to Redis 6.2 or later, or pass `allowNonAtomicPop: true` to state that losing a job is acceptable here.",
138
+ );
139
+ }
140
+ if (warned.has(client)) return;
70
141
  warned.add(client);
142
+
143
+ // Said even when the deployment opted in: agreeing to lose a job once, in a
144
+ // config file, is not the same as being reminded that this process is
145
+ // running that way. The line has to be in the logs of the incident.
146
+ const optedIn = inProduction() && allowNonAtomicPop;
71
147
  console.warn(
72
148
  "[bay] RedisDriver: client lacks LMOVE (Redis <6.2). pop() falls back to " +
73
149
  "a non-atomic lpop+rpush, downgrading delivery from at-least-once to " +
74
- "at-most-once — a crash between the two commands loses the in-flight job.",
150
+ "at-most-once — a crash between the two commands loses the in-flight job." +
151
+ (optedIn
152
+ ? "\n Running this way in PRODUCTION because allowNonAtomicPop was set."
153
+ : ""),
75
154
  );
76
155
  }
77
156
 
@@ -101,7 +180,7 @@ export class RedisDriver implements QueueDriver {
101
180
  if (this.#resolved) return this.#resolved;
102
181
  if (typeof this.#source !== "function") {
103
182
  this.#resolved = this.#source;
104
- warnWithoutLmove(this.#resolved);
183
+ checkLmove(this.#resolved, this.#allowNonAtomicPop);
105
184
  return this.#resolved;
106
185
  }
107
186
  if (!this.#pending) {
@@ -109,7 +188,7 @@ export class RedisDriver implements QueueDriver {
109
188
  this.#pending = Promise.resolve(resolver())
110
189
  .then((client) => {
111
190
  this.#resolved = client;
112
- warnWithoutLmove(client);
191
+ checkLmove(client, this.#allowNonAtomicPop);
113
192
  return client;
114
193
  })
115
194
  // Cleared on failure too. Clearing only on success left the
@@ -125,18 +204,47 @@ export class RedisDriver implements QueueDriver {
125
204
 
126
205
  constructor(
127
206
  source: RedisClientSource,
128
- options?: { prefix?: string; visibilityTimeoutMs?: number },
207
+ options?: {
208
+ prefix?: string;
209
+ visibilityTimeoutMs?: number;
210
+ /**
211
+ * Accept the non-atomic pop on a Redis older than 6.2, in
212
+ * production. Off by default: losing an accepted job is a choice a
213
+ * deployment makes, not one a version check makes for it.
214
+ */
215
+ allowNonAtomicPop?: boolean;
216
+ /**
217
+ * How many times a job may be reclaimed from a stalled worker before
218
+ * it is filed as failed instead of pushed round again. Default `1`,
219
+ * upstream's default for the same setting.
220
+ *
221
+ * Unbounded recovery is a job that kills its worker taking the whole
222
+ * queue down with it, forever: the crash never reaches the failure
223
+ * path, so `attempts` never moves and `maxAttempts` never applies.
224
+ */
225
+ maxStalledCount?: number;
226
+ /**
227
+ * How many failed jobs to keep. Default `1000` — the ceiling the
228
+ * memory driver already had. `0` keeps every one of them.
229
+ *
230
+ * Only enforced when the client answers `ltrim`.
231
+ */
232
+ maxFailedJobs?: number;
233
+ },
129
234
  ) {
130
235
  this.#source = source;
131
236
  // A client handed in directly can be checked now, so the warning keeps
132
237
  // landing at construction as it always did. A named connection has no
133
238
  // client yet — it is checked when the connection resolves.
134
- if (typeof source !== "function") warnWithoutLmove(source);
239
+ if (typeof source !== "function") {
240
+ checkLmove(source, options?.allowNonAtomicPop ?? false);
241
+ }
135
242
  // Normalised rather than documented: every key is built by concatenation
136
243
  // (`${prefix}pending`), so a prefix without a trailing separator yields
137
244
  // "myapppending" — unreadable, and able to collide with a neighbouring
138
245
  // prefix. Nothing warned, because nothing failed.
139
246
  this.#prefix = withSeparator(options?.prefix ?? "queue:");
247
+ this.#allowNonAtomicPop = options?.allowNonAtomicPop ?? false;
140
248
  const visibilityTimeout = options?.visibilityTimeoutMs ?? 30_000;
141
249
  // A non-positive / non-integer timeout makes pop()'s `SET … PX <ms>` fail
142
250
  // on a real Redis; the catch then removes the job from `processing` and
@@ -148,78 +256,168 @@ export class RedisDriver implements QueueDriver {
148
256
  );
149
257
  }
150
258
  this.#visibilityTimeout = visibilityTimeout;
259
+
260
+ const maxStalled = options?.maxStalledCount ?? 1;
261
+ if (!Number.isInteger(maxStalled) || maxStalled < 0) {
262
+ throw new Error(
263
+ `[bay] RedisDriver maxStalledCount must be a non-negative integer, got ${maxStalled}`,
264
+ );
265
+ }
266
+ this.#maxStalledCount = maxStalled;
267
+
268
+ const maxFailed = options?.maxFailedJobs ?? 1000;
269
+ if (!Number.isInteger(maxFailed) || maxFailed < 0) {
270
+ throw new Error(
271
+ `[bay] RedisDriver maxFailedJobs must be a non-negative integer, got ${maxFailed}`,
272
+ );
273
+ }
274
+ this.#maxFailedJobs = maxFailed;
151
275
  }
152
276
 
153
- #pendingKey = () => `${this.#prefix}pending`;
277
+ /**
278
+ * Renew a lease at half its length: two chances to be heard before the
279
+ * deadline, so one slow round-trip does not hand a running job to somebody
280
+ * else. Read by `QueueManager` while a handler runs.
281
+ */
282
+ get renewIntervalMs(): number {
283
+ return Math.max(1, Math.floor(this.#visibilityTimeout / 2));
284
+ }
285
+
286
+ /**
287
+ * Where one queue's jobs wait.
288
+ *
289
+ * The default queue keeps the key it always had. Naming it
290
+ * `queue:default:pending` would have been tidier and would have orphaned
291
+ * every job already sitting in `queue:pending` at the moment of the upgrade
292
+ * — a silent loss, since nothing reads the old key afterwards.
293
+ */
294
+ #pendingKey = (queue: string = DEFAULT_QUEUE) =>
295
+ queue === DEFAULT_QUEUE
296
+ ? `${this.#prefix}pending`
297
+ : `${this.#prefix}q:${queue}:pending`;
298
+ #delayedKey = (queue: string = DEFAULT_QUEUE) =>
299
+ queue === DEFAULT_QUEUE
300
+ ? `${this.#prefix}delayed`
301
+ : `${this.#prefix}q:${queue}:delayed`;
154
302
  #processingKey = () => `${this.#prefix}processing`;
303
+ #allowNonAtomicPop = false;
304
+ #maxStalledCount = 1;
305
+ #maxFailedJobs = 1000;
306
+ /** This driver instance, as a lease owner. */
307
+ #workerId = crypto.randomUUID();
155
308
  #failedKey = () => `${this.#prefix}failed`;
156
309
  #leaseKey = (jobId: string) => `${this.#prefix}lease:${jobId}`;
157
310
 
158
- async push(job: Job): Promise<void> {
311
+ async push(job: JobRecord): Promise<void> {
159
312
  const client = await this.#client();
160
- await client.rpush(this.#pendingKey(), JSON.stringify(job));
313
+ const queue = queueOf(job);
314
+ if (job.runAt !== undefined && job.runAt > Date.now()) {
315
+ if (!client.zadd) {
316
+ // Pushing it to the list instead would run it now, which is the
317
+ // one thing a delay exists to prevent.
318
+ throw new Error(
319
+ "This Redis client cannot hold a delayed job: it has no ZADD. " +
320
+ "Use a client with sorted-set commands (ioredis has them), or dispatch without `delay`.",
321
+ );
322
+ }
323
+ await client.zadd(
324
+ this.#delayedKey(queue),
325
+ job.runAt,
326
+ JSON.stringify(job),
327
+ );
328
+ return;
329
+ }
330
+ await client.rpush(this.#pendingKey(queue), JSON.stringify(job));
161
331
  }
162
332
 
163
- async pop(): Promise<Job | null> {
333
+ async pop(
334
+ queues: readonly string[] = [DEFAULT_QUEUE],
335
+ ): Promise<JobRecord | null> {
164
336
  const client = await this.#client();
165
337
  let raw: string | null = null;
166
338
 
167
- if (client.lmove) {
168
- raw = await client.lmove(
169
- this.#pendingKey(),
170
- this.#processingKey(),
171
- "LEFT",
172
- "RIGHT",
173
- );
174
- } else {
175
- raw = await client.lpop(this.#pendingKey());
176
- if (raw) await client.rpush(this.#processingKey(), raw);
339
+ // In the order given, so a worker can say which queue it drains first.
340
+ for (const queue of queues) {
341
+ await this.#promoteDue(client, queue);
342
+ raw = await this.#take(client, queue);
343
+ if (raw !== null) break;
177
344
  }
178
345
 
179
346
  if (!raw) return null;
347
+
348
+ // Only a payload that can never be run is purged. Everything past this
349
+ // point is a REAL job that already sits in `processing`, and deleting
350
+ // it there is the one thing that loses it for good: it is gone from
351
+ // pending too, and recoverStale() scans processing, so nothing would
352
+ // ever find it again.
353
+ let parsed: unknown;
180
354
  try {
181
- const parsed: unknown = JSON.parse(raw);
182
- if (!isValidJob(parsed)) {
183
- // Malformed payload — purge from `processing` so it can't sit
184
- // there indefinitely as a poison pill. recoverStale() also
185
- // catches survivors but pop()'s own move is the primary path.
186
- await client.lrem(this.#processingKey(), 1, raw);
187
- return null;
188
- }
189
- await client.set(
190
- this.#leaseKey(parsed.id),
191
- raw,
192
- "PX",
193
- String(this.#visibilityTimeout),
194
- );
195
- return parsed;
355
+ parsed = JSON.parse(raw);
196
356
  } catch {
357
+ // A poison pill: unparseable, and it would sit in processing
358
+ // forever blocking nothing but wasting every recovery pass.
359
+ await client.lrem(this.#processingKey(), 1, raw);
360
+ return null;
361
+ }
362
+ if (!isValidJob(parsed)) {
197
363
  await client.lrem(this.#processingKey(), 1, raw);
198
364
  return null;
199
365
  }
366
+
367
+ // A lease that cannot be written is a transient Redis failure, not a
368
+ // bad job. The error propagates and the job STAYS in processing with
369
+ // no lease, which is precisely the state recoverStale() puts back in
370
+ // pending — so the delivery guarantee survives the blip.
371
+ await client.set(
372
+ this.#leaseKey(parsed.id),
373
+ JSON.stringify({ owner: this.#workerId, raw } satisfies Lease),
374
+ "PX",
375
+ String(this.#visibilityTimeout),
376
+ );
377
+ return parsed;
378
+ }
379
+
380
+ /**
381
+ * Say the job is still being worked on, and push its deadline back.
382
+ *
383
+ * Answers `false` when there is nothing left to renew — the lease expired
384
+ * and the job was recovered, or it was recovered and re-popped by another
385
+ * worker, whose claim this one must not extend.
386
+ */
387
+ async renew(job: JobRecord): Promise<boolean> {
388
+ const client = await this.#client();
389
+ const key = this.#leaseKey(job.id);
390
+ const stored = await client.get(key);
391
+ if (stored === null) return false;
392
+ const lease = readLease(stored);
393
+ if (lease.owner !== undefined && lease.owner !== this.#workerId) {
394
+ return false;
395
+ }
396
+ await client.set(key, stored, "PX", String(this.#visibilityTimeout));
397
+ return true;
200
398
  }
201
399
 
202
- async complete(job: Job): Promise<void> {
400
+ async complete(job: JobRecord): Promise<void> {
203
401
  const client = await this.#client();
204
402
  await this.#removeFromProcessing(job);
205
403
  await client.del(this.#leaseKey(job.id));
206
404
  }
207
405
 
208
- async fail(job: Job, error: string): Promise<void> {
406
+ async fail(job: JobRecord, error: string): Promise<void> {
209
407
  const client = await this.#client();
210
408
  await this.#removeFromProcessing(job);
211
409
  await client.del(this.#leaseKey(job.id));
212
410
  job.error = error;
213
411
  job.status = "failed";
214
- await client.rpush(this.#failedKey(), JSON.stringify(job));
412
+ await this.#pushFailed(client, job);
215
413
  }
216
414
 
217
- async retry(job: Job): Promise<void> {
415
+ async retry(job: JobRecord): Promise<void> {
218
416
  const client = await this.#client();
219
417
  await this.#removeFromProcessing(job);
220
418
  await client.del(this.#leaseKey(job.id));
221
419
  job.status = "pending";
222
- await client.rpush(this.#pendingKey(), JSON.stringify(job));
420
+ await client.rpush(this.#pendingKey(queueOf(job)), JSON.stringify(job));
223
421
  }
224
422
 
225
423
  async recoverStale(): Promise<number> {
@@ -241,30 +439,77 @@ export class RedisDriver implements QueueDriver {
241
439
  continue;
242
440
  }
243
441
  const lease = await client.get(this.#leaseKey(parsed.id));
244
- if (lease === null) {
245
- await client.lrem(this.#processingKey(), 1, raw);
246
- parsed.status = "pending";
247
- await client.rpush(this.#pendingKey(), JSON.stringify(parsed));
248
- recovered++;
442
+ if (lease !== null) continue;
443
+
444
+ // The LREM is the claim, and its RESULT decides who acts. Two
445
+ // recovery passes overlapping — two workers, or one worker whose
446
+ // pass ran long — both read the same expired entry, and both used to
447
+ // push it back to pending: one job, delivered twice, from the
448
+ // mechanism that exists to make delivery reliable. Exactly one LREM
449
+ // can remove a given element, so exactly one pass continues past
450
+ // here. (A crash between this and the RPUSH below still loses the
451
+ // entry; closing that needs the whole pass in one server-side script,
452
+ // which is how upstream does it.)
453
+ const claimed = await client.lrem(this.#processingKey(), 1, raw);
454
+ if (claimed === 0) continue;
455
+
456
+ // A stall is not an attempt: the worker died before the handler
457
+ // could fail, so `attempts` never moved and `maxAttempts` never
458
+ // applied. A job that kills whatever picks it up was therefore
459
+ // recovered forever, taking the queue with it. Counted separately,
460
+ // and bounded — upstream bounds the same thing with the same
461
+ // default, failing the job once it is exceeded.
462
+ const stalled = (parsed.stalledCount ?? 0) + 1;
463
+ if (stalled > this.#maxStalledCount) {
464
+ parsed.stalledCount = stalled;
465
+ parsed.status = "failed";
466
+ parsed.error = `Stalled ${stalled} time(s) without completing (maxStalledCount ${this.#maxStalledCount})`;
467
+ await this.#pushFailed(client, parsed);
468
+ continue;
249
469
  }
470
+
471
+ parsed.stalledCount = stalled;
472
+ parsed.status = "pending";
473
+ // Back to the queue it came from, not to the default one: a recovered
474
+ // job whose queue nobody serves would never run again.
475
+ await client.rpush(
476
+ this.#pendingKey(queueOf(parsed)),
477
+ JSON.stringify(parsed),
478
+ );
479
+ recovered++;
250
480
  }
251
481
  return recovered;
252
482
  }
253
483
 
484
+ /**
485
+ * File a job as failed, keeping the list to `maxFailedJobs`.
486
+ *
487
+ * Unbounded, the failed list is a leak with no ceiling and no owner: nothing
488
+ * trims it, and `failed()` reads all of it in one LRANGE. The memory driver
489
+ * has capped its own at a thousand from the start; this is the same cap on
490
+ * the driver where the list actually survives a restart.
491
+ */
492
+ async #pushFailed(client: RedisClient, job: JobRecord): Promise<void> {
493
+ await client.rpush(this.#failedKey(), JSON.stringify(job));
494
+ if (this.#maxFailedJobs === 0 || !client.ltrim) return;
495
+ await client.ltrim(this.#failedKey(), -this.#maxFailedJobs, -1);
496
+ }
497
+
254
498
  /**
255
499
  * Remove the entry for `job` from the processing list. The string in
256
500
  * Redis is whatever pop() pushed, but QueueManager mutates `job` after
257
501
  * pop returns (attempts++, status="processing", processedAt, then
258
502
  * completed/failed/pending). LREM-ing on `JSON.stringify(job)` would
259
- * therefore miss every real-world entry. Use the lease — set to the
260
- * exact raw string at pop() time — and fall back to a list scan when
503
+ * therefore miss every real-world entry. Use the lease — which carries
504
+ * the exact raw string pop() moved — and fall back to a list scan when
261
505
  * the lease has expired (e.g. recoverStale already handled it).
262
506
  */
263
- async #removeFromProcessing(job: Job): Promise<void> {
507
+ async #removeFromProcessing(job: JobRecord): Promise<void> {
264
508
  const client = await this.#client();
265
509
  const stored = await client.get(this.#leaseKey(job.id));
266
510
  if (stored !== null) {
267
- const removed = await client.lrem(this.#processingKey(), 1, stored);
511
+ const { raw } = readLease(stored);
512
+ const removed = await client.lrem(this.#processingKey(), 1, raw);
268
513
  if (removed > 0) return;
269
514
  }
270
515
  // Lease missing or already-LREM'd entry not found — best-effort scan
@@ -277,14 +522,14 @@ export class RedisDriver implements QueueDriver {
277
522
  } catch {
278
523
  continue;
279
524
  }
280
- if (isValidJob(parsed) && (parsed as { id: string }).id === job.id) {
525
+ if (isValidJob(parsed) && parsed.id === job.id) {
281
526
  await client.lrem(this.#processingKey(), 1, item);
282
527
  return;
283
528
  }
284
529
  }
285
530
  }
286
531
 
287
- async failed(): Promise<Job[]> {
532
+ async failed(): Promise<JobRecord[]> {
288
533
  const client = await this.#client();
289
534
  const raws = await client.lrange(this.#failedKey(), 0, -1);
290
535
  return raws
@@ -296,11 +541,69 @@ export class RedisDriver implements QueueDriver {
296
541
  return null;
297
542
  }
298
543
  })
299
- .filter((j): j is Job => j !== null);
544
+ .filter((j): j is JobRecord => j !== null);
300
545
  }
301
546
 
302
- async size(): Promise<number> {
547
+ /**
548
+ * How many jobs are waiting on `queue` — its list plus its delayed set.
549
+ *
550
+ * A delayed job is queued; it is simply not due. Counting only the list
551
+ * reported an empty queue to anything draining one before shutdown.
552
+ */
553
+ async size(queue: string = DEFAULT_QUEUE): Promise<number> {
303
554
  const client = await this.#client();
304
- return client.llen(this.#pendingKey());
555
+ const waiting = await client.llen(this.#pendingKey(queue));
556
+ const delayed = client.zcard
557
+ ? await client.zcard(this.#delayedKey(queue))
558
+ : 0;
559
+ return waiting + delayed;
560
+ }
561
+
562
+ /**
563
+ * Move `queue`'s due jobs out of the delayed set and onto its list.
564
+ *
565
+ * `ZREM` is the claim: two workers can read the same due entry, and only
566
+ * the one whose removal returns 1 owns it. Without that the job is pushed
567
+ * onto the list once per worker that saw it.
568
+ */
569
+ async #promoteDue(client: RedisClient, queue: string): Promise<void> {
570
+ if (!client.zrangebyscore || !client.zrem) return;
571
+ const due = await client.zrangebyscore(
572
+ this.#delayedKey(queue),
573
+ 0,
574
+ Date.now(),
575
+ "LIMIT",
576
+ "0",
577
+ "100",
578
+ );
579
+ for (const raw of due) {
580
+ const claimed = await client.zrem(this.#delayedKey(queue), raw);
581
+ if (claimed !== 1) continue;
582
+ let parsed: unknown;
583
+ try {
584
+ parsed = JSON.parse(raw);
585
+ } catch {
586
+ // Already removed from the set; there is nothing runnable to push.
587
+ continue;
588
+ }
589
+ if (!isValidJob(parsed)) continue;
590
+ parsed.runAt = undefined;
591
+ await client.rpush(this.#pendingKey(queue), JSON.stringify(parsed));
592
+ }
593
+ }
594
+
595
+ /** Take the head of one queue, claiming it in `processing`. */
596
+ async #take(client: RedisClient, queue: string): Promise<string | null> {
597
+ if (client.lmove) {
598
+ return client.lmove(
599
+ this.#pendingKey(queue),
600
+ this.#processingKey(),
601
+ "LEFT",
602
+ "RIGHT",
603
+ );
604
+ }
605
+ const raw = await client.lpop(this.#pendingKey(queue));
606
+ if (raw) await client.rpush(this.#processingKey(), raw);
607
+ return raw;
305
608
  }
306
609
  }
package/src/index.ts CHANGED
@@ -1,17 +1,47 @@
1
1
  /**
2
2
  * @c9up/bay — Background job queue for the Ream framework.
3
3
  *
4
- * Dispatch/process/retry/fail pattern with pluggable drivers (Memory, Redis).
4
+ * A job is a class: it carries its own name, its options and the type of the
5
+ * payload it reads, and `dispatch(SomeJob, payload)` takes it. Named queues,
6
+ * `delay`, `timeout` and worker `concurrency` come from the same declaration.
5
7
  *
6
- * @implements MISS-11
8
+ * Dispatch/process/retry/fail, with pluggable drivers (Memory, Redis).
7
9
  */
8
10
 
9
- export type { BayProviderConfig } from "./BayProvider.js";
11
+ export {
12
+ type AdapterFactory,
13
+ drivers,
14
+ type QueueStoreFactory,
15
+ stores,
16
+ } from "./adapters.js";
17
+
18
+ import "./augmentations.js";
19
+
20
+ export type {
21
+ BayAppContext,
22
+ BayConfigStore,
23
+ BayContainer,
24
+ BayProviderConfig,
25
+ } from "./BayProvider.js";
10
26
  export { MemoryDriver } from "./drivers/MemoryDriver.js";
11
27
  export type { RedisClient } from "./drivers/RedisDriver.js";
12
28
  export { RedisDriver } from "./drivers/RedisDriver.js";
13
- export type { Job, JobHandler, QueueDriver } from "./QueueManager.js";
14
- export { QueueManager } from "./QueueManager.js";
29
+ export {
30
+ DEFAULT_QUEUE,
31
+ type Duration,
32
+ isJobClass,
33
+ Job,
34
+ type JobClass,
35
+ type JobOptions,
36
+ } from "./Job.js";
37
+ export type {
38
+ DispatchOptions,
39
+ JobHandler,
40
+ JobRecord,
41
+ QueueDriver,
42
+ WorkerOptions,
43
+ } from "./QueueManager.js";
44
+ export { QueueManager, queueOf } from "./QueueManager.js";
15
45
 
16
46
  import type { BayProviderConfig } from "./BayProvider.js";
17
47