@c9up/bay 0.1.13 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +122 -16
  2. package/dist/BayProvider.d.ts +51 -11
  3. package/dist/BayProvider.d.ts.map +1 -1
  4. package/dist/BayProvider.js +36 -13
  5. package/dist/BayProvider.js.map +1 -1
  6. package/dist/Job.d.ts +99 -0
  7. package/dist/Job.d.ts.map +1 -0
  8. package/dist/Job.js +78 -0
  9. package/dist/Job.js.map +1 -0
  10. package/dist/QueueManager.d.ts +140 -21
  11. package/dist/QueueManager.d.ts.map +1 -1
  12. package/dist/QueueManager.js +247 -53
  13. package/dist/QueueManager.js.map +1 -1
  14. package/dist/adapters.d.ts +68 -0
  15. package/dist/adapters.d.ts.map +1 -0
  16. package/dist/adapters.js +56 -0
  17. package/dist/adapters.js.map +1 -0
  18. package/dist/augmentations.d.ts +28 -0
  19. package/dist/augmentations.d.ts.map +1 -0
  20. package/dist/augmentations.js +17 -0
  21. package/dist/augmentations.js.map +1 -0
  22. package/dist/configure.d.ts +1 -0
  23. package/dist/configure.d.ts.map +1 -1
  24. package/dist/configure.js +24 -7
  25. package/dist/configure.js.map +1 -1
  26. package/dist/console/contract.d.ts +60 -0
  27. package/dist/console/contract.d.ts.map +1 -0
  28. package/dist/console/contract.js +36 -0
  29. package/dist/console/contract.js.map +1 -0
  30. package/dist/console/index.d.ts +29 -0
  31. package/dist/console/index.d.ts.map +1 -0
  32. package/dist/console/index.js +45 -0
  33. package/dist/console/index.js.map +1 -0
  34. package/dist/console/makeJob.d.ts +32 -0
  35. package/dist/console/makeJob.d.ts.map +1 -0
  36. package/dist/console/makeJob.js +118 -0
  37. package/dist/console/makeJob.js.map +1 -0
  38. package/dist/console/queueWork.d.ts +18 -0
  39. package/dist/console/queueWork.d.ts.map +1 -0
  40. package/dist/console/queueWork.js +58 -0
  41. package/dist/console/queueWork.js.map +1 -0
  42. package/dist/drivers/MemoryDriver.d.ts +14 -8
  43. package/dist/drivers/MemoryDriver.d.ts.map +1 -1
  44. package/dist/drivers/MemoryDriver.js +61 -7
  45. package/dist/drivers/MemoryDriver.js.map +1 -1
  46. package/dist/drivers/RedisDriver.d.ts +60 -8
  47. package/dist/drivers/RedisDriver.d.ts.map +1 -1
  48. package/dist/drivers/RedisDriver.js +214 -30
  49. package/dist/drivers/RedisDriver.js.map +1 -1
  50. package/dist/index.d.ts +10 -6
  51. package/dist/index.d.ts.map +1 -1
  52. package/dist/index.js +8 -4
  53. package/dist/index.js.map +1 -1
  54. package/dist/jobs.d.ts +55 -0
  55. package/dist/jobs.d.ts.map +1 -0
  56. package/dist/jobs.js +135 -0
  57. package/dist/jobs.js.map +1 -0
  58. package/dist/quasar.d.ts +6 -9
  59. package/dist/quasar.d.ts.map +1 -1
  60. package/dist/quasar.js +27 -55
  61. package/dist/quasar.js.map +1 -1
  62. package/dist/testing/FakeQueue.d.ts +15 -9
  63. package/dist/testing/FakeQueue.d.ts.map +1 -1
  64. package/dist/testing/FakeQueue.js +13 -3
  65. package/dist/testing/FakeQueue.js.map +1 -1
  66. package/dist/vendor/nodeEnv.d.ts +5 -0
  67. package/dist/vendor/nodeEnv.d.ts.map +1 -0
  68. package/dist/{nodeEnv.js → vendor/nodeEnv.js} +13 -7
  69. package/dist/vendor/nodeEnv.js.map +1 -0
  70. package/dist/vendor/quasarConnection.d.ts +35 -0
  71. package/dist/vendor/quasarConnection.d.ts.map +1 -0
  72. package/dist/vendor/quasarConnection.js +62 -0
  73. package/dist/vendor/quasarConnection.js.map +1 -0
  74. package/package.json +5 -3
  75. package/src/BayProvider.ts +92 -26
  76. package/src/Job.ts +137 -0
  77. package/src/QueueManager.ts +411 -56
  78. package/src/adapters.ts +75 -0
  79. package/src/augmentations.ts +31 -0
  80. package/src/configure.ts +25 -7
  81. package/src/console/contract.ts +94 -0
  82. package/src/console/index.ts +68 -0
  83. package/src/console/makeJob.ts +139 -0
  84. package/src/console/queueWork.ts +70 -0
  85. package/src/drivers/MemoryDriver.ts +66 -14
  86. package/src/drivers/RedisDriver.ts +303 -43
  87. package/src/index.ts +35 -6
  88. package/src/jobs.ts +144 -0
  89. package/src/quasar.ts +28 -72
  90. package/src/testing/FakeQueue.ts +25 -15
  91. package/src/{nodeEnv.ts → vendor/nodeEnv.ts} +14 -7
  92. package/src/vendor/quasarConnection.ts +129 -0
  93. package/dist/nodeEnv.d.ts +0 -16
  94. package/dist/nodeEnv.d.ts.map +0 -1
  95. package/dist/nodeEnv.js.map +0 -1
  96. package/dist/stores.d.ts +0 -41
  97. package/dist/stores.d.ts.map +0 -1
  98. package/dist/stores.js +0 -46
  99. package/dist/stores.js.map +0 -1
  100. package/src/stores.ts +0 -59
@@ -0,0 +1,58 @@
1
+ /**
2
+ * `queue:work` — run the worker that takes jobs off the queue.
3
+ *
4
+ * The command a deployment runs as its own process, beside the HTTP one:
5
+ *
6
+ * ream queue:work
7
+ * ream queue:work --queue=emails,notifications
8
+ * ream queue:work --concurrency=10
9
+ *
10
+ * It stays alive on purpose — `staysAlive` — and the loop it starts is the
11
+ * application's, so a SIGTERM reaches `BayProvider.shutdown()`, which stops the
12
+ * worker and lets the job in flight finish rather than dropping it.
13
+ */
14
+ import { getQueue } from "../services/main.js";
15
+ import { flag } from "./contract.js";
16
+ /** Split `--queue=a,b` into names, dropping the empties a trailing comma makes. */
17
+ export function parseQueues(value) {
18
+ if (value === undefined)
19
+ return undefined;
20
+ const names = value
21
+ .split(",")
22
+ .map((name) => name.trim())
23
+ .filter((name) => name.length > 0);
24
+ return names.length > 0 ? names : undefined;
25
+ }
26
+ export function queueWorkCommand() {
27
+ return class QueueWork {
28
+ static commandName = "queue:work";
29
+ static description = "Process queued jobs until the process is stopped";
30
+ // The worker needs the container: the queue it drains is the one the
31
+ // provider booted, with the driver the config named.
32
+ static options = { startApp: true, staysAlive: true };
33
+ static flags = [
34
+ flag("queue", "string", {
35
+ description: "Comma-separated queues to serve, in order (default: the default queue)",
36
+ }),
37
+ flag("concurrency", "number", {
38
+ description: "How many jobs to run at once (default: 1)",
39
+ }),
40
+ ];
41
+ async run() {
42
+ const manager = getQueue();
43
+ if (!manager) {
44
+ // Naming the wiring beats a stack trace out of the service proxy:
45
+ // the usual cause is a project that never added the provider.
46
+ throw new Error("[bay] queue:work found no queue. Add `() => import('@c9up/bay/provider')` " +
47
+ "to the providers in reamrc.ts, or call setQueue(myQueue) at boot.");
48
+ }
49
+ const queues = parseQueues(this.queue);
50
+ process.stdout.write(`[bay] worker started — queues: ${(queues ?? ["default"]).join(", ")}, concurrency: ${this.concurrency ?? 1}\n`);
51
+ await manager.work({
52
+ queues,
53
+ concurrency: this.concurrency,
54
+ });
55
+ }
56
+ };
57
+ }
58
+ //# sourceMappingURL=queueWork.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"queueWork.js","sourceRoot":"","sources":["../../src/console/queueWork.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAwB,IAAI,EAAE,MAAM,eAAe,CAAC;AAE3D,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,KAAyB;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK;SACjB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO,MAAM,SAAS;QACrB,MAAM,CAAC,WAAW,GAAG,YAAY,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,kDAAkD,CAAC;QACxE,qEAAqE;QACrE,qDAAqD;QACrD,MAAM,CAAC,OAAO,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACtD,MAAM,CAAC,KAAK,GAAG;YACd,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE;gBACvB,WAAW,EACV,wEAAwE;aACzE,CAAC;YACF,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE;gBAC7B,WAAW,EAAE,2CAA2C;aACxD,CAAC;SACF,CAAC;QAKF,KAAK,CAAC,GAAG;YACR,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,kEAAkE;gBAClE,8DAA8D;gBAC9D,MAAM,IAAI,KAAK,CACd,4EAA4E;oBAC3E,mEAAmE,CACpE,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,kCAAkC,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAC/G,CAAC;YAEF,MAAM,OAAO,CAAC,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC,CAAC;QACJ,CAAC;KACD,CAAC;AACH,CAAC"}
@@ -1,18 +1,24 @@
1
1
  /**
2
- * Memory queue driver — in-process queue for development.
2
+ * Memory queue driver — in-process queue for development and tests.
3
3
  */
4
- import type { Job, QueueDriver } from "../QueueManager.js";
4
+ import { type JobRecord, type QueueDriver } from "../QueueManager.js";
5
5
  export declare class MemoryDriver implements QueueDriver {
6
6
  #private;
7
7
  constructor(options?: {
8
8
  maxFailedJobs?: number;
9
9
  });
10
- push(job: Job): Promise<void>;
11
- pop(): Promise<Job | null>;
12
- fail(job: Job, error: string): Promise<void>;
13
- complete(_job: Job): Promise<void>;
14
- retry(job: Job): Promise<void>;
15
- failed(): Promise<Job[]>;
10
+ push(job: JobRecord): Promise<void>;
11
+ pop(queues?: readonly string[]): Promise<JobRecord | null>;
12
+ fail(job: JobRecord, error: string): Promise<void>;
13
+ complete(_job: JobRecord): Promise<void>;
14
+ retry(job: JobRecord): Promise<void>;
15
+ failed(): Promise<JobRecord[]>;
16
+ /**
17
+ * Everything waiting, across every queue — a delayed job included.
18
+ *
19
+ * It is queued; it is simply not due. Leaving it out would report an empty
20
+ * queue to anything draining one before shutdown.
21
+ */
16
22
  size(): Promise<number>;
17
23
  }
18
24
  //# sourceMappingURL=MemoryDriver.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"MemoryDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE3D,qBAAa,YAAa,YAAW,WAAW;;gBAKnC,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE;IAI1C,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7B,GAAG,IAAI,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAI1B,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS5C,QAAQ,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlC,KAAK,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9B,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAIxB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;CAG7B"}
1
+ {"version":3,"file":"MemoryDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAW,MAAM,oBAAoB,CAAC;AAE/E,qBAAa,YAAa,YAAW,WAAW;;gBAcnC,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE;IAwB1C,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IASnC,GAAG,CACR,MAAM,GAAE,SAAS,MAAM,EAAoB,GACzC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAWtB,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxC,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpC,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAIpC;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;CAK7B"}
@@ -1,18 +1,63 @@
1
1
  /**
2
- * Memory queue driver — in-process queue for development.
2
+ * Memory queue driver — in-process queue for development and tests.
3
3
  */
4
+ import { DEFAULT_QUEUE } from "../Job.js";
5
+ import { queueOf } from "../QueueManager.js";
4
6
  export class MemoryDriver {
5
- #pending = [];
7
+ /** One list per named queue, created on first use. */
8
+ #pending = new Map();
9
+ /**
10
+ * Jobs whose `runAt` has not arrived, oldest deadline first.
11
+ *
12
+ * Kept apart from the queues rather than filtered on the way out: a delayed
13
+ * job at the head of a list would otherwise be skipped over on every poll,
14
+ * and a queue whose head is not due would look empty while it is not.
15
+ */
16
+ #delayed = [];
6
17
  #failedJobs = [];
7
18
  #maxFailedJobs;
8
19
  constructor(options) {
9
20
  this.#maxFailedJobs = options?.maxFailedJobs ?? 1000;
10
21
  }
22
+ #queue(name) {
23
+ const existing = this.#pending.get(name);
24
+ if (existing !== undefined)
25
+ return existing;
26
+ const created = [];
27
+ this.#pending.set(name, created);
28
+ return created;
29
+ }
30
+ /** Move everything whose delay has elapsed into its queue. */
31
+ #promoteDue(now = Date.now()) {
32
+ if (this.#delayed.length === 0)
33
+ return;
34
+ const due = this.#delayed.filter((job) => (job.runAt ?? 0) <= now);
35
+ if (due.length === 0)
36
+ return;
37
+ this.#delayed = this.#delayed.filter((job) => (job.runAt ?? 0) > now);
38
+ for (const job of due) {
39
+ job.runAt = undefined;
40
+ this.#queue(queueOf(job)).push(job);
41
+ }
42
+ }
11
43
  async push(job) {
12
- this.#pending.push(job);
44
+ if (job.runAt !== undefined && job.runAt > Date.now()) {
45
+ this.#delayed.push(job);
46
+ this.#delayed.sort((a, b) => (a.runAt ?? 0) - (b.runAt ?? 0));
47
+ return;
48
+ }
49
+ this.#queue(queueOf(job)).push(job);
13
50
  }
14
- async pop() {
15
- return this.#pending.shift() ?? null;
51
+ async pop(queues = [DEFAULT_QUEUE]) {
52
+ this.#promoteDue();
53
+ // In the order given: naming `['critical', 'default']` is how a worker
54
+ // says which queue it would rather drain first.
55
+ for (const name of queues) {
56
+ const next = this.#pending.get(name)?.shift();
57
+ if (next !== undefined)
58
+ return next;
59
+ }
60
+ return null;
16
61
  }
17
62
  async fail(job, error) {
18
63
  job.error = error;
@@ -27,13 +72,22 @@ export class MemoryDriver {
27
72
  }
28
73
  async retry(job) {
29
74
  job.status = "pending";
30
- this.#pending.push(job);
75
+ this.#queue(queueOf(job)).push(job);
31
76
  }
32
77
  async failed() {
33
78
  return [...this.#failedJobs];
34
79
  }
80
+ /**
81
+ * Everything waiting, across every queue — a delayed job included.
82
+ *
83
+ * It is queued; it is simply not due. Leaving it out would report an empty
84
+ * queue to anything draining one before shutdown.
85
+ */
35
86
  async size() {
36
- return this.#pending.length;
87
+ let total = this.#delayed.length;
88
+ for (const jobs of this.#pending.values())
89
+ total += jobs.length;
90
+ return total;
37
91
  }
38
92
  }
39
93
  //# sourceMappingURL=MemoryDriver.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"MemoryDriver.js","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,OAAO,YAAY;IACxB,QAAQ,GAAU,EAAE,CAAC;IACrB,WAAW,GAAU,EAAE,CAAC;IACxB,cAAc,CAAS;IAEvB,YAAY,OAAoC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,aAAa,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAQ;QAClB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,GAAG;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAQ,EAAE,KAAa;QACjC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;QAClB,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3E,CAAC;IACF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAS;QACvB,kCAAkC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAQ;QACnB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,MAAM;QACX,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC7B,CAAC;CACD"}
1
+ {"version":3,"file":"MemoryDriver.js","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAoC,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE/E,MAAM,OAAO,YAAY;IACxB,sDAAsD;IACtD,QAAQ,GAA6B,IAAI,GAAG,EAAE,CAAC;IAC/C;;;;;;OAMG;IACH,QAAQ,GAAgB,EAAE,CAAC;IAC3B,WAAW,GAAgB,EAAE,CAAC;IAC9B,cAAc,CAAS;IAEvB,YAAY,OAAoC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,aAAa,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,QAAQ,CAAC;QAC5C,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjC,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,8DAA8D;IAC9D,WAAW,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;QACnE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YACvB,GAAG,CAAC,KAAK,GAAG,SAAS,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAc;QACxB,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9D,OAAO;QACR,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CACR,SAA4B,CAAC,aAAa,CAAC;QAE3C,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,uEAAuE;QACvE,gDAAgD;QAChD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC;YAC9C,IAAI,IAAI,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAc,EAAE,KAAa;QACvC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;QAClB,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3E,CAAC;IACF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAe;QAC7B,kCAAkC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAc;QACzB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM;QACX,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI;QACT,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;QAChE,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}
@@ -17,17 +17,32 @@
17
17
  * { PX: ms }) does NOT satisfy this interface and would drop the lease TTL — it
18
18
  * needs a thin adapter.
19
19
  */
20
- import type { Job, QueueDriver } from "../QueueManager.js";
20
+ import { type JobRecord, type QueueDriver } from "../QueueManager.js";
21
21
  export interface RedisClient {
22
22
  rpush(key: string, ...values: string[]): Promise<number>;
23
23
  lpop(key: string): Promise<string | null>;
24
24
  lmove?(source: string, destination: string, from: "LEFT" | "RIGHT", to: "LEFT" | "RIGHT"): Promise<string | null>;
25
25
  lrem(key: string, count: number, element: string): Promise<number>;
26
+ /**
27
+ * Optional, like `lmove`. Present on ioredis; without it the failed list
28
+ * simply keeps its entries, which is what this driver did before.
29
+ */
30
+ ltrim?(key: string, start: number, stop: number): Promise<string>;
26
31
  llen(key: string): Promise<number>;
27
32
  lrange(key: string, start: number, stop: number): Promise<string[]>;
28
33
  del(key: string): Promise<number>;
29
34
  set(key: string, value: string, ...args: string[]): Promise<string | null>;
30
35
  get(key: string): Promise<string | null>;
36
+ /**
37
+ * Sorted-set commands, for delayed jobs. Optional like `lmove`: a client
38
+ * without them can still run a queue, and `push` refuses a job carrying a
39
+ * `delay` rather than running it early — which is the one thing a delay
40
+ * must not do.
41
+ */
42
+ zadd?(key: string, score: number, member: string): Promise<number | string>;
43
+ zrangebyscore?(key: string, min: number | string, max: number | string, ...args: string[]): Promise<string[]>;
44
+ zrem?(key: string, ...members: string[]): Promise<number>;
45
+ zcard?(key: string): Promise<number>;
31
46
  }
32
47
  /**
33
48
  * Where the client comes from. A resolver is what lets a queue name its
@@ -47,14 +62,51 @@ export declare class RedisDriver implements QueueDriver {
47
62
  * deployment makes, not one a version check makes for it.
48
63
  */
49
64
  allowNonAtomicPop?: boolean;
65
+ /**
66
+ * How many times a job may be reclaimed from a stalled worker before
67
+ * it is filed as failed instead of pushed round again. Default `1`,
68
+ * upstream's default for the same setting.
69
+ *
70
+ * Unbounded recovery is a job that kills its worker taking the whole
71
+ * queue down with it, forever: the crash never reaches the failure
72
+ * path, so `attempts` never moves and `maxAttempts` never applies.
73
+ */
74
+ maxStalledCount?: number;
75
+ /**
76
+ * How many failed jobs to keep. Default `1000` — the ceiling the
77
+ * memory driver already had. `0` keeps every one of them.
78
+ *
79
+ * Only enforced when the client answers `ltrim`.
80
+ */
81
+ maxFailedJobs?: number;
50
82
  });
51
- push(job: Job): Promise<void>;
52
- pop(): Promise<Job | null>;
53
- complete(job: Job): Promise<void>;
54
- fail(job: Job, error: string): Promise<void>;
55
- retry(job: Job): Promise<void>;
83
+ /**
84
+ * Renew a lease at half its length: two chances to be heard before the
85
+ * deadline, so one slow round-trip does not hand a running job to somebody
86
+ * else. Read by `QueueManager` while a handler runs.
87
+ */
88
+ get renewIntervalMs(): number;
89
+ push(job: JobRecord): Promise<void>;
90
+ pop(queues?: readonly string[]): Promise<JobRecord | null>;
91
+ /**
92
+ * Say the job is still being worked on, and push its deadline back.
93
+ *
94
+ * Answers `false` when there is nothing left to renew — the lease expired
95
+ * and the job was recovered, or it was recovered and re-popped by another
96
+ * worker, whose claim this one must not extend.
97
+ */
98
+ renew(job: JobRecord): Promise<boolean>;
99
+ complete(job: JobRecord): Promise<void>;
100
+ fail(job: JobRecord, error: string): Promise<void>;
101
+ retry(job: JobRecord): Promise<void>;
56
102
  recoverStale(): Promise<number>;
57
- failed(): Promise<Job[]>;
58
- size(): Promise<number>;
103
+ failed(): Promise<JobRecord[]>;
104
+ /**
105
+ * How many jobs are waiting on `queue` — its list plus its delayed set.
106
+ *
107
+ * A delayed job is queued; it is simply not due. Counting only the list
108
+ * reported an empty queue to anything draining one before shutdown.
109
+ */
110
+ size(queue?: string): Promise<number>;
59
111
  }
60
112
  //# sourceMappingURL=RedisDriver.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"RedisDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/RedisDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE3D,MAAM,WAAW,WAAW;IAC3B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1C,KAAK,CAAC,CACL,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,GAAG,OAAO,EACtB,EAAE,EAAE,MAAM,GAAG,OAAO,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC3E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CACzC;AAcD;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAC1B,WAAW,GACX,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;AAmD9C,qBAAa,WAAY,YAAW,WAAW;;gBAsC7C,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B;;;;WAIG;QACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;KAC5B;IAkCI,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAK7B,GAAG,IAAI,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC;IAkD1B,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS5C,KAAK,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ9B,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IA8D/B,MAAM,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAexB,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;CAI7B"}
1
+ {"version":3,"file":"RedisDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/RedisDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAW,MAAM,oBAAoB,CAAC;AAG/E,MAAM,WAAW,WAAW;IAC3B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1C,KAAK,CAAC,CACL,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,GAAG,OAAO,EACtB,EAAE,EAAE,MAAM,GAAG,OAAO,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE;;;OAGG;IACH,KAAK,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC3E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACzC;;;;;OAKG;IACH,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAC5E,aAAa,CAAC,CACb,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,GAAG,IAAI,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrB,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1D,KAAK,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrC;AAiDD;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAC1B,WAAW,GACX,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;AAuD9C,qBAAa,WAAY,YAAW,WAAW;;gBAsC7C,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B;;;;WAIG;QACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B;;;;;;;;WAQG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;;;;;WAKG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB;IA4CF;;;;OAIG;IACH,IAAI,eAAe,IAAI,MAAM,CAE5B;IA2BK,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBnC,GAAG,CACR,MAAM,GAAE,SAAS,MAAM,EAAoB,GACzC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IA6C5B;;;;;;OAMG;IACG,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAavC,QAAQ,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAQpC,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IA6G/B,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAepC;;;;;OAKG;IACG,IAAI,CAAC,KAAK,GAAE,MAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;CAwD1D"}
@@ -17,16 +17,39 @@
17
17
  * { PX: ms }) does NOT satisfy this interface and would drop the lease TTL — it
18
18
  * needs a thin adapter.
19
19
  */
20
- import { inProduction } from "../nodeEnv.js";
20
+ import { DEFAULT_QUEUE } from "../Job.js";
21
+ import { queueOf } from "../QueueManager.js";
22
+ import { inProduction } from "../vendor/nodeEnv.js";
21
23
  function isValidJob(obj) {
22
24
  if (typeof obj !== "object" || obj === null)
23
25
  return false;
24
- const j = obj;
25
- return (typeof j.id === "string" &&
26
- typeof j.name === "string" &&
27
- typeof j.attempts === "number" &&
28
- typeof j.maxAttempts === "number" &&
29
- typeof j.status === "string");
26
+ return (typeof Reflect.get(obj, "id") === "string" &&
27
+ typeof Reflect.get(obj, "name") === "string" &&
28
+ typeof Reflect.get(obj, "attempts") === "number" &&
29
+ typeof Reflect.get(obj, "maxAttempts") === "number" &&
30
+ typeof Reflect.get(obj, "status") === "string");
31
+ }
32
+ /**
33
+ * Read a lease back. A value written by an older version of this driver is the
34
+ * raw job string on its own, with no owner — still usable for the one thing
35
+ * `#removeFromProcessing` needs it for.
36
+ */
37
+ function readLease(stored) {
38
+ let parsed;
39
+ try {
40
+ parsed = JSON.parse(stored);
41
+ }
42
+ catch {
43
+ return { raw: stored };
44
+ }
45
+ if (typeof parsed !== "object" || parsed === null)
46
+ return { raw: stored };
47
+ const raw = Reflect.get(parsed, "raw");
48
+ const owner = Reflect.get(parsed, "owner");
49
+ if (typeof raw !== "string" || typeof owner !== "string") {
50
+ return { raw: stored };
51
+ }
52
+ return { owner, raw };
30
53
  }
31
54
  /**
32
55
  * LMOVE (Redis 6.2+) is what makes pop() atomic. Warned once, when the client
@@ -42,6 +65,10 @@ function checkLmove(client, allowNonAtomicPop) {
42
65
  // job from pending before it reaches processing: nothing recovers it,
43
66
  // because nothing knows it existed. That is a different product, and in
44
67
  // production it must be asked for rather than fallen into.
68
+ // Read through `inProduction()` rather than `NODE_ENV === "production"`:
69
+ // `NODE_ENV=prod` is ordinary in a Dockerfile, and taken verbatim it answers
70
+ // "not production" — so this queue would silently accept a delivery
71
+ // guarantee weaker than the one it advertises.
45
72
  if (inProduction() && !allowNonAtomicPop) {
46
73
  throw new Error("[bay] this Redis client has no LMOVE (Redis < 6.2), so pop() would be a non-atomic lpop+rpush — " +
47
74
  "a crash between the two loses the in-flight job, turning at-least-once delivery into at-most-once.\n" +
@@ -131,26 +158,71 @@ export class RedisDriver {
131
158
  throw new Error(`[bay] RedisDriver visibilityTimeoutMs must be a positive integer (ms), got ${visibilityTimeout}`);
132
159
  }
133
160
  this.#visibilityTimeout = visibilityTimeout;
161
+ const maxStalled = options?.maxStalledCount ?? 1;
162
+ if (!Number.isInteger(maxStalled) || maxStalled < 0) {
163
+ throw new Error(`[bay] RedisDriver maxStalledCount must be a non-negative integer, got ${maxStalled}`);
164
+ }
165
+ this.#maxStalledCount = maxStalled;
166
+ const maxFailed = options?.maxFailedJobs ?? 1000;
167
+ if (!Number.isInteger(maxFailed) || maxFailed < 0) {
168
+ throw new Error(`[bay] RedisDriver maxFailedJobs must be a non-negative integer, got ${maxFailed}`);
169
+ }
170
+ this.#maxFailedJobs = maxFailed;
171
+ }
172
+ /**
173
+ * Renew a lease at half its length: two chances to be heard before the
174
+ * deadline, so one slow round-trip does not hand a running job to somebody
175
+ * else. Read by `QueueManager` while a handler runs.
176
+ */
177
+ get renewIntervalMs() {
178
+ return Math.max(1, Math.floor(this.#visibilityTimeout / 2));
134
179
  }
135
- #pendingKey = () => `${this.#prefix}pending`;
180
+ /**
181
+ * Where one queue's jobs wait.
182
+ *
183
+ * The default queue keeps the key it always had. Naming it
184
+ * `queue:default:pending` would have been tidier and would have orphaned
185
+ * every job already sitting in `queue:pending` at the moment of the upgrade
186
+ * — a silent loss, since nothing reads the old key afterwards.
187
+ */
188
+ #pendingKey = (queue = DEFAULT_QUEUE) => queue === DEFAULT_QUEUE
189
+ ? `${this.#prefix}pending`
190
+ : `${this.#prefix}q:${queue}:pending`;
191
+ #delayedKey = (queue = DEFAULT_QUEUE) => queue === DEFAULT_QUEUE
192
+ ? `${this.#prefix}delayed`
193
+ : `${this.#prefix}q:${queue}:delayed`;
136
194
  #processingKey = () => `${this.#prefix}processing`;
137
195
  #allowNonAtomicPop = false;
196
+ #maxStalledCount = 1;
197
+ #maxFailedJobs = 1000;
198
+ /** This driver instance, as a lease owner. */
199
+ #workerId = crypto.randomUUID();
138
200
  #failedKey = () => `${this.#prefix}failed`;
139
201
  #leaseKey = (jobId) => `${this.#prefix}lease:${jobId}`;
140
202
  async push(job) {
141
203
  const client = await this.#client();
142
- await client.rpush(this.#pendingKey(), JSON.stringify(job));
204
+ const queue = queueOf(job);
205
+ if (job.runAt !== undefined && job.runAt > Date.now()) {
206
+ if (!client.zadd) {
207
+ // Pushing it to the list instead would run it now, which is the
208
+ // one thing a delay exists to prevent.
209
+ throw new Error("This Redis client cannot hold a delayed job: it has no ZADD. " +
210
+ "Use a client with sorted-set commands (ioredis has them), or dispatch without `delay`.");
211
+ }
212
+ await client.zadd(this.#delayedKey(queue), job.runAt, JSON.stringify(job));
213
+ return;
214
+ }
215
+ await client.rpush(this.#pendingKey(queue), JSON.stringify(job));
143
216
  }
144
- async pop() {
217
+ async pop(queues = [DEFAULT_QUEUE]) {
145
218
  const client = await this.#client();
146
219
  let raw = null;
147
- if (client.lmove) {
148
- raw = await client.lmove(this.#pendingKey(), this.#processingKey(), "LEFT", "RIGHT");
149
- }
150
- else {
151
- raw = await client.lpop(this.#pendingKey());
152
- if (raw)
153
- await client.rpush(this.#processingKey(), raw);
220
+ // In the order given, so a worker can say which queue it drains first.
221
+ for (const queue of queues) {
222
+ await this.#promoteDue(client, queue);
223
+ raw = await this.#take(client, queue);
224
+ if (raw !== null)
225
+ break;
154
226
  }
155
227
  if (!raw)
156
228
  return null;
@@ -177,9 +249,29 @@ export class RedisDriver {
177
249
  // bad job. The error propagates and the job STAYS in processing with
178
250
  // no lease, which is precisely the state recoverStale() puts back in
179
251
  // pending — so the delivery guarantee survives the blip.
180
- await client.set(this.#leaseKey(parsed.id), raw, "PX", String(this.#visibilityTimeout));
252
+ await client.set(this.#leaseKey(parsed.id), JSON.stringify({ owner: this.#workerId, raw }), "PX", String(this.#visibilityTimeout));
181
253
  return parsed;
182
254
  }
255
+ /**
256
+ * Say the job is still being worked on, and push its deadline back.
257
+ *
258
+ * Answers `false` when there is nothing left to renew — the lease expired
259
+ * and the job was recovered, or it was recovered and re-popped by another
260
+ * worker, whose claim this one must not extend.
261
+ */
262
+ async renew(job) {
263
+ const client = await this.#client();
264
+ const key = this.#leaseKey(job.id);
265
+ const stored = await client.get(key);
266
+ if (stored === null)
267
+ return false;
268
+ const lease = readLease(stored);
269
+ if (lease.owner !== undefined && lease.owner !== this.#workerId) {
270
+ return false;
271
+ }
272
+ await client.set(key, stored, "PX", String(this.#visibilityTimeout));
273
+ return true;
274
+ }
183
275
  async complete(job) {
184
276
  const client = await this.#client();
185
277
  await this.#removeFromProcessing(job);
@@ -191,14 +283,14 @@ export class RedisDriver {
191
283
  await client.del(this.#leaseKey(job.id));
192
284
  job.error = error;
193
285
  job.status = "failed";
194
- await client.rpush(this.#failedKey(), JSON.stringify(job));
286
+ await this.#pushFailed(client, job);
195
287
  }
196
288
  async retry(job) {
197
289
  const client = await this.#client();
198
290
  await this.#removeFromProcessing(job);
199
291
  await client.del(this.#leaseKey(job.id));
200
292
  job.status = "pending";
201
- await client.rpush(this.#pendingKey(), JSON.stringify(job));
293
+ await client.rpush(this.#pendingKey(queueOf(job)), JSON.stringify(job));
202
294
  }
203
295
  async recoverStale() {
204
296
  const client = await this.#client();
@@ -220,29 +312,72 @@ export class RedisDriver {
220
312
  continue;
221
313
  }
222
314
  const lease = await client.get(this.#leaseKey(parsed.id));
223
- if (lease === null) {
224
- await client.lrem(this.#processingKey(), 1, raw);
225
- parsed.status = "pending";
226
- await client.rpush(this.#pendingKey(), JSON.stringify(parsed));
227
- recovered++;
315
+ if (lease !== null)
316
+ continue;
317
+ // The LREM is the claim, and its RESULT decides who acts. Two
318
+ // recovery passes overlapping — two workers, or one worker whose
319
+ // pass ran long — both read the same expired entry, and both used to
320
+ // push it back to pending: one job, delivered twice, from the
321
+ // mechanism that exists to make delivery reliable. Exactly one LREM
322
+ // can remove a given element, so exactly one pass continues past
323
+ // here. (A crash between this and the RPUSH below still loses the
324
+ // entry; closing that needs the whole pass in one server-side script,
325
+ // which is how upstream does it.)
326
+ const claimed = await client.lrem(this.#processingKey(), 1, raw);
327
+ if (claimed === 0)
328
+ continue;
329
+ // A stall is not an attempt: the worker died before the handler
330
+ // could fail, so `attempts` never moved and `maxAttempts` never
331
+ // applied. A job that kills whatever picks it up was therefore
332
+ // recovered forever, taking the queue with it. Counted separately,
333
+ // and bounded — upstream bounds the same thing with the same
334
+ // default, failing the job once it is exceeded.
335
+ const stalled = (parsed.stalledCount ?? 0) + 1;
336
+ if (stalled > this.#maxStalledCount) {
337
+ parsed.stalledCount = stalled;
338
+ parsed.status = "failed";
339
+ parsed.error = `Stalled ${stalled} time(s) without completing (maxStalledCount ${this.#maxStalledCount})`;
340
+ await this.#pushFailed(client, parsed);
341
+ continue;
228
342
  }
343
+ parsed.stalledCount = stalled;
344
+ parsed.status = "pending";
345
+ // Back to the queue it came from, not to the default one: a recovered
346
+ // job whose queue nobody serves would never run again.
347
+ await client.rpush(this.#pendingKey(queueOf(parsed)), JSON.stringify(parsed));
348
+ recovered++;
229
349
  }
230
350
  return recovered;
231
351
  }
352
+ /**
353
+ * File a job as failed, keeping the list to `maxFailedJobs`.
354
+ *
355
+ * Unbounded, the failed list is a leak with no ceiling and no owner: nothing
356
+ * trims it, and `failed()` reads all of it in one LRANGE. The memory driver
357
+ * has capped its own at a thousand from the start; this is the same cap on
358
+ * the driver where the list actually survives a restart.
359
+ */
360
+ async #pushFailed(client, job) {
361
+ await client.rpush(this.#failedKey(), JSON.stringify(job));
362
+ if (this.#maxFailedJobs === 0 || !client.ltrim)
363
+ return;
364
+ await client.ltrim(this.#failedKey(), -this.#maxFailedJobs, -1);
365
+ }
232
366
  /**
233
367
  * Remove the entry for `job` from the processing list. The string in
234
368
  * Redis is whatever pop() pushed, but QueueManager mutates `job` after
235
369
  * pop returns (attempts++, status="processing", processedAt, then
236
370
  * completed/failed/pending). LREM-ing on `JSON.stringify(job)` would
237
- * therefore miss every real-world entry. Use the lease — set to the
238
- * exact raw string at pop() time — and fall back to a list scan when
371
+ * therefore miss every real-world entry. Use the lease — which carries
372
+ * the exact raw string pop() moved — and fall back to a list scan when
239
373
  * the lease has expired (e.g. recoverStale already handled it).
240
374
  */
241
375
  async #removeFromProcessing(job) {
242
376
  const client = await this.#client();
243
377
  const stored = await client.get(this.#leaseKey(job.id));
244
378
  if (stored !== null) {
245
- const removed = await client.lrem(this.#processingKey(), 1, stored);
379
+ const { raw } = readLease(stored);
380
+ const removed = await client.lrem(this.#processingKey(), 1, raw);
246
381
  if (removed > 0)
247
382
  return;
248
383
  }
@@ -278,9 +413,58 @@ export class RedisDriver {
278
413
  })
279
414
  .filter((j) => j !== null);
280
415
  }
281
- async size() {
416
+ /**
417
+ * How many jobs are waiting on `queue` — its list plus its delayed set.
418
+ *
419
+ * A delayed job is queued; it is simply not due. Counting only the list
420
+ * reported an empty queue to anything draining one before shutdown.
421
+ */
422
+ async size(queue = DEFAULT_QUEUE) {
282
423
  const client = await this.#client();
283
- return client.llen(this.#pendingKey());
424
+ const waiting = await client.llen(this.#pendingKey(queue));
425
+ const delayed = client.zcard
426
+ ? await client.zcard(this.#delayedKey(queue))
427
+ : 0;
428
+ return waiting + delayed;
429
+ }
430
+ /**
431
+ * Move `queue`'s due jobs out of the delayed set and onto its list.
432
+ *
433
+ * `ZREM` is the claim: two workers can read the same due entry, and only
434
+ * the one whose removal returns 1 owns it. Without that the job is pushed
435
+ * onto the list once per worker that saw it.
436
+ */
437
+ async #promoteDue(client, queue) {
438
+ if (!client.zrangebyscore || !client.zrem)
439
+ return;
440
+ const due = await client.zrangebyscore(this.#delayedKey(queue), 0, Date.now(), "LIMIT", "0", "100");
441
+ for (const raw of due) {
442
+ const claimed = await client.zrem(this.#delayedKey(queue), raw);
443
+ if (claimed !== 1)
444
+ continue;
445
+ let parsed;
446
+ try {
447
+ parsed = JSON.parse(raw);
448
+ }
449
+ catch {
450
+ // Already removed from the set; there is nothing runnable to push.
451
+ continue;
452
+ }
453
+ if (!isValidJob(parsed))
454
+ continue;
455
+ parsed.runAt = undefined;
456
+ await client.rpush(this.#pendingKey(queue), JSON.stringify(parsed));
457
+ }
458
+ }
459
+ /** Take the head of one queue, claiming it in `processing`. */
460
+ async #take(client, queue) {
461
+ if (client.lmove) {
462
+ return client.lmove(this.#pendingKey(queue), this.#processingKey(), "LEFT", "RIGHT");
463
+ }
464
+ const raw = await client.lpop(this.#pendingKey(queue));
465
+ if (raw)
466
+ await client.rpush(this.#processingKey(), raw);
467
+ return raw;
284
468
  }
285
469
  }
286
470
  //# sourceMappingURL=RedisDriver.js.map