@c9up/bay 0.1.13 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -16
- package/dist/BayProvider.d.ts +39 -11
- package/dist/BayProvider.d.ts.map +1 -1
- package/dist/BayProvider.js +35 -13
- package/dist/BayProvider.js.map +1 -1
- package/dist/Job.d.ts +99 -0
- package/dist/Job.d.ts.map +1 -0
- package/dist/Job.js +78 -0
- package/dist/Job.js.map +1 -0
- package/dist/QueueManager.d.ts +140 -21
- package/dist/QueueManager.d.ts.map +1 -1
- package/dist/QueueManager.js +247 -53
- package/dist/QueueManager.js.map +1 -1
- package/dist/adapters.d.ts +68 -0
- package/dist/adapters.d.ts.map +1 -0
- package/dist/adapters.js +56 -0
- package/dist/adapters.js.map +1 -0
- package/dist/augmentations.d.ts +28 -0
- package/dist/augmentations.d.ts.map +1 -0
- package/dist/augmentations.js +17 -0
- package/dist/augmentations.js.map +1 -0
- package/dist/configure.d.ts +1 -0
- package/dist/configure.d.ts.map +1 -1
- package/dist/configure.js +24 -7
- package/dist/configure.js.map +1 -1
- package/dist/console/contract.d.ts +60 -0
- package/dist/console/contract.d.ts.map +1 -0
- package/dist/console/contract.js +36 -0
- package/dist/console/contract.js.map +1 -0
- package/dist/console/index.d.ts +29 -0
- package/dist/console/index.d.ts.map +1 -0
- package/dist/console/index.js +45 -0
- package/dist/console/index.js.map +1 -0
- package/dist/console/makeJob.d.ts +32 -0
- package/dist/console/makeJob.d.ts.map +1 -0
- package/dist/console/makeJob.js +118 -0
- package/dist/console/makeJob.js.map +1 -0
- package/dist/console/queueWork.d.ts +18 -0
- package/dist/console/queueWork.d.ts.map +1 -0
- package/dist/console/queueWork.js +58 -0
- package/dist/console/queueWork.js.map +1 -0
- package/dist/drivers/MemoryDriver.d.ts +14 -8
- package/dist/drivers/MemoryDriver.d.ts.map +1 -1
- package/dist/drivers/MemoryDriver.js +61 -7
- package/dist/drivers/MemoryDriver.js.map +1 -1
- package/dist/drivers/RedisDriver.d.ts +60 -8
- package/dist/drivers/RedisDriver.d.ts.map +1 -1
- package/dist/drivers/RedisDriver.js +209 -29
- package/dist/drivers/RedisDriver.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -4
- package/dist/index.js.map +1 -1
- package/dist/jobs.d.ts +43 -0
- package/dist/jobs.d.ts.map +1 -0
- package/dist/jobs.js +105 -0
- package/dist/jobs.js.map +1 -0
- package/dist/quasar.d.ts +1 -1
- package/dist/quasar.js +1 -1
- package/dist/testing/FakeQueue.d.ts +15 -9
- package/dist/testing/FakeQueue.d.ts.map +1 -1
- package/dist/testing/FakeQueue.js +13 -3
- package/dist/testing/FakeQueue.js.map +1 -1
- package/package.json +5 -3
- package/src/BayProvider.ts +79 -26
- package/src/Job.ts +137 -0
- package/src/QueueManager.ts +411 -56
- package/src/adapters.ts +75 -0
- package/src/augmentations.ts +31 -0
- package/src/configure.ts +25 -7
- package/src/console/contract.ts +94 -0
- package/src/console/index.ts +68 -0
- package/src/console/makeJob.ts +139 -0
- package/src/console/queueWork.ts +70 -0
- package/src/drivers/MemoryDriver.ts +66 -14
- package/src/drivers/RedisDriver.ts +298 -42
- package/src/index.ts +35 -6
- package/src/jobs.ts +111 -0
- package/src/quasar.ts +1 -1
- package/src/testing/FakeQueue.ts +25 -15
- package/dist/stores.d.ts +0 -41
- package/dist/stores.d.ts.map +0 -1
- package/dist/stores.js +0 -46
- package/dist/stores.js.map +0 -1
- package/src/stores.ts +0 -59
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `queue:work` — run the worker that takes jobs off the queue.
|
|
3
|
+
*
|
|
4
|
+
* The command a deployment runs as its own process, beside the HTTP one:
|
|
5
|
+
*
|
|
6
|
+
* ream queue:work
|
|
7
|
+
* ream queue:work --queue=emails,notifications
|
|
8
|
+
* ream queue:work --concurrency=10
|
|
9
|
+
*
|
|
10
|
+
* It stays alive on purpose — `staysAlive` — and the loop it starts is the
|
|
11
|
+
* application's, so a SIGTERM reaches `BayProvider.shutdown()`, which stops the
|
|
12
|
+
* worker and lets the job in flight finish rather than dropping it.
|
|
13
|
+
*/
|
|
14
|
+
import { getQueue } from "../services/main.js";
|
|
15
|
+
import { flag } from "./contract.js";
|
|
16
|
+
/** Split `--queue=a,b` into names, dropping the empties a trailing comma makes. */
|
|
17
|
+
export function parseQueues(value) {
|
|
18
|
+
if (value === undefined)
|
|
19
|
+
return undefined;
|
|
20
|
+
const names = value
|
|
21
|
+
.split(",")
|
|
22
|
+
.map((name) => name.trim())
|
|
23
|
+
.filter((name) => name.length > 0);
|
|
24
|
+
return names.length > 0 ? names : undefined;
|
|
25
|
+
}
|
|
26
|
+
export function queueWorkCommand() {
|
|
27
|
+
return class QueueWork {
|
|
28
|
+
static commandName = "queue:work";
|
|
29
|
+
static description = "Process queued jobs until the process is stopped";
|
|
30
|
+
// The worker needs the container: the queue it drains is the one the
|
|
31
|
+
// provider booted, with the driver the config named.
|
|
32
|
+
static options = { startApp: true, staysAlive: true };
|
|
33
|
+
static flags = [
|
|
34
|
+
flag("queue", "string", {
|
|
35
|
+
description: "Comma-separated queues to serve, in order (default: the default queue)",
|
|
36
|
+
}),
|
|
37
|
+
flag("concurrency", "number", {
|
|
38
|
+
description: "How many jobs to run at once (default: 1)",
|
|
39
|
+
}),
|
|
40
|
+
];
|
|
41
|
+
async run() {
|
|
42
|
+
const manager = getQueue();
|
|
43
|
+
if (!manager) {
|
|
44
|
+
// Naming the wiring beats a stack trace out of the service proxy:
|
|
45
|
+
// the usual cause is a project that never added the provider.
|
|
46
|
+
throw new Error("[bay] queue:work found no queue. Add `() => import('@c9up/bay/provider')` " +
|
|
47
|
+
"to the providers in reamrc.ts, or call setQueue(myQueue) at boot.");
|
|
48
|
+
}
|
|
49
|
+
const queues = parseQueues(this.queue);
|
|
50
|
+
process.stdout.write(`[bay] worker started — queues: ${(queues ?? ["default"]).join(", ")}, concurrency: ${this.concurrency ?? 1}\n`);
|
|
51
|
+
await manager.work({
|
|
52
|
+
queues,
|
|
53
|
+
concurrency: this.concurrency,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=queueWork.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queueWork.js","sourceRoot":"","sources":["../../src/console/queueWork.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAwB,IAAI,EAAE,MAAM,eAAe,CAAC;AAE3D,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,KAAyB;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK;SACjB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO,MAAM,SAAS;QACrB,MAAM,CAAC,WAAW,GAAG,YAAY,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,kDAAkD,CAAC;QACxE,qEAAqE;QACrE,qDAAqD;QACrD,MAAM,CAAC,OAAO,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACtD,MAAM,CAAC,KAAK,GAAG;YACd,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE;gBACvB,WAAW,EACV,wEAAwE;aACzE,CAAC;YACF,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE;gBAC7B,WAAW,EAAE,2CAA2C;aACxD,CAAC;SACF,CAAC;QAKF,KAAK,CAAC,GAAG;YACR,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,kEAAkE;gBAClE,8DAA8D;gBAC9D,MAAM,IAAI,KAAK,CACd,4EAA4E;oBAC3E,mEAAmE,CACpE,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,kCAAkC,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAC/G,CAAC;YAEF,MAAM,OAAO,CAAC,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC,CAAC;QACJ,CAAC;KACD,CAAC;AACH,CAAC"}
|
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Memory queue driver — in-process queue for development.
|
|
2
|
+
* Memory queue driver — in-process queue for development and tests.
|
|
3
3
|
*/
|
|
4
|
-
import type
|
|
4
|
+
import { type JobRecord, type QueueDriver } from "../QueueManager.js";
|
|
5
5
|
export declare class MemoryDriver implements QueueDriver {
|
|
6
6
|
#private;
|
|
7
7
|
constructor(options?: {
|
|
8
8
|
maxFailedJobs?: number;
|
|
9
9
|
});
|
|
10
|
-
push(job:
|
|
11
|
-
pop(): Promise<
|
|
12
|
-
fail(job:
|
|
13
|
-
complete(_job:
|
|
14
|
-
retry(job:
|
|
15
|
-
failed(): Promise<
|
|
10
|
+
push(job: JobRecord): Promise<void>;
|
|
11
|
+
pop(queues?: readonly string[]): Promise<JobRecord | null>;
|
|
12
|
+
fail(job: JobRecord, error: string): Promise<void>;
|
|
13
|
+
complete(_job: JobRecord): Promise<void>;
|
|
14
|
+
retry(job: JobRecord): Promise<void>;
|
|
15
|
+
failed(): Promise<JobRecord[]>;
|
|
16
|
+
/**
|
|
17
|
+
* Everything waiting, across every queue — a delayed job included.
|
|
18
|
+
*
|
|
19
|
+
* It is queued; it is simply not due. Leaving it out would report an empty
|
|
20
|
+
* queue to anything draining one before shutdown.
|
|
21
|
+
*/
|
|
16
22
|
size(): Promise<number>;
|
|
17
23
|
}
|
|
18
24
|
//# sourceMappingURL=MemoryDriver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MemoryDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"MemoryDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAW,MAAM,oBAAoB,CAAC;AAE/E,qBAAa,YAAa,YAAW,WAAW;;gBAcnC,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE;IAwB1C,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IASnC,GAAG,CACR,MAAM,GAAE,SAAS,MAAM,EAAoB,GACzC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAWtB,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxC,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpC,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAIpC;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;CAK7B"}
|
|
@@ -1,18 +1,63 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Memory queue driver — in-process queue for development.
|
|
2
|
+
* Memory queue driver — in-process queue for development and tests.
|
|
3
3
|
*/
|
|
4
|
+
import { DEFAULT_QUEUE } from "../Job.js";
|
|
5
|
+
import { queueOf } from "../QueueManager.js";
|
|
4
6
|
export class MemoryDriver {
|
|
5
|
-
|
|
7
|
+
/** One list per named queue, created on first use. */
|
|
8
|
+
#pending = new Map();
|
|
9
|
+
/**
|
|
10
|
+
* Jobs whose `runAt` has not arrived, oldest deadline first.
|
|
11
|
+
*
|
|
12
|
+
* Kept apart from the queues rather than filtered on the way out: a delayed
|
|
13
|
+
* job at the head of a list would otherwise be skipped over on every poll,
|
|
14
|
+
* and a queue whose head is not due would look empty while it is not.
|
|
15
|
+
*/
|
|
16
|
+
#delayed = [];
|
|
6
17
|
#failedJobs = [];
|
|
7
18
|
#maxFailedJobs;
|
|
8
19
|
constructor(options) {
|
|
9
20
|
this.#maxFailedJobs = options?.maxFailedJobs ?? 1000;
|
|
10
21
|
}
|
|
22
|
+
#queue(name) {
|
|
23
|
+
const existing = this.#pending.get(name);
|
|
24
|
+
if (existing !== undefined)
|
|
25
|
+
return existing;
|
|
26
|
+
const created = [];
|
|
27
|
+
this.#pending.set(name, created);
|
|
28
|
+
return created;
|
|
29
|
+
}
|
|
30
|
+
/** Move everything whose delay has elapsed into its queue. */
|
|
31
|
+
#promoteDue(now = Date.now()) {
|
|
32
|
+
if (this.#delayed.length === 0)
|
|
33
|
+
return;
|
|
34
|
+
const due = this.#delayed.filter((job) => (job.runAt ?? 0) <= now);
|
|
35
|
+
if (due.length === 0)
|
|
36
|
+
return;
|
|
37
|
+
this.#delayed = this.#delayed.filter((job) => (job.runAt ?? 0) > now);
|
|
38
|
+
for (const job of due) {
|
|
39
|
+
job.runAt = undefined;
|
|
40
|
+
this.#queue(queueOf(job)).push(job);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
11
43
|
async push(job) {
|
|
12
|
-
|
|
44
|
+
if (job.runAt !== undefined && job.runAt > Date.now()) {
|
|
45
|
+
this.#delayed.push(job);
|
|
46
|
+
this.#delayed.sort((a, b) => (a.runAt ?? 0) - (b.runAt ?? 0));
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
this.#queue(queueOf(job)).push(job);
|
|
13
50
|
}
|
|
14
|
-
async pop() {
|
|
15
|
-
|
|
51
|
+
async pop(queues = [DEFAULT_QUEUE]) {
|
|
52
|
+
this.#promoteDue();
|
|
53
|
+
// In the order given: naming `['critical', 'default']` is how a worker
|
|
54
|
+
// says which queue it would rather drain first.
|
|
55
|
+
for (const name of queues) {
|
|
56
|
+
const next = this.#pending.get(name)?.shift();
|
|
57
|
+
if (next !== undefined)
|
|
58
|
+
return next;
|
|
59
|
+
}
|
|
60
|
+
return null;
|
|
16
61
|
}
|
|
17
62
|
async fail(job, error) {
|
|
18
63
|
job.error = error;
|
|
@@ -27,13 +72,22 @@ export class MemoryDriver {
|
|
|
27
72
|
}
|
|
28
73
|
async retry(job) {
|
|
29
74
|
job.status = "pending";
|
|
30
|
-
this.#
|
|
75
|
+
this.#queue(queueOf(job)).push(job);
|
|
31
76
|
}
|
|
32
77
|
async failed() {
|
|
33
78
|
return [...this.#failedJobs];
|
|
34
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Everything waiting, across every queue — a delayed job included.
|
|
82
|
+
*
|
|
83
|
+
* It is queued; it is simply not due. Leaving it out would report an empty
|
|
84
|
+
* queue to anything draining one before shutdown.
|
|
85
|
+
*/
|
|
35
86
|
async size() {
|
|
36
|
-
|
|
87
|
+
let total = this.#delayed.length;
|
|
88
|
+
for (const jobs of this.#pending.values())
|
|
89
|
+
total += jobs.length;
|
|
90
|
+
return total;
|
|
37
91
|
}
|
|
38
92
|
}
|
|
39
93
|
//# sourceMappingURL=MemoryDriver.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MemoryDriver.js","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"MemoryDriver.js","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAoC,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE/E,MAAM,OAAO,YAAY;IACxB,sDAAsD;IACtD,QAAQ,GAA6B,IAAI,GAAG,EAAE,CAAC;IAC/C;;;;;;OAMG;IACH,QAAQ,GAAgB,EAAE,CAAC;IAC3B,WAAW,GAAgB,EAAE,CAAC;IAC9B,cAAc,CAAS;IAEvB,YAAY,OAAoC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,aAAa,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,QAAQ,CAAC;QAC5C,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjC,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,8DAA8D;IAC9D,WAAW,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;QACnE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YACvB,GAAG,CAAC,KAAK,GAAG,SAAS,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAc;QACxB,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9D,OAAO;QACR,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CACR,SAA4B,CAAC,aAAa,CAAC;QAE3C,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,uEAAuE;QACvE,gDAAgD;QAChD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC;YAC9C,IAAI,IAAI,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAc,EAAE,KAAa;QACvC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;QAClB,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3E,CAAC;IACF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAe;QAC7B,kCAAkC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAc;QACzB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM;QACX,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI;QACT,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;QAChE,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}
|
|
@@ -17,17 +17,32 @@
|
|
|
17
17
|
* { PX: ms }) does NOT satisfy this interface and would drop the lease TTL — it
|
|
18
18
|
* needs a thin adapter.
|
|
19
19
|
*/
|
|
20
|
-
import type
|
|
20
|
+
import { type JobRecord, type QueueDriver } from "../QueueManager.js";
|
|
21
21
|
export interface RedisClient {
|
|
22
22
|
rpush(key: string, ...values: string[]): Promise<number>;
|
|
23
23
|
lpop(key: string): Promise<string | null>;
|
|
24
24
|
lmove?(source: string, destination: string, from: "LEFT" | "RIGHT", to: "LEFT" | "RIGHT"): Promise<string | null>;
|
|
25
25
|
lrem(key: string, count: number, element: string): Promise<number>;
|
|
26
|
+
/**
|
|
27
|
+
* Optional, like `lmove`. Present on ioredis; without it the failed list
|
|
28
|
+
* simply keeps its entries, which is what this driver did before.
|
|
29
|
+
*/
|
|
30
|
+
ltrim?(key: string, start: number, stop: number): Promise<string>;
|
|
26
31
|
llen(key: string): Promise<number>;
|
|
27
32
|
lrange(key: string, start: number, stop: number): Promise<string[]>;
|
|
28
33
|
del(key: string): Promise<number>;
|
|
29
34
|
set(key: string, value: string, ...args: string[]): Promise<string | null>;
|
|
30
35
|
get(key: string): Promise<string | null>;
|
|
36
|
+
/**
|
|
37
|
+
* Sorted-set commands, for delayed jobs. Optional like `lmove`: a client
|
|
38
|
+
* without them can still run a queue, and `push` refuses a job carrying a
|
|
39
|
+
* `delay` rather than running it early — which is the one thing a delay
|
|
40
|
+
* must not do.
|
|
41
|
+
*/
|
|
42
|
+
zadd?(key: string, score: number, member: string): Promise<number | string>;
|
|
43
|
+
zrangebyscore?(key: string, min: number | string, max: number | string, ...args: string[]): Promise<string[]>;
|
|
44
|
+
zrem?(key: string, ...members: string[]): Promise<number>;
|
|
45
|
+
zcard?(key: string): Promise<number>;
|
|
31
46
|
}
|
|
32
47
|
/**
|
|
33
48
|
* Where the client comes from. A resolver is what lets a queue name its
|
|
@@ -47,14 +62,51 @@ export declare class RedisDriver implements QueueDriver {
|
|
|
47
62
|
* deployment makes, not one a version check makes for it.
|
|
48
63
|
*/
|
|
49
64
|
allowNonAtomicPop?: boolean;
|
|
65
|
+
/**
|
|
66
|
+
* How many times a job may be reclaimed from a stalled worker before
|
|
67
|
+
* it is filed as failed instead of pushed round again. Default `1`,
|
|
68
|
+
* upstream's default for the same setting.
|
|
69
|
+
*
|
|
70
|
+
* Unbounded recovery is a job that kills its worker taking the whole
|
|
71
|
+
* queue down with it, forever: the crash never reaches the failure
|
|
72
|
+
* path, so `attempts` never moves and `maxAttempts` never applies.
|
|
73
|
+
*/
|
|
74
|
+
maxStalledCount?: number;
|
|
75
|
+
/**
|
|
76
|
+
* How many failed jobs to keep. Default `1000` — the ceiling the
|
|
77
|
+
* memory driver already had. `0` keeps every one of them.
|
|
78
|
+
*
|
|
79
|
+
* Only enforced when the client answers `ltrim`.
|
|
80
|
+
*/
|
|
81
|
+
maxFailedJobs?: number;
|
|
50
82
|
});
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Renew a lease at half its length: two chances to be heard before the
|
|
85
|
+
* deadline, so one slow round-trip does not hand a running job to somebody
|
|
86
|
+
* else. Read by `QueueManager` while a handler runs.
|
|
87
|
+
*/
|
|
88
|
+
get renewIntervalMs(): number;
|
|
89
|
+
push(job: JobRecord): Promise<void>;
|
|
90
|
+
pop(queues?: readonly string[]): Promise<JobRecord | null>;
|
|
91
|
+
/**
|
|
92
|
+
* Say the job is still being worked on, and push its deadline back.
|
|
93
|
+
*
|
|
94
|
+
* Answers `false` when there is nothing left to renew — the lease expired
|
|
95
|
+
* and the job was recovered, or it was recovered and re-popped by another
|
|
96
|
+
* worker, whose claim this one must not extend.
|
|
97
|
+
*/
|
|
98
|
+
renew(job: JobRecord): Promise<boolean>;
|
|
99
|
+
complete(job: JobRecord): Promise<void>;
|
|
100
|
+
fail(job: JobRecord, error: string): Promise<void>;
|
|
101
|
+
retry(job: JobRecord): Promise<void>;
|
|
56
102
|
recoverStale(): Promise<number>;
|
|
57
|
-
failed(): Promise<
|
|
58
|
-
|
|
103
|
+
failed(): Promise<JobRecord[]>;
|
|
104
|
+
/**
|
|
105
|
+
* How many jobs are waiting on `queue` — its list plus its delayed set.
|
|
106
|
+
*
|
|
107
|
+
* A delayed job is queued; it is simply not due. Counting only the list
|
|
108
|
+
* reported an empty queue to anything draining one before shutdown.
|
|
109
|
+
*/
|
|
110
|
+
size(queue?: string): Promise<number>;
|
|
59
111
|
}
|
|
60
112
|
//# sourceMappingURL=RedisDriver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RedisDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/RedisDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;
|
|
1
|
+
{"version":3,"file":"RedisDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/RedisDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAW,MAAM,oBAAoB,CAAC;AAE/E,MAAM,WAAW,WAAW;IAC3B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1C,KAAK,CAAC,CACL,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,GAAG,OAAO,EACtB,EAAE,EAAE,MAAM,GAAG,OAAO,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE;;;OAGG;IACH,KAAK,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC3E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACzC;;;;;OAKG;IACH,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAC5E,aAAa,CAAC,CACb,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,GAAG,IAAI,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrB,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1D,KAAK,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrC;AAiDD;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAC1B,WAAW,GACX,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;AAmD9C,qBAAa,WAAY,YAAW,WAAW;;gBAsC7C,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B;;;;WAIG;QACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B;;;;;;;;WAQG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;;;;;WAKG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB;IA4CF;;;;OAIG;IACH,IAAI,eAAe,IAAI,MAAM,CAE5B;IA2BK,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBnC,GAAG,CACR,MAAM,GAAE,SAAS,MAAM,EAAoB,GACzC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IA6C5B;;;;;;OAMG;IACG,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAavC,QAAQ,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAQpC,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IA6G/B,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAepC;;;;;OAKG;IACG,IAAI,CAAC,KAAK,GAAE,MAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;CAwD1D"}
|
|
@@ -17,16 +17,39 @@
|
|
|
17
17
|
* { PX: ms }) does NOT satisfy this interface and would drop the lease TTL — it
|
|
18
18
|
* needs a thin adapter.
|
|
19
19
|
*/
|
|
20
|
+
import { DEFAULT_QUEUE } from "../Job.js";
|
|
20
21
|
import { inProduction } from "../nodeEnv.js";
|
|
22
|
+
import { queueOf } from "../QueueManager.js";
|
|
21
23
|
function isValidJob(obj) {
|
|
22
24
|
if (typeof obj !== "object" || obj === null)
|
|
23
25
|
return false;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
typeof
|
|
27
|
-
typeof
|
|
28
|
-
typeof
|
|
29
|
-
|
|
26
|
+
return (typeof Reflect.get(obj, "id") === "string" &&
|
|
27
|
+
typeof Reflect.get(obj, "name") === "string" &&
|
|
28
|
+
typeof Reflect.get(obj, "attempts") === "number" &&
|
|
29
|
+
typeof Reflect.get(obj, "maxAttempts") === "number" &&
|
|
30
|
+
typeof Reflect.get(obj, "status") === "string");
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Read a lease back. A value written by an older version of this driver is the
|
|
34
|
+
* raw job string on its own, with no owner — still usable for the one thing
|
|
35
|
+
* `#removeFromProcessing` needs it for.
|
|
36
|
+
*/
|
|
37
|
+
function readLease(stored) {
|
|
38
|
+
let parsed;
|
|
39
|
+
try {
|
|
40
|
+
parsed = JSON.parse(stored);
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return { raw: stored };
|
|
44
|
+
}
|
|
45
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
46
|
+
return { raw: stored };
|
|
47
|
+
const raw = Reflect.get(parsed, "raw");
|
|
48
|
+
const owner = Reflect.get(parsed, "owner");
|
|
49
|
+
if (typeof raw !== "string" || typeof owner !== "string") {
|
|
50
|
+
return { raw: stored };
|
|
51
|
+
}
|
|
52
|
+
return { owner, raw };
|
|
30
53
|
}
|
|
31
54
|
/**
|
|
32
55
|
* LMOVE (Redis 6.2+) is what makes pop() atomic. Warned once, when the client
|
|
@@ -131,26 +154,71 @@ export class RedisDriver {
|
|
|
131
154
|
throw new Error(`[bay] RedisDriver visibilityTimeoutMs must be a positive integer (ms), got ${visibilityTimeout}`);
|
|
132
155
|
}
|
|
133
156
|
this.#visibilityTimeout = visibilityTimeout;
|
|
157
|
+
const maxStalled = options?.maxStalledCount ?? 1;
|
|
158
|
+
if (!Number.isInteger(maxStalled) || maxStalled < 0) {
|
|
159
|
+
throw new Error(`[bay] RedisDriver maxStalledCount must be a non-negative integer, got ${maxStalled}`);
|
|
160
|
+
}
|
|
161
|
+
this.#maxStalledCount = maxStalled;
|
|
162
|
+
const maxFailed = options?.maxFailedJobs ?? 1000;
|
|
163
|
+
if (!Number.isInteger(maxFailed) || maxFailed < 0) {
|
|
164
|
+
throw new Error(`[bay] RedisDriver maxFailedJobs must be a non-negative integer, got ${maxFailed}`);
|
|
165
|
+
}
|
|
166
|
+
this.#maxFailedJobs = maxFailed;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Renew a lease at half its length: two chances to be heard before the
|
|
170
|
+
* deadline, so one slow round-trip does not hand a running job to somebody
|
|
171
|
+
* else. Read by `QueueManager` while a handler runs.
|
|
172
|
+
*/
|
|
173
|
+
get renewIntervalMs() {
|
|
174
|
+
return Math.max(1, Math.floor(this.#visibilityTimeout / 2));
|
|
134
175
|
}
|
|
135
|
-
|
|
176
|
+
/**
|
|
177
|
+
* Where one queue's jobs wait.
|
|
178
|
+
*
|
|
179
|
+
* The default queue keeps the key it always had. Naming it
|
|
180
|
+
* `queue:default:pending` would have been tidier and would have orphaned
|
|
181
|
+
* every job already sitting in `queue:pending` at the moment of the upgrade
|
|
182
|
+
* — a silent loss, since nothing reads the old key afterwards.
|
|
183
|
+
*/
|
|
184
|
+
#pendingKey = (queue = DEFAULT_QUEUE) => queue === DEFAULT_QUEUE
|
|
185
|
+
? `${this.#prefix}pending`
|
|
186
|
+
: `${this.#prefix}q:${queue}:pending`;
|
|
187
|
+
#delayedKey = (queue = DEFAULT_QUEUE) => queue === DEFAULT_QUEUE
|
|
188
|
+
? `${this.#prefix}delayed`
|
|
189
|
+
: `${this.#prefix}q:${queue}:delayed`;
|
|
136
190
|
#processingKey = () => `${this.#prefix}processing`;
|
|
137
191
|
#allowNonAtomicPop = false;
|
|
192
|
+
#maxStalledCount = 1;
|
|
193
|
+
#maxFailedJobs = 1000;
|
|
194
|
+
/** This driver instance, as a lease owner. */
|
|
195
|
+
#workerId = crypto.randomUUID();
|
|
138
196
|
#failedKey = () => `${this.#prefix}failed`;
|
|
139
197
|
#leaseKey = (jobId) => `${this.#prefix}lease:${jobId}`;
|
|
140
198
|
async push(job) {
|
|
141
199
|
const client = await this.#client();
|
|
142
|
-
|
|
200
|
+
const queue = queueOf(job);
|
|
201
|
+
if (job.runAt !== undefined && job.runAt > Date.now()) {
|
|
202
|
+
if (!client.zadd) {
|
|
203
|
+
// Pushing it to the list instead would run it now, which is the
|
|
204
|
+
// one thing a delay exists to prevent.
|
|
205
|
+
throw new Error("This Redis client cannot hold a delayed job: it has no ZADD. " +
|
|
206
|
+
"Use a client with sorted-set commands (ioredis has them), or dispatch without `delay`.");
|
|
207
|
+
}
|
|
208
|
+
await client.zadd(this.#delayedKey(queue), job.runAt, JSON.stringify(job));
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
await client.rpush(this.#pendingKey(queue), JSON.stringify(job));
|
|
143
212
|
}
|
|
144
|
-
async pop() {
|
|
213
|
+
async pop(queues = [DEFAULT_QUEUE]) {
|
|
145
214
|
const client = await this.#client();
|
|
146
215
|
let raw = null;
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
raw
|
|
152
|
-
|
|
153
|
-
await client.rpush(this.#processingKey(), raw);
|
|
216
|
+
// In the order given, so a worker can say which queue it drains first.
|
|
217
|
+
for (const queue of queues) {
|
|
218
|
+
await this.#promoteDue(client, queue);
|
|
219
|
+
raw = await this.#take(client, queue);
|
|
220
|
+
if (raw !== null)
|
|
221
|
+
break;
|
|
154
222
|
}
|
|
155
223
|
if (!raw)
|
|
156
224
|
return null;
|
|
@@ -177,9 +245,29 @@ export class RedisDriver {
|
|
|
177
245
|
// bad job. The error propagates and the job STAYS in processing with
|
|
178
246
|
// no lease, which is precisely the state recoverStale() puts back in
|
|
179
247
|
// pending — so the delivery guarantee survives the blip.
|
|
180
|
-
await client.set(this.#leaseKey(parsed.id), raw, "PX", String(this.#visibilityTimeout));
|
|
248
|
+
await client.set(this.#leaseKey(parsed.id), JSON.stringify({ owner: this.#workerId, raw }), "PX", String(this.#visibilityTimeout));
|
|
181
249
|
return parsed;
|
|
182
250
|
}
|
|
251
|
+
/**
|
|
252
|
+
* Say the job is still being worked on, and push its deadline back.
|
|
253
|
+
*
|
|
254
|
+
* Answers `false` when there is nothing left to renew — the lease expired
|
|
255
|
+
* and the job was recovered, or it was recovered and re-popped by another
|
|
256
|
+
* worker, whose claim this one must not extend.
|
|
257
|
+
*/
|
|
258
|
+
async renew(job) {
|
|
259
|
+
const client = await this.#client();
|
|
260
|
+
const key = this.#leaseKey(job.id);
|
|
261
|
+
const stored = await client.get(key);
|
|
262
|
+
if (stored === null)
|
|
263
|
+
return false;
|
|
264
|
+
const lease = readLease(stored);
|
|
265
|
+
if (lease.owner !== undefined && lease.owner !== this.#workerId) {
|
|
266
|
+
return false;
|
|
267
|
+
}
|
|
268
|
+
await client.set(key, stored, "PX", String(this.#visibilityTimeout));
|
|
269
|
+
return true;
|
|
270
|
+
}
|
|
183
271
|
async complete(job) {
|
|
184
272
|
const client = await this.#client();
|
|
185
273
|
await this.#removeFromProcessing(job);
|
|
@@ -191,14 +279,14 @@ export class RedisDriver {
|
|
|
191
279
|
await client.del(this.#leaseKey(job.id));
|
|
192
280
|
job.error = error;
|
|
193
281
|
job.status = "failed";
|
|
194
|
-
await
|
|
282
|
+
await this.#pushFailed(client, job);
|
|
195
283
|
}
|
|
196
284
|
async retry(job) {
|
|
197
285
|
const client = await this.#client();
|
|
198
286
|
await this.#removeFromProcessing(job);
|
|
199
287
|
await client.del(this.#leaseKey(job.id));
|
|
200
288
|
job.status = "pending";
|
|
201
|
-
await client.rpush(this.#pendingKey(), JSON.stringify(job));
|
|
289
|
+
await client.rpush(this.#pendingKey(queueOf(job)), JSON.stringify(job));
|
|
202
290
|
}
|
|
203
291
|
async recoverStale() {
|
|
204
292
|
const client = await this.#client();
|
|
@@ -220,29 +308,72 @@ export class RedisDriver {
|
|
|
220
308
|
continue;
|
|
221
309
|
}
|
|
222
310
|
const lease = await client.get(this.#leaseKey(parsed.id));
|
|
223
|
-
if (lease
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
311
|
+
if (lease !== null)
|
|
312
|
+
continue;
|
|
313
|
+
// The LREM is the claim, and its RESULT decides who acts. Two
|
|
314
|
+
// recovery passes overlapping — two workers, or one worker whose
|
|
315
|
+
// pass ran long — both read the same expired entry, and both used to
|
|
316
|
+
// push it back to pending: one job, delivered twice, from the
|
|
317
|
+
// mechanism that exists to make delivery reliable. Exactly one LREM
|
|
318
|
+
// can remove a given element, so exactly one pass continues past
|
|
319
|
+
// here. (A crash between this and the RPUSH below still loses the
|
|
320
|
+
// entry; closing that needs the whole pass in one server-side script,
|
|
321
|
+
// which is how upstream does it.)
|
|
322
|
+
const claimed = await client.lrem(this.#processingKey(), 1, raw);
|
|
323
|
+
if (claimed === 0)
|
|
324
|
+
continue;
|
|
325
|
+
// A stall is not an attempt: the worker died before the handler
|
|
326
|
+
// could fail, so `attempts` never moved and `maxAttempts` never
|
|
327
|
+
// applied. A job that kills whatever picks it up was therefore
|
|
328
|
+
// recovered forever, taking the queue with it. Counted separately,
|
|
329
|
+
// and bounded — upstream bounds the same thing with the same
|
|
330
|
+
// default, failing the job once it is exceeded.
|
|
331
|
+
const stalled = (parsed.stalledCount ?? 0) + 1;
|
|
332
|
+
if (stalled > this.#maxStalledCount) {
|
|
333
|
+
parsed.stalledCount = stalled;
|
|
334
|
+
parsed.status = "failed";
|
|
335
|
+
parsed.error = `Stalled ${stalled} time(s) without completing (maxStalledCount ${this.#maxStalledCount})`;
|
|
336
|
+
await this.#pushFailed(client, parsed);
|
|
337
|
+
continue;
|
|
228
338
|
}
|
|
339
|
+
parsed.stalledCount = stalled;
|
|
340
|
+
parsed.status = "pending";
|
|
341
|
+
// Back to the queue it came from, not to the default one: a recovered
|
|
342
|
+
// job whose queue nobody serves would never run again.
|
|
343
|
+
await client.rpush(this.#pendingKey(queueOf(parsed)), JSON.stringify(parsed));
|
|
344
|
+
recovered++;
|
|
229
345
|
}
|
|
230
346
|
return recovered;
|
|
231
347
|
}
|
|
348
|
+
/**
|
|
349
|
+
* File a job as failed, keeping the list to `maxFailedJobs`.
|
|
350
|
+
*
|
|
351
|
+
* Unbounded, the failed list is a leak with no ceiling and no owner: nothing
|
|
352
|
+
* trims it, and `failed()` reads all of it in one LRANGE. The memory driver
|
|
353
|
+
* has capped its own at a thousand from the start; this is the same cap on
|
|
354
|
+
* the driver where the list actually survives a restart.
|
|
355
|
+
*/
|
|
356
|
+
async #pushFailed(client, job) {
|
|
357
|
+
await client.rpush(this.#failedKey(), JSON.stringify(job));
|
|
358
|
+
if (this.#maxFailedJobs === 0 || !client.ltrim)
|
|
359
|
+
return;
|
|
360
|
+
await client.ltrim(this.#failedKey(), -this.#maxFailedJobs, -1);
|
|
361
|
+
}
|
|
232
362
|
/**
|
|
233
363
|
* Remove the entry for `job` from the processing list. The string in
|
|
234
364
|
* Redis is whatever pop() pushed, but QueueManager mutates `job` after
|
|
235
365
|
* pop returns (attempts++, status="processing", processedAt, then
|
|
236
366
|
* completed/failed/pending). LREM-ing on `JSON.stringify(job)` would
|
|
237
|
-
* therefore miss every real-world entry. Use the lease —
|
|
238
|
-
* exact raw string
|
|
367
|
+
* therefore miss every real-world entry. Use the lease — which carries
|
|
368
|
+
* the exact raw string pop() moved — and fall back to a list scan when
|
|
239
369
|
* the lease has expired (e.g. recoverStale already handled it).
|
|
240
370
|
*/
|
|
241
371
|
async #removeFromProcessing(job) {
|
|
242
372
|
const client = await this.#client();
|
|
243
373
|
const stored = await client.get(this.#leaseKey(job.id));
|
|
244
374
|
if (stored !== null) {
|
|
245
|
-
const
|
|
375
|
+
const { raw } = readLease(stored);
|
|
376
|
+
const removed = await client.lrem(this.#processingKey(), 1, raw);
|
|
246
377
|
if (removed > 0)
|
|
247
378
|
return;
|
|
248
379
|
}
|
|
@@ -278,9 +409,58 @@ export class RedisDriver {
|
|
|
278
409
|
})
|
|
279
410
|
.filter((j) => j !== null);
|
|
280
411
|
}
|
|
281
|
-
|
|
412
|
+
/**
|
|
413
|
+
* How many jobs are waiting on `queue` — its list plus its delayed set.
|
|
414
|
+
*
|
|
415
|
+
* A delayed job is queued; it is simply not due. Counting only the list
|
|
416
|
+
* reported an empty queue to anything draining one before shutdown.
|
|
417
|
+
*/
|
|
418
|
+
async size(queue = DEFAULT_QUEUE) {
|
|
282
419
|
const client = await this.#client();
|
|
283
|
-
|
|
420
|
+
const waiting = await client.llen(this.#pendingKey(queue));
|
|
421
|
+
const delayed = client.zcard
|
|
422
|
+
? await client.zcard(this.#delayedKey(queue))
|
|
423
|
+
: 0;
|
|
424
|
+
return waiting + delayed;
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Move `queue`'s due jobs out of the delayed set and onto its list.
|
|
428
|
+
*
|
|
429
|
+
* `ZREM` is the claim: two workers can read the same due entry, and only
|
|
430
|
+
* the one whose removal returns 1 owns it. Without that the job is pushed
|
|
431
|
+
* onto the list once per worker that saw it.
|
|
432
|
+
*/
|
|
433
|
+
async #promoteDue(client, queue) {
|
|
434
|
+
if (!client.zrangebyscore || !client.zrem)
|
|
435
|
+
return;
|
|
436
|
+
const due = await client.zrangebyscore(this.#delayedKey(queue), 0, Date.now(), "LIMIT", "0", "100");
|
|
437
|
+
for (const raw of due) {
|
|
438
|
+
const claimed = await client.zrem(this.#delayedKey(queue), raw);
|
|
439
|
+
if (claimed !== 1)
|
|
440
|
+
continue;
|
|
441
|
+
let parsed;
|
|
442
|
+
try {
|
|
443
|
+
parsed = JSON.parse(raw);
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
// Already removed from the set; there is nothing runnable to push.
|
|
447
|
+
continue;
|
|
448
|
+
}
|
|
449
|
+
if (!isValidJob(parsed))
|
|
450
|
+
continue;
|
|
451
|
+
parsed.runAt = undefined;
|
|
452
|
+
await client.rpush(this.#pendingKey(queue), JSON.stringify(parsed));
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
/** Take the head of one queue, claiming it in `processing`. */
|
|
456
|
+
async #take(client, queue) {
|
|
457
|
+
if (client.lmove) {
|
|
458
|
+
return client.lmove(this.#pendingKey(queue), this.#processingKey(), "LEFT", "RIGHT");
|
|
459
|
+
}
|
|
460
|
+
const raw = await client.lpop(this.#pendingKey(queue));
|
|
461
|
+
if (raw)
|
|
462
|
+
await client.rpush(this.#processingKey(), raw);
|
|
463
|
+
return raw;
|
|
284
464
|
}
|
|
285
465
|
}
|
|
286
466
|
//# sourceMappingURL=RedisDriver.js.map
|