@c9up/bay 0.1.13 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -16
- package/dist/BayProvider.d.ts +51 -11
- package/dist/BayProvider.d.ts.map +1 -1
- package/dist/BayProvider.js +36 -13
- package/dist/BayProvider.js.map +1 -1
- package/dist/Job.d.ts +99 -0
- package/dist/Job.d.ts.map +1 -0
- package/dist/Job.js +78 -0
- package/dist/Job.js.map +1 -0
- package/dist/QueueManager.d.ts +140 -21
- package/dist/QueueManager.d.ts.map +1 -1
- package/dist/QueueManager.js +247 -53
- package/dist/QueueManager.js.map +1 -1
- package/dist/adapters.d.ts +68 -0
- package/dist/adapters.d.ts.map +1 -0
- package/dist/adapters.js +56 -0
- package/dist/adapters.js.map +1 -0
- package/dist/augmentations.d.ts +28 -0
- package/dist/augmentations.d.ts.map +1 -0
- package/dist/augmentations.js +17 -0
- package/dist/augmentations.js.map +1 -0
- package/dist/configure.d.ts +1 -0
- package/dist/configure.d.ts.map +1 -1
- package/dist/configure.js +24 -7
- package/dist/configure.js.map +1 -1
- package/dist/console/contract.d.ts +60 -0
- package/dist/console/contract.d.ts.map +1 -0
- package/dist/console/contract.js +36 -0
- package/dist/console/contract.js.map +1 -0
- package/dist/console/index.d.ts +29 -0
- package/dist/console/index.d.ts.map +1 -0
- package/dist/console/index.js +45 -0
- package/dist/console/index.js.map +1 -0
- package/dist/console/makeJob.d.ts +32 -0
- package/dist/console/makeJob.d.ts.map +1 -0
- package/dist/console/makeJob.js +118 -0
- package/dist/console/makeJob.js.map +1 -0
- package/dist/console/queueWork.d.ts +18 -0
- package/dist/console/queueWork.d.ts.map +1 -0
- package/dist/console/queueWork.js +58 -0
- package/dist/console/queueWork.js.map +1 -0
- package/dist/drivers/MemoryDriver.d.ts +14 -8
- package/dist/drivers/MemoryDriver.d.ts.map +1 -1
- package/dist/drivers/MemoryDriver.js +61 -7
- package/dist/drivers/MemoryDriver.js.map +1 -1
- package/dist/drivers/RedisDriver.d.ts +60 -8
- package/dist/drivers/RedisDriver.d.ts.map +1 -1
- package/dist/drivers/RedisDriver.js +214 -30
- package/dist/drivers/RedisDriver.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -4
- package/dist/index.js.map +1 -1
- package/dist/jobs.d.ts +55 -0
- package/dist/jobs.d.ts.map +1 -0
- package/dist/jobs.js +135 -0
- package/dist/jobs.js.map +1 -0
- package/dist/quasar.d.ts +6 -9
- package/dist/quasar.d.ts.map +1 -1
- package/dist/quasar.js +27 -55
- package/dist/quasar.js.map +1 -1
- package/dist/testing/FakeQueue.d.ts +15 -9
- package/dist/testing/FakeQueue.d.ts.map +1 -1
- package/dist/testing/FakeQueue.js +13 -3
- package/dist/testing/FakeQueue.js.map +1 -1
- package/dist/vendor/nodeEnv.d.ts +5 -0
- package/dist/vendor/nodeEnv.d.ts.map +1 -0
- package/dist/{nodeEnv.js → vendor/nodeEnv.js} +13 -7
- package/dist/vendor/nodeEnv.js.map +1 -0
- package/dist/vendor/quasarConnection.d.ts +35 -0
- package/dist/vendor/quasarConnection.d.ts.map +1 -0
- package/dist/vendor/quasarConnection.js +62 -0
- package/dist/vendor/quasarConnection.js.map +1 -0
- package/package.json +5 -3
- package/src/BayProvider.ts +92 -26
- package/src/Job.ts +137 -0
- package/src/QueueManager.ts +411 -56
- package/src/adapters.ts +75 -0
- package/src/augmentations.ts +31 -0
- package/src/configure.ts +25 -7
- package/src/console/contract.ts +94 -0
- package/src/console/index.ts +68 -0
- package/src/console/makeJob.ts +139 -0
- package/src/console/queueWork.ts +70 -0
- package/src/drivers/MemoryDriver.ts +66 -14
- package/src/drivers/RedisDriver.ts +303 -43
- package/src/index.ts +35 -6
- package/src/jobs.ts +144 -0
- package/src/quasar.ts +28 -72
- package/src/testing/FakeQueue.ts +25 -15
- package/src/{nodeEnv.ts → vendor/nodeEnv.ts} +14 -7
- package/src/vendor/quasarConnection.ts +129 -0
- package/dist/nodeEnv.d.ts +0 -16
- package/dist/nodeEnv.d.ts.map +0 -1
- package/dist/nodeEnv.js.map +0 -1
- package/dist/stores.d.ts +0 -41
- package/dist/stores.d.ts.map +0 -1
- package/dist/stores.js +0 -46
- package/dist/stores.js.map +0 -1
- package/src/stores.ts +0 -59
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `queue:work` — run the worker that takes jobs off the queue.
|
|
3
|
+
*
|
|
4
|
+
* The command a deployment runs as its own process, beside the HTTP one:
|
|
5
|
+
*
|
|
6
|
+
* ream queue:work
|
|
7
|
+
* ream queue:work --queue=emails,notifications
|
|
8
|
+
* ream queue:work --concurrency=10
|
|
9
|
+
*
|
|
10
|
+
* It stays alive on purpose — `staysAlive` — and the loop it starts is the
|
|
11
|
+
* application's, so a SIGTERM reaches `BayProvider.shutdown()`, which stops the
|
|
12
|
+
* worker and lets the job in flight finish rather than dropping it.
|
|
13
|
+
*/
|
|
14
|
+
import { getQueue } from "../services/main.js";
|
|
15
|
+
import { flag } from "./contract.js";
|
|
16
|
+
/** Split `--queue=a,b` into names, dropping the empties a trailing comma makes. */
|
|
17
|
+
export function parseQueues(value) {
|
|
18
|
+
if (value === undefined)
|
|
19
|
+
return undefined;
|
|
20
|
+
const names = value
|
|
21
|
+
.split(",")
|
|
22
|
+
.map((name) => name.trim())
|
|
23
|
+
.filter((name) => name.length > 0);
|
|
24
|
+
return names.length > 0 ? names : undefined;
|
|
25
|
+
}
|
|
26
|
+
export function queueWorkCommand() {
|
|
27
|
+
return class QueueWork {
|
|
28
|
+
static commandName = "queue:work";
|
|
29
|
+
static description = "Process queued jobs until the process is stopped";
|
|
30
|
+
// The worker needs the container: the queue it drains is the one the
|
|
31
|
+
// provider booted, with the driver the config named.
|
|
32
|
+
static options = { startApp: true, staysAlive: true };
|
|
33
|
+
static flags = [
|
|
34
|
+
flag("queue", "string", {
|
|
35
|
+
description: "Comma-separated queues to serve, in order (default: the default queue)",
|
|
36
|
+
}),
|
|
37
|
+
flag("concurrency", "number", {
|
|
38
|
+
description: "How many jobs to run at once (default: 1)",
|
|
39
|
+
}),
|
|
40
|
+
];
|
|
41
|
+
async run() {
|
|
42
|
+
const manager = getQueue();
|
|
43
|
+
if (!manager) {
|
|
44
|
+
// Naming the wiring beats a stack trace out of the service proxy:
|
|
45
|
+
// the usual cause is a project that never added the provider.
|
|
46
|
+
throw new Error("[bay] queue:work found no queue. Add `() => import('@c9up/bay/provider')` " +
|
|
47
|
+
"to the providers in reamrc.ts, or call setQueue(myQueue) at boot.");
|
|
48
|
+
}
|
|
49
|
+
const queues = parseQueues(this.queue);
|
|
50
|
+
process.stdout.write(`[bay] worker started — queues: ${(queues ?? ["default"]).join(", ")}, concurrency: ${this.concurrency ?? 1}\n`);
|
|
51
|
+
await manager.work({
|
|
52
|
+
queues,
|
|
53
|
+
concurrency: this.concurrency,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=queueWork.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queueWork.js","sourceRoot":"","sources":["../../src/console/queueWork.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAwB,IAAI,EAAE,MAAM,eAAe,CAAC;AAE3D,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,KAAyB;IACpD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK;SACjB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO,MAAM,SAAS;QACrB,MAAM,CAAC,WAAW,GAAG,YAAY,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,kDAAkD,CAAC;QACxE,qEAAqE;QACrE,qDAAqD;QACrD,MAAM,CAAC,OAAO,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACtD,MAAM,CAAC,KAAK,GAAG;YACd,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE;gBACvB,WAAW,EACV,wEAAwE;aACzE,CAAC;YACF,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE;gBAC7B,WAAW,EAAE,2CAA2C;aACxD,CAAC;SACF,CAAC;QAKF,KAAK,CAAC,GAAG;YACR,MAAM,OAAO,GAAG,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,kEAAkE;gBAClE,8DAA8D;gBAC9D,MAAM,IAAI,KAAK,CACd,4EAA4E;oBAC3E,mEAAmE,CACpE,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,kCAAkC,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAC/G,CAAC;YAEF,MAAM,OAAO,CAAC,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW,EAAE,IAAI,CAAC,WAAW;aAC7B,CAAC,CAAC;QACJ,CAAC;KACD,CAAC;AACH,CAAC"}
|
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Memory queue driver — in-process queue for development.
|
|
2
|
+
* Memory queue driver — in-process queue for development and tests.
|
|
3
3
|
*/
|
|
4
|
-
import type
|
|
4
|
+
import { type JobRecord, type QueueDriver } from "../QueueManager.js";
|
|
5
5
|
export declare class MemoryDriver implements QueueDriver {
|
|
6
6
|
#private;
|
|
7
7
|
constructor(options?: {
|
|
8
8
|
maxFailedJobs?: number;
|
|
9
9
|
});
|
|
10
|
-
push(job:
|
|
11
|
-
pop(): Promise<
|
|
12
|
-
fail(job:
|
|
13
|
-
complete(_job:
|
|
14
|
-
retry(job:
|
|
15
|
-
failed(): Promise<
|
|
10
|
+
push(job: JobRecord): Promise<void>;
|
|
11
|
+
pop(queues?: readonly string[]): Promise<JobRecord | null>;
|
|
12
|
+
fail(job: JobRecord, error: string): Promise<void>;
|
|
13
|
+
complete(_job: JobRecord): Promise<void>;
|
|
14
|
+
retry(job: JobRecord): Promise<void>;
|
|
15
|
+
failed(): Promise<JobRecord[]>;
|
|
16
|
+
/**
|
|
17
|
+
* Everything waiting, across every queue — a delayed job included.
|
|
18
|
+
*
|
|
19
|
+
* It is queued; it is simply not due. Leaving it out would report an empty
|
|
20
|
+
* queue to anything draining one before shutdown.
|
|
21
|
+
*/
|
|
16
22
|
size(): Promise<number>;
|
|
17
23
|
}
|
|
18
24
|
//# sourceMappingURL=MemoryDriver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MemoryDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"MemoryDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAW,MAAM,oBAAoB,CAAC;AAE/E,qBAAa,YAAa,YAAW,WAAW;;gBAcnC,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE;IAwB1C,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IASnC,GAAG,CACR,MAAM,GAAE,SAAS,MAAM,EAAoB,GACzC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAWtB,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAIxC,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpC,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAIpC;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;CAK7B"}
|
|
@@ -1,18 +1,63 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Memory queue driver — in-process queue for development.
|
|
2
|
+
* Memory queue driver — in-process queue for development and tests.
|
|
3
3
|
*/
|
|
4
|
+
import { DEFAULT_QUEUE } from "../Job.js";
|
|
5
|
+
import { queueOf } from "../QueueManager.js";
|
|
4
6
|
export class MemoryDriver {
|
|
5
|
-
|
|
7
|
+
/** One list per named queue, created on first use. */
|
|
8
|
+
#pending = new Map();
|
|
9
|
+
/**
|
|
10
|
+
* Jobs whose `runAt` has not arrived, oldest deadline first.
|
|
11
|
+
*
|
|
12
|
+
* Kept apart from the queues rather than filtered on the way out: a delayed
|
|
13
|
+
* job at the head of a list would otherwise be skipped over on every poll,
|
|
14
|
+
* and a queue whose head is not due would look empty while it is not.
|
|
15
|
+
*/
|
|
16
|
+
#delayed = [];
|
|
6
17
|
#failedJobs = [];
|
|
7
18
|
#maxFailedJobs;
|
|
8
19
|
constructor(options) {
|
|
9
20
|
this.#maxFailedJobs = options?.maxFailedJobs ?? 1000;
|
|
10
21
|
}
|
|
22
|
+
#queue(name) {
|
|
23
|
+
const existing = this.#pending.get(name);
|
|
24
|
+
if (existing !== undefined)
|
|
25
|
+
return existing;
|
|
26
|
+
const created = [];
|
|
27
|
+
this.#pending.set(name, created);
|
|
28
|
+
return created;
|
|
29
|
+
}
|
|
30
|
+
/** Move everything whose delay has elapsed into its queue. */
|
|
31
|
+
#promoteDue(now = Date.now()) {
|
|
32
|
+
if (this.#delayed.length === 0)
|
|
33
|
+
return;
|
|
34
|
+
const due = this.#delayed.filter((job) => (job.runAt ?? 0) <= now);
|
|
35
|
+
if (due.length === 0)
|
|
36
|
+
return;
|
|
37
|
+
this.#delayed = this.#delayed.filter((job) => (job.runAt ?? 0) > now);
|
|
38
|
+
for (const job of due) {
|
|
39
|
+
job.runAt = undefined;
|
|
40
|
+
this.#queue(queueOf(job)).push(job);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
11
43
|
async push(job) {
|
|
12
|
-
|
|
44
|
+
if (job.runAt !== undefined && job.runAt > Date.now()) {
|
|
45
|
+
this.#delayed.push(job);
|
|
46
|
+
this.#delayed.sort((a, b) => (a.runAt ?? 0) - (b.runAt ?? 0));
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
this.#queue(queueOf(job)).push(job);
|
|
13
50
|
}
|
|
14
|
-
async pop() {
|
|
15
|
-
|
|
51
|
+
async pop(queues = [DEFAULT_QUEUE]) {
|
|
52
|
+
this.#promoteDue();
|
|
53
|
+
// In the order given: naming `['critical', 'default']` is how a worker
|
|
54
|
+
// says which queue it would rather drain first.
|
|
55
|
+
for (const name of queues) {
|
|
56
|
+
const next = this.#pending.get(name)?.shift();
|
|
57
|
+
if (next !== undefined)
|
|
58
|
+
return next;
|
|
59
|
+
}
|
|
60
|
+
return null;
|
|
16
61
|
}
|
|
17
62
|
async fail(job, error) {
|
|
18
63
|
job.error = error;
|
|
@@ -27,13 +72,22 @@ export class MemoryDriver {
|
|
|
27
72
|
}
|
|
28
73
|
async retry(job) {
|
|
29
74
|
job.status = "pending";
|
|
30
|
-
this.#
|
|
75
|
+
this.#queue(queueOf(job)).push(job);
|
|
31
76
|
}
|
|
32
77
|
async failed() {
|
|
33
78
|
return [...this.#failedJobs];
|
|
34
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Everything waiting, across every queue — a delayed job included.
|
|
82
|
+
*
|
|
83
|
+
* It is queued; it is simply not due. Leaving it out would report an empty
|
|
84
|
+
* queue to anything draining one before shutdown.
|
|
85
|
+
*/
|
|
35
86
|
async size() {
|
|
36
|
-
|
|
87
|
+
let total = this.#delayed.length;
|
|
88
|
+
for (const jobs of this.#pending.values())
|
|
89
|
+
total += jobs.length;
|
|
90
|
+
return total;
|
|
37
91
|
}
|
|
38
92
|
}
|
|
39
93
|
//# sourceMappingURL=MemoryDriver.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MemoryDriver.js","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"MemoryDriver.js","sourceRoot":"","sources":["../../src/drivers/MemoryDriver.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAoC,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAE/E,MAAM,OAAO,YAAY;IACxB,sDAAsD;IACtD,QAAQ,GAA6B,IAAI,GAAG,EAAE,CAAC;IAC/C;;;;;;OAMG;IACH,QAAQ,GAAgB,EAAE,CAAC;IAC3B,WAAW,GAAgB,EAAE,CAAC;IAC9B,cAAc,CAAS;IAEvB,YAAY,OAAoC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,aAAa,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,QAAQ,CAAC;QAC5C,MAAM,OAAO,GAAgB,EAAE,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjC,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,8DAA8D;IAC9D,WAAW,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;QACnE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YACvB,GAAG,CAAC,KAAK,GAAG,SAAS,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAc;QACxB,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9D,OAAO;QACR,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CACR,SAA4B,CAAC,aAAa,CAAC;QAE3C,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,uEAAuE;QACvE,gDAAgD;QAChD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC;YAC9C,IAAI,IAAI,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAc,EAAE,KAAa;QACvC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;QAClB,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC;QACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3E,CAAC;IACF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAe;QAC7B,kCAAkC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,GAAc;QACzB,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,MAAM;QACX,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI;QACT,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC;QAChE,OAAO,KAAK,CAAC;IACd,CAAC;CACD"}
|
|
@@ -17,17 +17,32 @@
|
|
|
17
17
|
* { PX: ms }) does NOT satisfy this interface and would drop the lease TTL — it
|
|
18
18
|
* needs a thin adapter.
|
|
19
19
|
*/
|
|
20
|
-
import type
|
|
20
|
+
import { type JobRecord, type QueueDriver } from "../QueueManager.js";
|
|
21
21
|
export interface RedisClient {
|
|
22
22
|
rpush(key: string, ...values: string[]): Promise<number>;
|
|
23
23
|
lpop(key: string): Promise<string | null>;
|
|
24
24
|
lmove?(source: string, destination: string, from: "LEFT" | "RIGHT", to: "LEFT" | "RIGHT"): Promise<string | null>;
|
|
25
25
|
lrem(key: string, count: number, element: string): Promise<number>;
|
|
26
|
+
/**
|
|
27
|
+
* Optional, like `lmove`. Present on ioredis; without it the failed list
|
|
28
|
+
* simply keeps its entries, which is what this driver did before.
|
|
29
|
+
*/
|
|
30
|
+
ltrim?(key: string, start: number, stop: number): Promise<string>;
|
|
26
31
|
llen(key: string): Promise<number>;
|
|
27
32
|
lrange(key: string, start: number, stop: number): Promise<string[]>;
|
|
28
33
|
del(key: string): Promise<number>;
|
|
29
34
|
set(key: string, value: string, ...args: string[]): Promise<string | null>;
|
|
30
35
|
get(key: string): Promise<string | null>;
|
|
36
|
+
/**
|
|
37
|
+
* Sorted-set commands, for delayed jobs. Optional like `lmove`: a client
|
|
38
|
+
* without them can still run a queue, and `push` refuses a job carrying a
|
|
39
|
+
* `delay` rather than running it early — which is the one thing a delay
|
|
40
|
+
* must not do.
|
|
41
|
+
*/
|
|
42
|
+
zadd?(key: string, score: number, member: string): Promise<number | string>;
|
|
43
|
+
zrangebyscore?(key: string, min: number | string, max: number | string, ...args: string[]): Promise<string[]>;
|
|
44
|
+
zrem?(key: string, ...members: string[]): Promise<number>;
|
|
45
|
+
zcard?(key: string): Promise<number>;
|
|
31
46
|
}
|
|
32
47
|
/**
|
|
33
48
|
* Where the client comes from. A resolver is what lets a queue name its
|
|
@@ -47,14 +62,51 @@ export declare class RedisDriver implements QueueDriver {
|
|
|
47
62
|
* deployment makes, not one a version check makes for it.
|
|
48
63
|
*/
|
|
49
64
|
allowNonAtomicPop?: boolean;
|
|
65
|
+
/**
|
|
66
|
+
* How many times a job may be reclaimed from a stalled worker before
|
|
67
|
+
* it is filed as failed instead of pushed round again. Default `1`,
|
|
68
|
+
* upstream's default for the same setting.
|
|
69
|
+
*
|
|
70
|
+
* Unbounded recovery is a job that kills its worker taking the whole
|
|
71
|
+
* queue down with it, forever: the crash never reaches the failure
|
|
72
|
+
* path, so `attempts` never moves and `maxAttempts` never applies.
|
|
73
|
+
*/
|
|
74
|
+
maxStalledCount?: number;
|
|
75
|
+
/**
|
|
76
|
+
* How many failed jobs to keep. Default `1000` — the ceiling the
|
|
77
|
+
* memory driver already had. `0` keeps every one of them.
|
|
78
|
+
*
|
|
79
|
+
* Only enforced when the client answers `ltrim`.
|
|
80
|
+
*/
|
|
81
|
+
maxFailedJobs?: number;
|
|
50
82
|
});
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
83
|
+
/**
|
|
84
|
+
* Renew a lease at half its length: two chances to be heard before the
|
|
85
|
+
* deadline, so one slow round-trip does not hand a running job to somebody
|
|
86
|
+
* else. Read by `QueueManager` while a handler runs.
|
|
87
|
+
*/
|
|
88
|
+
get renewIntervalMs(): number;
|
|
89
|
+
push(job: JobRecord): Promise<void>;
|
|
90
|
+
pop(queues?: readonly string[]): Promise<JobRecord | null>;
|
|
91
|
+
/**
|
|
92
|
+
* Say the job is still being worked on, and push its deadline back.
|
|
93
|
+
*
|
|
94
|
+
* Answers `false` when there is nothing left to renew — the lease expired
|
|
95
|
+
* and the job was recovered, or it was recovered and re-popped by another
|
|
96
|
+
* worker, whose claim this one must not extend.
|
|
97
|
+
*/
|
|
98
|
+
renew(job: JobRecord): Promise<boolean>;
|
|
99
|
+
complete(job: JobRecord): Promise<void>;
|
|
100
|
+
fail(job: JobRecord, error: string): Promise<void>;
|
|
101
|
+
retry(job: JobRecord): Promise<void>;
|
|
56
102
|
recoverStale(): Promise<number>;
|
|
57
|
-
failed(): Promise<
|
|
58
|
-
|
|
103
|
+
failed(): Promise<JobRecord[]>;
|
|
104
|
+
/**
|
|
105
|
+
* How many jobs are waiting on `queue` — its list plus its delayed set.
|
|
106
|
+
*
|
|
107
|
+
* A delayed job is queued; it is simply not due. Counting only the list
|
|
108
|
+
* reported an empty queue to anything draining one before shutdown.
|
|
109
|
+
*/
|
|
110
|
+
size(queue?: string): Promise<number>;
|
|
59
111
|
}
|
|
60
112
|
//# sourceMappingURL=RedisDriver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RedisDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/RedisDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"RedisDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/RedisDriver.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAW,MAAM,oBAAoB,CAAC;AAG/E,MAAM,WAAW,WAAW;IAC3B,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1C,KAAK,CAAC,CACL,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,GAAG,OAAO,EACtB,EAAE,EAAE,MAAM,GAAG,OAAO,GAClB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE;;;OAGG;IACH,KAAK,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC3E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IACzC;;;;;OAKG;IACH,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAC5E,aAAa,CAAC,CACb,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,GAAG,EAAE,MAAM,GAAG,MAAM,EACpB,GAAG,IAAI,EAAE,MAAM,EAAE,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrB,IAAI,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1D,KAAK,CAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrC;AAiDD;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAC1B,WAAW,GACX,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;AAuD9C,qBAAa,WAAY,YAAW,WAAW;;gBAsC7C,MAAM,EAAE,iBAAiB,EACzB,OAAO,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B;;;;WAIG;QACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B;;;;;;;;WAQG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;;;;;WAKG;QACH,aAAa,CAAC,EAAE,MAAM,CAAC;KACvB;IA4CF;;;;OAIG;IACH,IAAI,eAAe,IAAI,MAAM,CAE5B;IA2BK,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAsBnC,GAAG,CACR,MAAM,GAAE,SAAS,MAAM,EAAoB,GACzC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IA6C5B;;;;;;OAMG;IACG,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC;IAavC,QAAQ,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASlD,KAAK,CAAC,GAAG,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAQpC,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IA6G/B,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAepC;;;;;OAKG;IACG,IAAI,CAAC,KAAK,GAAE,MAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;CAwD1D"}
|
|
@@ -17,16 +17,39 @@
|
|
|
17
17
|
* { PX: ms }) does NOT satisfy this interface and would drop the lease TTL — it
|
|
18
18
|
* needs a thin adapter.
|
|
19
19
|
*/
|
|
20
|
-
import {
|
|
20
|
+
import { DEFAULT_QUEUE } from "../Job.js";
|
|
21
|
+
import { queueOf } from "../QueueManager.js";
|
|
22
|
+
import { inProduction } from "../vendor/nodeEnv.js";
|
|
21
23
|
function isValidJob(obj) {
|
|
22
24
|
if (typeof obj !== "object" || obj === null)
|
|
23
25
|
return false;
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
typeof
|
|
27
|
-
typeof
|
|
28
|
-
typeof
|
|
29
|
-
|
|
26
|
+
return (typeof Reflect.get(obj, "id") === "string" &&
|
|
27
|
+
typeof Reflect.get(obj, "name") === "string" &&
|
|
28
|
+
typeof Reflect.get(obj, "attempts") === "number" &&
|
|
29
|
+
typeof Reflect.get(obj, "maxAttempts") === "number" &&
|
|
30
|
+
typeof Reflect.get(obj, "status") === "string");
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Read a lease back. A value written by an older version of this driver is the
|
|
34
|
+
* raw job string on its own, with no owner — still usable for the one thing
|
|
35
|
+
* `#removeFromProcessing` needs it for.
|
|
36
|
+
*/
|
|
37
|
+
function readLease(stored) {
|
|
38
|
+
let parsed;
|
|
39
|
+
try {
|
|
40
|
+
parsed = JSON.parse(stored);
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return { raw: stored };
|
|
44
|
+
}
|
|
45
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
46
|
+
return { raw: stored };
|
|
47
|
+
const raw = Reflect.get(parsed, "raw");
|
|
48
|
+
const owner = Reflect.get(parsed, "owner");
|
|
49
|
+
if (typeof raw !== "string" || typeof owner !== "string") {
|
|
50
|
+
return { raw: stored };
|
|
51
|
+
}
|
|
52
|
+
return { owner, raw };
|
|
30
53
|
}
|
|
31
54
|
/**
|
|
32
55
|
* LMOVE (Redis 6.2+) is what makes pop() atomic. Warned once, when the client
|
|
@@ -42,6 +65,10 @@ function checkLmove(client, allowNonAtomicPop) {
|
|
|
42
65
|
// job from pending before it reaches processing: nothing recovers it,
|
|
43
66
|
// because nothing knows it existed. That is a different product, and in
|
|
44
67
|
// production it must be asked for rather than fallen into.
|
|
68
|
+
// Read through `inProduction()` rather than `NODE_ENV === "production"`:
|
|
69
|
+
// `NODE_ENV=prod` is ordinary in a Dockerfile, and taken verbatim it answers
|
|
70
|
+
// "not production" — so this queue would silently accept a delivery
|
|
71
|
+
// guarantee weaker than the one it advertises.
|
|
45
72
|
if (inProduction() && !allowNonAtomicPop) {
|
|
46
73
|
throw new Error("[bay] this Redis client has no LMOVE (Redis < 6.2), so pop() would be a non-atomic lpop+rpush — " +
|
|
47
74
|
"a crash between the two loses the in-flight job, turning at-least-once delivery into at-most-once.\n" +
|
|
@@ -131,26 +158,71 @@ export class RedisDriver {
|
|
|
131
158
|
throw new Error(`[bay] RedisDriver visibilityTimeoutMs must be a positive integer (ms), got ${visibilityTimeout}`);
|
|
132
159
|
}
|
|
133
160
|
this.#visibilityTimeout = visibilityTimeout;
|
|
161
|
+
const maxStalled = options?.maxStalledCount ?? 1;
|
|
162
|
+
if (!Number.isInteger(maxStalled) || maxStalled < 0) {
|
|
163
|
+
throw new Error(`[bay] RedisDriver maxStalledCount must be a non-negative integer, got ${maxStalled}`);
|
|
164
|
+
}
|
|
165
|
+
this.#maxStalledCount = maxStalled;
|
|
166
|
+
const maxFailed = options?.maxFailedJobs ?? 1000;
|
|
167
|
+
if (!Number.isInteger(maxFailed) || maxFailed < 0) {
|
|
168
|
+
throw new Error(`[bay] RedisDriver maxFailedJobs must be a non-negative integer, got ${maxFailed}`);
|
|
169
|
+
}
|
|
170
|
+
this.#maxFailedJobs = maxFailed;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Renew a lease at half its length: two chances to be heard before the
|
|
174
|
+
* deadline, so one slow round-trip does not hand a running job to somebody
|
|
175
|
+
* else. Read by `QueueManager` while a handler runs.
|
|
176
|
+
*/
|
|
177
|
+
get renewIntervalMs() {
|
|
178
|
+
return Math.max(1, Math.floor(this.#visibilityTimeout / 2));
|
|
134
179
|
}
|
|
135
|
-
|
|
180
|
+
/**
|
|
181
|
+
* Where one queue's jobs wait.
|
|
182
|
+
*
|
|
183
|
+
* The default queue keeps the key it always had. Naming it
|
|
184
|
+
* `queue:default:pending` would have been tidier and would have orphaned
|
|
185
|
+
* every job already sitting in `queue:pending` at the moment of the upgrade
|
|
186
|
+
* — a silent loss, since nothing reads the old key afterwards.
|
|
187
|
+
*/
|
|
188
|
+
#pendingKey = (queue = DEFAULT_QUEUE) => queue === DEFAULT_QUEUE
|
|
189
|
+
? `${this.#prefix}pending`
|
|
190
|
+
: `${this.#prefix}q:${queue}:pending`;
|
|
191
|
+
#delayedKey = (queue = DEFAULT_QUEUE) => queue === DEFAULT_QUEUE
|
|
192
|
+
? `${this.#prefix}delayed`
|
|
193
|
+
: `${this.#prefix}q:${queue}:delayed`;
|
|
136
194
|
#processingKey = () => `${this.#prefix}processing`;
|
|
137
195
|
#allowNonAtomicPop = false;
|
|
196
|
+
#maxStalledCount = 1;
|
|
197
|
+
#maxFailedJobs = 1000;
|
|
198
|
+
/** This driver instance, as a lease owner. */
|
|
199
|
+
#workerId = crypto.randomUUID();
|
|
138
200
|
#failedKey = () => `${this.#prefix}failed`;
|
|
139
201
|
#leaseKey = (jobId) => `${this.#prefix}lease:${jobId}`;
|
|
140
202
|
async push(job) {
|
|
141
203
|
const client = await this.#client();
|
|
142
|
-
|
|
204
|
+
const queue = queueOf(job);
|
|
205
|
+
if (job.runAt !== undefined && job.runAt > Date.now()) {
|
|
206
|
+
if (!client.zadd) {
|
|
207
|
+
// Pushing it to the list instead would run it now, which is the
|
|
208
|
+
// one thing a delay exists to prevent.
|
|
209
|
+
throw new Error("This Redis client cannot hold a delayed job: it has no ZADD. " +
|
|
210
|
+
"Use a client with sorted-set commands (ioredis has them), or dispatch without `delay`.");
|
|
211
|
+
}
|
|
212
|
+
await client.zadd(this.#delayedKey(queue), job.runAt, JSON.stringify(job));
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
await client.rpush(this.#pendingKey(queue), JSON.stringify(job));
|
|
143
216
|
}
|
|
144
|
-
async pop() {
|
|
217
|
+
async pop(queues = [DEFAULT_QUEUE]) {
|
|
145
218
|
const client = await this.#client();
|
|
146
219
|
let raw = null;
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
raw
|
|
152
|
-
|
|
153
|
-
await client.rpush(this.#processingKey(), raw);
|
|
220
|
+
// In the order given, so a worker can say which queue it drains first.
|
|
221
|
+
for (const queue of queues) {
|
|
222
|
+
await this.#promoteDue(client, queue);
|
|
223
|
+
raw = await this.#take(client, queue);
|
|
224
|
+
if (raw !== null)
|
|
225
|
+
break;
|
|
154
226
|
}
|
|
155
227
|
if (!raw)
|
|
156
228
|
return null;
|
|
@@ -177,9 +249,29 @@ export class RedisDriver {
|
|
|
177
249
|
// bad job. The error propagates and the job STAYS in processing with
|
|
178
250
|
// no lease, which is precisely the state recoverStale() puts back in
|
|
179
251
|
// pending — so the delivery guarantee survives the blip.
|
|
180
|
-
await client.set(this.#leaseKey(parsed.id), raw, "PX", String(this.#visibilityTimeout));
|
|
252
|
+
await client.set(this.#leaseKey(parsed.id), JSON.stringify({ owner: this.#workerId, raw }), "PX", String(this.#visibilityTimeout));
|
|
181
253
|
return parsed;
|
|
182
254
|
}
|
|
255
|
+
/**
|
|
256
|
+
* Say the job is still being worked on, and push its deadline back.
|
|
257
|
+
*
|
|
258
|
+
* Answers `false` when there is nothing left to renew — the lease expired
|
|
259
|
+
* and the job was recovered, or it was recovered and re-popped by another
|
|
260
|
+
* worker, whose claim this one must not extend.
|
|
261
|
+
*/
|
|
262
|
+
async renew(job) {
|
|
263
|
+
const client = await this.#client();
|
|
264
|
+
const key = this.#leaseKey(job.id);
|
|
265
|
+
const stored = await client.get(key);
|
|
266
|
+
if (stored === null)
|
|
267
|
+
return false;
|
|
268
|
+
const lease = readLease(stored);
|
|
269
|
+
if (lease.owner !== undefined && lease.owner !== this.#workerId) {
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
272
|
+
await client.set(key, stored, "PX", String(this.#visibilityTimeout));
|
|
273
|
+
return true;
|
|
274
|
+
}
|
|
183
275
|
async complete(job) {
|
|
184
276
|
const client = await this.#client();
|
|
185
277
|
await this.#removeFromProcessing(job);
|
|
@@ -191,14 +283,14 @@ export class RedisDriver {
|
|
|
191
283
|
await client.del(this.#leaseKey(job.id));
|
|
192
284
|
job.error = error;
|
|
193
285
|
job.status = "failed";
|
|
194
|
-
await
|
|
286
|
+
await this.#pushFailed(client, job);
|
|
195
287
|
}
|
|
196
288
|
async retry(job) {
|
|
197
289
|
const client = await this.#client();
|
|
198
290
|
await this.#removeFromProcessing(job);
|
|
199
291
|
await client.del(this.#leaseKey(job.id));
|
|
200
292
|
job.status = "pending";
|
|
201
|
-
await client.rpush(this.#pendingKey(), JSON.stringify(job));
|
|
293
|
+
await client.rpush(this.#pendingKey(queueOf(job)), JSON.stringify(job));
|
|
202
294
|
}
|
|
203
295
|
async recoverStale() {
|
|
204
296
|
const client = await this.#client();
|
|
@@ -220,29 +312,72 @@ export class RedisDriver {
|
|
|
220
312
|
continue;
|
|
221
313
|
}
|
|
222
314
|
const lease = await client.get(this.#leaseKey(parsed.id));
|
|
223
|
-
if (lease
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
315
|
+
if (lease !== null)
|
|
316
|
+
continue;
|
|
317
|
+
// The LREM is the claim, and its RESULT decides who acts. Two
|
|
318
|
+
// recovery passes overlapping — two workers, or one worker whose
|
|
319
|
+
// pass ran long — both read the same expired entry, and both used to
|
|
320
|
+
// push it back to pending: one job, delivered twice, from the
|
|
321
|
+
// mechanism that exists to make delivery reliable. Exactly one LREM
|
|
322
|
+
// can remove a given element, so exactly one pass continues past
|
|
323
|
+
// here. (A crash between this and the RPUSH below still loses the
|
|
324
|
+
// entry; closing that needs the whole pass in one server-side script,
|
|
325
|
+
// which is how upstream does it.)
|
|
326
|
+
const claimed = await client.lrem(this.#processingKey(), 1, raw);
|
|
327
|
+
if (claimed === 0)
|
|
328
|
+
continue;
|
|
329
|
+
// A stall is not an attempt: the worker died before the handler
|
|
330
|
+
// could fail, so `attempts` never moved and `maxAttempts` never
|
|
331
|
+
// applied. A job that kills whatever picks it up was therefore
|
|
332
|
+
// recovered forever, taking the queue with it. Counted separately,
|
|
333
|
+
// and bounded — upstream bounds the same thing with the same
|
|
334
|
+
// default, failing the job once it is exceeded.
|
|
335
|
+
const stalled = (parsed.stalledCount ?? 0) + 1;
|
|
336
|
+
if (stalled > this.#maxStalledCount) {
|
|
337
|
+
parsed.stalledCount = stalled;
|
|
338
|
+
parsed.status = "failed";
|
|
339
|
+
parsed.error = `Stalled ${stalled} time(s) without completing (maxStalledCount ${this.#maxStalledCount})`;
|
|
340
|
+
await this.#pushFailed(client, parsed);
|
|
341
|
+
continue;
|
|
228
342
|
}
|
|
343
|
+
parsed.stalledCount = stalled;
|
|
344
|
+
parsed.status = "pending";
|
|
345
|
+
// Back to the queue it came from, not to the default one: a recovered
|
|
346
|
+
// job whose queue nobody serves would never run again.
|
|
347
|
+
await client.rpush(this.#pendingKey(queueOf(parsed)), JSON.stringify(parsed));
|
|
348
|
+
recovered++;
|
|
229
349
|
}
|
|
230
350
|
return recovered;
|
|
231
351
|
}
|
|
352
|
+
/**
|
|
353
|
+
* File a job as failed, keeping the list to `maxFailedJobs`.
|
|
354
|
+
*
|
|
355
|
+
* Unbounded, the failed list is a leak with no ceiling and no owner: nothing
|
|
356
|
+
* trims it, and `failed()` reads all of it in one LRANGE. The memory driver
|
|
357
|
+
* has capped its own at a thousand from the start; this is the same cap on
|
|
358
|
+
* the driver where the list actually survives a restart.
|
|
359
|
+
*/
|
|
360
|
+
async #pushFailed(client, job) {
|
|
361
|
+
await client.rpush(this.#failedKey(), JSON.stringify(job));
|
|
362
|
+
if (this.#maxFailedJobs === 0 || !client.ltrim)
|
|
363
|
+
return;
|
|
364
|
+
await client.ltrim(this.#failedKey(), -this.#maxFailedJobs, -1);
|
|
365
|
+
}
|
|
232
366
|
/**
|
|
233
367
|
* Remove the entry for `job` from the processing list. The string in
|
|
234
368
|
* Redis is whatever pop() pushed, but QueueManager mutates `job` after
|
|
235
369
|
* pop returns (attempts++, status="processing", processedAt, then
|
|
236
370
|
* completed/failed/pending). LREM-ing on `JSON.stringify(job)` would
|
|
237
|
-
* therefore miss every real-world entry. Use the lease —
|
|
238
|
-
* exact raw string
|
|
371
|
+
* therefore miss every real-world entry. Use the lease — which carries
|
|
372
|
+
* the exact raw string pop() moved — and fall back to a list scan when
|
|
239
373
|
* the lease has expired (e.g. recoverStale already handled it).
|
|
240
374
|
*/
|
|
241
375
|
async #removeFromProcessing(job) {
|
|
242
376
|
const client = await this.#client();
|
|
243
377
|
const stored = await client.get(this.#leaseKey(job.id));
|
|
244
378
|
if (stored !== null) {
|
|
245
|
-
const
|
|
379
|
+
const { raw } = readLease(stored);
|
|
380
|
+
const removed = await client.lrem(this.#processingKey(), 1, raw);
|
|
246
381
|
if (removed > 0)
|
|
247
382
|
return;
|
|
248
383
|
}
|
|
@@ -278,9 +413,58 @@ export class RedisDriver {
|
|
|
278
413
|
})
|
|
279
414
|
.filter((j) => j !== null);
|
|
280
415
|
}
|
|
281
|
-
|
|
416
|
+
/**
|
|
417
|
+
* How many jobs are waiting on `queue` — its list plus its delayed set.
|
|
418
|
+
*
|
|
419
|
+
* A delayed job is queued; it is simply not due. Counting only the list
|
|
420
|
+
* reported an empty queue to anything draining one before shutdown.
|
|
421
|
+
*/
|
|
422
|
+
async size(queue = DEFAULT_QUEUE) {
|
|
282
423
|
const client = await this.#client();
|
|
283
|
-
|
|
424
|
+
const waiting = await client.llen(this.#pendingKey(queue));
|
|
425
|
+
const delayed = client.zcard
|
|
426
|
+
? await client.zcard(this.#delayedKey(queue))
|
|
427
|
+
: 0;
|
|
428
|
+
return waiting + delayed;
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Move `queue`'s due jobs out of the delayed set and onto its list.
|
|
432
|
+
*
|
|
433
|
+
* `ZREM` is the claim: two workers can read the same due entry, and only
|
|
434
|
+
* the one whose removal returns 1 owns it. Without that the job is pushed
|
|
435
|
+
* onto the list once per worker that saw it.
|
|
436
|
+
*/
|
|
437
|
+
async #promoteDue(client, queue) {
|
|
438
|
+
if (!client.zrangebyscore || !client.zrem)
|
|
439
|
+
return;
|
|
440
|
+
const due = await client.zrangebyscore(this.#delayedKey(queue), 0, Date.now(), "LIMIT", "0", "100");
|
|
441
|
+
for (const raw of due) {
|
|
442
|
+
const claimed = await client.zrem(this.#delayedKey(queue), raw);
|
|
443
|
+
if (claimed !== 1)
|
|
444
|
+
continue;
|
|
445
|
+
let parsed;
|
|
446
|
+
try {
|
|
447
|
+
parsed = JSON.parse(raw);
|
|
448
|
+
}
|
|
449
|
+
catch {
|
|
450
|
+
// Already removed from the set; there is nothing runnable to push.
|
|
451
|
+
continue;
|
|
452
|
+
}
|
|
453
|
+
if (!isValidJob(parsed))
|
|
454
|
+
continue;
|
|
455
|
+
parsed.runAt = undefined;
|
|
456
|
+
await client.rpush(this.#pendingKey(queue), JSON.stringify(parsed));
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
/** Take the head of one queue, claiming it in `processing`. */
|
|
460
|
+
async #take(client, queue) {
|
|
461
|
+
if (client.lmove) {
|
|
462
|
+
return client.lmove(this.#pendingKey(queue), this.#processingKey(), "LEFT", "RIGHT");
|
|
463
|
+
}
|
|
464
|
+
const raw = await client.lpop(this.#pendingKey(queue));
|
|
465
|
+
if (raw)
|
|
466
|
+
await client.rpush(this.#processingKey(), raw);
|
|
467
|
+
return raw;
|
|
284
468
|
}
|
|
285
469
|
}
|
|
286
470
|
//# sourceMappingURL=RedisDriver.js.map
|