@c9up/bay 0.1.12 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +172 -1
- package/dist/BayProvider.d.ts +55 -33
- package/dist/BayProvider.d.ts.map +1 -1
- package/dist/BayProvider.js +81 -10
- package/dist/BayProvider.js.map +1 -1
- package/dist/Job.d.ts +99 -0
- package/dist/Job.d.ts.map +1 -0
- package/dist/Job.js +78 -0
- package/dist/Job.js.map +1 -0
- package/dist/QueueManager.d.ts +147 -22
- package/dist/QueueManager.d.ts.map +1 -1
- package/dist/QueueManager.js +290 -52
- package/dist/QueueManager.js.map +1 -1
- package/dist/adapters.d.ts +68 -0
- package/dist/adapters.d.ts.map +1 -0
- package/dist/adapters.js +56 -0
- package/dist/adapters.js.map +1 -0
- package/dist/augmentations.d.ts +28 -0
- package/dist/augmentations.d.ts.map +1 -0
- package/dist/augmentations.js +17 -0
- package/dist/augmentations.js.map +1 -0
- package/dist/configure.d.ts +19 -0
- package/dist/configure.d.ts.map +1 -0
- package/dist/configure.js +48 -0
- package/dist/configure.js.map +1 -0
- package/dist/console/contract.d.ts +60 -0
- package/dist/console/contract.d.ts.map +1 -0
- package/dist/console/contract.js +36 -0
- package/dist/console/contract.js.map +1 -0
- package/dist/console/index.d.ts +29 -0
- package/dist/console/index.d.ts.map +1 -0
- package/dist/console/index.js +45 -0
- package/dist/console/index.js.map +1 -0
- package/dist/console/makeJob.d.ts +32 -0
- package/dist/console/makeJob.d.ts.map +1 -0
- package/dist/console/makeJob.js +118 -0
- package/dist/console/makeJob.js.map +1 -0
- package/dist/console/queueWork.d.ts +18 -0
- package/dist/console/queueWork.d.ts.map +1 -0
- package/dist/console/queueWork.js +58 -0
- package/dist/console/queueWork.js.map +1 -0
- package/dist/drivers/MemoryDriver.d.ts +14 -8
- package/dist/drivers/MemoryDriver.d.ts.map +1 -1
- package/dist/drivers/MemoryDriver.js +61 -7
- package/dist/drivers/MemoryDriver.js.map +1 -1
- package/dist/drivers/RedisDriver.d.ts +66 -8
- package/dist/drivers/RedisDriver.d.ts.map +1 -1
- package/dist/drivers/RedisDriver.js +257 -45
- package/dist/drivers/RedisDriver.js.map +1 -1
- package/dist/index.d.ts +10 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -3
- package/dist/index.js.map +1 -1
- package/dist/jobs.d.ts +43 -0
- package/dist/jobs.d.ts.map +1 -0
- package/dist/jobs.js +105 -0
- package/dist/jobs.js.map +1 -0
- package/dist/nodeEnv.d.ts +16 -0
- package/dist/nodeEnv.d.ts.map +1 -0
- package/dist/nodeEnv.js +32 -0
- package/dist/nodeEnv.js.map +1 -0
- package/dist/quasar.d.ts +1 -1
- package/dist/quasar.js +1 -1
- package/dist/services/main.d.ts +5 -0
- package/dist/services/main.d.ts.map +1 -1
- package/dist/services/main.js +7 -0
- package/dist/services/main.js.map +1 -1
- package/dist/testing/FakeQueue.d.ts +15 -9
- package/dist/testing/FakeQueue.d.ts.map +1 -1
- package/dist/testing/FakeQueue.js +13 -3
- package/dist/testing/FakeQueue.js.map +1 -1
- package/package.json +9 -3
- package/src/BayProvider.ts +143 -25
- package/src/Job.ts +137 -0
- package/src/QueueManager.ts +454 -56
- package/src/adapters.ts +75 -0
- package/src/augmentations.ts +31 -0
- package/src/configure.ts +63 -0
- package/src/console/contract.ts +94 -0
- package/src/console/index.ts +68 -0
- package/src/console/makeJob.ts +139 -0
- package/src/console/queueWork.ts +70 -0
- package/src/drivers/MemoryDriver.ts +66 -14
- package/src/drivers/RedisDriver.ts +366 -63
- package/src/index.ts +35 -5
- package/src/jobs.ts +111 -0
- package/src/nodeEnv.ts +30 -0
- package/src/quasar.ts +1 -1
- package/src/services/main.ts +8 -0
- package/src/testing/FakeQueue.ts +25 -15
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
* needs a thin adapter.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
import
|
|
21
|
+
import { DEFAULT_QUEUE } from "../Job.js";
|
|
22
|
+
import { inProduction } from "../nodeEnv.js";
|
|
23
|
+
import { type JobRecord, type QueueDriver, queueOf } from "../QueueManager.js";
|
|
22
24
|
|
|
23
25
|
export interface RedisClient {
|
|
24
26
|
rpush(key: string, ...values: string[]): Promise<number>;
|
|
@@ -30,25 +32,80 @@ export interface RedisClient {
|
|
|
30
32
|
to: "LEFT" | "RIGHT",
|
|
31
33
|
): Promise<string | null>;
|
|
32
34
|
lrem(key: string, count: number, element: string): Promise<number>;
|
|
35
|
+
/**
|
|
36
|
+
* Optional, like `lmove`. Present on ioredis; without it the failed list
|
|
37
|
+
* simply keeps its entries, which is what this driver did before.
|
|
38
|
+
*/
|
|
39
|
+
ltrim?(key: string, start: number, stop: number): Promise<string>;
|
|
33
40
|
llen(key: string): Promise<number>;
|
|
34
41
|
lrange(key: string, start: number, stop: number): Promise<string[]>;
|
|
35
42
|
del(key: string): Promise<number>;
|
|
36
43
|
set(key: string, value: string, ...args: string[]): Promise<string | null>;
|
|
37
44
|
get(key: string): Promise<string | null>;
|
|
45
|
+
/**
|
|
46
|
+
* Sorted-set commands, for delayed jobs. Optional like `lmove`: a client
|
|
47
|
+
* without them can still run a queue, and `push` refuses a job carrying a
|
|
48
|
+
* `delay` rather than running it early — which is the one thing a delay
|
|
49
|
+
* must not do.
|
|
50
|
+
*/
|
|
51
|
+
zadd?(key: string, score: number, member: string): Promise<number | string>;
|
|
52
|
+
zrangebyscore?(
|
|
53
|
+
key: string,
|
|
54
|
+
min: number | string,
|
|
55
|
+
max: number | string,
|
|
56
|
+
...args: string[]
|
|
57
|
+
): Promise<string[]>;
|
|
58
|
+
zrem?(key: string, ...members: string[]): Promise<number>;
|
|
59
|
+
zcard?(key: string): Promise<number>;
|
|
38
60
|
}
|
|
39
61
|
|
|
40
|
-
function isValidJob(obj: unknown): obj is
|
|
62
|
+
function isValidJob(obj: unknown): obj is JobRecord {
|
|
41
63
|
if (typeof obj !== "object" || obj === null) return false;
|
|
42
|
-
const j = obj as Record<string, unknown>;
|
|
43
64
|
return (
|
|
44
|
-
typeof
|
|
45
|
-
typeof
|
|
46
|
-
typeof
|
|
47
|
-
typeof
|
|
48
|
-
typeof
|
|
65
|
+
typeof Reflect.get(obj, "id") === "string" &&
|
|
66
|
+
typeof Reflect.get(obj, "name") === "string" &&
|
|
67
|
+
typeof Reflect.get(obj, "attempts") === "number" &&
|
|
68
|
+
typeof Reflect.get(obj, "maxAttempts") === "number" &&
|
|
69
|
+
typeof Reflect.get(obj, "status") === "string"
|
|
49
70
|
);
|
|
50
71
|
}
|
|
51
72
|
|
|
73
|
+
/**
|
|
74
|
+
* What a lease holds: the exact string pop() moved into `processing`, and the
|
|
75
|
+
* worker that moved it.
|
|
76
|
+
*
|
|
77
|
+
* The owner is what makes renewal safe. Without it a worker whose lease had
|
|
78
|
+
* already expired — its job recovered, re-popped by somebody else — would go on
|
|
79
|
+
* extending the deadline of a job it no longer had any claim on. Upstream draws
|
|
80
|
+
* the same line inside its renewal script: "Only the worker that currently owns
|
|
81
|
+
* the lease may renew it."
|
|
82
|
+
*/
|
|
83
|
+
interface Lease {
|
|
84
|
+
owner: string;
|
|
85
|
+
raw: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Read a lease back. A value written by an older version of this driver is the
|
|
90
|
+
* raw job string on its own, with no owner — still usable for the one thing
|
|
91
|
+
* `#removeFromProcessing` needs it for.
|
|
92
|
+
*/
|
|
93
|
+
function readLease(stored: string): { owner?: string; raw: string } {
|
|
94
|
+
let parsed: unknown;
|
|
95
|
+
try {
|
|
96
|
+
parsed = JSON.parse(stored);
|
|
97
|
+
} catch {
|
|
98
|
+
return { raw: stored };
|
|
99
|
+
}
|
|
100
|
+
if (typeof parsed !== "object" || parsed === null) return { raw: stored };
|
|
101
|
+
const raw = Reflect.get(parsed, "raw");
|
|
102
|
+
const owner = Reflect.get(parsed, "owner");
|
|
103
|
+
if (typeof raw !== "string" || typeof owner !== "string") {
|
|
104
|
+
return { raw: stored };
|
|
105
|
+
}
|
|
106
|
+
return { owner, raw };
|
|
107
|
+
}
|
|
108
|
+
|
|
52
109
|
/**
|
|
53
110
|
* Where the client comes from. A resolver is what lets a queue name its
|
|
54
111
|
* connection (`quasarConnection("jobs")`) instead of being handed a client:
|
|
@@ -65,13 +122,35 @@ export type RedisClientSource =
|
|
|
65
122
|
* has no client to inspect yet.
|
|
66
123
|
*/
|
|
67
124
|
const warned = new WeakSet<object>();
|
|
68
|
-
function
|
|
69
|
-
if (typeof client.lmove === "function"
|
|
125
|
+
function checkLmove(client: RedisClient, allowNonAtomicPop: boolean): void {
|
|
126
|
+
if (typeof client.lmove === "function") return;
|
|
127
|
+
|
|
128
|
+
// A queue's whole promise is that a job it accepted gets run. Without LMOVE
|
|
129
|
+
// the pop is `lpop` then `rpush`, and a crash between the two deletes the
|
|
130
|
+
// job from pending before it reaches processing: nothing recovers it,
|
|
131
|
+
// because nothing knows it existed. That is a different product, and in
|
|
132
|
+
// production it must be asked for rather than fallen into.
|
|
133
|
+
if (inProduction() && !allowNonAtomicPop) {
|
|
134
|
+
throw new Error(
|
|
135
|
+
"[bay] this Redis client has no LMOVE (Redis < 6.2), so pop() would be a non-atomic lpop+rpush — " +
|
|
136
|
+
"a crash between the two loses the in-flight job, turning at-least-once delivery into at-most-once.\n" +
|
|
137
|
+
" Upgrade to Redis 6.2 or later, or pass `allowNonAtomicPop: true` to state that losing a job is acceptable here.",
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
if (warned.has(client)) return;
|
|
70
141
|
warned.add(client);
|
|
142
|
+
|
|
143
|
+
// Said even when the deployment opted in: agreeing to lose a job once, in a
|
|
144
|
+
// config file, is not the same as being reminded that this process is
|
|
145
|
+
// running that way. The line has to be in the logs of the incident.
|
|
146
|
+
const optedIn = inProduction() && allowNonAtomicPop;
|
|
71
147
|
console.warn(
|
|
72
148
|
"[bay] RedisDriver: client lacks LMOVE (Redis <6.2). pop() falls back to " +
|
|
73
149
|
"a non-atomic lpop+rpush, downgrading delivery from at-least-once to " +
|
|
74
|
-
"at-most-once — a crash between the two commands loses the in-flight job."
|
|
150
|
+
"at-most-once — a crash between the two commands loses the in-flight job." +
|
|
151
|
+
(optedIn
|
|
152
|
+
? "\n Running this way in PRODUCTION because allowNonAtomicPop was set."
|
|
153
|
+
: ""),
|
|
75
154
|
);
|
|
76
155
|
}
|
|
77
156
|
|
|
@@ -101,7 +180,7 @@ export class RedisDriver implements QueueDriver {
|
|
|
101
180
|
if (this.#resolved) return this.#resolved;
|
|
102
181
|
if (typeof this.#source !== "function") {
|
|
103
182
|
this.#resolved = this.#source;
|
|
104
|
-
|
|
183
|
+
checkLmove(this.#resolved, this.#allowNonAtomicPop);
|
|
105
184
|
return this.#resolved;
|
|
106
185
|
}
|
|
107
186
|
if (!this.#pending) {
|
|
@@ -109,7 +188,7 @@ export class RedisDriver implements QueueDriver {
|
|
|
109
188
|
this.#pending = Promise.resolve(resolver())
|
|
110
189
|
.then((client) => {
|
|
111
190
|
this.#resolved = client;
|
|
112
|
-
|
|
191
|
+
checkLmove(client, this.#allowNonAtomicPop);
|
|
113
192
|
return client;
|
|
114
193
|
})
|
|
115
194
|
// Cleared on failure too. Clearing only on success left the
|
|
@@ -125,18 +204,47 @@ export class RedisDriver implements QueueDriver {
|
|
|
125
204
|
|
|
126
205
|
constructor(
|
|
127
206
|
source: RedisClientSource,
|
|
128
|
-
options?: {
|
|
207
|
+
options?: {
|
|
208
|
+
prefix?: string;
|
|
209
|
+
visibilityTimeoutMs?: number;
|
|
210
|
+
/**
|
|
211
|
+
* Accept the non-atomic pop on a Redis older than 6.2, in
|
|
212
|
+
* production. Off by default: losing an accepted job is a choice a
|
|
213
|
+
* deployment makes, not one a version check makes for it.
|
|
214
|
+
*/
|
|
215
|
+
allowNonAtomicPop?: boolean;
|
|
216
|
+
/**
|
|
217
|
+
* How many times a job may be reclaimed from a stalled worker before
|
|
218
|
+
* it is filed as failed instead of pushed round again. Default `1`,
|
|
219
|
+
* upstream's default for the same setting.
|
|
220
|
+
*
|
|
221
|
+
* Unbounded recovery is a job that kills its worker taking the whole
|
|
222
|
+
* queue down with it, forever: the crash never reaches the failure
|
|
223
|
+
* path, so `attempts` never moves and `maxAttempts` never applies.
|
|
224
|
+
*/
|
|
225
|
+
maxStalledCount?: number;
|
|
226
|
+
/**
|
|
227
|
+
* How many failed jobs to keep. Default `1000` — the ceiling the
|
|
228
|
+
* memory driver already had. `0` keeps every one of them.
|
|
229
|
+
*
|
|
230
|
+
* Only enforced when the client answers `ltrim`.
|
|
231
|
+
*/
|
|
232
|
+
maxFailedJobs?: number;
|
|
233
|
+
},
|
|
129
234
|
) {
|
|
130
235
|
this.#source = source;
|
|
131
236
|
// A client handed in directly can be checked now, so the warning keeps
|
|
132
237
|
// landing at construction as it always did. A named connection has no
|
|
133
238
|
// client yet — it is checked when the connection resolves.
|
|
134
|
-
if (typeof source !== "function")
|
|
239
|
+
if (typeof source !== "function") {
|
|
240
|
+
checkLmove(source, options?.allowNonAtomicPop ?? false);
|
|
241
|
+
}
|
|
135
242
|
// Normalised rather than documented: every key is built by concatenation
|
|
136
243
|
// (`${prefix}pending`), so a prefix without a trailing separator yields
|
|
137
244
|
// "myapppending" — unreadable, and able to collide with a neighbouring
|
|
138
245
|
// prefix. Nothing warned, because nothing failed.
|
|
139
246
|
this.#prefix = withSeparator(options?.prefix ?? "queue:");
|
|
247
|
+
this.#allowNonAtomicPop = options?.allowNonAtomicPop ?? false;
|
|
140
248
|
const visibilityTimeout = options?.visibilityTimeoutMs ?? 30_000;
|
|
141
249
|
// A non-positive / non-integer timeout makes pop()'s `SET … PX <ms>` fail
|
|
142
250
|
// on a real Redis; the catch then removes the job from `processing` and
|
|
@@ -148,78 +256,168 @@ export class RedisDriver implements QueueDriver {
|
|
|
148
256
|
);
|
|
149
257
|
}
|
|
150
258
|
this.#visibilityTimeout = visibilityTimeout;
|
|
259
|
+
|
|
260
|
+
const maxStalled = options?.maxStalledCount ?? 1;
|
|
261
|
+
if (!Number.isInteger(maxStalled) || maxStalled < 0) {
|
|
262
|
+
throw new Error(
|
|
263
|
+
`[bay] RedisDriver maxStalledCount must be a non-negative integer, got ${maxStalled}`,
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
this.#maxStalledCount = maxStalled;
|
|
267
|
+
|
|
268
|
+
const maxFailed = options?.maxFailedJobs ?? 1000;
|
|
269
|
+
if (!Number.isInteger(maxFailed) || maxFailed < 0) {
|
|
270
|
+
throw new Error(
|
|
271
|
+
`[bay] RedisDriver maxFailedJobs must be a non-negative integer, got ${maxFailed}`,
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
this.#maxFailedJobs = maxFailed;
|
|
151
275
|
}
|
|
152
276
|
|
|
153
|
-
|
|
277
|
+
/**
|
|
278
|
+
* Renew a lease at half its length: two chances to be heard before the
|
|
279
|
+
* deadline, so one slow round-trip does not hand a running job to somebody
|
|
280
|
+
* else. Read by `QueueManager` while a handler runs.
|
|
281
|
+
*/
|
|
282
|
+
get renewIntervalMs(): number {
|
|
283
|
+
return Math.max(1, Math.floor(this.#visibilityTimeout / 2));
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Where one queue's jobs wait.
|
|
288
|
+
*
|
|
289
|
+
* The default queue keeps the key it always had. Naming it
|
|
290
|
+
* `queue:default:pending` would have been tidier and would have orphaned
|
|
291
|
+
* every job already sitting in `queue:pending` at the moment of the upgrade
|
|
292
|
+
* — a silent loss, since nothing reads the old key afterwards.
|
|
293
|
+
*/
|
|
294
|
+
#pendingKey = (queue: string = DEFAULT_QUEUE) =>
|
|
295
|
+
queue === DEFAULT_QUEUE
|
|
296
|
+
? `${this.#prefix}pending`
|
|
297
|
+
: `${this.#prefix}q:${queue}:pending`;
|
|
298
|
+
#delayedKey = (queue: string = DEFAULT_QUEUE) =>
|
|
299
|
+
queue === DEFAULT_QUEUE
|
|
300
|
+
? `${this.#prefix}delayed`
|
|
301
|
+
: `${this.#prefix}q:${queue}:delayed`;
|
|
154
302
|
#processingKey = () => `${this.#prefix}processing`;
|
|
303
|
+
#allowNonAtomicPop = false;
|
|
304
|
+
#maxStalledCount = 1;
|
|
305
|
+
#maxFailedJobs = 1000;
|
|
306
|
+
/** This driver instance, as a lease owner. */
|
|
307
|
+
#workerId = crypto.randomUUID();
|
|
155
308
|
#failedKey = () => `${this.#prefix}failed`;
|
|
156
309
|
#leaseKey = (jobId: string) => `${this.#prefix}lease:${jobId}`;
|
|
157
310
|
|
|
158
|
-
async push(job:
|
|
311
|
+
async push(job: JobRecord): Promise<void> {
|
|
159
312
|
const client = await this.#client();
|
|
160
|
-
|
|
313
|
+
const queue = queueOf(job);
|
|
314
|
+
if (job.runAt !== undefined && job.runAt > Date.now()) {
|
|
315
|
+
if (!client.zadd) {
|
|
316
|
+
// Pushing it to the list instead would run it now, which is the
|
|
317
|
+
// one thing a delay exists to prevent.
|
|
318
|
+
throw new Error(
|
|
319
|
+
"This Redis client cannot hold a delayed job: it has no ZADD. " +
|
|
320
|
+
"Use a client with sorted-set commands (ioredis has them), or dispatch without `delay`.",
|
|
321
|
+
);
|
|
322
|
+
}
|
|
323
|
+
await client.zadd(
|
|
324
|
+
this.#delayedKey(queue),
|
|
325
|
+
job.runAt,
|
|
326
|
+
JSON.stringify(job),
|
|
327
|
+
);
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
await client.rpush(this.#pendingKey(queue), JSON.stringify(job));
|
|
161
331
|
}
|
|
162
332
|
|
|
163
|
-
async pop(
|
|
333
|
+
async pop(
|
|
334
|
+
queues: readonly string[] = [DEFAULT_QUEUE],
|
|
335
|
+
): Promise<JobRecord | null> {
|
|
164
336
|
const client = await this.#client();
|
|
165
337
|
let raw: string | null = null;
|
|
166
338
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
"RIGHT",
|
|
173
|
-
);
|
|
174
|
-
} else {
|
|
175
|
-
raw = await client.lpop(this.#pendingKey());
|
|
176
|
-
if (raw) await client.rpush(this.#processingKey(), raw);
|
|
339
|
+
// In the order given, so a worker can say which queue it drains first.
|
|
340
|
+
for (const queue of queues) {
|
|
341
|
+
await this.#promoteDue(client, queue);
|
|
342
|
+
raw = await this.#take(client, queue);
|
|
343
|
+
if (raw !== null) break;
|
|
177
344
|
}
|
|
178
345
|
|
|
179
346
|
if (!raw) return null;
|
|
347
|
+
|
|
348
|
+
// Only a payload that can never be run is purged. Everything past this
|
|
349
|
+
// point is a REAL job that already sits in `processing`, and deleting
|
|
350
|
+
// it there is the one thing that loses it for good: it is gone from
|
|
351
|
+
// pending too, and recoverStale() scans processing, so nothing would
|
|
352
|
+
// ever find it again.
|
|
353
|
+
let parsed: unknown;
|
|
180
354
|
try {
|
|
181
|
-
|
|
182
|
-
if (!isValidJob(parsed)) {
|
|
183
|
-
// Malformed payload — purge from `processing` so it can't sit
|
|
184
|
-
// there indefinitely as a poison pill. recoverStale() also
|
|
185
|
-
// catches survivors but pop()'s own move is the primary path.
|
|
186
|
-
await client.lrem(this.#processingKey(), 1, raw);
|
|
187
|
-
return null;
|
|
188
|
-
}
|
|
189
|
-
await client.set(
|
|
190
|
-
this.#leaseKey(parsed.id),
|
|
191
|
-
raw,
|
|
192
|
-
"PX",
|
|
193
|
-
String(this.#visibilityTimeout),
|
|
194
|
-
);
|
|
195
|
-
return parsed;
|
|
355
|
+
parsed = JSON.parse(raw);
|
|
196
356
|
} catch {
|
|
357
|
+
// A poison pill: unparseable, and it would sit in processing
|
|
358
|
+
// forever blocking nothing but wasting every recovery pass.
|
|
359
|
+
await client.lrem(this.#processingKey(), 1, raw);
|
|
360
|
+
return null;
|
|
361
|
+
}
|
|
362
|
+
if (!isValidJob(parsed)) {
|
|
197
363
|
await client.lrem(this.#processingKey(), 1, raw);
|
|
198
364
|
return null;
|
|
199
365
|
}
|
|
366
|
+
|
|
367
|
+
// A lease that cannot be written is a transient Redis failure, not a
|
|
368
|
+
// bad job. The error propagates and the job STAYS in processing with
|
|
369
|
+
// no lease, which is precisely the state recoverStale() puts back in
|
|
370
|
+
// pending — so the delivery guarantee survives the blip.
|
|
371
|
+
await client.set(
|
|
372
|
+
this.#leaseKey(parsed.id),
|
|
373
|
+
JSON.stringify({ owner: this.#workerId, raw } satisfies Lease),
|
|
374
|
+
"PX",
|
|
375
|
+
String(this.#visibilityTimeout),
|
|
376
|
+
);
|
|
377
|
+
return parsed;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* Say the job is still being worked on, and push its deadline back.
|
|
382
|
+
*
|
|
383
|
+
* Answers `false` when there is nothing left to renew — the lease expired
|
|
384
|
+
* and the job was recovered, or it was recovered and re-popped by another
|
|
385
|
+
* worker, whose claim this one must not extend.
|
|
386
|
+
*/
|
|
387
|
+
async renew(job: JobRecord): Promise<boolean> {
|
|
388
|
+
const client = await this.#client();
|
|
389
|
+
const key = this.#leaseKey(job.id);
|
|
390
|
+
const stored = await client.get(key);
|
|
391
|
+
if (stored === null) return false;
|
|
392
|
+
const lease = readLease(stored);
|
|
393
|
+
if (lease.owner !== undefined && lease.owner !== this.#workerId) {
|
|
394
|
+
return false;
|
|
395
|
+
}
|
|
396
|
+
await client.set(key, stored, "PX", String(this.#visibilityTimeout));
|
|
397
|
+
return true;
|
|
200
398
|
}
|
|
201
399
|
|
|
202
|
-
async complete(job:
|
|
400
|
+
async complete(job: JobRecord): Promise<void> {
|
|
203
401
|
const client = await this.#client();
|
|
204
402
|
await this.#removeFromProcessing(job);
|
|
205
403
|
await client.del(this.#leaseKey(job.id));
|
|
206
404
|
}
|
|
207
405
|
|
|
208
|
-
async fail(job:
|
|
406
|
+
async fail(job: JobRecord, error: string): Promise<void> {
|
|
209
407
|
const client = await this.#client();
|
|
210
408
|
await this.#removeFromProcessing(job);
|
|
211
409
|
await client.del(this.#leaseKey(job.id));
|
|
212
410
|
job.error = error;
|
|
213
411
|
job.status = "failed";
|
|
214
|
-
await
|
|
412
|
+
await this.#pushFailed(client, job);
|
|
215
413
|
}
|
|
216
414
|
|
|
217
|
-
async retry(job:
|
|
415
|
+
async retry(job: JobRecord): Promise<void> {
|
|
218
416
|
const client = await this.#client();
|
|
219
417
|
await this.#removeFromProcessing(job);
|
|
220
418
|
await client.del(this.#leaseKey(job.id));
|
|
221
419
|
job.status = "pending";
|
|
222
|
-
await client.rpush(this.#pendingKey(), JSON.stringify(job));
|
|
420
|
+
await client.rpush(this.#pendingKey(queueOf(job)), JSON.stringify(job));
|
|
223
421
|
}
|
|
224
422
|
|
|
225
423
|
async recoverStale(): Promise<number> {
|
|
@@ -241,30 +439,77 @@ export class RedisDriver implements QueueDriver {
|
|
|
241
439
|
continue;
|
|
242
440
|
}
|
|
243
441
|
const lease = await client.get(this.#leaseKey(parsed.id));
|
|
244
|
-
if (lease
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
442
|
+
if (lease !== null) continue;
|
|
443
|
+
|
|
444
|
+
// The LREM is the claim, and its RESULT decides who acts. Two
|
|
445
|
+
// recovery passes overlapping — two workers, or one worker whose
|
|
446
|
+
// pass ran long — both read the same expired entry, and both used to
|
|
447
|
+
// push it back to pending: one job, delivered twice, from the
|
|
448
|
+
// mechanism that exists to make delivery reliable. Exactly one LREM
|
|
449
|
+
// can remove a given element, so exactly one pass continues past
|
|
450
|
+
// here. (A crash between this and the RPUSH below still loses the
|
|
451
|
+
// entry; closing that needs the whole pass in one server-side script,
|
|
452
|
+
// which is how upstream does it.)
|
|
453
|
+
const claimed = await client.lrem(this.#processingKey(), 1, raw);
|
|
454
|
+
if (claimed === 0) continue;
|
|
455
|
+
|
|
456
|
+
// A stall is not an attempt: the worker died before the handler
|
|
457
|
+
// could fail, so `attempts` never moved and `maxAttempts` never
|
|
458
|
+
// applied. A job that kills whatever picks it up was therefore
|
|
459
|
+
// recovered forever, taking the queue with it. Counted separately,
|
|
460
|
+
// and bounded — upstream bounds the same thing with the same
|
|
461
|
+
// default, failing the job once it is exceeded.
|
|
462
|
+
const stalled = (parsed.stalledCount ?? 0) + 1;
|
|
463
|
+
if (stalled > this.#maxStalledCount) {
|
|
464
|
+
parsed.stalledCount = stalled;
|
|
465
|
+
parsed.status = "failed";
|
|
466
|
+
parsed.error = `Stalled ${stalled} time(s) without completing (maxStalledCount ${this.#maxStalledCount})`;
|
|
467
|
+
await this.#pushFailed(client, parsed);
|
|
468
|
+
continue;
|
|
249
469
|
}
|
|
470
|
+
|
|
471
|
+
parsed.stalledCount = stalled;
|
|
472
|
+
parsed.status = "pending";
|
|
473
|
+
// Back to the queue it came from, not to the default one: a recovered
|
|
474
|
+
// job whose queue nobody serves would never run again.
|
|
475
|
+
await client.rpush(
|
|
476
|
+
this.#pendingKey(queueOf(parsed)),
|
|
477
|
+
JSON.stringify(parsed),
|
|
478
|
+
);
|
|
479
|
+
recovered++;
|
|
250
480
|
}
|
|
251
481
|
return recovered;
|
|
252
482
|
}
|
|
253
483
|
|
|
484
|
+
/**
|
|
485
|
+
* File a job as failed, keeping the list to `maxFailedJobs`.
|
|
486
|
+
*
|
|
487
|
+
* Unbounded, the failed list is a leak with no ceiling and no owner: nothing
|
|
488
|
+
* trims it, and `failed()` reads all of it in one LRANGE. The memory driver
|
|
489
|
+
* has capped its own at a thousand from the start; this is the same cap on
|
|
490
|
+
* the driver where the list actually survives a restart.
|
|
491
|
+
*/
|
|
492
|
+
async #pushFailed(client: RedisClient, job: JobRecord): Promise<void> {
|
|
493
|
+
await client.rpush(this.#failedKey(), JSON.stringify(job));
|
|
494
|
+
if (this.#maxFailedJobs === 0 || !client.ltrim) return;
|
|
495
|
+
await client.ltrim(this.#failedKey(), -this.#maxFailedJobs, -1);
|
|
496
|
+
}
|
|
497
|
+
|
|
254
498
|
/**
|
|
255
499
|
* Remove the entry for `job` from the processing list. The string in
|
|
256
500
|
* Redis is whatever pop() pushed, but QueueManager mutates `job` after
|
|
257
501
|
* pop returns (attempts++, status="processing", processedAt, then
|
|
258
502
|
* completed/failed/pending). LREM-ing on `JSON.stringify(job)` would
|
|
259
|
-
* therefore miss every real-world entry. Use the lease —
|
|
260
|
-
* exact raw string
|
|
503
|
+
* therefore miss every real-world entry. Use the lease — which carries
|
|
504
|
+
* the exact raw string pop() moved — and fall back to a list scan when
|
|
261
505
|
* the lease has expired (e.g. recoverStale already handled it).
|
|
262
506
|
*/
|
|
263
|
-
async #removeFromProcessing(job:
|
|
507
|
+
async #removeFromProcessing(job: JobRecord): Promise<void> {
|
|
264
508
|
const client = await this.#client();
|
|
265
509
|
const stored = await client.get(this.#leaseKey(job.id));
|
|
266
510
|
if (stored !== null) {
|
|
267
|
-
const
|
|
511
|
+
const { raw } = readLease(stored);
|
|
512
|
+
const removed = await client.lrem(this.#processingKey(), 1, raw);
|
|
268
513
|
if (removed > 0) return;
|
|
269
514
|
}
|
|
270
515
|
// Lease missing or already-LREM'd entry not found — best-effort scan
|
|
@@ -277,14 +522,14 @@ export class RedisDriver implements QueueDriver {
|
|
|
277
522
|
} catch {
|
|
278
523
|
continue;
|
|
279
524
|
}
|
|
280
|
-
if (isValidJob(parsed) &&
|
|
525
|
+
if (isValidJob(parsed) && parsed.id === job.id) {
|
|
281
526
|
await client.lrem(this.#processingKey(), 1, item);
|
|
282
527
|
return;
|
|
283
528
|
}
|
|
284
529
|
}
|
|
285
530
|
}
|
|
286
531
|
|
|
287
|
-
async failed(): Promise<
|
|
532
|
+
async failed(): Promise<JobRecord[]> {
|
|
288
533
|
const client = await this.#client();
|
|
289
534
|
const raws = await client.lrange(this.#failedKey(), 0, -1);
|
|
290
535
|
return raws
|
|
@@ -296,11 +541,69 @@ export class RedisDriver implements QueueDriver {
|
|
|
296
541
|
return null;
|
|
297
542
|
}
|
|
298
543
|
})
|
|
299
|
-
.filter((j): j is
|
|
544
|
+
.filter((j): j is JobRecord => j !== null);
|
|
300
545
|
}
|
|
301
546
|
|
|
302
|
-
|
|
547
|
+
/**
|
|
548
|
+
* How many jobs are waiting on `queue` — its list plus its delayed set.
|
|
549
|
+
*
|
|
550
|
+
* A delayed job is queued; it is simply not due. Counting only the list
|
|
551
|
+
* reported an empty queue to anything draining one before shutdown.
|
|
552
|
+
*/
|
|
553
|
+
async size(queue: string = DEFAULT_QUEUE): Promise<number> {
|
|
303
554
|
const client = await this.#client();
|
|
304
|
-
|
|
555
|
+
const waiting = await client.llen(this.#pendingKey(queue));
|
|
556
|
+
const delayed = client.zcard
|
|
557
|
+
? await client.zcard(this.#delayedKey(queue))
|
|
558
|
+
: 0;
|
|
559
|
+
return waiting + delayed;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Move `queue`'s due jobs out of the delayed set and onto its list.
|
|
564
|
+
*
|
|
565
|
+
* `ZREM` is the claim: two workers can read the same due entry, and only
|
|
566
|
+
* the one whose removal returns 1 owns it. Without that the job is pushed
|
|
567
|
+
* onto the list once per worker that saw it.
|
|
568
|
+
*/
|
|
569
|
+
async #promoteDue(client: RedisClient, queue: string): Promise<void> {
|
|
570
|
+
if (!client.zrangebyscore || !client.zrem) return;
|
|
571
|
+
const due = await client.zrangebyscore(
|
|
572
|
+
this.#delayedKey(queue),
|
|
573
|
+
0,
|
|
574
|
+
Date.now(),
|
|
575
|
+
"LIMIT",
|
|
576
|
+
"0",
|
|
577
|
+
"100",
|
|
578
|
+
);
|
|
579
|
+
for (const raw of due) {
|
|
580
|
+
const claimed = await client.zrem(this.#delayedKey(queue), raw);
|
|
581
|
+
if (claimed !== 1) continue;
|
|
582
|
+
let parsed: unknown;
|
|
583
|
+
try {
|
|
584
|
+
parsed = JSON.parse(raw);
|
|
585
|
+
} catch {
|
|
586
|
+
// Already removed from the set; there is nothing runnable to push.
|
|
587
|
+
continue;
|
|
588
|
+
}
|
|
589
|
+
if (!isValidJob(parsed)) continue;
|
|
590
|
+
parsed.runAt = undefined;
|
|
591
|
+
await client.rpush(this.#pendingKey(queue), JSON.stringify(parsed));
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/** Take the head of one queue, claiming it in `processing`. */
|
|
596
|
+
async #take(client: RedisClient, queue: string): Promise<string | null> {
|
|
597
|
+
if (client.lmove) {
|
|
598
|
+
return client.lmove(
|
|
599
|
+
this.#pendingKey(queue),
|
|
600
|
+
this.#processingKey(),
|
|
601
|
+
"LEFT",
|
|
602
|
+
"RIGHT",
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
const raw = await client.lpop(this.#pendingKey(queue));
|
|
606
|
+
if (raw) await client.rpush(this.#processingKey(), raw);
|
|
607
|
+
return raw;
|
|
305
608
|
}
|
|
306
609
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,17 +1,47 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @c9up/bay — Background job queue for the Ream framework.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* A job is a class: it carries its own name, its options and the type of the
|
|
5
|
+
* payload it reads, and `dispatch(SomeJob, payload)` takes it. Named queues,
|
|
6
|
+
* `delay`, `timeout` and worker `concurrency` come from the same declaration.
|
|
5
7
|
*
|
|
6
|
-
*
|
|
8
|
+
* Dispatch/process/retry/fail, with pluggable drivers (Memory, Redis).
|
|
7
9
|
*/
|
|
8
10
|
|
|
9
|
-
export
|
|
11
|
+
export {
|
|
12
|
+
type AdapterFactory,
|
|
13
|
+
drivers,
|
|
14
|
+
type QueueStoreFactory,
|
|
15
|
+
stores,
|
|
16
|
+
} from "./adapters.js";
|
|
17
|
+
|
|
18
|
+
import "./augmentations.js";
|
|
19
|
+
|
|
20
|
+
export type {
|
|
21
|
+
BayAppContext,
|
|
22
|
+
BayConfigStore,
|
|
23
|
+
BayContainer,
|
|
24
|
+
BayProviderConfig,
|
|
25
|
+
} from "./BayProvider.js";
|
|
10
26
|
export { MemoryDriver } from "./drivers/MemoryDriver.js";
|
|
11
27
|
export type { RedisClient } from "./drivers/RedisDriver.js";
|
|
12
28
|
export { RedisDriver } from "./drivers/RedisDriver.js";
|
|
13
|
-
export
|
|
14
|
-
|
|
29
|
+
export {
|
|
30
|
+
DEFAULT_QUEUE,
|
|
31
|
+
type Duration,
|
|
32
|
+
isJobClass,
|
|
33
|
+
Job,
|
|
34
|
+
type JobClass,
|
|
35
|
+
type JobOptions,
|
|
36
|
+
} from "./Job.js";
|
|
37
|
+
export type {
|
|
38
|
+
DispatchOptions,
|
|
39
|
+
JobHandler,
|
|
40
|
+
JobRecord,
|
|
41
|
+
QueueDriver,
|
|
42
|
+
WorkerOptions,
|
|
43
|
+
} from "./QueueManager.js";
|
|
44
|
+
export { QueueManager, queueOf } from "./QueueManager.js";
|
|
15
45
|
|
|
16
46
|
import type { BayProviderConfig } from "./BayProvider.js";
|
|
17
47
|
|