@oneunit/redis 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +422 -0
- package/CHANGELOG.md +186 -0
- package/CONTRIBUTING.md +353 -0
- package/LICENSE +21 -0
- package/README.md +760 -2
- package/dist/client/check.d.ts +19 -0
- package/dist/client/check.d.ts.map +1 -0
- package/dist/client/check.js +44 -0
- package/dist/client/check.js.map +1 -0
- package/dist/client/client.d.ts +10 -0
- package/dist/client/client.d.ts.map +1 -0
- package/dist/client/client.js +25 -0
- package/dist/client/client.js.map +1 -0
- package/dist/client/events.d.ts +5 -0
- package/dist/client/events.d.ts.map +1 -0
- package/dist/client/events.js +66 -0
- package/dist/client/events.js.map +1 -0
- package/dist/client/index.d.ts +6 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +5 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client/shutdown.d.ts +4 -0
- package/dist/client/shutdown.d.ts.map +1 -0
- package/dist/client/shutdown.js +78 -0
- package/dist/client/shutdown.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +23 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +120 -0
- package/dist/logger.js.map +1 -0
- package/dist/pipeline/builder.d.ts +111 -0
- package/dist/pipeline/builder.d.ts.map +1 -0
- package/dist/pipeline/builder.js +197 -0
- package/dist/pipeline/builder.js.map +1 -0
- package/dist/pipeline/index.d.ts +3 -0
- package/dist/pipeline/index.d.ts.map +1 -0
- package/dist/pipeline/index.js +2 -0
- package/dist/pipeline/index.js.map +1 -0
- package/dist/queue/events.d.ts +13 -0
- package/dist/queue/events.d.ts.map +1 -0
- package/dist/queue/events.js +109 -0
- package/dist/queue/events.js.map +1 -0
- package/dist/queue/index.d.ts +7 -0
- package/dist/queue/index.d.ts.map +1 -0
- package/dist/queue/index.js +4 -0
- package/dist/queue/index.js.map +1 -0
- package/dist/queue/queue.d.ts +13 -0
- package/dist/queue/queue.d.ts.map +1 -0
- package/dist/queue/queue.js +37 -0
- package/dist/queue/queue.js.map +1 -0
- package/dist/queue/worker.d.ts +15 -0
- package/dist/queue/worker.d.ts.map +1 -0
- package/dist/queue/worker.js +20 -0
- package/dist/queue/worker.js.map +1 -0
- package/examples/README.md +86 -0
- package/examples/_setup.js +143 -0
- package/examples/cache.js +111 -0
- package/examples/pipeline.js +161 -0
- package/examples/pubsub.js +101 -0
- package/examples/queue-worker.js +189 -0
- package/examples/session.js +145 -0
- package/examples/standalone.js +58 -0
- package/package.json +100 -4
- package/src/client/check.ts +69 -0
- package/src/client/client.ts +45 -0
- package/src/client/events.ts +101 -0
- package/src/client/index.ts +5 -0
- package/src/client/shutdown.ts +97 -0
- package/src/index.ts +4 -0
- package/src/logger.ts +159 -0
- package/src/pipeline/builder.ts +307 -0
- package/src/pipeline/index.ts +7 -0
- package/src/queue/events.ts +158 -0
- package/src/queue/index.ts +6 -0
- package/src/queue/queue.ts +60 -0
- package/src/queue/worker.ts +44 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import {} from "ioredis";
|
|
2
|
+
import { normalizeLogger } from "../logger.js";
|
|
3
|
+
import { redactError } from "../client/events.js";
|
|
4
|
+
const DEFAULT_TIMEOUT_MS = 5000;
|
|
5
|
+
/** Whether a value is promise-like, i.e. something a caller forgot to await. */
|
|
6
|
+
function isThenable(value) {
|
|
7
|
+
return (typeof value === "object" &&
|
|
8
|
+
value !== null &&
|
|
9
|
+
typeof value.then === "function");
|
|
10
|
+
}
|
|
11
|
+
export class PipelineTimeoutError extends Error {
|
|
12
|
+
steps;
|
|
13
|
+
constructor(timeout, steps) {
|
|
14
|
+
super(`Redis pipeline of ${steps} command${steps === 1 ? "" : "s"} did not complete within ${timeout}ms`);
|
|
15
|
+
this.name = "PipelineTimeoutError";
|
|
16
|
+
this.steps = steps;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Thrown when `throwOnError` is set and a command failed.
|
|
21
|
+
*
|
|
22
|
+
* Carries the per-step results so the caller can see which command broke
|
|
23
|
+
* without re-running the batch. The `message` lists labels only — ioredis
|
|
24
|
+
* attaches command arguments to its errors, and a pipeline that batched an
|
|
25
|
+
* `AUTH` would otherwise put the password in an exception message.
|
|
26
|
+
*/
|
|
27
|
+
export class PipelineCommandError extends Error {
|
|
28
|
+
results;
|
|
29
|
+
constructor(results) {
|
|
30
|
+
const failed = results.filter((result) => result.error);
|
|
31
|
+
super(`Redis pipeline: ${failed.length} of ${results.length} commands failed (${failed
|
|
32
|
+
.map((result) => result.label)
|
|
33
|
+
.join(", ")})`);
|
|
34
|
+
this.name = "PipelineCommandError";
|
|
35
|
+
this.results = results;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Thrown when a step does not queue exactly one command.
|
|
40
|
+
*
|
|
41
|
+
* `PipelineStep.run` is typed as returning `void`, and TypeScript allows any
|
|
42
|
+
* value to be returned from a `void` signature — so a step that queues two
|
|
43
|
+
* commands, or an `async` step whose command is only queued after `runPipeline`
|
|
44
|
+
* has already called `exec`, compiles without complaint. Both shift the
|
|
45
|
+
* positional pairing between `steps` and the result tuples, which is the one
|
|
46
|
+
* thing the label exists to prevent, so the batch is rejected rather than
|
|
47
|
+
* returned with a value against the wrong label.
|
|
48
|
+
*/
|
|
49
|
+
export class PipelineStepError extends Error {
|
|
50
|
+
label;
|
|
51
|
+
queued;
|
|
52
|
+
constructor(label, queued) {
|
|
53
|
+
super(`Redis pipeline step "${label}" queued ${queued} command${queued === 1 ? "" : "s"}, expected exactly 1`);
|
|
54
|
+
this.name = "PipelineStepError";
|
|
55
|
+
this.label = label;
|
|
56
|
+
this.queued = queued;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Run a batch of commands in one round trip.
|
|
61
|
+
*
|
|
62
|
+
* Wraps `client.pipeline()` rather than reimplementing it. The value is in the
|
|
63
|
+
* four sharp edges this closes, all of which are silent in ioredis itself:
|
|
64
|
+
*
|
|
65
|
+
* 1. **A failed command does not fail the pipeline.** EXEC resolves with a
|
|
66
|
+
* `[error, null]` tuple for the command that failed. Code that reads
|
|
67
|
+
* `results.map(([, value]) => value)` gets `null` and carries on as if the
|
|
68
|
+
* write landed. Every result here carries an explicit `error` field.
|
|
69
|
+
* 2. **EXEC can hang forever.** ioredis parks queued commands while
|
|
70
|
+
* reconnecting. `timeout` bounds that.
|
|
71
|
+
* 3. **Command errors carry their arguments.** Redacted via `redactError`
|
|
72
|
+
* before they reach a logger or an exception — on both the per-command
|
|
73
|
+
* results and a rejected `exec()`.
|
|
74
|
+
* 4. **Results are positional.** Each step must queue exactly one command, or
|
|
75
|
+
* every later label is paired with the wrong value. A step that does not
|
|
76
|
+
* raises `PipelineStepError`.
|
|
77
|
+
*/
|
|
78
|
+
export async function runPipeline(client, steps, options = {}) {
|
|
79
|
+
const logger = normalizeLogger(options.logger);
|
|
80
|
+
const requested = options.timeout ?? DEFAULT_TIMEOUT_MS;
|
|
81
|
+
// setTimeout coerces a negative or NaN budget to 1ms and prints a warning for
|
|
82
|
+
// each; treat any invalid value as "no bound configured" instead.
|
|
83
|
+
const timeout = Number.isFinite(requested) && requested > 0
|
|
84
|
+
? requested
|
|
85
|
+
: DEFAULT_TIMEOUT_MS;
|
|
86
|
+
if (steps.length === 0) {
|
|
87
|
+
// No round trip is needed, and building a pipeline just to exec an empty
|
|
88
|
+
// one is a wasted allocation.
|
|
89
|
+
return { results: [], durationMs: 0, failed: 0 };
|
|
90
|
+
}
|
|
91
|
+
const pipeline = client.pipeline();
|
|
92
|
+
for (const step of steps) {
|
|
93
|
+
// Results are paired with steps by position, so the queue has to grow by
|
|
94
|
+
// exactly one per step. `length` is ioredis's own queue length, and it is
|
|
95
|
+
// read before and after rather than counted here so the check is against
|
|
96
|
+
// what was actually queued, not against what `run` appears to have done.
|
|
97
|
+
const before = pipeline.length;
|
|
98
|
+
let returned;
|
|
99
|
+
try {
|
|
100
|
+
returned = step.run(pipeline);
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
// A throwing `run` means the command was never queued, so the batch is
|
|
104
|
+
// already malformed. Fail the whole pipeline: silently dropping the
|
|
105
|
+
// command would shift every later result by one and return the wrong
|
|
106
|
+
// value against the wrong label.
|
|
107
|
+
logger?.error(`Redis pipeline step failed to queue: ${step.label}`, {
|
|
108
|
+
err: redactError(error),
|
|
109
|
+
});
|
|
110
|
+
throw error;
|
|
111
|
+
}
|
|
112
|
+
const queued = pipeline.length - before;
|
|
113
|
+
if (isThenable(returned)) {
|
|
114
|
+
// An `async` step is invisible to the count below: it queues nothing
|
|
115
|
+
// synchronously, so `exec()` runs first and the command it was going to
|
|
116
|
+
// queue lands in a pipeline that has already been sent. TypeScript
|
|
117
|
+
// permits returning a value from a `void`-typed signature, so this is
|
|
118
|
+
// not a compile error either.
|
|
119
|
+
//
|
|
120
|
+
// The step's promise is abandoned here, so its eventual rejection is
|
|
121
|
+
// nobody's to handle — which on Node 20 is a process-level crash that
|
|
122
|
+
// would mask the error naming the actual fault. Swallow it; the throw
|
|
123
|
+
// below is the real diagnosis.
|
|
124
|
+
void Promise.resolve(returned).catch(() => undefined);
|
|
125
|
+
logger?.error(`Redis pipeline step returned a promise instead of queuing: ${step.label}`);
|
|
126
|
+
throw new PipelineStepError(step.label, queued);
|
|
127
|
+
}
|
|
128
|
+
if (queued !== 1) {
|
|
129
|
+
logger?.error(`Redis pipeline step queued ${queued} commands: ${step.label}`);
|
|
130
|
+
throw new PipelineStepError(step.label, queued);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const started = performance.now();
|
|
134
|
+
let timer;
|
|
135
|
+
const expiry = new Promise((resolve) => {
|
|
136
|
+
timer = setTimeout(() => resolve("timeout"), timeout);
|
|
137
|
+
});
|
|
138
|
+
let raw;
|
|
139
|
+
try {
|
|
140
|
+
const outcome = await Promise.race([pipeline.exec(), expiry]);
|
|
141
|
+
if (outcome === "timeout") {
|
|
142
|
+
throw new PipelineTimeoutError(timeout, steps.length);
|
|
143
|
+
}
|
|
144
|
+
raw = outcome;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (error instanceof PipelineTimeoutError) {
|
|
148
|
+
throw error;
|
|
149
|
+
}
|
|
150
|
+
// A rejected `exec()` is a connection-level failure, and it bypasses the
|
|
151
|
+
// per-tuple redaction below entirely. The module documents that errors are
|
|
152
|
+
// redacted, so the rejection goes through the same path rather than
|
|
153
|
+
// reaching the caller — and from there a logger — as ioredis built it.
|
|
154
|
+
throw redactError(error);
|
|
155
|
+
}
|
|
156
|
+
finally {
|
|
157
|
+
// A pending timeout keeps the event loop alive. Always clear it, including
|
|
158
|
+
// on the timeout path where it has already fired.
|
|
159
|
+
clearTimeout(timer);
|
|
160
|
+
}
|
|
161
|
+
const results = raw.map(([error, value], index) => {
|
|
162
|
+
const step = steps[index];
|
|
163
|
+
return {
|
|
164
|
+
label: step?.label ?? `#${index}`,
|
|
165
|
+
value,
|
|
166
|
+
// Redacted here rather than at each call site so a caller that logs a
|
|
167
|
+
// whole result array cannot leak credentials by accident.
|
|
168
|
+
...(error ? { error: redactError(error) } : {}),
|
|
169
|
+
};
|
|
170
|
+
});
|
|
171
|
+
const failed = results.filter((result) => result.error).length;
|
|
172
|
+
const durationMs = Math.round(performance.now() - started);
|
|
173
|
+
if (failed > 0) {
|
|
174
|
+
logger?.warn(`Redis pipeline: ${failed} of ${results.length} commands failed`);
|
|
175
|
+
}
|
|
176
|
+
logger?.debug?.(`Redis pipeline completed ${results.length} command${results.length === 1 ? "" : "s"} in ${durationMs}ms`);
|
|
177
|
+
if (failed > 0 && options.throwOnError) {
|
|
178
|
+
throw new PipelineCommandError(results);
|
|
179
|
+
}
|
|
180
|
+
return { results, durationMs, failed };
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Values of the successful results, in order.
|
|
184
|
+
*
|
|
185
|
+
* Throws if any command failed, because the alternative is handing back a
|
|
186
|
+
* sparse array whose length matches the batch but whose contents silently
|
|
187
|
+
* include a failed write. Use `results` directly when a partial batch is
|
|
188
|
+
* expected and worth handling.
|
|
189
|
+
*/
|
|
190
|
+
export function pipelineValues(results) {
|
|
191
|
+
const failed = results.find((result) => result.error);
|
|
192
|
+
if (failed) {
|
|
193
|
+
throw new PipelineCommandError(results);
|
|
194
|
+
}
|
|
195
|
+
return results.map((result) => result.value);
|
|
196
|
+
}
|
|
197
|
+
//# sourceMappingURL=builder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"builder.js","sourceRoot":"","sources":["../../src/pipeline/builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsD,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAe,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAqDlD,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,gFAAgF;AAChF,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA4B,CAAC,IAAI,KAAK,UAAU,CACzD,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,KAAK,CAAS;IAEvB,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CACH,qBAAqB,KAAK,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,4BAA4B,OAAO,IAAI,CACnG,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,OAAO,CAAuB;IAEvC,YAAY,OAA6B;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,KAAK,CACH,mBAAmB,MAAM,CAAC,MAAM,OAAO,OAAO,CAAC,MAAM,qBAAqB,MAAM;aAC7E,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CACjB,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;QACnC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACjC,KAAK,CAAS;IACd,MAAM,CAAS;IAExB,YAAY,KAAa,EAAE,MAAc;QACvC,KAAK,CACH,wBAAwB,KAAK,YAAY,MAAM,WAC7C,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GACtB,sBAAsB,CACvB,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAmB,EACnB,KAAqB,EACrB,UAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,IAAI,kBAAkB,CAAC;IACxD,8EAA8E;IAC9E,kEAAkE;IAClE,MAAM,OAAO,GACX,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC;QACzC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,kBAAkB,CAAC;IAEzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,yEAAyE;QACzE,8BAA8B;QAC9B,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,yEAAyE;QACzE,0EAA0E;QAC1E,yEAAyE;QACzE,yEAAyE;QACzE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAE/B,IAAI,QAAiB,CAAC;QAEtB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,uEAAuE;YACvE,oEAAoE;YACpE,qEAAqE;YACrE,iCAAiC;YACjC,MAAM,EAAE,KAAK,CAAC,wCAAwC,IAAI,CAAC,KAAK,EAAE,EAAE;gBAClE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC;aACxB,CAAC,CAAC;YAEH,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;QAExC,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzB,qEAAqE;YACrE,wEAAwE;YACxE,mEAAmE;YACnE,sEAAsE;YACtE,8BAA8B;YAC9B,EAAE;YACF,qEAAqE;YACrE,sEAAsE;YACtE,sEAAsE;YACtE,+BAA+B;YAC/B,KAAK,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAEtD,MAAM,EAAE,KAAK,CACX,8DAA8D,IAAI,CAAC,KAAK,EAAE,CAC3E,CAAC;YAEF,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,KAAK,CACX,8BAA8B,MAAM,cAAc,IAAI,CAAC,KAAK,EAAE,CAC/D,CAAC;YAEF,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAElC,IAAI,KAAiC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QAChD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,IAAI,GAAmC,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QAE9D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,oBAAoB,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACxD,CAAC;QAED,GAAG,GAAG,OAAyC,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;YAC1C,MAAM,KAAK,CAAC;QACd,CAAC;QAED,yEAAyE;QACzE,2EAA2E;QAC3E,oEAAoE;QACpE,uEAAuE;QACvE,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,2EAA2E;QAC3E,kDAAkD;QAClD,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,OAAO,GAAyB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE;QACtE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1B,OAAO;YACL,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,KAAK,EAAE;YACjC,KAAK;YACL,sEAAsE;YACtE,0DAA0D;YAC1D,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,CAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;IAE3D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,MAAM,EAAE,IAAI,CACV,mBAAmB,MAAM,OAAO,OAAO,CAAC,MAAM,kBAAkB,CACjE,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,CACb,4BAA4B,OAAO,CAAC,MAAM,WACxC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAC9B,OAAO,UAAU,IAAI,CACtB,CAAC;IAEF,IAAI,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACvC,MAAM,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACzC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,OAA6B;IAC1D,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtD,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,IAAI,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/C,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,YAAY,EACV,YAAY,EACZ,eAAe,EACf,cAAc,EACd,kBAAkB,GACnB,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { QueueEvents, type QueueEventsOptions } from "bullmq";
|
|
2
|
+
import { Queue } from "bullmq";
|
|
3
|
+
import type { Logger } from "../logger.js";
|
|
4
|
+
export interface QueueEventsConfig {
|
|
5
|
+
queue: Queue;
|
|
6
|
+
logger?: Logger;
|
|
7
|
+
prefix?: string;
|
|
8
|
+
connection?: QueueEventsOptions["connection"];
|
|
9
|
+
}
|
|
10
|
+
export declare function attachQueueEvents(config: QueueEventsConfig): QueueEvents;
|
|
11
|
+
export type { QueueEventsOptions } from "bullmq";
|
|
12
|
+
export { QueueEvents } from "bullmq";
|
|
13
|
+
//# sourceMappingURL=events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../../src/queue/events.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,WAAW,EACX,KAAK,kBAAkB,EACxB,MAAM,QAAQ,CAAC;AAChB,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAK3C,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,kBAAkB,CAAC,YAAY,CAAC,CAAC;CAC/C;AAmFD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,GAAG,WAAW,CAuDxE;AAED,YAAY,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { ConnectionClosedError, QueueEvents, } from "bullmq";
|
|
2
|
+
import { Queue } from "bullmq";
|
|
3
|
+
import { normalizeLogger } from "../logger.js";
|
|
4
|
+
import { redactError } from "../client/events.js";
|
|
5
|
+
/**
|
|
6
|
+
* QueueEvents that can be closed after a failed startup.
|
|
7
|
+
*
|
|
8
|
+
* BullMQ's own `close()` awaits `this.client` before disconnecting, and that
|
|
9
|
+
* getter resolves to the connection's `initializing` promise. When the
|
|
10
|
+
* connection never became ready that promise has already rejected with
|
|
11
|
+
* "Connection is closed.", so `close()` throws before it reaches
|
|
12
|
+
* `connection.close()` and the duplicated ioredis client is left running its
|
|
13
|
+
* reconnect loop. Nothing else in the process can stop it, because the caller
|
|
14
|
+
* has no reference to the duplicate, so the process never exits.
|
|
15
|
+
*
|
|
16
|
+
* `Queue` does not have this problem: `RedisConnection.close()` handles
|
|
17
|
+
* `status === "initializing"` itself.
|
|
18
|
+
*
|
|
19
|
+
* Disconnecting the duplicate first is safe in both directions. When the
|
|
20
|
+
* connection is healthy, `disconnect()` ends it and the subsequent
|
|
21
|
+
* `connection.close()` sees a client already at `end` and skips its own quit.
|
|
22
|
+
*/
|
|
23
|
+
class ManagedQueueEvents extends QueueEvents {
|
|
24
|
+
async close() {
|
|
25
|
+
const connection = this.connection;
|
|
26
|
+
// A duplicate that is already gone does not need disconnecting, and
|
|
27
|
+
// `disconnect()` is a no-op at `end`.
|
|
28
|
+
connection._client?.disconnect();
|
|
29
|
+
try {
|
|
30
|
+
await super.close();
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
// `super.close()` propagates the rejection from the failed startup
|
|
34
|
+
// instead of reporting that the connection is now closed. The connection
|
|
35
|
+
// itself does know how to close from `initializing`, so drive it directly
|
|
36
|
+
// and only rethrow errors that are not about the connection being gone.
|
|
37
|
+
await connection.close(true).catch(() => undefined);
|
|
38
|
+
if (!isConnectionGone(error)) {
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* True for the "this connection is already gone" family of errors.
|
|
46
|
+
*
|
|
47
|
+
* `ConnectionClosedError` is checked first and structurally, because BullMQ
|
|
48
|
+
* introduced it for exactly this reason — its own comment on the class says it
|
|
49
|
+
* exists so `isNotConnectionError` can "do a structural `instanceof` check
|
|
50
|
+
* rather than fragile message-substring matching". Matching the message cannot
|
|
51
|
+
* work here: only some of BullMQ's construction sites pass ioredis's
|
|
52
|
+
* `CONNECTION_CLOSED_ERROR_MSG`, and the others pass their own wording or no
|
|
53
|
+
* message at all, in which case the class default (`"Connection is closed"`,
|
|
54
|
+
* with no trailing period) applies. An exact string comparison therefore
|
|
55
|
+
* rethrows precisely the failures this function exists to absorb, and the
|
|
56
|
+
* caller gets an exception from teardown instead of a clean close.
|
|
57
|
+
*
|
|
58
|
+
* The string clauses stay as a fallback: they still cover a `bullmq` error that
|
|
59
|
+
* predates the class, and an error forwarded from another adapter. `instanceof`
|
|
60
|
+
* is identity-based, so a consumer with two copies of `bullmq` in one tree
|
|
61
|
+
* would miss the class check — the fallbacks catch the ioredis wording, and
|
|
62
|
+
* missing them is the safe direction to fail.
|
|
63
|
+
*/
|
|
64
|
+
function isConnectionGone(error) {
|
|
65
|
+
if (error instanceof ConnectionClosedError) {
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
if (!(error instanceof Error)) {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
return (error.message === "Connection is closed." ||
|
|
72
|
+
error.message.includes("ECONNREFUSED") ||
|
|
73
|
+
error.code === "ECONNREFUSED");
|
|
74
|
+
}
|
|
75
|
+
export function attachQueueEvents(config) {
|
|
76
|
+
const { queue, connection } = config;
|
|
77
|
+
// BullMQ swallows a throwing event listener and re-emits the failure as an
|
|
78
|
+
// "error" event, which then throws again and lands on console.error. A logger
|
|
79
|
+
// missing `info` would trigger that on every completed job.
|
|
80
|
+
const logger = normalizeLogger(config.logger);
|
|
81
|
+
// QueueEvents subscribes to a key derived from the prefix. Defaulting to a
|
|
82
|
+
// literal "queue" here silently dropped every event for a queue created
|
|
83
|
+
// with a custom prefix, so inherit the queue's own prefix instead.
|
|
84
|
+
const prefix = config.prefix ?? queue.opts.prefix ?? "queue";
|
|
85
|
+
const queueEvents = new ManagedQueueEvents(queue.name, {
|
|
86
|
+
prefix,
|
|
87
|
+
connection: connection ?? queue.opts.connection,
|
|
88
|
+
});
|
|
89
|
+
queueEvents.on("completed", (args, _id) => {
|
|
90
|
+
logger?.info(`Job ${args.jobId} completed`);
|
|
91
|
+
});
|
|
92
|
+
queueEvents.on("failed", (args, _id) => {
|
|
93
|
+
logger?.error(`Job ${args.jobId} failed`, args.failedReason);
|
|
94
|
+
});
|
|
95
|
+
queueEvents.on("progress", (args, _id) => {
|
|
96
|
+
logger?.info(`Job ${args.jobId} progress`, args.data);
|
|
97
|
+
});
|
|
98
|
+
queueEvents.on("error", (error) => {
|
|
99
|
+
// QueueEvents duplicates the caller's client, so it authenticates with the
|
|
100
|
+
// same password and BullMQ re-emits any AUTH failure here. ioredis attaches
|
|
101
|
+
// the failing command to that error, and for AUTH its args are the password
|
|
102
|
+
// in plaintext — logging it as-is would write the credential to the app's
|
|
103
|
+
// log on every reconnect attempt against a misconfigured server.
|
|
104
|
+
logger?.error(`Queue "${queue.name}" events error`, redactError(error));
|
|
105
|
+
});
|
|
106
|
+
return queueEvents;
|
|
107
|
+
}
|
|
108
|
+
export { QueueEvents } from "bullmq";
|
|
109
|
+
//# sourceMappingURL=events.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.js","sourceRoot":"","sources":["../../src/queue/events.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,WAAW,GAEZ,MAAM,QAAQ,CAAC;AAChB,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAG/B,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AASlD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,kBAAmB,SAAQ,WAAW;IACjC,KAAK,CAAC,KAAK;QAClB,MAAM,UAAU,GAAG,IAAI,CAAC,UAGvB,CAAC;QAEF,oEAAoE;QACpE,sCAAsC;QACtC,UAAU,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;QAEjC,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,mEAAmE;YACnE,yEAAyE;YACzE,0EAA0E;YAC1E,wEAAwE;YACxE,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAEpD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,YAAY,qBAAqB,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,CACL,KAAK,CAAC,OAAO,KAAK,uBAAuB;QACzC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;QACrC,KAA+B,CAAC,IAAI,KAAK,cAAc,CACzD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAyB;IACzD,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAErC,2EAA2E;IAC3E,8EAA8E;IAC9E,4DAA4D;IAC5D,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE9C,2EAA2E;IAC3E,wEAAwE;IACxE,mEAAmE;IACnE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC;IAE7D,MAAM,WAAW,GAAG,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE;QACrD,MAAM;QACN,UAAU,EAAE,UAAU,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU;KAChD,CAAC,CAAC;IAEH,WAAW,CAAC,EAAE,CACZ,WAAW,EACX,CACE,IAA2D,EAC3D,GAAW,EACX,EAAE;QACF,MAAM,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,YAAY,CAAC,CAAC;IAC9C,CAAC,CACF,CAAC;IAEF,WAAW,CAAC,EAAE,CACZ,QAAQ,EACR,CACE,IAA4D,EAC5D,GAAW,EACX,EAAE;QACF,MAAM,EAAE,KAAK,CAAC,OAAO,IAAI,CAAC,KAAK,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC/D,CAAC,CACF,CAAC;IAEF,WAAW,CAAC,EAAE,CACZ,UAAU,EACV,CAAC,IAAsC,EAAE,GAAW,EAAE,EAAE;QACtD,MAAM,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC,CACF,CAAC;IAEF,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;QACvC,2EAA2E;QAC3E,4EAA4E;QAC5E,4EAA4E;QAC5E,0EAA0E;QAC1E,iEAAiE;QACjE,MAAM,EAAE,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,gBAAgB,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC;AAGD,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export * from "./queue.js";
|
|
2
|
+
export * from "./worker.js";
|
|
3
|
+
export * from "./events.js";
|
|
4
|
+
export type { QueueConfig } from "./queue.js";
|
|
5
|
+
export type { WorkerConfig } from "./worker.js";
|
|
6
|
+
export type { QueueEventsConfig } from "./events.js";
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/queue/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/queue/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Queue, type QueueOptions, type JobsOptions } from "bullmq";
|
|
2
|
+
import { type Redis as RedisClient } from "ioredis";
|
|
3
|
+
export interface QueueConfig {
|
|
4
|
+
name: string;
|
|
5
|
+
connection: RedisClient;
|
|
6
|
+
prefix?: string;
|
|
7
|
+
defaultJobOptions?: JobsOptions;
|
|
8
|
+
settings?: QueueOptions["settings"];
|
|
9
|
+
}
|
|
10
|
+
export declare function createQueue(config: QueueConfig): Queue;
|
|
11
|
+
export type { QueueOptions, JobsOptions } from "bullmq";
|
|
12
|
+
export { Queue } from "bullmq";
|
|
13
|
+
//# sourceMappingURL=queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../../src/queue/queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,MAAM,QAAQ,CAAC;AACpE,OAAO,EAAE,KAAK,KAAK,IAAI,WAAW,EAAE,MAAM,SAAS,CAAC;AAEpD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,WAAW,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iBAAiB,CAAC,EAAE,WAAW,CAAC;IAChC,QAAQ,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;CACrC;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,KAAK,CA6CtD;AAED,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { Queue } from "bullmq";
|
|
2
|
+
import {} from "ioredis";
|
|
3
|
+
export function createQueue(config) {
|
|
4
|
+
const { name, connection, prefix = "queue", defaultJobOptions, settings, ...options } = config;
|
|
5
|
+
const defaults = {
|
|
6
|
+
removeOnComplete: 100,
|
|
7
|
+
removeOnFail: 1000,
|
|
8
|
+
attempts: 3,
|
|
9
|
+
backoff: {
|
|
10
|
+
type: "exponential",
|
|
11
|
+
delay: 1000,
|
|
12
|
+
},
|
|
13
|
+
};
|
|
14
|
+
// Merged key by key rather than by spreading. A spread writes `undefined` for
|
|
15
|
+
// every key the caller left unset, which erases the default instead of falling
|
|
16
|
+
// through to it — and that is how a config built by spreading another object
|
|
17
|
+
// (`{ ...base, attempts: maybeUndefined }`) quietly loses the package default.
|
|
18
|
+
// `null` is a deliberate value rather than a missing one, so it passes
|
|
19
|
+
// through: `removeOnComplete: null` is how BullMQ is told to keep a job.
|
|
20
|
+
const merged = { ...defaults };
|
|
21
|
+
for (const [key, value] of Object.entries(defaultJobOptions ?? {})) {
|
|
22
|
+
if (value !== undefined) {
|
|
23
|
+
merged[key] = value;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const queueOptions = {
|
|
27
|
+
prefix,
|
|
28
|
+
connection,
|
|
29
|
+
defaultJobOptions: merged,
|
|
30
|
+
...(settings === undefined ? {} : { settings }),
|
|
31
|
+
...options,
|
|
32
|
+
};
|
|
33
|
+
const queue = new Queue(name, queueOptions);
|
|
34
|
+
return queue;
|
|
35
|
+
}
|
|
36
|
+
export { Queue } from "bullmq";
|
|
37
|
+
//# sourceMappingURL=queue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../src/queue/queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAuC,MAAM,QAAQ,CAAC;AACpE,OAAO,EAA6B,MAAM,SAAS,CAAC;AAUpD,MAAM,UAAU,WAAW,CAAC,MAAmB;IAC7C,MAAM,EACJ,IAAI,EACJ,UAAU,EACV,MAAM,GAAG,OAAO,EAChB,iBAAiB,EACjB,QAAQ,EACR,GAAG,OAAO,EACX,GAAG,MAAM,CAAC;IAEX,MAAM,QAAQ,GAAgB;QAC5B,gBAAgB,EAAE,GAAG;QACrB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,CAAC;QACX,OAAO,EAAE;YACP,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,IAAI;SACZ;KACF,CAAC;IAEF,8EAA8E;IAC9E,+EAA+E;IAC/E,6EAA6E;IAC7E,+EAA+E;IAC/E,uEAAuE;IACvE,yEAAyE;IACzE,MAAM,MAAM,GAA4B,EAAE,GAAG,QAAQ,EAAE,CAAC;IAExD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,EAAE,CAAC;QACnE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAiB;QACjC,MAAM;QACN,UAAU;QACV,iBAAiB,EAAE,MAAqB;QACxC,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,GAAG,OAAO;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAE5C,OAAO,KAAK,CAAC;AACf,CAAC;AAGD,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { Worker, type WorkerOptions, type Processor } from "bullmq";
|
|
2
|
+
import { type Redis as RedisClient } from "ioredis";
|
|
3
|
+
export interface WorkerConfig<T = unknown> {
|
|
4
|
+
name: string;
|
|
5
|
+
processor: Processor<T>;
|
|
6
|
+
connection: RedisClient;
|
|
7
|
+
prefix?: string;
|
|
8
|
+
concurrency?: number;
|
|
9
|
+
limiter?: WorkerOptions["limiter"];
|
|
10
|
+
settings?: WorkerOptions["settings"];
|
|
11
|
+
}
|
|
12
|
+
export declare function createWorker<T = unknown>(config: WorkerConfig<T>): Worker<T>;
|
|
13
|
+
export type { WorkerOptions, Job, Processor } from "bullmq";
|
|
14
|
+
export { Worker } from "bullmq";
|
|
15
|
+
//# sourceMappingURL=worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/queue/worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAE,MAAM,QAAQ,CAAC;AACpE,OAAO,EAAE,KAAK,KAAK,IAAI,WAAW,EAAE,MAAM,SAAS,CAAC;AAEpD,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,OAAO;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IACxB,UAAU,EAAE,WAAW,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IACnC,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;CACtC;AAED,wBAAgB,YAAY,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CA2B5E;AAED,YAAY,EAAE,aAAa,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC5D,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Worker } from "bullmq";
|
|
2
|
+
import {} from "ioredis";
|
|
3
|
+
export function createWorker(config) {
|
|
4
|
+
const { name, processor, connection, prefix = "queue", concurrency, limiter, settings, ...options } = config;
|
|
5
|
+
// BullMQ only falls back to its own defaults for keys that are absent.
|
|
6
|
+
// Spreading an explicit `undefined` overrides `concurrency: 1` and trips
|
|
7
|
+
// its `concurrency must be a finite number greater than 0` setter.
|
|
8
|
+
const workerOptions = {
|
|
9
|
+
prefix,
|
|
10
|
+
connection,
|
|
11
|
+
...(concurrency === undefined ? {} : { concurrency }),
|
|
12
|
+
...(limiter === undefined ? {} : { limiter }),
|
|
13
|
+
...(settings === undefined ? {} : { settings }),
|
|
14
|
+
...options,
|
|
15
|
+
};
|
|
16
|
+
const worker = new Worker(name, processor, workerOptions);
|
|
17
|
+
return worker;
|
|
18
|
+
}
|
|
19
|
+
export { Worker } from "bullmq";
|
|
20
|
+
//# sourceMappingURL=worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/queue/worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAsC,MAAM,QAAQ,CAAC;AACpE,OAAO,EAA6B,MAAM,SAAS,CAAC;AAYpD,MAAM,UAAU,YAAY,CAAc,MAAuB;IAC/D,MAAM,EACJ,IAAI,EACJ,SAAS,EACT,UAAU,EACV,MAAM,GAAG,OAAO,EAChB,WAAW,EACX,OAAO,EACP,QAAQ,EACR,GAAG,OAAO,EACX,GAAG,MAAM,CAAC;IAEX,uEAAuE;IACvE,yEAAyE;IACzE,mEAAmE;IACnE,MAAM,aAAa,GAAkB;QACnC,MAAM;QACN,UAAU;QACV,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;QACrD,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;QAC7C,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,GAAG,OAAO;KACX,CAAC;IAEF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAI,IAAI,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;IAE7D,OAAO,MAAM,CAAC;AAChB,CAAC;AAGD,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Examples
|
|
2
|
+
|
|
3
|
+
Runnable scripts for `@oneunit/redis`. Each one is standalone and needs nothing
|
|
4
|
+
but a Redis server and the package built.
|
|
5
|
+
|
|
6
|
+
```bash
|
|
7
|
+
cd packages/redis
|
|
8
|
+
npm install
|
|
9
|
+
npm run build # the example scripts import the package by name, which
|
|
10
|
+
# resolves through the exports map to dist/
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Then start Redis (any 6.x or 7.x server) and run one:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
npm run example:standalone # client: health, read/write, TTL
|
|
17
|
+
npm run example:cache # read-through cache with TTL and invalidation
|
|
18
|
+
npm run example:session # session store: create, update, expire, delete
|
|
19
|
+
npm run example:pubsub # publish/subscribe across two clients
|
|
20
|
+
npm run example:queue-worker # BullMQ queue, worker, retries, queue events
|
|
21
|
+
npm run example:pipeline # batching many commands into one round trip
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
`example:queue-worker` finishes on its own once the demo jobs settle. The others
|
|
25
|
+
exit immediately.
|
|
26
|
+
|
|
27
|
+
## Environment
|
|
28
|
+
|
|
29
|
+
| Variable | Default | Purpose |
|
|
30
|
+
| :------------------- | :----------------------- | :------------------------------------------------- |
|
|
31
|
+
| `REDIS_URL` | `redis://localhost:6379` | Connection URL |
|
|
32
|
+
| `REDIS_SILENT` | unset | Set to `true` to suppress connection-event logging |
|
|
33
|
+
| `EXAMPLE_TIMEOUT_MS` | `5000` | Bound for the pub/sub subscription check |
|
|
34
|
+
|
|
35
|
+
## What each one covers
|
|
36
|
+
|
|
37
|
+
**`standalone.js`** — the client on its own. Health check, `SET`/`GET`, the
|
|
38
|
+
atomic `INCRBY`, and `EXPIRE`/`TTL`. Shows the defaults that make one client work
|
|
39
|
+
for both plain commands and BullMQ.
|
|
40
|
+
|
|
41
|
+
**`cache.js`** — read-through caching. The same read served in ~1ms from Redis
|
|
42
|
+
versus ~50ms from the "database", plus cache invalidation and why `SCAN` is
|
|
43
|
+
preferred over `KEYS`.
|
|
44
|
+
|
|
45
|
+
**`session.js`** — a session store. Create, read, update, extend, and destroy,
|
|
46
|
+
including the sliding-TTL pattern and how to refresh a TTL without rewriting the
|
|
47
|
+
payload.
|
|
48
|
+
|
|
49
|
+
**`pubsub.js`** — two clients, because a Redis connection in subscriber mode
|
|
50
|
+
cannot issue other commands. Also shows why you confirm the subscription count
|
|
51
|
+
before publishing: pub/sub does not queue for a subscriber that is not attached.
|
|
52
|
+
|
|
53
|
+
**`queue-worker.js`** — the full BullMQ surface. Job defaults from `createQueue`,
|
|
54
|
+
a worker with retries and exponential backoff (one job deliberately fails all
|
|
55
|
+
three attempts), and `attachQueueEvents` observing completion and failure over
|
|
56
|
+
its own connection. Shows the shutdown order.
|
|
57
|
+
|
|
58
|
+
**`pipeline.js`** — `runPipeline` for batch work. Writes and reads 100 keys in
|
|
59
|
+
one round trip, then deliberately includes an `INCR` on a string key so a single
|
|
60
|
+
command fails: the batch still resolves, and the example prints which command
|
|
61
|
+
failed. That is the case a bare `client.pipeline()` hides. Also shows
|
|
62
|
+
`pipelineValues` and `throwOnError`.
|
|
63
|
+
|
|
64
|
+
Note the shape of every step: one command, and the step returns nothing. That is
|
|
65
|
+
the contract, not a style preference — `runPipeline` matches results to steps by
|
|
66
|
+
position, so a step that queued two commands would report a real value against
|
|
67
|
+
the wrong label, and it raises `PipelineStepError` rather than letting that
|
|
68
|
+
through. Each entry here is a single `void pipeline.<cmd>(...)`.
|
|
69
|
+
|
|
70
|
+
## Things the examples avoid
|
|
71
|
+
|
|
72
|
+
Queue names contain no `:` — BullMQ rejects them, which is what stops one queue
|
|
73
|
+
addressing another's keys.
|
|
74
|
+
|
|
75
|
+
`attachQueueEvents` takes no `prefix` here so it inherits the queue's. Setting a
|
|
76
|
+
prefix on the queue and worker but not the listener is the easy way to end up
|
|
77
|
+
with a worker polling one key and a listener subscribed to another, receiving no
|
|
78
|
+
events and no error.
|
|
79
|
+
|
|
80
|
+
Pub/sub uses explicit `sleep` between publishes instead of nesting `setTimeout`
|
|
81
|
+
callbacks, so an error in one publish is caught rather than becoming an unhandled
|
|
82
|
+
rejection after the script has already claimed success.
|
|
83
|
+
|
|
84
|
+
Every example reports failure through `run()` in `_setup.js`, which sets a
|
|
85
|
+
non-zero exit code. `catch(console.error)` on its own leaves the process exiting
|
|
86
|
+
`0`, so a broken example passes any check that looks at the exit status.
|