@oneunit/redis 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +422 -0
- package/CHANGELOG.md +186 -0
- package/CONTRIBUTING.md +353 -0
- package/LICENSE +21 -0
- package/README.md +760 -2
- package/dist/client/check.d.ts +19 -0
- package/dist/client/check.d.ts.map +1 -0
- package/dist/client/check.js +44 -0
- package/dist/client/check.js.map +1 -0
- package/dist/client/client.d.ts +10 -0
- package/dist/client/client.d.ts.map +1 -0
- package/dist/client/client.js +25 -0
- package/dist/client/client.js.map +1 -0
- package/dist/client/events.d.ts +5 -0
- package/dist/client/events.d.ts.map +1 -0
- package/dist/client/events.js +66 -0
- package/dist/client/events.js.map +1 -0
- package/dist/client/index.d.ts +6 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +5 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client/shutdown.d.ts +4 -0
- package/dist/client/shutdown.d.ts.map +1 -0
- package/dist/client/shutdown.js +78 -0
- package/dist/client/shutdown.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +23 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +120 -0
- package/dist/logger.js.map +1 -0
- package/dist/pipeline/builder.d.ts +111 -0
- package/dist/pipeline/builder.d.ts.map +1 -0
- package/dist/pipeline/builder.js +197 -0
- package/dist/pipeline/builder.js.map +1 -0
- package/dist/pipeline/index.d.ts +3 -0
- package/dist/pipeline/index.d.ts.map +1 -0
- package/dist/pipeline/index.js +2 -0
- package/dist/pipeline/index.js.map +1 -0
- package/dist/queue/events.d.ts +13 -0
- package/dist/queue/events.d.ts.map +1 -0
- package/dist/queue/events.js +109 -0
- package/dist/queue/events.js.map +1 -0
- package/dist/queue/index.d.ts +7 -0
- package/dist/queue/index.d.ts.map +1 -0
- package/dist/queue/index.js +4 -0
- package/dist/queue/index.js.map +1 -0
- package/dist/queue/queue.d.ts +13 -0
- package/dist/queue/queue.d.ts.map +1 -0
- package/dist/queue/queue.js +37 -0
- package/dist/queue/queue.js.map +1 -0
- package/dist/queue/worker.d.ts +15 -0
- package/dist/queue/worker.d.ts.map +1 -0
- package/dist/queue/worker.js +20 -0
- package/dist/queue/worker.js.map +1 -0
- package/examples/README.md +86 -0
- package/examples/_setup.js +143 -0
- package/examples/cache.js +111 -0
- package/examples/pipeline.js +161 -0
- package/examples/pubsub.js +101 -0
- package/examples/queue-worker.js +189 -0
- package/examples/session.js +145 -0
- package/examples/standalone.js +58 -0
- package/package.json +100 -4
- package/src/client/check.ts +69 -0
- package/src/client/client.ts +45 -0
- package/src/client/events.ts +101 -0
- package/src/client/index.ts +5 -0
- package/src/client/shutdown.ts +97 -0
- package/src/index.ts +4 -0
- package/src/logger.ts +159 -0
- package/src/pipeline/builder.ts +307 -0
- package/src/pipeline/index.ts +7 -0
- package/src/queue/events.ts +158 -0
- package/src/queue/index.ts +6 -0
- package/src/queue/queue.ts +60 -0
- package/src/queue/worker.ts +44 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared setup for the examples in this directory.
|
|
3
|
+
*
|
|
4
|
+
* Every example needs the same three things: a Redis URL, a decision about
|
|
5
|
+
* whether to log connection events, and a clean shutdown. Keeping that here
|
|
6
|
+
* means each example file can be about its actual subject.
|
|
7
|
+
*
|
|
8
|
+
* These are run straight from a checkout (`npm run example:<name>`), so they
|
|
9
|
+
* import the package by name rather than by relative path. Node resolves that
|
|
10
|
+
* through the package's own `exports` map, which means the examples exercise
|
|
11
|
+
* the same entry points a consumer gets.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { silentLogger } from "@oneunit/redis";
|
|
15
|
+
|
|
16
|
+
export const REDIS_URL = process.env.REDIS_URL ?? "redis://localhost:6379";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Connection-event logging is noisy in a demo, so `REDIS_SILENT=true` turns it
|
|
20
|
+
* off. Passing `undefined` instead lets `createClient` skip the log calls while
|
|
21
|
+
* still registering the ioredis `error` listener, which is what keeps a
|
|
22
|
+
* transient outage from becoming an unhandled exception.
|
|
23
|
+
*/
|
|
24
|
+
export const exampleLogger =
|
|
25
|
+
process.env.REDIS_SILENT === "true" ? silentLogger : undefined;
|
|
26
|
+
|
|
27
|
+
/** Resolve after `ms`, used to space out publishes and let I/O settle. */
|
|
28
|
+
export function sleep(ms) {
|
|
29
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Resolve to `"timeout"` if `promise` has not settled within `ms`.
|
|
34
|
+
*
|
|
35
|
+
* Worth having in every example. ioredis queues commands while reconnecting and
|
|
36
|
+
* BullMQ waits on its own readiness, so against an unreachable server calls like
|
|
37
|
+
* `subscribe()` or `worker.waitUntilReady()` never settle at all. Without a
|
|
38
|
+
* bound the script hangs until someone kills it, which looks identical to a
|
|
39
|
+
* Redis that is merely slow.
|
|
40
|
+
*
|
|
41
|
+
* The timer is always cleared. A pending timeout keeps the event loop alive,
|
|
42
|
+
* so a losing race would silently add its full duration to the script's
|
|
43
|
+
* runtime.
|
|
44
|
+
*/
|
|
45
|
+
export async function within(promise, ms = 5000) {
|
|
46
|
+
let timer;
|
|
47
|
+
try {
|
|
48
|
+
return await Promise.race([
|
|
49
|
+
promise,
|
|
50
|
+
new Promise((resolve) => {
|
|
51
|
+
timer = setTimeout(() => resolve("timeout"), ms);
|
|
52
|
+
}),
|
|
53
|
+
]);
|
|
54
|
+
} finally {
|
|
55
|
+
clearTimeout(timer);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* True if `promise` settles successfully within `ms`, false on timeout or
|
|
61
|
+
* rejection.
|
|
62
|
+
*
|
|
63
|
+
* BullMQ readiness needs both guards. It rejects outright against a dead server
|
|
64
|
+
* ("Connection is closed") rather than hanging, so a timeout race alone is not
|
|
65
|
+
* enough. Letting that rejection escape would abort the example before it could
|
|
66
|
+
* shut down, and BullMQ's reconnect loops would then keep the process alive
|
|
67
|
+
* with nothing left to stop them.
|
|
68
|
+
*/
|
|
69
|
+
export async function isReady(promise, label, ms = 5000) {
|
|
70
|
+
try {
|
|
71
|
+
if ((await within(promise, ms)) === "timeout") {
|
|
72
|
+
console.log(` ${label} not ready within ${ms}ms.`);
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
} catch (error) {
|
|
76
|
+
console.log(` ${label} unavailable: ${error.message}`);
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Resources registered here, closed in reverse order when the example finishes.
|
|
84
|
+
*
|
|
85
|
+
* Every example creates its own clients, and ioredis keeps a socket open for
|
|
86
|
+
* each one. If `main()` returns early or throws partway through, whoever was
|
|
87
|
+
* going to call `shutdown` never gets to run, so the open socket keeps the event
|
|
88
|
+
* loop alive and the script hangs until it is killed from outside. That turns a
|
|
89
|
+
* clear stack trace or a tidy "Redis is not reachable" message into a timeout.
|
|
90
|
+
*
|
|
91
|
+
* Cleanup always runs, not only on failure. Every registered close is a no-op
|
|
92
|
+
* once the resource is already gone, so running them unconditionally covers the
|
|
93
|
+
* early-return paths without asking each example to repeat its own teardown.
|
|
94
|
+
*/
|
|
95
|
+
const cleanups = [];
|
|
96
|
+
|
|
97
|
+
/** Register `close` to run when the example ends. Later registrations close first. */
|
|
98
|
+
export function onFailure(close) {
|
|
99
|
+
cleanups.push(close);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Close everything registered so far, in reverse order.
|
|
104
|
+
*
|
|
105
|
+
* Draining the list matters for more than tidiness. `shutdown` waits out a 5s
|
|
106
|
+
* QUIT deadline against an unreachable server, so a resource closed here and
|
|
107
|
+
* then closed again by `run` would cost that deadline twice. Once released, it
|
|
108
|
+
* is out of the list and cannot be closed again.
|
|
109
|
+
*
|
|
110
|
+
* Call this when the example shuts down by hand, so its own "disconnected
|
|
111
|
+
* cleanly" message is only printed after the connection is really gone.
|
|
112
|
+
*/
|
|
113
|
+
export async function release() {
|
|
114
|
+
const pending = cleanups.splice(0).reverse();
|
|
115
|
+
|
|
116
|
+
for (const close of pending) {
|
|
117
|
+
try {
|
|
118
|
+
await close();
|
|
119
|
+
} catch {
|
|
120
|
+
// A cleanup that throws must not hide the original failure.
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Run an example's main function and make a failure visible in the exit code.
|
|
127
|
+
*
|
|
128
|
+
* `catch(console.error)` alone leaves the process exiting 0, so a broken example
|
|
129
|
+
* looks like a passing one in any script that checks `$?`.
|
|
130
|
+
*/
|
|
131
|
+
export async function run(name, main) {
|
|
132
|
+
try {
|
|
133
|
+
await main();
|
|
134
|
+
} catch (error) {
|
|
135
|
+
console.error(
|
|
136
|
+
`\n${name} failed:`,
|
|
137
|
+
error instanceof Error ? error.message : error,
|
|
138
|
+
);
|
|
139
|
+
process.exitCode = 1;
|
|
140
|
+
} finally {
|
|
141
|
+
await release();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-through cache: Redis in front of a slow "database".
|
|
3
|
+
*
|
|
4
|
+
* npm run example:cache
|
|
5
|
+
*
|
|
6
|
+
* Environment:
|
|
7
|
+
* REDIS_URL connection URL (default redis://localhost:6379)
|
|
8
|
+
* REDIS_SILENT set to "true" to suppress connection-event logging
|
|
9
|
+
*
|
|
10
|
+
* The point of the pattern is that the cache key, the TTL, and the
|
|
11
|
+
* serialisation all live in one place, so callers only ever see a plain object.
|
|
12
|
+
* The two failure modes worth knowing about are both shown below: a TTL that is
|
|
13
|
+
* too long serves stale data, and no TTL at all grows Redis without limit.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { createClient, health, shutdown } from "@oneunit/redis";
|
|
17
|
+
import { REDIS_URL, exampleLogger, onFailure, release, run } from "./_setup.js";
|
|
18
|
+
|
|
19
|
+
const CACHE_TTL_SECONDS = 60;
|
|
20
|
+
const KEY_PREFIX = "example:cache:";
|
|
21
|
+
|
|
22
|
+
await run("cache", async () => {
|
|
23
|
+
const client = createClient({ url: REDIS_URL }, exampleLogger);
|
|
24
|
+
onFailure(() => shutdown(client, exampleLogger));
|
|
25
|
+
|
|
26
|
+
const healthResult = await health(client);
|
|
27
|
+
if (healthResult.status === "down") {
|
|
28
|
+
// Always shut down, including on this early exit: ioredis retries in the
|
|
29
|
+
// background, so a client left open keeps the event loop alive and the
|
|
30
|
+
// script never exits.
|
|
31
|
+
console.log("Redis is not reachable, stopping here.");
|
|
32
|
+
console.log("Health:", healthResult);
|
|
33
|
+
await release();
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function fetchUserFromDatabase(userId) {
|
|
38
|
+
// Stands in for a real query. The delay is what makes the cache worth
|
|
39
|
+
// having and makes the timing difference visible below.
|
|
40
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
41
|
+
return {
|
|
42
|
+
id: userId,
|
|
43
|
+
name: `User ${userId}`,
|
|
44
|
+
email: `user${userId}@example.com`,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function getUser(userId) {
|
|
49
|
+
const key = `${KEY_PREFIX}user:${userId}`;
|
|
50
|
+
|
|
51
|
+
const cached = await client.get(key);
|
|
52
|
+
if (cached !== null) {
|
|
53
|
+
console.log(` HIT ${key}`);
|
|
54
|
+
return JSON.parse(cached);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
console.log(` MISS ${key}`);
|
|
58
|
+
const user = await fetchUserFromDatabase(userId);
|
|
59
|
+
|
|
60
|
+
// SETEX writes the value and its TTL in one round trip, so the key can
|
|
61
|
+
// never end up stored without an expiry if the process dies midway.
|
|
62
|
+
await client.setex(key, CACHE_TTL_SECONDS, JSON.stringify(user));
|
|
63
|
+
|
|
64
|
+
return user;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
console.log("First request:");
|
|
68
|
+
const started = Date.now();
|
|
69
|
+
const first = await getUser(123);
|
|
70
|
+
console.log(" ->", first, `(${Date.now() - started}ms, hit the database)`);
|
|
71
|
+
|
|
72
|
+
console.log("\nSecond request:");
|
|
73
|
+
const cachedStart = Date.now();
|
|
74
|
+
const second = await getUser(123);
|
|
75
|
+
console.log(
|
|
76
|
+
" ->",
|
|
77
|
+
second,
|
|
78
|
+
`(${Date.now() - cachedStart}ms, served from Redis)`,
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
const ttl = await client.ttl(`${KEY_PREFIX}user:123`);
|
|
82
|
+
console.log(`\nCached entry expires in ${ttl}s.`);
|
|
83
|
+
|
|
84
|
+
// A negative TTL means the key has no expiry set, which is the bug to avoid
|
|
85
|
+
// in production: it survives deploys and restarts indefinitely.
|
|
86
|
+
if (ttl === -1) {
|
|
87
|
+
console.log(" WARNING: key has no TTL; it will never expire on its own.");
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Cache invalidation: delete the key so the next read repopulates it.
|
|
91
|
+
await client.del(`${KEY_PREFIX}user:123`);
|
|
92
|
+
console.log("\nInvalidated the entry.");
|
|
93
|
+
|
|
94
|
+
// SCAN rather than KEYS. KEYS blocks the server for the length of the scan,
|
|
95
|
+
// which on a large keyspace is a production outage.
|
|
96
|
+
const cursor = "0";
|
|
97
|
+
const [next, keys] = await client.scan(
|
|
98
|
+
cursor,
|
|
99
|
+
"MATCH",
|
|
100
|
+
`${KEY_PREFIX}*`,
|
|
101
|
+
"COUNT",
|
|
102
|
+
100,
|
|
103
|
+
);
|
|
104
|
+
console.log(
|
|
105
|
+
`SCAN from ${cursor} returned cursor ${next} and ${keys.length} matching keys.`,
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
await client.del(`${KEY_PREFIX}user:123`);
|
|
109
|
+
await release();
|
|
110
|
+
console.log("\nDisconnected cleanly.");
|
|
111
|
+
});
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batching many commands into one round trip with `runPipeline`.
|
|
3
|
+
*
|
|
4
|
+
* npm run example:pipeline
|
|
5
|
+
*
|
|
6
|
+
* Environment:
|
|
7
|
+
* REDIS_URL connection URL (default redis://localhost:6379)
|
|
8
|
+
* REDIS_SILENT set to "true" to suppress connection-event logging
|
|
9
|
+
*
|
|
10
|
+
* The point of the pattern is that N commands cost one network round trip
|
|
11
|
+
* instead of N. That only pays off for a batch you already have in hand: if
|
|
12
|
+
* each command depends on the previous one's result, it is a sequential script
|
|
13
|
+
* and a pipeline cannot help.
|
|
14
|
+
*
|
|
15
|
+
* The failure worth seeing is at the bottom. A command that fails inside a
|
|
16
|
+
* pipeline does not fail the pipeline — ioredis resolves the batch and reports
|
|
17
|
+
* the error per command — so a batch can look successful while having silently
|
|
18
|
+
* dropped a write.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import {
|
|
22
|
+
createClient,
|
|
23
|
+
health,
|
|
24
|
+
pipelineValues,
|
|
25
|
+
runPipeline,
|
|
26
|
+
shutdown,
|
|
27
|
+
} from "@oneunit/redis";
|
|
28
|
+
import { REDIS_URL, exampleLogger, onFailure, release, run } from "./_setup.js";
|
|
29
|
+
|
|
30
|
+
const KEY_PREFIX = "example:pipeline:";
|
|
31
|
+
const BATCH_SIZE = 100;
|
|
32
|
+
|
|
33
|
+
await run("pipeline", async () => {
|
|
34
|
+
const client = createClient({ url: REDIS_URL }, exampleLogger);
|
|
35
|
+
onFailure(() => shutdown(client, exampleLogger));
|
|
36
|
+
|
|
37
|
+
const healthResult = await health(client);
|
|
38
|
+
if (healthResult.status === "down") {
|
|
39
|
+
// Shut down before returning: ioredis retries in the background, so a
|
|
40
|
+
// client left open keeps the event loop alive and the script never exits.
|
|
41
|
+
console.log("Redis is not reachable, stopping here.");
|
|
42
|
+
console.log("Health:", healthResult);
|
|
43
|
+
await release();
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// --- 1. A write batch ---------------------------------------------------
|
|
48
|
+
// Every key is independent, so all of them can go in one round trip.
|
|
49
|
+
const writes = Array.from({ length: BATCH_SIZE }, (_, index) => ({
|
|
50
|
+
label: `set:${index}`,
|
|
51
|
+
run: (pipeline) => void pipeline.set(`${KEY_PREFIX}item:${index}`, index),
|
|
52
|
+
}));
|
|
53
|
+
|
|
54
|
+
const started = Date.now();
|
|
55
|
+
const writeResult = await runPipeline(client, writes, {
|
|
56
|
+
logger: exampleLogger,
|
|
57
|
+
});
|
|
58
|
+
console.log(
|
|
59
|
+
`Wrote ${writeResult.results.length} keys in ${writeResult.durationMs}ms ` +
|
|
60
|
+
`(${Date.now() - started}ms wall clock).`,
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
// `pipelineValues` is the convenience path when every command is expected to
|
|
64
|
+
// succeed. It throws rather than hand back a sparse array if any of them did
|
|
65
|
+
// not, so a failed write cannot be mistaken for a successful one.
|
|
66
|
+
const values = pipelineValues(writeResult.results);
|
|
67
|
+
console.log(` all writes reported OK: ${values.every((v) => v === "OK")}`);
|
|
68
|
+
|
|
69
|
+
// --- 2. A read batch ----------------------------------------------------
|
|
70
|
+
// MSET/MSETNX aside, reads are the case pipelines are built for: a dashboard
|
|
71
|
+
// that renders 100 rows can fetch them all at once.
|
|
72
|
+
const reads = Array.from({ length: BATCH_SIZE }, (_, index) => ({
|
|
73
|
+
label: `get:${index}`,
|
|
74
|
+
run: (pipeline) => void pipeline.get(`${KEY_PREFIX}item:${index}`),
|
|
75
|
+
}));
|
|
76
|
+
|
|
77
|
+
const readResult = await runPipeline(client, reads, {
|
|
78
|
+
logger: exampleLogger,
|
|
79
|
+
});
|
|
80
|
+
const readValues = pipelineValues(readResult.results);
|
|
81
|
+
console.log(
|
|
82
|
+
`Read ${readValues.length} keys in ${readResult.durationMs}ms, ` +
|
|
83
|
+
`all present: ${readValues.every((v) => v !== null)}.`,
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
// --- 3. A batch where one command fails ---------------------------------
|
|
87
|
+
// INCR on a string key is a real Redis error: WRONGTYPE, not a connection
|
|
88
|
+
// problem. The batch still resolves, and every other command still runs.
|
|
89
|
+
await client.set(`${KEY_PREFIX}string`, "not-a-number");
|
|
90
|
+
|
|
91
|
+
const mixed = await runPipeline(
|
|
92
|
+
client,
|
|
93
|
+
[
|
|
94
|
+
{
|
|
95
|
+
label: "incr:string",
|
|
96
|
+
run: (pipeline) => void pipeline.incr(`${KEY_PREFIX}string`),
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
label: "set:fine",
|
|
100
|
+
run: (pipeline) => void pipeline.set(`${KEY_PREFIX}fine`, "1"),
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
label: "get:fine",
|
|
104
|
+
run: (pipeline) => void pipeline.get(`${KEY_PREFIX}fine`),
|
|
105
|
+
},
|
|
106
|
+
],
|
|
107
|
+
{ logger: exampleLogger },
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
console.log(
|
|
111
|
+
`\nMixed batch: ${mixed.failed} of ${mixed.results.length} failed.`,
|
|
112
|
+
);
|
|
113
|
+
for (const step of mixed.results) {
|
|
114
|
+
// The per-command error is the whole reason to use runPipeline over a bare
|
|
115
|
+
// `client.pipeline()`. Reading only the values here would show `null` for
|
|
116
|
+
// the failed INCR and look like a successful batch.
|
|
117
|
+
console.log(
|
|
118
|
+
` ${step.error ? "FAIL" : " ok "} ${step.label}: ` +
|
|
119
|
+
(step.error ? step.error.message : JSON.stringify(step.value)),
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// `throwOnError: true` is the strict alternative for batches where a partial
|
|
124
|
+
// write is not acceptable. It raises a PipelineCommandError carrying the
|
|
125
|
+
// per-step results, so the caller can see which command broke without
|
|
126
|
+
// re-running the batch.
|
|
127
|
+
try {
|
|
128
|
+
await runPipeline(
|
|
129
|
+
client,
|
|
130
|
+
[
|
|
131
|
+
{
|
|
132
|
+
label: "incr:string",
|
|
133
|
+
run: (pipeline) => void pipeline.incr(`${KEY_PREFIX}string`),
|
|
134
|
+
},
|
|
135
|
+
],
|
|
136
|
+
{ throwOnError: true },
|
|
137
|
+
);
|
|
138
|
+
} catch (error) {
|
|
139
|
+
console.log(`\nWith throwOnError: ${error.name}: ${error.message}`);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// --- 4. Clean up --------------------------------------------------------
|
|
143
|
+
const removals = Array.from({ length: BATCH_SIZE }, (_, index) => ({
|
|
144
|
+
label: `del:${index}`,
|
|
145
|
+
run: (pipeline) => void pipeline.del(`${KEY_PREFIX}item:${index}`),
|
|
146
|
+
}));
|
|
147
|
+
removals.push({
|
|
148
|
+
label: "del:string",
|
|
149
|
+
run: (pipeline) => void pipeline.del(`${KEY_PREFIX}string`),
|
|
150
|
+
});
|
|
151
|
+
removals.push({
|
|
152
|
+
label: "del:fine",
|
|
153
|
+
run: (pipeline) => void pipeline.del(`${KEY_PREFIX}fine`),
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
await runPipeline(client, removals, { logger: exampleLogger });
|
|
157
|
+
console.log(`\nCleaned up ${removals.length} keys.`);
|
|
158
|
+
|
|
159
|
+
await release();
|
|
160
|
+
console.log("Disconnected cleanly.");
|
|
161
|
+
});
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Publish/subscribe.
|
|
3
|
+
*
|
|
4
|
+
* npm run example:pubsub
|
|
5
|
+
*
|
|
6
|
+
* Environment:
|
|
7
|
+
* REDIS_URL connection URL (default redis://localhost:6379)
|
|
8
|
+
* REDIS_SILENT set to "true" to suppress connection-event logging
|
|
9
|
+
*
|
|
10
|
+
* A Redis connection in subscriber mode can only issue subscribe-family commands,
|
|
11
|
+
* so pub/sub always needs two clients. `subscribe` and `psubscribe` are also
|
|
12
|
+
* fire-and-forget: Redis does not queue a message published before a subscriber
|
|
13
|
+
* is attached, which is why this example subscribes and waits for the
|
|
14
|
+
* subscription count to confirm before publishing anything.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { createClient, shutdown } from "@oneunit/redis";
|
|
18
|
+
import {
|
|
19
|
+
REDIS_URL,
|
|
20
|
+
exampleLogger,
|
|
21
|
+
onFailure,
|
|
22
|
+
release,
|
|
23
|
+
run,
|
|
24
|
+
sleep,
|
|
25
|
+
within,
|
|
26
|
+
} from "./_setup.js";
|
|
27
|
+
|
|
28
|
+
const NOTIFICATIONS = "example:notifications";
|
|
29
|
+
const ALERTS = "example:alerts";
|
|
30
|
+
|
|
31
|
+
await run("pubsub", async () => {
|
|
32
|
+
// Two clients on purpose: one cannot both subscribe and publish. Each is
|
|
33
|
+
// registered the moment it exists, and release() closes in reverse order, so
|
|
34
|
+
// the subscriber goes first.
|
|
35
|
+
const publisher = createClient({ url: REDIS_URL }, exampleLogger);
|
|
36
|
+
onFailure(() => shutdown(publisher, exampleLogger));
|
|
37
|
+
|
|
38
|
+
const subscriber = createClient({ url: REDIS_URL }, exampleLogger);
|
|
39
|
+
onFailure(() => shutdown(subscriber, exampleLogger));
|
|
40
|
+
|
|
41
|
+
const received = [];
|
|
42
|
+
|
|
43
|
+
subscriber.on("message", (channel, message) => {
|
|
44
|
+
received.push({ channel, message });
|
|
45
|
+
console.log(` [${channel}]`, message);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
// subscribe() resolves with the number of channels this client is now
|
|
49
|
+
// attached to. That count is the confirmation we need: a client already in
|
|
50
|
+
// subscriber mode may not run `PUBSUB CHANNELS`, since only subscriber
|
|
51
|
+
// commands are permitted in that mode.
|
|
52
|
+
//
|
|
53
|
+
// The wait is bounded because subscribe() is a queued command. Against an
|
|
54
|
+
// unreachable server it never settles, and nothing else would notice.
|
|
55
|
+
const attached = await within(
|
|
56
|
+
subscriber.subscribe(NOTIFICATIONS, ALERTS),
|
|
57
|
+
5000,
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
if (attached !== 2) {
|
|
61
|
+
console.log(
|
|
62
|
+
attached === "timeout"
|
|
63
|
+
? "Subscribe did not complete within 5s. Is Redis running?"
|
|
64
|
+
: `Expected to attach to 2 channels, attached to ${attached}.`,
|
|
65
|
+
);
|
|
66
|
+
await release();
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
console.log(`Subscribed to ${attached} channels. Publishing...\n`);
|
|
71
|
+
|
|
72
|
+
await publisher.publish(
|
|
73
|
+
NOTIFICATIONS,
|
|
74
|
+
JSON.stringify({ type: "info", message: "Deploy finished" }),
|
|
75
|
+
);
|
|
76
|
+
await sleep(50);
|
|
77
|
+
|
|
78
|
+
await publisher.publish(
|
|
79
|
+
ALERTS,
|
|
80
|
+
JSON.stringify({ level: "warning", message: "High memory usage" }),
|
|
81
|
+
);
|
|
82
|
+
await sleep(50);
|
|
83
|
+
|
|
84
|
+
await publisher.publish(
|
|
85
|
+
NOTIFICATIONS,
|
|
86
|
+
JSON.stringify({ type: "info", message: "Second message" }),
|
|
87
|
+
);
|
|
88
|
+
await sleep(100);
|
|
89
|
+
|
|
90
|
+
console.log(`\nReceived ${received.length} of 3 published messages.`);
|
|
91
|
+
|
|
92
|
+
// unsubscribe() resolves with how many channels the client is *still*
|
|
93
|
+
// subscribed to, so 0 here means both channels were detached. A client that
|
|
94
|
+
// has been in subscriber mode cannot issue any other command afterwards, so
|
|
95
|
+
// unsubscribe before reusing it as a normal client.
|
|
96
|
+
const stillSubscribed = await subscriber.unsubscribe(NOTIFICATIONS, ALERTS);
|
|
97
|
+
console.log(`Unsubscribed; ${stillSubscribed} channel(s) still attached.`);
|
|
98
|
+
|
|
99
|
+
await release();
|
|
100
|
+
console.log("Disconnected cleanly.");
|
|
101
|
+
});
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BullMQ queue and worker.
|
|
3
|
+
*
|
|
4
|
+
* npm run example:queue-worker
|
|
5
|
+
*
|
|
6
|
+
* Environment:
|
|
7
|
+
* REDIS_URL connection URL (default redis://localhost:6379)
|
|
8
|
+
* REDIS_SILENT set to "true" to suppress connection-event logging
|
|
9
|
+
*
|
|
10
|
+
* Demonstrates the whole queue surface: `createQueue` applies job defaults,
|
|
11
|
+
* `createWorker` processes jobs with retries and backoff, and
|
|
12
|
+
* `attachQueueEvents` observes progress from a separate connection. One client
|
|
13
|
+
* serves all three, which works because `createClient` defaults
|
|
14
|
+
* `maxRetriesPerRequest` to null as BullMQ requires.
|
|
15
|
+
*
|
|
16
|
+
* `prefix` is per-call on all three factories and must match. `attachQueueEvents`
|
|
17
|
+
* inherits it from the queue when you do not pass one, so setting it on the
|
|
18
|
+
* queue and worker is enough.
|
|
19
|
+
*
|
|
20
|
+
* The script exits once the demo jobs settle. Press Ctrl+C to stop early; the
|
|
21
|
+
* signal handler closes everything in reverse order of creation.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import {
|
|
25
|
+
createClient,
|
|
26
|
+
createQueue,
|
|
27
|
+
createWorker,
|
|
28
|
+
attachQueueEvents,
|
|
29
|
+
shutdown,
|
|
30
|
+
} from "@oneunit/redis";
|
|
31
|
+
import {
|
|
32
|
+
REDIS_URL,
|
|
33
|
+
exampleLogger,
|
|
34
|
+
isReady,
|
|
35
|
+
onFailure,
|
|
36
|
+
release,
|
|
37
|
+
run,
|
|
38
|
+
} from "./_setup.js";
|
|
39
|
+
|
|
40
|
+
const QUEUE_NAME = "example-demo-jobs";
|
|
41
|
+
|
|
42
|
+
await run("queue-worker", async () => {
|
|
43
|
+
const client = createClient({ url: REDIS_URL }, exampleLogger);
|
|
44
|
+
onFailure(() => shutdown(client, exampleLogger));
|
|
45
|
+
|
|
46
|
+
// Job defaults from createQueue: 3 attempts with exponential backoff,
|
|
47
|
+
// keeping the last 100 completed and 1000 failed jobs. Retention matters
|
|
48
|
+
// because BullMQ keeps finished jobs in Redis until something removes them.
|
|
49
|
+
const queue = createQueue({ name: QUEUE_NAME, connection: client });
|
|
50
|
+
onFailure(() => queue.close());
|
|
51
|
+
|
|
52
|
+
// Events come from a separate listener with its own blocking connection, so
|
|
53
|
+
// they cannot interfere with the worker's polling.
|
|
54
|
+
const events = attachQueueEvents({ queue, logger: exampleLogger });
|
|
55
|
+
onFailure(() => events.close());
|
|
56
|
+
|
|
57
|
+
// Every readiness wait is bounded and rejection-tolerant. BullMQ polls Redis
|
|
58
|
+
// with commands that ioredis queues while reconnecting, so against an
|
|
59
|
+
// unreachable server these never settle and the script would hang
|
|
60
|
+
// indefinitely with BullMQ's reconnect loops still running.
|
|
61
|
+
if (!(await isReady(queue.waitUntilReady(), "Queue"))) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (!(await isReady(events.waitUntilReady(), "Queue events"))) {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const completed = [];
|
|
70
|
+
const failures = [];
|
|
71
|
+
|
|
72
|
+
// `concurrency` is optional. Omitting it leaves BullMQ's default of 1, which
|
|
73
|
+
// is deliberately not overridden by an explicit undefined.
|
|
74
|
+
const worker = createWorker({
|
|
75
|
+
name: QUEUE_NAME,
|
|
76
|
+
connection: client,
|
|
77
|
+
concurrency: 2,
|
|
78
|
+
processor: async (job) => {
|
|
79
|
+
console.log(
|
|
80
|
+
` processing ${job.name} (attempt ${job.attemptsMade + 1}/${job.opts.attempts ?? 3})`,
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
84
|
+
|
|
85
|
+
// One in four jobs fails, to show the retry and backoff path.
|
|
86
|
+
if (job.name === "report" && job.data.fail) {
|
|
87
|
+
throw new Error("report generation failed");
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return { processed: true, jobId: job.id, name: job.name };
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
// Registered before the readiness wait, since a worker that is never ready
|
|
95
|
+
// still holds its blocking connection open.
|
|
96
|
+
onFailure(() => worker.close(true));
|
|
97
|
+
|
|
98
|
+
if (!(await isReady(worker.waitUntilReady(), "Worker"))) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Track settlement per job so the script knows when the demo is finished.
|
|
103
|
+
const settled = new Map();
|
|
104
|
+
const markSettled = (jobId, outcome) => {
|
|
105
|
+
settled.set(jobId, outcome);
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
events.on("completed", ({ jobId, returnvalue }) => {
|
|
109
|
+
completed.push(jobId);
|
|
110
|
+
console.log(` completed ${jobId} ->`, returnvalue);
|
|
111
|
+
markSettled(jobId, "completed");
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
events.on("failed", ({ jobId, failedReason }) => {
|
|
115
|
+
failures.push({ jobId, failedReason });
|
|
116
|
+
console.log(` failed ${jobId} -> ${failedReason}`);
|
|
117
|
+
markSettled(jobId, "failed");
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// Clear anything left by a previous run so the counts below are meaningful.
|
|
121
|
+
// Clear anything a previous run left behind. obliterate() refuses an
|
|
122
|
+
// active queue, hence force.
|
|
123
|
+
await queue.obliterate({ force: true }).catch(() => {});
|
|
124
|
+
|
|
125
|
+
console.log("Enqueuing jobs...\n");
|
|
126
|
+
|
|
127
|
+
const welcome = await queue.add("welcome", {
|
|
128
|
+
userId: "user-1",
|
|
129
|
+
message: "Welcome!",
|
|
130
|
+
});
|
|
131
|
+
const notification = await queue.add("notification", {
|
|
132
|
+
type: "email",
|
|
133
|
+
to: "user@example.com",
|
|
134
|
+
});
|
|
135
|
+
// Succeeds on the first attempt.
|
|
136
|
+
await queue.add("report", { format: "pdf" });
|
|
137
|
+
// Fails every attempt, so it exercises the full retry budget.
|
|
138
|
+
const doomed = await queue.add("report", { format: "csv", fail: true });
|
|
139
|
+
|
|
140
|
+
console.log(
|
|
141
|
+
`Enqueued 4 jobs (${welcome.id}, ${notification.id}, ${doomed.id}, +1).\n`,
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
// Wait for all four to reach a terminal state. Bounded, so a worker that
|
|
145
|
+
// never picks the jobs up cannot hang the script forever.
|
|
146
|
+
// Clear the loser of the race. A pending timer keeps the event loop alive, so
|
|
147
|
+
// leaving it armed would hold the process open for the full 20s even after
|
|
148
|
+
// every job has settled.
|
|
149
|
+
let deadline;
|
|
150
|
+
const allSettled = await Promise.race([
|
|
151
|
+
new Promise((resolve) => {
|
|
152
|
+
const check = setInterval(() => {
|
|
153
|
+
if (settled.size >= 4) {
|
|
154
|
+
clearInterval(check);
|
|
155
|
+
resolve(true);
|
|
156
|
+
}
|
|
157
|
+
}, 50);
|
|
158
|
+
}),
|
|
159
|
+
new Promise((resolve) => {
|
|
160
|
+
deadline = setTimeout(() => resolve(false), 20000);
|
|
161
|
+
}),
|
|
162
|
+
]).finally(() => clearTimeout(deadline));
|
|
163
|
+
|
|
164
|
+
if (!allSettled) {
|
|
165
|
+
console.log("Timed out waiting for jobs to settle.");
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const counts = await queue.getJobCounts();
|
|
169
|
+
console.log("\nQueue counts:", counts);
|
|
170
|
+
console.log(
|
|
171
|
+
`Completed: ${completed.length}, failed permanently: ${failures.length}`,
|
|
172
|
+
);
|
|
173
|
+
|
|
174
|
+
if (failures.length > 0) {
|
|
175
|
+
const attemptsMade = await queue
|
|
176
|
+
.getJob(doomed.id)
|
|
177
|
+
.then((job) => job?.attemptsMade);
|
|
178
|
+
console.log(
|
|
179
|
+
`The failing job was attempted ${attemptsMade} times before BullMQ gave up.`,
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Close in reverse order of creation: worker, events, queue, then the shared
|
|
184
|
+
// client. BullMQ closes its own blocking connections for the first three.
|
|
185
|
+
await worker.close();
|
|
186
|
+
await release();
|
|
187
|
+
|
|
188
|
+
console.log("\nClosed cleanly.");
|
|
189
|
+
});
|