@mastra/valkey-streams 0.0.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/LICENSE.md +30 -0
- package/dist/client.d.ts +83 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/index.cjs +751 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +139 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +750 -0
- package/dist/index.js.map +1 -0
- package/package.json +23 -23
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,751 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
let crypto = require("crypto");
|
|
3
|
+
let _mastra_core_events = require("@mastra/core/events");
|
|
4
|
+
let _valkey_valkey_glide = require("@valkey/valkey-glide");
|
|
5
|
+
//#region src/client.ts
|
|
6
|
+
const text = (value) => Buffer.isBuffer(value) ? value.toString() : String(value);
|
|
7
|
+
var ValkeyMulti = class {
|
|
8
|
+
getClient;
|
|
9
|
+
#batch = new _valkey_valkey_glide.Batch(true);
|
|
10
|
+
constructor(getClient) {
|
|
11
|
+
this.getClient = getClient;
|
|
12
|
+
}
|
|
13
|
+
xAdd(key, id, fields, options = {}) {
|
|
14
|
+
const args = ["XADD", key];
|
|
15
|
+
if (options.TRIM) args.push("MAXLEN", "~", String(options.TRIM.threshold));
|
|
16
|
+
args.push(id, ...Object.entries(fields).flat());
|
|
17
|
+
this.#batch.customCommand(args);
|
|
18
|
+
return this;
|
|
19
|
+
}
|
|
20
|
+
pExpire(key, ttl) {
|
|
21
|
+
this.#batch.customCommand([
|
|
22
|
+
"PEXPIRE",
|
|
23
|
+
key,
|
|
24
|
+
String(ttl)
|
|
25
|
+
]);
|
|
26
|
+
return this;
|
|
27
|
+
}
|
|
28
|
+
xGroupCreate(key, group, id, options = {}) {
|
|
29
|
+
this.#batch.xgroupCreate(key, group, id, { mkStream: options.MKSTREAM });
|
|
30
|
+
return this;
|
|
31
|
+
}
|
|
32
|
+
async exec() {
|
|
33
|
+
return await (await this.getClient()).exec(this.#batch, true, { decoder: _valkey_valkey_glide.Decoder.String }) ?? [];
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
var ValkeyStreamsClient = class {
|
|
37
|
+
options;
|
|
38
|
+
#client;
|
|
39
|
+
constructor(options) {
|
|
40
|
+
this.options = options;
|
|
41
|
+
}
|
|
42
|
+
get isOpen() {
|
|
43
|
+
return this.#client !== void 0;
|
|
44
|
+
}
|
|
45
|
+
on(_event, _listener) {
|
|
46
|
+
return this;
|
|
47
|
+
}
|
|
48
|
+
async connect() {
|
|
49
|
+
await this.getClient();
|
|
50
|
+
}
|
|
51
|
+
async quit() {
|
|
52
|
+
(await this.getClient()).close();
|
|
53
|
+
}
|
|
54
|
+
multi() {
|
|
55
|
+
return new ValkeyMulti(() => this.getClient());
|
|
56
|
+
}
|
|
57
|
+
async command(args) {
|
|
58
|
+
return (await this.getClient()).customCommand(args, { decoder: _valkey_valkey_glide.Decoder.String });
|
|
59
|
+
}
|
|
60
|
+
async xAdd(key, id, fields, options = {}) {
|
|
61
|
+
const args = ["XADD", key];
|
|
62
|
+
if (options.TRIM) args.push("MAXLEN", "~", String(options.TRIM.threshold));
|
|
63
|
+
args.push(id, ...Object.entries(fields).flat());
|
|
64
|
+
return text(await this.command(args));
|
|
65
|
+
}
|
|
66
|
+
async pExpire(key, ttl) {
|
|
67
|
+
return Number(await this.command([
|
|
68
|
+
"PEXPIRE",
|
|
69
|
+
key,
|
|
70
|
+
String(ttl)
|
|
71
|
+
]));
|
|
72
|
+
}
|
|
73
|
+
async xGroupCreate(key, group, id, options = {}) {
|
|
74
|
+
return (await this.getClient()).xgroupCreate(key, group, id, { mkStream: options.MKSTREAM });
|
|
75
|
+
}
|
|
76
|
+
async xGroupDestroy(key, group) {
|
|
77
|
+
return Number(await this.command([
|
|
78
|
+
"XGROUP",
|
|
79
|
+
"DESTROY",
|
|
80
|
+
key,
|
|
81
|
+
group
|
|
82
|
+
]));
|
|
83
|
+
}
|
|
84
|
+
async xReadGroup(group, consumer, streams, options = {}) {
|
|
85
|
+
return (await (await this.getClient()).xreadgroup(group, consumer, Object.fromEntries(streams.map((stream) => [stream.key, stream.id])), {
|
|
86
|
+
count: options.COUNT,
|
|
87
|
+
block: options.BLOCK,
|
|
88
|
+
decoder: _valkey_valkey_glide.Decoder.String
|
|
89
|
+
}))?.map((stream) => ({
|
|
90
|
+
name: text(stream.key),
|
|
91
|
+
messages: Object.entries(stream.value ?? {}).map(([id, fields]) => ({
|
|
92
|
+
id,
|
|
93
|
+
message: Object.fromEntries(fields?.map(([field, value]) => [text(field), text(value)]) ?? [])
|
|
94
|
+
}))
|
|
95
|
+
})) ?? null;
|
|
96
|
+
}
|
|
97
|
+
async xAutoClaim(key, group, consumer, minIdle, start, options = {}) {
|
|
98
|
+
const result = await (await this.getClient()).xautoclaim(key, group, consumer, minIdle, start, {
|
|
99
|
+
count: options.COUNT,
|
|
100
|
+
decoder: _valkey_valkey_glide.Decoder.String
|
|
101
|
+
});
|
|
102
|
+
return {
|
|
103
|
+
nextId: text(result[0]),
|
|
104
|
+
messages: Object.entries(result[1] ?? {}).map(([id, fields]) => ({
|
|
105
|
+
id,
|
|
106
|
+
message: Object.fromEntries(fields.map(([field, value]) => [text(field), text(value)]))
|
|
107
|
+
}))
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
async xAck(key, group, id) {
|
|
111
|
+
return (await this.getClient()).xack(key, group, [id]);
|
|
112
|
+
}
|
|
113
|
+
async del(key) {
|
|
114
|
+
return Number(await this.command(["DEL", key]));
|
|
115
|
+
}
|
|
116
|
+
async set(key, value, options = {}) {
|
|
117
|
+
const args = [
|
|
118
|
+
"SET",
|
|
119
|
+
key,
|
|
120
|
+
value
|
|
121
|
+
];
|
|
122
|
+
if (options.NX) args.push("NX");
|
|
123
|
+
if (options.PX) args.push("PX", String(options.PX));
|
|
124
|
+
const result = await this.command(args);
|
|
125
|
+
return result === null ? null : text(result);
|
|
126
|
+
}
|
|
127
|
+
async get(key) {
|
|
128
|
+
const result = await this.command(["GET", key]);
|
|
129
|
+
return result === null ? null : text(result);
|
|
130
|
+
}
|
|
131
|
+
async eval(script, options) {
|
|
132
|
+
return Number(await this.command([
|
|
133
|
+
"EVAL",
|
|
134
|
+
script,
|
|
135
|
+
String(options.keys.length),
|
|
136
|
+
...options.keys,
|
|
137
|
+
...options.arguments
|
|
138
|
+
]));
|
|
139
|
+
}
|
|
140
|
+
getClient() {
|
|
141
|
+
if (!this.#client) {
|
|
142
|
+
const { url, ...configuration } = this.options;
|
|
143
|
+
const parsed = url ? new URL(url) : void 0;
|
|
144
|
+
this.#client = _valkey_valkey_glide.GlideClient.createClient({
|
|
145
|
+
...configuration,
|
|
146
|
+
addresses: configuration.addresses ?? [{
|
|
147
|
+
host: parsed?.hostname ?? "localhost",
|
|
148
|
+
port: Number(parsed?.port || 6379)
|
|
149
|
+
}],
|
|
150
|
+
credentials: configuration.credentials ?? (parsed?.password ? {
|
|
151
|
+
username: parsed.username || "default",
|
|
152
|
+
password: parsed.password
|
|
153
|
+
} : void 0),
|
|
154
|
+
databaseId: configuration.databaseId ?? (parsed?.pathname ? Number(parsed.pathname.slice(1) || 0) : void 0)
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
return this.#client;
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
const createClient = (options) => new ValkeyStreamsClient(options);
|
|
161
|
+
//#endregion
|
|
162
|
+
//#region src/index.ts
|
|
163
|
+
/**
|
|
164
|
+
* Flatten an error into searchable text. node-redis MULTI failures throw a
|
|
165
|
+
* `MultiErrorReply` whose own message is just "N commands failed…" — the real
|
|
166
|
+
* per-command errors (e.g. BUSYGROUP) live in `err.replies`, so those are
|
|
167
|
+
* folded in too.
|
|
168
|
+
*/
|
|
169
|
+
function errorText(err) {
|
|
170
|
+
const parts = [err instanceof Error ? err.message : String(err)];
|
|
171
|
+
if (err instanceof Error && "replies" in err && Array.isArray(err.replies)) parts.push(...err.replies.map((r) => String(r)));
|
|
172
|
+
return parts.join("; ");
|
|
173
|
+
}
|
|
174
|
+
var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
175
|
+
get supportedModes() {
|
|
176
|
+
return ["pull"];
|
|
177
|
+
}
|
|
178
|
+
get supportsOffsets() {
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
onUnsupportedOffset(offset) {
|
|
182
|
+
this.#logger?.warn?.(`valkey-streams: numeric replay offset ${offset} is unsupported; falling back to full replay`);
|
|
183
|
+
}
|
|
184
|
+
#writeClient;
|
|
185
|
+
#connectOptions;
|
|
186
|
+
#keyPrefix;
|
|
187
|
+
#blockMs;
|
|
188
|
+
#maxStreamLength;
|
|
189
|
+
#streamIdleTtlMs;
|
|
190
|
+
#reclaimIntervalMs;
|
|
191
|
+
#reclaimIdleMs;
|
|
192
|
+
#maxDeliveryAttempts;
|
|
193
|
+
#logger;
|
|
194
|
+
#subscriptions = /* @__PURE__ */ new Map();
|
|
195
|
+
#cbIds = /* @__PURE__ */ new WeakMap();
|
|
196
|
+
#pendingPublishes = /* @__PURE__ */ new Set();
|
|
197
|
+
#localCallbacks = /* @__PURE__ */ new Map();
|
|
198
|
+
#closed = false;
|
|
199
|
+
constructor(options = {}) {
|
|
200
|
+
super();
|
|
201
|
+
const url = options.url ?? options.valkeyOptions?.url ?? "valkey://localhost:6379";
|
|
202
|
+
this.#connectOptions = {
|
|
203
|
+
...options.valkeyOptions,
|
|
204
|
+
url
|
|
205
|
+
};
|
|
206
|
+
this.#writeClient = createClient(this.#connectOptions);
|
|
207
|
+
this.#logger = options.logger;
|
|
208
|
+
this.#attachErrorLogger(this.#writeClient, "write");
|
|
209
|
+
this.#keyPrefix = options.keyPrefix ?? "mastra:topic";
|
|
210
|
+
this.#blockMs = options.blockMs ?? 1e3;
|
|
211
|
+
this.#maxStreamLength = options.maxStreamLength ?? 1e4;
|
|
212
|
+
const ttl = options.streamIdleTtlMs ?? 0;
|
|
213
|
+
if (!Number.isInteger(ttl) || ttl < 0) throw new Error(`valkey-streams: streamIdleTtlMs must be a non-negative integer (milliseconds), got ${ttl}`);
|
|
214
|
+
this.#streamIdleTtlMs = ttl;
|
|
215
|
+
this.#reclaimIntervalMs = options.reclaimIntervalMs ?? 3e4;
|
|
216
|
+
this.#reclaimIdleMs = options.reclaimIdleMs ?? 6e4;
|
|
217
|
+
const cap = options.maxDeliveryAttempts ?? 5;
|
|
218
|
+
if (cap === 0) {
|
|
219
|
+
options.logger?.warn?.("valkey-streams: maxDeliveryAttempts=0 is treated as Infinity for back-compat; pass Infinity to disable the cap explicitly.");
|
|
220
|
+
this.#maxDeliveryAttempts = Infinity;
|
|
221
|
+
} else if (cap < 0 || Number.isNaN(cap)) throw new Error(`valkey-streams: maxDeliveryAttempts must be >= 1 or Infinity, got ${cap}`);
|
|
222
|
+
else this.#maxDeliveryAttempts = cap;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Attach the `'error'` listener node-redis requires on every client. Without
|
|
226
|
+
* one, a mid-life socket close makes the client emit an unhandled `'error'`,
|
|
227
|
+
* which (per EventEmitter semantics) throws from inside ValkeySocket's error
|
|
228
|
+
* handler BEFORE the built-in reconnect is scheduled — the process gets an
|
|
229
|
+
* uncaughtException and the client is left permanently un-reconnected
|
|
230
|
+
* (isOpen=true, isReady=false), hanging every subsequent command. Merely
|
|
231
|
+
* having a listener lets node-redis's own reconnect run.
|
|
232
|
+
*
|
|
233
|
+
* Logged at warn so operators can see connection churn, throttled to one line
|
|
234
|
+
* per 30s per client: node-redis re-emits `'error'` roughly every 500ms while
|
|
235
|
+
* a server is unreachable, and with one reader client per subscription an
|
|
236
|
+
* outage would otherwise flood the logs with thousands of identical lines.
|
|
237
|
+
*/
|
|
238
|
+
#attachErrorLogger(client, role, extra = {}) {
|
|
239
|
+
let lastWarnAt = 0;
|
|
240
|
+
client.on("error", (err) => {
|
|
241
|
+
const now = Date.now();
|
|
242
|
+
if (now - lastWarnAt < 3e4) return;
|
|
243
|
+
lastWarnAt = now;
|
|
244
|
+
this.#logger?.warn?.(`valkey-streams: ${role} client connection error (node-redis will reconnect)`, {
|
|
245
|
+
...extra,
|
|
246
|
+
err: err instanceof Error ? err.message : err
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
#subKey(topic, cb) {
|
|
251
|
+
let cbId = this.#cbIds.get(cb);
|
|
252
|
+
if (!cbId) {
|
|
253
|
+
cbId = (0, crypto.randomUUID)();
|
|
254
|
+
this.#cbIds.set(cb, cbId);
|
|
255
|
+
}
|
|
256
|
+
return `${topic}::${cbId}`;
|
|
257
|
+
}
|
|
258
|
+
/** Lazily connect the shared writer client. Idempotent. */
|
|
259
|
+
async #ensureWriterConnected() {
|
|
260
|
+
if (this.#writeClient.isOpen) return;
|
|
261
|
+
await this.#writeClient.connect();
|
|
262
|
+
}
|
|
263
|
+
#streamKey(topic) {
|
|
264
|
+
return `${this.#keyPrefix}:${topic}`;
|
|
265
|
+
}
|
|
266
|
+
async publish(topic, event, options) {
|
|
267
|
+
if (this.#closed) throw new Error("ValkeyStreamsPubSub: cannot publish on closed client");
|
|
268
|
+
if (options?.localOnly) {
|
|
269
|
+
const localEvent = {
|
|
270
|
+
...event,
|
|
271
|
+
id: (0, crypto.randomUUID)(),
|
|
272
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
273
|
+
deliveryAttempt: event.deliveryAttempt ?? 1
|
|
274
|
+
};
|
|
275
|
+
this.#deliverLocal(topic, localEvent);
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
await this.#ensureWriterConnected();
|
|
279
|
+
const id = (0, crypto.randomUUID)();
|
|
280
|
+
const createdAt = /* @__PURE__ */ new Date();
|
|
281
|
+
const payload = {
|
|
282
|
+
...event,
|
|
283
|
+
id,
|
|
284
|
+
createdAt,
|
|
285
|
+
deliveryAttempt: event.deliveryAttempt ?? 1
|
|
286
|
+
};
|
|
287
|
+
const xaddOptions = {};
|
|
288
|
+
if (this.#maxStreamLength > 0) xaddOptions.TRIM = {
|
|
289
|
+
strategy: "MAXLEN",
|
|
290
|
+
strategyModifier: "~",
|
|
291
|
+
threshold: this.#maxStreamLength
|
|
292
|
+
};
|
|
293
|
+
const streamKey = this.#streamKey(topic);
|
|
294
|
+
const promise = this.#streamIdleTtlMs > 0 ? this.#writeClient.multi().xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions).pExpire(streamKey, this.#streamIdleTtlMs).exec().then((replies) => {
|
|
295
|
+
if (Number(replies[1]) !== 1) this.#logger?.warn?.("valkey-streams: PEXPIRE inside publish MULTI did not apply", {
|
|
296
|
+
streamKey,
|
|
297
|
+
reply: String(replies[1])
|
|
298
|
+
});
|
|
299
|
+
}) : this.#writeClient.xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions);
|
|
300
|
+
this.#pendingPublishes.add(promise);
|
|
301
|
+
try {
|
|
302
|
+
await promise;
|
|
303
|
+
} finally {
|
|
304
|
+
this.#pendingPublishes.delete(promise);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
async subscribe(topic, cb, options) {
|
|
308
|
+
if (this.#closed) throw new Error("ValkeyStreamsPubSub: cannot subscribe on closed client");
|
|
309
|
+
const key = this.#subKey(topic, cb);
|
|
310
|
+
if (this.#subscriptions.has(key)) return;
|
|
311
|
+
let localBucket = this.#localCallbacks.get(topic);
|
|
312
|
+
if (!localBucket) {
|
|
313
|
+
localBucket = /* @__PURE__ */ new Set();
|
|
314
|
+
this.#localCallbacks.set(topic, localBucket);
|
|
315
|
+
}
|
|
316
|
+
localBucket.add(cb);
|
|
317
|
+
await this.#ensureWriterConnected();
|
|
318
|
+
const isGrouped = !!options?.group;
|
|
319
|
+
const group = options?.group ?? `__fanout-${(0, crypto.randomUUID)()}`;
|
|
320
|
+
const consumer = `${group}-${(0, crypto.randomUUID)()}`;
|
|
321
|
+
const streamKey = this.#streamKey(topic);
|
|
322
|
+
const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
|
|
323
|
+
try {
|
|
324
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true }).pExpire(streamKey, this.#streamIdleTtlMs).exec();
|
|
325
|
+
else await this.#writeClient.xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true });
|
|
326
|
+
} catch (err) {
|
|
327
|
+
if (!errorText(err).includes("BUSYGROUP")) throw err;
|
|
328
|
+
this.#logger?.debug?.("valkey-streams: consumer group already exists", {
|
|
329
|
+
topic,
|
|
330
|
+
group
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
const readClient = createClient(this.#connectOptions);
|
|
334
|
+
this.#attachErrorLogger(readClient, "read", { topic });
|
|
335
|
+
await readClient.connect();
|
|
336
|
+
const sub = {
|
|
337
|
+
cb,
|
|
338
|
+
topic,
|
|
339
|
+
streamKey,
|
|
340
|
+
group,
|
|
341
|
+
consumer,
|
|
342
|
+
isGrouped,
|
|
343
|
+
groupAnchor,
|
|
344
|
+
lastId: void 0,
|
|
345
|
+
readClient,
|
|
346
|
+
stopped: false,
|
|
347
|
+
loop: void 0,
|
|
348
|
+
reclaimTimer: void 0
|
|
349
|
+
};
|
|
350
|
+
this.#subscriptions.set(key, sub);
|
|
351
|
+
sub.loop = this.#runReadLoop(sub);
|
|
352
|
+
this.#startReclaimLoop(sub);
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Periodically run XAUTOCLAIM against this subscription's group so that
|
|
356
|
+
* messages a crashed/stuck consumer read but never acked get redelivered to
|
|
357
|
+
* a live sibling. Runs only for grouped subscriptions — fan-out groups are
|
|
358
|
+
* private to one consumer, so there's no sibling to claim from.
|
|
359
|
+
*/
|
|
360
|
+
#startReclaimLoop(sub) {
|
|
361
|
+
if (this.#reclaimIntervalMs <= 0) return;
|
|
362
|
+
if (!sub.isGrouped) return;
|
|
363
|
+
const tick = async () => {
|
|
364
|
+
if (sub.stopped || this.#closed) return;
|
|
365
|
+
try {
|
|
366
|
+
const messages = (await this.#writeClient.xAutoClaim(sub.streamKey, sub.group, sub.consumer, this.#reclaimIdleMs, "0-0", { COUNT: 100 }))?.messages ?? [];
|
|
367
|
+
for (const entry of messages) {
|
|
368
|
+
if (sub.stopped || this.#closed) return;
|
|
369
|
+
if (!entry) continue;
|
|
370
|
+
await this.#deliverMessage(sub, entry.id, entry.message);
|
|
371
|
+
}
|
|
372
|
+
} catch (err) {
|
|
373
|
+
this.#logger?.debug?.("valkey-streams: XAUTOCLAIM failed", {
|
|
374
|
+
topic: sub.topic,
|
|
375
|
+
group: sub.group,
|
|
376
|
+
err: err instanceof Error ? err.message : err
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
if (sub.stopped || this.#closed) return;
|
|
380
|
+
sub.reclaimTimer = setTimeout(tick, this.#reclaimIntervalMs);
|
|
381
|
+
};
|
|
382
|
+
sub.reclaimTimer = setTimeout(tick, this.#reclaimIntervalMs);
|
|
383
|
+
}
|
|
384
|
+
async unsubscribe(topic, cb) {
|
|
385
|
+
const localBucket = this.#localCallbacks.get(topic);
|
|
386
|
+
if (localBucket) {
|
|
387
|
+
localBucket.delete(cb);
|
|
388
|
+
if (localBucket.size === 0) this.#localCallbacks.delete(topic);
|
|
389
|
+
}
|
|
390
|
+
const key = this.#subKey(topic, cb);
|
|
391
|
+
const sub = this.#subscriptions.get(key);
|
|
392
|
+
if (!sub) return;
|
|
393
|
+
this.#subscriptions.delete(key);
|
|
394
|
+
sub.stopped = true;
|
|
395
|
+
if (sub.reclaimTimer) {
|
|
396
|
+
clearTimeout(sub.reclaimTimer);
|
|
397
|
+
sub.reclaimTimer = void 0;
|
|
398
|
+
}
|
|
399
|
+
try {
|
|
400
|
+
await sub.readClient.quit();
|
|
401
|
+
} catch (err) {
|
|
402
|
+
this.#logger?.debug?.("valkey-streams: reader quit failed", {
|
|
403
|
+
topic: sub.topic,
|
|
404
|
+
err: err instanceof Error ? err.message : err
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
if (sub.loop) try {
|
|
408
|
+
await sub.loop;
|
|
409
|
+
} catch (err) {
|
|
410
|
+
this.#logger?.debug?.("valkey-streams: read loop exited with error", {
|
|
411
|
+
topic: sub.topic,
|
|
412
|
+
err: err instanceof Error ? err.message : err
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
if (!sub.isGrouped) try {
|
|
416
|
+
await this.#writeClient.xGroupDestroy(sub.streamKey, sub.group);
|
|
417
|
+
} catch (err) {
|
|
418
|
+
this.#logger?.debug?.("valkey-streams: xGroupDestroy failed", {
|
|
419
|
+
topic: sub.topic,
|
|
420
|
+
group: sub.group,
|
|
421
|
+
err: err instanceof Error ? err.message : err
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
async flush() {
|
|
426
|
+
if (this.#pendingPublishes.size > 0) await Promise.allSettled([...this.#pendingPublishes]);
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Delete a topic's stream, and with it every consumer group on that stream.
|
|
430
|
+
* This is the Valkey implementation of the optional `clearTopic` hook that
|
|
431
|
+
* consumers (e.g. `DurableAgent`) call to free a topic once its lifecycle is
|
|
432
|
+
* over — without it, a persistent backend accumulates every finished run's
|
|
433
|
+
* stream until Valkey exhausts its memory.
|
|
434
|
+
*
|
|
435
|
+
* Because it removes events for ALL consumers and destroys their groups, only
|
|
436
|
+
* call it once nothing will read the topic again. A subscriber still attached
|
|
437
|
+
* when the stream is deleted recovers on its own (the read loop recreates the
|
|
438
|
+
* group on NOGROUP) but will have missed the deleted entries.
|
|
439
|
+
*
|
|
440
|
+
* Best-effort and non-throwing: callers commonly invoke this fire-and-forget
|
|
441
|
+
* (`void clearTopic(...)`), so a rejection would surface as an
|
|
442
|
+
* unhandledRejection. Failures are swallowed and logged; the `streamIdleTtlMs`
|
|
443
|
+
* backstop (when configured) reclaims anything a failed delete leaves behind.
|
|
444
|
+
*/
|
|
445
|
+
async clearTopic(topic) {
|
|
446
|
+
if (this.#closed) return;
|
|
447
|
+
try {
|
|
448
|
+
await this.#ensureWriterConnected();
|
|
449
|
+
await this.#writeClient.del(this.#streamKey(topic));
|
|
450
|
+
for (const sub of this.#subscriptions.values()) if (sub.topic === topic) {
|
|
451
|
+
sub.lastId = void 0;
|
|
452
|
+
sub.groupAnchor = "0";
|
|
453
|
+
}
|
|
454
|
+
} catch (err) {
|
|
455
|
+
this.#logger?.warn?.("valkey-streams: clearTopic failed", {
|
|
456
|
+
topic,
|
|
457
|
+
err: err instanceof Error ? err.message : err
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Lease key used in Valkey. Distinct prefix from streams so leases and
|
|
463
|
+
* streams can't collide on key namespace.
|
|
464
|
+
*/
|
|
465
|
+
#leaseKey(key) {
|
|
466
|
+
return `${this.#keyPrefix}:lease:${key}`;
|
|
467
|
+
}
|
|
468
|
+
/**
|
|
469
|
+
* Atomic claim via SET NX PX. Idempotent for the same owner: if the
|
|
470
|
+
* current value is already this owner, we refresh the TTL instead of
|
|
471
|
+
* failing. Cross-process callers race here; Valkey serializes them.
|
|
472
|
+
*/
|
|
473
|
+
async acquireLease(key, owner, ttlMs) {
|
|
474
|
+
if (this.#closed) return { acquired: false };
|
|
475
|
+
await this.#ensureWriterConnected();
|
|
476
|
+
const redisKey = this.#leaseKey(key);
|
|
477
|
+
if (await this.#writeClient.set(redisKey, owner, {
|
|
478
|
+
NX: true,
|
|
479
|
+
PX: ttlMs
|
|
480
|
+
}) === "OK") return {
|
|
481
|
+
acquired: true,
|
|
482
|
+
owner
|
|
483
|
+
};
|
|
484
|
+
if (await this.#writeClient.eval(`
|
|
485
|
+
local current = redis.call("GET", KEYS[1])
|
|
486
|
+
if current == ARGV[1] then
|
|
487
|
+
redis.call("PEXPIRE", KEYS[1], ARGV[2])
|
|
488
|
+
return 1
|
|
489
|
+
end
|
|
490
|
+
return 0
|
|
491
|
+
`, {
|
|
492
|
+
keys: [redisKey],
|
|
493
|
+
arguments: [owner, String(ttlMs)]
|
|
494
|
+
}) === 1) return {
|
|
495
|
+
acquired: true,
|
|
496
|
+
owner
|
|
497
|
+
};
|
|
498
|
+
return {
|
|
499
|
+
acquired: false,
|
|
500
|
+
owner: await this.#writeClient.get(redisKey) ?? void 0
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
async getLeaseOwner(key) {
|
|
504
|
+
if (this.#closed) return void 0;
|
|
505
|
+
await this.#ensureWriterConnected();
|
|
506
|
+
return await this.#writeClient.get(this.#leaseKey(key)) ?? void 0;
|
|
507
|
+
}
|
|
508
|
+
/**
|
|
509
|
+
* Release only if we still own it. Implemented as GET+DEL with a Lua
|
|
510
|
+
* script so the check-and-delete is atomic against concurrent renewals
|
|
511
|
+
* from other processes.
|
|
512
|
+
*/
|
|
513
|
+
async releaseLease(key, owner) {
|
|
514
|
+
if (this.#closed) return;
|
|
515
|
+
await this.#ensureWriterConnected();
|
|
516
|
+
await this.#writeClient.eval(`
|
|
517
|
+
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
|
518
|
+
return redis.call("DEL", KEYS[1])
|
|
519
|
+
else
|
|
520
|
+
return 0
|
|
521
|
+
end
|
|
522
|
+
`, {
|
|
523
|
+
keys: [this.#leaseKey(key)],
|
|
524
|
+
arguments: [owner]
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* Extend the TTL only if we still own the lease. Returns false if the
|
|
529
|
+
* lease was lost (expired or another owner took it).
|
|
530
|
+
*/
|
|
531
|
+
async renewLease(key, owner, ttlMs) {
|
|
532
|
+
if (this.#closed) return false;
|
|
533
|
+
await this.#ensureWriterConnected();
|
|
534
|
+
return await this.#writeClient.eval(`
|
|
535
|
+
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
|
536
|
+
return redis.call("PEXPIRE", KEYS[1], ARGV[2])
|
|
537
|
+
else
|
|
538
|
+
return 0
|
|
539
|
+
end
|
|
540
|
+
`, {
|
|
541
|
+
keys: [this.#leaseKey(key)],
|
|
542
|
+
arguments: [owner, String(ttlMs)]
|
|
543
|
+
}) === 1;
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* Atomically hand the lease from `fromOwner` to `toOwner`, refreshing the
|
|
547
|
+
* TTL to the full `ttlMs`, without the key ever going empty.
|
|
548
|
+
*
|
|
549
|
+
* Implemented as a single Lua script (GET == fromOwner -> SET toOwner PX)
|
|
550
|
+
* so a racing process cannot win the key between a release and a re-acquire.
|
|
551
|
+
* Returns false if `fromOwner` no longer holds the lease (expired or taken),
|
|
552
|
+
* in which case the caller should fall back to a fresh `acquireLease`.
|
|
553
|
+
*/
|
|
554
|
+
async transferLease(key, fromOwner, toOwner, ttlMs) {
|
|
555
|
+
if (this.#closed) return false;
|
|
556
|
+
await this.#ensureWriterConnected();
|
|
557
|
+
return await this.#writeClient.eval(`
|
|
558
|
+
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
|
559
|
+
redis.call("SET", KEYS[1], ARGV[2], "PX", ARGV[3])
|
|
560
|
+
return 1
|
|
561
|
+
else
|
|
562
|
+
return 0
|
|
563
|
+
end
|
|
564
|
+
`, {
|
|
565
|
+
keys: [this.#leaseKey(key)],
|
|
566
|
+
arguments: [
|
|
567
|
+
fromOwner,
|
|
568
|
+
toOwner,
|
|
569
|
+
String(ttlMs)
|
|
570
|
+
]
|
|
571
|
+
}) === 1;
|
|
572
|
+
}
|
|
573
|
+
/**
|
|
574
|
+
* Disconnect all clients and stop all subscription loops.
|
|
575
|
+
*/
|
|
576
|
+
async close() {
|
|
577
|
+
if (this.#closed) return;
|
|
578
|
+
this.#closed = true;
|
|
579
|
+
const subs = [...this.#subscriptions.values()];
|
|
580
|
+
await Promise.all(subs.map((sub) => this.unsubscribe(sub.topic, sub.cb)));
|
|
581
|
+
this.#localCallbacks.clear();
|
|
582
|
+
if (this.#writeClient.isOpen) try {
|
|
583
|
+
await this.#writeClient.quit();
|
|
584
|
+
} catch (err) {
|
|
585
|
+
this.#logger?.debug?.("valkey-streams: writer quit failed", { err: err instanceof Error ? err.message : err });
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
async #runReadLoop(sub) {
|
|
589
|
+
while (!sub.stopped) {
|
|
590
|
+
let result;
|
|
591
|
+
try {
|
|
592
|
+
result = await sub.readClient.xReadGroup(sub.group, sub.consumer, [{
|
|
593
|
+
key: sub.streamKey,
|
|
594
|
+
id: ">"
|
|
595
|
+
}], {
|
|
596
|
+
COUNT: 10,
|
|
597
|
+
BLOCK: this.#blockMs
|
|
598
|
+
});
|
|
599
|
+
} catch (err) {
|
|
600
|
+
if (sub.stopped) return;
|
|
601
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
602
|
+
if (msg.includes("NOGROUP")) {
|
|
603
|
+
const recoveryAnchor = sub.lastId ?? sub.groupAnchor;
|
|
604
|
+
try {
|
|
605
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xGroupCreate(sub.streamKey, sub.group, recoveryAnchor, { MKSTREAM: true }).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
606
|
+
else await this.#writeClient.xGroupCreate(sub.streamKey, sub.group, recoveryAnchor, { MKSTREAM: true });
|
|
607
|
+
this.#logger?.debug?.("valkey-streams: recreated consumer group after NOGROUP", {
|
|
608
|
+
topic: sub.topic,
|
|
609
|
+
group: sub.group
|
|
610
|
+
});
|
|
611
|
+
} catch (recreateErr) {
|
|
612
|
+
const rmsg = errorText(recreateErr);
|
|
613
|
+
if (!rmsg.includes("BUSYGROUP")) this.#logger?.debug?.("valkey-streams: consumer group re-create failed", {
|
|
614
|
+
topic: sub.topic,
|
|
615
|
+
err: rmsg
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
} else this.#logger?.debug?.("valkey-streams: xReadGroup failed", {
|
|
619
|
+
topic: sub.topic,
|
|
620
|
+
group: sub.group,
|
|
621
|
+
err: msg
|
|
622
|
+
});
|
|
623
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
if (!result || result.length === 0) continue;
|
|
627
|
+
for (const stream of result) for (const entry of stream.messages) {
|
|
628
|
+
if (sub.stopped) return;
|
|
629
|
+
sub.lastId = entry.id;
|
|
630
|
+
await this.#deliverMessage(sub, entry.id, entry.message);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
async #deliverMessage(sub, streamId, fields) {
|
|
635
|
+
let event;
|
|
636
|
+
try {
|
|
637
|
+
event = JSON.parse(fields.event ?? "{}");
|
|
638
|
+
if (typeof event.createdAt === "string") event.createdAt = new Date(event.createdAt);
|
|
639
|
+
} catch (err) {
|
|
640
|
+
this.#logger?.debug?.("valkey-streams: malformed payload, dropping", {
|
|
641
|
+
topic: sub.topic,
|
|
642
|
+
streamId,
|
|
643
|
+
err: err instanceof Error ? err.message : err
|
|
644
|
+
});
|
|
645
|
+
try {
|
|
646
|
+
await this.#writeClient.xAck(sub.streamKey, sub.group, streamId);
|
|
647
|
+
} catch (ackErr) {
|
|
648
|
+
this.#logger?.debug?.("valkey-streams: xAck after malformed payload failed", {
|
|
649
|
+
topic: sub.topic,
|
|
650
|
+
err: ackErr instanceof Error ? ackErr.message : ackErr
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
return;
|
|
654
|
+
}
|
|
655
|
+
let settled = false;
|
|
656
|
+
const ack = async () => {
|
|
657
|
+
if (settled) return;
|
|
658
|
+
settled = true;
|
|
659
|
+
try {
|
|
660
|
+
await this.#writeClient.xAck(sub.streamKey, sub.group, streamId);
|
|
661
|
+
} catch (err) {
|
|
662
|
+
this.#logger?.debug?.("valkey-streams: ack cleanup failed", {
|
|
663
|
+
topic: sub.topic,
|
|
664
|
+
err: err instanceof Error ? err.message : err
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
};
|
|
668
|
+
const nack = async () => {
|
|
669
|
+
if (settled) return;
|
|
670
|
+
settled = true;
|
|
671
|
+
const attempt = event.deliveryAttempt ?? 1;
|
|
672
|
+
if (attempt >= this.#maxDeliveryAttempts) {
|
|
673
|
+
this.#logger?.warn?.("valkey-streams: dropping event after max delivery attempts", {
|
|
674
|
+
topic: sub.topic,
|
|
675
|
+
eventType: event.type,
|
|
676
|
+
eventId: event.id,
|
|
677
|
+
attempt,
|
|
678
|
+
max: this.#maxDeliveryAttempts
|
|
679
|
+
});
|
|
680
|
+
try {
|
|
681
|
+
await this.#writeClient.xAck(sub.streamKey, sub.group, streamId);
|
|
682
|
+
} catch (err) {
|
|
683
|
+
this.#logger?.debug?.("valkey-streams: ack on dropped poison message failed", {
|
|
684
|
+
topic: sub.topic,
|
|
685
|
+
err: err instanceof Error ? err.message : err
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
const next = {
|
|
691
|
+
...event,
|
|
692
|
+
deliveryAttempt: attempt + 1
|
|
693
|
+
};
|
|
694
|
+
try {
|
|
695
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
696
|
+
else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) });
|
|
697
|
+
} catch (err) {
|
|
698
|
+
this.#logger?.warn?.("valkey-streams: nack republish failed; leaving original pending for reclaim", {
|
|
699
|
+
topic: sub.topic,
|
|
700
|
+
eventId: event.id,
|
|
701
|
+
err: err instanceof Error ? err.message : err
|
|
702
|
+
});
|
|
703
|
+
settled = false;
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
try {
|
|
707
|
+
await this.#writeClient.xAck(sub.streamKey, sub.group, streamId);
|
|
708
|
+
} catch (err) {
|
|
709
|
+
this.#logger?.debug?.("valkey-streams: xAck after nack failed", {
|
|
710
|
+
topic: sub.topic,
|
|
711
|
+
err: err instanceof Error ? err.message : err
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
};
|
|
715
|
+
try {
|
|
716
|
+
const result = sub.cb(event, ack, nack);
|
|
717
|
+
if (result && typeof result.catch === "function") result.catch(async () => {
|
|
718
|
+
await nack();
|
|
719
|
+
});
|
|
720
|
+
} catch {
|
|
721
|
+
await nack();
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
#deliverLocal(topic, event) {
|
|
725
|
+
const callbacks = this.#localCallbacks.get(topic);
|
|
726
|
+
if (!callbacks || callbacks.size === 0) return;
|
|
727
|
+
for (const cb of callbacks) this.#invokeLocalCallback(topic, event, cb);
|
|
728
|
+
}
|
|
729
|
+
#invokeLocalCallback(topic, event, cb) {
|
|
730
|
+
const ack = async () => {};
|
|
731
|
+
const nack = async () => {};
|
|
732
|
+
try {
|
|
733
|
+
const result = cb(event, ack, nack);
|
|
734
|
+
if (result && typeof result.catch === "function") result.catch((err) => {
|
|
735
|
+
this.#logger?.debug?.("valkey-streams: local subscriber rejected", {
|
|
736
|
+
topic,
|
|
737
|
+
err: err instanceof Error ? err.message : err
|
|
738
|
+
});
|
|
739
|
+
});
|
|
740
|
+
} catch (err) {
|
|
741
|
+
this.#logger?.debug?.("valkey-streams: local subscriber threw", {
|
|
742
|
+
topic,
|
|
743
|
+
err: err instanceof Error ? err.message : err
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
};
|
|
748
|
+
//#endregion
|
|
749
|
+
exports.ValkeyStreamsPubSub = ValkeyStreamsPubSub;
|
|
750
|
+
|
|
751
|
+
//# sourceMappingURL=index.cjs.map
|