@mastra/valkey-streams 0.5.3 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +8 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/index.cjs +254 -20
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +254 -20
- package/dist/index.js.map +1 -1
- package/package.json +7 -7
package/dist/client.d.ts
CHANGED
|
@@ -72,6 +72,14 @@ export declare class ValkeyStreamsClient {
|
|
|
72
72
|
[k: string]: string;
|
|
73
73
|
};
|
|
74
74
|
}[]>;
|
|
75
|
+
xRange(key: string, start: string, end: string, options?: {
|
|
76
|
+
COUNT?: number;
|
|
77
|
+
}): Promise<StreamMessage[]>;
|
|
78
|
+
xRevRange(key: string, end: string, start: string, options?: {
|
|
79
|
+
COUNT?: number;
|
|
80
|
+
}): Promise<StreamMessage[]>;
|
|
81
|
+
/** XINFO GROUPS <key>, one object per group keyed by field name. */
|
|
82
|
+
xInfoGroups(key: string): Promise<Array<Record<string, unknown>>>;
|
|
75
83
|
xAck(key: string, group: string, id: string): Promise<number>;
|
|
76
84
|
del(key: string): Promise<number>;
|
|
77
85
|
set(key: string, value: string, options?: {
|
package/dist/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,WAAW,EAEX,KAAK,wBAAwB,EAC7B,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAE9B,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,WAAW,CAAC,GAAG;IACvF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,wBAAwB,CAAC,WAAW,CAAC,CAAC;CACnD,CAAC;AAEF,KAAK,aAAa,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC;AACrE,KAAK,WAAW,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,WAAW,EAEX,KAAK,wBAAwB,EAC7B,KAAK,WAAW,EACjB,MAAM,sBAAsB,CAAC;AAE9B,MAAM,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,WAAW,CAAC,GAAG;IACvF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,wBAAwB,CAAC,WAAW,CAAC,CAAC;CACnD,CAAC;AAEF,KAAK,aAAa,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC;AACrE,KAAK,WAAW,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC;AAoB/D,cAAM,WAAW;;IAEH,OAAO,CAAC,QAAQ,CAAC,SAAS;IAAtC,YAA6B,SAAS,EAAE,MAAM,OAAO,CAAC,WAAW,CAAC,EAAI;IACtE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAA;SAAE,CAAA;KAAO,QAM3G;IACD,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,QAG/B;IACD,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,QAGxF;IACK,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAE/B;CACF;AAED,qBAAa,mBAAmB;;IAElB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAApC,YAA6B,OAAO,EAAE,mBAAmB,EAAI;IAC7D,IAAI,MAAM,YAET;IACD,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,QAEtD;IACK,OAAO,kBAEZ;IACK,IAAI,kBAET;IACD,KAAK,gBAEJ;IACK,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,2DAEhC;IACK,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAA;SAAE,CAAA;KAAO,mBAKjH;IACK,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,mBAErC;IACK,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,iBAE9F;IACK,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,mBAE7C;IACK,UAAU,CACd,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,EAC3C,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAC/C,OAAO,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,CAiB/B;IACD;;;OAGG;IACG,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO;;;;;SAe/G;IACK,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE;;;;;SAQxF;IACK,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,4BAErF;IACK,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,4BAExF;IACD,oEAAoE;IAC9D,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAGtE;IACK,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,mBAEhD;IACK,GAAG,CAAC,GAAG,EAAE,MAAM,mBAEpB;IACK,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,EAAE,CAAC,EAAE,OAAO,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAO,0BAMhF;IACK,GAAG,CAAC,GAAG,EAAE,MAAM,0BAGpB;IACK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,mBAI1E;IACD,OAAO,CAAC,SAAS;CAiBlB;AAED,eAAO,MAAM,YAAY,YAAa,mBAAmB,wBAAqC,CAAC;AAC/F,MAAM,MAAM,gBAAgB,GAAG,mBAAmB,CAAC"}
|
package/dist/index.cjs
CHANGED
|
@@ -3,6 +3,12 @@ let _mastra_core_events = require("@mastra/core/events");
|
|
|
3
3
|
let _valkey_valkey_glide = require("@valkey/valkey-glide");
|
|
4
4
|
//#region src/client.ts
|
|
5
5
|
const text = (value) => Buffer.isBuffer(value) ? value.toString() : String(value);
|
|
6
|
+
const countArgs = (options) => options.COUNT === void 0 ? [] : ["COUNT", String(options.COUNT)];
|
|
7
|
+
/** Decode an XRANGE/XREVRANGE reply (`[{ key: id, value: [[field, value], …] }]`). */
|
|
8
|
+
const records = (value) => (value ?? []).map((entry) => ({
|
|
9
|
+
id: text(entry.key),
|
|
10
|
+
message: Object.fromEntries((entry.value ?? []).map(([field, fieldValue]) => [text(field), text(fieldValue)]))
|
|
11
|
+
}));
|
|
6
12
|
var ValkeyMulti = class {
|
|
7
13
|
getClient;
|
|
8
14
|
#batch = new _valkey_valkey_glide.Batch(true);
|
|
@@ -120,6 +126,32 @@ var ValkeyStreamsClient = class {
|
|
|
120
126
|
message: Object.fromEntries(fields.map(([field, value]) => [text(field), text(value)]))
|
|
121
127
|
}));
|
|
122
128
|
}
|
|
129
|
+
async xRange(key, start, end, options = {}) {
|
|
130
|
+
return records(await this.command([
|
|
131
|
+
"XRANGE",
|
|
132
|
+
key,
|
|
133
|
+
start,
|
|
134
|
+
end,
|
|
135
|
+
...countArgs(options)
|
|
136
|
+
]));
|
|
137
|
+
}
|
|
138
|
+
async xRevRange(key, end, start, options = {}) {
|
|
139
|
+
return records(await this.command([
|
|
140
|
+
"XREVRANGE",
|
|
141
|
+
key,
|
|
142
|
+
end,
|
|
143
|
+
start,
|
|
144
|
+
...countArgs(options)
|
|
145
|
+
]));
|
|
146
|
+
}
|
|
147
|
+
/** XINFO GROUPS <key>, one object per group keyed by field name. */
|
|
148
|
+
async xInfoGroups(key) {
|
|
149
|
+
return (await this.command([
|
|
150
|
+
"XINFO",
|
|
151
|
+
"GROUPS",
|
|
152
|
+
key
|
|
153
|
+
]) ?? []).map((group) => Object.fromEntries(group.map(({ key, value }) => [text(key), value])));
|
|
154
|
+
}
|
|
123
155
|
async xAck(key, group, id) {
|
|
124
156
|
return (await this.getClient()).xack(key, group, [id]);
|
|
125
157
|
}
|
|
@@ -175,6 +207,8 @@ const createClient = (options) => new ValkeyStreamsClient(options);
|
|
|
175
207
|
//#region src/index.ts
|
|
176
208
|
/** Page size for the reclaim loop's XPENDING scan and the max entries claimed per tick. */
|
|
177
209
|
const RECLAIM_PAGE_SIZE = 100;
|
|
210
|
+
/** Page size for replaying existing entries to a fan-out subscriber joining a shared reader. */
|
|
211
|
+
const REPLAY_PAGE_SIZE = 500;
|
|
178
212
|
/**
|
|
179
213
|
* Heartbeat: reset a pending entry's idle time only if this consumer still
|
|
180
214
|
* owns it, so a late heartbeat never takes an entry back from a sibling.
|
|
@@ -223,6 +257,15 @@ function errorText(err) {
|
|
|
223
257
|
if (err instanceof Error && "replies" in err && Array.isArray(err.replies)) parts.push(...err.replies.map((r) => String(r)));
|
|
224
258
|
return parts.join("; ");
|
|
225
259
|
}
|
|
260
|
+
/** Order two stream ids (`<ms>-<seq>`). */
|
|
261
|
+
function compareStreamIds(a, b) {
|
|
262
|
+
const [aMs = "0", aSeq = "0"] = a.split("-");
|
|
263
|
+
const [bMs = "0", bSeq = "0"] = b.split("-");
|
|
264
|
+
const ms = BigInt(aMs) - BigInt(bMs);
|
|
265
|
+
if (ms !== 0n) return ms < 0n ? -1 : 1;
|
|
266
|
+
const seq = BigInt(aSeq) - BigInt(bSeq);
|
|
267
|
+
return seq === 0n ? 0 : seq < 0n ? -1 : 1;
|
|
268
|
+
}
|
|
226
269
|
var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
227
270
|
get supportedModes() {
|
|
228
271
|
return ["pull"];
|
|
@@ -245,6 +288,8 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
245
288
|
#inFlightTimeoutMs;
|
|
246
289
|
#logger;
|
|
247
290
|
#subscriptions = /* @__PURE__ */ new Map();
|
|
291
|
+
#fanoutReaders = /* @__PURE__ */ new Map();
|
|
292
|
+
#pendingFanoutReaders = /* @__PURE__ */ new Map();
|
|
248
293
|
#cbIds = /* @__PURE__ */ new WeakMap();
|
|
249
294
|
#pendingPublishes = /* @__PURE__ */ new Set();
|
|
250
295
|
#localCallbacks = /* @__PURE__ */ new Map();
|
|
@@ -374,11 +419,93 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
374
419
|
}
|
|
375
420
|
localBucket.add(cb);
|
|
376
421
|
await this.#ensureWriterConnected();
|
|
377
|
-
const
|
|
378
|
-
|
|
422
|
+
const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
|
|
423
|
+
if (options?.group) {
|
|
424
|
+
const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
|
|
425
|
+
this.#subscriptions.set(key, sub);
|
|
426
|
+
sub.loop = this.#runReadLoop(sub);
|
|
427
|
+
this.#startReclaimLoop(sub);
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
for (;;) {
|
|
431
|
+
const reader = this.#fanoutReaders.get(topic);
|
|
432
|
+
if (reader) return this.#joinFanoutReader(reader, cb, key, groupAnchor);
|
|
433
|
+
const opening = this.#pendingFanoutReaders.get(topic);
|
|
434
|
+
if (!opening) break;
|
|
435
|
+
await opening.catch(() => {});
|
|
436
|
+
}
|
|
437
|
+
const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
|
|
438
|
+
this.#pendingFanoutReaders.set(topic, opening);
|
|
439
|
+
try {
|
|
440
|
+
const sub = await opening;
|
|
441
|
+
this.#fanoutReaders.set(topic, sub);
|
|
442
|
+
this.#subscriptions.set(key, sub);
|
|
443
|
+
sub.loop = this.#runReadLoop(sub);
|
|
444
|
+
this.#startReclaimLoop(sub);
|
|
445
|
+
} finally {
|
|
446
|
+
this.#pendingFanoutReaders.delete(topic);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* Add a callback to a topic's open fan-out reader. The reader may already
|
|
451
|
+
* have delivered entries this subscriber should see (`startFrom: 'earliest'`)
|
|
452
|
+
* or still be catching up on entries it should not (`startFrom: 'latest'`),
|
|
453
|
+
* so the new member buffers live entries while its starting point is
|
|
454
|
+
* resolved, replays what it missed, then switches to live delivery.
|
|
455
|
+
*/
|
|
456
|
+
async #joinFanoutReader(sub, cb, key, anchor) {
|
|
457
|
+
if (sub.members.has(cb)) {
|
|
458
|
+
this.#subscriptions.set(key, sub);
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
const member = { joining: [] };
|
|
462
|
+
sub.members.set(cb, member);
|
|
463
|
+
this.#subscriptions.set(key, sub);
|
|
464
|
+
const active = () => sub.members.get(cb) === member && !sub.stopped;
|
|
465
|
+
try {
|
|
466
|
+
if (anchor === "$") {
|
|
467
|
+
const [tail] = await this.#writeClient.xRevRange(sub.streamKey, "+", "-", { COUNT: 1 });
|
|
468
|
+
member.skipThrough = tail?.id;
|
|
469
|
+
} else {
|
|
470
|
+
let boundary;
|
|
471
|
+
try {
|
|
472
|
+
const info = (await this.#writeClient.xInfoGroups(sub.streamKey)).find((g) => String(g.name) === sub.group);
|
|
473
|
+
boundary = info ? String(info["last-delivered-id"]) : void 0;
|
|
474
|
+
} catch (err) {
|
|
475
|
+
if (!errorText(err).match(/no such key|NOGROUP/i)) throw err;
|
|
476
|
+
}
|
|
477
|
+
if (boundary && boundary !== "0-0") {
|
|
478
|
+
member.skipThrough = boundary;
|
|
479
|
+
let start = "-";
|
|
480
|
+
while (active()) {
|
|
481
|
+
const page = await this.#writeClient.xRange(sub.streamKey, start, boundary, { COUNT: REPLAY_PAGE_SIZE });
|
|
482
|
+
for (const entry of page) {
|
|
483
|
+
if (!active()) break;
|
|
484
|
+
this.#deliverDetached(sub, cb, entry.message);
|
|
485
|
+
}
|
|
486
|
+
if (page.length < REPLAY_PAGE_SIZE) break;
|
|
487
|
+
start = `(${page[page.length - 1].id}`;
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
} catch (err) {
|
|
492
|
+
sub.members.delete(cb);
|
|
493
|
+
this.#subscriptions.delete(key);
|
|
494
|
+
throw err;
|
|
495
|
+
}
|
|
496
|
+
if (!active()) return;
|
|
497
|
+
const buffered = member.joining ?? [];
|
|
498
|
+
for (const entry of buffered) {
|
|
499
|
+
if (member.skipThrough && compareStreamIds(entry.id, member.skipThrough) <= 0) continue;
|
|
500
|
+
this.#deliverDetached(sub, cb, entry.fields);
|
|
501
|
+
}
|
|
502
|
+
member.joining = void 0;
|
|
503
|
+
}
|
|
504
|
+
async #openSubscription(topic, cb, groupName, groupAnchor) {
|
|
505
|
+
const isGrouped = groupName !== void 0;
|
|
506
|
+
const group = groupName ?? `__fanout-${globalThis.crypto.randomUUID()}`;
|
|
379
507
|
const consumer = `${group}-${globalThis.crypto.randomUUID()}`;
|
|
380
508
|
const streamKey = this.#streamKey(topic);
|
|
381
|
-
const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
|
|
382
509
|
try {
|
|
383
510
|
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true }).pExpire(streamKey, this.#streamIdleTtlMs).exec();
|
|
384
511
|
else await this.#writeClient.xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true });
|
|
@@ -392,8 +519,9 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
392
519
|
const readClient = createClient(this.#connectOptions);
|
|
393
520
|
this.#attachErrorLogger(readClient, "read", { topic });
|
|
394
521
|
await readClient.connect();
|
|
395
|
-
|
|
396
|
-
cb,
|
|
522
|
+
return {
|
|
523
|
+
members: /* @__PURE__ */ new Map([[cb, {}]]),
|
|
524
|
+
settling: /* @__PURE__ */ new Map(),
|
|
397
525
|
topic,
|
|
398
526
|
streamKey,
|
|
399
527
|
group,
|
|
@@ -407,9 +535,6 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
407
535
|
reclaimTimer: void 0,
|
|
408
536
|
inFlight: /* @__PURE__ */ new Map()
|
|
409
537
|
};
|
|
410
|
-
this.#subscriptions.set(key, sub);
|
|
411
|
-
sub.loop = this.#runReadLoop(sub);
|
|
412
|
-
this.#startReclaimLoop(sub);
|
|
413
538
|
}
|
|
414
539
|
/**
|
|
415
540
|
* Periodically reclaim idle pending entries in this subscription's group so
|
|
@@ -500,11 +625,21 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
500
625
|
const sub = this.#subscriptions.get(key);
|
|
501
626
|
if (!sub) return;
|
|
502
627
|
this.#subscriptions.delete(key);
|
|
628
|
+
if (sub.teardown) return sub.teardown;
|
|
629
|
+
if (sub.members.size > 1 && sub.members.delete(cb)) {
|
|
630
|
+
for (const settle of [...sub.settling.values()]) settle(cb);
|
|
631
|
+
return;
|
|
632
|
+
}
|
|
633
|
+
if (this.#fanoutReaders.get(topic) === sub) this.#fanoutReaders.delete(topic);
|
|
503
634
|
sub.stopped = true;
|
|
504
635
|
if (sub.reclaimTimer) {
|
|
505
636
|
clearTimeout(sub.reclaimTimer);
|
|
506
637
|
sub.reclaimTimer = void 0;
|
|
507
638
|
}
|
|
639
|
+
sub.teardown = this.#stopSubscription(sub);
|
|
640
|
+
return sub.teardown;
|
|
641
|
+
}
|
|
642
|
+
async #stopSubscription(sub) {
|
|
508
643
|
try {
|
|
509
644
|
await sub.readClient.quit();
|
|
510
645
|
} catch (err) {
|
|
@@ -560,6 +695,7 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
560
695
|
for (const sub of this.#subscriptions.values()) if (sub.topic === topic) {
|
|
561
696
|
sub.lastId = void 0;
|
|
562
697
|
sub.groupAnchor = "0";
|
|
698
|
+
for (const member of sub.members.values()) member.skipThrough = void 0;
|
|
563
699
|
}
|
|
564
700
|
} catch (err) {
|
|
565
701
|
this.#logger?.warn?.("valkey-streams: clearTopic failed", {
|
|
@@ -686,8 +822,8 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
686
822
|
async close() {
|
|
687
823
|
if (this.#closed) return;
|
|
688
824
|
this.#closed = true;
|
|
689
|
-
const subs =
|
|
690
|
-
await Promise.all(subs.
|
|
825
|
+
const subs = new Set(this.#subscriptions.values());
|
|
826
|
+
await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
|
|
691
827
|
this.#localCallbacks.clear();
|
|
692
828
|
await this.flush();
|
|
693
829
|
if (this.#writeClient.isOpen) try {
|
|
@@ -743,15 +879,11 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
743
879
|
}
|
|
744
880
|
}
|
|
745
881
|
async #deliverMessage(sub, streamId, fields) {
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
event = JSON.parse(fields.event ?? "{}");
|
|
749
|
-
if (typeof event.createdAt === "string") event.createdAt = new Date(event.createdAt);
|
|
750
|
-
} catch (err) {
|
|
882
|
+
const event = this.#parseEvent(fields);
|
|
883
|
+
if (!event) {
|
|
751
884
|
this.#logger?.debug?.("valkey-streams: malformed payload, dropping", {
|
|
752
885
|
topic: sub.topic,
|
|
753
|
-
streamId
|
|
754
|
-
err: err instanceof Error ? err.message : err
|
|
886
|
+
streamId
|
|
755
887
|
});
|
|
756
888
|
try {
|
|
757
889
|
await this.#writeClient.xAck(sub.streamKey, sub.group, streamId);
|
|
@@ -833,6 +965,7 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
833
965
|
const expire = async () => {
|
|
834
966
|
if (settled) return;
|
|
835
967
|
settled = true;
|
|
968
|
+
sub.settling.delete(streamId);
|
|
836
969
|
const attempt = event.deliveryAttempt ?? 1;
|
|
837
970
|
const drop = attempt >= this.#maxDeliveryAttempts;
|
|
838
971
|
const payload = drop ? "" : JSON.stringify({
|
|
@@ -858,7 +991,12 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
858
991
|
err: err instanceof Error ? err.message : err
|
|
859
992
|
});
|
|
860
993
|
settled = false;
|
|
861
|
-
sub.
|
|
994
|
+
sub.settling.set(streamId, (cb) => settleMember(cb, false));
|
|
995
|
+
if (sub.isGrouped) sub.inFlight.delete(streamId);
|
|
996
|
+
else sub.inFlight.set(streamId, {
|
|
997
|
+
since: Date.now(),
|
|
998
|
+
expire
|
|
999
|
+
});
|
|
862
1000
|
return;
|
|
863
1001
|
}
|
|
864
1002
|
sub.inFlight.delete(streamId);
|
|
@@ -891,19 +1029,115 @@ var ValkeyStreamsPubSub = class extends _mastra_core_events.PubSub {
|
|
|
891
1029
|
]
|
|
892
1030
|
});
|
|
893
1031
|
};
|
|
1032
|
+
const waiting = /* @__PURE__ */ new Set();
|
|
1033
|
+
let anyNack = false;
|
|
1034
|
+
const settleMember = async (cb, nacked) => {
|
|
1035
|
+
if (settled || !waiting.delete(cb)) return;
|
|
1036
|
+
if (nacked) anyNack = true;
|
|
1037
|
+
if (waiting.size > 0) return;
|
|
1038
|
+
sub.settling.delete(streamId);
|
|
1039
|
+
await (anyNack ? nack() : ack());
|
|
1040
|
+
};
|
|
1041
|
+
const targets = [];
|
|
1042
|
+
for (const [cb, member] of sub.members) {
|
|
1043
|
+
if (member.joining) {
|
|
1044
|
+
member.joining.push({
|
|
1045
|
+
id: streamId,
|
|
1046
|
+
fields
|
|
1047
|
+
});
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
if (member.skipThrough) {
|
|
1051
|
+
if (compareStreamIds(streamId, member.skipThrough) <= 0) continue;
|
|
1052
|
+
member.skipThrough = void 0;
|
|
1053
|
+
}
|
|
1054
|
+
waiting.add(cb);
|
|
1055
|
+
targets.push(cb);
|
|
1056
|
+
}
|
|
1057
|
+
if (targets.length === 0) {
|
|
1058
|
+
await ack();
|
|
1059
|
+
return;
|
|
1060
|
+
}
|
|
894
1061
|
sub.inFlight.set(streamId, {
|
|
895
1062
|
since: Date.now(),
|
|
896
1063
|
expire
|
|
897
1064
|
});
|
|
1065
|
+
sub.settling.set(streamId, (cb) => settleMember(cb, false));
|
|
1066
|
+
for (const cb of targets) {
|
|
1067
|
+
const memberEvent = targets.length === 1 ? event : this.#parseEvent(fields);
|
|
1068
|
+
this.#invokeCallback(cb, memberEvent, () => settleMember(cb, false), () => settleMember(cb, true), extend);
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
/**
|
|
1072
|
+
* Invoke a handler without awaiting it. EventCallback is typed `=> void`
|
|
1073
|
+
* but handlers commonly return a promise (TS allows Promise<void> to satisfy
|
|
1074
|
+
* void); a rejection or synchronous throw routes to nack instead of silently
|
|
1075
|
+
* dropping the message. We do NOT await — serializing messages on a
|
|
1076
|
+
* subscription would deadlock orchestration callbacks that await their own
|
|
1077
|
+
* future events.
|
|
1078
|
+
*/
|
|
1079
|
+
#invokeCallback(cb, event, ack, nack, extend) {
|
|
898
1080
|
try {
|
|
899
|
-
const result =
|
|
1081
|
+
const result = cb(event, ack, nack, extend);
|
|
900
1082
|
if (result && typeof result.catch === "function") result.catch(async () => {
|
|
901
1083
|
await nack();
|
|
902
1084
|
});
|
|
903
1085
|
} catch {
|
|
904
|
-
|
|
1086
|
+
nack();
|
|
905
1087
|
}
|
|
906
1088
|
}
|
|
1089
|
+
#parseEvent(fields) {
|
|
1090
|
+
try {
|
|
1091
|
+
const event = JSON.parse(fields.event ?? "{}");
|
|
1092
|
+
if (typeof event.createdAt === "string") event.createdAt = new Date(event.createdAt);
|
|
1093
|
+
return event;
|
|
1094
|
+
} catch {
|
|
1095
|
+
return;
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* Deliver an entry to one fan-out member outside the shared group's PEL:
|
|
1100
|
+
* entries the topic's reader handed out before the member joined. Ack is a
|
|
1101
|
+
* no-op (the group settles the entry on its own); nack republishes, which
|
|
1102
|
+
* redelivers to every member just as a nack on the shared entry would.
|
|
1103
|
+
*/
|
|
1104
|
+
#deliverDetached(sub, cb, fields) {
|
|
1105
|
+
const event = this.#parseEvent(fields);
|
|
1106
|
+
if (!event) return;
|
|
1107
|
+
let settled = false;
|
|
1108
|
+
const nack = async () => {
|
|
1109
|
+
if (settled) return;
|
|
1110
|
+
settled = true;
|
|
1111
|
+
const attempt = event.deliveryAttempt ?? 1;
|
|
1112
|
+
if (attempt >= this.#maxDeliveryAttempts) {
|
|
1113
|
+
this.#logger?.warn?.("valkey-streams: dropping event after max delivery attempts", {
|
|
1114
|
+
topic: sub.topic,
|
|
1115
|
+
eventType: event.type,
|
|
1116
|
+
eventId: event.id,
|
|
1117
|
+
attempt,
|
|
1118
|
+
max: this.#maxDeliveryAttempts
|
|
1119
|
+
});
|
|
1120
|
+
return;
|
|
1121
|
+
}
|
|
1122
|
+
const payload = { event: JSON.stringify({
|
|
1123
|
+
...event,
|
|
1124
|
+
deliveryAttempt: attempt + 1
|
|
1125
|
+
}) };
|
|
1126
|
+
try {
|
|
1127
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
1128
|
+
else await this.#writeClient.xAdd(sub.streamKey, "*", payload);
|
|
1129
|
+
} catch (err) {
|
|
1130
|
+
this.#logger?.warn?.("valkey-streams: nack republish failed for replayed event", {
|
|
1131
|
+
topic: sub.topic,
|
|
1132
|
+
eventId: event.id,
|
|
1133
|
+
err: err instanceof Error ? err.message : err
|
|
1134
|
+
});
|
|
1135
|
+
}
|
|
1136
|
+
};
|
|
1137
|
+
this.#invokeCallback(cb, event, async () => {
|
|
1138
|
+
settled = true;
|
|
1139
|
+
}, nack);
|
|
1140
|
+
}
|
|
907
1141
|
#deliverLocal(topic, event) {
|
|
908
1142
|
const callbacks = this.#localCallbacks.get(topic);
|
|
909
1143
|
if (!callbacks || callbacks.size === 0) return;
|