@mastra/redis-streams 0.6.3 → 0.6.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +104 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +104 -45
- package/dist/index.js.map +1 -1
- package/package.json +10 -11
package/dist/index.cjs
CHANGED
|
@@ -26,7 +26,8 @@ const EXTEND_IF_OWNED_SCRIPT = `
|
|
|
26
26
|
* KEYS[1] stream key
|
|
27
27
|
* ARGV[1] group, ARGV[2] consumer, ARGV[3] stream entry id,
|
|
28
28
|
* ARGV[4] republish payload ('' = drop without republish),
|
|
29
|
-
* ARGV[5] stream idle TTL in ms (0 = none)
|
|
29
|
+
* ARGV[5] stream idle TTL in ms (0 = none),
|
|
30
|
+
* ARGV[6] approximate MAXLEN for the republish (0 = no trim)
|
|
30
31
|
*/
|
|
31
32
|
const NACK_IF_OWNED_SCRIPT = `
|
|
32
33
|
local pending = redis.call("XPENDING", KEYS[1], ARGV[1], ARGV[3], ARGV[3], 1)
|
|
@@ -34,7 +35,11 @@ const NACK_IF_OWNED_SCRIPT = `
|
|
|
34
35
|
return 0
|
|
35
36
|
end
|
|
36
37
|
if ARGV[4] ~= "" then
|
|
37
|
-
|
|
38
|
+
if tonumber(ARGV[6]) > 0 then
|
|
39
|
+
redis.call("XADD", KEYS[1], "MAXLEN", "~", ARGV[6], "*", "event", ARGV[4])
|
|
40
|
+
else
|
|
41
|
+
redis.call("XADD", KEYS[1], "*", "event", ARGV[4])
|
|
42
|
+
end
|
|
38
43
|
if tonumber(ARGV[5]) > 0 then
|
|
39
44
|
redis.call("PEXPIRE", KEYS[1], ARGV[5])
|
|
40
45
|
end
|
|
@@ -240,12 +245,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
240
245
|
createdAt,
|
|
241
246
|
deliveryAttempt: event.deliveryAttempt ?? 1
|
|
242
247
|
};
|
|
243
|
-
const xaddOptions =
|
|
244
|
-
if (this.#maxStreamLength > 0) xaddOptions.TRIM = {
|
|
245
|
-
strategy: "MAXLEN",
|
|
246
|
-
strategyModifier: "~",
|
|
247
|
-
threshold: this.#maxStreamLength
|
|
248
|
-
};
|
|
248
|
+
const xaddOptions = this.#xaddTrimOptions();
|
|
249
249
|
const streamKey = this.#streamKey(topic);
|
|
250
250
|
await (this.#streamIdleTtlMs > 0 ? this.#writeClient.multi().xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions).pExpire(streamKey, this.#streamIdleTtlMs).exec().then((replies) => {
|
|
251
251
|
if (Number(replies[1]) !== 1) this.#logger?.warn?.("redis-streams: PEXPIRE inside publish MULTI did not apply", {
|
|
@@ -254,6 +254,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
254
254
|
});
|
|
255
255
|
}) : this.#writeClient.xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions));
|
|
256
256
|
}
|
|
257
|
+
#xaddTrimOptions() {
|
|
258
|
+
if (this.#maxStreamLength <= 0) return {};
|
|
259
|
+
return { TRIM: {
|
|
260
|
+
strategy: "MAXLEN",
|
|
261
|
+
strategyModifier: "~",
|
|
262
|
+
threshold: this.#maxStreamLength
|
|
263
|
+
} };
|
|
264
|
+
}
|
|
257
265
|
async subscribe(topic, cb, options) {
|
|
258
266
|
if (this.#closed) throw new Error("RedisStreamsPubSub: cannot subscribe on closed client");
|
|
259
267
|
const key = this.#subKey(topic, cb);
|
|
@@ -269,36 +277,47 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
269
277
|
async #doSubscribe(topic, cb, key, options) {
|
|
270
278
|
let localBucket = this.#localCallbacks.get(topic);
|
|
271
279
|
if (!localBucket) {
|
|
272
|
-
localBucket = /* @__PURE__ */ new
|
|
280
|
+
localBucket = /* @__PURE__ */ new Map();
|
|
273
281
|
this.#localCallbacks.set(topic, localBucket);
|
|
274
282
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
|
|
278
|
-
if (options?.group) {
|
|
279
|
-
const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
|
|
280
|
-
this.#subscriptions.set(key, sub);
|
|
281
|
-
sub.loop = this.#runReadLoop(sub);
|
|
282
|
-
this.#startReclaimLoop(sub);
|
|
283
|
-
return;
|
|
284
|
-
}
|
|
285
|
-
for (;;) {
|
|
286
|
-
const reader = this.#fanoutReaders.get(topic);
|
|
287
|
-
if (reader) return this.#joinFanoutReader(reader, cb, key, groupAnchor);
|
|
288
|
-
const opening = this.#pendingFanoutReaders.get(topic);
|
|
289
|
-
if (!opening) break;
|
|
290
|
-
await opening.catch(() => {});
|
|
291
|
-
}
|
|
292
|
-
const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
|
|
293
|
-
this.#pendingFanoutReaders.set(topic, opening);
|
|
283
|
+
const timers = /* @__PURE__ */ new Set();
|
|
284
|
+
localBucket.set(cb, timers);
|
|
294
285
|
try {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
286
|
+
await this.#ensureWriterConnected();
|
|
287
|
+
const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
|
|
288
|
+
if (options?.group) {
|
|
289
|
+
const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
|
|
290
|
+
this.#subscriptions.set(key, sub);
|
|
291
|
+
sub.loop = this.#runReadLoop(sub);
|
|
292
|
+
this.#startReclaimLoop(sub);
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
for (;;) {
|
|
296
|
+
const reader = this.#fanoutReaders.get(topic);
|
|
297
|
+
if (reader) return await this.#joinFanoutReader(reader, cb, key, groupAnchor);
|
|
298
|
+
const opening = this.#pendingFanoutReaders.get(topic);
|
|
299
|
+
if (!opening) break;
|
|
300
|
+
await opening.catch(() => {});
|
|
301
|
+
}
|
|
302
|
+
const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
|
|
303
|
+
this.#pendingFanoutReaders.set(topic, opening);
|
|
304
|
+
try {
|
|
305
|
+
const sub = await opening;
|
|
306
|
+
this.#fanoutReaders.set(topic, sub);
|
|
307
|
+
this.#subscriptions.set(key, sub);
|
|
308
|
+
sub.loop = this.#runReadLoop(sub);
|
|
309
|
+
this.#startReclaimLoop(sub);
|
|
310
|
+
} finally {
|
|
311
|
+
this.#pendingFanoutReaders.delete(topic);
|
|
312
|
+
}
|
|
313
|
+
} catch (err) {
|
|
314
|
+
for (const timer of timers) clearTimeout(timer);
|
|
315
|
+
timers.clear();
|
|
316
|
+
if (this.#localCallbacks.get(topic)?.get(cb) === timers) {
|
|
317
|
+
localBucket.delete(cb);
|
|
318
|
+
if (localBucket.size === 0) this.#localCallbacks.delete(topic);
|
|
319
|
+
}
|
|
320
|
+
throw err;
|
|
302
321
|
}
|
|
303
322
|
}
|
|
304
323
|
/**
|
|
@@ -465,6 +484,11 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
465
484
|
async unsubscribe(topic, cb) {
|
|
466
485
|
const localBucket = this.#localCallbacks.get(topic);
|
|
467
486
|
if (localBucket) {
|
|
487
|
+
const timers = localBucket.get(cb);
|
|
488
|
+
if (timers) {
|
|
489
|
+
for (const timer of timers) clearTimeout(timer);
|
|
490
|
+
timers.clear();
|
|
491
|
+
}
|
|
468
492
|
localBucket.delete(cb);
|
|
469
493
|
if (localBucket.size === 0) this.#localCallbacks.delete(topic);
|
|
470
494
|
}
|
|
@@ -711,10 +735,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
711
735
|
async close() {
|
|
712
736
|
if (this.#closed) return;
|
|
713
737
|
this.#closed = true;
|
|
738
|
+
for (const callbacks of this.#localCallbacks.values()) for (const timers of callbacks.values()) {
|
|
739
|
+
for (const timer of timers) clearTimeout(timer);
|
|
740
|
+
timers.clear();
|
|
741
|
+
}
|
|
742
|
+
this.#localCallbacks.clear();
|
|
714
743
|
if (this.#pendingSubscribes.size > 0) await Promise.allSettled([...this.#pendingSubscribes.values()]);
|
|
715
744
|
const subs = new Set(this.#subscriptions.values());
|
|
716
745
|
await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
|
|
717
|
-
this.#localCallbacks.clear();
|
|
718
746
|
await this.flush();
|
|
719
747
|
if (this.#writeClient.isOpen) try {
|
|
720
748
|
await disconnect(this.#writeClient);
|
|
@@ -828,8 +856,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
828
856
|
deliveryAttempt: attempt + 1
|
|
829
857
|
};
|
|
830
858
|
try {
|
|
831
|
-
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
832
|
-
else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) });
|
|
859
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
860
|
+
else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions());
|
|
833
861
|
} catch (err) {
|
|
834
862
|
this.#logger?.warn?.("redis-streams: nack republish failed; leaving original pending for reclaim", {
|
|
835
863
|
topic: sub.topic,
|
|
@@ -870,7 +898,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
870
898
|
sub.consumer,
|
|
871
899
|
streamId,
|
|
872
900
|
payload,
|
|
873
|
-
String(this.#streamIdleTtlMs)
|
|
901
|
+
String(this.#streamIdleTtlMs),
|
|
902
|
+
String(this.#maxStreamLength > 0 ? this.#maxStreamLength : 0)
|
|
874
903
|
]
|
|
875
904
|
});
|
|
876
905
|
} catch (err) {
|
|
@@ -1013,8 +1042,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
1013
1042
|
deliveryAttempt: attempt + 1
|
|
1014
1043
|
}) };
|
|
1015
1044
|
try {
|
|
1016
|
-
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
1017
|
-
else await this.#writeClient.xAdd(sub.streamKey, "*", payload);
|
|
1045
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
1046
|
+
else await this.#writeClient.xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions());
|
|
1018
1047
|
} catch (err) {
|
|
1019
1048
|
this.#logger?.warn?.("redis-streams: nack republish failed for replayed event", {
|
|
1020
1049
|
topic: sub.topic,
|
|
@@ -1030,11 +1059,39 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
1030
1059
|
#deliverLocal(topic, event) {
|
|
1031
1060
|
const callbacks = this.#localCallbacks.get(topic);
|
|
1032
1061
|
if (!callbacks || callbacks.size === 0) return;
|
|
1033
|
-
for (const cb of callbacks) this.#invokeLocalCallback(topic, event, cb);
|
|
1062
|
+
for (const [cb, timers] of callbacks) this.#invokeLocalCallback(topic, event, cb, timers);
|
|
1034
1063
|
}
|
|
1035
|
-
#invokeLocalCallback(topic, event, cb) {
|
|
1036
|
-
|
|
1037
|
-
|
|
1064
|
+
#invokeLocalCallback(topic, event, cb, timers) {
|
|
1065
|
+
if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
|
|
1066
|
+
let settled = false;
|
|
1067
|
+
const ack = async () => {
|
|
1068
|
+
settled = true;
|
|
1069
|
+
};
|
|
1070
|
+
const nack = async () => {
|
|
1071
|
+
if (settled) return;
|
|
1072
|
+
settled = true;
|
|
1073
|
+
if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
|
|
1074
|
+
const attempt = event.deliveryAttempt ?? 1;
|
|
1075
|
+
if (attempt >= this.#maxDeliveryAttempts) {
|
|
1076
|
+
this.#logger?.warn?.("redis-streams: dropping local event after max delivery attempts", {
|
|
1077
|
+
topic,
|
|
1078
|
+
eventId: event.id,
|
|
1079
|
+
eventType: event.type,
|
|
1080
|
+
attempt,
|
|
1081
|
+
max: this.#maxDeliveryAttempts
|
|
1082
|
+
});
|
|
1083
|
+
return;
|
|
1084
|
+
}
|
|
1085
|
+
const delay = Math.min(1e3 * 2 ** Math.min(attempt - 1, 5), 3e4);
|
|
1086
|
+
const timer = setTimeout(() => {
|
|
1087
|
+
timers.delete(timer);
|
|
1088
|
+
this.#invokeLocalCallback(topic, {
|
|
1089
|
+
...event,
|
|
1090
|
+
deliveryAttempt: attempt + 1
|
|
1091
|
+
}, cb, timers);
|
|
1092
|
+
}, delay);
|
|
1093
|
+
timers.add(timer);
|
|
1094
|
+
};
|
|
1038
1095
|
try {
|
|
1039
1096
|
const result = cb(event, ack, nack);
|
|
1040
1097
|
if (result && typeof result.catch === "function") result.catch((err) => {
|
|
@@ -1042,12 +1099,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
1042
1099
|
topic,
|
|
1043
1100
|
err: err instanceof Error ? err.message : err
|
|
1044
1101
|
});
|
|
1102
|
+
nack();
|
|
1045
1103
|
});
|
|
1046
1104
|
} catch (err) {
|
|
1047
1105
|
this.#logger?.debug?.("redis-streams: local subscriber threw", {
|
|
1048
1106
|
topic,
|
|
1049
1107
|
err: err instanceof Error ? err.message : err
|
|
1050
1108
|
});
|
|
1109
|
+
nack();
|
|
1051
1110
|
}
|
|
1052
1111
|
}
|
|
1053
1112
|
};
|