@mastra/redis-streams 0.6.3 → 0.6.4-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +104 -45
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +104 -45
- package/dist/index.js.map +1 -1
- package/package.json +10 -11
package/dist/index.js
CHANGED
|
@@ -25,7 +25,8 @@ const EXTEND_IF_OWNED_SCRIPT = `
|
|
|
25
25
|
* KEYS[1] stream key
|
|
26
26
|
* ARGV[1] group, ARGV[2] consumer, ARGV[3] stream entry id,
|
|
27
27
|
* ARGV[4] republish payload ('' = drop without republish),
|
|
28
|
-
* ARGV[5] stream idle TTL in ms (0 = none)
|
|
28
|
+
* ARGV[5] stream idle TTL in ms (0 = none),
|
|
29
|
+
* ARGV[6] approximate MAXLEN for the republish (0 = no trim)
|
|
29
30
|
*/
|
|
30
31
|
const NACK_IF_OWNED_SCRIPT = `
|
|
31
32
|
local pending = redis.call("XPENDING", KEYS[1], ARGV[1], ARGV[3], ARGV[3], 1)
|
|
@@ -33,7 +34,11 @@ const NACK_IF_OWNED_SCRIPT = `
|
|
|
33
34
|
return 0
|
|
34
35
|
end
|
|
35
36
|
if ARGV[4] ~= "" then
|
|
36
|
-
|
|
37
|
+
if tonumber(ARGV[6]) > 0 then
|
|
38
|
+
redis.call("XADD", KEYS[1], "MAXLEN", "~", ARGV[6], "*", "event", ARGV[4])
|
|
39
|
+
else
|
|
40
|
+
redis.call("XADD", KEYS[1], "*", "event", ARGV[4])
|
|
41
|
+
end
|
|
37
42
|
if tonumber(ARGV[5]) > 0 then
|
|
38
43
|
redis.call("PEXPIRE", KEYS[1], ARGV[5])
|
|
39
44
|
end
|
|
@@ -239,12 +244,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
239
244
|
createdAt,
|
|
240
245
|
deliveryAttempt: event.deliveryAttempt ?? 1
|
|
241
246
|
};
|
|
242
|
-
const xaddOptions =
|
|
243
|
-
if (this.#maxStreamLength > 0) xaddOptions.TRIM = {
|
|
244
|
-
strategy: "MAXLEN",
|
|
245
|
-
strategyModifier: "~",
|
|
246
|
-
threshold: this.#maxStreamLength
|
|
247
|
-
};
|
|
247
|
+
const xaddOptions = this.#xaddTrimOptions();
|
|
248
248
|
const streamKey = this.#streamKey(topic);
|
|
249
249
|
await (this.#streamIdleTtlMs > 0 ? this.#writeClient.multi().xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions).pExpire(streamKey, this.#streamIdleTtlMs).exec().then((replies) => {
|
|
250
250
|
if (Number(replies[1]) !== 1) this.#logger?.warn?.("redis-streams: PEXPIRE inside publish MULTI did not apply", {
|
|
@@ -253,6 +253,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
253
253
|
});
|
|
254
254
|
}) : this.#writeClient.xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions));
|
|
255
255
|
}
|
|
256
|
+
#xaddTrimOptions() {
|
|
257
|
+
if (this.#maxStreamLength <= 0) return {};
|
|
258
|
+
return { TRIM: {
|
|
259
|
+
strategy: "MAXLEN",
|
|
260
|
+
strategyModifier: "~",
|
|
261
|
+
threshold: this.#maxStreamLength
|
|
262
|
+
} };
|
|
263
|
+
}
|
|
256
264
|
async subscribe(topic, cb, options) {
|
|
257
265
|
if (this.#closed) throw new Error("RedisStreamsPubSub: cannot subscribe on closed client");
|
|
258
266
|
const key = this.#subKey(topic, cb);
|
|
@@ -268,36 +276,47 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
268
276
|
async #doSubscribe(topic, cb, key, options) {
|
|
269
277
|
let localBucket = this.#localCallbacks.get(topic);
|
|
270
278
|
if (!localBucket) {
|
|
271
|
-
localBucket = /* @__PURE__ */ new
|
|
279
|
+
localBucket = /* @__PURE__ */ new Map();
|
|
272
280
|
this.#localCallbacks.set(topic, localBucket);
|
|
273
281
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
|
|
277
|
-
if (options?.group) {
|
|
278
|
-
const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
|
|
279
|
-
this.#subscriptions.set(key, sub);
|
|
280
|
-
sub.loop = this.#runReadLoop(sub);
|
|
281
|
-
this.#startReclaimLoop(sub);
|
|
282
|
-
return;
|
|
283
|
-
}
|
|
284
|
-
for (;;) {
|
|
285
|
-
const reader = this.#fanoutReaders.get(topic);
|
|
286
|
-
if (reader) return this.#joinFanoutReader(reader, cb, key, groupAnchor);
|
|
287
|
-
const opening = this.#pendingFanoutReaders.get(topic);
|
|
288
|
-
if (!opening) break;
|
|
289
|
-
await opening.catch(() => {});
|
|
290
|
-
}
|
|
291
|
-
const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
|
|
292
|
-
this.#pendingFanoutReaders.set(topic, opening);
|
|
282
|
+
const timers = /* @__PURE__ */ new Set();
|
|
283
|
+
localBucket.set(cb, timers);
|
|
293
284
|
try {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
285
|
+
await this.#ensureWriterConnected();
|
|
286
|
+
const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
|
|
287
|
+
if (options?.group) {
|
|
288
|
+
const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
|
|
289
|
+
this.#subscriptions.set(key, sub);
|
|
290
|
+
sub.loop = this.#runReadLoop(sub);
|
|
291
|
+
this.#startReclaimLoop(sub);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
for (;;) {
|
|
295
|
+
const reader = this.#fanoutReaders.get(topic);
|
|
296
|
+
if (reader) return await this.#joinFanoutReader(reader, cb, key, groupAnchor);
|
|
297
|
+
const opening = this.#pendingFanoutReaders.get(topic);
|
|
298
|
+
if (!opening) break;
|
|
299
|
+
await opening.catch(() => {});
|
|
300
|
+
}
|
|
301
|
+
const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
|
|
302
|
+
this.#pendingFanoutReaders.set(topic, opening);
|
|
303
|
+
try {
|
|
304
|
+
const sub = await opening;
|
|
305
|
+
this.#fanoutReaders.set(topic, sub);
|
|
306
|
+
this.#subscriptions.set(key, sub);
|
|
307
|
+
sub.loop = this.#runReadLoop(sub);
|
|
308
|
+
this.#startReclaimLoop(sub);
|
|
309
|
+
} finally {
|
|
310
|
+
this.#pendingFanoutReaders.delete(topic);
|
|
311
|
+
}
|
|
312
|
+
} catch (err) {
|
|
313
|
+
for (const timer of timers) clearTimeout(timer);
|
|
314
|
+
timers.clear();
|
|
315
|
+
if (this.#localCallbacks.get(topic)?.get(cb) === timers) {
|
|
316
|
+
localBucket.delete(cb);
|
|
317
|
+
if (localBucket.size === 0) this.#localCallbacks.delete(topic);
|
|
318
|
+
}
|
|
319
|
+
throw err;
|
|
301
320
|
}
|
|
302
321
|
}
|
|
303
322
|
/**
|
|
@@ -464,6 +483,11 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
464
483
|
async unsubscribe(topic, cb) {
|
|
465
484
|
const localBucket = this.#localCallbacks.get(topic);
|
|
466
485
|
if (localBucket) {
|
|
486
|
+
const timers = localBucket.get(cb);
|
|
487
|
+
if (timers) {
|
|
488
|
+
for (const timer of timers) clearTimeout(timer);
|
|
489
|
+
timers.clear();
|
|
490
|
+
}
|
|
467
491
|
localBucket.delete(cb);
|
|
468
492
|
if (localBucket.size === 0) this.#localCallbacks.delete(topic);
|
|
469
493
|
}
|
|
@@ -710,10 +734,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
710
734
|
async close() {
|
|
711
735
|
if (this.#closed) return;
|
|
712
736
|
this.#closed = true;
|
|
737
|
+
for (const callbacks of this.#localCallbacks.values()) for (const timers of callbacks.values()) {
|
|
738
|
+
for (const timer of timers) clearTimeout(timer);
|
|
739
|
+
timers.clear();
|
|
740
|
+
}
|
|
741
|
+
this.#localCallbacks.clear();
|
|
713
742
|
if (this.#pendingSubscribes.size > 0) await Promise.allSettled([...this.#pendingSubscribes.values()]);
|
|
714
743
|
const subs = new Set(this.#subscriptions.values());
|
|
715
744
|
await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
|
|
716
|
-
this.#localCallbacks.clear();
|
|
717
745
|
await this.flush();
|
|
718
746
|
if (this.#writeClient.isOpen) try {
|
|
719
747
|
await disconnect(this.#writeClient);
|
|
@@ -827,8 +855,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
827
855
|
deliveryAttempt: attempt + 1
|
|
828
856
|
};
|
|
829
857
|
try {
|
|
830
|
-
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
831
|
-
else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) });
|
|
858
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
859
|
+
else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions());
|
|
832
860
|
} catch (err) {
|
|
833
861
|
this.#logger?.warn?.("redis-streams: nack republish failed; leaving original pending for reclaim", {
|
|
834
862
|
topic: sub.topic,
|
|
@@ -869,7 +897,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
869
897
|
sub.consumer,
|
|
870
898
|
streamId,
|
|
871
899
|
payload,
|
|
872
|
-
String(this.#streamIdleTtlMs)
|
|
900
|
+
String(this.#streamIdleTtlMs),
|
|
901
|
+
String(this.#maxStreamLength > 0 ? this.#maxStreamLength : 0)
|
|
873
902
|
]
|
|
874
903
|
});
|
|
875
904
|
} catch (err) {
|
|
@@ -1012,8 +1041,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
1012
1041
|
deliveryAttempt: attempt + 1
|
|
1013
1042
|
}) };
|
|
1014
1043
|
try {
|
|
1015
|
-
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
1016
|
-
else await this.#writeClient.xAdd(sub.streamKey, "*", payload);
|
|
1044
|
+
if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
|
|
1045
|
+
else await this.#writeClient.xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions());
|
|
1017
1046
|
} catch (err) {
|
|
1018
1047
|
this.#logger?.warn?.("redis-streams: nack republish failed for replayed event", {
|
|
1019
1048
|
topic: sub.topic,
|
|
@@ -1029,11 +1058,39 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
1029
1058
|
#deliverLocal(topic, event) {
|
|
1030
1059
|
const callbacks = this.#localCallbacks.get(topic);
|
|
1031
1060
|
if (!callbacks || callbacks.size === 0) return;
|
|
1032
|
-
for (const cb of callbacks) this.#invokeLocalCallback(topic, event, cb);
|
|
1061
|
+
for (const [cb, timers] of callbacks) this.#invokeLocalCallback(topic, event, cb, timers);
|
|
1033
1062
|
}
|
|
1034
|
-
#invokeLocalCallback(topic, event, cb) {
|
|
1035
|
-
|
|
1036
|
-
|
|
1063
|
+
#invokeLocalCallback(topic, event, cb, timers) {
|
|
1064
|
+
if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
|
|
1065
|
+
let settled = false;
|
|
1066
|
+
const ack = async () => {
|
|
1067
|
+
settled = true;
|
|
1068
|
+
};
|
|
1069
|
+
const nack = async () => {
|
|
1070
|
+
if (settled) return;
|
|
1071
|
+
settled = true;
|
|
1072
|
+
if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
|
|
1073
|
+
const attempt = event.deliveryAttempt ?? 1;
|
|
1074
|
+
if (attempt >= this.#maxDeliveryAttempts) {
|
|
1075
|
+
this.#logger?.warn?.("redis-streams: dropping local event after max delivery attempts", {
|
|
1076
|
+
topic,
|
|
1077
|
+
eventId: event.id,
|
|
1078
|
+
eventType: event.type,
|
|
1079
|
+
attempt,
|
|
1080
|
+
max: this.#maxDeliveryAttempts
|
|
1081
|
+
});
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
const delay = Math.min(1e3 * 2 ** Math.min(attempt - 1, 5), 3e4);
|
|
1085
|
+
const timer = setTimeout(() => {
|
|
1086
|
+
timers.delete(timer);
|
|
1087
|
+
this.#invokeLocalCallback(topic, {
|
|
1088
|
+
...event,
|
|
1089
|
+
deliveryAttempt: attempt + 1
|
|
1090
|
+
}, cb, timers);
|
|
1091
|
+
}, delay);
|
|
1092
|
+
timers.add(timer);
|
|
1093
|
+
};
|
|
1037
1094
|
try {
|
|
1038
1095
|
const result = cb(event, ack, nack);
|
|
1039
1096
|
if (result && typeof result.catch === "function") result.catch((err) => {
|
|
@@ -1041,12 +1098,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
|
|
|
1041
1098
|
topic,
|
|
1042
1099
|
err: err instanceof Error ? err.message : err
|
|
1043
1100
|
});
|
|
1101
|
+
nack();
|
|
1044
1102
|
});
|
|
1045
1103
|
} catch (err) {
|
|
1046
1104
|
this.#logger?.debug?.("redis-streams: local subscriber threw", {
|
|
1047
1105
|
topic,
|
|
1048
1106
|
err: err instanceof Error ? err.message : err
|
|
1049
1107
|
});
|
|
1108
|
+
nack();
|
|
1050
1109
|
}
|
|
1051
1110
|
}
|
|
1052
1111
|
};
|