@mastra/redis-streams 0.6.3 → 0.6.4-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -26,7 +26,8 @@ const EXTEND_IF_OWNED_SCRIPT = `
26
26
  * KEYS[1] stream key
27
27
  * ARGV[1] group, ARGV[2] consumer, ARGV[3] stream entry id,
28
28
  * ARGV[4] republish payload ('' = drop without republish),
29
- * ARGV[5] stream idle TTL in ms (0 = none)
29
+ * ARGV[5] stream idle TTL in ms (0 = none),
30
+ * ARGV[6] approximate MAXLEN for the republish (0 = no trim)
30
31
  */
31
32
  const NACK_IF_OWNED_SCRIPT = `
32
33
  local pending = redis.call("XPENDING", KEYS[1], ARGV[1], ARGV[3], ARGV[3], 1)
@@ -34,7 +35,11 @@ const NACK_IF_OWNED_SCRIPT = `
34
35
  return 0
35
36
  end
36
37
  if ARGV[4] ~= "" then
37
- redis.call("XADD", KEYS[1], "*", "event", ARGV[4])
38
+ if tonumber(ARGV[6]) > 0 then
39
+ redis.call("XADD", KEYS[1], "MAXLEN", "~", ARGV[6], "*", "event", ARGV[4])
40
+ else
41
+ redis.call("XADD", KEYS[1], "*", "event", ARGV[4])
42
+ end
38
43
  if tonumber(ARGV[5]) > 0 then
39
44
  redis.call("PEXPIRE", KEYS[1], ARGV[5])
40
45
  end
@@ -240,12 +245,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
240
245
  createdAt,
241
246
  deliveryAttempt: event.deliveryAttempt ?? 1
242
247
  };
243
- const xaddOptions = {};
244
- if (this.#maxStreamLength > 0) xaddOptions.TRIM = {
245
- strategy: "MAXLEN",
246
- strategyModifier: "~",
247
- threshold: this.#maxStreamLength
248
- };
248
+ const xaddOptions = this.#xaddTrimOptions();
249
249
  const streamKey = this.#streamKey(topic);
250
250
  await (this.#streamIdleTtlMs > 0 ? this.#writeClient.multi().xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions).pExpire(streamKey, this.#streamIdleTtlMs).exec().then((replies) => {
251
251
  if (Number(replies[1]) !== 1) this.#logger?.warn?.("redis-streams: PEXPIRE inside publish MULTI did not apply", {
@@ -254,6 +254,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
254
254
  });
255
255
  }) : this.#writeClient.xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions));
256
256
  }
257
+ #xaddTrimOptions() {
258
+ if (this.#maxStreamLength <= 0) return {};
259
+ return { TRIM: {
260
+ strategy: "MAXLEN",
261
+ strategyModifier: "~",
262
+ threshold: this.#maxStreamLength
263
+ } };
264
+ }
257
265
  async subscribe(topic, cb, options) {
258
266
  if (this.#closed) throw new Error("RedisStreamsPubSub: cannot subscribe on closed client");
259
267
  const key = this.#subKey(topic, cb);
@@ -269,36 +277,47 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
269
277
  async #doSubscribe(topic, cb, key, options) {
270
278
  let localBucket = this.#localCallbacks.get(topic);
271
279
  if (!localBucket) {
272
- localBucket = /* @__PURE__ */ new Set();
280
+ localBucket = /* @__PURE__ */ new Map();
273
281
  this.#localCallbacks.set(topic, localBucket);
274
282
  }
275
- localBucket.add(cb);
276
- await this.#ensureWriterConnected();
277
- const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
278
- if (options?.group) {
279
- const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
280
- this.#subscriptions.set(key, sub);
281
- sub.loop = this.#runReadLoop(sub);
282
- this.#startReclaimLoop(sub);
283
- return;
284
- }
285
- for (;;) {
286
- const reader = this.#fanoutReaders.get(topic);
287
- if (reader) return this.#joinFanoutReader(reader, cb, key, groupAnchor);
288
- const opening = this.#pendingFanoutReaders.get(topic);
289
- if (!opening) break;
290
- await opening.catch(() => {});
291
- }
292
- const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
293
- this.#pendingFanoutReaders.set(topic, opening);
283
+ const timers = /* @__PURE__ */ new Set();
284
+ localBucket.set(cb, timers);
294
285
  try {
295
- const sub = await opening;
296
- this.#fanoutReaders.set(topic, sub);
297
- this.#subscriptions.set(key, sub);
298
- sub.loop = this.#runReadLoop(sub);
299
- this.#startReclaimLoop(sub);
300
- } finally {
301
- this.#pendingFanoutReaders.delete(topic);
286
+ await this.#ensureWriterConnected();
287
+ const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
288
+ if (options?.group) {
289
+ const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
290
+ this.#subscriptions.set(key, sub);
291
+ sub.loop = this.#runReadLoop(sub);
292
+ this.#startReclaimLoop(sub);
293
+ return;
294
+ }
295
+ for (;;) {
296
+ const reader = this.#fanoutReaders.get(topic);
297
+ if (reader) return await this.#joinFanoutReader(reader, cb, key, groupAnchor);
298
+ const opening = this.#pendingFanoutReaders.get(topic);
299
+ if (!opening) break;
300
+ await opening.catch(() => {});
301
+ }
302
+ const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
303
+ this.#pendingFanoutReaders.set(topic, opening);
304
+ try {
305
+ const sub = await opening;
306
+ this.#fanoutReaders.set(topic, sub);
307
+ this.#subscriptions.set(key, sub);
308
+ sub.loop = this.#runReadLoop(sub);
309
+ this.#startReclaimLoop(sub);
310
+ } finally {
311
+ this.#pendingFanoutReaders.delete(topic);
312
+ }
313
+ } catch (err) {
314
+ for (const timer of timers) clearTimeout(timer);
315
+ timers.clear();
316
+ if (this.#localCallbacks.get(topic)?.get(cb) === timers) {
317
+ localBucket.delete(cb);
318
+ if (localBucket.size === 0) this.#localCallbacks.delete(topic);
319
+ }
320
+ throw err;
302
321
  }
303
322
  }
304
323
  /**
@@ -465,6 +484,11 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
465
484
  async unsubscribe(topic, cb) {
466
485
  const localBucket = this.#localCallbacks.get(topic);
467
486
  if (localBucket) {
487
+ const timers = localBucket.get(cb);
488
+ if (timers) {
489
+ for (const timer of timers) clearTimeout(timer);
490
+ timers.clear();
491
+ }
468
492
  localBucket.delete(cb);
469
493
  if (localBucket.size === 0) this.#localCallbacks.delete(topic);
470
494
  }
@@ -711,10 +735,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
711
735
  async close() {
712
736
  if (this.#closed) return;
713
737
  this.#closed = true;
738
+ for (const callbacks of this.#localCallbacks.values()) for (const timers of callbacks.values()) {
739
+ for (const timer of timers) clearTimeout(timer);
740
+ timers.clear();
741
+ }
742
+ this.#localCallbacks.clear();
714
743
  if (this.#pendingSubscribes.size > 0) await Promise.allSettled([...this.#pendingSubscribes.values()]);
715
744
  const subs = new Set(this.#subscriptions.values());
716
745
  await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
717
- this.#localCallbacks.clear();
718
746
  await this.flush();
719
747
  if (this.#writeClient.isOpen) try {
720
748
  await disconnect(this.#writeClient);
@@ -828,8 +856,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
828
856
  deliveryAttempt: attempt + 1
829
857
  };
830
858
  try {
831
- if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
832
- else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) });
859
+ if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
860
+ else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions());
833
861
  } catch (err) {
834
862
  this.#logger?.warn?.("redis-streams: nack republish failed; leaving original pending for reclaim", {
835
863
  topic: sub.topic,
@@ -870,7 +898,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
870
898
  sub.consumer,
871
899
  streamId,
872
900
  payload,
873
- String(this.#streamIdleTtlMs)
901
+ String(this.#streamIdleTtlMs),
902
+ String(this.#maxStreamLength > 0 ? this.#maxStreamLength : 0)
874
903
  ]
875
904
  });
876
905
  } catch (err) {
@@ -1013,8 +1042,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
1013
1042
  deliveryAttempt: attempt + 1
1014
1043
  }) };
1015
1044
  try {
1016
- if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
1017
- else await this.#writeClient.xAdd(sub.streamKey, "*", payload);
1045
+ if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
1046
+ else await this.#writeClient.xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions());
1018
1047
  } catch (err) {
1019
1048
  this.#logger?.warn?.("redis-streams: nack republish failed for replayed event", {
1020
1049
  topic: sub.topic,
@@ -1030,11 +1059,39 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
1030
1059
  #deliverLocal(topic, event) {
1031
1060
  const callbacks = this.#localCallbacks.get(topic);
1032
1061
  if (!callbacks || callbacks.size === 0) return;
1033
- for (const cb of callbacks) this.#invokeLocalCallback(topic, event, cb);
1062
+ for (const [cb, timers] of callbacks) this.#invokeLocalCallback(topic, event, cb, timers);
1034
1063
  }
1035
- #invokeLocalCallback(topic, event, cb) {
1036
- const ack = async () => {};
1037
- const nack = async () => {};
1064
+ #invokeLocalCallback(topic, event, cb, timers) {
1065
+ if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
1066
+ let settled = false;
1067
+ const ack = async () => {
1068
+ settled = true;
1069
+ };
1070
+ const nack = async () => {
1071
+ if (settled) return;
1072
+ settled = true;
1073
+ if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
1074
+ const attempt = event.deliveryAttempt ?? 1;
1075
+ if (attempt >= this.#maxDeliveryAttempts) {
1076
+ this.#logger?.warn?.("redis-streams: dropping local event after max delivery attempts", {
1077
+ topic,
1078
+ eventId: event.id,
1079
+ eventType: event.type,
1080
+ attempt,
1081
+ max: this.#maxDeliveryAttempts
1082
+ });
1083
+ return;
1084
+ }
1085
+ const delay = Math.min(1e3 * 2 ** Math.min(attempt - 1, 5), 3e4);
1086
+ const timer = setTimeout(() => {
1087
+ timers.delete(timer);
1088
+ this.#invokeLocalCallback(topic, {
1089
+ ...event,
1090
+ deliveryAttempt: attempt + 1
1091
+ }, cb, timers);
1092
+ }, delay);
1093
+ timers.add(timer);
1094
+ };
1038
1095
  try {
1039
1096
  const result = cb(event, ack, nack);
1040
1097
  if (result && typeof result.catch === "function") result.catch((err) => {
@@ -1042,12 +1099,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
1042
1099
  topic,
1043
1100
  err: err instanceof Error ? err.message : err
1044
1101
  });
1102
+ nack();
1045
1103
  });
1046
1104
  } catch (err) {
1047
1105
  this.#logger?.debug?.("redis-streams: local subscriber threw", {
1048
1106
  topic,
1049
1107
  err: err instanceof Error ? err.message : err
1050
1108
  });
1109
+ nack();
1051
1110
  }
1052
1111
  }
1053
1112
  };