@mastra/redis-streams 0.6.3 → 0.6.4-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -25,7 +25,8 @@ const EXTEND_IF_OWNED_SCRIPT = `
25
25
  * KEYS[1] stream key
26
26
  * ARGV[1] group, ARGV[2] consumer, ARGV[3] stream entry id,
27
27
  * ARGV[4] republish payload ('' = drop without republish),
28
- * ARGV[5] stream idle TTL in ms (0 = none)
28
+ * ARGV[5] stream idle TTL in ms (0 = none),
29
+ * ARGV[6] approximate MAXLEN for the republish (0 = no trim)
29
30
  */
30
31
  const NACK_IF_OWNED_SCRIPT = `
31
32
  local pending = redis.call("XPENDING", KEYS[1], ARGV[1], ARGV[3], ARGV[3], 1)
@@ -33,7 +34,11 @@ const NACK_IF_OWNED_SCRIPT = `
33
34
  return 0
34
35
  end
35
36
  if ARGV[4] ~= "" then
36
- redis.call("XADD", KEYS[1], "*", "event", ARGV[4])
37
+ if tonumber(ARGV[6]) > 0 then
38
+ redis.call("XADD", KEYS[1], "MAXLEN", "~", ARGV[6], "*", "event", ARGV[4])
39
+ else
40
+ redis.call("XADD", KEYS[1], "*", "event", ARGV[4])
41
+ end
37
42
  if tonumber(ARGV[5]) > 0 then
38
43
  redis.call("PEXPIRE", KEYS[1], ARGV[5])
39
44
  end
@@ -239,12 +244,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
239
244
  createdAt,
240
245
  deliveryAttempt: event.deliveryAttempt ?? 1
241
246
  };
242
- const xaddOptions = {};
243
- if (this.#maxStreamLength > 0) xaddOptions.TRIM = {
244
- strategy: "MAXLEN",
245
- strategyModifier: "~",
246
- threshold: this.#maxStreamLength
247
- };
247
+ const xaddOptions = this.#xaddTrimOptions();
248
248
  const streamKey = this.#streamKey(topic);
249
249
  await (this.#streamIdleTtlMs > 0 ? this.#writeClient.multi().xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions).pExpire(streamKey, this.#streamIdleTtlMs).exec().then((replies) => {
250
250
  if (Number(replies[1]) !== 1) this.#logger?.warn?.("redis-streams: PEXPIRE inside publish MULTI did not apply", {
@@ -253,6 +253,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
253
253
  });
254
254
  }) : this.#writeClient.xAdd(streamKey, "*", { event: JSON.stringify(payload) }, xaddOptions));
255
255
  }
256
+ #xaddTrimOptions() {
257
+ if (this.#maxStreamLength <= 0) return {};
258
+ return { TRIM: {
259
+ strategy: "MAXLEN",
260
+ strategyModifier: "~",
261
+ threshold: this.#maxStreamLength
262
+ } };
263
+ }
256
264
  async subscribe(topic, cb, options) {
257
265
  if (this.#closed) throw new Error("RedisStreamsPubSub: cannot subscribe on closed client");
258
266
  const key = this.#subKey(topic, cb);
@@ -268,36 +276,47 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
268
276
  async #doSubscribe(topic, cb, key, options) {
269
277
  let localBucket = this.#localCallbacks.get(topic);
270
278
  if (!localBucket) {
271
- localBucket = /* @__PURE__ */ new Set();
279
+ localBucket = /* @__PURE__ */ new Map();
272
280
  this.#localCallbacks.set(topic, localBucket);
273
281
  }
274
- localBucket.add(cb);
275
- await this.#ensureWriterConnected();
276
- const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
277
- if (options?.group) {
278
- const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
279
- this.#subscriptions.set(key, sub);
280
- sub.loop = this.#runReadLoop(sub);
281
- this.#startReclaimLoop(sub);
282
- return;
283
- }
284
- for (;;) {
285
- const reader = this.#fanoutReaders.get(topic);
286
- if (reader) return this.#joinFanoutReader(reader, cb, key, groupAnchor);
287
- const opening = this.#pendingFanoutReaders.get(topic);
288
- if (!opening) break;
289
- await opening.catch(() => {});
290
- }
291
- const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
292
- this.#pendingFanoutReaders.set(topic, opening);
282
+ const timers = /* @__PURE__ */ new Set();
283
+ localBucket.set(cb, timers);
293
284
  try {
294
- const sub = await opening;
295
- this.#fanoutReaders.set(topic, sub);
296
- this.#subscriptions.set(key, sub);
297
- sub.loop = this.#runReadLoop(sub);
298
- this.#startReclaimLoop(sub);
299
- } finally {
300
- this.#pendingFanoutReaders.delete(topic);
285
+ await this.#ensureWriterConnected();
286
+ const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
287
+ if (options?.group) {
288
+ const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
289
+ this.#subscriptions.set(key, sub);
290
+ sub.loop = this.#runReadLoop(sub);
291
+ this.#startReclaimLoop(sub);
292
+ return;
293
+ }
294
+ for (;;) {
295
+ const reader = this.#fanoutReaders.get(topic);
296
+ if (reader) return await this.#joinFanoutReader(reader, cb, key, groupAnchor);
297
+ const opening = this.#pendingFanoutReaders.get(topic);
298
+ if (!opening) break;
299
+ await opening.catch(() => {});
300
+ }
301
+ const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
302
+ this.#pendingFanoutReaders.set(topic, opening);
303
+ try {
304
+ const sub = await opening;
305
+ this.#fanoutReaders.set(topic, sub);
306
+ this.#subscriptions.set(key, sub);
307
+ sub.loop = this.#runReadLoop(sub);
308
+ this.#startReclaimLoop(sub);
309
+ } finally {
310
+ this.#pendingFanoutReaders.delete(topic);
311
+ }
312
+ } catch (err) {
313
+ for (const timer of timers) clearTimeout(timer);
314
+ timers.clear();
315
+ if (this.#localCallbacks.get(topic)?.get(cb) === timers) {
316
+ localBucket.delete(cb);
317
+ if (localBucket.size === 0) this.#localCallbacks.delete(topic);
318
+ }
319
+ throw err;
301
320
  }
302
321
  }
303
322
  /**
@@ -464,6 +483,11 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
464
483
  async unsubscribe(topic, cb) {
465
484
  const localBucket = this.#localCallbacks.get(topic);
466
485
  if (localBucket) {
486
+ const timers = localBucket.get(cb);
487
+ if (timers) {
488
+ for (const timer of timers) clearTimeout(timer);
489
+ timers.clear();
490
+ }
467
491
  localBucket.delete(cb);
468
492
  if (localBucket.size === 0) this.#localCallbacks.delete(topic);
469
493
  }
@@ -710,10 +734,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
710
734
  async close() {
711
735
  if (this.#closed) return;
712
736
  this.#closed = true;
737
+ for (const callbacks of this.#localCallbacks.values()) for (const timers of callbacks.values()) {
738
+ for (const timer of timers) clearTimeout(timer);
739
+ timers.clear();
740
+ }
741
+ this.#localCallbacks.clear();
713
742
  if (this.#pendingSubscribes.size > 0) await Promise.allSettled([...this.#pendingSubscribes.values()]);
714
743
  const subs = new Set(this.#subscriptions.values());
715
744
  await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
716
- this.#localCallbacks.clear();
717
745
  await this.flush();
718
746
  if (this.#writeClient.isOpen) try {
719
747
  await disconnect(this.#writeClient);
@@ -827,8 +855,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
827
855
  deliveryAttempt: attempt + 1
828
856
  };
829
857
  try {
830
- if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
831
- else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) });
858
+ if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
859
+ else await this.#writeClient.xAdd(sub.streamKey, "*", { event: JSON.stringify(next) }, this.#xaddTrimOptions());
832
860
  } catch (err) {
833
861
  this.#logger?.warn?.("redis-streams: nack republish failed; leaving original pending for reclaim", {
834
862
  topic: sub.topic,
@@ -869,7 +897,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
869
897
  sub.consumer,
870
898
  streamId,
871
899
  payload,
872
- String(this.#streamIdleTtlMs)
900
+ String(this.#streamIdleTtlMs),
901
+ String(this.#maxStreamLength > 0 ? this.#maxStreamLength : 0)
873
902
  ]
874
903
  });
875
904
  } catch (err) {
@@ -1012,8 +1041,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
1012
1041
  deliveryAttempt: attempt + 1
1013
1042
  }) };
1014
1043
  try {
1015
- if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
1016
- else await this.#writeClient.xAdd(sub.streamKey, "*", payload);
1044
+ if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions()).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
1045
+ else await this.#writeClient.xAdd(sub.streamKey, "*", payload, this.#xaddTrimOptions());
1017
1046
  } catch (err) {
1018
1047
  this.#logger?.warn?.("redis-streams: nack republish failed for replayed event", {
1019
1048
  topic: sub.topic,
@@ -1029,11 +1058,39 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
1029
1058
  #deliverLocal(topic, event) {
1030
1059
  const callbacks = this.#localCallbacks.get(topic);
1031
1060
  if (!callbacks || callbacks.size === 0) return;
1032
- for (const cb of callbacks) this.#invokeLocalCallback(topic, event, cb);
1061
+ for (const [cb, timers] of callbacks) this.#invokeLocalCallback(topic, event, cb, timers);
1033
1062
  }
1034
- #invokeLocalCallback(topic, event, cb) {
1035
- const ack = async () => {};
1036
- const nack = async () => {};
1063
+ #invokeLocalCallback(topic, event, cb, timers) {
1064
+ if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
1065
+ let settled = false;
1066
+ const ack = async () => {
1067
+ settled = true;
1068
+ };
1069
+ const nack = async () => {
1070
+ if (settled) return;
1071
+ settled = true;
1072
+ if (this.#closed || this.#localCallbacks.get(topic)?.get(cb) !== timers) return;
1073
+ const attempt = event.deliveryAttempt ?? 1;
1074
+ if (attempt >= this.#maxDeliveryAttempts) {
1075
+ this.#logger?.warn?.("redis-streams: dropping local event after max delivery attempts", {
1076
+ topic,
1077
+ eventId: event.id,
1078
+ eventType: event.type,
1079
+ attempt,
1080
+ max: this.#maxDeliveryAttempts
1081
+ });
1082
+ return;
1083
+ }
1084
+ const delay = Math.min(1e3 * 2 ** Math.min(attempt - 1, 5), 3e4);
1085
+ const timer = setTimeout(() => {
1086
+ timers.delete(timer);
1087
+ this.#invokeLocalCallback(topic, {
1088
+ ...event,
1089
+ deliveryAttempt: attempt + 1
1090
+ }, cb, timers);
1091
+ }, delay);
1092
+ timers.add(timer);
1093
+ };
1037
1094
  try {
1038
1095
  const result = cb(event, ack, nack);
1039
1096
  if (result && typeof result.catch === "function") result.catch((err) => {
@@ -1041,12 +1098,14 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
1041
1098
  topic,
1042
1099
  err: err instanceof Error ? err.message : err
1043
1100
  });
1101
+ nack();
1044
1102
  });
1045
1103
  } catch (err) {
1046
1104
  this.#logger?.debug?.("redis-streams: local subscriber threw", {
1047
1105
  topic,
1048
1106
  err: err instanceof Error ? err.message : err
1049
1107
  });
1108
+ nack();
1050
1109
  }
1051
1110
  }
1052
1111
  };