@mastra/redis-streams 0.7.0 → 0.8.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -24,7 +24,7 @@ export const mastra = new Mastra({
24
24
  });
25
25
  ```
26
26
 
27
- For production deployments, set `streamIdleTtlMs` to put a sliding idle TTL on every stream (for example, `7 * 24 * 60 * 60 * 1000` for 7 days). Every write refreshes it, so actively written topics never expire, while abandoned ones (a crashed run, or open-ended topics that never reach a `clearTopic` call) are reclaimed by Redis once they go idle. Reads do not refresh the TTL, so keep it above the longest expected gap between writes on a live topic, including the time a grouped consumer may need to replay a backlog. Without it (the default), those streams stay in Redis forever and its memory grows without bound.
27
+ Every stream gets a sliding idle TTL, controlled by `streamIdleTtlMs` (default: 7 days). Every write refreshes it, so actively written topics never expire, while abandoned ones (a crashed run, or open-ended topics that never reach a `clearTopic` call) are reclaimed by Redis once they go idle. Reads do not refresh the TTL, so keep it above the longest expected gap between writes on a live topic, including the time a grouped consumer may need to replay a backlog. Setting `streamIdleTtlMs: 0` disables expiry; those streams then stay in Redis forever and its memory grows without bound.
28
28
 
29
29
  ### Redis Cluster
30
30
 
package/dist/index.cjs CHANGED
@@ -4,6 +4,7 @@ let redis = require("redis");
4
4
  // src/index.ts
5
5
  /** Page size for the reclaim loop's XPENDING scan and the max entries claimed per tick. */
6
6
  const RECLAIM_PAGE_SIZE = 100;
7
+ const DEFAULT_STREAM_IDLE_TTL_MS = 6048e5;
7
8
  const TRIM_PAGE_SIZE = 500;
8
9
  /**
9
10
  * Heartbeat: reset a pending entry's idle time only if this consumer still
@@ -68,6 +69,71 @@ function compareStreamIds(a, b) {
68
69
  return seq === 0n ? 0 : seq < 0n ? -1 : 1;
69
70
  }
70
71
  /**
72
+ * Connect a client, standalone or cluster, within its `socket.connectTimeout`
73
+ * (node-redis default: 5000 ms; 0 disables the bound).
74
+ *
75
+ * node-redis applies `connectTimeout` only to opening the TCP socket. The
76
+ * handshake that follows (CLIENT SETINFO, HELLO/AUTH, SELECT) has no deadline,
77
+ * and the default reconnectStrategy retries a failed attempt forever. So a
78
+ * server that accepts the socket but never replies (a SIGSTOPped process, or a
79
+ * proxy that stopped forwarding) makes `connect()` hang forever. The same
80
+ * budget therefore bounds the whole connect here. On expiry the client is
81
+ * destroyed, which stops its retry loop and lets a later call connect again.
82
+ *
83
+ * destroy() does not stop a cluster's slot discovery: once the nodes answer it
84
+ * still connects node clients that nothing would close. So when the abandoned
85
+ * connect settles, the client is destroyed again unless it is open. Until then
86
+ * no new connect starts on that client: a second discovery would share the
87
+ * first one's node entries, settle before they connect, and its destroy()
88
+ * would miss them. A call made meanwhile waits for the abandoned connect
89
+ * within its own deadline, then connects.
90
+ */
91
+ const abandonedConnects = /* @__PURE__ */ new WeakMap();
92
+ async function connectWithin(client, role, isReleased = () => false) {
93
+ const cluster = client;
94
+ const timeoutMs = (cluster._options ? cluster._options.defaults?.socket : client.options?.socket)?.connectTimeout ?? 5e3;
95
+ let timer;
96
+ let connecting;
97
+ const timedOut = new Promise((_, reject) => {
98
+ if (!timeoutMs) return;
99
+ timer = setTimeout(() => {
100
+ if (connecting) {
101
+ const settled = connecting.catch(() => {}).then(() => {
102
+ if (abandonedConnects.get(client) === settled) abandonedConnects.delete(client);
103
+ if (client.isOpen) return;
104
+ try {
105
+ client.destroy();
106
+ } catch {}
107
+ });
108
+ abandonedConnects.set(client, settled);
109
+ if (client.isOpen) client.destroy();
110
+ }
111
+ reject(/* @__PURE__ */ new Error(`redis-streams: ${role} client did not connect to Redis within ${timeoutMs}ms (socket.connectTimeout); the server is unreachable or not responding`));
112
+ }, timeoutMs);
113
+ });
114
+ try {
115
+ const abandoned = abandonedConnects.get(client);
116
+ if (abandoned) {
117
+ await Promise.race([abandoned, timedOut]);
118
+ if (isReleased()) throw new Error(`redis-streams: ${role} client closed while connecting`);
119
+ }
120
+ connecting = client.connect();
121
+ await Promise.race([connecting, timedOut]);
122
+ } finally {
123
+ clearTimeout(timer);
124
+ }
125
+ }
126
+ /**
127
+ * Parse `maxmemory` / `maxmemory_policy` out of an `INFO memory` reply and
128
+ * return the policy when the server can evict keys under memory pressure.
129
+ */
130
+ function evictingMaxmemoryPolicy(info) {
131
+ const field = (name) => info.match(new RegExp(`^${name}:(.*)$`, "m"))?.[1]?.trim();
132
+ const policy = field("maxmemory_policy");
133
+ if (!policy || policy === "noeviction" || Number(field("maxmemory") ?? 0) <= 0) return void 0;
134
+ return policy;
135
+ }
136
+ /**
71
137
  * Disconnect a client, standalone or cluster.
72
138
  *
73
139
  * node-redis 5 `quit()` never settles on a client that is open but waiting to
@@ -116,6 +182,9 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
116
182
  #pendingPublishes = /* @__PURE__ */ new Set();
117
183
  #localCallbacks = /* @__PURE__ */ new Map();
118
184
  #closed = false;
185
+ /** Set once close() has drained publishes and is about to release the writer. */
186
+ #writerReleased = false;
187
+ #evictionPolicyChecked = false;
119
188
  constructor(options = {}) {
120
189
  super();
121
190
  this.#writeClient = RedisStreamsPubSub.#createWriteClient(options);
@@ -124,7 +193,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
124
193
  this.#keyPrefix = options.keyPrefix ?? "mastra:topic";
125
194
  this.#blockMs = options.blockMs ?? 1e3;
126
195
  this.#maxStreamLength = options.maxStreamLength ?? 1e4;
127
- const ttl = options.streamIdleTtlMs ?? 0;
196
+ const ttl = options.streamIdleTtlMs ?? DEFAULT_STREAM_IDLE_TTL_MS;
128
197
  if (!Number.isInteger(ttl) || ttl < 0) throw new Error(`redis-streams: streamIdleTtlMs must be a non-negative integer (milliseconds), got ${ttl}`);
129
198
  this.#streamIdleTtlMs = ttl;
130
199
  this.#reclaimIntervalMs = options.reclaimIntervalMs ?? 3e4;
@@ -207,11 +276,33 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
207
276
  /** Lazily connect the shared writer client. Idempotent and safe under concurrent callers. */
208
277
  async #ensureWriterConnected() {
209
278
  if (!this.#writerConnecting && this.#writeClient.isOpen) return;
210
- if (!this.#writerConnecting) this.#writerConnecting = this.#writeClient.connect().then(() => void 0).finally(() => {
279
+ if (!this.#writerConnecting) this.#writerConnecting = connectWithin(this.#writeClient, "write", () => this.#writerReleased).then(() => {
280
+ if (!this.#evictionPolicyChecked) {
281
+ this.#evictionPolicyChecked = true;
282
+ this.#warnOnEvictingPolicy();
283
+ }
284
+ }).finally(() => {
211
285
  this.#writerConnecting = void 0;
212
286
  });
213
287
  return this.#writerConnecting;
214
288
  }
289
+ /**
290
+ * Consumer-group state, lease keys and TTL'd streams are all live data. A
291
+ * server with `maxmemory` set and any policy other than `noeviction` deletes
292
+ * them under memory pressure (silently losing events and leases), so warn
293
+ * once. INFO rather than CONFIG GET because managed Redis often blocks CONFIG.
294
+ */
295
+ async #warnOnEvictingPolicy() {
296
+ try {
297
+ const client = this.#writeClient;
298
+ const reply = Array.isArray(client.masters) ? await this.#writeClient.sendCommand(void 0, true, ["INFO", "memory"]) : await this.#writeClient.sendCommand(["INFO", "memory"]);
299
+ const policy = evictingMaxmemoryPolicy(String(reply));
300
+ if (!policy) return;
301
+ (this.#logger?.warn ? this.#logger.warn.bind(this.#logger) : console.warn)(`redis-streams: Redis maxmemory-policy is "${policy}". Under memory pressure Redis will evict pubsub keys (TTL'd streams, consumer groups, lease keys), losing events and leases. Set maxmemory-policy to "noeviction".`);
302
+ } catch (err) {
303
+ this.#logger?.debug?.("redis-streams: maxmemory-policy check failed", { err: err instanceof Error ? err.message : String(err) });
304
+ }
305
+ }
215
306
  #streamKey(topic) {
216
307
  return `${this.#keyPrefix}:${topic}`;
217
308
  }
@@ -388,7 +479,18 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
388
479
  }
389
480
  const readClient = this.#createReadClient();
390
481
  this.#attachErrorLogger(readClient, "read", { topic });
391
- await readClient.connect();
482
+ try {
483
+ await connectWithin(readClient, "read");
484
+ } catch (err) {
485
+ if (!isGrouped) this.#writeClient.xGroupDestroy(streamKey, group).catch((destroyErr) => {
486
+ this.#logger?.debug?.("redis-streams: xGroupDestroy failed", {
487
+ topic,
488
+ group,
489
+ error: errorText(destroyErr)
490
+ });
491
+ });
492
+ throw err;
493
+ }
392
494
  return {
393
495
  members: /* @__PURE__ */ new Map([[cb, {}]]),
394
496
  settling: /* @__PURE__ */ new Map(),
@@ -750,6 +852,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
750
852
  const subs = new Set(this.#subscriptions.values());
751
853
  await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
752
854
  await this.flush();
855
+ this.#writerReleased = true;
753
856
  if (this.#writeClient.isOpen) try {
754
857
  await disconnect(this.#writeClient);
755
858
  } catch (err) {