@mastra/redis-streams 0.7.0-alpha.2 → 0.8.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -3,6 +3,7 @@ import { createClient, createCluster } from "redis";
3
3
  // src/index.ts
4
4
  /** Page size for the reclaim loop's XPENDING scan and the max entries claimed per tick. */
5
5
  const RECLAIM_PAGE_SIZE = 100;
6
+ const DEFAULT_STREAM_IDLE_TTL_MS = 6048e5;
6
7
  const TRIM_PAGE_SIZE = 500;
7
8
  /**
8
9
  * Heartbeat: reset a pending entry's idle time only if this consumer still
@@ -67,6 +68,71 @@ function compareStreamIds(a, b) {
67
68
  return seq === 0n ? 0 : seq < 0n ? -1 : 1;
68
69
  }
69
70
  /**
71
+ * Connect a client, standalone or cluster, within its `socket.connectTimeout`
72
+ * (node-redis default: 5000 ms; 0 disables the bound).
73
+ *
74
+ * node-redis applies `connectTimeout` only to opening the TCP socket. The
75
+ * handshake that follows (CLIENT SETINFO, HELLO/AUTH, SELECT) has no deadline,
76
+ * and the default reconnectStrategy retries a failed attempt forever. So a
77
+ * server that accepts the socket but never replies (a SIGSTOPped process, or a
78
+ * proxy that stopped forwarding) makes `connect()` hang forever. The same
79
+ * budget therefore bounds the whole connect here. On expiry the client is
80
+ * destroyed, which stops its retry loop and lets a later call connect again.
81
+ *
82
+ * destroy() does not stop a cluster's slot discovery: once the nodes answer it
83
+ * still connects node clients that nothing would close. So when the abandoned
84
+ * connect settles, the client is destroyed again unless it is open. Until then
85
+ * no new connect starts on that client: a second discovery would share the
86
+ * first one's node entries, settle before they connect, and its destroy()
87
+ * would miss them. A call made meanwhile waits for the abandoned connect
88
+ * within its own deadline, then connects.
89
+ */
90
+ const abandonedConnects = /* @__PURE__ */ new WeakMap();
91
+ async function connectWithin(client, role, isReleased = () => false) {
92
+ const cluster = client;
93
+ const timeoutMs = (cluster._options ? cluster._options.defaults?.socket : client.options?.socket)?.connectTimeout ?? 5e3;
94
+ let timer;
95
+ let connecting;
96
+ const timedOut = new Promise((_, reject) => {
97
+ if (!timeoutMs) return;
98
+ timer = setTimeout(() => {
99
+ if (connecting) {
100
+ const settled = connecting.catch(() => {}).then(() => {
101
+ if (abandonedConnects.get(client) === settled) abandonedConnects.delete(client);
102
+ if (client.isOpen) return;
103
+ try {
104
+ client.destroy();
105
+ } catch {}
106
+ });
107
+ abandonedConnects.set(client, settled);
108
+ if (client.isOpen) client.destroy();
109
+ }
110
+ reject(/* @__PURE__ */ new Error(`redis-streams: ${role} client did not connect to Redis within ${timeoutMs}ms (socket.connectTimeout); the server is unreachable or not responding`));
111
+ }, timeoutMs);
112
+ });
113
+ try {
114
+ const abandoned = abandonedConnects.get(client);
115
+ if (abandoned) {
116
+ await Promise.race([abandoned, timedOut]);
117
+ if (isReleased()) throw new Error(`redis-streams: ${role} client closed while connecting`);
118
+ }
119
+ connecting = client.connect();
120
+ await Promise.race([connecting, timedOut]);
121
+ } finally {
122
+ clearTimeout(timer);
123
+ }
124
+ }
125
+ /**
126
+ * Parse `maxmemory` / `maxmemory_policy` out of an `INFO memory` reply and
127
+ * return the policy when the server can evict keys under memory pressure.
128
+ */
129
+ function evictingMaxmemoryPolicy(info) {
130
+ const field = (name) => info.match(new RegExp(`^${name}:(.*)$`, "m"))?.[1]?.trim();
131
+ const policy = field("maxmemory_policy");
132
+ if (!policy || policy === "noeviction" || Number(field("maxmemory") ?? 0) <= 0) return void 0;
133
+ return policy;
134
+ }
135
+ /**
70
136
  * Disconnect a client, standalone or cluster.
71
137
  *
72
138
  * node-redis 5 `quit()` never settles on a client that is open but waiting to
@@ -115,6 +181,9 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
115
181
  #pendingPublishes = /* @__PURE__ */ new Set();
116
182
  #localCallbacks = /* @__PURE__ */ new Map();
117
183
  #closed = false;
184
+ /** Set once close() has drained publishes and is about to release the writer. */
185
+ #writerReleased = false;
186
+ #evictionPolicyChecked = false;
118
187
  constructor(options = {}) {
119
188
  super();
120
189
  this.#writeClient = RedisStreamsPubSub.#createWriteClient(options);
@@ -123,7 +192,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
123
192
  this.#keyPrefix = options.keyPrefix ?? "mastra:topic";
124
193
  this.#blockMs = options.blockMs ?? 1e3;
125
194
  this.#maxStreamLength = options.maxStreamLength ?? 1e4;
126
- const ttl = options.streamIdleTtlMs ?? 0;
195
+ const ttl = options.streamIdleTtlMs ?? DEFAULT_STREAM_IDLE_TTL_MS;
127
196
  if (!Number.isInteger(ttl) || ttl < 0) throw new Error(`redis-streams: streamIdleTtlMs must be a non-negative integer (milliseconds), got ${ttl}`);
128
197
  this.#streamIdleTtlMs = ttl;
129
198
  this.#reclaimIntervalMs = options.reclaimIntervalMs ?? 3e4;
@@ -206,11 +275,33 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
206
275
  /** Lazily connect the shared writer client. Idempotent and safe under concurrent callers. */
207
276
  async #ensureWriterConnected() {
208
277
  if (!this.#writerConnecting && this.#writeClient.isOpen) return;
209
- if (!this.#writerConnecting) this.#writerConnecting = this.#writeClient.connect().then(() => void 0).finally(() => {
278
+ if (!this.#writerConnecting) this.#writerConnecting = connectWithin(this.#writeClient, "write", () => this.#writerReleased).then(() => {
279
+ if (!this.#evictionPolicyChecked) {
280
+ this.#evictionPolicyChecked = true;
281
+ this.#warnOnEvictingPolicy();
282
+ }
283
+ }).finally(() => {
210
284
  this.#writerConnecting = void 0;
211
285
  });
212
286
  return this.#writerConnecting;
213
287
  }
288
+ /**
289
+ * Consumer-group state, lease keys and TTL'd streams are all live data. A
290
+ * server with `maxmemory` set and any policy other than `noeviction` deletes
291
+ * them under memory pressure (silently losing events and leases), so warn
292
+ * once. INFO rather than CONFIG GET because managed Redis often blocks CONFIG.
293
+ */
294
+ async #warnOnEvictingPolicy() {
295
+ try {
296
+ const client = this.#writeClient;
297
+ const reply = Array.isArray(client.masters) ? await this.#writeClient.sendCommand(void 0, true, ["INFO", "memory"]) : await this.#writeClient.sendCommand(["INFO", "memory"]);
298
+ const policy = evictingMaxmemoryPolicy(String(reply));
299
+ if (!policy) return;
300
+ (this.#logger?.warn ? this.#logger.warn.bind(this.#logger) : console.warn)(`redis-streams: Redis maxmemory-policy is "${policy}". Under memory pressure Redis will evict pubsub keys (TTL'd streams, consumer groups, lease keys), losing events and leases. Set maxmemory-policy to "noeviction".`);
301
+ } catch (err) {
302
+ this.#logger?.debug?.("redis-streams: maxmemory-policy check failed", { err: err instanceof Error ? err.message : String(err) });
303
+ }
304
+ }
214
305
  #streamKey(topic) {
215
306
  return `${this.#keyPrefix}:${topic}`;
216
307
  }
@@ -387,7 +478,18 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
387
478
  }
388
479
  const readClient = this.#createReadClient();
389
480
  this.#attachErrorLogger(readClient, "read", { topic });
390
- await readClient.connect();
481
+ try {
482
+ await connectWithin(readClient, "read");
483
+ } catch (err) {
484
+ if (!isGrouped) this.#writeClient.xGroupDestroy(streamKey, group).catch((destroyErr) => {
485
+ this.#logger?.debug?.("redis-streams: xGroupDestroy failed", {
486
+ topic,
487
+ group,
488
+ error: errorText(destroyErr)
489
+ });
490
+ });
491
+ throw err;
492
+ }
391
493
  return {
392
494
  members: /* @__PURE__ */ new Map([[cb, {}]]),
393
495
  settling: /* @__PURE__ */ new Map(),
@@ -749,6 +851,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
749
851
  const subs = new Set(this.#subscriptions.values());
750
852
  await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
751
853
  await this.flush();
854
+ this.#writerReleased = true;
752
855
  if (this.#writeClient.isOpen) try {
753
856
  await disconnect(this.#writeClient);
754
857
  } catch (err) {