@mastra/redis-streams 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.cjs +106 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +106 -3
- package/dist/index.js.map +1 -1
- package/package.json +10 -9
package/README.md
CHANGED
|
@@ -24,7 +24,7 @@ export const mastra = new Mastra({
|
|
|
24
24
|
});
|
|
25
25
|
```
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
Every stream gets a sliding idle TTL, controlled by `streamIdleTtlMs` (default: 7 days). Every write refreshes it, so actively written topics never expire, while abandoned ones (a crashed run, or open-ended topics that never reach a `clearTopic` call) are reclaimed by Redis once they go idle. Reads do not refresh the TTL, so keep it above the longest expected gap between writes on a live topic, including the time a grouped consumer may need to replay a backlog. Setting `streamIdleTtlMs: 0` disables expiry; those streams then stay in Redis forever and its memory grows without bound.
|
|
28
28
|
|
|
29
29
|
### Redis Cluster
|
|
30
30
|
|
package/dist/index.cjs
CHANGED
|
@@ -4,6 +4,7 @@ let redis = require("redis");
|
|
|
4
4
|
// src/index.ts
|
|
5
5
|
/** Page size for the reclaim loop's XPENDING scan and the max entries claimed per tick. */
|
|
6
6
|
const RECLAIM_PAGE_SIZE = 100;
|
|
7
|
+
const DEFAULT_STREAM_IDLE_TTL_MS = 6048e5;
|
|
7
8
|
const TRIM_PAGE_SIZE = 500;
|
|
8
9
|
/**
|
|
9
10
|
* Heartbeat: reset a pending entry's idle time only if this consumer still
|
|
@@ -68,6 +69,71 @@ function compareStreamIds(a, b) {
|
|
|
68
69
|
return seq === 0n ? 0 : seq < 0n ? -1 : 1;
|
|
69
70
|
}
|
|
70
71
|
/**
|
|
72
|
+
* Connect a client, standalone or cluster, within its `socket.connectTimeout`
|
|
73
|
+
* (node-redis default: 5000 ms; 0 disables the bound).
|
|
74
|
+
*
|
|
75
|
+
* node-redis applies `connectTimeout` only to opening the TCP socket. The
|
|
76
|
+
* handshake that follows (CLIENT SETINFO, HELLO/AUTH, SELECT) has no deadline,
|
|
77
|
+
* and the default reconnectStrategy retries a failed attempt forever. So a
|
|
78
|
+
* server that accepts the socket but never replies (a SIGSTOPped process, or a
|
|
79
|
+
* proxy that stopped forwarding) makes `connect()` hang forever. The same
|
|
80
|
+
* budget therefore bounds the whole connect here. On expiry the client is
|
|
81
|
+
* destroyed, which stops its retry loop and lets a later call connect again.
|
|
82
|
+
*
|
|
83
|
+
* destroy() does not stop a cluster's slot discovery: once the nodes answer it
|
|
84
|
+
* still connects node clients that nothing would close. So when the abandoned
|
|
85
|
+
* connect settles, the client is destroyed again unless it is open. Until then
|
|
86
|
+
* no new connect starts on that client: a second discovery would share the
|
|
87
|
+
* first one's node entries, settle before they connect, and its destroy()
|
|
88
|
+
* would miss them. A call made meanwhile waits for the abandoned connect
|
|
89
|
+
* within its own deadline, then connects.
|
|
90
|
+
*/
|
|
91
|
+
const abandonedConnects = /* @__PURE__ */ new WeakMap();
|
|
92
|
+
async function connectWithin(client, role, isReleased = () => false) {
|
|
93
|
+
const cluster = client;
|
|
94
|
+
const timeoutMs = (cluster._options ? cluster._options.defaults?.socket : client.options?.socket)?.connectTimeout ?? 5e3;
|
|
95
|
+
let timer;
|
|
96
|
+
let connecting;
|
|
97
|
+
const timedOut = new Promise((_, reject) => {
|
|
98
|
+
if (!timeoutMs) return;
|
|
99
|
+
timer = setTimeout(() => {
|
|
100
|
+
if (connecting) {
|
|
101
|
+
const settled = connecting.catch(() => {}).then(() => {
|
|
102
|
+
if (abandonedConnects.get(client) === settled) abandonedConnects.delete(client);
|
|
103
|
+
if (client.isOpen) return;
|
|
104
|
+
try {
|
|
105
|
+
client.destroy();
|
|
106
|
+
} catch {}
|
|
107
|
+
});
|
|
108
|
+
abandonedConnects.set(client, settled);
|
|
109
|
+
if (client.isOpen) client.destroy();
|
|
110
|
+
}
|
|
111
|
+
reject(/* @__PURE__ */ new Error(`redis-streams: ${role} client did not connect to Redis within ${timeoutMs}ms (socket.connectTimeout); the server is unreachable or not responding`));
|
|
112
|
+
}, timeoutMs);
|
|
113
|
+
});
|
|
114
|
+
try {
|
|
115
|
+
const abandoned = abandonedConnects.get(client);
|
|
116
|
+
if (abandoned) {
|
|
117
|
+
await Promise.race([abandoned, timedOut]);
|
|
118
|
+
if (isReleased()) throw new Error(`redis-streams: ${role} client closed while connecting`);
|
|
119
|
+
}
|
|
120
|
+
connecting = client.connect();
|
|
121
|
+
await Promise.race([connecting, timedOut]);
|
|
122
|
+
} finally {
|
|
123
|
+
clearTimeout(timer);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Parse `maxmemory` / `maxmemory_policy` out of an `INFO memory` reply and
|
|
128
|
+
* return the policy when the server can evict keys under memory pressure.
|
|
129
|
+
*/
|
|
130
|
+
function evictingMaxmemoryPolicy(info) {
|
|
131
|
+
const field = (name) => info.match(new RegExp(`^${name}:(.*)$`, "m"))?.[1]?.trim();
|
|
132
|
+
const policy = field("maxmemory_policy");
|
|
133
|
+
if (!policy || policy === "noeviction" || Number(field("maxmemory") ?? 0) <= 0) return void 0;
|
|
134
|
+
return policy;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
71
137
|
* Disconnect a client, standalone or cluster.
|
|
72
138
|
*
|
|
73
139
|
* node-redis 5 `quit()` never settles on a client that is open but waiting to
|
|
@@ -116,6 +182,9 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
116
182
|
#pendingPublishes = /* @__PURE__ */ new Set();
|
|
117
183
|
#localCallbacks = /* @__PURE__ */ new Map();
|
|
118
184
|
#closed = false;
|
|
185
|
+
/** Set once close() has drained publishes and is about to release the writer. */
|
|
186
|
+
#writerReleased = false;
|
|
187
|
+
#evictionPolicyChecked = false;
|
|
119
188
|
constructor(options = {}) {
|
|
120
189
|
super();
|
|
121
190
|
this.#writeClient = RedisStreamsPubSub.#createWriteClient(options);
|
|
@@ -124,7 +193,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
124
193
|
this.#keyPrefix = options.keyPrefix ?? "mastra:topic";
|
|
125
194
|
this.#blockMs = options.blockMs ?? 1e3;
|
|
126
195
|
this.#maxStreamLength = options.maxStreamLength ?? 1e4;
|
|
127
|
-
const ttl = options.streamIdleTtlMs ??
|
|
196
|
+
const ttl = options.streamIdleTtlMs ?? DEFAULT_STREAM_IDLE_TTL_MS;
|
|
128
197
|
if (!Number.isInteger(ttl) || ttl < 0) throw new Error(`redis-streams: streamIdleTtlMs must be a non-negative integer (milliseconds), got ${ttl}`);
|
|
129
198
|
this.#streamIdleTtlMs = ttl;
|
|
130
199
|
this.#reclaimIntervalMs = options.reclaimIntervalMs ?? 3e4;
|
|
@@ -207,11 +276,33 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
207
276
|
/** Lazily connect the shared writer client. Idempotent and safe under concurrent callers. */
|
|
208
277
|
async #ensureWriterConnected() {
|
|
209
278
|
if (!this.#writerConnecting && this.#writeClient.isOpen) return;
|
|
210
|
-
if (!this.#writerConnecting) this.#writerConnecting = this.#writeClient
|
|
279
|
+
if (!this.#writerConnecting) this.#writerConnecting = connectWithin(this.#writeClient, "write", () => this.#writerReleased).then(() => {
|
|
280
|
+
if (!this.#evictionPolicyChecked) {
|
|
281
|
+
this.#evictionPolicyChecked = true;
|
|
282
|
+
this.#warnOnEvictingPolicy();
|
|
283
|
+
}
|
|
284
|
+
}).finally(() => {
|
|
211
285
|
this.#writerConnecting = void 0;
|
|
212
286
|
});
|
|
213
287
|
return this.#writerConnecting;
|
|
214
288
|
}
|
|
289
|
+
/**
|
|
290
|
+
* Consumer-group state, lease keys and TTL'd streams are all live data. A
|
|
291
|
+
* server with `maxmemory` set and any policy other than `noeviction` deletes
|
|
292
|
+
* them under memory pressure (silently losing events and leases), so warn
|
|
293
|
+
* once. INFO rather than CONFIG GET because managed Redis often blocks CONFIG.
|
|
294
|
+
*/
|
|
295
|
+
async #warnOnEvictingPolicy() {
|
|
296
|
+
try {
|
|
297
|
+
const client = this.#writeClient;
|
|
298
|
+
const reply = Array.isArray(client.masters) ? await this.#writeClient.sendCommand(void 0, true, ["INFO", "memory"]) : await this.#writeClient.sendCommand(["INFO", "memory"]);
|
|
299
|
+
const policy = evictingMaxmemoryPolicy(String(reply));
|
|
300
|
+
if (!policy) return;
|
|
301
|
+
(this.#logger?.warn ? this.#logger.warn.bind(this.#logger) : console.warn)(`redis-streams: Redis maxmemory-policy is "${policy}". Under memory pressure Redis will evict pubsub keys (TTL'd streams, consumer groups, lease keys), losing events and leases. Set maxmemory-policy to "noeviction".`);
|
|
302
|
+
} catch (err) {
|
|
303
|
+
this.#logger?.debug?.("redis-streams: maxmemory-policy check failed", { err: err instanceof Error ? err.message : String(err) });
|
|
304
|
+
}
|
|
305
|
+
}
|
|
215
306
|
#streamKey(topic) {
|
|
216
307
|
return `${this.#keyPrefix}:${topic}`;
|
|
217
308
|
}
|
|
@@ -388,7 +479,18 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
388
479
|
}
|
|
389
480
|
const readClient = this.#createReadClient();
|
|
390
481
|
this.#attachErrorLogger(readClient, "read", { topic });
|
|
391
|
-
|
|
482
|
+
try {
|
|
483
|
+
await connectWithin(readClient, "read");
|
|
484
|
+
} catch (err) {
|
|
485
|
+
if (!isGrouped) this.#writeClient.xGroupDestroy(streamKey, group).catch((destroyErr) => {
|
|
486
|
+
this.#logger?.debug?.("redis-streams: xGroupDestroy failed", {
|
|
487
|
+
topic,
|
|
488
|
+
group,
|
|
489
|
+
error: errorText(destroyErr)
|
|
490
|
+
});
|
|
491
|
+
});
|
|
492
|
+
throw err;
|
|
493
|
+
}
|
|
392
494
|
return {
|
|
393
495
|
members: /* @__PURE__ */ new Map([[cb, {}]]),
|
|
394
496
|
settling: /* @__PURE__ */ new Map(),
|
|
@@ -750,6 +852,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends _mastra_core_events.Pu
|
|
|
750
852
|
const subs = new Set(this.#subscriptions.values());
|
|
751
853
|
await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
|
|
752
854
|
await this.flush();
|
|
855
|
+
this.#writerReleased = true;
|
|
753
856
|
if (this.#writeClient.isOpen) try {
|
|
754
857
|
await disconnect(this.#writeClient);
|
|
755
858
|
} catch (err) {
|