@mastra/redis-streams 0.6.1 → 0.6.2-alpha.0

package/dist/index.js

@@ -52,6 +52,15 @@ function errorText(err) {
 	if (err instanceof Error && "replies" in err && Array.isArray(err.replies)) parts.push(...err.replies.map((r) => String(r)));
 	return parts.join("; ");
 }
+/** Order two Redis stream ids (`<ms>-<seq>`). */
+function compareStreamIds(a, b) {
+	const [aMs = "0", aSeq = "0"] = a.split("-");
+	const [bMs = "0", bSeq = "0"] = b.split("-");
+	const ms = BigInt(aMs) - BigInt(bMs);
+	if (ms !== 0n) return ms < 0n ? -1 : 1;
+	const seq = BigInt(aSeq) - BigInt(bSeq);
+	return seq === 0n ? 0 : seq < 0n ? -1 : 1;
+}
 var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 	get supportedModes() {
 		return ["pull"];
@@ -74,6 +83,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 	#inFlightTimeoutMs;
 	#logger;
 	#subscriptions = /* @__PURE__ */ new Map();
+	#fanoutReaders = /* @__PURE__ */ new Map();
+	#pendingFanoutReaders = /* @__PURE__ */ new Map();
 	#pendingSubscribes = /* @__PURE__ */ new Map();
 	#cbIds = /* @__PURE__ */ new WeakMap();
 	#pendingPublishes = /* @__PURE__ */ new Set();
@@ -126,8 +137,9 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 		});
 	}
 	/**
-	* A fresh, unconnected client with the writer's configuration. Each
-	* subscription needs its own because XREADGROUP BLOCK holds the connection.
+	* A fresh, unconnected client with the writer's configuration. Each reader
+	* (one per fan-out topic, one per grouped subscription) needs its own
+	* because XREADGROUP BLOCK holds the connection.
 	*/
 	#createReadClient() {
 		return this.#writeClient.duplicate();
@@ -241,11 +253,89 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 		}
 		localBucket.add(cb);
 		await this.#ensureWriterConnected();
-		const isGrouped = !!options?.group;
-		const group = options?.group ?? `__fanout-${globalThis.crypto.randomUUID()}`;
+		const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
+		if (options?.group) {
+			const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
+			this.#subscriptions.set(key, sub);
+			sub.loop = this.#runReadLoop(sub);
+			this.#startReclaimLoop(sub);
+			return;
+		}
+		for (;;) {
+			const reader = this.#fanoutReaders.get(topic);
+			if (reader) return this.#joinFanoutReader(reader, cb, key, groupAnchor);
+			const opening = this.#pendingFanoutReaders.get(topic);
+			if (!opening) break;
+			await opening.catch(() => {});
+		}
+		const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
+		this.#pendingFanoutReaders.set(topic, opening);
+		try {
+			const sub = await opening;
+			this.#fanoutReaders.set(topic, sub);
+			this.#subscriptions.set(key, sub);
+			sub.loop = this.#runReadLoop(sub);
+			this.#startReclaimLoop(sub);
+		} finally {
+			this.#pendingFanoutReaders.delete(topic);
+		}
+	}
+	/**
+	* Add a callback to a topic's open fan-out reader. The reader may already
+	* have delivered entries this subscriber should see (`startFrom: 'earliest'`)
+	* or still be catching up on entries it should not (`startFrom: 'latest'`),
+	* so the new member buffers live entries while its starting point is
+	* resolved, replays what it missed, then switches to live delivery.
+	*/
+	async #joinFanoutReader(sub, cb, key, anchor) {
+		const member = { joining: [] };
+		sub.members.set(cb, member);
+		this.#subscriptions.set(key, sub);
+		const active = () => sub.members.get(cb) === member && !sub.stopped;
+		try {
+			if (anchor === "$") {
+				const [tail] = await this.#writeClient.xRevRange(sub.streamKey, "+", "-", { COUNT: 1 });
+				member.skipThrough = tail?.id;
+			} else {
+				let boundary;
+				try {
+					const info = (await this.#writeClient.xInfoGroups(sub.streamKey)).find((g) => String(g.name) === sub.group);
+					boundary = info ? String(info["last-delivered-id"]) : void 0;
+				} catch (err) {
+					if (!errorText(err).match(/no such key|NOGROUP/i)) throw err;
+				}
+				if (boundary && boundary !== "0-0") {
+					member.skipThrough = boundary;
+					let start = "-";
+					while (active()) {
+						const page = await this.#writeClient.xRange(sub.streamKey, start, boundary, { COUNT: TRIM_PAGE_SIZE });
+						for (const entry of page) {
+							if (!active()) break;
+							this.#deliverDetached(sub, cb, entry.message);
+						}
+						if (page.length < TRIM_PAGE_SIZE) break;
+						start = `(${page[page.length - 1].id}`;
+					}
+				}
+			}
+		} catch (err) {
+			sub.members.delete(cb);
+			this.#subscriptions.delete(key);
+			throw err;
+		}
+		if (!active()) return;
+		const buffered = member.joining ?? [];
+		for (const entry of buffered) {
+			if (member.skipThrough && compareStreamIds(entry.id, member.skipThrough) <= 0) continue;
+			this.#deliverDetached(sub, cb, entry.fields);
+		}
+		member.joining = void 0;
+	}
+	async #openSubscription(topic, cb, groupName, groupAnchor) {
+		const isGrouped = groupName !== void 0;
+		const group = groupName ?? `__fanout-${globalThis.crypto.randomUUID()}`;
 		const consumer = `${group}-${globalThis.crypto.randomUUID()}`;
 		const streamKey = this.#streamKey(topic);
-		const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
 		try {
 			if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true }).pExpire(streamKey, this.#streamIdleTtlMs).exec();
 			else await this.#writeClient.xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true });
@@ -259,8 +349,9 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 		const readClient = this.#createReadClient();
 		this.#attachErrorLogger(readClient, "read", { topic });
 		await readClient.connect();
-		const sub = {
-			cb,
+		return {
+			members: /* @__PURE__ */ new Map([[cb, {}]]),
+			settling: /* @__PURE__ */ new Map(),
 			topic,
 			streamKey,
 			group,
@@ -274,9 +365,6 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 			reclaimTimer: void 0,
 			inFlight: /* @__PURE__ */ new Map()
 		};
-		this.#subscriptions.set(key, sub);
-		sub.loop = this.#runReadLoop(sub);
-		this.#startReclaimLoop(sub);
 	}
 	/**
 	* Periodically reclaim idle pending entries in this subscription's group so that
@@ -365,6 +453,12 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 		const sub = this.#subscriptions.get(key);
 		if (!sub) return;
 		if (sub.teardown) return sub.teardown;
+		if (sub.members.size > 1 && sub.members.delete(cb)) {
+			this.#subscriptions.delete(key);
+			for (const settle of [...sub.settling.values()]) settle(cb);
+			return;
+		}
+		if (this.#fanoutReaders.get(topic) === sub) this.#fanoutReaders.delete(topic);
 		sub.stopped = true;
 		if (sub.reclaimTimer) {
 			clearTimeout(sub.reclaimTimer);
@@ -431,6 +525,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 			for (const sub of this.#subscriptions.values()) if (sub.topic === topic) {
 				sub.lastId = void 0;
 				sub.groupAnchor = "0";
+				for (const member of sub.members.values()) member.skipThrough = void 0;
 			}
 		} catch (err) {
 			this.#logger?.warn?.("redis-streams: clearTopic failed", {
@@ -596,8 +691,8 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 		if (this.#closed) return;
 		this.#closed = true;
 		if (this.#pendingSubscribes.size > 0) await Promise.allSettled([...this.#pendingSubscribes.values()]);
-		const subs = [...this.#subscriptions.values()];
-		await Promise.all(subs.map((sub) => this.unsubscribe(sub.topic, sub.cb)));
+		const subs = new Set(this.#subscriptions.values());
+		await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
 		this.#localCallbacks.clear();
 		await this.flush();
 		if (this.#writeClient.isOpen) try {
@@ -652,15 +747,11 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 		}
 	}
 	async #deliverMessage(sub, streamId, fields) {
-		let event;
-		try {
-			event = JSON.parse(fields.event ?? "{}");
-			if (typeof event.createdAt === "string") event.createdAt = new Date(event.createdAt);
-		} catch (err) {
+		const event = this.#parseEvent(fields);
+		if (!event) {
 			this.#logger?.debug?.("redis-streams: malformed payload, dropping", {
 				topic: sub.topic,
-				streamId,
-				err: err instanceof Error ? err.message : err
+				streamId
 			});
 			try {
 				await this.#writeClient.xAck(sub.streamKey, sub.group, streamId);
@@ -742,6 +833,7 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 		const expire = async () => {
 			if (settled) return;
 			settled = true;
+			sub.settling.delete(streamId);
 			const attempt = event.deliveryAttempt ?? 1;
 			const drop = attempt >= this.#maxDeliveryAttempts;
 			const payload = drop ? "" : JSON.stringify({
@@ -767,7 +859,12 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 					err: err instanceof Error ? err.message : err
 				});
 				settled = false;
-				sub.inFlight.delete(streamId);
+				sub.settling.set(streamId, (cb) => settleMember(cb, false));
+				if (sub.isGrouped) sub.inFlight.delete(streamId);
+				else sub.inFlight.set(streamId, {
+					since: Date.now(),
+					expire
+				});
 				return;
 			}
 			sub.inFlight.delete(streamId);
@@ -800,19 +897,115 @@ var RedisStreamsPubSub = class RedisStreamsPubSub extends PubSub {
 				]
 			});
 		};
+		const waiting = /* @__PURE__ */ new Set();
+		let anyNack = false;
+		const settleMember = async (cb, nacked) => {
+			if (settled || !waiting.delete(cb)) return;
+			if (nacked) anyNack = true;
+			if (waiting.size > 0) return;
+			sub.settling.delete(streamId);
+			await (anyNack ? nack() : ack());
+		};
+		const targets = [];
+		for (const [cb, member] of sub.members) {
+			if (member.joining) {
+				member.joining.push({
+					id: streamId,
+					fields
+				});
+				continue;
+			}
+			if (member.skipThrough) {
+				if (compareStreamIds(streamId, member.skipThrough) <= 0) continue;
+				member.skipThrough = void 0;
+			}
+			waiting.add(cb);
+			targets.push(cb);
+		}
+		if (targets.length === 0) {
+			await ack();
+			return;
+		}
 		sub.inFlight.set(streamId, {
 			since: Date.now(),
 			expire
 		});
+		sub.settling.set(streamId, (cb) => settleMember(cb, false));
+		for (const cb of targets) {
+			const memberEvent = targets.length === 1 ? event : this.#parseEvent(fields);
+			this.#invokeCallback(cb, memberEvent, () => settleMember(cb, false), () => settleMember(cb, true), extend);
+		}
+	}
+	/**
+	* Invoke a handler without awaiting it. EventCallback is typed `=> void`
+	* but handlers commonly return a promise (TS allows Promise<void> to satisfy
+	* void); a rejection or synchronous throw routes to nack instead of silently
+	* dropping the message. We do NOT await — serializing messages on a
+	* subscription would deadlock orchestration callbacks that await their own
+	* future events.
+	*/
+	#invokeCallback(cb, event, ack, nack, extend) {
 		try {
-			const result = sub.cb(event, ack, nack, extend);
+			const result = cb(event, ack, nack, extend);
 			if (result && typeof result.catch === "function") result.catch(async () => {
 				await nack();
 			});
 		} catch {
-			await nack();
+			nack();
 		}
 	}
+	#parseEvent(fields) {
+		try {
+			const event = JSON.parse(fields.event ?? "{}");
+			if (typeof event.createdAt === "string") event.createdAt = new Date(event.createdAt);
+			return event;
+		} catch {
+			return;
+		}
+	}
+	/**
+	* Deliver an entry to one fan-out member outside the shared group's PEL:
+	* entries the topic's reader handed out before the member joined. Ack is a
+	* no-op (the group settles the entry on its own); nack republishes, which
+	* redelivers to every member just as a nack on the shared entry would.
+	*/
+	#deliverDetached(sub, cb, fields) {
+		const event = this.#parseEvent(fields);
+		if (!event) return;
+		let settled = false;
+		const nack = async () => {
+			if (settled) return;
+			settled = true;
+			const attempt = event.deliveryAttempt ?? 1;
+			if (attempt >= this.#maxDeliveryAttempts) {
+				this.#logger?.warn?.("redis-streams: dropping event after max delivery attempts", {
+					topic: sub.topic,
+					eventType: event.type,
+					eventId: event.id,
+					attempt,
+					max: this.#maxDeliveryAttempts
+				});
+				return;
+			}
+			const payload = { event: JSON.stringify({
+				...event,
+				deliveryAttempt: attempt + 1
+			}) };
+			try {
+				if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
+				else await this.#writeClient.xAdd(sub.streamKey, "*", payload);
+			} catch (err) {
+				this.#logger?.warn?.("redis-streams: nack republish failed for replayed event", {
+					topic: sub.topic,
+					eventId: event.id,
+					err: err instanceof Error ? err.message : err
+				});
+			}
+		};
+		this.#invokeCallback(cb, event, async () => {
+			settled = true;
+		}, nack);
+	}
 	#deliverLocal(topic, event) {
 		const callbacks = this.#localCallbacks.get(topic);
 		if (!callbacks || callbacks.size === 0) return;