@mastra/valkey-streams 0.5.3-alpha.0 → 0.5.4-alpha.0

package/dist/index.js

@@ -2,6 +2,12 @@ import { PubSub } from "@mastra/core/events";
 import { Batch, Decoder, GlideClient, InfBoundary } from "@valkey/valkey-glide";
 //#region src/client.ts
 const text = (value) => Buffer.isBuffer(value) ? value.toString() : String(value);
+const countArgs = (options) => options.COUNT === void 0 ? [] : ["COUNT", String(options.COUNT)];
+/** Decode an XRANGE/XREVRANGE reply (`[{ key: id, value: [[field, value], …] }]`). */
+const records = (value) => (value ?? []).map((entry) => ({
+	id: text(entry.key),
+	message: Object.fromEntries((entry.value ?? []).map(([field, fieldValue]) => [text(field), text(fieldValue)]))
+}));
 var ValkeyMulti = class {
 	getClient;
 	#batch = new Batch(true);
@@ -119,6 +125,32 @@ var ValkeyStreamsClient = class {
 			message: Object.fromEntries(fields.map(([field, value]) => [text(field), text(value)]))
 		}));
 	}
+	async xRange(key, start, end, options = {}) {
+		return records(await this.command([
+			"XRANGE",
+			key,
+			start,
+			end,
+			...countArgs(options)
+		]));
+	}
+	async xRevRange(key, end, start, options = {}) {
+		return records(await this.command([
+			"XREVRANGE",
+			key,
+			end,
+			start,
+			...countArgs(options)
+		]));
+	}
+	/** XINFO GROUPS <key>, one object per group keyed by field name. */
+	async xInfoGroups(key) {
+		return (await this.command([
+			"XINFO",
+			"GROUPS",
+			key
+		]) ?? []).map((group) => Object.fromEntries(group.map(({ key, value }) => [text(key), value])));
+	}
 	async xAck(key, group, id) {
 		return (await this.getClient()).xack(key, group, [id]);
 	}
@@ -174,6 +206,8 @@ const createClient = (options) => new ValkeyStreamsClient(options);
 //#region src/index.ts
 /** Page size for the reclaim loop's XPENDING scan and the max entries claimed per tick. */
 const RECLAIM_PAGE_SIZE = 100;
+/** Page size for replaying existing entries to a fan-out subscriber joining a shared reader. */
+const REPLAY_PAGE_SIZE = 500;
 /**
 * Heartbeat: reset a pending entry's idle time only if this consumer still
 * owns it, so a late heartbeat never takes an entry back from a sibling.
@@ -222,6 +256,15 @@ function errorText(err) {
 	if (err instanceof Error && "replies" in err && Array.isArray(err.replies)) parts.push(...err.replies.map((r) => String(r)));
 	return parts.join("; ");
 }
+/** Order two stream ids (`<ms>-<seq>`). */
+function compareStreamIds(a, b) {
+	const [aMs = "0", aSeq = "0"] = a.split("-");
+	const [bMs = "0", bSeq = "0"] = b.split("-");
+	const ms = BigInt(aMs) - BigInt(bMs);
+	if (ms !== 0n) return ms < 0n ? -1 : 1;
+	const seq = BigInt(aSeq) - BigInt(bSeq);
+	return seq === 0n ? 0 : seq < 0n ? -1 : 1;
+}
 var ValkeyStreamsPubSub = class extends PubSub {
 	get supportedModes() {
 		return ["pull"];
@@ -244,6 +287,8 @@ var ValkeyStreamsPubSub = class extends PubSub {
 	#inFlightTimeoutMs;
 	#logger;
 	#subscriptions = /* @__PURE__ */ new Map();
+	#fanoutReaders = /* @__PURE__ */ new Map();
+	#pendingFanoutReaders = /* @__PURE__ */ new Map();
 	#cbIds = /* @__PURE__ */ new WeakMap();
 	#pendingPublishes = /* @__PURE__ */ new Set();
 	#localCallbacks = /* @__PURE__ */ new Map();
@@ -373,11 +418,93 @@ var ValkeyStreamsPubSub = class extends PubSub {
 		}
 		localBucket.add(cb);
 		await this.#ensureWriterConnected();
-		const isGrouped = !!options?.group;
-		const group = options?.group ?? `__fanout-${globalThis.crypto.randomUUID()}`;
+		const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
+		if (options?.group) {
+			const sub = await this.#openSubscription(topic, cb, options.group, groupAnchor);
+			this.#subscriptions.set(key, sub);
+			sub.loop = this.#runReadLoop(sub);
+			this.#startReclaimLoop(sub);
+			return;
+		}
+		for (;;) {
+			const reader = this.#fanoutReaders.get(topic);
+			if (reader) return this.#joinFanoutReader(reader, cb, key, groupAnchor);
+			const opening = this.#pendingFanoutReaders.get(topic);
+			if (!opening) break;
+			await opening.catch(() => {});
+		}
+		const opening = this.#openSubscription(topic, cb, void 0, groupAnchor);
+		this.#pendingFanoutReaders.set(topic, opening);
+		try {
+			const sub = await opening;
+			this.#fanoutReaders.set(topic, sub);
+			this.#subscriptions.set(key, sub);
+			sub.loop = this.#runReadLoop(sub);
+			this.#startReclaimLoop(sub);
+		} finally {
+			this.#pendingFanoutReaders.delete(topic);
+		}
+	}
+	/**
+	* Add a callback to a topic's open fan-out reader. The reader may already
+	* have delivered entries this subscriber should see (`startFrom: 'earliest'`)
+	* or still be catching up on entries it should not (`startFrom: 'latest'`),
+	* so the new member buffers live entries while its starting point is
+	* resolved, replays what it missed, then switches to live delivery.
+	*/
+	async #joinFanoutReader(sub, cb, key, anchor) {
+		if (sub.members.has(cb)) {
+			this.#subscriptions.set(key, sub);
+			return;
+		}
+		const member = { joining: [] };
+		sub.members.set(cb, member);
+		this.#subscriptions.set(key, sub);
+		const active = () => sub.members.get(cb) === member && !sub.stopped;
+		try {
+			if (anchor === "$") {
+				const [tail] = await this.#writeClient.xRevRange(sub.streamKey, "+", "-", { COUNT: 1 });
+				member.skipThrough = tail?.id;
+			} else {
+				let boundary;
+				try {
+					const info = (await this.#writeClient.xInfoGroups(sub.streamKey)).find((g) => String(g.name) === sub.group);
+					boundary = info ? String(info["last-delivered-id"]) : void 0;
+				} catch (err) {
+					if (!errorText(err).match(/no such key|NOGROUP/i)) throw err;
+				}
+				if (boundary && boundary !== "0-0") {
+					member.skipThrough = boundary;
+					let start = "-";
+					while (active()) {
+						const page = await this.#writeClient.xRange(sub.streamKey, start, boundary, { COUNT: REPLAY_PAGE_SIZE });
+						for (const entry of page) {
+							if (!active()) break;
+							this.#deliverDetached(sub, cb, entry.message);
+						}
+						if (page.length < REPLAY_PAGE_SIZE) break;
+						start = `(${page[page.length - 1].id}`;
+					}
+				}
+			}
+		} catch (err) {
+			sub.members.delete(cb);
+			this.#subscriptions.delete(key);
+			throw err;
+		}
+		if (!active()) return;
+		const buffered = member.joining ?? [];
+		for (const entry of buffered) {
+			if (member.skipThrough && compareStreamIds(entry.id, member.skipThrough) <= 0) continue;
+			this.#deliverDetached(sub, cb, entry.fields);
+		}
+		member.joining = void 0;
+	}
+	async #openSubscription(topic, cb, groupName, groupAnchor) {
+		const isGrouped = groupName !== void 0;
+		const group = groupName ?? `__fanout-${globalThis.crypto.randomUUID()}`;
 		const consumer = `${group}-${globalThis.crypto.randomUUID()}`;
 		const streamKey = this.#streamKey(topic);
-		const groupAnchor = options?.startFrom === "latest" ? "$" : "0";
 		try {
 			if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true }).pExpire(streamKey, this.#streamIdleTtlMs).exec();
 			else await this.#writeClient.xGroupCreate(streamKey, group, groupAnchor, { MKSTREAM: true });
@@ -391,8 +518,9 @@ var ValkeyStreamsPubSub = class extends PubSub {
 		const readClient = createClient(this.#connectOptions);
 		this.#attachErrorLogger(readClient, "read", { topic });
 		await readClient.connect();
-		const sub = {
-			cb,
+		return {
+			members: /* @__PURE__ */ new Map([[cb, {}]]),
+			settling: /* @__PURE__ */ new Map(),
 			topic,
 			streamKey,
 			group,
@@ -406,9 +534,6 @@ var ValkeyStreamsPubSub = class extends PubSub {
 			reclaimTimer: void 0,
 			inFlight: /* @__PURE__ */ new Map()
 		};
-		this.#subscriptions.set(key, sub);
-		sub.loop = this.#runReadLoop(sub);
-		this.#startReclaimLoop(sub);
 	}
 	/**
 	* Periodically reclaim idle pending entries in this subscription's group so
@@ -499,11 +624,21 @@ var ValkeyStreamsPubSub = class extends PubSub {
 		const sub = this.#subscriptions.get(key);
 		if (!sub) return;
 		this.#subscriptions.delete(key);
+		if (sub.teardown) return sub.teardown;
+		if (sub.members.size > 1 && sub.members.delete(cb)) {
+			for (const settle of [...sub.settling.values()]) settle(cb);
+			return;
+		}
+		if (this.#fanoutReaders.get(topic) === sub) this.#fanoutReaders.delete(topic);
 		sub.stopped = true;
 		if (sub.reclaimTimer) {
 			clearTimeout(sub.reclaimTimer);
 			sub.reclaimTimer = void 0;
 		}
+		sub.teardown = this.#stopSubscription(sub);
+		return sub.teardown;
+	}
+	async #stopSubscription(sub) {
 		try {
 			await sub.readClient.quit();
 		} catch (err) {
@@ -559,6 +694,7 @@ var ValkeyStreamsPubSub = class extends PubSub {
 			for (const sub of this.#subscriptions.values()) if (sub.topic === topic) {
 				sub.lastId = void 0;
 				sub.groupAnchor = "0";
+				for (const member of sub.members.values()) member.skipThrough = void 0;
 			}
 		} catch (err) {
 			this.#logger?.warn?.("valkey-streams: clearTopic failed", {
@@ -685,8 +821,8 @@ var ValkeyStreamsPubSub = class extends PubSub {
 	async close() {
 		if (this.#closed) return;
 		this.#closed = true;
-		const subs = [...this.#subscriptions.values()];
-		await Promise.all(subs.map((sub) => this.unsubscribe(sub.topic, sub.cb)));
+		const subs = new Set(this.#subscriptions.values());
+		await Promise.all([...subs].flatMap((sub) => [...sub.members.keys()].map((cb) => this.unsubscribe(sub.topic, cb))));
 		this.#localCallbacks.clear();
 		await this.flush();
 		if (this.#writeClient.isOpen) try {
@@ -742,15 +878,11 @@ var ValkeyStreamsPubSub = class extends PubSub {
 		}
 	}
 	async #deliverMessage(sub, streamId, fields) {
-		let event;
-		try {
-			event = JSON.parse(fields.event ?? "{}");
-			if (typeof event.createdAt === "string") event.createdAt = new Date(event.createdAt);
-		} catch (err) {
+		const event = this.#parseEvent(fields);
+		if (!event) {
 			this.#logger?.debug?.("valkey-streams: malformed payload, dropping", {
 				topic: sub.topic,
-				streamId,
-				err: err instanceof Error ? err.message : err
+				streamId
 			});
 			try {
 				await this.#writeClient.xAck(sub.streamKey, sub.group, streamId);
@@ -832,6 +964,7 @@ var ValkeyStreamsPubSub = class extends PubSub {
 		const expire = async () => {
 			if (settled) return;
 			settled = true;
+			sub.settling.delete(streamId);
 			const attempt = event.deliveryAttempt ?? 1;
 			const drop = attempt >= this.#maxDeliveryAttempts;
 			const payload = drop ? "" : JSON.stringify({
@@ -857,7 +990,12 @@ var ValkeyStreamsPubSub = class extends PubSub {
 					err: err instanceof Error ? err.message : err
 				});
 				settled = false;
-				sub.inFlight.delete(streamId);
+				sub.settling.set(streamId, (cb) => settleMember(cb, false));
+				if (sub.isGrouped) sub.inFlight.delete(streamId);
+				else sub.inFlight.set(streamId, {
+					since: Date.now(),
+					expire
+				});
 				return;
 			}
 			sub.inFlight.delete(streamId);
@@ -890,19 +1028,115 @@ var ValkeyStreamsPubSub = class extends PubSub {
 				]
 			});
 		};
+		const waiting = /* @__PURE__ */ new Set();
+		let anyNack = false;
+		const settleMember = async (cb, nacked) => {
+			if (settled || !waiting.delete(cb)) return;
+			if (nacked) anyNack = true;
+			if (waiting.size > 0) return;
+			sub.settling.delete(streamId);
+			await (anyNack ? nack() : ack());
+		};
+		const targets = [];
+		for (const [cb, member] of sub.members) {
+			if (member.joining) {
+				member.joining.push({
+					id: streamId,
+					fields
+				});
+				continue;
+			}
+			if (member.skipThrough) {
+				if (compareStreamIds(streamId, member.skipThrough) <= 0) continue;
+				member.skipThrough = void 0;
+			}
+			waiting.add(cb);
+			targets.push(cb);
+		}
+		if (targets.length === 0) {
+			await ack();
+			return;
+		}
 		sub.inFlight.set(streamId, {
 			since: Date.now(),
 			expire
 		});
+		sub.settling.set(streamId, (cb) => settleMember(cb, false));
+		for (const cb of targets) {
+			const memberEvent = targets.length === 1 ? event : this.#parseEvent(fields);
+			this.#invokeCallback(cb, memberEvent, () => settleMember(cb, false), () => settleMember(cb, true), extend);
+		}
+	}
+	/**
+	* Invoke a handler without awaiting it. EventCallback is typed `=> void`
+	* but handlers commonly return a promise (TS allows Promise<void> to satisfy
+	* void); a rejection or synchronous throw routes to nack instead of silently
+	* dropping the message. We do NOT await — serializing messages on a
+	* subscription would deadlock orchestration callbacks that await their own
+	* future events.
+	*/
+	#invokeCallback(cb, event, ack, nack, extend) {
 		try {
-			const result = sub.cb(event, ack, nack, extend);
+			const result = cb(event, ack, nack, extend);
 			if (result && typeof result.catch === "function") result.catch(async () => {
 				await nack();
 			});
 		} catch {
-			await nack();
+			nack();
 		}
 	}
+	#parseEvent(fields) {
+		try {
+			const event = JSON.parse(fields.event ?? "{}");
+			if (typeof event.createdAt === "string") event.createdAt = new Date(event.createdAt);
+			return event;
+		} catch {
+			return;
+		}
+	}
+	/**
+	* Deliver an entry to one fan-out member outside the shared group's PEL:
+	* entries the topic's reader handed out before the member joined. Ack is a
+	* no-op (the group settles the entry on its own); nack republishes, which
+	* redelivers to every member just as a nack on the shared entry would.
+	*/
+	#deliverDetached(sub, cb, fields) {
+		const event = this.#parseEvent(fields);
+		if (!event) return;
+		let settled = false;
+		const nack = async () => {
+			if (settled) return;
+			settled = true;
+			const attempt = event.deliveryAttempt ?? 1;
+			if (attempt >= this.#maxDeliveryAttempts) {
+				this.#logger?.warn?.("valkey-streams: dropping event after max delivery attempts", {
+					topic: sub.topic,
+					eventType: event.type,
+					eventId: event.id,
+					attempt,
+					max: this.#maxDeliveryAttempts
+				});
+				return;
+			}
+			const payload = { event: JSON.stringify({
+				...event,
+				deliveryAttempt: attempt + 1
+			}) };
+			try {
+				if (this.#streamIdleTtlMs > 0) await this.#writeClient.multi().xAdd(sub.streamKey, "*", payload).pExpire(sub.streamKey, this.#streamIdleTtlMs).exec();
+				else await this.#writeClient.xAdd(sub.streamKey, "*", payload);
+			} catch (err) {
+				this.#logger?.warn?.("valkey-streams: nack republish failed for replayed event", {
+					topic: sub.topic,
+					eventId: event.id,
+					err: err instanceof Error ? err.message : err
+				});
+			}
+		};
+		this.#invokeCallback(cb, event, async () => {
+			settled = true;
+		}, nack);
+	}
 	#deliverLocal(topic, event) {
 		const callbacks = this.#localCallbacks.get(topic);
 		if (!callbacks || callbacks.size === 0) return;