@mastra/livekit 0.3.0 → 0.3.1-alpha.1

package/dist/remote-D7n50m8S.js

@@ -0,0 +1,629 @@
+import { ReadableStream } from "stream/web";
+import { APIConnectionError, APIError, APIStatusError, APITimeoutError, llm, voice } from "@livekit/agents";
+import { RequestContext } from "@mastra/core/request-context";
+function textOfMessage(message) {
+	const parts = [];
+	for (const part of message.content) if (typeof part === "string") parts.push(part);
+	else if (part.type === "instructions") parts.push(part.value);
+	else if (part.type === "audio_content" && part.transcript) parts.push(part.transcript);
+	return parts.join("\n").trim();
+}
+function toVoiceTurnMessage(item) {
+	if (item.type !== "message") return void 0;
+	const content = textOfMessage(item);
+	if (!content) return void 0;
+	const id = item.id;
+	if (item.role === "user") return {
+		role: "user",
+		content,
+		id
+	};
+	if (item.role === "assistant") return {
+		role: "assistant",
+		content,
+		id
+	};
+	return {
+		role: "system",
+		content,
+		id
+	};
+}
+/**
+* Extracts only the messages added since the agent last spoke. Used when Mastra Memory is
+* the source of truth for conversation history: prior turns are already persisted in the
+* thread, so re-sending them would duplicate history.
+*
+* Two extensions over the naive "slice after the last assistant message":
+*
+* - **Interrupted-turn self-heal:** when the last assistant message was cut off by barge-in
+*   (`interrupted: true`), the server never persisted it — aborted runs skip persistence — so
+*   its heard-only text is missing from the thread. Re-send that fragment (ordered first) this
+*   turn to backfill it. It stops being "the last assistant message" once a full reply lands,
+*   so each interrupted fragment is sent exactly once, on the following turn.
+* - **Instructions filter:** LiveKit injects the customer Agent's `instructions` as a
+*   leading `system` message ({@link LIVEKIT_INSTRUCTIONS_MESSAGE_ID}); the server-side Mastra
+*   agent owns its own system prompt, so drop it (it would otherwise ship on the first turn,
+*   before any assistant message).
+*/
+function extractNewTurnMessages(chatCtx) {
+	const items = chatCtx.items;
+	let lastAssistantIdx = -1;
+	for (let i = items.length - 1; i >= 0; i--) {
+		const item = items[i];
+		if (item?.type === "message" && item.role === "assistant") {
+			lastAssistantIdx = i;
+			break;
+		}
+	}
+	const lastAssistant = lastAssistantIdx >= 0 ? items[lastAssistantIdx] : void 0;
+	const startIdx = lastAssistant?.type === "message" && lastAssistant.role === "assistant" && lastAssistant.interrupted ? lastAssistantIdx : lastAssistantIdx + 1;
+	const messages = [];
+	for (const item of items.slice(startIdx)) {
+		if (item.type === "message" && item.id === "lk.agent_task.instructions") continue;
+		const message = toVoiceTurnMessage(item);
+		if (message) messages.push(message);
+	}
+	return messages;
+}
+/**
+* Converts the full LiveKit chat context to Mastra messages. Used when the bridge runs
+* without Mastra Memory and LiveKit's in-session context is the only history. The agent's
+* LiveKit-level instructions are excluded — the Mastra agent applies its own instructions.
+*/
+function chatContextToMessages(chatCtx) {
+	const withoutInstructions = chatCtx.copy({
+		excludeInstructions: true,
+		excludeFunctionCall: true
+	});
+	const messages = [];
+	for (const item of withoutInstructions.items) {
+		const message = toVoiceTurnMessage(item);
+		if (message) messages.push(message);
+	}
+	return messages;
+}
+//#endregion
+//#region src/bridge.ts
+const DEFAULT_INSTRUCTIONS = "You are a helpful voice assistant powered by a Mastra agent.";
+/**
+* Tracks periodic AI re-disclosure for a single call. `due()` returns the reminder text once
+* `everyMs` has elapsed since the last disclosure (resetting the clock), otherwise `undefined`.
+* Time is injectable so the interval logic is deterministically testable.
+*/
+var DisclosureReminder = class {
+	everyMs;
+	text;
+	lastAt;
+	constructor(everyMs, text, now = Date.now()) {
+		this.everyMs = everyMs;
+		this.text = text;
+		this.lastAt = now;
+	}
+	/** Call once per turn: the reminder text if it's due, else `undefined`. Does not reset the clock —
+	* call {@link DisclosureReminder.markDelivered} once the reminder is actually threaded into the
+	* outgoing reply, so a reminder that never makes it out isn't silently skipped for a full interval. */
+	due(now = Date.now()) {
+		if (now - this.lastAt < this.everyMs) return void 0;
+		return this.text;
+	}
+	/** Resets the clock. Call only once the reminder text from {@link due} was actually emitted. */
+	markDelivered(now = Date.now()) {
+		this.lastAt = now;
+	}
+};
+/**
+* Wraps `source` in a stream that emits `text` as a single leading chunk (with a trailing space, so
+* TTS pauses before the reply) before piping the rest of `source` through unchanged. Cancelling the
+* wrapper cancels `source` — so barge-in still aborts the underlying generation.
+*/
+function prependText(source, text) {
+	const prefix = text.endsWith(" ") ? text : `${text} `;
+	const reader = source.getReader();
+	return new ReadableStream({
+		start(controller) {
+			controller.enqueue(prefix);
+		},
+		async pull(controller) {
+			try {
+				const { done, value } = await reader.read();
+				if (done) controller.close();
+				else controller.enqueue(value);
+			} catch (error) {
+				controller.error(error);
+			}
+		},
+		cancel(reason) {
+			return reader.cancel(reason);
+		}
+	});
+}
+/**
+* Maps a Mastra `finish` chunk's usage (`payload.output.usage`, AI-SDK `LanguageModelUsage`) to the
+* LiveKit-shaped {@link VoiceTurnUsage}, or `undefined` when the chunk carries no token counts.
+* Handles both the flat V2 usage shape (`inputTokens`/`outputTokens`) and the nested V3 shape
+* (`inputTokens.total`/`outputTokens.total`).
+*/
+function mapTurnUsage(usage) {
+	if (!usage || typeof usage !== "object") return void 0;
+	const u = usage;
+	const totalOf = (v) => {
+		if (typeof v === "number") return v;
+		if (v && typeof v === "object" && typeof v.total === "number") return v.total;
+	};
+	const cacheReadOf = (v) => v && typeof v === "object" && typeof v.cacheRead === "number" ? v.cacheRead : void 0;
+	const promptTokens = totalOf(u.inputTokens) ?? 0;
+	const completionTokens = totalOf(u.outputTokens) ?? 0;
+	const promptCachedTokens = (typeof u.cachedInputTokens === "number" ? u.cachedInputTokens : cacheReadOf(u.inputTokens)) ?? 0;
+	const totalTokens = typeof u.totalTokens === "number" ? u.totalTokens : promptTokens + completionTokens;
+	if (promptTokens === 0 && completionTokens === 0 && totalTokens === 0 && promptCachedTokens === 0) return;
+	return {
+		promptTokens,
+		completionTokens,
+		promptCachedTokens,
+		totalTokens
+	};
+}
+/**
+* A {@link VoiceReplyGenerator} backed by a Mastra agent: runs the agent's full loop (model,
+* tools, memory) and streams its text deltas. On barge-in the returned stream is cancelled,
+* which aborts the in-flight `agent.stream()`.
+*/
+function createAgentReplyGenerator(options) {
+	const { agent, streamOptions, toolFeedback, onToolCall, onTurnComplete } = options;
+	return (ctx) => {
+		if (ctx.messages.length === 0) return null;
+		const abortController = new AbortController();
+		const mergedOptions = {
+			...streamOptions,
+			abortSignal: abortController.signal
+		};
+		if (ctx.memory) mergedOptions.memory = ctx.memory;
+		if (ctx.requestContext) mergedOptions.requestContext = ctx.requestContext;
+		let cancelled = false;
+		let replyText = "";
+		const toolCalls = [];
+		let usage;
+		const emitTurnComplete = (interrupted) => {
+			if (!onTurnComplete) return;
+			const completeCtx = {
+				...ctx,
+				result: {
+					text: replyText,
+					toolCalls,
+					interrupted,
+					usage
+				}
+			};
+			Promise.resolve().then(() => onTurnComplete(completeCtx)).catch((error) => {
+				console.warn("@mastra/livekit: onTurnComplete hook threw", error);
+			});
+		};
+		return new ReadableStream({
+			start: async (controller) => {
+				try {
+					const result = await agent.stream(ctx.messages, mergedOptions);
+					for await (const chunk of result.fullStream) {
+						if (cancelled) break;
+						if (chunk.type === "text-delta") {
+							if (chunk.payload.text) {
+								replyText += chunk.payload.text;
+								controller.enqueue(chunk.payload.text);
+							}
+						} else if (chunk.type === "tool-call") {
+							const toolCall = {
+								toolCallId: chunk.payload.toolCallId,
+								toolName: chunk.payload.toolName,
+								args: chunk.payload.args
+							};
+							toolCalls.push(toolCall);
+							try {
+								onToolCall?.(toolCall);
+							} catch (error) {
+								console.warn("@mastra/livekit: onToolCall hook threw", error);
+							}
+							if (toolFeedback) {
+								let filler;
+								try {
+									filler = toolFeedback(toolCall);
+								} catch (error) {
+									console.warn("@mastra/livekit: toolFeedback hook threw", error);
+								}
+								if (filler) controller.enqueue(filler.endsWith(" ") ? filler : `${filler} `);
+							}
+						} else if (chunk.type === "finish") {
+							const output = chunk.payload.output;
+							const turnUsage = mapTurnUsage(output?.usage);
+							if (turnUsage) {
+								usage = turnUsage;
+								try {
+									ctx.onUsage?.(turnUsage);
+								} catch (error) {
+									console.warn("@mastra/livekit: onUsage hook threw", error);
+								}
+							}
+						} else if (chunk.type === "error") {
+							const error = chunk.payload.error;
+							throw error instanceof Error ? error : new Error(String(error));
+						}
+					}
+					if (!cancelled) controller.close();
+					emitTurnComplete(cancelled);
+				} catch (error) {
+					if (cancelled || abortController.signal.aborted) {
+						emitTurnComplete(true);
+						return;
+					}
+					controller.error(error);
+				}
+			},
+			cancel: () => {
+				cancelled = true;
+				abortController.abort();
+			}
+		});
+	};
+}
+function toRequestContext(value) {
+	if (!value) return void 0;
+	if (value instanceof RequestContext) return value;
+	return new RequestContext(Object.entries(value));
+}
+/**
+* The session only runs its cascaded reply pipeline when an `llm` instance is present —
+* `llmNode` replaces the inference step, but the gate checks `llm instanceof LLM`. This
+* placeholder satisfies the gate; the Mastra agent/workflow does the actual generation.
+*/
+var MastraPlaceholderLLM = class extends llm.LLM {
+	label() {
+		return "mastra.MastraVoiceAgent";
+	}
+	get model() {
+		return "mastra-agent";
+	}
+	get provider() {
+		return "mastra";
+	}
+	chat() {
+		throw new Error("@mastra/livekit: reply generation runs through the Mastra agent via llmNode; the placeholder LLM cannot be used for inference.");
+	}
+};
+/**
+* A LiveKit `voice.Agent` whose replies come from a Mastra agent or workflow.
+*
+* LiveKit keeps ownership of the audio loop (VAD, STT, turn detection, TTS, barge-in) and calls
+* `llmNode` once per detected user turn; the node delegates to a {@link VoiceReplyGenerator}
+* which streams text deltas back. On barge-in LiveKit cancels the returned stream, which aborts
+* the in-flight generation.
+*/
+var MastraVoiceAgent = class extends voice.Agent {
+	mastraAgent;
+	memory;
+	requestContext;
+	streamOptions;
+	replyGenerator;
+	reminder;
+	constructor(options) {
+		if (options.agent && options.generate) throw new Error("@mastra/livekit: MastraVoiceAgent requires `agent` or `generate`, not both — they are mutually exclusive reply sources.");
+		super({
+			id: options.id,
+			instructions: options.instructions ?? DEFAULT_INSTRUCTIONS,
+			stt: options.stt,
+			vad: options.vad,
+			llm: new MastraPlaceholderLLM(),
+			tts: options.tts,
+			turnHandling: options.turnHandling
+		});
+		this.memory = options.memory ?? false;
+		this.requestContext = toRequestContext(options.requestContext);
+		this.streamOptions = options.streamOptions;
+		if (options.greetingReminder) this.reminder = new DisclosureReminder(options.greetingReminder.everyMs, options.greetingReminder.text?.trim() || "Just a reminder, you're speaking with an AI assistant.");
+		if (options.generate) this.replyGenerator = options.generate;
+		else if (options.agent) {
+			this.mastraAgent = options.agent;
+			this.replyGenerator = createAgentReplyGenerator({
+				agent: options.agent,
+				streamOptions: options.streamOptions,
+				toolFeedback: options.toolFeedback,
+				onToolCall: options.onToolCall,
+				onTurnComplete: options.onTurnComplete
+			});
+		} else throw new Error("@mastra/livekit: MastraVoiceAgent requires `agent` or `generate`.");
+	}
+	async llmNode(chatCtx, _toolCtx, _modelSettings) {
+		const messages = this.memory === false ? chatContextToMessages(chatCtx) : extractNewTurnMessages(chatCtx);
+		if (messages.length === 0) return null;
+		const reply = await this.replyGenerator({
+			messages,
+			chatCtx,
+			memory: this.memory,
+			requestContext: this.requestContext,
+			tracingContext: this.streamOptions?.tracingContext
+		});
+		if (!reply) return null;
+		const reminder = this.reminder?.due();
+		if (!reminder) return reply;
+		this.reminder?.markDelivered();
+		return prependText(reply, reminder);
+	}
+};
+function createMastraVoiceAgent(options) {
+	return new MastraVoiceAgent(options);
+}
+//#endregion
+//#region src/remote.ts
+const DEFAULT_API_PREFIX = "/api";
+/** Connect + first-token budget when not overridden. Plugin mode passes `connOptions.timeoutMs`. */
+const DEFAULT_REMOTE_TIMEOUT_MS = 1e4;
+/** Thrown (as a plain, non-retryable error) when the server emits a chunk that needs client action. */
+const HITL_UNSUPPORTED_MESSAGE = "@mastra/livekit: the agent requested tool approval or suspended a tool call; human-in-the-loop flows (approve-tool-call / resume-stream) are not supported on the voice path. Remove requireApproval or suspend from the tools this agent uses on voice calls.";
+function trimTrailingSlash(url) {
+	return url.endsWith("/") ? url.slice(0, -1) : url;
+}
+function toMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+async function resolveHeaders(headers) {
+	if (!headers) return {};
+	if (typeof headers === "function") return await headers() ?? {};
+	return headers;
+}
+function serializeRequestContext(requestContext) {
+	if (!requestContext) return void 0;
+	if (requestContext instanceof RequestContext) return Object.fromEntries(requestContext.entries());
+	return requestContext;
+}
+async function safeReadBody(response) {
+	try {
+		const text = await response.text();
+		if (!text) return null;
+		try {
+			const parsed = JSON.parse(text);
+			return parsed && typeof parsed === "object" ? parsed : { message: String(parsed) };
+		} catch {
+			return { message: text };
+		}
+	} catch {
+		return null;
+	}
+}
+/**
+* Reads a Mastra SSE stream and yields each event's parsed JSON. Framing matches the server's
+* `processMastraStream` (buffer, split on `\n\n`, strip `data: `, stop on `[DONE]`); undecodable
+* `data:` lines are skipped. Aborting `signal` cancels the underlying reader.
+*/
+async function* readMastraSSE(body, signal) {
+	const reader = body.getReader();
+	const decoder = new TextDecoder();
+	let buffer = "";
+	const onAbort = () => void reader.cancel().catch(() => {});
+	if (signal.aborted) {
+		reader.cancel().catch(() => {});
+		return;
+	}
+	signal.addEventListener("abort", onAbort, { once: true });
+	try {
+		for (;;) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			buffer += decoder.decode(value, { stream: true });
+			const events = buffer.split("\n\n");
+			buffer = events.pop() ?? "";
+			for (const event of events) {
+				if (!event.startsWith("data:")) continue;
+				const data = event.slice(event.startsWith("data: ") ? 6 : 5).trim();
+				if (data === "[DONE]") return;
+				if (!data) continue;
+				let json;
+				try {
+					json = JSON.parse(data);
+				} catch {
+					continue;
+				}
+				if (json && typeof json === "object") yield json;
+			}
+		}
+	} finally {
+		signal.removeEventListener("abort", onAbort);
+		try {
+			reader.releaseLock();
+		} catch {}
+	}
+}
+/**
+* A {@link VoiceReplyGenerator} that runs the Mastra agent loop on a **remote** Mastra server over
+* HTTP/SSE. Shaped exactly like the in-process `createAgentReplyGenerator`: it consumes the same
+* chunk vocabulary, drives the same `toolFeedback` / `onToolCall` / `onTurnComplete` seams, and
+* cancelling the returned stream (LiveKit does this on barge-in) tears down the HTTP request so the
+* server aborts generation.
+*
+* Usable standalone via the worker's `generate:` hatch (a minimum-viable remote worker mode), and as
+* the transport `MastraLLM` wraps. Errors are thrown as LiveKit `APIError` subclasses so the plugin's
+* base-class retry loop and `FallbackAdapter` behave; a connect + first-token watchdog prevents
+* indefinite dead air.
+*/
+function createRemoteAgentReplyGenerator(options) {
+	const { baseUrl, agentId, apiPrefix = DEFAULT_API_PREFIX, headers, fetch: fetchImpl = globalThis.fetch, timeoutMs = DEFAULT_REMOTE_TIMEOUT_MS, retries = 2, body: extraBody, toolFeedback, onToolCall, onTurnComplete } = options;
+	if (!fetchImpl) throw new Error("@mastra/livekit: no fetch implementation available; pass `fetch` or run on Node ≥ 22.");
+	const url = `${trimTrailingSlash(baseUrl)}${apiPrefix}/agents/${agentId}/stream`;
+	return (ctx) => {
+		if (ctx.messages.length === 0) return null;
+		let currentAbortController;
+		let cancelled = false;
+		let replyText = "";
+		const toolCalls = [];
+		let usage;
+		const emitTurnComplete = (interrupted) => {
+			if (!onTurnComplete) return;
+			const completeCtx = {
+				...ctx,
+				result: {
+					text: replyText,
+					toolCalls,
+					interrupted,
+					usage
+				}
+			};
+			Promise.resolve().then(() => onTurnComplete(completeCtx)).catch((error) => {
+				console.warn("@mastra/livekit: onTurnComplete hook threw", error);
+			});
+		};
+		const requestBody = {
+			messages: ctx.messages,
+			memory: ctx.memory ? {
+				thread: ctx.memory.thread,
+				resource: ctx.memory.resource ?? ctx.memory.thread
+			} : void 0,
+			requestContext: serializeRequestContext(ctx.requestContext),
+			...extraBody
+		};
+		return new ReadableStream({
+			start: async (controller) => {
+				let retryable = true;
+				try {
+					for (let attempt = 0;; attempt++) {
+						const abortController = new AbortController();
+						currentAbortController = abortController;
+						let timedOut = false;
+						let watchdog;
+						const clearWatchdog = () => {
+							if (watchdog) {
+								clearTimeout(watchdog);
+								watchdog = void 0;
+							}
+						};
+						try {
+							watchdog = setTimeout(() => {
+								timedOut = true;
+								abortController.abort();
+							}, timeoutMs);
+							watchdog.unref?.();
+							const resolvedHeaders = await resolveHeaders(headers);
+							if (cancelled) {
+								clearWatchdog();
+								break;
+							}
+							const response = await fetchImpl(url, {
+								method: "POST",
+								headers: {
+									"content-type": "application/json",
+									accept: "text/event-stream",
+									...resolvedHeaders
+								},
+								body: JSON.stringify(requestBody),
+								signal: abortController.signal
+							});
+							if (!response.ok) {
+								const errorBody = await safeReadBody(response);
+								throw new APIStatusError({
+									message: `@mastra/livekit: Mastra agent stream request failed with status ${response.status}`,
+									options: {
+										statusCode: response.status,
+										body: errorBody,
+										retryable
+									}
+								});
+							}
+							if (!response.body) throw new APIConnectionError({
+								message: "@mastra/livekit: Mastra agent stream returned an empty response body",
+								options: { retryable }
+							});
+							for await (const chunk of readMastraSSE(response.body, abortController.signal)) {
+								if (cancelled) break;
+								retryable = false;
+								const payload = chunk.payload ?? {};
+								switch (chunk.type) {
+									case "text-delta": {
+										const text = payload.text;
+										if (typeof text === "string" && text) {
+											clearWatchdog();
+											replyText += text;
+											controller.enqueue(text);
+										}
+										break;
+									}
+									case "tool-call": {
+										clearWatchdog();
+										const toolCall = {
+											toolCallId: String(payload.toolCallId ?? ""),
+											toolName: String(payload.toolName ?? ""),
+											args: payload.args
+										};
+										toolCalls.push(toolCall);
+										try {
+											onToolCall?.(toolCall);
+										} catch (error) {
+											console.warn("@mastra/livekit: onToolCall hook threw", error);
+										}
+										if (toolFeedback) {
+											let filler;
+											try {
+												filler = toolFeedback(toolCall);
+											} catch (error) {
+												console.warn("@mastra/livekit: toolFeedback hook threw", error);
+											}
+											if (filler) controller.enqueue(filler.endsWith(" ") ? filler : `${filler} `);
+										}
+										break;
+									}
+									case "finish": {
+										clearWatchdog();
+										const output = payload.output;
+										const turnUsage = mapTurnUsage(output?.usage);
+										if (turnUsage) {
+											usage = turnUsage;
+											try {
+												ctx.onUsage?.(turnUsage);
+											} catch (error) {
+												console.warn("@mastra/livekit: onUsage hook threw", error);
+											}
+										}
+										break;
+									}
+									case "tool-call-approval":
+									case "tool-call-suspended": throw new Error(HITL_UNSUPPORTED_MESSAGE);
+									case "error": {
+										const error = payload.error;
+										throw error instanceof Error ? error : new Error(String(error));
+									}
+									default: break;
+								}
+							}
+							clearWatchdog();
+							break;
+						} catch (error) {
+							clearWatchdog();
+							if (cancelled) break;
+							let typed;
+							if (timedOut) typed = new APITimeoutError({ options: { retryable } });
+							else if (error instanceof APIError) typed = error;
+							else if (retryable) typed = new APIConnectionError({
+								message: toMessage(error),
+								options: { retryable }
+							});
+							else throw error;
+							if (typed.retryable && attempt < retries) continue;
+							throw typed;
+						}
+					}
+					if (!cancelled) controller.close();
+					emitTurnComplete(cancelled);
+				} catch (error) {
+					if (cancelled) {
+						emitTurnComplete(true);
+						return;
+					}
+					controller.error(error);
+				}
+			},
+			cancel: () => {
+				cancelled = true;
+				currentAbortController?.abort();
+			}
+		});
+	};
+}
+//#endregion
+export { chatContextToMessages as a, createMastraVoiceAgent as i, MastraVoiceAgent as n, extractNewTurnMessages as o, createAgentReplyGenerator as r, createRemoteAgentReplyGenerator as t };
+
+//# sourceMappingURL=remote-D7n50m8S.js.map