@juspay/neurolink 13.3.0 → 13.4.0

package/CHANGELOG.md

@@ -1,8 +1,8 @@
-## [13.3.0](https://github.com/juspay/neurolink/compare/v13.2.0...v13.3.0) (2026-10-18)
+## [13.4.0](https://github.com/juspay/neurolink/compare/v13.3.0...v13.4.0) (2026-10-18)
 
 ### Features
 
-- **(middleware):** add constructor and global middleware defaults ([a61c7bd](https://github.com/juspay/neurolink/commit/a61c7bd61ae25f6629dd0ec0c97f1f9f158fb044))
+- **(voice):** add an xAI realtime backend beside Gemini Live ([7ae5ca4](https://github.com/juspay/neurolink/commit/7ae5ca4c5b313e8ca50c5c20d8fc33d1fa529ac5))
 
 ## [11.2.3](https://github.com/juspay/neurolink/compare/v11.2.2...v11.2.3) (2026-08-19)
 

package/dist/types/livekit.d.ts

@@ -421,30 +421,52 @@ export type LiveKitServerCredentials = {
     apiKey: string;
     apiSecret: string;
 };
+/**
+ * The realtime speech-to-speech model a call runs on: Gemini Live on Vertex
+ * (the default) or xAI Grok Voice through LiveKit's xAI plugin.
+ */
+export type RealtimeVoiceBackend = "gemini" | "xai";
+/**
+ * A LiveKit realtime model as far as callers of the model factory can rely on.
+ * It is declared structurally so the published declarations name no optional
+ * LiveKit plugin; at runtime it is a `RealtimeModel` subclass of the backend's
+ * plugin.
+ */
+export type RealtimeLlm = {
+    readonly model: string;
+    label: () => string;
+    close: () => Promise<void>;
+};
 /**
  * Realtime voice configuration resolved from the environment.
  *
- * In speech-to-speech mode one realtime model (Gemini Live on Vertex) does STT,
- * reasoning, TTS, and turn detection — so there is no separate STT/TTS/VAD/EOU
- * config. `resolveRealtimeVoiceConfig` fills every field from `process.env`
- * (with defaults); `RealtimeVoiceAgentConfig` lets a caller override any of them.
+ * In speech-to-speech mode one realtime model (Gemini Live on Vertex, or xAI
+ * Grok Voice) does STT, reasoning, TTS, and turn detection — so there is no
+ * separate STT/TTS/VAD/EOU config. `resolveRealtimeVoiceConfig` fills every
+ * field from `process.env` (with defaults); `RealtimeVoiceAgentConfig` lets a
+ * caller override any of them. The API key of the xai backend is deliberately
+ * not part of the config: the xAI plugin reads `XAI_API_KEY` itself.
  */
 export type RealtimeVoiceConfig = {
-    /** Vertex project id (from VERTEX_PROJECT / GOOGLE_AUTH_* / GOOGLE_CLOUD_PROJECT_ID). */
+    /** Which realtime model family handles the call (VOICE_REALTIME_BACKEND; default "gemini"). */
+    backend: RealtimeVoiceBackend;
+    /** Vertex project id (from VERTEX_PROJECT / GOOGLE_AUTH_* / GOOGLE_CLOUD_PROJECT_ID). Gemini only. */
     project: string | undefined;
-    /** Vertex location; native-audio Live is served on `global`, not regionally. */
+    /** Vertex location; native-audio Live is served on `global`, not regionally. Gemini only. */
     location: string;
-    /** Realtime model id (e.g. "gemini-live-2.5-flash"). */
+    /** Realtime model id (e.g. "gemini-live-2.5-flash" for gemini, "grok-voice-3.0" for xai). */
     model: string;
-    /** Optional Gemini voice name; omit for the plugin default. */
+    /** Optional voice name for either backend; omit for the plugin default. */
     voice: string | undefined;
-    /** Response modality: "AUDIO" (native S2S) or "TEXT" (half-cascade). */
+    /** xai only: WebSocket base URL of the realtime endpoint (XAI_REALTIME_BASE_URL); unset keeps the plugin's own endpoint. */
+    xaiBaseURL: string | undefined;
+    /** Response modality: "AUDIO" (native S2S) or "TEXT" (half-cascade). Gemini only. */
     responseModality: string;
     /** System prompt / instructions for the agent. */
     systemPrompt: string;
     /** Opening line the agent speaks on connect ("" disables). */
     greeting: string;
-    /** Whether to bridge Lighthouse MCP tools as Gemini function tools. */
+    /** Whether to bridge Lighthouse MCP tools as function tools of the realtime model (either backend). */
     toolsEnabled: boolean;
     /** Full URL of the MCP server the tools are bridged from. */
     mcpUrl: string;
@@ -474,10 +496,14 @@ export type RealtimeVoiceLogContext = {
  * everything via env.
  */
 export type RealtimeVoiceAgentConfig = {
+    /** Realtime backend; omitted means the environment's VOICE_REALTIME_BACKEND, and without it "gemini". */
+    backend?: RealtimeVoiceBackend;
     project?: string;
     location?: string;
     model?: string;
     voice?: string;
+    /** xai only: WebSocket base URL of the realtime endpoint. */
+    xaiBaseURL?: string;
     responseModality?: string;
     systemPrompt?: string;
     greeting?: string;
@@ -736,7 +762,7 @@ export type BuildRealtimeMcpToolsParams = {
      * Hard cap per MCP tool call, in milliseconds (default 30000).
      *
      * Without one, a stalled MCP server holds the realtime turn open forever:
-     * Gemini waits on the function result, so the user gets silence rather than
+     * The realtime model waits on the function result, so the user gets silence rather than
      * an error. Bounding the call turns that into a normal tool failure the
      * model can talk about.
      */

package/dist/voice/livekit/config.d.ts

@@ -7,7 +7,7 @@
  *
  * See docs/features/livekit-voice-agent.md.
  */
-import type { LiveKitServerConfig, LiveKitBrainDefaults, RealtimeVoiceConfig } from "../../types/index.js";
+import type { LiveKitServerConfig, LiveKitBrainDefaults, RealtimeVoiceAgentConfig, RealtimeVoiceConfig } from "../../types/index.js";
 /**
  * Resolve LiveKit server connection settings from the environment.
  *
@@ -24,16 +24,21 @@ export declare function resolveLiveKitServerConfig(): LiveKitServerConfig;
  */
 export declare function resolveBrainDefaults(): LiveKitBrainDefaults;
 /**
- * Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
- * the environment.
+ * Resolve the realtime (speech-to-speech) voice configuration from the
+ * environment.
  *
  * Every value has a safe default so a worker can run with no realtime-specific
- * env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
- * service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
- * `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
- * `${LIGHTHOUSE_URL}/ai/mcp/v2`.
+ * env at all. `VOICE_REALTIME_BACKEND` picks the model family (`gemini`, the
+ * default, or `xai`); `backend` here overrides it and then the environment's
+ * value is not read. A non-blank `xaiBaseURL` here likewise replaces
+ * `XAI_REALTIME_BASE_URL` after the same ws or wss check.
+ * `VOICE_LIVE_MODEL` overrides the backend's default model.
+ * Vertex `project` is resolved from `VERTEX_PROJECT`, then the service-account
+ * project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then `GOOGLE_CLOUD_PROJECT_ID`;
+ * project, location and modality only matter to the gemini backend. The MCP URL
+ * is `VOICE_MCP_URL` if set, otherwise `${LIGHTHOUSE_URL}/ai/mcp/v2`.
  */
-export declare function resolveRealtimeVoiceConfig(): RealtimeVoiceConfig;
+export declare function resolveRealtimeVoiceConfig(overrides?: Pick<RealtimeVoiceAgentConfig, "backend" | "xaiBaseURL">): RealtimeVoiceConfig;
 /**
  * Resolve the semantic end-of-utterance (EOU) turn-detection settings.
  *

package/dist/voice/livekit/config.js

@@ -68,6 +68,9 @@ export function resolveBrainDefaults() {
     };
 }
 const DEFAULT_REALTIME_MODEL = "gemini-live-2.5-flash";
+// xAI's documented flagship Grok Voice id. Never exercised against the real
+// service; the plugin's own default is `grok-voice-think-fast-1.0`.
+const DEFAULT_XAI_REALTIME_MODEL = "grok-voice-3.0";
 const DEFAULT_REALTIME_LOCATION = "global";
 const DEFAULT_RESPONSE_MODALITY = "AUDIO";
 const DEFAULT_SYSTEM_PROMPT = "You are a concise voice assistant for a merchant dashboard. Keep replies short and spoken.";
@@ -75,25 +78,86 @@ const DEFAULT_GREETING = "Briefly greet the merchant and ask how you can help wi
 const DEFAULT_LIGHTHOUSE_URL = "http://localhost:5173";
 const DEFAULT_MCP_PATH = "/ai/mcp/v2";
 /**
- * Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
- * the environment.
+ * The backend named by `VOICE_REALTIME_BACKEND`: `gemini` or `xai`, matched
+ * case-insensitively, with blank or absent meaning `gemini`. The error names the
+ * variable and the choices and repeats nothing from the environment.
+ */
+function readRealtimeBackend() {
+    const value = readEnv("VOICE_REALTIME_BACKEND", "gemini").toLowerCase();
+    if (value === "gemini" || value === "xai") {
+        return value;
+    }
+    throw new Error('VOICE_REALTIME_BACKEND must be "gemini" or "xai" (leave it unset for gemini)');
+}
+/**
+ * A backend passed in code. The type already limits it, but a plain-JavaScript
+ * caller is not held by the type, and a typo would otherwise take the xai
+ * branch and fail on a missing key. The error repeats nothing it was given.
+ */
+function requireKnownBackend(value) {
+    if (value === "gemini" || value === "xai") {
+        return value;
+    }
+    throw new Error('The realtime voice backend must be "gemini" or "xai"');
+}
+function isWebSocketUrl(value) {
+    try {
+        const { protocol } = new URL(value);
+        return protocol === "ws:" || protocol === "wss:";
+    }
+    catch {
+        return false;
+    }
+}
+/** A ws or wss URL. The error names the setting and does not repeat the value. */
+function requireWebSocketUrl(value, name) {
+    if (!isWebSocketUrl(value)) {
+        throw new Error(`${name} must be a ws:// or wss:// URL`);
+    }
+    return value;
+}
+/** `XAI_REALTIME_BASE_URL`, which must be a ws or wss URL when set. */
+function readXaiBaseUrl() {
+    const value = readOptionalEnv("XAI_REALTIME_BASE_URL");
+    return value === undefined
+        ? undefined
+        : requireWebSocketUrl(value, "XAI_REALTIME_BASE_URL");
+}
+/** A caller-supplied `xaiBaseURL` gets the same check as the variable; blank means unset, so the variable applies. */
+function resolveXaiBaseUrl(override) {
+    const value = override?.trim();
+    return value ? requireWebSocketUrl(value, "xaiBaseURL") : readXaiBaseUrl();
+}
+/**
+ * Resolve the realtime (speech-to-speech) voice configuration from the
+ * environment.
  *
  * Every value has a safe default so a worker can run with no realtime-specific
- * env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
- * service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
- * `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
- * `${LIGHTHOUSE_URL}/ai/mcp/v2`.
+ * env at all. `VOICE_REALTIME_BACKEND` picks the model family (`gemini`, the
+ * default, or `xai`); `backend` here overrides it and then the environment's
+ * value is not read. A non-blank `xaiBaseURL` here likewise replaces
+ * `XAI_REALTIME_BASE_URL` after the same ws or wss check.
+ * `VOICE_LIVE_MODEL` overrides the backend's default model.
+ * Vertex `project` is resolved from `VERTEX_PROJECT`, then the service-account
+ * project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then `GOOGLE_CLOUD_PROJECT_ID`;
+ * project, location and modality only matter to the gemini backend. The MCP URL
+ * is `VOICE_MCP_URL` if set, otherwise `${LIGHTHOUSE_URL}/ai/mcp/v2`.
  */
-export function resolveRealtimeVoiceConfig() {
+export function resolveRealtimeVoiceConfig(overrides = {}) {
+    const backend = overrides.backend === undefined
+        ? readRealtimeBackend()
+        : requireKnownBackend(overrides.backend);
     const lighthouseUrl = readEnv("LIGHTHOUSE_URL", DEFAULT_LIGHTHOUSE_URL);
     const mcpUrl = readEnv("VOICE_MCP_URL", `${lighthouseUrl}${DEFAULT_MCP_PATH}`);
     return {
+        backend,
         project: readOptionalEnv("VERTEX_PROJECT") ??
             readOptionalEnv("GOOGLE_AUTH_BREEZE_PROJECT_ID") ??
             readOptionalEnv("GOOGLE_CLOUD_PROJECT_ID"),
         location: readEnv("VERTEX_LOCATION", DEFAULT_REALTIME_LOCATION),
-        model: readEnv("VOICE_LIVE_MODEL", DEFAULT_REALTIME_MODEL),
+        model: readEnv("VOICE_LIVE_MODEL", backend === "xai" ? DEFAULT_XAI_REALTIME_MODEL : DEFAULT_REALTIME_MODEL),
         voice: readOptionalEnv("VOICE_S2S_VOICE"),
+        xaiBaseURL: resolveXaiBaseUrl(overrides.xaiBaseURL),
         responseModality: readEnv("VOICE_RESPONSE_MODALITY", DEFAULT_RESPONSE_MODALITY).toUpperCase(),
         systemPrompt: readEnv("VOICE_SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT),
         greeting: readEnv("VOICE_GREETING", DEFAULT_GREETING),

package/dist/voice/livekit/index.d.ts

@@ -15,6 +15,7 @@ export { mintJoinToken } from "./tokens.js";
 export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
 export { defineVoiceAgent } from "./voiceAgent.js";
 export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
+export { createRealtimeLlm } from "./realtimeModel.js";
 export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
 export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
 export { readCallContextFromRoom } from "./roomContext.js";

package/dist/voice/livekit/index.js

@@ -15,6 +15,7 @@ export { mintJoinToken } from "./tokens.js";
 export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
 export { defineVoiceAgent } from "./voiceAgent.js";
 export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
+export { createRealtimeLlm } from "./realtimeModel.js";
 export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
 export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
 export { readCallContextFromRoom } from "./roomContext.js";

package/dist/voice/livekit/realtimeMcpTools.d.ts

@@ -1,12 +1,13 @@
 /**
- * MCP tools → Gemini function tools bridge (realtime mode).
+ * MCP tools → realtime function tools bridge (realtime mode).
  *
- * In speech-to-speech mode Gemini owns the tool loop, so an external MCP
- * server's tools are registered as LiveKit/Gemini function tools rather than
- * NeuroLink tools. This connects to the MCP server as the calling user
- * (`x-auth-token` + base64 `x-context`), sanitizes each tool's schema into the
- * Gemini-safe subset, wires WRITE-labeled (destructive) tools through a HITL
- * confirmation, and forwards tool start/result events to the browser bridge.
+ * In speech-to-speech mode the realtime model (Gemini Live, or xAI Grok Voice)
+ * owns the tool loop, so an external MCP server's tools are registered as
+ * LiveKit function tools rather than NeuroLink tools. This connects to the MCP
+ * server as the calling user (`x-auth-token` + base64 `x-context`), sanitizes
+ * each tool's schema into the Gemini-safe subset (kept for both backends), wires
+ * WRITE-labeled (destructive) tools through a HITL confirmation, and forwards
+ * tool start/result events to the browser bridge.
  *
  * `@livekit/agents` and `@modelcontextprotocol/sdk` are imported dynamically so
  * the core package does not require them unless the realtime agent is used.

package/dist/voice/livekit/realtimeMcpTools.js

@@ -1,12 +1,13 @@
 /**
- * MCP tools → Gemini function tools bridge (realtime mode).
+ * MCP tools → realtime function tools bridge (realtime mode).
  *
- * In speech-to-speech mode Gemini owns the tool loop, so an external MCP
- * server's tools are registered as LiveKit/Gemini function tools rather than
- * NeuroLink tools. This connects to the MCP server as the calling user
- * (`x-auth-token` + base64 `x-context`), sanitizes each tool's schema into the
- * Gemini-safe subset, wires WRITE-labeled (destructive) tools through a HITL
- * confirmation, and forwards tool start/result events to the browser bridge.
+ * In speech-to-speech mode the realtime model (Gemini Live, or xAI Grok Voice)
+ * owns the tool loop, so an external MCP server's tools are registered as
+ * LiveKit function tools rather than NeuroLink tools. This connects to the MCP
+ * server as the calling user (`x-auth-token` + base64 `x-context`), sanitizes
+ * each tool's schema into the Gemini-safe subset (kept for both backends), wires
+ * WRITE-labeled (destructive) tools through a HITL confirmation, and forwards
+ * tool start/result events to the browser bridge.
  *
  * `@livekit/agents` and `@modelcontextprotocol/sdk` are imported dynamically so
  * the core package does not require them unless the realtime agent is used.
@@ -52,7 +53,7 @@ export function mcpResultToText(result) {
     return text || "Tool returned no content.";
 }
 /**
- * The function-result text returned to Gemini when the user rejects a HITL tool.
+ * The function-result text returned to the model when the user rejects a HITL tool.
  * Scope the rejection to THIS attempt — a "never retry" wording made the model
  * refuse the tool permanently even when the user later asked for it again.
  */
@@ -163,7 +164,7 @@ export async function buildRealtimeMcpTools(params) {
                     try {
                         // Third argument is RequestOptions; the SDK aborts the in-flight
                         // request when `timeout` elapses. Without it a stalled server holds
-                        // the turn open indefinitely — Gemini blocks on the function
+                        // the turn open indefinitely — the model waits on the function
                         // result, so the user hears nothing at all rather than an error.
                         const result = await client.callTool({
                             name: mcpTool.name,

package/dist/voice/livekit/realtimeMemory.d.ts

@@ -17,6 +17,14 @@
  * See docs/features/livekit-voice-agent.md.
  */
 import type { JobContext } from "@livekit/agents";
-import type { RealtimeCallMemory, RealtimeVoiceMemoryConfig } from "../../types/index.js";
-/** Verify the call's binding and prepare its recorder; `undefined` means "no memory for this call". */
-export declare function setUpRealtimeCallMemory(ctx: Pick<JobContext, "room" | "addShutdownCallback">, config: RealtimeVoiceMemoryConfig | undefined, model?: string): Promise<RealtimeCallMemory | undefined>;
+import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig } from "../../types/index.js";
+/**
+ * Verify the call's binding and prepare its recorder; `undefined` means "no
+ * memory for this call". `call` names the model and the realtime backend the
+ * call runs on, which the turn spans carry as `model` and `provider`; the
+ * backend defaults to `gemini`, the original and default one.
+ */
+export declare function setUpRealtimeCallMemory(ctx: Pick<JobContext, "room" | "addShutdownCallback">, config: RealtimeVoiceMemoryConfig | undefined, call?: {
+    model?: string;
+    backend?: RealtimeVoiceBackend;
+}): Promise<RealtimeCallMemory | undefined>;

package/dist/voice/livekit/realtimeMemory.js

@@ -17,6 +17,7 @@
  * See docs/features/livekit-voice-agent.md.
  */
 import { context, trace } from "@opentelemetry/api";
+import { AIProviderName } from "../../constants/enums.js";
 import { setLangfuseContext } from "../../services/server/ai/observability/instrumentation.js";
 import { observe, observeTrace } from "../../telemetry/observation.js";
 import { tracers } from "../../telemetry/tracers.js";
@@ -24,8 +25,13 @@ import { logger } from "../../utils/logger.js";
 import { VOICE_MEMORY_SECRET_ENV, verifyVoiceMemoryBinding, } from "./memoryBinding.js";
 import { createVoiceTurnRecorder } from "./turnRecorder.js";
 const DEFAULT_FLUSH_BUDGET_MS = 800;
+/** The provider a backend's turn spans are attributed to: Gemini Live runs on Vertex AI, Grok Voice on xAI. */
+const TRACE_PROVIDER = {
+    gemini: AIProviderName.VERTEX,
+    xai: AIProviderName.XAI,
+};
 function traceTurn(turn, call) {
-    const { sessionId, userId, roomName, model } = call;
+    const { sessionId, userId, roomName, model, provider } = call;
     try {
         setLangfuseContext({
             sessionId,
@@ -55,7 +61,7 @@ function traceTurn(turn, call) {
                 input: [{ role: "user", content: turn.userMessage }],
                 output: [{ role: "assistant", content: turn.aiResponse }],
                 level,
-                metadata: { provider: "vertex" },
+                metadata: { provider },
             });
             generation.end(endTime);
             const events = turn.events ?? [];
@@ -80,8 +86,14 @@ function traceTurn(turn, call) {
         });
     }
 }
-/** Verify the call's binding and prepare its recorder; `undefined` means "no memory for this call". */
-export async function setUpRealtimeCallMemory(ctx, config, model) {
+/**
+ * Verify the call's binding and prepare its recorder; `undefined` means "no
+ * memory for this call". `call` names the model and the realtime backend the
+ * call runs on, which the turn spans carry as `model` and `provider`; the
+ * backend defaults to `gemini`, the original and default one.
+ */
+export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
+    const { model, backend = "gemini" } = call;
     if (!config) {
         return undefined;
     }
@@ -116,7 +128,13 @@ export async function setUpRealtimeCallMemory(ctx, config, model) {
         write: async (turn) => {
             if (!tracedTurns.has(turn.turnId)) {
                 tracedTurns.add(turn.turnId);
-                traceTurn(turn, { sessionId, userId, roomName: ctx.room.name, model });
+                traceTurn(turn, {
+                    sessionId,
+                    userId,
+                    roomName: ctx.room.name,
+                    model,
+                    provider: TRACE_PROVIDER[backend],
+                });
             }
             const result = await neurolink.recordConversationTurn({
                 sessionId,

package/dist/voice/livekit/realtimeModel.d.ts

@@ -0,0 +1,33 @@
+/**
+ * The realtime model factory: builds the LiveKit speech-to-speech model for the
+ * configured backend.
+ *
+ * `gemini` is Gemini Live on Vertex through `@livekit/agents-plugin-google`;
+ * `xai` is Grok Voice through `@livekit/agents-plugin-xai`, which speaks the
+ * OpenAI realtime protocol to `wss://api.x.ai/v1`. Both plugins and
+ * `@google/genai` are imported dynamically so the core package does not require
+ * them unless the realtime agent is used.
+ *
+ * The xai backend has only been exercised against a stub WebSocket server. It
+ * has never talked to the real xAI service.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import type { RealtimeLlm, RealtimeVoiceConfig } from "../../types/index.js";
+/**
+ * Fail early, naming the variable, when `XAI_API_KEY` is blank or absent.
+ *
+ * The plugin reads the key itself and falls back to `OPENAI_API_KEY` when its
+ * own is missing, which would send an OpenAI key to the xAI host, so a missing
+ * key must stop here. The message never carries a value.
+ */
+export declare function requireXaiApiKey(): void;
+/**
+ * Build the realtime model for `cfg.backend`.
+ *
+ * The gemini branch builds exactly what the agent has always built: Vertex AI,
+ * the configured model, transcripts of both sides, and project, location,
+ * voice and modality when set. The xai branch needs `XAI_API_KEY` in the
+ * environment and throws before touching the plugin when it is missing.
+ */
+export declare function createRealtimeLlm(cfg: RealtimeVoiceConfig): Promise<RealtimeLlm>;

package/dist/voice/livekit/realtimeModel.js

@@ -0,0 +1,89 @@
+/**
+ * The realtime model factory: builds the LiveKit speech-to-speech model for the
+ * configured backend.
+ *
+ * `gemini` is Gemini Live on Vertex through `@livekit/agents-plugin-google`;
+ * `xai` is Grok Voice through `@livekit/agents-plugin-xai`, which speaks the
+ * OpenAI realtime protocol to `wss://api.x.ai/v1`. Both plugins and
+ * `@google/genai` are imported dynamically so the core package does not require
+ * them unless the realtime agent is used.
+ *
+ * The xai backend has only been exercised against a stub WebSocket server. It
+ * has never talked to the real xAI service.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+/**
+ * Fail early, naming the variable, when `XAI_API_KEY` is blank or absent.
+ *
+ * The plugin reads the key itself and falls back to `OPENAI_API_KEY` when its
+ * own is missing, which would send an OpenAI key to the xAI host, so a missing
+ * key must stop here. The message never carries a value.
+ */
+export function requireXaiApiKey() {
+    const key = process.env.XAI_API_KEY;
+    if (typeof key !== "string" || key.trim().length === 0) {
+        throw new Error("XAI_API_KEY is not set; the xai realtime backend reads its key from that environment variable");
+    }
+}
+async function createGeminiRealtimeLlm(cfg) {
+    const google = await import("@livekit/agents-plugin-google");
+    const { Modality } = await import("@google/genai");
+    const modality = Object.values(Modality).find((value) => value === cfg.responseModality);
+    const modelOptions = {
+        vertexai: true,
+        model: cfg.model,
+        // Emit text transcripts of BOTH sides; LiveKit forwards them to the room.
+        inputAudioTranscription: {},
+        outputAudioTranscription: {},
+    };
+    if (modality !== undefined) {
+        modelOptions.modalities = [modality];
+    }
+    if (cfg.project) {
+        modelOptions.project = cfg.project;
+    }
+    if (cfg.location) {
+        modelOptions.location = cfg.location;
+    }
+    if (cfg.voice) {
+        modelOptions.voice = cfg.voice;
+    }
+    return new google.realtime.RealtimeModel(modelOptions);
+}
+async function createXaiRealtimeLlm(cfg) {
+    requireXaiApiKey();
+    const xai = await import("@livekit/agents-plugin-xai");
+    // Keys are added only when set. The plugin spreads these options over its
+    // own defaults, so a key holding `undefined` would erase the default: a
+    // missing `baseURL` would send the xAI key to OPENAI_BASE_URL or
+    // api.openai.com, and no `apiKey` option is passed at all because the plugin
+    // reads XAI_API_KEY itself. The plugin forwards `baseURL` although its
+    // declared option type leaves it out.
+    const modelOptions = { model: cfg.model };
+    if (cfg.voice) {
+        modelOptions.voice = cfg.voice;
+    }
+    if (cfg.xaiBaseURL) {
+        modelOptions.baseURL = cfg.xaiBaseURL;
+    }
+    return new xai.realtime.RealtimeModel(modelOptions);
+}
+/**
+ * Build the realtime model for `cfg.backend`.
+ *
+ * The gemini branch builds exactly what the agent has always built: Vertex AI,
+ * the configured model, transcripts of both sides, and project, location,
+ * voice and modality when set. The xai branch needs `XAI_API_KEY` in the
+ * environment and throws before touching the plugin when it is missing.
+ */
+export async function createRealtimeLlm(cfg) {
+    switch (cfg.backend) {
+        case "gemini":
+            return createGeminiRealtimeLlm(cfg);
+        case "xai":
+            return createXaiRealtimeLlm(cfg);
+        default:
+            throw new Error('The realtime backend must be "gemini" or "xai"');
+    }
+}

package/dist/voice/livekit/realtimeVoiceAgent.d.ts

@@ -1,21 +1,25 @@
 /**
- * LiveKit Agents agent definition — realtime (Gemini Live speech-to-speech).
+ * LiveKit Agents agent definition — realtime (speech-to-speech).
  *
  * `defineRealtimeVoiceAgent` returns the agent object placed as the default
  * export of a worker entry file. Unlike the cascaded `defineVoiceAgent`
  * (Silero VAD → STT → NeuroLink → TTS), here a single realtime model (Gemini
- * Live on Vertex) does STT + reasoning + TTS + turn detection over one
+ * Live on Vertex, or xAI Grok Voice) does STT + reasoning + TTS + turn
+ * detection over one connection. The backend is chosen by `VOICE_REALTIME_BACKEND`
+ * or the `backend` option; Gemini Live is the default.
  *
- * `@livekit/agents`, `@livekit/agents-plugin-google`, `@livekit/rtc-node`, and
- * `@google/genai` are imported dynamically so the core package does not require
- * them unless the realtime agent is used. Type-only imports are erased at build.
+ * `@livekit/agents`, the backend's LiveKit plugin (see `realtimeModel.ts`),
+ * `@livekit/rtc-node`, and `@google/genai` are imported dynamically so the core
+ * package does not require them unless the realtime agent is used. Type-only
+ * imports are erased at build.
  *
  * See docs/features/livekit-voice-agent.md.
  */
 import type { JobContext } from "@livekit/agents";
 import type { RealtimeVoiceAgentConfig } from "../../types/index.js";
 /**
- * Define a realtime (Gemini Live S2S) LiveKit voice agent.
+ * Define a realtime (speech-to-speech) LiveKit voice agent: Gemini Live by
+ * default, xAI Grok Voice with `VOICE_REALTIME_BACKEND=xai` or `backend: "xai"`.
  *
  * Place the result as the default export of the worker entry file and launch it
  * with `startRealtimeVoiceAgentWorker`. With no `config` everything is resolved

package/dist/voice/livekit/realtimeVoiceAgent.js

@@ -1,14 +1,17 @@
 /**
- * LiveKit Agents agent definition — realtime (Gemini Live speech-to-speech).
+ * LiveKit Agents agent definition — realtime (speech-to-speech).
  *
  * `defineRealtimeVoiceAgent` returns the agent object placed as the default
  * export of a worker entry file. Unlike the cascaded `defineVoiceAgent`
  * (Silero VAD → STT → NeuroLink → TTS), here a single realtime model (Gemini
- * Live on Vertex) does STT + reasoning + TTS + turn detection over one
+ * Live on Vertex, or xAI Grok Voice) does STT + reasoning + TTS + turn
+ * detection over one connection. The backend is chosen by `VOICE_REALTIME_BACKEND`
+ * or the `backend` option; Gemini Live is the default.
  *
- * `@livekit/agents`, `@livekit/agents-plugin-google`, `@livekit/rtc-node`, and
- * `@google/genai` are imported dynamically so the core package does not require
- * them unless the realtime agent is used. Type-only imports are erased at build.
+ * `@livekit/agents`, the backend's LiveKit plugin (see `realtimeModel.ts`),
+ * `@livekit/rtc-node`, and `@google/genai` are imported dynamically so the core
+ * package does not require them unless the realtime agent is used. Type-only
+ * imports are erased at build.
  *
  * See docs/features/livekit-voice-agent.md.
  */
@@ -16,6 +19,7 @@ import { z } from "zod";
 import { logger } from "../../utils/logger.js";
 import { resolveRealtimeVoiceConfig } from "./config.js";
 import { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
+import { createRealtimeLlm, requireXaiApiKey } from "./realtimeModel.js";
 import { readCallContextFromRoom } from "./roomContext.js";
 import { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
 import { buildRealtimeMcpTools } from "./realtimeMcpTools.js";
@@ -119,7 +123,10 @@ async function installRealtimeJobLifecycle(ctx, cfg) {
 }
 /** Merge caller overrides over the env-resolved realtime config. */
 function resolveConfig(overrides) {
-    const base = resolveRealtimeVoiceConfig();
+    const base = resolveRealtimeVoiceConfig({
+        backend: overrides.backend,
+        xaiBaseURL: overrides.xaiBaseURL,
+    });
     return {
         ...base,
         project: overrides.project ?? base.project,
@@ -134,7 +141,8 @@ function resolveConfig(overrides) {
     };
 }
 /**
- * Define a realtime (Gemini Live S2S) LiveKit voice agent.
+ * Define a realtime (speech-to-speech) LiveKit voice agent: Gemini Live by
+ * default, xAI Grok Voice with `VOICE_REALTIME_BACKEND=xai` or `backend: "xai"`.
  *
  * Place the result as the default export of the worker entry file and launch it
  * with `startRealtimeVoiceAgentWorker`. With no `config` everything is resolved
@@ -147,16 +155,26 @@ export function defineRealtimeVoiceAgent(config = {}) {
     const onLog = config.onLog;
     const memoryConfig = config.memory;
     async function entry(ctx) {
-        // The Gemini Live WS authenticates to Vertex via ADC; materialise it and
-        // force ADC (not an API key) auth before any realtime connection.
-        ensureVertexAdc();
-        clearGeminiApiKeyEnv();
+        const isGemini = cfg.backend === "gemini";
+        if (isGemini) {
+            // The Gemini Live WS authenticates to Vertex via ADC; materialise it and
+            // force ADC (not an API key) auth before any realtime connection.
+            ensureVertexAdc();
+            clearGeminiApiKeyEnv();
+        }
+        else {
+            requireXaiApiKey();
+        }
         logger.info("realtime.config", {
+            backend: cfg.backend,
             model: cfg.model,
-            location: cfg.location,
-            project: cfg.project ?? null,
+            ...(isGemini
+                ? { location: cfg.location, project: cfg.project ?? null }
+                : {}),
             voice: cfg.voice ?? null,
-            modality: cfg.responseModality,
+            ...(isGemini
+                ? { modality: cfg.responseModality }
+                : { customBaseURL: cfg.xaiBaseURL !== undefined }),
             toolsEnabled: cfg.toolsEnabled,
             mcpUrl: cfg.mcpUrl,
         });
@@ -190,17 +208,20 @@ export function defineRealtimeVoiceAgent(config = {}) {
         }
         logger.info("realtime.room.joined", {
             room: ctx.room.name ?? "unknown",
+            backend: cfg.backend,
             model: cfg.model,
-            location: cfg.location,
+            ...(isGemini ? { location: cfg.location } : {}),
             ms: sinceConnect(),
         });
         // Decide whether this call may be remembered BEFORE the lifecycle installs
         // its process reaper, so the shutdown flush is registered first.
-        const memory = await setUpRealtimeCallMemory(ctx, memoryConfig, cfg.model);
+        const memory = await setUpRealtimeCallMemory(ctx, memoryConfig, {
+            model: cfg.model,
+            backend: cfg.backend,
+        });
         await installRealtimeJobLifecycle(ctx, cfg);
         const { voice } = await import("@livekit/agents");
-        const google = await import("@livekit/agents-plugin-google");
-        const { Modality } = await import("@google/genai");
+        const realtimeLlm = await createRealtimeLlm(cfg);
         const bridge = await attachRealtimeEventBridge({
             room: ctx.room,
             hitlTimeoutMs: cfg.hitlTimeoutMs,
@@ -254,28 +275,10 @@ export function defineRealtimeVoiceAgent(config = {}) {
                 }
             }
         }
-        const modality = Object.values(Modality).find((value) => value === cfg.responseModality);
-        const modelOptions = {
-            vertexai: true,
-            model: cfg.model,
-            // Emit text transcripts of BOTH sides; LiveKit forwards them to the room.
-            inputAudioTranscription: {},
-            outputAudioTranscription: {},
-        };
-        if (modality !== undefined) {
-            modelOptions.modalities = [modality];
-        }
-        if (cfg.project) {
-            modelOptions.project = cfg.project;
-        }
-        if (cfg.location) {
-            modelOptions.location = cfg.location;
-        }
-        if (cfg.voice) {
-            modelOptions.voice = cfg.voice;
-        }
         const session = new voice.AgentSession({
-            llm: new google.realtime.RealtimeModel(modelOptions),
+            // `RealtimeLlm` is declared structurally so the published declarations
+            // name no optional plugin; both backends return a `RealtimeModel`.
+            llm: realtimeLlm,
         });
         // The conversation so far, so a call that appends to an open conversation
         // starts knowing it. Empty (and omitted) when there is nothing to seed.

package/dist/voice/livekit/schemaSanitizer.d.ts

@@ -9,6 +9,8 @@
  *
  * Pure, dependency-free, and assertion-free — values arrive as `unknown` and are
  * narrowed with the `isRecord` guard. Safe to reuse for any Gemini tool path.
+ * The xai realtime backend uses it too: the output is plain JSON Schema, and
+ * keeping it avoids sending xAI `$ref` or union shapes it has not been tried with.
  */
 /**
  * Rebuild a JSON Schema node into the Gemini-safe subset. Returns a fresh object

package/dist/voice/livekit/schemaSanitizer.js

@@ -9,6 +9,8 @@
  *
  * Pure, dependency-free, and assertion-free — values arrive as `unknown` and are
  * narrowed with the `isRecord` guard. Safe to reuse for any Gemini tool path.
+ * The xai realtime backend uses it too: the output is plain JSON Schema, and
+ * keeping it avoids sending xAI `$ref` or union shapes it has not been tried with.
  */
 const GEMINI_TYPES = new Set([
     "string",

package/docs-site/static/search-index.json

@@ -3635,9 +3635,9 @@
 {"objectID":"ca876274dff5d751bc09c60e4de804f44917d9e0599697c2cd17362e3e6148bd","title":"Advanced Integrations","url":"/docs/features#advanced-integrations","content":"| Integration                                                    | Description                                                                                         |\n| -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- |\n| LiteLLM Integration           | Access 100+ models across a broad range of AI providers via LiteLLM routing with unified interface. |\n| SageMaker Integration       | Deploy and call custom endpoints directly from NeuroLink CLI/SDK with full control.                 |\n| Memory                                        | Per-user condensed memory with S3/Redis/SQLite storage and LLM-powered condensation.                |\n| Enterprise Proxy           | Configure outbound policies and compliance posture for corporate environments.                      |\n| Configuration Management | Manage environments, regions, and credentials safely across deployments.                            |","hierarchy":{"lvl0":"Features","lvl1":"Feature Guides","lvl2":"Advanced Integrations","lvl3":""}},
 {"objectID":"0d74f47963b8a59cf58901d344f8d1e987d3bb286cb012ed22278f415c051d3f","title":"Advanced Features","url":"/docs/features#advanced-features","content":"| Feature                                                                                   | Description                                                                        |\n| ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- |\n| 🏭 Factory Pattern Architecture | Unified provider interface with automatic fallbacks and type-safe implementations. |\n| 🗄️ Conversation Memory              | Deep dive into memory management and Redis integration.                            |\n| 🔧 Custom Middleware              | Build request/response hooks for logging, filtering, and custom processing.        |\n| ⚡ Performance Optimization     | Caching, connection pooling, and latency optimization strategies.                  |\n| 📊 Telemetry & Observability               | OpenTelemetry integration for distributed tracing and monitoring.                  |\n| 🧪 Testing Guide                                   | Provider-agnostic testing, mocking, and quality assurance strategies.              |\n| 📊 Analytics & Evaluation               | Usage tracking, cost monitoring, and quality scoring for AI responses.             |\n| ⚡ Streaming                                | Real-time token streaming with provider-specific optimizations.                    |\n|  Thinking Configuration               | Configure extended thinking levels for supported models (Anthropic, Gemini 2.5+).  |\n|  Structured Output                        | JSON schema-based structured output with provider-specific formatting.             |\n|  Text-to-Speech (TTS)                              | Basic TTS support via Google Cloud TTS (Neural2, Wavenet, Standard voices).        |","hierarchy":{"lvl0":"Features","lvl1":"Feature Guides","lvl2":"Advanced Features","lvl3":""}},
 {"objectID":"9c1dd453df8293552c84f52e9dd08ca4cd003afe799a2561dc4101aa9c2a304a","title":"See Also","url":"/docs/features#see-also","content":"Getting Started - Quick start and installation\nCLI Reference - Command-line interface documentation\nSDK Reference - TypeScript API documentation\nEnterprise Guides - Production deployment patterns\nTutorials - Step-by-step implementation guides\nExamples - Real-world code samples","hierarchy":{"lvl0":"Features","lvl1":"Feature Guides","lvl2":"See Also","lvl3":""}},
-{"objectID":"83f5b0de036a8a17898acf6e57b876b24b8eab4db20af614b766816e4da47913","title":"LiveKit Voice Agent — Real-Time Voice over WebRTC","url":"/docs/features/livekit-voice-agent","content":"LiveKit Voice Agent — Real-Time Voice over WebRTC\n\nA WebRTC voice agent that uses LiveKit for the real-time media loop and NeuroLink as the brain (LLM, tools, memory).\n\nTable of Contents\nProblem Statement & Solution\nArchitecture Overview\nDeployment Topologies (Cloud & Self-Hosted)\nCore Components\nHow NeuroLink Owns the Brain\nRuntime Flow\nUsage Example\nSource Layout\nConfiguration\nTuning the Voice Loop (VAD, Turn Detection, Interruption, Language)\nConversation Memory\nImplementation Plan\nOperational Behavior\nError Handling & Troubleshooting\nExtensibility Roadmap\n\nProblem Statement & Solution\n\nThe Challenge\n\nThe original NeuroLink voice agent (see voice-agent.md) runs a browser-to-server loop over a WebSocket. That design works, but a WebSocket transport carries structural limits for real-time audio:\nTCP head-of-line blocking and no jitter buffer cause choppy audio on lossy networks\nno built-in acoustic echo cancellation — the assistant can be transcribed by its own mic input\nraw PCM is ~8–10× the bandwidth of a compressed codec, and all of it flows through the application server\nvoice-activity detection runs on the application server's event loop, capping per-process concurrency\nSvelteKit and similar frameworks cannot accept the WebSocket upgrade without a custom server entry\n\nThe Solution\n\nThe LiveKit voice agent moves the transport to WebRTC via LiveKit, while keeping NeuroLink as the brain. LiveKit (an open-source WebRTC platform with a managed cloud and a self-hostable server) provides the parts that are hard to build correctly:\nWebRTC transport with echo cancellation, jitter buffering, packet-loss concealment, and Opus compression\nvoice-activity detection, turn detection, and interruption handling\na worker/job model that runs each call in its own process for isolation and horizontal scaling\n\nNeuroLink remains responsible for the conversation itself:\nthe LLM (any NeuroLink provider — Bedrock/Claude, OpenAI, Gemini, etc.)\ntool calling (MCP and registered tools), decided and executed inside neurolink.stream()\nconversation memory, keyed by a stable conversationId\n\nKey Benefits\nProduction-grade real-time audio without building media plumbing\nNeuroLink stays the brain — generate()/stream(), tools, and memory are unchanged\nWorker-per-call scaling provided by the LiveKit Agents runtime\nCloud or self-hosted with identical application code\nProvider-agnostic brain layer that can later back other transports\n\nArchitecture Overview\n\nSystem Flow Diagram\n\nDivision of Responsibility\n\n| Concern                                     | Owner                               |\n| ------------------------------------------- | ----------------------------------- |\n| WebRTC transport, AEC, jitter, Opus         | LiveKit                             |\n| VAD, turn detection, interruption           | LiveKit Agents                      |\n| Worker-per-call process isolation & scaling | LiveKit Agents                      |\n| STT / TTS                                   | LiveKit plugins (configurable)      |\n| LLM, tool-calling, memory                   | NeuroLink                           |\n| Conversation history source of truth        | NeuroLink memory (conversationId) |\n\nDeployment Topologies (Cloud & Self-Hosted)\n\nThe application code is identical across topologies; only LIVEKIT_URL and credentials change.\n\nTopology A — LiveKit Cloud (managed)\nRooms are created automatically on LiveKit's servers on first join.\nThe worker connects outbound to Cloud and receives dispatched Jobs over that connection — no inbound exposure or tunneling is required, even in local development.\nBilling is per participant-minute (a free Build tier is suitable for development).\nUse when: fastest setup, minimal media ops, dev/staging, or production without running media infrastructure.\n\nTopology B — Self-Hosted LiveKit (in-house)\nThe livekit-server (open source) runs on your own infrastructure (for example, Kubernetes behind your ingress/service mesh).\nMedia stays inside your network; there is no per-minute media fee — you pay only for compute and bandwidth.\nUse when: cost control at scale, data-residency/compliance requirements, or full control over the media path.\n\nLocal Development\nConsole mode: the worker runs standalone using the host machine's microphone and speakers — no LiveKit server and no browser required. Best for iterating on the brain loop.\nLocal server: livekit-server --dev (placeholder credentials, no external dependencies) with the browser and worker on localhost.\nCloud from local: point local LIVEKIT_URL at a Cloud project. Because the worker connects outbound, Cloud can dispatch Jobs to a locally-running worker without tunneling.\n\nCore Components\nLiveKit Agents Worker\n\nA long-lived Node process built on @livekit/agents. It registers with the LiveKit server under an agentName (for example, neurolink-voice). For each room, LiveKit dispatches a Job, which the runtime runs in its own process — this is the worker-per-call isolation that bounds the ","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"","lvl3":""}},
+{"objectID":"83f5b0de036a8a17898acf6e57b876b24b8eab4db20af614b766816e4da47913","title":"LiveKit Voice Agent — Real-Time Voice over WebRTC","url":"/docs/features/livekit-voice-agent","content":"LiveKit Voice Agent — Real-Time Voice over WebRTC\n\nA WebRTC voice agent that uses LiveKit for the real-time media loop and NeuroLink as the brain (LLM, tools, memory).\n\nTable of Contents\nProblem Statement & Solution\nArchitecture Overview\nDeployment Topologies (Cloud & Self-Hosted)\nCore Components\nHow NeuroLink Owns the Brain\nRuntime Flow\nUsage Example\nSource Layout\nConfiguration\nTuning the Voice Loop (VAD, Turn Detection, Interruption, Language)\nConversation Memory\nRealtime Speech-to-Speech Backends (Gemini Live and xAI Grok Voice)\nImplementation Plan\nOperational Behavior\nError Handling & Troubleshooting\nExtensibility Roadmap\n\nProblem Statement & Solution\n\nThe Challenge\n\nThe original NeuroLink voice agent (see voice-agent.md) runs a browser-to-server loop over a WebSocket. That design works, but a WebSocket transport carries structural limits for real-time audio:\nTCP head-of-line blocking and no jitter buffer cause choppy audio on lossy networks\nno built-in acoustic echo cancellation — the assistant can be transcribed by its own mic input\nraw PCM is ~8–10× the bandwidth of a compressed codec, and all of it flows through the application server\nvoice-activity detection runs on the application server's event loop, capping per-process concurrency\nSvelteKit and similar frameworks cannot accept the WebSocket upgrade without a custom server entry\n\nThe Solution\n\nThe LiveKit voice agent moves the transport to WebRTC via LiveKit, while keeping NeuroLink as the brain. LiveKit (an open-source WebRTC platform with a managed cloud and a self-hostable server) provides the parts that are hard to build correctly:\nWebRTC transport with echo cancellation, jitter buffering, packet-loss concealment, and Opus compression\nvoice-activity detection, turn detection, and interruption handling\na worker/job model that runs each call in its own process for isolation and horizontal scaling\n\nNeuroLink remains responsible for the conversation itself:\nthe LLM (any NeuroLink provider — Bedrock/Claude, OpenAI, Gemini, etc.)\ntool calling (MCP and registered tools), decided and executed inside neurolink.stream()\nconversation memory, keyed by a stable conversationId\n\nKey Benefits\nProduction-grade real-time audio without building media plumbing\nNeuroLink stays the brain — generate()/stream(), tools, and memory are unchanged\nWorker-per-call scaling provided by the LiveKit Agents runtime\nCloud or self-hosted with identical application code\nProvider-agnostic brain layer that can later back other transports\n\nArchitecture Overview\n\nSystem Flow Diagram\n\nDivision of Responsibility\n\n| Concern                                     | Owner                               |\n| ------------------------------------------- | ----------------------------------- |\n| WebRTC transport, AEC, jitter, Opus         | LiveKit                             |\n| VAD, turn detection, interruption           | LiveKit Agents                      |\n| Worker-per-call process isolation & scaling | LiveKit Agents                      |\n| STT / TTS                                   | LiveKit plugins (configurable)      |\n| LLM, tool-calling, memory                   | NeuroLink                           |\n| Conversation history source of truth        | NeuroLink memory (conversationId) |\n\nDeployment Topologies (Cloud & Self-Hosted)\n\nThe application code is identical across topologies; only LIVEKIT_URL and credentials change.\n\nTopology A — LiveKit Cloud (managed)\nRooms are created automatically on LiveKit's servers on first join.\nThe worker connects outbound to Cloud and receives dispatched Jobs over that connection — no inbound exposure or tunneling is required, even in local development.\nBilling is per participant-minute (a free Build tier is suitable for development).\nUse when: fastest setup, minimal media ops, dev/staging, or production without running media infrastructure.\n\nTopology B — Self-Hosted LiveKit (in-house)\nThe livekit-server (open source) runs on your own infrastructure (for example, Kubernetes behind your ingress/service mesh).\nMedia stays inside your network; there is no per-minute media fee — you pay only for compute and bandwidth.\nUse when: cost control at scale, data-residency/compliance requirements, or full control over the media path.\n\nLocal Development\nConsole mode: the worker runs standalone using the host machine's microphone and speakers — no LiveKit server and no browser required. Best for iterating on the brain loop.\nLocal server: livekit-server --dev (placeholder credentials, no external dependencies) with the browser and worker on localhost.\nCloud from local: point local LIVEKIT_URL at a Cloud project. Because the worker connects outbound, Cloud can dispatch Jobs to a locally-running worker without tunneling.\n\nCore Components\nLiveKit Agents Worker\n\nA long-lived Node process built on @livekit/agents. It registers with the LiveKit server under an agentName (for example, neurolink-voice). For each room, LiveKit dispatches a Job, which the runtime runs in its ","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"","lvl3":""}},
 {"objectID":"18162c78ef8435753a2a960c9aaacf301d553ede68ef97184f924649726c9b59","title":"LiveKit Voice Agent — Real-Time Voice over WebRTC","url":"/docs/features/livekit-voice-agent#livekit-voice-agent-real-time-voice-over-webrtc","content":"A WebRTC voice agent that uses LiveKit for the real-time media loop and NeuroLink as the brain (LLM, tools, memory).","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl3":""}},
-{"objectID":"1f733cc14879f19626ff1c687558c535be79cba162237c90533f45426f7d98cc","title":"Table of Contents","url":"/docs/features/livekit-voice-agent#table-of-contents","content":"Problem Statement & Solution\nArchitecture Overview\nDeployment Topologies (Cloud & Self-Hosted)\nCore Components\nHow NeuroLink Owns the Brain\nRuntime Flow\nUsage Example\nSource Layout\nConfiguration\nTuning the Voice Loop (VAD, Turn Detection, Interruption, Language)\nConversation Memory\nImplementation Plan\nOperational Behavior\nError Handling & Troubleshooting\nExtensibility Roadmap","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Table of Contents","lvl3":""}},
+{"objectID":"1f733cc14879f19626ff1c687558c535be79cba162237c90533f45426f7d98cc","title":"Table of Contents","url":"/docs/features/livekit-voice-agent#table-of-contents","content":"Problem Statement & Solution\nArchitecture Overview\nDeployment Topologies (Cloud & Self-Hosted)\nCore Components\nHow NeuroLink Owns the Brain\nRuntime Flow\nUsage Example\nSource Layout\nConfiguration\nTuning the Voice Loop (VAD, Turn Detection, Interruption, Language)\nConversation Memory\nRealtime Speech-to-Speech Backends (Gemini Live and xAI Grok Voice)\nImplementation Plan\nOperational Behavior\nError Handling & Troubleshooting\nExtensibility Roadmap","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Table of Contents","lvl3":""}},
 {"objectID":"91eb3bc5f2e53ccf0662d056db82c3261cd9995803814a4bd489a21b509d5ac8","title":"The Challenge","url":"/docs/features/livekit-voice-agent#the-challenge","content":"The original NeuroLink voice agent (see voice-agent.md) runs a browser-to-server loop over a WebSocket. That design works, but a WebSocket transport carries structural limits for real-time audio:\nTCP head-of-line blocking and no jitter buffer cause choppy audio on lossy networks\nno built-in acoustic echo cancellation — the assistant can be transcribed by its own mic input\nraw PCM is ~8–10× the bandwidth of a compressed codec, and all of it flows through the application server\nvoice-activity detection runs on the application server's event loop, capping per-process concurrency\nSvelteKit and similar frameworks cannot accept the WebSocket upgrade without a custom server entry","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"The Challenge","lvl3":""}},
 {"objectID":"1cb9d49f576a65ba650f409acd589e1409d749a9e793c8007fcdd653049227cd","title":"The Solution","url":"/docs/features/livekit-voice-agent#the-solution","content":"The LiveKit voice agent moves the transport to WebRTC via LiveKit, while keeping NeuroLink as the brain. LiveKit (an open-source WebRTC platform with a managed cloud and a self-hostable server) provides the parts that are hard to build correctly:\nWebRTC transport with echo cancellation, jitter buffering, packet-loss concealment, and Opus compression\nvoice-activity detection, turn detection, and interruption handling\na worker/job model that runs each call in its own process for isolation and horizontal scaling\n\nNeuroLink remains responsible for the conversation itself:\nthe LLM (any NeuroLink provider — Bedrock/Claude, OpenAI, Gemini, etc.)\ntool calling (MCP and registered tools), decided and executed inside neurolink.stream()\nconversation memory, keyed by a stable conversationId","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"The Solution","lvl3":""}},
 {"objectID":"8c89b1ddf6fdca318ba346bdddbd0dc7a7d0e750cf4eeffdde705f11e43e7dc7","title":"Key Benefits","url":"/docs/features/livekit-voice-agent#key-benefits","content":"Production-grade real-time audio without building media plumbing\nNeuroLink stays the brain — generate()/stream(), tools, and memory are unchanged\nWorker-per-call scaling provided by the LiveKit Agents runtime\nCloud or self-hosted with identical application code\nProvider-agnostic brain layer that can later back other transports","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Key Benefits","lvl3":""}},
@@ -3676,6 +3676,14 @@
 {"objectID":"804a64dd17c69cb8d4af469682d01d12f00bec2d6869a5faef30865da99f176d","title":"Speech provider selection","url":"/docs/features/livekit-voice-agent#speech-provider-selection","content":"STT and TTS plugins are chosen per agent and configured by environment credentials.\nSTT: soniox, deepgram. TTS: cartesia, elevenlabs.\nOnly set voice / model if your account supports them; otherwise omit those\n  fields to use the plugin's own defaults.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Speech provider selection","lvl3":""}},
 {"objectID":"5589375c3b5b8ee71271f32386ef6b0f878bc88f7097a13ee7d8ef03b694655e","title":"Conversation Memory","url":"/docs/features/livekit-voice-agent#conversation-memory","content":"The agent remembers earlier turns automatically when the NeuroLink instance you\nbuild inside createNeuroLink has conversation memory enabled. History is the\nagent's source of truth — LiveKit's own transcript context is not used for\ngeneration.\n\nHow it behaves:\nKeyed per call. Each room/call is an isolated conversation; the id is\n  derived from the room name. Override the prefix with conversationIdPrefix\n  (default \"voice\").\nIn-memory by default; Redis for persistence. Set REDIS_URL to use a\n  shared store that survives worker restarts and is shared across worker\n  replicas — important because each call runs in its own job process.\nWorks across turns within the session. The user can say \"my name is Alex\"\n  and later ask \"what's my name?\" and the agent recalls it.\n\nMemory persists only when the instance is configured with\nconversationMemory.enabled. Without it, each turn is independent.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Conversation Memory","lvl3":""}},
 {"objectID":"e6ef692117fe2ad968f2bb5bd1692e50783a069a189d3dbfd6ac49f71ef84c73","title":"Remembering a realtime call in the user's own conversation","url":"/docs/features/livekit-voice-agent#remembering-a-realtime-call-in-the-users-own-conversation","content":"A speech-to-speech call (defineRealtimeVoiceAgent) never goes through\nstream() or generate(), so by default nothing it says is stored. Opt in with\nmemory and the call's turns are written, through\nNeuroLink.recordConversationTurn, to the same conversation memory typed chat\nuses, appended to the conversation the person already has open. The model is\nalso seeded with that conversation's recent history when the call starts, so it\nknows what was said before.\n\nWhose conversation is never taken from the client. The host that creates\nthe room signs a binding and puts it in the room metadata next to authToken\nand mcpContext. The worker stores nothing unless it verifies.\nThe signature is HMAC-SHA256 over [\"neurolink.voice.memory.v1\", room,\nuserId, sessionId, exp] (a JSON array, so no choice of values can make two\n  bindings collide), base64url-encoded. It covers the room name, so a binding\n  copied into another room does not verify. exp is Unix seconds; the default\n  lifetime is two hours, the cap 24 hours.\nThe shared secret is at least 16 characters, read from\n  NEUROLINK_VOICE_MEMORY_SECRET unless memory.secret is set on the worker.\nsessionId matches ^[A-Za-z0-9_-]{1,128}$ (no :, which is part of the\n  stored key). userId is 1 to 256 characters with no whitespace or control\n  characters; : is allowed because chat user ids contain it.\nFail closed. No secret, no binding, a malformed or expired binding, or a\n  bad signature: the call runs normally, nothing is stored, and a warning\n  names the reason (never the ids).\nA client may name the conversation to append to. The host accepts the name\n  only when neurolink.conversationExists({ sessionId, userId }) is true for\n  the verified user. Otherwise it mints a new voice-<uuid>.\n\nWhat is stored. Each finished exchange becomes a user message and an\nassistant message with the ids ${turnId}:user and ${turnId}:assistant, where\nturnId is the realtime session's assistant message id, and\nmetadata.modality: \"voice\".","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Remembering a realtime call in the user's own conversation","lvl3":""}},
+{"objectID":"2b5f572d39aad7e08324d267d06f5dcbc4e90af9c0c31f1d536c84cc4243c5c1","title":"Realtime Speech-to-Speech Backends (Gemini Live and xAI Grok Voice)","url":"/docs/features/livekit-voice-agent#realtime-speech-to-speech-backends-gemini-live-and-xai-grok-voice","content":"defineRealtimeVoiceAgent runs one realtime model that does speech-to-text,\nreasoning, text-to-speech and turn detection over a single connection. Gemini\nLive on Vertex is the default. xAI's Grok Voice is a second backend, reached\nthrough LiveKit's xAI plugin.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Realtime Speech-to-Speech Backends (Gemini Live and xAI Grok Voice)","lvl3":""}},
+{"objectID":"3ea0de3512723dda2eba65763b09a6e3eff2ded8a4d0aa2fa41581d46b88e6c7","title":"Status","url":"/docs/features/livekit-voice-agent#status","content":"Tested against a stub WebSocket server only. The stub listens on\n  127.0.0.1. It shows what the LiveKit xAI plugin sends to whatever it is\n  pointed at, and how this package reads the OpenAI-style events it is given.\n  It shows nothing about how xAI answers.\nNo request has been made to xAI by this project. No xAI key was available\n  when the backend was written.\nNot established here: behaviour against the live service, latency,\n  accuracy, and every xAI benchmark. Nothing in this section says the xai\n  backend works with the real service. Treat it as untested against xAI until\n  someone runs a call with a real key.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Status","lvl3":""}},
+{"objectID":"6676179c44d375115fe98161f9377ba40357b82eebf4a99b7b330ede9721e8c3","title":"Choosing the backend","url":"/docs/features/livekit-voice-agent#choosing-the-backend","content":"| Variable                                                                          | Backend | Meaning                                                                                                                                                                                                                                                                                                                                                                            |\n| --------------------------------------------------------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| VOICE_REALTIME_BACKEND                                                          | both    | gemini or xai. Any other value throws at startup, naming the variable and the two choices and repeating nothing from the environment.                                                                                                                                                                                                                                          |\n| VOICE_LIVE_MODEL                                                                | both    | Realtime model id. Default gemini-live-2.5-flash for gemini, grok-voice-3.0 for xai.                                                                                                                                                                                                                                                                                           |\n| VOICE_S2S_VOICE                                                                 | both    | Voice name.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Choosing the backend","lvl3":""}},
+{"objectID":"632f3752058fa7c7c6d48b02af1e8d5212b7daa55001e8cc931227404009278f","title":"Dependencies","url":"/docs/features/livekit-voice-agent#dependencies","content":"The xai backend needs @livekit/agents-plugin-xai and\n@livekit/agents-plugin-openai. Both are optional dependencies of this package,\nso they are installed with its other optional dependencies (an install that\nomits optional dependencies has to add them by hand), and both resolved to\n1.4.6 in its lockfile. The xAI plugin asks for @livekit/agents 1.4.6 exactly,\nwhich is why that package moved from 1.4.5 to 1.4.6 with it. The options the\nGemini backend builds are unchanged by the move, but no live Gemini call was\nrun after it.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Dependencies","lvl3":""}},
+{"objectID":"a890643483526cae888fdc083535c83d5313ddef29c4a24f6d2771e94dcf3f71","title":"What the xai backend does","url":"/docs/features/livekit-voice-agent#what-the-xai-backend-does","content":"The LiveKit xAI plugin extends the OpenAI realtime model. Against a stub it\nconnects to <base URL>/realtime?model=<id> with Authorization: Bearer\n<XAI_API_KEY> (the key is never put in the URL) and its first event is a\nsession.update that carries the model, the voice, output_modalities:\n[\"audio\"], server-side voice-activity detection (threshold 0.5, 300 ms prefix\npadding, 200 ms silence, response creation and interruption on) and an input\ntranscription setting. The plugin's default endpoint is wss://api.x.ai/v1.\n\nTwo things in that list are the plugin's and are not verified against xAI:\nthe plugin sends gpt-4o-mini-transcribe, an OpenAI model name, as the input\n  transcription model;\nthe plugin's own default model is grok-voice-think-fast-1.0. NeuroLink\n  defaults to grok-voice-3.0, the flagship id xAI's documentation lists\n  (together with the alias grok-voice-latest and a deprecated\n  grok-voice-2.0). The plugin passes the id through; whether the service\n  accepts it through the plugin has not been tested.\n\nA missing key stops the call before it connects. If XAI_API_KEY is unset\nthe plugin falls back to OPENAI_API_KEY, and a stub server showed that\nfallback sending the OpenAI key as the Bearer token to the configured xAI URL.\n(An empty value does not fall back: the plugin throws its own OpenAI API key is\nrequired error. A whitespace-only value is sent as the key.) NeuroLink\ntherefore refuses an unset, empty or whitespace-only key: it throws\nXAI_API_KEY is not set ... when the agent starts and again in\ncreateRealtimeLlm, and no connection is made.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"What the xai backend does","lvl3":""}},
+{"objectID":"b4b0a14667e072deb333f9771c062361a617e33da9a0142087c0ba9502d3e4f7","title":"How each part of the agent behaves on xai","url":"/docs/features/livekit-voice-agent#how-each-part-of-the-agent-behaves-on-xai","content":"| Part                                                               | On xai                                                                                                                                                                                                                            |\n| ------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Vertex ADC and GEMINI_API_KEY clearing                           | Not run. The xai backend never touches the Gemini environment.                                                                                                                                                                    |\n| Data-channel event bridge, HITL, room context                      | Unchanged. They work on the LiveKit room and the agent session, not on a model.                                                                                                                                                   |\n| Session events (user transcript, assistant text, turn done, usage) | Unchanged code. Against a stub, OpenAI-style transcript and usage events reach the shapes the agent reads.                                                                                                                        |\n| MCP tool bridge and schema sanitizer                               | Unchanged. The sanitizer produces plain JSON Schema (the OpenAI-style API takes full JSON Schema), and the plugin forwards it as is. Keeping it is the safer choice: it drops $ref and unions that xAI has not been tried with. |\n| Conversation memory (recording and seeding)                        | Recording is shared by both backends.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"How each part of the agent behaves on xai","lvl3":""}},
+{"objectID":"f2c4f00431a110b4ec6b67653e157a21d79cc3d52751253260a13758eec90e66","title":"Still unknown without a real key","url":"/docs/features/livekit-voice-agent#still-unknown-without-a-real-key","content":"Whether xAI accepts the model ids above through the plugin; whether it accepts\nthe input transcription setting; whether it accepts the sanitized tool schemas\nand underscore tool names; the event names and ordering xAI really sends (the\nplugin line pinned here, 1.4.6, predates later xAI-specific handling in newer\nplugin releases); whether the seeded history is accepted; and how interruption,\ntruncation and tool calls behave. A live smoke test with a key supplied by the\nowner is the missing step.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Still unknown without a real key","lvl3":""}},
+{"objectID":"9eb38592a55bd3da33ee8a936b4648d18645a49909579f1fccb19844c8263ac7","title":"Troubleshooting","url":"/docs/features/livekit-voice-agent#troubleshooting","content":"VOICE_REALTIME_BACKEND must be \"gemini\" or \"xai\": fix the value, or unset it\n  for gemini.\nXAI_API_KEY is not set: set it in the worker's environment. The check also\n  rejects a whitespace-only value.\nXAI_REALTIME_BASE_URL must be a ws:// or wss:// URL: use a WebSocket URL,\n  not an https:// one.\nA call that will not start on the xai backend: confirm that VOICE_LIVE_MODEL\n  and VOICE_S2S_VOICE hold xAI values and not leftover Gemini ones, since both\n  variables are shared.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Troubleshooting","lvl3":""}},
 {"objectID":"7ffe7e32a7c2baea6c491d743617448eb70eeb4afe439960050efdc3ead18c48","title":"Implementation Plan","url":"/docs/features/livekit-voice-agent#implementation-plan","content":"The integration is built and validated in phases. Each phase is independently testable.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Implementation Plan","lvl3":""}},
 {"objectID":"d4c9885eea3272e250fa1a0dbc1af9503b1a24f0217a82dd0c30d93bf6c5df31","title":"Phase 0 — Console-mode spike (no infrastructure)","url":"/docs/features/livekit-voice-agent#phase-0-console-mode-spike-no-infrastructure","content":"Build a minimal agent (Silero VAD + Deepgram STT + ElevenLabs TTS + llmNode → neurolink.stream()) and run it in console mode using the host machine's mic/speakers. Validates the NeuroLink brain loop, conversationId history, and a tool call — with no LiveKit server and no browser. Requires only STT/TTS and LLM credentials.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Phase 0 — Console-mode spike (no infrastructure)","lvl3":""}},
 {"objectID":"9dddf98f952af643b5357bdefbcb5554a3ff2d9c33a16b0c432243fd68db5569","title":"Phase 1 — NeuroLink LiveKit module","url":"/docs/features/livekit-voice-agent#phase-1-neurolink-livekit-module","content":"Implement brain.ts, voiceAgent.ts, voiceAgentWorker.ts, tokens.ts, config.ts; add the @juspay/neurolink/livekit export and optional/peer dependencies. The worker factory accepts an external NeuroLink instance so a host application's registered tools are available. Wire abort-on-interrupt. Verify build, type-check, and lint.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Phase 1 — NeuroLink LiveKit module","lvl3":""}},
@@ -3987,7 +3995,7 @@
 {"objectID":"2f01481e1bd920ff04b7d26da5bc86efababd09cbd1f763a1f119fb7d1443582","title":"Serverless and short-lived processes","url":"/docs/features/observability#serverless-and-short-lived-processes","content":"Spans are batched. Flush before the process freezes or exits:\n\nOr send each span as it ends:\n\nshutdownOpenTelemetry() is terminal for the process: telemetry cannot restart afterwards. While other NeuroLink instances are still alive it only flushes.","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Serverless and short-lived processes","lvl3":""}},
 {"objectID":"721428a68bdcef28a5437443fe5a57565c6aa3b324a48ccc39728e5d7ff25e2d","title":"Health","url":"/docs/features/observability#health","content":"isHealthy and hasProcessor are true only while a Langfuse exporter is registered, whether NeuroLink owns the provider or runs beside a host provider. With a host provider, the exporter still needs createExporterSpanProcessor() on that provider to receive spans.","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Health","lvl3":""}},
 {"objectID":"68b0ab87bc1ff79e31454e9550e76c174e5a1a4628f3c9de0e74a7dc6e07bdd2","title":"Proxy","url":"/docs/features/observability#proxy","content":"The proxy records the pool account label (not an email address), sets userId and sessionId from x-neurolink-user-id and x-neurolink-session-id, names traces proxy:<model>, and records request and response bodies, usage, cost and time to first token. See the Telemetry Guide and Claude Proxy Observability.","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Proxy","lvl3":""}},
-{"objectID":"818b18185bdd4b8b7003f170da3113452c78895619ec762279322c8b2b587801","title":"Voice (LiveKit)","url":"/docs/features/observability#voice-livekit","content":"Pipeline mode traces through stream() and carries the voice tag. Realtime mode records one trace per conversation turn, with transcripts as input and output and tool calls as tool records, and only when voice memory is enabled. Full transcripts are sent to Langfuse by default; use mask for sensitive data.","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Voice (LiveKit)","lvl3":""}},
+{"objectID":"818b18185bdd4b8b7003f170da3113452c78895619ec762279322c8b2b587801","title":"Voice (LiveKit)","url":"/docs/features/observability#voice-livekit","content":"Pipeline mode traces through stream() and carries the voice tag. Realtime mode records one trace per conversation turn, with transcripts as input and output and tool calls as tool records, and only when voice memory is enabled. The realtime generation record carries the model id and the provider of the realtime backend (vertex for Gemini Live, xai for Grok Voice). Full transcripts are sent to Langfuse by default; use mask for sensitive data.","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Voice (LiveKit)","lvl3":""}},
 {"objectID":"6ddae5fd4f96e8c2a1c9d924054685440eb46353183984ff0b8d3167e12071b8","title":"Troubleshooting","url":"/docs/features/observability#troubleshooting","content":"| Symptom                           | Check                                                                                                                                                                                                                              |\n| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Nothing reaches Langfuse          | Both keys set and langfuse not false; getLangfuseHealthStatus(); call flushOpenTelemetry() on exit                                                                                                                         |\n| Spans missing with own OTEL setup | Add createExporterSpanProcessor() to your provider and pass the Langfuse keys to NeuroLink (or set LANGFUSE_*), or use new LangfuseSpanProcessor() with its own keys; register the processor before the first NeuroLink call |\n| Wrong host                        | Variable is LANGFUSE_BASE_URL, not LANGFUSE_BASEURL                                                                                                                                                                            |\n| Traces stop after shutdown        | shutdownOpenTelemetry() cannot be undone in the same process; use flushOpenTelemetry()                                                                                                                                         |","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Troubleshooting","lvl3":""}},
 {"objectID":"c26071897c848ba9cf00651234ebd459171008740460f6075451e866420e5881","title":"Migrating from the previous version","url":"/docs/features/observability#migrating-from-the-previous-version","content":"This release is breaking.","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Migrating from the previous version","lvl3":""}},
 {"objectID":"310e635468902de6eca8789d9ce5905c4ecb08a665b802119f33a6ae78a6f429","title":"Configuration fields","url":"/docs/features/observability#configuration-fields","content":"| Removed                                                       | Replacement                                                             |\n| ------------------------------------------------------------- | ----------------------------------------------------------------------- |\n| langfuse.enabled                                            | Set the keys to enable; langfuse: false disables                      |\n| langfuse.userId, langfuse.sessionId                       | Per-call context.userId, context.sessionId, or setLangfuseContext |\n| useExternalTracerProvider, autoDetectExternalProvider     | Automatic: a registered global provider is detected                     |\n| skipLangfuseSpanProcessor                                   | Register your own provider; NeuroLink then adds nothing                 |\n| mediaUploadEnabled, LANGFUSE_MEDIA_UPLOAD_ENABLED         | Removed; the exporter does not upload media                             |\n| autoDetectOperationName, operationName, traceNameFormat | context.traceName                                                     |\n| customAttributes                                            | context.metadata                                                      |\n| conversationId, requestId on the config                   | context.conversationId, context.requestId (stored as metadata)      |\n| NEUROLINK_TELEMETRY_ENABLED                                 | Removed; the LANGFUSE_* keys enable tracing                           |\n| LANGFUSE_BASEURL hint                                       | LANGFUSE_BASE_URL                                                     |","hierarchy":{"lvl0":"Features","lvl1":"Observability Guide","lvl2":"Configuration fields","lvl3":""}},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "13.3.0",
+  "version": "13.4.0",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -115,6 +115,7 @@
     "test:forced-tool-declared": "pnpm exec tsx test/continuous-test-suite-forced-tool-declared.ts",
     "test:span-content-redaction": "pnpm exec tsx test/continuous-test-suite-span-content-redaction.ts",
     "test:voice-turn-memory": "pnpm exec tsx test/continuous-test-suite-voice-turn-memory.ts",
+    "test:realtime-voice": "pnpm exec tsx test/continuous-test-suite-realtime-voice.ts",
     "test:error-classification-e2e": "pnpm exec tsx test/continuous-test-suite-error-classification-e2e.ts",
     "test:error-classifier-contract": "pnpm exec tsx test/continuous-test-suite-error-classifier-contract.ts",
     "test:bedrock-inference-profile": "pnpm exec tsx test/continuous-test-suite-bedrock-inference-profile.ts",
@@ -235,7 +236,7 @@
     "// CI tier — fast, no live AI calls, safe for every commit (test:unit; also see the separate provider-safety-net CI job, which runs build + test:providers-mocked + test:provider-structure + test:error-classifier-contract on every PR)": "",
     "test:tool-routing": "pnpm exec tsx test/continuous-test-suite-tool-routing.ts",
     "test:tool-routing-semantic": "pnpm exec tsx test/continuous-test-suite-tool-routing-semantic.ts",
-    "test:unit": "pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:registry && pnpm run test:mcp:spans && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:classifier-router && pnpm run test:tool-routing-semantic && pnpm run test:mcp-result-cache && pnpm run test:mcp-direct-name-repair && pnpm run test:mcp-breaker-resolved-errors && pnpm run test:archive:security && pnpm run test:office:security && pnpm run test:image-exif && pnpm run test:vector-chroma && pnpm run test:vector-pgvector && pnpm run test:vector-pinecone && pnpm run test:provider-wiring && pnpm run test:docs-mcp && pnpm run test:retired-model-defaults && pnpm run test:cli-fallback && pnpm run test:embedding-response-validation && pnpm run test:xiaomi-mimo-pilot && pnpm run test:rag-bounds && pnpm run test:longcat-pilot && pnpm run test:runinfra-pilot && pnpm run test:catalog-output-ceiling && pnpm run test:package-declarations",
+    "test:unit": "pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:registry && pnpm run test:mcp:spans && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:classifier-router && pnpm run test:tool-routing-semantic && pnpm run test:mcp-result-cache && pnpm run test:mcp-direct-name-repair && pnpm run test:mcp-breaker-resolved-errors && pnpm run test:archive:security && pnpm run test:office:security && pnpm run test:image-exif && pnpm run test:vector-chroma && pnpm run test:vector-pgvector && pnpm run test:vector-pinecone && pnpm run test:provider-wiring && pnpm run test:docs-mcp && pnpm run test:retired-model-defaults && pnpm run test:cli-fallback && pnpm run test:embedding-response-validation && pnpm run test:xiaomi-mimo-pilot && pnpm run test:rag-bounds && pnpm run test:longcat-pilot && pnpm run test:runinfra-pilot && pnpm run test:catalog-output-ceiling && pnpm run test:package-declarations && pnpm run test:realtime-voice",
     "// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit; test:matrix, a different suite covering the full provider capability matrix, runs nightly via .github/workflows/live-matrix.yml — test:providers itself is still only wired into test:live, not any GitHub Actions workflow)": "",
     "test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic && pnpm run test:model-not-found-retryable",
     "// CI tier — product output (image/video/TTS/PPT) — costs $$ per run (not wired into any GitHub Actions workflow as of this comment; run manually or add to live-matrix.yml if nightly coverage is needed)": "",
@@ -560,14 +561,16 @@
     "@hono/node-server": "^1.19.15",
     "@koa/cors": "^5.0.0",
     "@koa/router": "^15.3.1",
-    "@livekit/agents": "^1.4.5",
+    "@livekit/agents": "^1.4.6",
     "@livekit/agents-plugin-cartesia": "^1.4.5",
     "@livekit/agents-plugin-deepgram": "^1.4.5",
     "@livekit/agents-plugin-elevenlabs": "^1.4.5",
     "@livekit/agents-plugin-google": "^1.4.5",
     "@livekit/agents-plugin-livekit": "^1.4.5",
+    "@livekit/agents-plugin-openai": "^1.4.6",
     "@livekit/agents-plugin-silero": "^1.4.5",
     "@livekit/agents-plugin-soniox": "^1.4.5",
+    "@livekit/agents-plugin-xai": "^1.4.6",
     "@livekit/rtc-node": "^0.13.29",
     "@picovoice/cobra-node": "^3.0.2",
     "bullmq": "^5.52.2",