@juspay/neurolink 13.2.0 → 13.4.0

package/dist/voice/livekit/config.d.ts

@@ -7,7 +7,7 @@
  *
  * See docs/features/livekit-voice-agent.md.
  */
-import type { LiveKitServerConfig, LiveKitBrainDefaults, RealtimeVoiceConfig } from "../../types/index.js";
+import type { LiveKitServerConfig, LiveKitBrainDefaults, RealtimeVoiceAgentConfig, RealtimeVoiceConfig } from "../../types/index.js";
 /**
  * Resolve LiveKit server connection settings from the environment.
  *
@@ -24,16 +24,21 @@ export declare function resolveLiveKitServerConfig(): LiveKitServerConfig;
  */
 export declare function resolveBrainDefaults(): LiveKitBrainDefaults;
 /**
- * Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
- * the environment.
+ * Resolve the realtime (speech-to-speech) voice configuration from the
+ * environment.
  *
  * Every value has a safe default so a worker can run with no realtime-specific
- * env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
- * service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
- * `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
- * `${LIGHTHOUSE_URL}/ai/mcp/v2`.
+ * env at all. `VOICE_REALTIME_BACKEND` picks the model family (`gemini`, the
+ * default, or `xai`); `backend` here overrides it and then the environment's
+ * value is not read. A non-blank `xaiBaseURL` here likewise replaces
+ * `XAI_REALTIME_BASE_URL` after the same ws or wss check.
+ * `VOICE_LIVE_MODEL` overrides the backend's default model.
+ * Vertex `project` is resolved from `VERTEX_PROJECT`, then the service-account
+ * project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then `GOOGLE_CLOUD_PROJECT_ID`;
+ * project, location and modality only matter to the gemini backend. The MCP URL
+ * is `VOICE_MCP_URL` if set, otherwise `${LIGHTHOUSE_URL}/ai/mcp/v2`.
  */
-export declare function resolveRealtimeVoiceConfig(): RealtimeVoiceConfig;
+export declare function resolveRealtimeVoiceConfig(overrides?: Pick<RealtimeVoiceAgentConfig, "backend" | "xaiBaseURL">): RealtimeVoiceConfig;
 /**
  * Resolve the semantic end-of-utterance (EOU) turn-detection settings.
  *

package/dist/voice/livekit/config.js

@@ -68,6 +68,9 @@ export function resolveBrainDefaults() {
     };
 }
 const DEFAULT_REALTIME_MODEL = "gemini-live-2.5-flash";
+// xAI's documented flagship Grok Voice id. Never exercised against the real
+// service; the plugin's own default is `grok-voice-think-fast-1.0`.
+const DEFAULT_XAI_REALTIME_MODEL = "grok-voice-3.0";
 const DEFAULT_REALTIME_LOCATION = "global";
 const DEFAULT_RESPONSE_MODALITY = "AUDIO";
 const DEFAULT_SYSTEM_PROMPT = "You are a concise voice assistant for a merchant dashboard. Keep replies short and spoken.";
@@ -75,25 +78,86 @@ const DEFAULT_GREETING = "Briefly greet the merchant and ask how you can help wi
 const DEFAULT_LIGHTHOUSE_URL = "http://localhost:5173";
 const DEFAULT_MCP_PATH = "/ai/mcp/v2";
 /**
- * Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
- * the environment.
+ * The backend named by `VOICE_REALTIME_BACKEND`: `gemini` or `xai`, matched
+ * case-insensitively, with blank or absent meaning `gemini`. The error names the
+ * variable and the choices and repeats nothing from the environment.
+ */
+function readRealtimeBackend() {
+    const value = readEnv("VOICE_REALTIME_BACKEND", "gemini").toLowerCase();
+    if (value === "gemini" || value === "xai") {
+        return value;
+    }
+    throw new Error('VOICE_REALTIME_BACKEND must be "gemini" or "xai" (leave it unset for gemini)');
+}
+/**
+ * A backend passed in code. The type already limits it, but a plain-JavaScript
+ * caller is not held by the type, and a typo would otherwise take the xai
+ * branch and fail on a missing key. The error repeats nothing it was given.
+ */
+function requireKnownBackend(value) {
+    if (value === "gemini" || value === "xai") {
+        return value;
+    }
+    throw new Error('The realtime voice backend must be "gemini" or "xai"');
+}
+function isWebSocketUrl(value) {
+    try {
+        const { protocol } = new URL(value);
+        return protocol === "ws:" || protocol === "wss:";
+    }
+    catch {
+        return false;
+    }
+}
+/** A ws or wss URL. The error names the setting and does not repeat the value. */
+function requireWebSocketUrl(value, name) {
+    if (!isWebSocketUrl(value)) {
+        throw new Error(`${name} must be a ws:// or wss:// URL`);
+    }
+    return value;
+}
+/** `XAI_REALTIME_BASE_URL`, which must be a ws or wss URL when set. */
+function readXaiBaseUrl() {
+    const value = readOptionalEnv("XAI_REALTIME_BASE_URL");
+    return value === undefined
+        ? undefined
+        : requireWebSocketUrl(value, "XAI_REALTIME_BASE_URL");
+}
+/** A caller-supplied `xaiBaseURL` gets the same check as the variable; blank means unset, so the variable applies. */
+function resolveXaiBaseUrl(override) {
+    const value = override?.trim();
+    return value ? requireWebSocketUrl(value, "xaiBaseURL") : readXaiBaseUrl();
+}
+/**
+ * Resolve the realtime (speech-to-speech) voice configuration from the
+ * environment.
  *
  * Every value has a safe default so a worker can run with no realtime-specific
- * env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
- * service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
- * `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
- * `${LIGHTHOUSE_URL}/ai/mcp/v2`.
+ * env at all. `VOICE_REALTIME_BACKEND` picks the model family (`gemini`, the
+ * default, or `xai`); `backend` here overrides it and then the environment's
+ * value is not read. A non-blank `xaiBaseURL` here likewise replaces
+ * `XAI_REALTIME_BASE_URL` after the same ws or wss check.
+ * `VOICE_LIVE_MODEL` overrides the backend's default model.
+ * Vertex `project` is resolved from `VERTEX_PROJECT`, then the service-account
+ * project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then `GOOGLE_CLOUD_PROJECT_ID`;
+ * project, location and modality only matter to the gemini backend. The MCP URL
+ * is `VOICE_MCP_URL` if set, otherwise `${LIGHTHOUSE_URL}/ai/mcp/v2`.
  */
-export function resolveRealtimeVoiceConfig() {
+export function resolveRealtimeVoiceConfig(overrides = {}) {
+    const backend = overrides.backend === undefined
+        ? readRealtimeBackend()
+        : requireKnownBackend(overrides.backend);
     const lighthouseUrl = readEnv("LIGHTHOUSE_URL", DEFAULT_LIGHTHOUSE_URL);
     const mcpUrl = readEnv("VOICE_MCP_URL", `${lighthouseUrl}${DEFAULT_MCP_PATH}`);
     return {
+        backend,
         project: readOptionalEnv("VERTEX_PROJECT") ??
             readOptionalEnv("GOOGLE_AUTH_BREEZE_PROJECT_ID") ??
             readOptionalEnv("GOOGLE_CLOUD_PROJECT_ID"),
         location: readEnv("VERTEX_LOCATION", DEFAULT_REALTIME_LOCATION),
-        model: readEnv("VOICE_LIVE_MODEL", DEFAULT_REALTIME_MODEL),
+        model: readEnv("VOICE_LIVE_MODEL", backend === "xai" ? DEFAULT_XAI_REALTIME_MODEL : DEFAULT_REALTIME_MODEL),
         voice: readOptionalEnv("VOICE_S2S_VOICE"),
+        xaiBaseURL: resolveXaiBaseUrl(overrides.xaiBaseURL),
         responseModality: readEnv("VOICE_RESPONSE_MODALITY", DEFAULT_RESPONSE_MODALITY).toUpperCase(),
         systemPrompt: readEnv("VOICE_SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT),
         greeting: readEnv("VOICE_GREETING", DEFAULT_GREETING),

package/dist/voice/livekit/index.d.ts

@@ -15,6 +15,7 @@ export { mintJoinToken } from "./tokens.js";
 export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
 export { defineVoiceAgent } from "./voiceAgent.js";
 export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
+export { createRealtimeLlm } from "./realtimeModel.js";
 export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
 export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
 export { readCallContextFromRoom } from "./roomContext.js";

package/dist/voice/livekit/index.js

@@ -15,6 +15,7 @@ export { mintJoinToken } from "./tokens.js";
 export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
 export { defineVoiceAgent } from "./voiceAgent.js";
 export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
+export { createRealtimeLlm } from "./realtimeModel.js";
 export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
 export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
 export { readCallContextFromRoom } from "./roomContext.js";

package/dist/voice/livekit/realtimeMcpTools.d.ts

@@ -1,12 +1,13 @@
 /**
- * MCP tools → Gemini function tools bridge (realtime mode).
+ * MCP tools → realtime function tools bridge (realtime mode).
  *
- * In speech-to-speech mode Gemini owns the tool loop, so an external MCP
- * server's tools are registered as LiveKit/Gemini function tools rather than
- * NeuroLink tools. This connects to the MCP server as the calling user
- * (`x-auth-token` + base64 `x-context`), sanitizes each tool's schema into the
- * Gemini-safe subset, wires WRITE-labeled (destructive) tools through a HITL
- * confirmation, and forwards tool start/result events to the browser bridge.
+ * In speech-to-speech mode the realtime model (Gemini Live, or xAI Grok Voice)
+ * owns the tool loop, so an external MCP server's tools are registered as
+ * LiveKit function tools rather than NeuroLink tools. This connects to the MCP
+ * server as the calling user (`x-auth-token` + base64 `x-context`), sanitizes
+ * each tool's schema into the Gemini-safe subset (kept for both backends), wires
+ * WRITE-labeled (destructive) tools through a HITL confirmation, and forwards
+ * tool start/result events to the browser bridge.
  *
  * `@livekit/agents` and `@modelcontextprotocol/sdk` are imported dynamically so
  * the core package does not require them unless the realtime agent is used.

package/dist/voice/livekit/realtimeMcpTools.js

@@ -1,12 +1,13 @@
 /**
- * MCP tools → Gemini function tools bridge (realtime mode).
+ * MCP tools → realtime function tools bridge (realtime mode).
  *
- * In speech-to-speech mode Gemini owns the tool loop, so an external MCP
- * server's tools are registered as LiveKit/Gemini function tools rather than
- * NeuroLink tools. This connects to the MCP server as the calling user
- * (`x-auth-token` + base64 `x-context`), sanitizes each tool's schema into the
- * Gemini-safe subset, wires WRITE-labeled (destructive) tools through a HITL
- * confirmation, and forwards tool start/result events to the browser bridge.
+ * In speech-to-speech mode the realtime model (Gemini Live, or xAI Grok Voice)
+ * owns the tool loop, so an external MCP server's tools are registered as
+ * LiveKit function tools rather than NeuroLink tools. This connects to the MCP
+ * server as the calling user (`x-auth-token` + base64 `x-context`), sanitizes
+ * each tool's schema into the Gemini-safe subset (kept for both backends), wires
+ * WRITE-labeled (destructive) tools through a HITL confirmation, and forwards
+ * tool start/result events to the browser bridge.
  *
  * `@livekit/agents` and `@modelcontextprotocol/sdk` are imported dynamically so
  * the core package does not require them unless the realtime agent is used.
@@ -52,7 +53,7 @@ export function mcpResultToText(result) {
     return text || "Tool returned no content.";
 }
 /**
- * The function-result text returned to Gemini when the user rejects a HITL tool.
+ * The function-result text returned to the model when the user rejects a HITL tool.
  * Scope the rejection to THIS attempt — a "never retry" wording made the model
  * refuse the tool permanently even when the user later asked for it again.
  */
@@ -163,7 +164,7 @@ export async function buildRealtimeMcpTools(params) {
                     try {
                         // Third argument is RequestOptions; the SDK aborts the in-flight
                         // request when `timeout` elapses. Without it a stalled server holds
-                        // the turn open indefinitely — Gemini blocks on the function
+                        // the turn open indefinitely — the model waits on the function
                         // result, so the user hears nothing at all rather than an error.
                         const result = await client.callTool({
                             name: mcpTool.name,

package/dist/voice/livekit/realtimeMemory.d.ts

@@ -17,6 +17,14 @@
  * See docs/features/livekit-voice-agent.md.
  */
 import type { JobContext } from "@livekit/agents";
-import type { RealtimeCallMemory, RealtimeVoiceMemoryConfig } from "../../types/index.js";
-/** Verify the call's binding and prepare its recorder; `undefined` means "no memory for this call". */
-export declare function setUpRealtimeCallMemory(ctx: Pick<JobContext, "room" | "addShutdownCallback">, config: RealtimeVoiceMemoryConfig | undefined, model?: string): Promise<RealtimeCallMemory | undefined>;
+import type { RealtimeCallMemory, RealtimeVoiceBackend, RealtimeVoiceMemoryConfig } from "../../types/index.js";
+/**
+ * Verify the call's binding and prepare its recorder; `undefined` means "no
+ * memory for this call". `call` names the model and the realtime backend the
+ * call runs on, which the turn spans carry as `model` and `provider`; the
+ * backend defaults to `gemini`, the original and default one.
+ */
+export declare function setUpRealtimeCallMemory(ctx: Pick<JobContext, "room" | "addShutdownCallback">, config: RealtimeVoiceMemoryConfig | undefined, call?: {
+    model?: string;
+    backend?: RealtimeVoiceBackend;
+}): Promise<RealtimeCallMemory | undefined>;

package/dist/voice/livekit/realtimeMemory.js

@@ -17,6 +17,7 @@
  * See docs/features/livekit-voice-agent.md.
  */
 import { context, trace } from "@opentelemetry/api";
+import { AIProviderName } from "../../constants/enums.js";
 import { setLangfuseContext } from "../../services/server/ai/observability/instrumentation.js";
 import { observe, observeTrace } from "../../telemetry/observation.js";
 import { tracers } from "../../telemetry/tracers.js";
@@ -24,8 +25,13 @@ import { logger } from "../../utils/logger.js";
 import { VOICE_MEMORY_SECRET_ENV, verifyVoiceMemoryBinding, } from "./memoryBinding.js";
 import { createVoiceTurnRecorder } from "./turnRecorder.js";
 const DEFAULT_FLUSH_BUDGET_MS = 800;
+/** The provider a backend's turn spans are attributed to: Gemini Live runs on Vertex AI, Grok Voice on xAI. */
+const TRACE_PROVIDER = {
+    gemini: AIProviderName.VERTEX,
+    xai: AIProviderName.XAI,
+};
 function traceTurn(turn, call) {
-    const { sessionId, userId, roomName, model } = call;
+    const { sessionId, userId, roomName, model, provider } = call;
     try {
         setLangfuseContext({
             sessionId,
@@ -55,7 +61,7 @@ function traceTurn(turn, call) {
                 input: [{ role: "user", content: turn.userMessage }],
                 output: [{ role: "assistant", content: turn.aiResponse }],
                 level,
-                metadata: { provider: "vertex" },
+                metadata: { provider },
             });
             generation.end(endTime);
             const events = turn.events ?? [];
@@ -80,8 +86,14 @@ function traceTurn(turn, call) {
         });
     }
 }
-/** Verify the call's binding and prepare its recorder; `undefined` means "no memory for this call". */
-export async function setUpRealtimeCallMemory(ctx, config, model) {
+/**
+ * Verify the call's binding and prepare its recorder; `undefined` means "no
+ * memory for this call". `call` names the model and the realtime backend the
+ * call runs on, which the turn spans carry as `model` and `provider`; the
+ * backend defaults to `gemini`, the original and default one.
+ */
+export async function setUpRealtimeCallMemory(ctx, config, call = {}) {
+    const { model, backend = "gemini" } = call;
     if (!config) {
         return undefined;
     }
@@ -116,7 +128,13 @@ export async function setUpRealtimeCallMemory(ctx, config, model) {
         write: async (turn) => {
             if (!tracedTurns.has(turn.turnId)) {
                 tracedTurns.add(turn.turnId);
-                traceTurn(turn, { sessionId, userId, roomName: ctx.room.name, model });
+                traceTurn(turn, {
+                    sessionId,
+                    userId,
+                    roomName: ctx.room.name,
+                    model,
+                    provider: TRACE_PROVIDER[backend],
+                });
             }
             const result = await neurolink.recordConversationTurn({
                 sessionId,

package/dist/voice/livekit/realtimeModel.d.ts

@@ -0,0 +1,33 @@
+/**
+ * The realtime model factory: builds the LiveKit speech-to-speech model for the
+ * configured backend.
+ *
+ * `gemini` is Gemini Live on Vertex through `@livekit/agents-plugin-google`;
+ * `xai` is Grok Voice through `@livekit/agents-plugin-xai`, which speaks the
+ * OpenAI realtime protocol to `wss://api.x.ai/v1`. Both plugins and
+ * `@google/genai` are imported dynamically so the core package does not require
+ * them unless the realtime agent is used.
+ *
+ * The xai backend has only been exercised against a stub WebSocket server. It
+ * has never talked to the real xAI service.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import type { RealtimeLlm, RealtimeVoiceConfig } from "../../types/index.js";
+/**
+ * Fail early, naming the variable, when `XAI_API_KEY` is blank or absent.
+ *
+ * The plugin reads the key itself and falls back to `OPENAI_API_KEY` when its
+ * own is missing, which would send an OpenAI key to the xAI host, so a missing
+ * key must stop here. The message never carries a value.
+ */
+export declare function requireXaiApiKey(): void;
+/**
+ * Build the realtime model for `cfg.backend`.
+ *
+ * The gemini branch builds exactly what the agent has always built: Vertex AI,
+ * the configured model, transcripts of both sides, and project, location,
+ * voice and modality when set. The xai branch needs `XAI_API_KEY` in the
+ * environment and throws before touching the plugin when it is missing.
+ */
+export declare function createRealtimeLlm(cfg: RealtimeVoiceConfig): Promise<RealtimeLlm>;

package/dist/voice/livekit/realtimeModel.js

@@ -0,0 +1,89 @@
+/**
+ * The realtime model factory: builds the LiveKit speech-to-speech model for the
+ * configured backend.
+ *
+ * `gemini` is Gemini Live on Vertex through `@livekit/agents-plugin-google`;
+ * `xai` is Grok Voice through `@livekit/agents-plugin-xai`, which speaks the
+ * OpenAI realtime protocol to `wss://api.x.ai/v1`. Both plugins and
+ * `@google/genai` are imported dynamically so the core package does not require
+ * them unless the realtime agent is used.
+ *
+ * The xai backend has only been exercised against a stub WebSocket server. It
+ * has never talked to the real xAI service.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+/**
+ * Fail early, naming the variable, when `XAI_API_KEY` is blank or absent.
+ *
+ * The plugin reads the key itself and falls back to `OPENAI_API_KEY` when its
+ * own is missing, which would send an OpenAI key to the xAI host, so a missing
+ * key must stop here. The message never carries a value.
+ */
+export function requireXaiApiKey() {
+    const key = process.env.XAI_API_KEY;
+    if (typeof key !== "string" || key.trim().length === 0) {
+        throw new Error("XAI_API_KEY is not set; the xai realtime backend reads its key from that environment variable");
+    }
+}
+async function createGeminiRealtimeLlm(cfg) {
+    const google = await import("@livekit/agents-plugin-google");
+    const { Modality } = await import("@google/genai");
+    const modality = Object.values(Modality).find((value) => value === cfg.responseModality);
+    const modelOptions = {
+        vertexai: true,
+        model: cfg.model,
+        // Emit text transcripts of BOTH sides; LiveKit forwards them to the room.
+        inputAudioTranscription: {},
+        outputAudioTranscription: {},
+    };
+    if (modality !== undefined) {
+        modelOptions.modalities = [modality];
+    }
+    if (cfg.project) {
+        modelOptions.project = cfg.project;
+    }
+    if (cfg.location) {
+        modelOptions.location = cfg.location;
+    }
+    if (cfg.voice) {
+        modelOptions.voice = cfg.voice;
+    }
+    return new google.realtime.RealtimeModel(modelOptions);
+}
+async function createXaiRealtimeLlm(cfg) {
+    requireXaiApiKey();
+    const xai = await import("@livekit/agents-plugin-xai");
+    // Keys are added only when set. The plugin spreads these options over its
+    // own defaults, so a key holding `undefined` would erase the default: a
+    // missing `baseURL` would send the xAI key to OPENAI_BASE_URL or
+    // api.openai.com, and no `apiKey` option is passed at all because the plugin
+    // reads XAI_API_KEY itself. The plugin forwards `baseURL` although its
+    // declared option type leaves it out.
+    const modelOptions = { model: cfg.model };
+    if (cfg.voice) {
+        modelOptions.voice = cfg.voice;
+    }
+    if (cfg.xaiBaseURL) {
+        modelOptions.baseURL = cfg.xaiBaseURL;
+    }
+    return new xai.realtime.RealtimeModel(modelOptions);
+}
+/**
+ * Build the realtime model for `cfg.backend`.
+ *
+ * The gemini branch builds exactly what the agent has always built: Vertex AI,
+ * the configured model, transcripts of both sides, and project, location,
+ * voice and modality when set. The xai branch needs `XAI_API_KEY` in the
+ * environment and throws before touching the plugin when it is missing.
+ */
+export async function createRealtimeLlm(cfg) {
+    switch (cfg.backend) {
+        case "gemini":
+            return createGeminiRealtimeLlm(cfg);
+        case "xai":
+            return createXaiRealtimeLlm(cfg);
+        default:
+            throw new Error('The realtime backend must be "gemini" or "xai"');
+    }
+}

package/dist/voice/livekit/realtimeVoiceAgent.d.ts

@@ -1,21 +1,25 @@
 /**
- * LiveKit Agents agent definition — realtime (Gemini Live speech-to-speech).
+ * LiveKit Agents agent definition — realtime (speech-to-speech).
  *
  * `defineRealtimeVoiceAgent` returns the agent object placed as the default
  * export of a worker entry file. Unlike the cascaded `defineVoiceAgent`
  * (Silero VAD → STT → NeuroLink → TTS), here a single realtime model (Gemini
- * Live on Vertex) does STT + reasoning + TTS + turn detection over one
+ * Live on Vertex, or xAI Grok Voice) does STT + reasoning + TTS + turn
+ * detection over one connection. The backend is chosen by `VOICE_REALTIME_BACKEND`
+ * or the `backend` option; Gemini Live is the default.
  *
- * `@livekit/agents`, `@livekit/agents-plugin-google`, `@livekit/rtc-node`, and
- * `@google/genai` are imported dynamically so the core package does not require
- * them unless the realtime agent is used. Type-only imports are erased at build.
+ * `@livekit/agents`, the backend's LiveKit plugin (see `realtimeModel.ts`),
+ * `@livekit/rtc-node`, and `@google/genai` are imported dynamically so the core
+ * package does not require them unless the realtime agent is used. Type-only
+ * imports are erased at build.
  *
  * See docs/features/livekit-voice-agent.md.
  */
 import type { JobContext } from "@livekit/agents";
 import type { RealtimeVoiceAgentConfig } from "../../types/index.js";
 /**
- * Define a realtime (Gemini Live S2S) LiveKit voice agent.
+ * Define a realtime (speech-to-speech) LiveKit voice agent: Gemini Live by
+ * default, xAI Grok Voice with `VOICE_REALTIME_BACKEND=xai` or `backend: "xai"`.
  *
  * Place the result as the default export of the worker entry file and launch it
  * with `startRealtimeVoiceAgentWorker`. With no `config` everything is resolved

package/dist/voice/livekit/realtimeVoiceAgent.js

@@ -1,14 +1,17 @@
 /**
- * LiveKit Agents agent definition — realtime (Gemini Live speech-to-speech).
+ * LiveKit Agents agent definition — realtime (speech-to-speech).
  *
  * `defineRealtimeVoiceAgent` returns the agent object placed as the default
  * export of a worker entry file. Unlike the cascaded `defineVoiceAgent`
  * (Silero VAD → STT → NeuroLink → TTS), here a single realtime model (Gemini
- * Live on Vertex) does STT + reasoning + TTS + turn detection over one
+ * Live on Vertex, or xAI Grok Voice) does STT + reasoning + TTS + turn
+ * detection over one connection. The backend is chosen by `VOICE_REALTIME_BACKEND`
+ * or the `backend` option; Gemini Live is the default.
  *
- * `@livekit/agents`, `@livekit/agents-plugin-google`, `@livekit/rtc-node`, and
- * `@google/genai` are imported dynamically so the core package does not require
- * them unless the realtime agent is used. Type-only imports are erased at build.
+ * `@livekit/agents`, the backend's LiveKit plugin (see `realtimeModel.ts`),
+ * `@livekit/rtc-node`, and `@google/genai` are imported dynamically so the core
+ * package does not require them unless the realtime agent is used. Type-only
+ * imports are erased at build.
  *
  * See docs/features/livekit-voice-agent.md.
  */
@@ -16,6 +19,7 @@ import { z } from "zod";
 import { logger } from "../../utils/logger.js";
 import { resolveRealtimeVoiceConfig } from "./config.js";
 import { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
+import { createRealtimeLlm, requireXaiApiKey } from "./realtimeModel.js";
 import { readCallContextFromRoom } from "./roomContext.js";
 import { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
 import { buildRealtimeMcpTools } from "./realtimeMcpTools.js";
@@ -119,7 +123,10 @@ async function installRealtimeJobLifecycle(ctx, cfg) {
 }
 /** Merge caller overrides over the env-resolved realtime config. */
 function resolveConfig(overrides) {
-    const base = resolveRealtimeVoiceConfig();
+    const base = resolveRealtimeVoiceConfig({
+        backend: overrides.backend,
+        xaiBaseURL: overrides.xaiBaseURL,
+    });
     return {
         ...base,
         project: overrides.project ?? base.project,
@@ -134,7 +141,8 @@ function resolveConfig(overrides) {
     };
 }
 /**
- * Define a realtime (Gemini Live S2S) LiveKit voice agent.
+ * Define a realtime (speech-to-speech) LiveKit voice agent: Gemini Live by
+ * default, xAI Grok Voice with `VOICE_REALTIME_BACKEND=xai` or `backend: "xai"`.
  *
  * Place the result as the default export of the worker entry file and launch it
  * with `startRealtimeVoiceAgentWorker`. With no `config` everything is resolved
@@ -147,16 +155,26 @@ export function defineRealtimeVoiceAgent(config = {}) {
     const onLog = config.onLog;
     const memoryConfig = config.memory;
     async function entry(ctx) {
-        // The Gemini Live WS authenticates to Vertex via ADC; materialise it and
-        // force ADC (not an API key) auth before any realtime connection.
-        ensureVertexAdc();
-        clearGeminiApiKeyEnv();
+        const isGemini = cfg.backend === "gemini";
+        if (isGemini) {
+            // The Gemini Live WS authenticates to Vertex via ADC; materialise it and
+            // force ADC (not an API key) auth before any realtime connection.
+            ensureVertexAdc();
+            clearGeminiApiKeyEnv();
+        }
+        else {
+            requireXaiApiKey();
+        }
         logger.info("realtime.config", {
+            backend: cfg.backend,
             model: cfg.model,
-            location: cfg.location,
-            project: cfg.project ?? null,
+            ...(isGemini
+                ? { location: cfg.location, project: cfg.project ?? null }
+                : {}),
             voice: cfg.voice ?? null,
-            modality: cfg.responseModality,
+            ...(isGemini
+                ? { modality: cfg.responseModality }
+                : { customBaseURL: cfg.xaiBaseURL !== undefined }),
             toolsEnabled: cfg.toolsEnabled,
             mcpUrl: cfg.mcpUrl,
         });
@@ -190,17 +208,20 @@ export function defineRealtimeVoiceAgent(config = {}) {
         }
         logger.info("realtime.room.joined", {
             room: ctx.room.name ?? "unknown",
+            backend: cfg.backend,
             model: cfg.model,
-            location: cfg.location,
+            ...(isGemini ? { location: cfg.location } : {}),
             ms: sinceConnect(),
         });
         // Decide whether this call may be remembered BEFORE the lifecycle installs
         // its process reaper, so the shutdown flush is registered first.
-        const memory = await setUpRealtimeCallMemory(ctx, memoryConfig, cfg.model);
+        const memory = await setUpRealtimeCallMemory(ctx, memoryConfig, {
+            model: cfg.model,
+            backend: cfg.backend,
+        });
         await installRealtimeJobLifecycle(ctx, cfg);
         const { voice } = await import("@livekit/agents");
-        const google = await import("@livekit/agents-plugin-google");
-        const { Modality } = await import("@google/genai");
+        const realtimeLlm = await createRealtimeLlm(cfg);
         const bridge = await attachRealtimeEventBridge({
             room: ctx.room,
             hitlTimeoutMs: cfg.hitlTimeoutMs,
@@ -254,28 +275,10 @@ export function defineRealtimeVoiceAgent(config = {}) {
                 }
             }
         }
-        const modality = Object.values(Modality).find((value) => value === cfg.responseModality);
-        const modelOptions = {
-            vertexai: true,
-            model: cfg.model,
-            // Emit text transcripts of BOTH sides; LiveKit forwards them to the room.
-            inputAudioTranscription: {},
-            outputAudioTranscription: {},
-        };
-        if (modality !== undefined) {
-            modelOptions.modalities = [modality];
-        }
-        if (cfg.project) {
-            modelOptions.project = cfg.project;
-        }
-        if (cfg.location) {
-            modelOptions.location = cfg.location;
-        }
-        if (cfg.voice) {
-            modelOptions.voice = cfg.voice;
-        }
         const session = new voice.AgentSession({
-            llm: new google.realtime.RealtimeModel(modelOptions),
+            // `RealtimeLlm` is declared structurally so the published declarations
+            // name no optional plugin; both backends return a `RealtimeModel`.
+            llm: realtimeLlm,
         });
         // The conversation so far, so a call that appends to an open conversation
         // starts knowing it. Empty (and omitted) when there is nothing to seed.

package/dist/voice/livekit/schemaSanitizer.d.ts

@@ -9,6 +9,8 @@
  *
  * Pure, dependency-free, and assertion-free — values arrive as `unknown` and are
  * narrowed with the `isRecord` guard. Safe to reuse for any Gemini tool path.
+ * The xai realtime backend uses it too: the output is plain JSON Schema, and
+ * keeping it avoids sending xAI `$ref` or union shapes it has not been tried with.
  */
 /**
  * Rebuild a JSON Schema node into the Gemini-safe subset. Returns a fresh object

package/dist/voice/livekit/schemaSanitizer.js

@@ -9,6 +9,8 @@
  *
  * Pure, dependency-free, and assertion-free — values arrive as `unknown` and are
  * narrowed with the `isRecord` guard. Safe to reuse for any Gemini tool path.
+ * The xai realtime backend uses it too: the output is plain JSON Schema, and
+ * keeping it avoids sending xAI `$ref` or union shapes it has not been tried with.
  */
 const GEMINI_TYPES = new Set([
     "string",