@juspay/neurolink 13.6.1 → 13.6.2

package/dist/providers/catalog/provider-catalog.schema.json

@@ -311,7 +311,13 @@
                     "description": "The rule also needs the error message to match this case-insensitive regular expression (first 2,000 characters, key removed). Bounded: outside a character class no unescaped *, + or {n,}, no quantifier on a group, no count above 255, and the counts side by side multiply to at most 4,096 ways (each ? is 2, each {n,m} is m-n+1)."
                   },
                   "kind": { "$ref": "#/definitions/decisionErrorKind" },
-                  "retryable": { "type": "boolean" }
+                  "retryable": { "type": "boolean" },
+                  "hint": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 300,
+                    "description": "Text added after the error message, behind a dash, when this rule matches. For a status the vendor uses for more than one cause. {apiKeyEnvVar} is the only placeholder."
+                  }
                 }
               }
             },

package/dist/providers/catalog/scaledown.json

@@ -24,6 +24,7 @@
             }
         }
     },
+    "timeouts": { "decideMs": 30000 },
     "decide": {
         "dialect": "system-one",
         "order": 8,
@@ -35,14 +36,27 @@
         "errors": {
             "rules": [
                 { "status": 401, "kind": "authentication" },
-                { "status": [402, 422], "kind": "invalid_request" },
+                {
+                    "status": 402,
+                    "kind": "invalid_request",
+                    "hint": "ScaleDown answers HTTP 402 both when the account has no credit left and when it does not recognise the API key; check {apiKeyEnvVar} first."
+                },
+                { "status": 422, "kind": "invalid_request" },
                 { "status": 429, "kind": "rate_limit" },
                 { "status": 502, "kind": "server" },
                 { "status": 504, "kind": "timeout" },
                 { "statusMin": 500, "kind": "server" }
             ],
             "default": "invalid_request",
-            "bodyShapes": [{ "path": "detail.message" }, { "path": "detail" }]
+            "bodyShapes": [
+                { "path": "detail.message" },
+                { "path": "detail" },
+                {
+                    "path": "detail",
+                    "isArray": true,
+                    "formatter": "fastapi-validation"
+                }
+            ]
         }
     },
     "setup": {
@@ -54,7 +68,7 @@
             "1. Create an account and an API key, following https://docs.scaledown.ai",
             "2. Set {apiKeyEnvVar} in your .env file",
             "3. ScaleDown's documentation says the key goes in an x-api-key header rather than Authorization: Bearer, and NeuroLink sends it that way",
-            "4. ScaleDown's documentation says a request made with no credit left returns HTTP 402"
+            "4. ScaleDown's documentation says a request made with no credit left returns HTTP 402; a live test on 2026-10-20 saw it return 402 for an unrecognised key as well, so check {apiKeyEnvVar} before adding credit"
         ]
     },
     "evidence": {

package/dist/providers/catalog/schema.js

@@ -482,6 +482,13 @@ const decideMessagePatternSchema = z
     .refine((pattern) => !hasRiskyQuantifier(pattern), {
     message: "messagePattern must be bounded: an unescaped * or +, an open-ended {n,}, a {n,m} count above 255, or any quantifier on a group, outside a character class, can backtrack catastrophically on an error message. Put ?, or {n,m} up to 255, only on a plain character, a character class or an escape, and keep the counts side by side to 4096 ways in all (each ? is 2, each {n,m} is m-n+1)",
 });
+const decideErrorHintSchema = z
+    .string()
+    .min(1)
+    .max(300)
+    .refine((hint) => !/[{}]/.test(hint.split("{apiKeyEnvVar}").join("")), {
+    message: "hint may only use the {apiKeyEnvVar} placeholder",
+});
 const decideErrorRuleSchema = z.strictObject({
     status: z
         .union([decideStatusSchema, z.array(decideStatusSchema).min(1)])
@@ -490,6 +497,7 @@ const decideErrorRuleSchema = z.strictObject({
     messagePattern: decideMessagePatternSchema.optional(),
     kind: decideErrorKindSchema,
     retryable: z.boolean().optional(),
+    hint: decideErrorHintSchema.optional(),
 });
 const UNSAFE_PATH_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]);
 const HTTP_TOKEN_PATTERN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/;

package/dist/providers/configuredDecision.js

@@ -136,6 +136,7 @@ export class ConfiguredDecisionProvider extends SystemOneDecisionProvider {
             requestId,
             label: this.entry.displayName,
             apiKey: this.apiKey,
+            apiKeyEnvVar: catalogEnvVar(this.entry, "apiKey"),
         });
     }
     readRequestId(headers) {

package/dist/providers/decide/errors.d.ts

@@ -5,4 +5,5 @@ export declare function classifyDecisionError(spec: DecideErrors, input: {
     requestId: string | undefined;
     label: string;
     apiKey: string;
+    apiKeyEnvVar?: string;
 }): DecisionError;

package/dist/providers/decide/errors.js

@@ -52,7 +52,7 @@ function firstShapeText(shapes, payload) {
     return undefined;
 }
 export function classifyDecisionError(spec, input) {
-    const { status, payload, requestId, label, apiKey } = input;
+    const { status, payload, requestId, label, apiKey, apiKeyEnvVar } = input;
     // Redacted once, here: it stands in for a shape's text that redaction
     // empties, and the label is catalog data a key can match. Redaction is not
     // idempotent for a key such as "e" (it matches inside "[redacted]"), so the
@@ -65,9 +65,16 @@ export function classifyDecisionError(spec, input) {
     const rule = spec.rules.find((candidate) => ruleMatches(candidate, status, message));
     const ruleKind = rule?.kind ?? spec.default;
     const ruleRetryable = rule?.retryable ?? isRetryableByDefault(ruleKind);
+    // Added after the rule is chosen, so a pattern never has to allow for it,
+    // and redacted on its own, once, for the same reason as the fallback above.
+    const hinted = rule?.hint
+        ? `${message} — ${redactCredentials(rule.hint
+            .split("{apiKeyEnvVar}")
+            .join(apiKeyEnvVar ?? "the API key variable"), apiKey)}`
+        : message;
     const build = (kind, retryable) => ({
         kind,
-        message,
+        message: hinted,
         status,
         requestId,
         retryable,

package/dist/types/generate.d.ts

@@ -1838,7 +1838,12 @@ export type NativeGenerateLoopArgs = {
     conversation: Array<Record<string, unknown>>;
     /** Tool declarations in the v3 shape doGenerate already knows how to convert. */
     tools?: Array<Record<string, unknown>>;
-    /** Registered tools, used to execute a call the model asks for. */
+    /**
+     * Registered tools, used to execute a call the model asks for. Typed
+     * `Record<string, Tool>`; it was `Record<string, unknown>` before discovery
+     * learned to declare tools that join it mid-turn, so a caller that builds
+     * this record from loosely typed tools now needs a cast to `Tool`.
+     */
     toolsRecord: Record<string, Tool>;
     /**
      * Builds the v3 declaration for one tool. When set, every step declares any

package/dist/types/livekit.d.ts

@@ -9,6 +9,7 @@
  *
  * See docs/features/livekit-voice-agent.md.
  */
+import type { EventEmitter } from "node:events";
 import type { NeuroLinkEvents, TypedEventEmitter } from "./common.js";
 import type { StreamOptions, StreamResult } from "./stream.js";
 import type { ConversationSeedMessage, ConversationSeedOptions, RecordConversationTurnOptions, RecordConversationTurnResult, StreamEventSequence } from "./conversation.js";
@@ -437,6 +438,24 @@ export type RealtimeLlm = {
     label: () => string;
     close: () => Promise<void>;
 };
+/**
+ * A session of the xAI plugin as far as the xai model's watcher needs it: an
+ * event emitter (the plugin republishes every server event as
+ * `openai_server_event_received`) that can be closed.
+ */
+export type XaiObservedSession = EventEmitter & {
+    close: () => Promise<void>;
+};
+/**
+ * Token counts of one xAI response. xAI sends them at the top level of
+ * `response.done`; the plugin reads `response.usage`, which xAI leaves empty.
+ */
+export type XaiResponseUsage = {
+    inputTokens: number;
+    inputAudioTokens: number;
+    outputTokens: number;
+    outputAudioTokens: number;
+};
 /**
  * Realtime voice configuration resolved from the environment.
  *

package/dist/types/providerCatalog.d.ts

@@ -290,6 +290,14 @@ export type DecideErrorRule = {
     messagePattern?: string;
     kind: DecisionErrorKind;
     retryable?: boolean;
+    /**
+     * Text added after the error message, behind a dash, when this rule is the
+     * one that matches. For a status the vendor uses for more than one cause,
+     * where the body's own words name only one of them. `{apiKeyEnvVar}` is the
+     * only placeholder. A `messagePattern` is judged on the message without it,
+     * and the key is removed from it as from the message.
+     */
+    hint?: string;
 };
 export type DecideBodyShape = {
     /** Dot path into the parsed error body. */

package/dist/voice/livekit/config.js

@@ -68,8 +68,9 @@ export function resolveBrainDefaults() {
     };
 }
 const DEFAULT_REALTIME_MODEL = "gemini-live-2.5-flash";
-// xAI's documented flagship Grok Voice id. Never exercised against the real
-// service; the plugin's own default is `grok-voice-think-fast-1.0`.
+// xAI's documented flagship Grok Voice id, accepted and honoured by the live
+// service (2026-10-20). The plugin's own default, `grok-voice-think-fast-1.0`,
+// ran as `grok-voice-3.0` there.
 const DEFAULT_XAI_REALTIME_MODEL = "grok-voice-3.0";
 const DEFAULT_REALTIME_LOCATION = "global";
 const DEFAULT_RESPONSE_MODALITY = "AUDIO";

package/dist/voice/livekit/realtimeModel.d.ts

@@ -8,8 +8,8 @@
  * `@google/genai` are imported dynamically so the core package does not require
  * them unless the realtime agent is used.
  *
- * The xai backend has only been exercised against a stub WebSocket server. It
- * has never talked to the real xAI service.
+ * The xai backend was run against the real xAI service on 2026-10-20; what that
+ * showed, and what is still unmeasured, is in the document below.
  *
  * See docs/features/livekit-voice-agent.md.
  */

package/dist/voice/livekit/realtimeModel.js

@@ -8,12 +8,13 @@
  * `@google/genai` are imported dynamically so the core package does not require
  * them unless the realtime agent is used.
  *
- * The xai backend has only been exercised against a stub WebSocket server. It
- * has never talked to the real xAI service.
+ * The xai backend was run against the real xAI service on 2026-10-20; what that
+ * showed, and what is still unmeasured, is in the document below.
  *
  * See docs/features/livekit-voice-agent.md.
  */
 import { requireWebSocketUrl } from "./config.js";
+import { observeXaiSession } from "./xaiSessionWatch.js";
 /**
  * Fail early, naming the variable, when `XAI_API_KEY` is blank or absent.
  *
@@ -66,16 +67,34 @@ async function createXaiRealtimeLlm(cfg) {
     // own defaults, so a key holding `undefined` would erase the default: a
     // missing `baseURL` would send the xAI key to OPENAI_BASE_URL or
     // api.openai.com, and no `apiKey` option is passed at all because the plugin
-    // reads XAI_API_KEY itself. The plugin forwards `baseURL` although its
-    // declared option type leaves it out.
-    const modelOptions = { model: cfg.model };
+    // reads XAI_API_KEY itself. The plugin forwards `baseURL` and
+    // `inputAudioTranscription` although its declared option type leaves them out.
+    //
+    // `inputAudioTranscription` is the one key set to `undefined` on purpose, to
+    // erase the plugin's OpenAI default of `gpt-4o-mini-transcribe`. With it xAI
+    // answered one spoken utterance with five "completed" transcripts, which the
+    // turn recorder stored as one user message with the utterance repeated;
+    // without it xAI sends one. `undefined`, unlike `null`, keeps the plugin's
+    // user-transcription capability on, and JSON.stringify leaves the key out of
+    // the `session.update` it sends.
+    const modelOptions = {
+        model: cfg.model,
+        inputAudioTranscription: undefined,
+    };
     if (cfg.voice) {
         modelOptions.voice = cfg.voice;
     }
     if (cfg.xaiBaseURL) {
         modelOptions.baseURL = cfg.xaiBaseURL;
     }
-    return new xai.realtime.RealtimeModel(modelOptions);
+    class ObservedXaiRealtimeModel extends xai.realtime.RealtimeModel {
+        session() {
+            const session = super.session();
+            observeXaiSession(session, cfg.model);
+            return session;
+        }
+    }
+    return new ObservedXaiRealtimeModel(modelOptions);
 }
 /**
  * Build the realtime model for `cfg.backend`.

package/dist/voice/livekit/realtimeVoiceAgent.js

@@ -343,7 +343,9 @@ export function defineRealtimeVoiceAgent(config = {}) {
         };
         session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
             const metrics = ev.metrics;
-            if (metrics.type !== "realtime_model_metrics") {
+            // xAI leaves `response.usage` empty, which is where the plugin reads, so
+            // its metrics are zero; the xai model logs `realtime.usage` itself.
+            if (!isGemini || metrics.type !== "realtime_model_metrics") {
                 return;
             }
             const inAudio = metrics.inputTokenDetails.audioTokens;

package/dist/voice/livekit/xaiSessionWatch.d.ts

@@ -0,0 +1,26 @@
+/**
+ * What this package adds around an xAI realtime session.
+ *
+ * The LiveKit xAI plugin is the OpenAI realtime plugin pointed at xAI, and the
+ * live service differs from OpenAI in three ways the plugin does not cover
+ * (measured 2026-10-20, see docs/features/livekit-voice-agent.md):
+ *
+ *  - xAI sends usage at the top level of `response.done` and leaves
+ *    `response.usage` an empty object, which is what the plugin reads, so its
+ *    metrics are zero for every response;
+ *  - an unknown model id is accepted and runs a default model, with no error;
+ *  - a wrong key is refused with HTTP 400 at the WebSocket upgrade, and the
+ *    plugin opens its socket without an 'error' listener, so the refusal is an
+ *    uncaught exception.
+ *
+ * `observeXaiSession` reads the stream of server events the plugin republishes
+ * as `openai_server_event_received` for the first two, and watches for the
+ * third.
+ */
+import type { XaiObservedSession } from "../../types/index.js";
+/**
+ * Log the usage of every response, warn when the service runs another model
+ * than the one asked for, and report a refused upgrade as a session error.
+ * `requestedModel` is the id the session was configured with.
+ */
+export declare function observeXaiSession(session: XaiObservedSession, requestedModel: string): void;

package/dist/voice/livekit/xaiSessionWatch.js

@@ -0,0 +1,157 @@
+/**
+ * What this package adds around an xAI realtime session.
+ *
+ * The LiveKit xAI plugin is the OpenAI realtime plugin pointed at xAI, and the
+ * live service differs from OpenAI in three ways the plugin does not cover
+ * (measured 2026-10-20, see docs/features/livekit-voice-agent.md):
+ *
+ *  - xAI sends usage at the top level of `response.done` and leaves
+ *    `response.usage` an empty object, which is what the plugin reads, so its
+ *    metrics are zero for every response;
+ *  - an unknown model id is accepted and runs a default model, with no error;
+ *  - a wrong key is refused with HTTP 400 at the WebSocket upgrade, and the
+ *    plugin opens its socket without an 'error' listener, so the refusal is an
+ *    uncaught exception.
+ *
+ * `observeXaiSession` reads the stream of server events the plugin republishes
+ * as `openai_server_event_received` for the first two, and watches for the
+ * third.
+ */
+import { logger } from "../../utils/logger.js";
+const TOKEN_COUNT_KEYS = ["input_tokens", "output_tokens"];
+const REFUSED_UPGRADE = /^Unexpected server response: (\d{3})$/;
+const KEY_REFUSAL_STATUSES = new Set([400, 401, 403]);
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function countOf(source, key) {
+    const value = source[key];
+    return typeof value === "number" && Number.isFinite(value) && value > 0
+        ? value
+        : 0;
+}
+function detailOf(source, group, key) {
+    const details = source[group];
+    return isRecord(details) ? countOf(details, key) : 0;
+}
+/**
+ * The token counts of a `response.done` event, or `undefined` when it reports
+ * none. xAI's own place is the top level of the event; `response.usage`, where
+ * OpenAI puts them, is the fallback, so a service that fills it still works.
+ * A report with seconds but no token count (a cancelled response) is not usage
+ * and would log as zeros.
+ */
+function readXaiResponseUsage(event) {
+    if (!isRecord(event) || event.type !== "response.done") {
+        return undefined;
+    }
+    const nested = isRecord(event.response) ? event.response.usage : undefined;
+    const usage = [event.usage, nested].find((candidate) => isRecord(candidate) &&
+        TOKEN_COUNT_KEYS.some((key) => typeof candidate[key] === "number"));
+    if (usage === undefined) {
+        return undefined;
+    }
+    return {
+        inputTokens: countOf(usage, "input_tokens"),
+        inputAudioTokens: detailOf(usage, "input_token_details", "audio_tokens"),
+        outputTokens: countOf(usage, "output_tokens"),
+        outputAudioTokens: detailOf(usage, "output_token_details", "audio_tokens"),
+    };
+}
+/** The same record the agent logs for the Gemini backend, built from the same three counts. */
+function logUsage(usage) {
+    logger.info("realtime.usage", {
+        inputTokens: usage.inputTokens,
+        inputAudioTokens: usage.inputAudioTokens,
+        inputTextTokens: Math.max(0, usage.inputTokens - usage.inputAudioTokens),
+        outputTokens: usage.outputTokens,
+        outputAudioTokens: usage.outputAudioTokens,
+        outputTextTokens: Math.max(0, usage.outputTokens - usage.outputAudioTokens),
+    });
+}
+function describeRefusal(status) {
+    const keyHint = KEY_REFUSAL_STATUSES.has(status)
+        ? " xAI answers a wrong XAI_API_KEY with HTTP 400, not 401, so check the key first."
+        : "";
+    return `xAI refused the realtime connection with HTTP ${status}.${keyHint}`;
+}
+/**
+ * Turn the plugin's uncaught socket error into the error event the plugin
+ * meant to emit. Returns the function that ends the watch.
+ *
+ * `uncaughtExceptionMonitor` observes a crash without handling it, so the
+ * process still fails exactly as it would have; what changes is that the
+ * session reports a classified, unrecoverable error first, which the agent
+ * entry already logs and publishes. The watch is on only while the connection
+ * is being made, so it cannot take a crash elsewhere in the process for a
+ * refused upgrade once the call is running.
+ *
+ * A crash event carries no socket, so a refusal cannot be tied to one session:
+ * every session still connecting is told. That is accurate, because the crash
+ * ends the process for all of them, and the sessions of one process share
+ * `XAI_API_KEY`, so a refused key is refused for each.
+ */
+function watchForRefusedUpgrade(session) {
+    const onCrash = (error) => {
+        const status = error instanceof Error
+            ? Number(REFUSED_UPGRADE.exec(error.message)?.[1])
+            : Number.NaN;
+        if (Number.isNaN(status)) {
+            return;
+        }
+        stop();
+        const message = describeRefusal(status);
+        logger.error("realtime.xai.connectRefused", { status, error: message });
+        if (session.listenerCount("error") === 0) {
+            return;
+        }
+        try {
+            session.emit("error", {
+                type: "realtime_model_error",
+                timestamp: Date.now(),
+                label: "xai.RealtimeModel",
+                error: new Error(message),
+                recoverable: false,
+            });
+        }
+        catch {
+            /* the process is already failing; a listener's error must not mask it */
+        }
+    };
+    const stop = () => {
+        process.off("uncaughtExceptionMonitor", onCrash);
+    };
+    process.on("uncaughtExceptionMonitor", onCrash);
+    return stop;
+}
+/**
+ * Log the usage of every response, warn when the service runs another model
+ * than the one asked for, and report a refused upgrade as a session error.
+ * `requestedModel` is the id the session was configured with.
+ */
+export function observeXaiSession(session, requestedModel) {
+    const stopWatching = watchForRefusedUpgrade(session);
+    session.on("openai_server_event_received", (event) => {
+        if (isRecord(event) && event.type === "session.created") {
+            stopWatching();
+            const served = isRecord(event.session) ? event.session.model : undefined;
+            if (typeof served === "string" && served !== requestedModel) {
+                logger.warn("realtime.xai.modelMismatch", {
+                    requested: requestedModel,
+                    served,
+                    note: "xAI runs a default model for an id it does not know, without an error; an alias comes back as the model it points to",
+                });
+            }
+            return;
+        }
+        const usage = readXaiResponseUsage(event);
+        if (usage !== undefined) {
+            logUsage(usage);
+        }
+    });
+    const close = session.close.bind(session);
+    session.close = async () => {
+        stopWatching();
+        await close();
+    };
+}

package/docs-site/static/search-index.json

@@ -3149,13 +3149,13 @@
 {"objectID":"5589375c3b5b8ee71271f32386ef6b0f878bc88f7097a13ee7d8ef03b694655e","title":"Conversation Memory","url":"/docs/features/livekit-voice-agent#conversation-memory","content":"The agent remembers earlier turns automatically when the NeuroLink instance you\nbuild inside createNeuroLink has conversation memory enabled. History is the\nagent's source of truth — LiveKit's own transcript context is not used for\ngeneration.\n\nHow it behaves:\nKeyed per call. Each room/call is an isolated conversation; the id is\n  derived from the room name. Override the prefix with conversationIdPrefix\n  (default \"voice\").\nIn-memory by default; Redis for persistence. Set REDIS_URL to use a\n  shared store that survives worker restarts and is shared across worker\n  replicas — important because each call runs in its own job process.\nWorks across turns within the session. The user can say \"my name is Alex\"\n  and later ask \"what's my name?\" and the agent recalls it.\n\nMemory persists only when the instance is configured with\nconversationMemory.enabled. Without it, each turn is independent.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Conversation Memory","lvl3":""}},
 {"objectID":"e6ef692117fe2ad968f2bb5bd1692e50783a069a189d3dbfd6ac49f71ef84c73","title":"Remembering a realtime call in the user's own conversation","url":"/docs/features/livekit-voice-agent#remembering-a-realtime-call-in-the-users-own-conversation","content":"A speech-to-speech call (defineRealtimeVoiceAgent) never goes through\nstream() or generate(), so by default nothing it says is stored. Opt in with\nmemory and the call's turns are written, through\nNeuroLink.recordConversationTurn, to the same conversation memory typed chat\nuses, appended to the conversation the person already has open. The model is\nalso seeded with that conversation's recent history when the call starts, so it\nknows what was said before.\n\nWhose conversation is never taken from the client. The host that creates\nthe room signs a binding and puts it in the room metadata next to authToken\nand mcpContext. The worker stores nothing unless it verifies.\nThe signature is HMAC-SHA256 over [\"neurolink.voice.memory.v1\", room,\nuserId, sessionId, exp] (a JSON array, so no choice of values can make two\n  bindings collide), base64url-encoded. It covers the room name, so a binding\n  copied into another room does not verify. exp is Unix seconds; the default\n  lifetime is two hours, the cap 24 hours.\nThe shared secret is at least 16 characters, read from\n  NEUROLINK_VOICE_MEMORY_SECRET unless memory.secret is set on the worker.\nsessionId matches ^[A-Za-z0-9_-]{1,128}$ (no :, which is part of the\n  stored key). userId is 1 to 256 characters with no whitespace or control\n  characters; : is allowed because chat user ids contain it.\nFail closed. No secret, no binding, a malformed or expired binding, or a\n  bad signature: the call runs normally, nothing is stored, and a warning\n  names the reason (never the ids).\nA client may name the conversation to append to. The host accepts the name\n  only when neurolink.conversationExists({ sessionId, userId }) is true for\n  the verified user. Otherwise it mints a new voice-<uuid>.\n\nWhat is stored. Each finished exchange becomes a user message and an\nassistant message with the ids ${turnId}:user and ${turnId}:assistant, where\nturnId is the realtime session's assistant message id, and\nmetadata.modality: \"voice\".","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Remembering a realtime call in the user's own conversation","lvl3":""}},
 {"objectID":"2b5f572d39aad7e08324d267d06f5dcbc4e90af9c0c31f1d536c84cc4243c5c1","title":"Realtime Speech-to-Speech Backends (Gemini Live and xAI Grok Voice)","url":"/docs/features/livekit-voice-agent#realtime-speech-to-speech-backends-gemini-live-and-xai-grok-voice","content":"defineRealtimeVoiceAgent runs one realtime model that does speech-to-text,\nreasoning, text-to-speech and turn detection over a single connection. Gemini\nLive on Vertex is the default. xAI's Grok Voice is a second backend, reached\nthrough LiveKit's xAI plugin.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Realtime Speech-to-Speech Backends (Gemini Live and xAI Grok Voice)","lvl3":""}},
-{"objectID":"3ea0de3512723dda2eba65763b09a6e3eff2ded8a4d0aa2fa41581d46b88e6c7","title":"Status","url":"/docs/features/livekit-voice-agent#status","content":"Tested against a stub WebSocket server only. The stub listens on\n  127.0.0.1. It shows what the LiveKit xAI plugin sends to whatever it is\n  pointed at, and how this package reads the OpenAI-style events it is given.\n  It shows nothing about how xAI answers.\nNo request has been made to xAI by this project. No xAI key was available\n  when the backend was written.\nNot established here: behaviour against the live service, latency,\n  accuracy, and every xAI benchmark. Nothing in this section says the xai\n  backend works with the real service. Treat it as untested against xAI until\n  someone runs a call with a real key.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Status","lvl3":""}},
+{"objectID":"3ea0de3512723dda2eba65763b09a6e3eff2ded8a4d0aa2fa41581d46b88e6c7","title":"Status","url":"/docs/features/livekit-voice-agent#status","content":"Run against the live xAI service on 2026-10-20 with synthetic speech (a\ntext-to-speech voice, not a person), inside a recorded budget: 21 WebSocket\nsessions, about 95 seconds connected, under US$0.20. Plugin version 1.4.6. What\nthat showed:\nIt works through the plugin. A greeting, a text turn, a spoken turn\n  answered in voice, a function call with its continuation, a seeded history and\n  two voices all completed with no error event.\ngrok-voice-3.0 is accepted and honoured: session.created names it.\nFirst audio arrived 985 ms after a text turn (counted from\n  response.create) and 1,229 ms after a spoken turn (counted from\n  input_audio_buffer.speech_stopped). One sample each, from one network\n  location, so neither is a distribution.\nPrice: xAI's own pages list USD 0.08 per minute. That is a list price; no\n  invoice was checked.\nFunction calling works for one function tool with a flat object schema.\n  Tool names with underscores and the $schema and additionalProperties\n  fields the sanitizer leaves in were accepted.\nThe plugin's session.update layout is accepted and its voice and\n  turn-detection settings take effect. The service's echo of the turn detection\n  leaves out threshold and interrupt_response; whether it ignores them was\n  not measured.\nSeeded history is accepted: a user and an assistant message sent before the\n  first turn were used by the model.\n\nThe same run found the differences from OpenAI that the sections below handle:\nrepeated final transcripts, usage reported in another place, an unknown model or\nvoice accepted without an error, and a wrong key that crashes the plugin's\nsocket. It did not cover a full agent in a LiveKit room, interruption, or\ncalls longer than a few seconds; see Still unmeasured.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Status","lvl3":""}},
 {"objectID":"6676179c44d375115fe98161f9377ba40357b82eebf4a99b7b330ede9721e8c3","title":"Choosing the backend","url":"/docs/features/livekit-voice-agent#choosing-the-backend","content":"| Variable                                                                          | Backend | Meaning                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |\n| --------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| VOICE_REALTIME_BACKEND                                                          | both    | gemini or xai. Any other value throws at startup, naming the variable and the two choices and repeating nothing from the environment.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Choosing the backend","lvl3":""}},
 {"objectID":"632f3752058fa7c7c6d48b02af1e8d5212b7daa55001e8cc931227404009278f","title":"Dependencies","url":"/docs/features/livekit-voice-agent#dependencies","content":"The xai backend needs @livekit/agents-plugin-xai and\n@livekit/agents-plugin-openai. Both are optional dependencies of this package,\nso they are installed with its other optional dependencies (an install that\nomits optional dependencies has to add them by hand), and both resolved to\n1.4.6 in its lockfile. The xAI plugin asks for @livekit/agents 1.4.6 exactly,\nwhich is why that package moved from 1.4.5 to 1.4.6 with it. The options the\nGemini backend builds are unchanged by the move, but no live Gemini call was\nrun after it.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Dependencies","lvl3":""}},
-{"objectID":"a890643483526cae888fdc083535c83d5313ddef29c4a24f6d2771e94dcf3f71","title":"What the xai backend does","url":"/docs/features/livekit-voice-agent#what-the-xai-backend-does","content":"The LiveKit xAI plugin extends the OpenAI realtime model. Against a stub it\nconnects to <base URL>/realtime?model=<id> with Authorization: Bearer\n<XAI_API_KEY> (the key is never put in the URL) and its first event is a\nsession.update that carries the model, the voice, output_modalities:\n[\"audio\"], server-side voice-activity detection (threshold 0.5, 300 ms prefix\npadding, 200 ms silence, response creation and interruption on) and an input\ntranscription setting. The plugin's default endpoint is wss://api.x.ai/v1.\n\nTwo things in that list are the plugin's and are not verified against xAI:\nthe plugin sends gpt-4o-mini-transcribe, an OpenAI model name, as the input\n  transcription model;\nthe plugin's own default model is grok-voice-think-fast-1.0. NeuroLink\n  defaults to grok-voice-3.0, the flagship id xAI's documentation lists\n  (together with the alias grok-voice-latest and a deprecated\n  grok-voice-2.0). The plugin passes the id through; whether the service\n  accepts it through the plugin has not been tested.\n\nA missing key stops the call before it connects. If XAI_API_KEY is unset\nthe plugin falls back to OPENAI_API_KEY, and a stub server showed that\nfallback sending the OpenAI key as the Bearer token to the configured xAI URL.\n(An empty value does not fall back: the plugin throws its own OpenAI API key is\nrequired error. A whitespace-only value is sent as the key.) NeuroLink\ntherefore refuses an unset, empty or whitespace-only key: it throws\nXAI_API_KEY is not set ... when the agent starts and again in\ncreateRealtimeLlm, and no connection is made.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"What the xai backend does","lvl3":""}},
-{"objectID":"b4b0a14667e072deb333f9771c062361a617e33da9a0142087c0ba9502d3e4f7","title":"How each part of the agent behaves on xai","url":"/docs/features/livekit-voice-agent#how-each-part-of-the-agent-behaves-on-xai","content":"| Part                                                               | On xai                                                                                                                                                                                                                            |\n| ------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Vertex ADC and GEMINI_API_KEY clearing                           | Not run. The xai backend never touches the Gemini environment.                                                                                                                                                                    |\n| Data-channel event bridge, HITL, room context                      | Unchanged. They work on the LiveKit room and the agent session, not on a model.                                                                                                                                                   |\n| Session events (user transcript, assistant text, turn done, usage) | Unchanged code. Against a stub, OpenAI-style transcript and usage events reach the shapes the agent reads.                                                                                                                        |\n| MCP tool bridge and schema sanitizer                               | Unchanged. The sanitizer produces plain JSON Schema (the OpenAI-style API takes full JSON Schema), and the plugin forwards it as is. Keeping it is the safer choice: it drops $ref and unions that xAI has not been tried with. |\n| Conversation memory (recording and seeding)                        | Recording is shared by both backends.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"How each part of the agent behaves on xai","lvl3":""}},
-{"objectID":"f2c4f00431a110b4ec6b67653e157a21d79cc3d52751253260a13758eec90e66","title":"Still unknown without a real key","url":"/docs/features/livekit-voice-agent#still-unknown-without-a-real-key","content":"Whether xAI accepts the model ids above through the plugin; whether it accepts\nthe input transcription setting; whether it accepts the sanitized tool schemas\nand underscore tool names; the event names and ordering xAI really sends (the\nplugin line pinned here, 1.4.6, predates later xAI-specific handling in newer\nplugin releases); whether the seeded history is accepted; and how interruption,\ntruncation and tool calls behave. A live smoke test with a key supplied by the\nowner is the missing step.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Still unknown without a real key","lvl3":""}},
-{"objectID":"9eb38592a55bd3da33ee8a936b4648d18645a49909579f1fccb19844c8263ac7","title":"Troubleshooting","url":"/docs/features/livekit-voice-agent#troubleshooting","content":"VOICE_REALTIME_BACKEND must be \"gemini\" or \"xai\": fix the value, or unset it\n  for gemini.\nXAI_API_KEY is not set: set it in the worker's environment. The check also\n  rejects a whitespace-only value.\nXAI_REALTIME_BASE_URL must be a ws:// or wss:// URL (or the same for\n  xaiBaseURL): use a WebSocket URL, not an https:// one.\nXAI_REALTIME_BASE_URL is a ws:// URL to a host other than localhost, ... (or\n  the same for xaiBaseURL): the URL would send XAI_API_KEY unencrypted to a\n  host that is not loopback. Use a wss:// URL. Only if the path to that host is\n  one you trust, set XAI_REALTIME_ALLOW_INSECURE=true, knowing the key then\n  travels in cleartext.\nA call that will not start on the xai backend: confirm that VOICE_LIVE_MODEL\n  and VOICE_S2S_VOICE hold xAI values and not leftover Gemini ones, since both\n  variables are shared.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Troubleshooting","lvl3":""}},
+{"objectID":"a890643483526cae888fdc083535c83d5313ddef29c4a24f6d2771e94dcf3f71","title":"What the xai backend does","url":"/docs/features/livekit-voice-agent#what-the-xai-backend-does","content":"The LiveKit xAI plugin extends the OpenAI realtime model. It connects to\n<base URL>/realtime?model=<id> with Authorization: Bearer <XAI_API_KEY> (the\nkey is never put in the URL) and its first event is a session.update that\ncarries the model, the voice, output_modalities: [\"audio\"], server-side\nvoice-activity detection (threshold 0.5, 300 ms prefix padding, 200 ms silence,\nresponse creation and interruption on). The plugin's default endpoint is\nwss://api.x.ai/v1. xAI documents a flatter layout than the plugin sends and\naccepts the plugin's anyway.\n\nFour places where the live service differs from what the plugin assumes, and\nwhat this package does about each:\nInput transcription is left out of the layout. The plugin's default is\n  { model: \"gpt-4o-mini-transcribe\" }, an OpenAI model name. xAI accepts it,\n  and then answers one spoken utterance with five completed transcripts of\n  growing text (20, 36, 58, 58 and 58 characters for one 58-character sentence)\n  plus three updated ones. Any transcription key does this, including xAI's\n  own grok-transcribe. The plugin turns each completed event into a final\n  user transcript, so the turn recorder stored 234 characters: the sentence\n  repeated. The factory therefore builds the model with\n  inputAudioTranscription: undefined, the one place it erases a plugin\n  default on purpose; the plugin then sends no transcription key and xAI sends\n  one completed transcript per utterance. Transcription itself stays on.\nUsage is in another place. xAI sends the token counts at the top level of\n  response.done and leaves response.usage, where the plugin reads, an empty\n  object, so the plugin's own metrics are zero for every response. The xai\n  model reads the top-level counts (and response.usage when the top level has\n  none) and logs realtime.usage with the same six fields the Gemini backend\n  logs. A response that reports nothing, such as a cancelled one, logs nothing.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"What the xai backend does","lvl3":""}},
+{"objectID":"b4b0a14667e072deb333f9771c062361a617e33da9a0142087c0ba9502d3e4f7","title":"How each part of the agent behaves on xai","url":"/docs/features/livekit-voice-agent#how-each-part-of-the-agent-behaves-on-xai","content":"| Part                                                               | On xai                                                                                                                                                                                                                                                                                           |\n| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |\n| Vertex ADC and GEMINI_API_KEY clearing                           | Not run. The xai backend never touches the Gemini environment.                                                                                                                                                                                                                                   |\n| Data-channel event bridge, HITL, room context                      | Unchanged. They work on the LiveKit room and the agent session, not on a model.                                                                                                                                                                                                                  |\n| Session events (user transcript, assistant text, turn done, usage) | Unchanged code, with one difference. The user transcript, assistant text and turn events reach the shapes the agent reads. Usage does not come through the plugin's metrics (they are zero on xai); the xai model logs realtime.usage itself and the agent skips the plugin's metrics for xai. |\n| MCP tool bridge and schema sanitizer                               | Unchanged.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"How each part of the agent behaves on xai","lvl3":""}},
+{"objectID":"9da227b121f2046a8dce8100205da4cb9f470302ac8a60d8f65827f64fd7d333","title":"Still unmeasured","url":"/docs/features/livekit-voice-agent#still-unmeasured","content":"A full agent inside a LiveKit room: a voice.AgentSession with the event\nbridge, human confirmations and MCP tools going through the model (the model was\ndriven directly, without a room). Interruption and truncation\n(conversation.item.truncate, response.cancel). Reconnection, the plugin's\n20-minute session cap, idle timeouts and calls longer than a few seconds.\nParallel and server-side tools, and tool schemas with $ref or unions (the\nsanitizer removes those). Latency beyond the two single samples above. Accuracy\nwith human speech and accents. The real charge in xAI's console. The results\nabove are for plugin 1.4.6, which predates later xAI-specific handling in newer\nplugin releases.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Still unmeasured","lvl3":""}},
+{"objectID":"9eb38592a55bd3da33ee8a936b4648d18645a49909579f1fccb19844c8263ac7","title":"Troubleshooting","url":"/docs/features/livekit-voice-agent#troubleshooting","content":"VOICE_REALTIME_BACKEND must be \"gemini\" or \"xai\": fix the value, or unset it\n  for gemini.\nXAI_API_KEY is not set: set it in the worker's environment. The check also\n  rejects a whitespace-only value.\nXAI_REALTIME_BASE_URL must be a ws:// or wss:// URL (or the same for\n  xaiBaseURL): use a WebSocket URL, not an https:// one.\nXAI_REALTIME_BASE_URL is a ws:// URL to a host other than localhost, ... (or\n  the same for xaiBaseURL): the URL would send XAI_API_KEY unencrypted to a\n  host that is not loopback. Use a wss:// URL. Only if the path to that host is\n  one you trust, set XAI_REALTIME_ALLOW_INSECURE=true, knowing the key then\n  travels in cleartext.\nA call that will not start on the xai backend: confirm that VOICE_LIVE_MODEL\n  and VOICE_S2S_VOICE hold xAI values and not leftover Gemini ones, since both\n  variables are shared.\nxAI refused the realtime connection with HTTP 400 (in the log as\n  realtime.xai.connectRefused), or Unexpected server response: 400 in a\n  stack trace: a wrong, revoked or expired XAI_API_KEY. xAI answers a wrong\n  key with HTTP 400 at the WebSocket upgrade, not 401. The plugin opens its\n  socket with no error listener, so the refusal is an uncaught exception, and\n  a Node process with no handler for it exits with code 1. This package cannot\n  stop that. What it does before the exception takes effect is log the\n  classified error and emit it as an unrecoverable error on the model session;\n  the LiveKit agent session forwards that to the handler the agent entry already\n  has, which logs realtime.session.error and publishes status: error (read\n  from the LiveKit source; not run inside a room). With\n  installVoiceWorkerProcessGuards() installed in the process, the exception is\n  logged and a job child exits one second later instead.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Troubleshooting","lvl3":""}},
 {"objectID":"7ffe7e32a7c2baea6c491d743617448eb70eeb4afe439960050efdc3ead18c48","title":"Implementation Plan","url":"/docs/features/livekit-voice-agent#implementation-plan","content":"The integration is built and validated in phases. Each phase is independently testable.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Implementation Plan","lvl3":""}},
 {"objectID":"d4c9885eea3272e250fa1a0dbc1af9503b1a24f0217a82dd0c30d93bf6c5df31","title":"Phase 0 — Console-mode spike (no infrastructure)","url":"/docs/features/livekit-voice-agent#phase-0-console-mode-spike-no-infrastructure","content":"Build a minimal agent (Silero VAD + Deepgram STT + ElevenLabs TTS + llmNode → neurolink.stream()) and run it in console mode using the host machine's mic/speakers. Validates the NeuroLink brain loop, conversationId history, and a tool call — with no LiveKit server and no browser. Requires only STT/TTS and LLM credentials.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Phase 0 — Console-mode spike (no infrastructure)","lvl3":""}},
 {"objectID":"9dddf98f952af643b5357bdefbcb5554a3ff2d9c33a16b0c432243fd68db5569","title":"Phase 1 — NeuroLink LiveKit module","url":"/docs/features/livekit-voice-agent#phase-1-neurolink-livekit-module","content":"Implement brain.ts, voiceAgent.ts, voiceAgentWorker.ts, tokens.ts, config.ts; add the @juspay/neurolink/livekit export and optional/peer dependencies. The worker factory accepts an external NeuroLink instance so a host application's registered tools are available. Wire abort-on-interrupt. Verify build, type-check, and lint.","hierarchy":{"lvl0":"Features","lvl1":"LiveKit Voice Agent — Real-Time Voice over WebRTC","lvl2":"Phase 1 — NeuroLink LiveKit module","lvl3":""}},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "13.6.1",
+  "version": "13.6.2",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {