@juspay/neurolink 9.72.0 → 9.74.0

package/dist/lib/types/toolRouting.d.ts

@@ -64,6 +64,30 @@ export type ToolRoutingConfig = {
      * always appended by the SDK regardless of this value.
      */
     routerPromptPrefix?: string;
+    /**
+     * LRU+TTL cache for routing decisions. When enabled, identical routing
+     * queries within the TTL window skip the router LLM entirely and reuse
+     * the cached exclusion list.
+     */
+    cache?: {
+        /** Whether the cache is active. Default: false. */
+        enabled?: boolean;
+        /** Time-to-live in milliseconds for each cached entry. Default: 60000. */
+        ttlMs?: number;
+        /** Maximum number of entries in the LRU cache. Default: 256. */
+        maxEntries?: number;
+    };
+    /**
+     * Session stickiness: once the router picks a set of servers for a session,
+     * those servers are kept warm (not excluded) for the next N turns to prevent
+     * flapping.
+     */
+    stickiness?: {
+        /** Whether session stickiness is active. Default: false. */
+        enabled?: boolean;
+        /** Number of turns for which a previously-selected server stays warm. Default: 3. */
+        turns?: number;
+    };
 };
 /** Catalog entry pairing a server descriptor with its registered tool names. */
 export type ToolRoutingCatalogEntry = {
@@ -72,6 +96,63 @@ export type ToolRoutingCatalogEntry = {
     /** Registered tool names for this server, i.e. `${serverId}_${toolName}`. */
     toolNames: string[];
 };
+/** Internal cache entry for `ToolRoutingCache`. */
+export type ToolRoutingCacheEntry = {
+    excludedToolNames: string[];
+    selectedServerIds: string[];
+    /** Absolute expiry timestamp (from the injected `now()` clock). */
+    expiresAt: number;
+    /** LRU eviction order — lower = older. Bumped on each get/set. */
+    accessOrder: number;
+};
+/** Internal stickiness entry for `ToolRoutingCache`. */
+export type ToolRoutingStickiness = {
+    serverIds: string[];
+    /** Turn counter — decremented on each routing turn, removed when it hits 0. */
+    turnsRemaining: number;
+};
+/** Constructor options for `ToolRoutingCache`. */
+export type ToolRoutingCacheOptions = {
+    /** Time-to-live in milliseconds for each cached entry. Default: 60_000. */
+    ttlMs?: number;
+    /** Maximum number of entries kept in the LRU before eviction. Default: 256. */
+    maxEntries?: number;
+    /** Number of turns a selected server remains sticky per session. Default: 3. */
+    stickyTurns?: number;
+    /**
+     * Clock function for TTL calculations. Defaults to `Date.now`.
+     * Inject a deterministic function in tests to control time.
+     */
+    now?: () => number;
+};
+/**
+ * Outcome classifier for a single routing resolution. Used in
+ * `ToolRoutingDecision` and emitted as an OTel span attribute.
+ */
+export type ToolRoutingOutcome = "applied" | "skipped-no-query" | "skipped-single-server" | "empty-pick" | "failed-open-parse" | "failed-open-timeout" | "failed-open-error" | "cache-hit";
+/**
+ * Machine-readable summary of one routing resolution. Emitted via the
+ * `emitDecision` callback so the caller can attach it as OTel span attributes
+ * or record it in any other telemetry sink.
+ */
+export type ToolRoutingDecision = {
+    /** How the routing turn concluded. */
+    outcome: ToolRoutingOutcome;
+    /** Server ids the router kept (selected as relevant). */
+    selectedServerIds: string[];
+    /** Server ids whose tools were excluded (router considered them irrelevant). */
+    excludedServerIds: string[];
+    /** Server ids the router returned that did not exist in the catalog. */
+    hallucinatedIds: string[];
+    /** Total number of individual tool names added to the exclusion list. */
+    excludedToolCount: number;
+    /** Number of servers that were offered to the router (always-include excluded). */
+    routableServerCount: number;
+    /** True when the result was served from cache, skipping the router LLM. */
+    cacheHit: boolean;
+    /** Wall-clock time spent in the routing resolution in milliseconds. */
+    durationMs: number;
+};
 /** Parameters for `resolveToolRoutingExclusions()`. */
 export type ToolRoutingResolutionParams = {
     /** Full catalog; always-include servers are filtered out internally. */
@@ -88,4 +169,11 @@ export type ToolRoutingResolutionParams = {
     timeoutMs: number;
     /** Invokes the router LLM — `NeuroLink.generate` bound by the caller. */
     generateFn: (options: GenerateOptions) => Promise<GenerateResult>;
+    /**
+     * Optional callback invoked once per resolution with a structured summary of
+     * the routing decision. Called on every return path (applied, skipped,
+     * failed-open). Must never throw — any error inside is swallowed by
+     * the resolver.
+     */
+    emitDecision?: (decision: ToolRoutingDecision) => void;
 };

package/dist/lib/voice/livekit/brain.js

@@ -61,7 +61,7 @@ export function createVoiceBrain(config) {
         });
         for await (const chunk of result.stream) {
             if (signal?.aborted) {
-                logger.debug("[LiveKitBrain] Turn aborted mid-stream; stopping");
+                logger.debug("voice.brain.turnAborted", { reason: "signal-aborted" });
                 return;
             }
             const delta = extractTextDelta(chunk);

package/dist/lib/voice/livekit/config.d.ts

@@ -7,7 +7,7 @@
  *
  * See docs/features/livekit-voice-agent.md.
  */
-import type { LiveKitServerConfig, LiveKitBrainDefaults } from "../../types/index.js";
+import type { LiveKitServerConfig, LiveKitBrainDefaults, RealtimeVoiceConfig } from "../../types/index.js";
 /**
  * Resolve LiveKit server connection settings from the environment.
  *
@@ -23,6 +23,17 @@ export declare function resolveLiveKitServerConfig(): LiveKitServerConfig;
  * `VOICE_LLM_MODEL`.
  */
 export declare function resolveBrainDefaults(): LiveKitBrainDefaults;
+/**
+ * Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
+ * the environment.
+ *
+ * Every value has a safe default so a worker can run with no realtime-specific
+ * env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
+ * service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
+ * `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
+ * `${LIGHTHOUSE_URL}/ai/mcp/v2`.
+ */
+export declare function resolveRealtimeVoiceConfig(): RealtimeVoiceConfig;
 /**
  * Resolve the semantic end-of-utterance (EOU) turn-detection settings.
  *

package/dist/lib/voice/livekit/config.js

@@ -25,6 +25,22 @@ function readEnv(name, fallback) {
     }
     return fallback;
 }
+function readOptionalEnv(name) {
+    const value = process.env[name];
+    return typeof value === "string" && value.trim().length > 0
+        ? value.trim()
+        : undefined;
+}
+function readNumberEnv(name, fallback) {
+    const raw = process.env[name];
+    if (typeof raw === "string" && raw.trim().length > 0) {
+        const parsed = Number(raw.trim());
+        if (Number.isFinite(parsed) && parsed > 0) {
+            return parsed;
+        }
+    }
+    return fallback;
+}
 /**
  * Resolve LiveKit server connection settings from the environment.
  *
@@ -51,6 +67,44 @@ export function resolveBrainDefaults() {
         model: readEnv("VOICE_LLM_MODEL", DEFAULT_LLM_MODEL),
     };
 }
+const DEFAULT_REALTIME_MODEL = "gemini-live-2.5-flash";
+const DEFAULT_REALTIME_LOCATION = "global";
+const DEFAULT_RESPONSE_MODALITY = "AUDIO";
+const DEFAULT_SYSTEM_PROMPT = "You are a concise voice assistant for a merchant dashboard. Keep replies short and spoken.";
+const DEFAULT_GREETING = "Briefly greet the merchant and ask how you can help with their store today.";
+const DEFAULT_LIGHTHOUSE_URL = "http://localhost:5173";
+const DEFAULT_MCP_PATH = "/ai/mcp/v2";
+/**
+ * Resolve the realtime (Gemini Live speech-to-speech) voice configuration from
+ * the environment.
+ *
+ * Every value has a safe default so a worker can run with no realtime-specific
+ * env at all. Vertex `project` is resolved from `VERTEX_PROJECT`, then the
+ * service-account project (`GOOGLE_AUTH_BREEZE_PROJECT_ID`), then
+ * `GOOGLE_CLOUD_PROJECT_ID`. The MCP URL is `VOICE_MCP_URL` if set, otherwise
+ * `${LIGHTHOUSE_URL}/ai/mcp/v2`.
+ */
+export function resolveRealtimeVoiceConfig() {
+    const lighthouseUrl = readEnv("LIGHTHOUSE_URL", DEFAULT_LIGHTHOUSE_URL);
+    const mcpUrl = readEnv("VOICE_MCP_URL", `${lighthouseUrl}${DEFAULT_MCP_PATH}`);
+    return {
+        project: readOptionalEnv("VERTEX_PROJECT") ??
+            readOptionalEnv("GOOGLE_AUTH_BREEZE_PROJECT_ID") ??
+            readOptionalEnv("GOOGLE_CLOUD_PROJECT_ID"),
+        location: readEnv("VERTEX_LOCATION", DEFAULT_REALTIME_LOCATION),
+        model: readEnv("VOICE_LIVE_MODEL", DEFAULT_REALTIME_MODEL),
+        voice: readOptionalEnv("VOICE_S2S_VOICE"),
+        responseModality: readEnv("VOICE_RESPONSE_MODALITY", DEFAULT_RESPONSE_MODALITY).toUpperCase(),
+        systemPrompt: readEnv("VOICE_SYSTEM_PROMPT", DEFAULT_SYSTEM_PROMPT),
+        greeting: readEnv("VOICE_GREETING", DEFAULT_GREETING),
+        toolsEnabled: readEnv("VOICE_S2S_TOOLS", "").toLowerCase() === "true",
+        mcpUrl,
+        emptyRoomGraceMs: readNumberEnv("VOICE_EMPTY_ROOM_GRACE_MS", 30_000),
+        joinDeadlineMs: readNumberEnv("VOICE_JOIN_DEADLINE_MS", 60_000),
+        hitlTimeoutMs: readNumberEnv("VOICE_HITL_TIMEOUT_MS", 45_000),
+        metricsIntervalMs: readNumberEnv("VOICE_METRICS_INTERVAL_MS", 10_000),
+    };
+}
 const EOU_TRUTHY = new Set(["1", "true", "english", "en", "on", "yes"]);
 /**
  * Resolve the semantic end-of-utterance (EOU) turn-detection settings.

package/dist/lib/voice/livekit/eventBridge.js

@@ -191,7 +191,7 @@ export async function attachEventBridge(params) {
         const json = JSON.stringify(envelope);
         const bytes = encoder.encode(json);
         const onError = (transport) => (error) => {
-            logger.warn("[LiveKitEventBridge] Failed to publish event", {
+            logger.warn("voice.bridge.publishFailed", {
                 transport,
                 type: event.type,
                 error: error instanceof Error ? error.message : String(error),
@@ -304,7 +304,7 @@ export async function attachEventBridge(params) {
             parsed = JSON.parse(decoder.decode(payload));
         }
         catch (error) {
-            logger.warn("[LiveKitEventBridge] Dropping malformed control message", {
+            logger.warn("voice.bridge.controlMalformed", {
                 error: error instanceof Error ? error.message : String(error),
             });
             return;
@@ -332,7 +332,7 @@ export async function attachEventBridge(params) {
     };
     room.on(RoomEvent.DataReceived, onData);
     publish({ type: "status", data: { state: "listening" } });
-    logger.info("[LiveKitEventBridge] Attached", {
+    logger.info("voice.bridge.attached", {
         eventsTopic,
         controlTopic,
         filtered: include !== undefined,
@@ -353,7 +353,7 @@ export async function attachEventBridge(params) {
             emitter.off(STREAM_END_EVENT, onStreamComplete);
             emitter.off(STREAM_ERROR_EVENT, onStreamError);
             room.off(RoomEvent.DataReceived, onData);
-            logger.info("[LiveKitEventBridge] Disposed");
+            logger.info("voice.bridge.disposed", {});
         },
     };
 }

package/dist/lib/voice/livekit/index.d.ts

@@ -8,8 +8,15 @@
  * See docs/features/livekit-voice-agent.md.
  */
 export { createVoiceBrain } from "./brain.js";
-export { resolveLiveKitServerConfig, resolveBrainDefaults } from "./config.js";
+export { resolveLiveKitServerConfig, resolveBrainDefaults, resolveRealtimeVoiceConfig, } from "./config.js";
 export { attachEventBridge } from "./eventBridge.js";
+export { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
 export { mintJoinToken } from "./tokens.js";
+export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
 export { defineVoiceAgent } from "./voiceAgent.js";
-export { startVoiceAgentWorker } from "./voiceAgentWorker.js";
+export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
+export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
+export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
+export { readCallContextFromRoom } from "./roomContext.js";
+export { sanitizeSchema, sanitizeToolParameters, findSchemaIssue, } from "./schemaSanitizer.js";
+export { buildRealtimeMcpTools, mcpResultToText } from "./realtimeMcpTools.js";

package/dist/lib/voice/livekit/index.js

@@ -8,9 +8,16 @@
  * See docs/features/livekit-voice-agent.md.
  */
 export { createVoiceBrain } from "./brain.js";
-export { resolveLiveKitServerConfig, resolveBrainDefaults } from "./config.js";
+export { resolveLiveKitServerConfig, resolveBrainDefaults, resolveRealtimeVoiceConfig, } from "./config.js";
 export { attachEventBridge } from "./eventBridge.js";
+export { attachRealtimeEventBridge } from "./realtimeEventBridge.js";
 export { mintJoinToken } from "./tokens.js";
+export { createVoiceRoom, dispatchVoiceAgent } from "./roomDispatch.js";
 export { defineVoiceAgent } from "./voiceAgent.js";
-export { startVoiceAgentWorker } from "./voiceAgentWorker.js";
+export { defineRealtimeVoiceAgent } from "./realtimeVoiceAgent.js";
+export { startVoiceAgentWorker, startRealtimeVoiceAgentWorker, installVoiceWorkerProcessGuards, } from "./voiceAgentWorker.js";
+export { ensureVertexAdc, clearGeminiApiKeyEnv } from "./vertexAuth.js";
+export { readCallContextFromRoom } from "./roomContext.js";
+export { sanitizeSchema, sanitizeToolParameters, findSchemaIssue, } from "./schemaSanitizer.js";
+export { buildRealtimeMcpTools, mcpResultToText } from "./realtimeMcpTools.js";
 //# sourceMappingURL=index.js.map

package/dist/lib/voice/livekit/realtimeEventBridge.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Realtime data-channel event bridge.
+ *
+ * The cascaded bridge (`eventBridge.ts`) is driven by NeuroLink's event emitter.
+ */
+import type { RealtimeEventBridgeHandle, RealtimeEventBridgeParams } from "../../types/index.js";
+/**
+ * Attach the realtime event bridge to a room.
+ *
+ * Returns `publishEvent` (worker → browser), `requestConfirmation` (HITL
+ * round-trip), and `dispose` (remove the control listener, decline anything
+ * still pending).
+ */
+export declare function attachRealtimeEventBridge(params: RealtimeEventBridgeParams): Promise<RealtimeEventBridgeHandle>;

package/dist/lib/voice/livekit/realtimeEventBridge.js

@@ -0,0 +1,161 @@
+/**
+ * Realtime data-channel event bridge.
+ *
+ * The cascaded bridge (`eventBridge.ts`) is driven by NeuroLink's event emitter.
+ */
+import { z } from "zod";
+import { logger } from "../../utils/logger.js";
+const DEFAULT_EVENTS_TOPIC = "ai-events";
+const DEFAULT_CONTROL_TOPIC = "ai-control";
+const DEFAULT_MAX_INLINE_BYTES = 12_000;
+const DEFAULT_HITL_TIMEOUT_MS = 45_000;
+const hitlControlMessageSchema = z.object({
+    action: z.enum(["hitl:accept", "hitl:reject"]),
+    confirmationId: z.string(),
+});
+/**
+ * Attach the realtime event bridge to a room.
+ *
+ * Returns `publishEvent` (worker → browser), `requestConfirmation` (HITL
+ * round-trip), and `dispose` (remove the control listener, decline anything
+ * still pending).
+ */
+export async function attachRealtimeEventBridge(params) {
+    const { room } = params;
+    const eventsTopic = params.eventsTopic ?? DEFAULT_EVENTS_TOPIC;
+    const controlTopic = params.controlTopic ?? DEFAULT_CONTROL_TOPIC;
+    const maxInlineBytes = params.maxInlineBytes ?? DEFAULT_MAX_INLINE_BYTES;
+    const hitlTimeoutMs = params.hitlTimeoutMs ?? DEFAULT_HITL_TIMEOUT_MS;
+    const { RoomEvent } = await import("@livekit/rtc-node");
+    const encoder = new TextEncoder();
+    const decoder = new TextDecoder();
+    let seq = 0;
+    let disposed = false;
+    const startedAt = Date.now();
+    const publishEvent = (type, data) => {
+        if (disposed) {
+            return;
+        }
+        const participant = room.localParticipant;
+        if (!participant) {
+            return;
+        }
+        try {
+            seq += 1;
+            const json = JSON.stringify({ seq, ts: Date.now(), type, data });
+            const bytes = encoder.encode(json);
+            const via = bytes.byteLength <= maxInlineBytes ? "data" : "stream";
+            logger.debug("realtime.bridge.publish", {
+                ms: Date.now() - startedAt,
+                seq,
+                type,
+                via,
+                bytes: bytes.byteLength,
+            });
+            if (via === "data") {
+                void participant.publishData(bytes, {
+                    reliable: true,
+                    topic: eventsTopic,
+                });
+            }
+            else {
+                void participant.sendText(json, { topic: eventsTopic });
+            }
+        }
+        catch {
+            /* non-fatal — the UI bridge is best-effort */
+        }
+    };
+    // HITL: WRITE-labeled tools pause for user confirmation. We publish a
+    // `hitl-prompt`, the UI replies on the control topic, and we resolve the
+    // pending request by confirmationId. A timeout is treated as a decline so a
+    // turn can never hang waiting on the user.
+    const pendingHitl = new Map();
+    let hitlSeq = 0;
+    const onData = (...args) => {
+        const payload = args[0];
+        const topic = args[3];
+        if (topic !== controlTopic || !(payload instanceof Uint8Array)) {
+            return;
+        }
+        let raw;
+        try {
+            raw = JSON.parse(decoder.decode(payload));
+        }
+        catch {
+            return;
+        }
+        const message = hitlControlMessageSchema.safeParse(raw);
+        if (!message.success) {
+            return;
+        }
+        const { action, confirmationId } = message.data;
+        const resolver = pendingHitl.get(confirmationId);
+        if (resolver) {
+            logger.debug("realtime.bridge.controlReceived", {
+                action,
+                confirmationId,
+            });
+            pendingHitl.delete(confirmationId);
+            resolver(action === "hitl:accept");
+        }
+        else {
+            logger.warn("realtime.bridge.controlUnmatched", {
+                action,
+                confirmationId,
+            });
+        }
+    };
+    room.on(RoomEvent.DataReceived, onData);
+    const requestConfirmation = (toolName, args) => {
+        if (disposed) {
+            logger.warn("realtime.bridge.hitlDisposed", { toolName });
+            return Promise.resolve(false);
+        }
+        hitlSeq += 1;
+        const confirmationId = `hitl_${seq}_${hitlSeq}`;
+        publishEvent("hitl-prompt", {
+            confirmationId,
+            toolName,
+            actionType: `Run ${toolName}`,
+            arguments: args ?? {},
+        });
+        logger.info("realtime.bridge.hitlPrompt", { toolName, confirmationId });
+        return new Promise((resolve) => {
+            const timer = setTimeout(() => {
+                pendingHitl.delete(confirmationId);
+                logger.warn("realtime.bridge.hitlTimeout", {
+                    toolName,
+                    confirmationId,
+                    timeoutMs: hitlTimeoutMs,
+                });
+                resolve(false);
+            }, hitlTimeoutMs);
+            pendingHitl.set(confirmationId, (approved) => {
+                clearTimeout(timer);
+                logger.info("realtime.bridge.hitlResolved", {
+                    toolName,
+                    confirmationId,
+                    approved,
+                });
+                resolve(approved);
+            });
+        });
+    };
+    return {
+        publishEvent,
+        requestConfirmation,
+        dispose() {
+            if (disposed) {
+                return;
+            }
+            disposed = true;
+            room.off(RoomEvent.DataReceived, onData);
+            for (const [, resolver] of pendingHitl) {
+                resolver(false);
+            }
+            pendingHitl.clear();
+        },
+    };
+}
+//# sourceMappingURL=realtimeEventBridge.js.map

package/dist/lib/voice/livekit/realtimeMcpTools.d.ts

@@ -0,0 +1,31 @@
+/**
+ * MCP tools → Gemini function tools bridge (realtime mode).
+ *
+ * In speech-to-speech mode Gemini owns the tool loop, so an external MCP
+ * server's tools are registered as LiveKit/Gemini function tools rather than
+ * NeuroLink tools. This connects to the MCP server as the calling user
+ * (`x-auth-token` + base64 `x-context`), sanitizes each tool's schema into the
+ * Gemini-safe subset, wires WRITE-labeled (destructive) tools through a HITL
+ * confirmation, and forwards tool start/result events to the browser bridge.
+ *
+ * `@livekit/agents` and `@modelcontextprotocol/sdk` are imported dynamically so
+ * the core package does not require them unless the realtime agent is used.
+ *
+ * See docs/features/livekit-voice-agent.md.
+ */
+import type { llm as llmNs } from "@livekit/agents";
+import type { BuildRealtimeMcpToolsParams } from "../../types/index.js";
+/** Flatten an MCP tool result's text content into a single string for the model. */
+export declare function mcpResultToText(result: unknown): string;
+/**
+ * Connect to the MCP server as the calling user and bridge every tool into a
+ * LiveKit `ToolContext`. Returns the tool map + the client (close on shutdown).
+ * The server already scopes the tool list by `x-context`, so no client-side
+ * filtering is needed.
+ */
+export declare function buildRealtimeMcpTools(params: BuildRealtimeMcpToolsParams): Promise<{
+    tools: llmNs.ToolContext;
+    client: {
+        close: () => Promise<void>;
+    };
+}>;