@juspay/neurolink 9.53.0 → 9.54.1

package/dist/cli/commands/voiceServer.js

@@ -0,0 +1,17 @@
+import { startVoiceServer } from "../../lib/server/voice/voiceServerApp.js";
+import { configureVoiceServerEnvironment } from "../../lib/server/voice/voiceWebSocketHandler.js";
+export const voiceServerCommand = {
+    command: "voice-server",
+    describe: "Start the real-time voice assistant server (Soniox STT + Cartesia TTS + Cobra VAD)",
+    builder: (yargs) => yargs.option("port", {
+        alias: "p",
+        type: "number",
+        default: 3000,
+        describe: "Port to listen on",
+    }),
+    handler: async (argv) => {
+        configureVoiceServerEnvironment();
+        await startVoiceServer(argv.port);
+    },
+};
+//# sourceMappingURL=voiceServer.js.map

package/dist/cli/parser.js

@@ -17,6 +17,7 @@ import { proxyStartCommand, proxyStatusCommand, proxyTelemetryCommand, proxySetu
 import { EvaluateCommandFactory } from "./commands/evaluate.js";
 import { TaskCommandFactory } from "./commands/task.js";
 import { AutoresearchCommandFactory } from "./commands/autoresearch.js";
+import { voiceServerCommand } from "./commands/voiceServer.js";
 // Enhanced CLI with Professional UX
 export function initializeCliParser() {
     return (yargs(hideBin(process.argv))
@@ -206,6 +207,8 @@ export function initializeCliParser() {
         // Task Command Group - Scheduled and self-running tasks
         .command(TaskCommandFactory.createTaskCommands())
         // AutoResearch Command Group - Automated AI-driven research experiments
-        .command(AutoresearchCommandFactory.createAutoresearchCommands())); // Close the main return statement
+        .command(AutoresearchCommandFactory.createAutoresearchCommands())
+        // Real-time voice server (Soniox STT + Cartesia TTS + Cobra VAD)
+        .command(voiceServerCommand)); // Close the main return statement
 }
 //# sourceMappingURL=parser.js.map

package/dist/lib/adapters/tts/cartesiaHandler.d.ts

@@ -0,0 +1,12 @@
+import { EventEmitter } from "events";
+export declare function getCartesiaWsUrl(): string;
+export declare class CartesiaStream extends EventEmitter {
+    private ws;
+    private contextId;
+    private isReady;
+    constructor(contextId: string);
+    ready(): Promise<void>;
+    send(text: string, cont?: boolean): void;
+    flush(): void;
+    close(): void;
+}

package/dist/lib/adapters/tts/cartesiaHandler.js

@@ -0,0 +1,131 @@
+import WebSocket from "ws";
+import { EventEmitter } from "events";
+import { logger } from "../../utils/logger.js";
+import { withTimeout } from "../../utils/async/withTimeout.js";
+export function getCartesiaWsUrl() {
+    const baseUrl = process.env.CARTESIA_WS_BASE_URL ?? "wss://api.cartesia.ai/tts/websocket";
+    const cartesiaVersion = process.env.CARTESIA_API_VERSION ?? "2025-04-16";
+    const wsUrl = new URL(baseUrl);
+    wsUrl.searchParams.set("cartesia_version", cartesiaVersion);
+    return wsUrl.toString();
+}
+export class CartesiaStream extends EventEmitter {
+    ws = null;
+    contextId;
+    isReady = false;
+    constructor(contextId) {
+        super();
+        this.contextId = contextId;
+        const apiKey = process.env.CARTESIA_API_KEY;
+        if (!apiKey) {
+            throw new Error("CARTESIA_API_KEY is not set in environment");
+        }
+        this.ws = new WebSocket(getCartesiaWsUrl(), {
+            headers: { "X-API-Key": apiKey },
+        });
+        this.ws.on("open", () => {
+            this.isReady = true;
+            logger.info("[CARTESIA] WS connected");
+            this.emit("ready");
+        });
+        this.ws.on("message", (data) => {
+            let msg;
+            try {
+                msg = JSON.parse(data.toString());
+            }
+            catch {
+                logger.error("[CARTESIA] Failed to parse message");
+                return;
+            }
+            // Handle error first so it always surfaces, even mid-stream
+            if (msg.error) {
+                const err = new Error(msg.error);
+                if (this.listenerCount("error") > 0) {
+                    this.emit("error", err);
+                }
+                else {
+                    logger.error("[CARTESIA] Unhandled error:", msg.error);
+                }
+                return;
+            }
+            if (msg.data) {
+                const audio = Buffer.from(msg.data, "base64");
+                this.emit("audio", audio);
+            }
+            if (msg.done) {
+                this.emit("done");
+            }
+        });
+        this.ws.on("error", (err) => {
+            if (this.listenerCount("error") > 0) {
+                this.emit("error", err);
+            }
+            else {
+                logger.error("[CARTESIA] Unhandled WebSocket error:", err.message);
+            }
+        });
+        this.ws.on("close", () => {
+            this.isReady = false;
+            this.emit("close");
+        });
+    }
+    async ready() {
+        if (this.isReady) {
+            return;
+        }
+        const connectPromise = new Promise((resolve, reject) => {
+            if (!this.ws) {
+                reject(new Error("Cartesia WebSocket is not initialized"));
+                return;
+            }
+            this.ws.once("open", resolve);
+            this.ws.once("error", reject);
+            this.ws.once("close", () => reject(new Error("Cartesia WebSocket closed before ready")));
+        });
+        return withTimeout(connectPromise, 5000, "Cartesia WS connect timed out");
+    }
+    send(text, cont = true) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            return;
+        }
+        this.ws.send(JSON.stringify({
+            context_id: this.contextId,
+            model_id: "sonic-3",
+            transcript: text,
+            voice: {
+                mode: "id",
+                id: "694f9389-aac1-45b6-b726-9d9369183238",
+            },
+            output_format: {
+                container: "raw",
+                encoding: "pcm_s16le",
+                sample_rate: 24000,
+            },
+            continue: cont,
+        }));
+    }
+    flush() {
+        this.send("", false);
+    }
+    close() {
+        if (!this.ws) {
+            return;
+        }
+        const ws = this.ws;
+        this.ws = null;
+        this.isReady = false;
+        if (ws.readyState === WebSocket.OPEN ||
+            ws.readyState === WebSocket.CONNECTING) {
+            try {
+                ws.close();
+            }
+            catch (error) {
+                logger.warn("[CARTESIA] Failed to close WebSocket cleanly", error);
+            }
+        }
+        ws.once("close", () => {
+            ws.removeAllListeners();
+        });
+    }
+}
+//# sourceMappingURL=cartesiaHandler.js.map

package/dist/lib/providers/azureOpenai.d.ts

@@ -13,7 +13,10 @@ export declare class AzureOpenAIProvider extends BaseProvider {
     getProviderName(): AIProviderName;
     getDefaultModel(): string;
     /**
-     * Returns the Vercel AI SDK model instance for Azure OpenAI
+     * Returns the Vercel AI SDK model instance for Azure OpenAI.
+     * Uses .chat() explicitly because @ai-sdk/azure v3+ defaults the bare
+     * provider() call to the Responses API, which many Azure deployments
+     * do not support yet.
      */
     getAISDKModel(): LanguageModel;
     protected formatProviderError(error: unknown): Error;

package/dist/lib/providers/azureOpenai.js

@@ -43,11 +43,14 @@ export class AzureOpenAIProvider extends BaseProvider {
             validateApiKey(createAzureEndpointConfig());
         }
         // Create the Azure provider instance with proxy support
-        // Let the Azure SDK handle all URL construction automatically
+        // useDeploymentBasedUrls is required because @ai-sdk/azure v3+ defaults to
+        // the /v1/ URL format, but most Azure deployments still require the legacy
+        // /deployments/{deployment}/ URL pattern.
         this.azureProvider = createAzure({
             resourceName: this.resourceName,
             apiKey: this.apiKey,
             apiVersion: this.apiVersion,
+            useDeploymentBasedUrls: true,
             fetch: createProxyFetch(),
         });
         logger.debug("Azure Vercel Provider initialized", {
@@ -63,10 +66,13 @@ export class AzureOpenAIProvider extends BaseProvider {
         return this.deployment;
     }
     /**
-     * Returns the Vercel AI SDK model instance for Azure OpenAI
+     * Returns the Vercel AI SDK model instance for Azure OpenAI.
+     * Uses .chat() explicitly because @ai-sdk/azure v3+ defaults the bare
+     * provider() call to the Responses API, which many Azure deployments
+     * do not support yet.
      */
     getAISDKModel() {
-        return this.azureProvider(this.deployment);
+        return this.azureProvider.chat(this.deployment);
     }
     formatProviderError(error) {
         if (error instanceof TimeoutError) {

package/dist/lib/server/voice/frameBus.d.ts

@@ -0,0 +1,8 @@
+import type { Frame } from "./types.js";
+export declare class FrameBus {
+    private handlers;
+    subscribe<T extends Frame["type"]>(type: T, fn: (frame: Extract<Frame, {
+        type: T;
+    }>) => void): void;
+    publish(frame: Frame): void;
+}

package/dist/lib/server/voice/frameBus.js

@@ -0,0 +1,25 @@
+import { logger } from "../../utils/logger.js";
+export class FrameBus {
+    handlers = {};
+    subscribe(type, fn) {
+        if (!this.handlers[type]) {
+            this.handlers[type] = [];
+        }
+        this.handlers[type].push(fn);
+    }
+    publish(frame) {
+        const subs = this.handlers[frame.type];
+        if (!subs) {
+            return;
+        }
+        for (const fn of subs) {
+            try {
+                fn(frame);
+            }
+            catch (err) {
+                logger.error(`[FrameBus] Subscriber threw on ${frame.type}:`, err);
+            }
+        }
+    }
+}
+//# sourceMappingURL=frameBus.js.map

package/dist/lib/server/voice/turnManager.d.ts

@@ -0,0 +1,15 @@
+import type { FrameBus } from "./frameBus.js";
+export declare enum TurnState {
+    IDLE = 0,
+    USER_SPEAKING = 1,
+    PROCESSING = 2,
+    ASSISTANT_SPEAKING = 3
+}
+export declare class TurnManager {
+    state: TurnState;
+    constructor(bus: FrameBus);
+    private onVadStart;
+    private onVadStop;
+    assistantSpeaking(): void;
+    reset(): void;
+}

package/dist/lib/server/voice/turnManager.js

@@ -0,0 +1,36 @@
+export var TurnState;
+(function (TurnState) {
+    TurnState[TurnState["IDLE"] = 0] = "IDLE";
+    TurnState[TurnState["USER_SPEAKING"] = 1] = "USER_SPEAKING";
+    TurnState[TurnState["PROCESSING"] = 2] = "PROCESSING";
+    TurnState[TurnState["ASSISTANT_SPEAKING"] = 3] = "ASSISTANT_SPEAKING";
+})(TurnState || (TurnState = {}));
+export class TurnManager {
+    state = TurnState.IDLE;
+    constructor(bus) {
+        bus.subscribe("vad_start", () => this.onVadStart());
+        bus.subscribe("vad_stop", () => this.onVadStop());
+    }
+    onVadStart() {
+        // Only update state if TTS is NOT playing. During ASSISTANT_SPEAKING, the
+        // barge-in interrupt is triggered by Soniox non-final tokens — which arrive
+        // after a network round-trip. If we let VAD immediately flip state to
+        // USER_SPEAKING, the state check in handleSonioxMessage fails and the
+        // interrupt never fires.
+        if (this.state !== TurnState.ASSISTANT_SPEAKING) {
+            this.state = TurnState.USER_SPEAKING;
+        }
+    }
+    onVadStop() {
+        if (this.state === TurnState.USER_SPEAKING) {
+            this.state = TurnState.PROCESSING;
+        }
+    }
+    assistantSpeaking() {
+        this.state = TurnState.ASSISTANT_SPEAKING;
+    }
+    reset() {
+        this.state = TurnState.IDLE;
+    }
+}
+//# sourceMappingURL=turnManager.js.map

package/dist/lib/server/voice/types.d.ts

@@ -0,0 +1,20 @@
+export type Frame = {
+    type: "audio";
+    data: Int16Array;
+} | {
+    type: "vad_start";
+} | {
+    type: "vad_stop";
+} | {
+    type: "transcript";
+    text: string;
+    final: boolean;
+} | {
+    type: "llm_token";
+    text: string;
+} | {
+    type: "tts_audio";
+    data: Buffer;
+} | {
+    type: "interrupt";
+};

package/dist/lib/server/voice/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/lib/server/voice/voiceServerApp.d.ts

@@ -0,0 +1 @@
+export declare function startVoiceServer(port?: number): Promise<void>;

package/dist/lib/server/voice/voiceServerApp.js

@@ -0,0 +1,118 @@
+import express from "express";
+import fs from "fs";
+import http from "http";
+import path from "path";
+import { fileURLToPath } from "url";
+import { setupWebSocket } from "./voiceWebSocketHandler.js";
+import { NeuroLink } from "../../neurolink.js";
+import { logger } from "../../utils/logger.js";
+import { withTimeout } from "../../utils/async/withTimeout.js";
+import { getCartesiaWsUrl } from "../../adapters/tts/cartesiaHandler.js";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+/**
+ * Resolve the public/ directory containing static assets.
+ * The CLI build (tsc) only emits .ts → .js and does NOT copy non-TS assets,
+ * so __dirname/public may not exist when running from dist/.
+ * Fall back to the original source path in that case.
+ */
+function resolvePublicPath() {
+    const compiled = path.join(__dirname, "public");
+    if (fs.existsSync(compiled)) {
+        return compiled;
+    }
+    // Resolve from project root → src/lib/server/voice/public
+    const source = path.resolve(__dirname, "../../../../src/lib/server/voice/public");
+    if (fs.existsSync(source)) {
+        return source;
+    }
+    return compiled; // let express.static handle the 404
+}
+export async function startVoiceServer(port = 3000) {
+    const app = express();
+    /* ---------- STATIC FILES ---------- */
+    const publicPath = resolvePublicPath();
+    logger.info("[SERVER] Serving static from:", publicPath);
+    app.use(express.static(publicPath));
+    app.get("/", (_, res) => {
+        res.sendFile(path.join(publicPath, "index.html"));
+    });
+    /* ---------- HEALTH CHECK ---------- */
+    app.get("/health", (_, res) => {
+        res.json({ status: "ok" });
+    });
+    const server = http.createServer(app);
+    /* ---------- WS ---------- */
+    setupWebSocket(server);
+    /* ---------- START ---------- */
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(port, () => {
+            server.removeListener("error", reject);
+            logger.info(`[SERVER] Voice server running at http://localhost:${port}`);
+            resolve();
+        });
+    });
+    /* ---------- WARMUP ---------- */
+    // Pre-warm NeuroLink + Azure on startup so the first real user request isn't
+    // slow. NeuroLink's MCP init + Azure's connection pool both have cold-start
+    // overhead that shows up as 3-4s on the very first call. We also open and
+    // immediately close a Cartesia WS to prime the TLS handshake.
+    warmup().catch((err) => {
+        logger.warn("[WARMUP] Failed (non-fatal):", err.message);
+    });
+}
+async function warmup() {
+    const t = Date.now();
+    logger.info("[WARMUP] Warming up LLM + TTS...");
+    const neurolink = new NeuroLink();
+    const provider = process.env.VOICE_LLM_PROVIDER ?? "azure";
+    const model = process.env.VOICE_LLM_MODEL ?? "gpt-4o-automatic";
+    try {
+        const result = await withTimeout(neurolink.stream({
+            provider,
+            model,
+            input: { text: "hi" },
+            maxTokens: 3,
+            disableTools: true,
+            enableAnalytics: false,
+            enableEvaluation: false,
+        }), 15000, "LLM warmup timed out");
+        // Drain the stream so the connection is fully exercised.
+        for await (const _chunk of result.stream) {
+            /* drain */
+        }
+        logger.info(`[WARMUP] LLM warmup done in ${Date.now() - t}ms`);
+    }
+    catch (err) {
+        logger.warn("[WARMUP] LLM warmup failed (non-fatal):", err.message);
+    }
+    // Cartesia TLS warmup — open WS, wait for connect, then close.
+    try {
+        const { default: WebSocket } = await import("ws");
+        const apiKey = process.env.CARTESIA_API_KEY;
+        await new Promise((resolve) => {
+            const ws = new WebSocket(getCartesiaWsUrl(), {
+                headers: apiKey ? { "X-API-Key": apiKey } : undefined,
+            });
+            const timeout = setTimeout(() => {
+                ws.terminate();
+                resolve(); // non-fatal, just move on
+            }, 5000);
+            ws.once("open", () => {
+                clearTimeout(timeout);
+                ws.close();
+                resolve();
+            });
+            ws.once("error", () => {
+                clearTimeout(timeout);
+                resolve(); // non-fatal
+            });
+        });
+        logger.info(`[WARMUP] Cartesia warmup done in ${Date.now() - t}ms`);
+    }
+    catch {
+        // non-fatal
+    }
+}
+//# sourceMappingURL=voiceServerApp.js.map

package/dist/lib/server/voice/voiceWebSocketHandler.d.ts

@@ -0,0 +1,11 @@
+import type { Server as HttpServer } from "http";
+/**
+ * Call from the voice-server command handler BEFORE importing anything else
+ * so the env change is scoped to voice mode only.
+ */
+export declare function configureVoiceServerEnvironment(): void;
+export type Message = {
+    role: "system" | "user" | "assistant";
+    content: string;
+};
+export declare function setupWebSocket(server: HttpServer): void;