@juspay/neurolink 9.53.0 → 9.54.0

package/dist/server/voice/public/app.js

@@ -0,0 +1,275 @@
+const {
+  WebSocket: BrowserWebSocket,
+  HTMLElement: BrowserHTMLElement,
+  HTMLButtonElement: BrowserHTMLButtonElement,
+  AudioContext: BrowserAudioContext,
+  Blob: BrowserBlob,
+  alert: browserAlert,
+} = globalThis;
+
+const socketProtocol =
+  globalThis.location.protocol === "https:" ? "wss:" : "ws:";
+const socketUrl = `${socketProtocol}//${globalThis.location.host}`;
+
+/** @type {WebSocket | null} */
+let socket = null;
+
+const orb = document.getElementById("orb");
+const statusEl = document.getElementById("status");
+const toggleBtn = document.getElementById("toggleBtn");
+
+if (!(orb instanceof BrowserHTMLElement)) {
+  throw new Error("Missing #orb element");
+}
+if (!(statusEl instanceof BrowserHTMLElement)) {
+  throw new Error("Missing #status element");
+}
+if (!(toggleBtn instanceof BrowserHTMLButtonElement)) {
+  throw new Error("Missing #toggleBtn element");
+}
+
+const orbEl = orb;
+const statusNode = statusEl;
+const toggleBtnEl = toggleBtn;
+
+// Cobra VAD on the server requires 16kHz raw PCM
+const CAPTURE_SAMPLE_RATE = 16000;
+
+let isActive = false;
+
+/* ---- PLAYBACK (TTS arrives as 24kHz raw PCM from server) ---- */
+
+const playbackCtx = new BrowserAudioContext({ sampleRate: 24000 });
+let playbackTime = 0;
+/** @type {AudioBufferSourceNode[]} */
+let activeSources = []; // track all scheduled sources so we can stop them on interrupt
+let playbackCanceled = false;
+
+/* ---- WEBSOCKET ---- */
+
+/**
+ * Create (or recreate) the WebSocket connection.
+ * Called at page load and on every startConversation so the page
+ * is recoverable after a disconnect.
+ */
+function connectSocket() {
+  if (socket && socket.readyState <= BrowserWebSocket.OPEN) {
+    return; // already connected or connecting
+  }
+  socket = new BrowserWebSocket(socketUrl);
+  socket.binaryType = "blob";
+
+  socket.onopen = () => {
+    console.log("Connected");
+    statusNode.textContent = "Connected";
+  };
+  /** @param {Event} e */
+  socket.onerror = (e) => console.error("WS error", e);
+  socket.onclose = () => {
+    stopConversation();
+    statusNode.textContent = "Disconnected";
+  };
+
+  socket.onmessage = onSocketMessage;
+}
+
+/* ---- INCOMING: TTS audio + control messages ---- */
+
+/** @param {MessageEvent<string | Blob>} event */
+async function onSocketMessage(event) {
+  // JSON control message
+  if (typeof event.data === "string") {
+    let msg;
+    try {
+      msg = JSON.parse(event.data);
+    } catch {
+      return;
+    }
+
+    if (msg.type === "interrupt") {
+      console.log("Interrupt — stopping playback");
+      playbackCanceled = true;
+      activeSources.forEach((s) => {
+        try {
+          s.stop(0);
+        } catch {
+          /* already stopped */
+        }
+      });
+      activeSources = [];
+      playbackTime = playbackCtx.currentTime;
+      orbEl.className = isActive ? "listening" : "idle";
+      statusNode.textContent = isActive ? "Listening..." : "Stopped.";
+    }
+    return;
+  }
+
+  // Binary: raw 16-bit PCM, 24kHz, mono
+  if (!(event.data instanceof BrowserBlob)) {
+    return;
+  }
+
+  orbEl.className = "speaking";
+  statusNode.textContent = "Assistant speaking...";
+  playbackCanceled = false;
+
+  const arrayBuffer = await event.data.arrayBuffer();
+  const pcm16 = new Int16Array(arrayBuffer);
+  const float32 = new Float32Array(pcm16.length);
+  for (let i = 0; i < pcm16.length; i++) {
+    float32[i] = pcm16[i] / 32768;
+  }
+
+  const audioBuffer = playbackCtx.createBuffer(1, float32.length, 24000);
+  audioBuffer.getChannelData(0).set(float32);
+
+  const source = playbackCtx.createBufferSource();
+  source.buffer = audioBuffer;
+  source.connect(playbackCtx.destination);
+
+  const nowT = playbackCtx.currentTime;
+  if (playbackTime < nowT) {
+    playbackTime = nowT;
+  }
+  source.start(playbackTime);
+  playbackTime += audioBuffer.duration;
+  activeSources.push(source);
+
+  source.onended = () => {
+    activeSources = activeSources.filter((s) => s !== source);
+    // When the last chunk finishes, notify server and reset UI.
+    // Skip if playback was intentionally canceled (interrupt/stop) —
+    // stale onended callbacks must not send playback_done mid-barge-in.
+    if (
+      !playbackCanceled &&
+      activeSources.length === 0 &&
+      playbackTime <= playbackCtx.currentTime + 0.05
+    ) {
+      if (socket && socket.readyState === BrowserWebSocket.OPEN) {
+        socket.send(JSON.stringify({ type: "playback_done" }));
+      }
+      orbEl.className = isActive ? "listening" : "idle";
+      statusNode.textContent = isActive ? "Listening..." : "Stopped.";
+    }
+  };
+}
+
+// Connect immediately on page load
+connectSocket();
+
+/* ---- CAPTURE ---- */
+
+/** @type {AudioContext | null} */
+let captureCtx = null;
+/** @type {ScriptProcessorNode | null} */
+let scriptProcessor = null;
+/** @type {MediaStream | null} */
+let micStream = null;
+
+async function startConversation() {
+  try {
+    // Ensure we have a live WebSocket (reconnects after previous disconnect)
+    connectSocket();
+    await playbackCtx.resume();
+
+    micStream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+        channelCount: 1,
+      },
+    });
+
+    // Separate AudioContext at 16kHz — keeps capture and playback sample rates independent
+    captureCtx = new BrowserAudioContext({ sampleRate: CAPTURE_SAMPLE_RATE });
+    const micSource = captureCtx.createMediaStreamSource(micStream);
+
+    // 1024 samples = 64ms per callback; server splits into 512-sample Cobra frames
+    scriptProcessor = captureCtx.createScriptProcessor(1024, 1, 1);
+    scriptProcessor.onaudioprocess = (e) => {
+      if (!isActive || !socket || socket.readyState !== BrowserWebSocket.OPEN) {
+        return;
+      }
+      const input = e.inputBuffer.getChannelData(0);
+      const int16 = new Int16Array(input.length);
+      for (let i = 0; i < input.length; i++) {
+        const s = Math.max(-1, Math.min(1, input[i]));
+        int16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+      }
+      socket.send(int16.buffer);
+    };
+
+    micSource.connect(scriptProcessor);
+    // ScriptProcessor requires a destination in the graph to fire onaudioprocess,
+    // but we must NOT route mic audio to speakers — that feeds back into the mic,
+    // defeats browser AEC, and causes Soniox to transcribe TTS audio as user speech.
+    // A zero-gain node keeps the graph alive while staying completely silent.
+    const silentGain = captureCtx.createGain();
+    silentGain.gain.value = 0;
+    scriptProcessor.connect(silentGain);
+    silentGain.connect(captureCtx.destination);
+
+    isActive = true;
+    orbEl.className = "listening";
+    statusNode.textContent = "Listening...";
+    toggleBtnEl.textContent = "Stop Conversation";
+    toggleBtnEl.classList.add("active");
+  } catch (err) {
+    // Release any partially-initialized resources
+    if (scriptProcessor) {
+      scriptProcessor.disconnect();
+      scriptProcessor = null;
+    }
+    if (micStream) {
+      micStream.getTracks().forEach((t) => t.stop());
+      micStream = null;
+    }
+    if (captureCtx) {
+      captureCtx.close();
+      captureCtx = null;
+    }
+    isActive = false;
+    console.error("Failed to start:", err);
+    const message = err instanceof Error ? err.message : String(err);
+    browserAlert("Error: " + message);
+  }
+}
+
+function stopConversation() {
+  isActive = false;
+  playbackCanceled = true;
+  activeSources.forEach((s) => {
+    try {
+      s.stop(0);
+    } catch {
+      /* already stopped */
+    }
+  });
+  activeSources = [];
+  playbackTime = playbackCtx.currentTime;
+  if (scriptProcessor) {
+    scriptProcessor.disconnect();
+    scriptProcessor = null;
+  }
+  if (micStream) {
+    micStream.getTracks().forEach((t) => t.stop());
+    micStream = null;
+  }
+  if (captureCtx) {
+    captureCtx.close();
+    captureCtx = null;
+  }
+  orbEl.className = "idle";
+  statusNode.textContent = "Stopped.";
+  toggleBtnEl.textContent = "Start Conversation";
+  toggleBtnEl.classList.remove("active");
+}
+
+toggleBtnEl.onclick = async () => {
+  if (!isActive) {
+    await startConversation();
+  } else {
+    stopConversation();
+  }
+};

package/dist/server/voice/public/index.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Voice Assistant</title>
+    <link rel="stylesheet" href="styles.css" />
+  </head>
+
+  <body>
+    <div id="orb" class="idle"></div>
+
+    <button id="toggleBtn">Start Conversation</button>
+    <p id="status" aria-live="polite">Ready</p>
+
+    <script type="module" src="app.js"></script>
+  </body>
+</html>

package/dist/server/voice/public/pcm-worklet.js

@@ -0,0 +1,67 @@
+/** @type {any} */
+const g = globalThis;
+
+/** @type {any} */
+const AudioWorkletProcessorBase = g["AudioWorkletProcessor"];
+
+/** @type {(name: string, ctor: any) => void} */
+const registerProcessorFn = g["registerProcessor"];
+
+/** Max queued chunks before dropping oldest (~10s of 24kHz audio). */
+const HIGH_WATER_MARK = 500;
+
+class PCMProcessor extends AudioWorkletProcessorBase {
+  constructor() {
+    super();
+    /** @type {Float32Array[]} */
+    this.queue = [];
+    /** @type {number} */
+    this.offset = 0;
+
+    /** @param {MessageEvent<ArrayLike<number>>} event */
+    this.port.onmessage = (event) => {
+      if (this.queue.length >= HIGH_WATER_MARK) {
+        // Drop oldest chunks to prevent unbounded memory growth
+        this.queue.splice(0, this.queue.length - HIGH_WATER_MARK + 1);
+        this.offset = 0;
+      }
+      this.queue.push(new Float32Array(event.data));
+    };
+  }
+
+  /**
+   * @param {Float32Array[][]} _inputs
+   * @param {Float32Array[][]} outputs
+   * @returns {boolean}
+   */
+  process(_inputs, outputs) {
+    const output = outputs[0][0];
+
+    if (!this.queue.length) {
+      output.fill(0);
+      return true;
+    }
+
+    let chunk = this.queue[0];
+
+    for (let i = 0; i < output.length; i++) {
+      if (this.offset >= chunk.length) {
+        this.queue.shift();
+        this.offset = 0;
+
+        if (!this.queue.length) {
+          output.fill(0, i);
+          break;
+        }
+
+        chunk = this.queue[0];
+      }
+
+      output[i] = chunk[this.offset++];
+    }
+
+    return true;
+  }
+}
+
+registerProcessorFn("pcm-processor", PCMProcessor);

package/dist/server/voice/public/styles.css

@@ -0,0 +1,102 @@
+/* Base layout */
+body {
+  background: radial-gradient(circle at center, #0b0f19 0%, #05070d 70%);
+  color: white;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  height: 100vh;
+  margin: 0;
+  font-family:
+    -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+}
+
+/* Voice orb */
+#orb {
+  width: 140px;
+  height: 140px;
+  border-radius: 50%;
+  margin-bottom: 32px;
+  transition: all 0.35s ease;
+}
+
+/* Idle */
+#orb.idle {
+  background: radial-gradient(circle, #222, #0a0a0a);
+  box-shadow: 0 0 18px rgba(255, 255, 255, 0.08);
+}
+
+/* Listening */
+#orb.listening {
+  background: radial-gradient(circle, #00ff99, #006644);
+  box-shadow: 0 0 70px rgba(0, 255, 153, 0.9);
+  animation: pulse 1.6s infinite;
+}
+
+/* Speaking */
+#orb.speaking {
+  background: radial-gradient(circle, #00e5ff, #003b44);
+  box-shadow: 0 0 90px rgba(0, 229, 255, 1);
+  animation: speak 0.35s infinite alternate;
+}
+
+/* Toggle button */
+#toggleBtn {
+  padding: 14px 32px;
+  font-size: 16px;
+  font-weight: 700;
+  border-radius: 999px;
+  border: none;
+  cursor: pointer;
+  background: linear-gradient(135deg, #2979ff, #00e5ff);
+  color: #001018;
+  letter-spacing: 0.6px;
+  box-shadow: 0 8px 30px rgba(0, 229, 255, 0.45);
+  transition: all 0.3s ease;
+}
+
+#toggleBtn:hover {
+  transform: translateY(-2px);
+  box-shadow: 0 12px 40px rgba(0, 229, 255, 0.7);
+}
+
+#toggleBtn.active {
+  background: linear-gradient(135deg, #ff1744, #ff616f);
+  color: white;
+  box-shadow: 0 8px 30px rgba(255, 23, 68, 0.5);
+}
+
+/* Status text */
+#status {
+  margin-top: 18px;
+  font-size: 15px;
+  color: #9aa4b2;
+  min-height: 20px;
+  text-align: center;
+}
+
+/* Animations */
+@keyframes pulse {
+  0% {
+    transform: scale(1);
+    opacity: 1;
+  }
+  50% {
+    transform: scale(1.08);
+    opacity: 0.85;
+  }
+  100% {
+    transform: scale(1);
+    opacity: 1;
+  }
+}
+
+@keyframes speak {
+  from {
+    transform: scale(1);
+  }
+  to {
+    transform: scale(1.05);
+  }
+}

package/dist/server/voice/turnManager.d.ts

@@ -0,0 +1,15 @@
+import type { FrameBus } from "./frameBus.js";
+export declare enum TurnState {
+    IDLE = 0,
+    USER_SPEAKING = 1,
+    PROCESSING = 2,
+    ASSISTANT_SPEAKING = 3
+}
+export declare class TurnManager {
+    state: TurnState;
+    constructor(bus: FrameBus);
+    private onVadStart;
+    private onVadStop;
+    assistantSpeaking(): void;
+    reset(): void;
+}

package/dist/server/voice/turnManager.js

@@ -0,0 +1,35 @@
+export var TurnState;
+(function (TurnState) {
+    TurnState[TurnState["IDLE"] = 0] = "IDLE";
+    TurnState[TurnState["USER_SPEAKING"] = 1] = "USER_SPEAKING";
+    TurnState[TurnState["PROCESSING"] = 2] = "PROCESSING";
+    TurnState[TurnState["ASSISTANT_SPEAKING"] = 3] = "ASSISTANT_SPEAKING";
+})(TurnState || (TurnState = {}));
+export class TurnManager {
+    state = TurnState.IDLE;
+    constructor(bus) {
+        bus.subscribe("vad_start", () => this.onVadStart());
+        bus.subscribe("vad_stop", () => this.onVadStop());
+    }
+    onVadStart() {
+        // Only update state if TTS is NOT playing. During ASSISTANT_SPEAKING, the
+        // barge-in interrupt is triggered by Soniox non-final tokens — which arrive
+        // after a network round-trip. If we let VAD immediately flip state to
+        // USER_SPEAKING, the state check in handleSonioxMessage fails and the
+        // interrupt never fires.
+        if (this.state !== TurnState.ASSISTANT_SPEAKING) {
+            this.state = TurnState.USER_SPEAKING;
+        }
+    }
+    onVadStop() {
+        if (this.state === TurnState.USER_SPEAKING) {
+            this.state = TurnState.PROCESSING;
+        }
+    }
+    assistantSpeaking() {
+        this.state = TurnState.ASSISTANT_SPEAKING;
+    }
+    reset() {
+        this.state = TurnState.IDLE;
+    }
+}

package/dist/server/voice/types.d.ts

@@ -0,0 +1,20 @@
+export type Frame = {
+    type: "audio";
+    data: Int16Array;
+} | {
+    type: "vad_start";
+} | {
+    type: "vad_stop";
+} | {
+    type: "transcript";
+    text: string;
+    final: boolean;
+} | {
+    type: "llm_token";
+    text: string;
+} | {
+    type: "tts_audio";
+    data: Buffer;
+} | {
+    type: "interrupt";
+};

package/dist/server/voice/types.js

@@ -0,0 +1 @@
+export {};

package/dist/server/voice/voiceServerApp.d.ts

@@ -0,0 +1 @@
+export declare function startVoiceServer(port?: number): Promise<void>;

package/dist/server/voice/voiceServerApp.js

@@ -0,0 +1,117 @@
+import express from "express";
+import fs from "fs";
+import http from "http";
+import path from "path";
+import { fileURLToPath } from "url";
+import { setupWebSocket } from "./voiceWebSocketHandler.js";
+import { NeuroLink } from "../../neurolink.js";
+import { logger } from "../../utils/logger.js";
+import { withTimeout } from "../../utils/async/withTimeout.js";
+import { getCartesiaWsUrl } from "../../adapters/tts/cartesiaHandler.js";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+/**
+ * Resolve the public/ directory containing static assets.
+ * The CLI build (tsc) only emits .ts → .js and does NOT copy non-TS assets,
+ * so __dirname/public may not exist when running from dist/.
+ * Fall back to the original source path in that case.
+ */
+function resolvePublicPath() {
+    const compiled = path.join(__dirname, "public");
+    if (fs.existsSync(compiled)) {
+        return compiled;
+    }
+    // Resolve from project root → src/lib/server/voice/public
+    const source = path.resolve(__dirname, "../../../../src/lib/server/voice/public");
+    if (fs.existsSync(source)) {
+        return source;
+    }
+    return compiled; // let express.static handle the 404
+}
+export async function startVoiceServer(port = 3000) {
+    const app = express();
+    /* ---------- STATIC FILES ---------- */
+    const publicPath = resolvePublicPath();
+    logger.info("[SERVER] Serving static from:", publicPath);
+    app.use(express.static(publicPath));
+    app.get("/", (_, res) => {
+        res.sendFile(path.join(publicPath, "index.html"));
+    });
+    /* ---------- HEALTH CHECK ---------- */
+    app.get("/health", (_, res) => {
+        res.json({ status: "ok" });
+    });
+    const server = http.createServer(app);
+    /* ---------- WS ---------- */
+    setupWebSocket(server);
+    /* ---------- START ---------- */
+    await new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(port, () => {
+            server.removeListener("error", reject);
+            logger.info(`[SERVER] Voice server running at http://localhost:${port}`);
+            resolve();
+        });
+    });
+    /* ---------- WARMUP ---------- */
+    // Pre-warm NeuroLink + Azure on startup so the first real user request isn't
+    // slow. NeuroLink's MCP init + Azure's connection pool both have cold-start
+    // overhead that shows up as 3-4s on the very first call. We also open and
+    // immediately close a Cartesia WS to prime the TLS handshake.
+    warmup().catch((err) => {
+        logger.warn("[WARMUP] Failed (non-fatal):", err.message);
+    });
+}
+async function warmup() {
+    const t = Date.now();
+    logger.info("[WARMUP] Warming up LLM + TTS...");
+    const neurolink = new NeuroLink();
+    const provider = process.env.VOICE_LLM_PROVIDER ?? "azure";
+    const model = process.env.VOICE_LLM_MODEL ?? "gpt-4o-automatic";
+    try {
+        const result = await withTimeout(neurolink.stream({
+            provider,
+            model,
+            input: { text: "hi" },
+            maxTokens: 3,
+            disableTools: true,
+            enableAnalytics: false,
+            enableEvaluation: false,
+        }), 15000, "LLM warmup timed out");
+        // Drain the stream so the connection is fully exercised.
+        for await (const _chunk of result.stream) {
+            /* drain */
+        }
+        logger.info(`[WARMUP] LLM warmup done in ${Date.now() - t}ms`);
+    }
+    catch (err) {
+        logger.warn("[WARMUP] LLM warmup failed (non-fatal):", err.message);
+    }
+    // Cartesia TLS warmup — open WS, wait for connect, then close.
+    try {
+        const { default: WebSocket } = await import("ws");
+        const apiKey = process.env.CARTESIA_API_KEY;
+        await new Promise((resolve) => {
+            const ws = new WebSocket(getCartesiaWsUrl(), {
+                headers: apiKey ? { "X-API-Key": apiKey } : undefined,
+            });
+            const timeout = setTimeout(() => {
+                ws.terminate();
+                resolve(); // non-fatal, just move on
+            }, 5000);
+            ws.once("open", () => {
+                clearTimeout(timeout);
+                ws.close();
+                resolve();
+            });
+            ws.once("error", () => {
+                clearTimeout(timeout);
+                resolve(); // non-fatal
+            });
+        });
+        logger.info(`[WARMUP] Cartesia warmup done in ${Date.now() - t}ms`);
+    }
+    catch {
+        // non-fatal
+    }
+}

package/dist/server/voice/voiceWebSocketHandler.d.ts

@@ -0,0 +1,11 @@
+import type { Server as HttpServer } from "http";
+/**
+ * Call from the voice-server command handler BEFORE importing anything else
+ * so the env change is scoped to voice mode only.
+ */
+export declare function configureVoiceServerEnvironment(): void;
+export type Message = {
+    role: "system" | "user" | "assistant";
+    content: string;
+};
+export declare function setupWebSocket(server: HttpServer): void;