@lokutor/sdk 1.1.42 → 1.2.1

package/dist/index.mjs

@@ -60,7 +60,8 @@ var AUDIO_CONFIG = {
 };
 var DEFAULT_URLS = {
   VOICE_AGENT: "wss://api.lokutor.com/ws/agent",
-  TTS: "wss://api.lokutor.com/ws/tts"
+  TTS: "wss://api.lokutor.com/ws/tts",
+  STT: "wss://api.lokutor.com/ws/stt"
 };
 var LokutorError = class extends Error {
   code;
@@ -240,6 +241,10 @@ var BrowserAudioManager = class {
   mediaStreamAudioSourceNode = null;
   scriptProcessor = null;
   analyserNode = null;
+  // Reused across getAmplitude() calls instead of allocating a new
+  // Uint8Array on every call — this runs once per animation frame (~60/sec)
+  // from the visualizer, on the same main thread as audio capture.
+  amplitudeBuffer = null;
   mediaStream = null;
   resampler = null;
   // Playback scheduling
@@ -306,7 +311,7 @@ var BrowserAudioManager = class {
         }
       });
       this.mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.mediaStream);
-      const bufferSize = 4096;
+      const bufferSize = 1024;
       this.scriptProcessor = this.audioContext.createScriptProcessor(
         bufferSize,
         1,
@@ -340,7 +345,6 @@ var BrowserAudioManager = class {
    */
   _processAudioInput(event) {
     if (!this.onAudioInput || !this.audioContext || !this.isListening) return;
-    if (this.isMuted) return;
     const inputBuffer = event.inputBuffer;
     const inputData = inputBuffer.getChannelData(0);
     const outputBuffer = event.outputBuffer;
@@ -352,6 +356,9 @@ var BrowserAudioManager = class {
       processedData = this.resampler.process(processedData);
     }
     if (processedData.length === 0) return;
+    if (this.isMuted) {
+      processedData = new Float32Array(processedData.length);
+    }
     const int16Data = float32ToPcm16(processedData);
     const uint8Data = new Uint8Array(
       int16Data.buffer,
@@ -475,9 +482,11 @@ var BrowserAudioManager = class {
    */
   getAmplitude() {
     if (!this.analyserNode) return 0;
-    const dataArray = new Uint8Array(this.analyserNode.frequencyBinCount);
-    this.analyserNode.getByteTimeDomainData(dataArray);
-    const rms = calculateRMS(dataArray);
+    if (!this.amplitudeBuffer || this.amplitudeBuffer.length !== this.analyserNode.frequencyBinCount) {
+      this.amplitudeBuffer = new Uint8Array(this.analyserNode.frequencyBinCount);
+    }
+    this.analyserNode.getByteTimeDomainData(this.amplitudeBuffer);
+    const rms = calculateRMS(this.amplitudeBuffer);
     return Math.min(rms * 10, 1);
   }
   /**
@@ -584,6 +593,14 @@ function base64ToUint8Array(base64) {
   }
   return bytes;
 }
+function uint8ArrayToBase64(bytes) {
+  let binaryString = "";
+  const chunkSize = 32768;
+  for (let i = 0; i < bytes.length; i += chunkSize) {
+    binaryString += String.fromCharCode(...bytes.subarray(i, i + chunkSize));
+  }
+  return btoa(binaryString);
+}
 function normalizeVisemes(payload) {
   if (!Array.isArray(payload)) return [];
   const normalized = [];
@@ -620,6 +637,7 @@ function extractVisemePayload(msg) {
 var VoiceAgentClient = class {
   ws = null;
   apiKey;
+  agentId = "";
   prompt;
   voice;
   language;
@@ -652,6 +670,7 @@ var VoiceAgentClient = class {
   serverUrl;
   constructor(config) {
     this.apiKey = config.apiKey;
+    this.agentId = config.agentId || "";
     this.prompt = config.prompt;
     this.voice = config.voice || "F1" /* F1 */;
     this.language = config.language || "en" /* ENGLISH */;
@@ -696,6 +715,10 @@ var VoiceAgentClient = class {
           const separator = url.includes("?") ? "&" : "?";
           url += `${separator}api_key=${this.apiKey}`;
         }
+        if (this.agentId) {
+          const separator = url.includes("?") ? "&" : "?";
+          url += `${separator}agent_id=${this.agentId}`;
+        }
         const redactedUrl = url.replace(/api_key=[^&]+/, "api_key=***");
         sdkTrace("ws.connect", {
           endpoint: this.serverUrl,
@@ -1315,6 +1338,164 @@ var TTSClient = class {
     });
   }
 };
+var STTClient = class {
+  apiKey;
+  baseUrl;
+  constructor(config) {
+    this.apiKey = config.apiKey;
+    this.baseUrl = wsToHttp(config.serverUrl || DEFAULT_URLS.STT).replace(/\/ws\/stt\/?$/, "");
+  }
+  /**
+   * Transcribe a complete audio clip. Pass a Blob/File (e.g. from a file
+   * input or MediaRecorder) for WAV/compressed audio, or a raw PCM16
+   * buffer with `format: "pcm16"` and `sampleRate` set.
+   */
+  async transcribe(options) {
+    const url = `${this.baseUrl}/stt/transcribe`;
+    let res;
+    if (typeof Blob !== "undefined" && options.audio instanceof Blob) {
+      const form = new FormData();
+      form.append("audio", options.audio, "audio.wav");
+      if (options.language) form.append("lang", options.language);
+      if (options.sampleRate) form.append("sample_rate", String(options.sampleRate));
+      res = await fetch(url, {
+        method: "POST",
+        headers: { "X-API-Key": this.apiKey },
+        body: form
+      });
+    } else {
+      const bytes = options.audio instanceof Uint8Array ? options.audio : new Uint8Array(options.audio);
+      res = await fetch(url, {
+        method: "POST",
+        headers: { "X-API-Key": this.apiKey, "Content-Type": "application/json" },
+        body: JSON.stringify({
+          audio: uint8ArrayToBase64(bytes),
+          format: options.format || "pcm16",
+          sample_rate: options.sampleRate,
+          lang: options.language
+        })
+      });
+    }
+    if (!res.ok) {
+      const detail = await res.text().catch(() => "");
+      throw new LokutorError("internal.error", `HTTP ${res.status} from ${url}`, {
+        detail,
+        retryable: res.status >= 500
+      });
+    }
+    const data = await res.json();
+    return {
+      text: data.text ?? "",
+      latencyMs: data.latency_ms ?? 0,
+      engine: data.engine ?? "",
+      sampleRate: data.sample_rate ?? 0,
+      language: data.language ?? "",
+      durationSeconds: data.duration_seconds ?? 0,
+      segments: data.segments
+    };
+  }
+};
+var SpeechToTextClient = class {
+  apiKey;
+  serverUrl;
+  language;
+  vad;
+  onPartialTranscript;
+  onFinalTranscript;
+  onError;
+  onStatusChange;
+  ws = null;
+  audioManager = null;
+  isConnected = false;
+  constructor(config) {
+    this.apiKey = config.apiKey;
+    this.serverUrl = config.serverUrl || DEFAULT_URLS.STT;
+    this.language = config.language;
+    this.vad = config.vad || "silero";
+    this.onPartialTranscript = config.onPartialTranscript;
+    this.onFinalTranscript = config.onFinalTranscript;
+    this.onError = config.onError;
+    this.onStatusChange = config.onStatusChange;
+  }
+  /**
+   * Connect and start streaming microphone audio.
+   * @param customAudioManager Optional replacement for the default audio hardware handler (e.g. NodeAudioManager for CLI use)
+   */
+  async connect(customAudioManager) {
+    this.audioManager = customAudioManager || (typeof window !== "undefined" ? new BrowserAudioManager() : null);
+    if (!this.audioManager) {
+      throw new LokutorError("internal.error", "No audio manager available \u2014 pass one explicitly outside the browser (e.g. NodeAudioManager).");
+    }
+    await this.audioManager.init();
+    this.onStatusChange?.("connecting");
+    return new Promise((resolve, reject) => {
+      let settled = false;
+      const settle = (fn) => {
+        if (!settled) {
+          settled = true;
+          fn();
+        }
+      };
+      try {
+        let url = this.serverUrl;
+        const separator = url.includes("?") ? "&" : "?";
+        url += `${separator}api_key=${this.apiKey}`;
+        this.ws = new WebSocket(url);
+        this.ws.onopen = async () => {
+          this.isConnected = true;
+          this.onStatusChange?.("connected");
+          this.ws.send(JSON.stringify({ lang: this.language || "en" /* ENGLISH */, vad: this.vad }));
+          await this.audioManager.startMicrophone((data) => {
+            if (this.isConnected && this.ws?.readyState === WebSocket.OPEN) {
+              this.ws.send(data);
+            }
+          });
+          settle(() => resolve(true));
+        };
+        this.ws.onmessage = (event) => {
+          if (typeof event.data !== "string") return;
+          try {
+            const msg = JSON.parse(event.data);
+            if (msg.type === "transcript") {
+              if (msg.isFinal) {
+                this.onFinalTranscript?.(msg.data ?? "");
+              } else {
+                this.onPartialTranscript?.(msg.data ?? "");
+              }
+            } else if (msg.type === "error") {
+              this.onError?.(new LokutorError("internal.error", msg.data ?? "STT error"));
+            }
+          } catch {
+          }
+        };
+        this.ws.onerror = (err) => {
+          const lokutorErr = new LokutorError("internal.error", "WebSocket error", { original: err });
+          this.onError?.(lokutorErr);
+          settle(() => reject(lokutorErr));
+        };
+        this.ws.onclose = () => {
+          this.isConnected = false;
+          this.onStatusChange?.("disconnected");
+        };
+      } catch (err) {
+        settle(() => reject(err));
+      }
+    });
+  }
+  /** Force-finalize whatever utterance is currently in progress. */
+  endUtterance() {
+    if (this.ws?.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify({ type: "end" }));
+    }
+  }
+  disconnect() {
+    this.isConnected = false;
+    this.audioManager?.stopMicrophone();
+    this.audioManager?.cleanup();
+    this.ws?.close();
+    this.ws = null;
+  }
+};
 async function simpleConversation(config) {
   const client = new VoiceAgentClient(config);
   await client.connect();
@@ -1324,6 +1505,102 @@ async function simpleTTS(options) {
   const client = new TTSClient({ apiKey: options.apiKey });
   return client.synthesize(options);
 }
+async function simpleTranscribe(options) {
+  const client = new STTClient({ apiKey: options.apiKey, serverUrl: options.serverUrl });
+  return client.transcribe(options);
+}
+
+// src/node-audio.ts
+function optionalImport(moduleName) {
+  return import(moduleName);
+}
+var NodeAudioManager = class {
+  speaker = null;
+  recorder = null;
+  recordingStream = null;
+  isMuted = false;
+  isListening = false;
+  constructor() {
+  }
+  async init() {
+    try {
+      const Speaker = await optionalImport("speaker").catch(() => null);
+      if (!Speaker) {
+        console.warn('\u26A0\uFE0F  Package "speaker" is missing. Hardware output will be disabled.');
+        console.warn("\u{1F449} Run: npm install speaker");
+      }
+    } catch (e) {
+      console.error("Error initializing Node audio:", e);
+    }
+  }
+  async startMicrophone(onAudioInput) {
+    if (this.isListening) return;
+    try {
+      const recorder = await optionalImport("node-record-lpcm16").catch(() => null);
+      if (!recorder) {
+        throw new Error('Package "node-record-lpcm16" is missing. Microphone input failed.\n\u{1F449} Run: npm install node-record-lpcm16');
+      }
+      console.log("\u{1F3A4} Starting microphone (Node.js)...");
+      this.recordingStream = recorder.record({
+        sampleRate: AUDIO_CONFIG.SAMPLE_RATE,
+        threshold: 0,
+        verbose: false,
+        recordProgram: "sox"
+        // default
+      });
+      this.recordingStream.stream().on("data", (chunk) => {
+        if (!this.isMuted && onAudioInput) {
+          onAudioInput(new Uint8Array(chunk));
+        }
+      });
+      this.isListening = true;
+    } catch (e) {
+      console.error("Failed to start microphone:", e.message);
+      throw e;
+    }
+  }
+  stopMicrophone() {
+    if (this.recordingStream) {
+      this.recordingStream.stop();
+      this.recordingStream = null;
+    }
+    this.isListening = false;
+  }
+  async playAudio(pcm16Data) {
+    try {
+      if (!this.speaker) {
+        const Speaker = (await optionalImport("speaker")).default;
+        this.speaker = new Speaker({
+          channels: AUDIO_CONFIG.CHANNELS,
+          bitDepth: 16,
+          sampleRate: AUDIO_CONFIG.SPEAKER_SAMPLE_RATE
+        });
+      }
+      this.speaker.write(Buffer.from(pcm16Data));
+    } catch (e) {
+      console.error("NodeAudioManager: speaker playback failed:", e);
+    }
+  }
+  stopPlayback() {
+    if (this.speaker) {
+      this.speaker.end();
+      this.speaker = null;
+    }
+  }
+  cleanup() {
+    this.stopMicrophone();
+    this.stopPlayback();
+  }
+  isMicMuted() {
+    return this.isMuted;
+  }
+  setMuted(muted) {
+    this.isMuted = muted;
+  }
+  getAmplitude() {
+    return 0;
+  }
+};
 
 // src/conversational-panel.ts
 var PANEL_CSS = (
@@ -1419,7 +1696,7 @@ var PANEL_CSS = (
   }
   .cv-curtain-btn svg { transition: transform 0.3s ease; }
   .cv-curtain-btn:hover svg { transform: translateX(4px); }
-  .cv-header {
+  .cv-error {
     display: flex;
     flex-direction: column;
     align-items: center;
@@ -1545,12 +1822,47 @@ var PANEL_CSS = (
   }
   .cv-error.is-visible { display: flex; }
   .cv-error-icon { color: var(--cv-accent); flex-shrink: 0; }
+  .cv-voice-picker {
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: center;
+    gap: 0.35rem;
+    z-index: 2;
+  }
+  .cv-voice-btn {
+    padding: 0.25rem 0.5rem;
+    border-radius: 100px;
+    border: 1px solid rgba(255,255,255,0.15);
+    background: rgba(255,255,255,0.05);
+    color: rgba(255,255,255,0.6);
+    font-size: 0.65rem;
+    font-weight: 600;
+    cursor: pointer;
+    transition: all 0.2s ease;
+  }
+  .cv-voice-btn:hover {
+    background: rgba(255,255,255,0.12);
+    color: #fff;
+  }
+  .cv-voice-btn.is-selected {
+    background: #fff;
+    color: #000;
+    border-color: #fff;
+  }
+  .cv-voice-label {
+    font-size: 0.55rem;
+    text-transform: uppercase;
+    letter-spacing: 0.08em;
+    color: rgba(255,255,255,0.35);
+    z-index: 2;
+  }
   
   /* === COMPACT MODE: < 300px width === */
   @container (max-width: 299px) {
     .cv-curtain-content { padding: 0.5rem; gap: 0.3rem; }
     .cv-curtain-title { font-size: 0.8rem; }
     .cv-curtain-desc { display: none; }
+    .cv-voice-picker { display: none; }
     .cv-curtain-btn { padding: 0.3rem 0.6rem; font-size: 0.6rem; gap: 0.3rem; }
     .cv-curtain-btn svg { display: none; }
     .cv-visualizer-wrap { width: 60px; height: 60px; }
@@ -1635,11 +1947,13 @@ var ConversationalPanel = class {
   isRunning = false;
   _locked = false;
   _lastSpeechTime = 0;
-  // Cached DOM refs
+  selectedVoice = "M1";
   el;
   curtain;
   curtainTitle;
   curtainDesc;
+  curtainBg;
+  curtainOverlay;
   startBtn;
   errorEl;
   errorText;
@@ -1659,6 +1973,7 @@ var ConversationalPanel = class {
   constructor(cfg) {
     this.cfg = cfg;
     this.container = cfg.container;
+    this.selectedVoice = cfg.voice || "M1";
     injectStyles();
     this.buildDOM();
   }
@@ -1674,9 +1989,9 @@ var ConversationalPanel = class {
         <div class="cv-curtain-bg"></div>
         <div class="cv-curtain-overlay"></div>
         <div class="cv-curtain-content">
-          <h3 class="cv-curtain-title">${this.esc(this.cfg.title)}</h3>
-          <p class="cv-curtain-desc">${this.esc(this.cfg.description)}</p>
-          <button class="cv-curtain-btn">
+          <h3 class="cv-curtain-title">${this.esc(this.cfg.title || "Voice Chat")}</h3>
+          <p class="cv-curtain-desc">${this.esc(this.cfg.description || "")}</p>
+           <button class="cv-curtain-btn">
             <span>Start Conversation</span>
             <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" width="20" height="20">
               <path d="M5 12h14M12 5l7 7-7 7"/>
@@ -1694,7 +2009,7 @@ var ConversationalPanel = class {
         </div>
       </div>
       <div class="cv-header">
-        <h2 class="cv-title">${this.esc(this.cfg.title)} <span class="cv-timer">00:00</span></h2>
+        <h2 class="cv-title">${this.esc(this.cfg.title || "Voice Chat")} <span class="cv-timer">00:00</span></h2>
       </div>
       <div class="cv-visualizer-wrap">
         <canvas class="cv-canvas"></canvas>
@@ -1772,7 +2087,7 @@ var ConversationalPanel = class {
         apiKey: this.cfg.apiKey,
         tools: this.cfg.tools,
         prompt: this.cfg.prompt,
-        voice: this.cfg.voice || "M1",
+        voice: this.selectedVoice,
         language: this.cfg.language || "en",
         onStatusChange: (status) => {
           this.el.classList.remove("cv-is-speaking", "cv-is-thinking");
@@ -1780,6 +2095,7 @@ var ConversationalPanel = class {
           else if (status === "thinking") this.el.classList.add("cv-is-thinking");
         },
         onTranscription: (text) => {
+          this._lastSpeechTime = Date.now();
           this.onTranscription?.(text);
         },
         onResponse: (text) => {
@@ -1904,6 +2220,7 @@ var ConversationalPanel = class {
     const h2 = this.el.querySelector(".cv-title");
     if (h2) h2.innerHTML = `${this.esc(title)} <span class="cv-timer">${this.timerEl?.textContent || "00:00"}</span>`;
   }
+  /** Select a voice from the picker */
   /** Update description text */
   setDescription(desc) {
     this.cfg.description = desc;
@@ -1939,7 +2256,7 @@ var ConversationalPanel = class {
   startTimer() {
     let elapsed = 0;
     const maxDur = this.cfg.maxDuration || 300;
-    const silentMax = this.cfg.silenceTimeout || 30;
+    const silentMax = this.cfg.silenceTimeout || 60;
     this._lastSpeechTime = Date.now();
     this.timerEl.textContent = this.fmt(maxDur);
     this.timerTicker = window.setInterval(() => {
@@ -2033,6 +2350,9 @@ export {
   DEFAULT_URLS,
   Language,
   LokutorError,
+  NodeAudioManager,
+  STTClient,
+  SpeechToTextClient,
   StreamResampler,
   TTSClient,
   VoiceAgentClient,
@@ -2048,5 +2368,6 @@ export {
   resample,
   resampleWithAntiAliasing,
   simpleConversation,
-  simpleTTS
+  simpleTTS,
+  simpleTranscribe
 };

package/package.json

@@ -1,7 +1,6 @@
 {
   "name": "@lokutor/sdk",
-
-  "version": "1.1.42",
+  "version": "1.2.1",
   "description": "JavaScript/TypeScript SDK for Lokutor Real-time Voice AI",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",

package/src/browser-audio.ts

@@ -30,6 +30,10 @@ export class BrowserAudioManager {
   private mediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null;
   private scriptProcessor: ScriptProcessorNode | null = null;
   private analyserNode: AnalyserNode | null = null;
+  // Reused across getAmplitude() calls instead of allocating a new
+  // Uint8Array on every call — this runs once per animation frame (~60/sec)
+  // from the visualizer, on the same main thread as audio capture.
+  private amplitudeBuffer: Uint8Array<ArrayBuffer> | null = null;
   private mediaStream: MediaStream | null = null;
   private resampler: StreamResampler | null = null;
 
@@ -123,7 +127,17 @@ export class BrowserAudioManager {
       // Create script processor for PCM extraction
       // Note: ScriptProcessorNode is deprecated but widely supported.
       // AudioWorklet would be better but requires additional setup.
-      const bufferSize = 4096;
+      //
+      // 1024 samples is ~21ms at a typical 48kHz hardware rate — matched to
+      // AUDIO_CONFIG.CHUNK_DURATION_MS (20ms), the granularity the backend's
+      // VAD is tuned around. The previous 4096 was ~85ms of audio buffered
+      // before a single byte reached the server: on top of adding raw
+      // latency, ScriptProcessorNode callbacks run on the main thread, so
+      // that much buffering meant audio could arrive in large, uneven
+      // bursts whenever the main thread was briefly busy (a re-render, the
+      // visualizer) instead of a steady stream — exactly the kind of
+      // mistimed input that can trip server-side VAD into a false barge-in.
+      const bufferSize = 1024;
       this.scriptProcessor = this.audioContext!.createScriptProcessor(
         bufferSize,
         1, // input channels
@@ -165,7 +179,6 @@ export class BrowserAudioManager {
    */
   private _processAudioInput(event: AudioProcessingEvent): void {
     if (!this.onAudioInput || !this.audioContext || !this.isListening) return;
-    if (this.isMuted) return;
 
     const inputBuffer = event.inputBuffer;
     const inputData = inputBuffer.getChannelData(0);
@@ -185,6 +198,18 @@ export class BrowserAudioManager {
 
     if (processedData.length === 0) return; // Need more data for resampler
 
+    // While muted, keep sending — silent — chunks instead of sending
+    // nothing at all (the old `if (this.isMuted) return` above this).
+    // The server's turn-taking/VAD is purely reactive to incoming chunks:
+    // if the client stops sending entirely mid-utterance, the server never
+    // observes the silence it needs to close out the turn, so muting
+    // mid-sentence left the conversation stuck instead of handing off to
+    // the agent. Explicitly zeroed here rather than relying solely on the
+    // disabled MediaStreamTrack to already read as silence.
+    if (this.isMuted) {
+      processedData = new Float32Array(processedData.length);
+    }
+
     // Convert Float32 to Int16 PCM
     const int16Data = float32ToPcm16(processedData);
     const uint8Data = new Uint8Array(
@@ -339,10 +364,12 @@ export class BrowserAudioManager {
   getAmplitude(): number {
     if (!this.analyserNode) return 0;
 
-    const dataArray = new Uint8Array(this.analyserNode.frequencyBinCount);
-    this.analyserNode.getByteTimeDomainData(dataArray);
+    if (!this.amplitudeBuffer || this.amplitudeBuffer.length !== this.analyserNode.frequencyBinCount) {
+      this.amplitudeBuffer = new Uint8Array(this.analyserNode.frequencyBinCount);
+    }
+    this.analyserNode.getByteTimeDomainData(this.amplitudeBuffer);
 
-    const rms = calculateRMS(dataArray);
+    const rms = calculateRMS(this.amplitudeBuffer);
     return Math.min(rms * 10, 1); // Boost for visualization
   }