@keyframelabs/elements 0.2.0 → 0.2.1

package/dist/agents/elevenlabs.d.ts

@@ -20,6 +20,10 @@ export declare class ElevenLabsAgent extends BaseAgent {
     private sourceInputSampleRate;
     private initialized;
     private lastInterruptId;
+    private agentResponseReceived;
+    private turnStartTime;
+    private accumulatedDurationMs;
+    private turnEndTimer;
     connect(config: ElevenLabsConfig): Promise<void>;
     protected handleParsedMessage(message: unknown): void;
     private handleInitMetadata;
@@ -27,6 +31,13 @@ export declare class ElevenLabsAgent extends BaseAgent {
     private handleAudio;
     private handleUserTranscript;
     private handleAgentResponse;
+    /**
+     * Schedule a timer to emit turnEnd when the virtual audio buffer
+     * "would have" finished playing. Replicates the ElevenLabs SDK's
+     * AudioWorklet buffer-empty detection without requiring local playback.
+     */
+    private scheduleVirtualBufferCheck;
+    private resetTurnState;
     private handleClientToolCall;
     private handleInterruption;
     sendAudio(pcmData: Uint8Array): void;

package/dist/index.js

@@ -1,12 +1,12 @@
 import { createClient as f } from "@keyframelabs/sdk";
-const l = 24e3;
-function g(i) {
+const o = 24e3;
+function m(i) {
   const e = atob(i), t = new Uint8Array(e.length);
   for (let s = 0; s < e.length; s++)
     t[s] = e.charCodeAt(s);
   return t;
 }
-function m(i) {
+function g(i) {
   let e = "";
   for (let t = 0; t < i.length; t++)
     e += String.fromCharCode(i[t]);
@@ -16,9 +16,9 @@ function h(i, e, t) {
   if (e === t)
     return i;
   const s = new Int16Array(i.buffer, i.byteOffset, i.length / 2), n = e / t, a = Math.floor(s.length / n), d = new Int16Array(a);
-  for (let o = 0; o < a; o++) {
-    const S = o * n, p = Math.floor(S), b = Math.min(p + 1, s.length - 1), v = S - p;
-    d[o] = Math.round(
+  for (let r = 0; r < a; r++) {
+    const _ = r * n, p = Math.floor(_), b = Math.min(p + 1, s.length - 1), v = _ - p;
+    d[r] = Math.round(
       s[p] * (1 - v) + s[b] * v
     );
   }
@@ -113,7 +113,7 @@ class u {
     this.events.emit("closed", { code: e, reason: t });
   }
 }
-const A = "gemini-2.5-flash-native-audio-preview-12-2025", I = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", k = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
+const k = "gemini-2.5-flash-native-audio-preview-12-2025", A = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", I = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
 class R extends u {
   agentName = "GeminiLive";
   async connect(e) {
@@ -122,10 +122,10 @@ class R extends u {
     if (!e.apiKey)
       throw new Error("Gemini API key is required");
     e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate);
-    const t = e.model ?? A, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${k}?access_token=${encodeURIComponent(e.apiKey)}` : `${I}?key=${encodeURIComponent(e.apiKey)}`;
+    const t = e.model ?? k, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${I}?access_token=${encodeURIComponent(e.apiKey)}` : `${A}?key=${encodeURIComponent(e.apiKey)}`;
     return new Promise((a, d) => {
       this.ws = new WebSocket(n), this.ws.onopen = () => {
-        const o = {
+        const r = {
           setup: {
             model: `models/${t}`,
             generationConfig: {
@@ -134,13 +134,13 @@ class R extends u {
             systemInstruction: e.systemPrompt ? { parts: [{ text: e.systemPrompt }] } : void 0
           }
         };
-        this.ws.send(JSON.stringify(o)), this.setState("listening"), a();
+        this.ws.send(JSON.stringify(r)), this.setState("listening"), a();
       }, this.ws.onerror = () => {
         d(new Error("Failed to connect to Gemini Live"));
-      }, this.ws.onclose = (o) => {
-        this.ws = null, this.setState("idle"), this.emitClosed(o.code, o.reason);
-      }, this.ws.onmessage = (o) => {
-        this.handleMessage(o.data);
+      }, this.ws.onclose = (r) => {
+        this.ws = null, this.setState("idle"), this.emitClosed(r.code, r.reason);
+      }, this.ws.onmessage = (r) => {
+        this.handleMessage(r.data);
       };
     });
   }
@@ -159,7 +159,7 @@ class R extends u {
         this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
         for (const n of s.modelTurn.parts) {
           if (n.inlineData?.data) {
-            const a = g(n.inlineData.data);
+            const a = m(n.inlineData.data);
             this.events.emit("audio", a);
           }
           n.text && this.events.emit("transcript", {
@@ -181,7 +181,7 @@ class R extends u {
         mediaChunks: [
           {
             mimeType: `audio/pcm;rate=${this.inputSampleRate}`,
-            data: m(e)
+            data: g(e)
           }
         ]
       }
@@ -190,7 +190,7 @@ class R extends u {
   }
 }
 const M = ["neutral", "angry", "sad", "happy"], T = "wss://api.elevenlabs.io/v1/convai/conversation";
-class _ extends u {
+class S extends u {
   agentName = "ElevenLabs";
   outputSampleRate = 24e3;
   // Default, updated from metadata
@@ -202,6 +202,12 @@ class _ extends u {
   // True after conversation_initiation_metadata received
   lastInterruptId = 0;
   // Track interruptions to filter stale audio
+  // Virtual buffer turn-end detection: track audio duration and emit turnEnd
+  // when agent_response has arrived and all audio "would have" finished playing.
+  agentResponseReceived = !1;
+  turnStartTime = 0;
+  accumulatedDurationMs = 0;
+  turnEndTimer = null;
   async connect(e) {
     if (this.ws)
       throw new Error("Already connected");
@@ -275,8 +281,12 @@ class _ extends u {
     if (!t?.audio_base_64 || (t.event_id ?? 0) <= this.lastInterruptId)
       return;
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    let n = g(t.audio_base_64);
-    this.outputSampleRate !== l && (n = h(n, this.outputSampleRate, l)), this.events.emit("audio", n);
+    let n = m(t.audio_base_64);
+    this.outputSampleRate !== o && (n = h(n, this.outputSampleRate, o)), this.events.emit("audio", n);
+    const a = n.length / 2 / o * 1e3;
+    this.turnStartTime === 0 && (this.turnStartTime = Date.now()), this.accumulatedDurationMs += a, console.debug(
+      `[ElevenLabs] audio chunk: ${n.length} bytes, +${a.toFixed(0)}ms, totalDuration=${this.accumulatedDurationMs.toFixed(0)}ms, agentResponse=${this.agentResponseReceived}`
+    ), this.scheduleVirtualBufferCheck();
   }
   handleUserTranscript(e) {
     const t = e.user_transcription_event;
@@ -288,11 +298,31 @@ class _ extends u {
   }
   handleAgentResponse(e) {
     const t = e.agent_response_event;
-    t?.agent_response && (this.events.emit("turnEnd", void 0), this.setState("listening"), this.events.emit("transcript", {
+    t?.agent_response && (this.events.emit("transcript", {
       role: "assistant",
       text: t.agent_response,
       isFinal: !0
-    }));
+    }), this.agentResponseReceived = !0, console.debug(
+      `[ElevenLabs] agent_response received: totalDuration=${this.accumulatedDurationMs.toFixed(0)}ms, text="${t.agent_response.slice(0, 60)}${t.agent_response.length > 60 ? "..." : ""}"`
+    ), this.scheduleVirtualBufferCheck());
+  }
+  /**
+   * Schedule a timer to emit turnEnd when the virtual audio buffer
+   * "would have" finished playing. Replicates the ElevenLabs SDK's
+   * AudioWorklet buffer-empty detection without requiring local playback.
+   */
+  scheduleVirtualBufferCheck() {
+    if (!this.agentResponseReceived || this.turnStartTime === 0) return;
+    this.turnEndTimer !== null && clearTimeout(this.turnEndTimer);
+    const e = Date.now() - this.turnStartTime, t = Math.max(0, this.accumulatedDurationMs - e);
+    console.debug(
+      `[ElevenLabs] virtual buffer: elapsed=${e.toFixed(0)}ms, accumulated=${this.accumulatedDurationMs.toFixed(0)}ms, remaining=${t.toFixed(0)}ms`
+    ), this.turnEndTimer = setTimeout(() => {
+      this.turnEndTimer = null, console.debug("[ElevenLabs] virtual buffer drained, emitting turnEnd"), this.resetTurnState(), this.events.emit("turnEnd", void 0), this.setState("listening");
+    }, t);
+  }
+  resetTurnState() {
+    this.agentResponseReceived = !1, this.turnStartTime = 0, this.accumulatedDurationMs = 0, this.turnEndTimer !== null && (clearTimeout(this.turnEndTimer), this.turnEndTimer = null);
   }
   handleClientToolCall(e) {
     const t = e.client_tool_call;
@@ -311,14 +341,16 @@ class _ extends u {
   }
   handleInterruption(e) {
     const t = e.interruption_event;
-    t?.event_id && (this.lastInterruptId = t.event_id), this.events.emit("interrupted", void 0), this.setState("listening");
+    t?.event_id && (this.lastInterruptId = t.event_id), (this.agentResponseReceived || this.accumulatedDurationMs > 0) && console.debug(
+      `[ElevenLabs] interruption: discarding pending turn (duration=${this.accumulatedDurationMs.toFixed(0)}ms, agentResponse=${this.agentResponseReceived})`
+    ), this.resetTurnState(), this.events.emit("interrupted", void 0), this.setState("listening");
   }
   sendAudio(e) {
     if (!this.ws || this.ws.readyState !== WebSocket.OPEN || !this.initialized)
       return;
     let t = e;
     this.sourceInputSampleRate !== this.expectedInputSampleRate && (t = h(e, this.sourceInputSampleRate, this.expectedInputSampleRate)), this.ws.send(JSON.stringify({
-      user_audio_chunk: m(t)
+      user_audio_chunk: g(t)
     }));
   }
   /**
@@ -348,11 +380,11 @@ class _ extends u {
     }));
   }
   close() {
-    this.initialized = !1, this.lastInterruptId = 0, super.close();
+    this.initialized = !1, this.lastInterruptId = 0, this.resetTurnState(), super.close();
   }
 }
-const P = "wss://api.cartesia.ai/agents/stream", x = "2025-04-16";
-class O extends u {
+const x = "wss://api.cartesia.ai/agents/stream", P = "2025-04-16";
+class D extends u {
   agentName = "Cartesia";
   // Audio configuration
   cartesiaInputFormat = "pcm_16000";
@@ -371,7 +403,7 @@ class O extends u {
     if (!e.apiKey)
       throw new Error("Cartesia API Key is required");
     e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate), this.inputSampleRate === 16e3 ? this.cartesiaInputFormat = "pcm_16000" : this.inputSampleRate === 24e3 ? this.cartesiaInputFormat = "pcm_24000" : this.inputSampleRate === 44100 ? this.cartesiaInputFormat = "pcm_44100" : this.cartesiaInputFormat = "pcm_16000";
-    const t = `${P}/${e.agentId}?api_key=${e.apiKey}&cartesia_version=${x}`;
+    const t = `${x}/${e.agentId}?api_key=${e.apiKey}&cartesia_version=${P}`;
     return new Promise((s, n) => {
       this.ws = new WebSocket(t), this.ws.onopen = () => {
         this.sendStartEvent(), this.startHeartbeat(), s();
@@ -433,8 +465,8 @@ class O extends u {
   handleMediaOutput(e) {
     if (!e.media?.payload) return;
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    let t = g(e.media.payload);
-    this.cartesiaOutputRate !== l && (t = h(t, this.cartesiaOutputRate, l)), this.events.emit("audio", t);
+    let t = m(e.media.payload);
+    this.cartesiaOutputRate !== o && (t = h(t, this.cartesiaOutputRate, o)), this.events.emit("audio", t);
   }
   handleClear() {
     this.events.emit("interrupted", void 0), this.setState("listening");
@@ -448,7 +480,7 @@ class O extends u {
       event: "media_input",
       stream_id: this.streamId,
       media: {
-        payload: m(t)
+        payload: g(t)
       }
     }));
   }
@@ -456,7 +488,7 @@ class O extends u {
     this.stopHeartbeat(), this.isReady = !1, this.streamId = null, super.close();
   }
 }
-class N extends u {
+class O extends u {
   agentName = "Vapi";
   // Audio configuration - Vapi uses 16kHz PCM by default
   vapiSampleRate = 16e3;
@@ -483,7 +515,7 @@ class N extends u {
    */
   handleBinaryAudio(e) {
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    const t = new Uint8Array(e), s = this.vapiSampleRate !== l ? h(t, this.vapiSampleRate, l) : t;
+    const t = new Uint8Array(e), s = this.vapiSampleRate !== o ? h(t, this.vapiSampleRate, o) : t;
     this.events.emit("audio", s);
   }
   handleParsedMessage(e) {
@@ -535,7 +567,7 @@ class N extends u {
     this.hangup(), super.close();
   }
 }
-const U = [
+const N = [
   { id: "gemini", name: "Gemini Live", description: "Google Gemini Live API" },
   { id: "elevenlabs", name: "ElevenLabs", description: "ElevenLabs Conversational AI" },
   { id: "cartesia", name: "Cartesia", description: "Cartesia Agents API" },
@@ -546,19 +578,19 @@ function y(i) {
     case "gemini":
       return new R();
     case "elevenlabs":
-      return new _();
+      return new S();
     case "cartesia":
-      return new O();
+      return new D();
     case "vapi":
-      return new N();
+      return new O();
     default:
       throw new Error(`Unknown agent type: ${i}`);
   }
 }
 function F(i) {
-  return U.find((e) => e.id === i);
+  return N.find((e) => e.id === i);
 }
-class D extends Error {
+class L extends Error {
   status;
   payload;
   url;
@@ -566,8 +598,8 @@ class D extends Error {
     super(e.message), this.name = "ApiError", this.status = e.status, this.payload = e.payload, this.url = e.url;
   }
 }
-const r = /* @__PURE__ */ new Set();
-class K {
+const l = /* @__PURE__ */ new Set();
+class $ {
   apiBaseUrl;
   publishableKey;
   callbacks;
@@ -611,31 +643,31 @@ class K {
   }
   /** Connect to the embed session */
   async connect() {
-    if (r.has(this.publishableKey)) {
+    if (l.has(this.publishableKey)) {
       console.log("[PersonaEmbed] Connection already in progress, skipping");
       return;
     }
-    r.add(this.publishableKey), this.mounted = !0, this.abortController = new AbortController(), this.setStatus("connecting");
+    l.add(this.publishableKey), this.mounted = !0, this.abortController = new AbortController(), this.setStatus("connecting");
     try {
       const e = await this.fetchSession(this.abortController.signal);
       if (!this.mounted) {
-        r.delete(this.publishableKey);
+        l.delete(this.publishableKey);
         return;
       }
       if (await this.initSession(e), await this.initMicrophone(), await this.connectAgent(e.voice_agent_details), !this.mounted) {
-        this.cleanup(), r.delete(this.publishableKey);
+        this.cleanup(), l.delete(this.publishableKey);
         return;
       }
       this.setStatus("connected");
     } catch (e) {
-      if (r.delete(this.publishableKey), e instanceof Error && e.name === "AbortError")
+      if (l.delete(this.publishableKey), e instanceof Error && e.name === "AbortError")
         return;
       console.error("[PersonaEmbed]", e), this.mounted && (this.setStatus("error"), this.callbacks.onError?.(e));
     }
   }
   /** Disconnect and cleanup */
   disconnect() {
-    this.mounted = !1, this.abortController?.abort(), this.abortController = null, r.delete(this.publishableKey), this.cleanup(), this.setStatus("disconnected");
+    this.mounted = !1, this.abortController?.abort(), this.abortController = null, l.delete(this.publishableKey), this.cleanup(), this.setStatus("disconnected");
   }
   /** Toggle microphone mute */
   toggleMute() {
@@ -660,7 +692,7 @@ class K {
         s = await t.json();
       } catch {
       }
-      throw new D({
+      throw new L({
         message: s?.message ?? "create_session failed",
         status: t.status,
         payload: s,
@@ -701,7 +733,7 @@ class K {
       this.session?.endAudioTurn(), this.session?.interrupt();
     }), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
-    }), this.agent instanceof _ && this.agent.on("emotion", (t) => this.session?.setEmotion(t)), await this.session.connect();
+    }), this.agent instanceof S && this.agent.on("emotion", (t) => this.session?.setEmotion(t)), await this.session.connect();
   }
   async initMicrophone() {
     this.stream = await navigator.mediaDevices.getUserMedia({
@@ -841,7 +873,7 @@ class B {
       this.session?.endAudioTurn(), this.session?.interrupt();
     }), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
-    }), this.agent instanceof _ && this.agent.on("emotion", (e) => this.session?.setEmotion(e)), await this.session.connect();
+    }), this.agent instanceof S && this.agent.on("emotion", (e) => this.session?.setEmotion(e)), await this.session.connect();
   }
   async initMicrophone() {
     this.stream = await navigator.mediaDevices.getUserMedia({
@@ -880,17 +912,17 @@ class B {
   }
 }
 export {
-  U as AGENT_REGISTRY,
+  N as AGENT_REGISTRY,
   u as BaseAgent,
-  O as CartesiaAgent,
-  _ as ElevenLabsAgent,
+  D as CartesiaAgent,
+  S as ElevenLabsAgent,
   R as GeminiLiveAgent,
-  D as KeyframeApiError,
-  K as PersonaEmbed,
+  L as KeyframeApiError,
+  $ as PersonaEmbed,
   B as PersonaView,
-  l as SAMPLE_RATE,
-  g as base64ToBytes,
-  m as bytesToBase64,
+  o as SAMPLE_RATE,
+  m as base64ToBytes,
+  g as bytesToBase64,
   y as createAgent,
   E as createEventEmitter,
   w as floatTo16BitPCM,

package/package.json

@@ -4,7 +4,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.2.0",
+  "version": "0.2.1",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",