@keyframelabs/elements 0.0.5 → 0.0.7

package/README.md

@@ -71,7 +71,7 @@ view.disconnect();
 
 ## Supported agents and real-time LLMs
 
-Supports ElevenLabs Agents, Cartesia Line, and Gemini Live.
+Supports Cartesia, ElevenLabs, Vapi, Gemini Live (closed alpha), OpenAI Realtime (closed alpha).
 
 For `PersonaEmbed`, this is determined by the values you set in the Keyframe platform dashboard.
 
@@ -145,10 +145,10 @@ type SessionDetails = {
 };
 
 type VoiceAgentDetails = {
-  type: 'gemini' | 'elevenlabs' | 'cartesia';
+  type: 'cartesia' | 'elevenlabs' | 'vapi' | 'gemini' | 'openai';
   token?: string;       // For gemini, cartesia
   agent_id?: string;    // For elevenlabs, cartesia
-  signed_url?: string;  // For elevenlabs
+  signed_url?: string;  // For elevenlabs, vapi
 };
 ```
 

package/dist/agents/index.d.ts

@@ -1,6 +1,7 @@
 import { GeminiLiveAgent, GeminiLiveConfig } from './gemini-live';
 import { ElevenLabsAgent, ElevenLabsConfig } from './elevenlabs';
 import { CartesiaAgent, CartesiaConfig } from './cartesia';
+import { VapiAgent, VapiConfig } from './vapi';
 /**
  * Agent implementations for voice AI platforms.
  *
@@ -12,9 +13,10 @@ export type { Agent, AgentConfig, AgentEventMap, AgentState } from './types';
 export { GeminiLiveAgent, type GeminiLiveConfig };
 export { ElevenLabsAgent, type ElevenLabsConfig };
 export { CartesiaAgent, type CartesiaConfig };
+export { VapiAgent, type VapiConfig };
 export { SAMPLE_RATE, base64ToBytes, bytesToBase64, resamplePcm, createEventEmitter, floatTo16BitPCM } from './audio-utils';
 /** Supported agent types */
-export type AgentType = 'gemini' | 'elevenlabs' | 'cartesia';
+export type AgentType = 'gemini' | 'elevenlabs' | 'cartesia' | 'vapi';
 /** Agent type metadata */
 export interface AgentTypeInfo {
     id: AgentType;
@@ -28,9 +30,10 @@ export interface AgentConfigMap {
     gemini: GeminiLiveConfig;
     elevenlabs: ElevenLabsConfig;
     cartesia: CartesiaConfig;
+    vapi: VapiConfig;
 }
 /** Union type of all agent instances */
-export type AnyAgent = GeminiLiveAgent | ElevenLabsAgent | CartesiaAgent;
+export type AnyAgent = GeminiLiveAgent | ElevenLabsAgent | CartesiaAgent | VapiAgent;
 /**
  * Create an agent instance by type.
  *
@@ -43,6 +46,7 @@ export type AnyAgent = GeminiLiveAgent | ElevenLabsAgent | CartesiaAgent;
 export declare function createAgent(type: 'gemini'): GeminiLiveAgent;
 export declare function createAgent(type: 'elevenlabs'): ElevenLabsAgent;
 export declare function createAgent(type: 'cartesia'): CartesiaAgent;
+export declare function createAgent(type: 'vapi'): VapiAgent;
 export declare function createAgent(type: AgentType): AnyAgent;
 /**
  * Get agent type metadata by ID.

package/dist/agents/vapi.d.ts

@@ -0,0 +1,30 @@
+import { AgentConfig } from './types';
+import { BaseAgent } from './base';
+/** Vapi specific configuration */
+export interface VapiConfig extends AgentConfig {
+    /** Pre-authenticated WebSocket URL from backend (same concept as ElevenLabs signed_url) */
+    signedUrl: string;
+}
+/**
+ * Vapi WebSocket Transport agent implementation.
+ *
+ * Handles WebSocket connection to Vapi and converts
+ * audio responses to events that Persona SDK can consume.
+ */
+export declare class VapiAgent extends BaseAgent {
+    protected readonly agentName = "Vapi";
+    private vapiSampleRate;
+    connect(config: VapiConfig): Promise<void>;
+    /**
+     * Handle binary audio data from Vapi.
+     * Vapi sends raw PCM 16-bit little-endian audio.
+     */
+    private handleBinaryAudio;
+    protected handleParsedMessage(message: unknown): void;
+    sendAudio(pcmData: Uint8Array): void;
+    /**
+     * Send a control message to end the call.
+     */
+    hangup(): void;
+    close(): void;
+}

package/dist/index.js

@@ -1,28 +1,28 @@
 import { createClient as v } from "@keyframelabs/sdk";
-const c = 24e3;
-function p(i) {
+const h = 24e3;
+function g(i) {
   const e = atob(i), t = new Uint8Array(e.length);
   for (let s = 0; s < e.length; s++)
     t[s] = e.charCodeAt(s);
   return t;
 }
-function g(i) {
+function m(i) {
   let e = "";
   for (let t = 0; t < i.length; t++)
     e += String.fromCharCode(i[t]);
   return btoa(e);
 }
-function d(i, e, t) {
+function c(i, e, t) {
   if (e === t)
     return i;
-  const s = new Int16Array(i.buffer, i.byteOffset, i.length / 2), n = e / t, a = Math.floor(s.length / n), r = new Int16Array(a);
-  for (let o = 0; o < a; o++) {
-    const _ = o * n, u = Math.floor(_), w = Math.min(u + 1, s.length - 1), S = _ - u;
-    r[o] = Math.round(
-      s[u] * (1 - S) + s[w] * S
+  const s = new Int16Array(i.buffer, i.byteOffset, i.length / 2), n = e / t, a = Math.floor(s.length / n), o = new Int16Array(a);
+  for (let r = 0; r < a; r++) {
+    const S = r * n, p = Math.floor(S), f = Math.min(p + 1, s.length - 1), _ = S - p;
+    o[r] = Math.round(
+      s[p] * (1 - _) + s[f] * _
     );
   }
-  return new Uint8Array(r.buffer);
+  return new Uint8Array(o.buffer);
 }
 function b() {
   const i = /* @__PURE__ */ new Map();
@@ -41,7 +41,7 @@ function b() {
     }
   };
 }
-function y(i) {
+function w(i) {
   const e = new Int16Array(i.length);
   for (let t = 0; t < i.length; t++) {
     const s = Math.max(-1, Math.min(1, i[t]));
@@ -50,7 +50,7 @@ function y(i) {
   return new Uint8Array(e.buffer);
 }
 const I = 16e3;
-class m {
+class u {
   ws = null;
   _state = "idle";
   events = b();
@@ -113,8 +113,8 @@ class m {
     this.events.emit("closed", { code: e, reason: t });
   }
 }
-const C = "gemini-2.5-flash-native-audio-preview-12-2025", E = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", A = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
-class k extends m {
+const E = "gemini-2.5-flash-native-audio-preview-12-2025", C = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", k = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
+class A extends u {
   agentName = "GeminiLive";
   async connect(e) {
     if (this.ws)
@@ -122,10 +122,10 @@ class k extends m {
     if (!e.apiKey)
       throw new Error("Gemini API key is required");
     e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate);
-    const t = e.model ?? C, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${A}?access_token=${encodeURIComponent(e.apiKey)}` : `${E}?key=${encodeURIComponent(e.apiKey)}`;
-    return new Promise((a, r) => {
+    const t = e.model ?? E, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${k}?access_token=${encodeURIComponent(e.apiKey)}` : `${C}?key=${encodeURIComponent(e.apiKey)}`;
+    return new Promise((a, o) => {
       this.ws = new WebSocket(n), this.ws.onopen = () => {
-        const o = {
+        const r = {
           setup: {
             model: `models/${t}`,
             generationConfig: {
@@ -134,13 +134,13 @@ class k extends m {
             systemInstruction: e.systemPrompt ? { parts: [{ text: e.systemPrompt }] } : void 0
           }
         };
-        this.ws.send(JSON.stringify(o)), this.setState("listening"), a();
+        this.ws.send(JSON.stringify(r)), this.setState("listening"), a();
       }, this.ws.onerror = () => {
-        r(new Error("Failed to connect to Gemini Live"));
-      }, this.ws.onclose = (o) => {
-        this.ws = null, this.setState("idle"), this.emitClosed(o.code, o.reason);
-      }, this.ws.onmessage = (o) => {
-        this.handleMessage(o.data);
+        o(new Error("Failed to connect to Gemini Live"));
+      }, this.ws.onclose = (r) => {
+        this.ws = null, this.setState("idle"), this.emitClosed(r.code, r.reason);
+      }, this.ws.onmessage = (r) => {
+        this.handleMessage(r.data);
       };
     });
   }
@@ -159,7 +159,7 @@ class k extends m {
         this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
         for (const n of s.modelTurn.parts) {
           if (n.inlineData?.data) {
-            const a = p(n.inlineData.data);
+            const a = g(n.inlineData.data);
             this.events.emit("audio", a);
           }
           n.text && this.events.emit("transcript", {
@@ -181,7 +181,7 @@ class k extends m {
         mediaChunks: [
           {
             mimeType: `audio/pcm;rate=${this.inputSampleRate}`,
-            data: g(e)
+            data: m(e)
           }
         ]
       }
@@ -190,7 +190,7 @@ class k extends m {
   }
 }
 const R = "wss://api.elevenlabs.io/v1/convai/conversation";
-class M extends m {
+class M extends u {
   agentName = "ElevenLabs";
   outputSampleRate = 24e3;
   // Default, updated from metadata
@@ -272,15 +272,15 @@ class M extends m {
     if (!t?.audio_base_64 || (t.event_id ?? 0) <= this.lastInterruptId)
       return;
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    let n = p(t.audio_base_64);
-    this.outputSampleRate !== c && (n = d(n, this.outputSampleRate, c));
+    let n = g(t.audio_base_64);
+    this.outputSampleRate !== h && (n = c(n, this.outputSampleRate, h));
     const a = 4800;
     if (n.length <= a)
       this.events.emit("audio", n);
     else
-      for (let r = 0; r < n.length; r += a) {
-        const o = n.slice(r, Math.min(r + a, n.length));
-        this.events.emit("audio", o);
+      for (let o = 0; o < n.length; o += a) {
+        const r = n.slice(o, Math.min(o + a, n.length));
+        this.events.emit("audio", r);
       }
   }
   handleUserTranscript(e) {
@@ -307,8 +307,8 @@ class M extends m {
     if (!this.ws || this.ws.readyState !== WebSocket.OPEN || !this.initialized)
       return;
     let t = e;
-    this.sourceInputSampleRate !== this.expectedInputSampleRate && (t = d(e, this.sourceInputSampleRate, this.expectedInputSampleRate)), this.ws.send(JSON.stringify({
-      user_audio_chunk: g(t)
+    this.sourceInputSampleRate !== this.expectedInputSampleRate && (t = c(e, this.sourceInputSampleRate, this.expectedInputSampleRate)), this.ws.send(JSON.stringify({
+      user_audio_chunk: m(t)
     }));
   }
   /**
@@ -342,7 +342,7 @@ class M extends m {
   }
 }
 const x = "wss://api.cartesia.ai/agents/stream", P = "2025-04-16";
-class T extends m {
+class T extends u {
   agentName = "Cartesia";
   // Audio configuration
   cartesiaInputFormat = "pcm_16000";
@@ -423,8 +423,8 @@ class T extends m {
   handleMediaOutput(e) {
     if (!e.media?.payload) return;
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    let t = p(e.media.payload);
-    this.cartesiaOutputRate !== c && (t = d(t, this.cartesiaOutputRate, c)), this.events.emit("audio", t);
+    let t = g(e.media.payload);
+    this.cartesiaOutputRate !== h && (t = c(t, this.cartesiaOutputRate, h)), this.events.emit("audio", t);
   }
   handleClear() {
     this.events.emit("interrupted", void 0), this.setState("listening");
@@ -434,11 +434,11 @@ class T extends m {
       return;
     let t = e;
     const s = parseInt(this.cartesiaInputFormat.split("_")[1]);
-    this.inputSampleRate !== s && (t = d(e, this.inputSampleRate, s)), this.ws.send(JSON.stringify({
+    this.inputSampleRate !== s && (t = c(e, this.inputSampleRate, s)), this.ws.send(JSON.stringify({
       event: "media_input",
       stream_id: this.streamId,
       media: {
-        payload: g(t)
+        payload: m(t)
       }
     }));
   }
@@ -446,27 +446,109 @@ class T extends m {
     this.stopHeartbeat(), this.isReady = !1, this.streamId = null, super.close();
   }
 }
-const O = [
+class O extends u {
+  agentName = "Vapi";
+  // Audio configuration - Vapi uses 16kHz PCM by default
+  vapiSampleRate = 16e3;
+  async connect(e) {
+    if (this.ws)
+      throw new Error("Already connected");
+    if (!e.signedUrl)
+      throw new Error("Vapi signed URL is required");
+    return e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate), new Promise((t, s) => {
+      this.ws = new WebSocket(e.signedUrl), this.ws.binaryType = "arraybuffer", this.ws.onopen = () => {
+        this.setState("listening"), t();
+      }, this.ws.onerror = () => {
+        s(new Error("Failed to connect to Vapi"));
+      }, this.ws.onclose = (n) => {
+        this.ws = null, this.setState("idle"), this.emitClosed(n.code, n.reason);
+      }, this.ws.onmessage = (n) => {
+        n.data instanceof ArrayBuffer ? this.handleBinaryAudio(n.data) : this.handleMessage(n.data);
+      };
+    });
+  }
+  /**
+   * Handle binary audio data from Vapi.
+   * Vapi sends raw PCM 16-bit little-endian audio.
+   */
+  handleBinaryAudio(e) {
+    this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
+    const t = new Uint8Array(e), s = this.vapiSampleRate !== h ? c(t, this.vapiSampleRate, h) : t;
+    this.events.emit("audio", s);
+  }
+  handleParsedMessage(e) {
+    const t = e;
+    switch (t.type) {
+      case "conversation-update":
+        t.role === "user" && t.transcript ? this.events.emit("transcript", {
+          role: "user",
+          text: t.transcript,
+          isFinal: !0
+        }) : t.role === "assistant" && t.transcript && this.events.emit("transcript", {
+          role: "assistant",
+          text: t.transcript,
+          isFinal: !0
+        });
+        break;
+      case "speech-update":
+        t.status === "started" ? (this.events.emit("turnStart", void 0), this.setState("speaking")) : t.status === "stopped" && (this.events.emit("turnEnd", void 0), this.setState("listening"));
+        break;
+      case "transcript":
+        this.events.emit("transcript", {
+          role: t.role === "user" ? "user" : "assistant",
+          text: t.transcript || "",
+          isFinal: t.transcriptType === "final"
+        });
+        break;
+      case "hang":
+      case "end-of-call-report":
+        this.events.emit("turnEnd", void 0), this.setState("idle");
+        break;
+      case "error":
+        console.error("[Vapi] Server error:", t);
+        break;
+    }
+  }
+  sendAudio(e) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN)
+      return;
+    let t = e;
+    this.inputSampleRate !== this.vapiSampleRate && (t = c(e, this.inputSampleRate, this.vapiSampleRate)), this.ws.send(t.buffer);
+  }
+  /**
+   * Send a control message to end the call.
+   */
+  hangup() {
+    this.ws && this.ws.readyState === WebSocket.OPEN && this.ws.send(JSON.stringify({ type: "end-call" }));
+  }
+  close() {
+    this.hangup(), super.close();
+  }
+}
+const N = [
   { id: "gemini", name: "Gemini Live", description: "Google Gemini Live API" },
   { id: "elevenlabs", name: "ElevenLabs", description: "ElevenLabs Conversational AI" },
-  { id: "cartesia", name: "Cartesia", description: "Cartesia Agents API" }
+  { id: "cartesia", name: "Cartesia", description: "Cartesia Agents API" },
+  { id: "vapi", name: "Vapi", description: "Vapi WebSocket Transport" }
 ];
-function f(i) {
+function y(i) {
   switch (i) {
     case "gemini":
-      return new k();
+      return new A();
     case "elevenlabs":
       return new M();
     case "cartesia":
       return new T();
+    case "vapi":
+      return new O();
     default:
       throw new Error(`Unknown agent type: ${i}`);
   }
 }
 function L(i) {
-  return O.find((e) => e.id === i);
+  return N.find((e) => e.id === i);
 }
-class D extends Error {
+class U extends Error {
   status;
   payload;
   url;
@@ -475,7 +557,7 @@ class D extends Error {
   }
 }
 const l = /* @__PURE__ */ new Set();
-class U {
+class F {
   apiBaseUrl;
   publishableKey;
   callbacks;
@@ -568,7 +650,7 @@ class U {
         s = await t.json();
       } catch {
       }
-      throw new D({
+      throw new U({
         message: s?.message ?? "create_session failed",
         status: t.status,
         payload: s,
@@ -602,7 +684,7 @@ class U {
       onClose: () => {
         this.mounted && this.callbacks.onDisconnect?.();
       }
-    }), this.agent = f(e.voice_agent_details.type), this.agent.on("audio", (t) => this.session?.sendAudio(t)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (t) => this.setAgentState(t)), this.agent.on("closed", () => {
+    }), this.agent = y(e.voice_agent_details.type), this.agent.on("audio", (t) => this.session?.sendAudio(t)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (t) => this.setAgentState(t)), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
     }), await this.session.connect();
   }
@@ -613,7 +695,7 @@ class U {
     const e = this.audioContext.createMediaStreamSource(this.stream);
     this.processor = this.audioContext.createScriptProcessor(4096, 1, 1), this.processor.onaudioprocess = (t) => {
       if (!this._isMuted) {
-        const s = y(t.inputBuffer.getChannelData(0));
+        const s = w(t.inputBuffer.getChannelData(0));
         this.agent?.sendAudio(s);
       }
     }, e.connect(this.processor), this.processor.connect(this.audioContext.destination);
@@ -629,17 +711,20 @@ class U {
       ...t,
       agentId: e.agent_id,
       signedUrl: e.signed_url
-    }) : e.type === "cartesia" && await this.agent.connect({
+    }) : e.type === "cartesia" ? await this.agent.connect({
       ...t,
       agentId: e.agent_id,
       apiKey: e.token
+    }) : e.type === "vapi" && await this.agent.connect({
+      ...t,
+      signedUrl: e.signed_url
     });
   }
   cleanup() {
     this.stream?.getTracks().forEach((e) => e.stop()), this.processor?.disconnect(), this.audioContext?.close(), this.agent?.close(), this.session?.close(), this.stream = null, this.processor = null, this.audioContext = null, this.agent = null, this.session = null;
   }
 }
-const h = /* @__PURE__ */ new Set();
+const d = /* @__PURE__ */ new Set();
 class K {
   voiceAgentDetails;
   sessionDetails;
@@ -684,24 +769,24 @@ class K {
   }
   /** Connect to the session */
   async connect() {
-    if (h.has(this.connectionId)) {
+    if (d.has(this.connectionId)) {
       console.log("[PersonaView] Connection already in progress, skipping");
       return;
     }
-    h.add(this.connectionId), this.mounted = !0, this.setStatus("connecting");
+    d.add(this.connectionId), this.mounted = !0, this.setStatus("connecting");
     try {
       if (await this.initSession(), await this.initMicrophone(), await this.connectAgent(), !this.mounted) {
-        this.cleanup(), h.delete(this.connectionId);
+        this.cleanup(), d.delete(this.connectionId);
         return;
       }
       this.setStatus("connected");
     } catch (e) {
-      h.delete(this.connectionId), console.error("[PersonaView]", e), this.mounted && (this.setStatus("error"), this.callbacks.onError?.(e));
+      d.delete(this.connectionId), console.error("[PersonaView]", e), this.mounted && (this.setStatus("error"), this.callbacks.onError?.(e));
     }
   }
   /** Disconnect and cleanup */
   disconnect() {
-    this.mounted = !1, h.delete(this.connectionId), this.cleanup(), this.setStatus("disconnected");
+    this.mounted = !1, d.delete(this.connectionId), this.cleanup(), this.setStatus("disconnected");
   }
   /** Toggle microphone mute */
   toggleMute() {
@@ -734,7 +819,7 @@ class K {
       onClose: () => {
         this.mounted && this.callbacks.onDisconnect?.();
       }
-    }), this.agent = f(this.voiceAgentDetails.type), this.agent.on("audio", (e) => this.session?.sendAudio(e)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (e) => this.setAgentState(e)), this.agent.on("closed", () => {
+    }), this.agent = y(this.voiceAgentDetails.type), this.agent.on("audio", (e) => this.session?.sendAudio(e)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (e) => this.setAgentState(e)), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
     }), await this.session.connect();
   }
@@ -745,7 +830,7 @@ class K {
     const e = this.audioContext.createMediaStreamSource(this.stream);
     this.processor = this.audioContext.createScriptProcessor(4096, 1, 1), this.processor.onaudioprocess = (t) => {
       if (!this._isMuted) {
-        const s = y(t.inputBuffer.getChannelData(0));
+        const s = w(t.inputBuffer.getChannelData(0));
         this.agent?.sendAudio(s);
       }
     }, e.connect(this.processor), this.processor.connect(this.audioContext.destination);
@@ -761,10 +846,13 @@ class K {
       ...t,
       agentId: e.agent_id,
       signedUrl: e.signed_url
-    }) : e.type === "cartesia" && await this.agent.connect({
+    }) : e.type === "cartesia" ? await this.agent.connect({
       ...t,
       agentId: e.agent_id,
       apiKey: e.token
+    }) : e.type === "vapi" && await this.agent.connect({
+      ...t,
+      signedUrl: e.signed_url
     });
   }
   cleanup() {
@@ -772,20 +860,20 @@ class K {
   }
 }
 export {
-  O as AGENT_REGISTRY,
-  m as BaseAgent,
+  N as AGENT_REGISTRY,
+  u as BaseAgent,
   T as CartesiaAgent,
   M as ElevenLabsAgent,
-  k as GeminiLiveAgent,
-  D as KeyframeApiError,
-  U as PersonaEmbed,
+  A as GeminiLiveAgent,
+  U as KeyframeApiError,
+  F as PersonaEmbed,
   K as PersonaView,
-  c as SAMPLE_RATE,
-  p as base64ToBytes,
-  g as bytesToBase64,
-  f as createAgent,
+  h as SAMPLE_RATE,
+  g as base64ToBytes,
+  m as bytesToBase64,
+  y as createAgent,
   b as createEventEmitter,
-  y as floatTo16BitPCM,
+  w as floatTo16BitPCM,
   L as getAgentInfo,
-  d as resamplePcm
+  c as resamplePcm
 };

package/package.json

@@ -4,7 +4,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.5",
+  "version": "0.0.7",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",