@keyframelabs/elements 0.0.1 → 0.0.2

package/dist/PersonaEmbed.d.ts

@@ -0,0 +1,73 @@
+import { AgentState } from './agents';
+export type EmbedStatus = 'connecting' | 'connected' | 'error' | 'disconnected';
+export type VideoFit = 'cover' | 'contain';
+export interface PersonaEmbedOptions {
+    /** Target container element */
+    container: HTMLElement;
+    /** Publishable key from your embed config */
+    publishableKey: string;
+    /** Base API URL to generate embed sessions. Defaults to https://api.keyframelabs.com. */
+    apiBaseUrl?: string;
+    /** Video fit mode. 'cover' fills container (may crop), 'contain' shows full video (may have black bars). Default: 'cover' */
+    videoFit?: VideoFit;
+    /** Called when session disconnects */
+    onDisconnect?: () => void;
+    /** Called on error */
+    onError?: (err: Error) => void;
+    /** Called when status changes */
+    onStateChange?: (status: EmbedStatus) => void;
+    /** Called when agent state changes */
+    onAgentStateChange?: (state: AgentState) => void;
+}
+/**
+ * Headless Persona avatar with voice agent integration.
+ *
+ * Creates video/audio elements and handles all wiring.
+ * UI (overlays, controls, status) is the consumer's responsibility.
+ *
+ * @example
+ * ```ts
+ * const embed = new PersonaEmbed({
+ *   container: document.getElementById('avatar'),
+ *   publishableKey: 'kfl_pk_live_...',
+ *   onStateChange: (status) => updateUI(status),
+ * });
+ * await embed.connect();
+ * ```
+ */
+export declare class PersonaEmbed {
+    private readonly apiBaseUrl;
+    private readonly publishableKey;
+    private readonly callbacks;
+    private readonly _video;
+    private readonly _audio;
+    private session;
+    private agent;
+    private audioContext;
+    private processor;
+    private stream;
+    private abortController;
+    private _status;
+    private _agentState;
+    private _isMuted;
+    private mounted;
+    constructor(options: PersonaEmbedOptions);
+    get status(): EmbedStatus;
+    get agentState(): AgentState;
+    get isMuted(): boolean;
+    get videoElement(): HTMLVideoElement;
+    get audioElement(): HTMLAudioElement;
+    /** Connect to the embed session */
+    connect(): Promise<void>;
+    /** Disconnect and cleanup */
+    disconnect(): void;
+    /** Toggle microphone mute */
+    toggleMute(): void;
+    private setStatus;
+    private setAgentState;
+    private fetchSession;
+    private initSession;
+    private initMicrophone;
+    private connectAgent;
+    private cleanup;
+}

package/dist/agents/audio-utils.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Audio utilities for agent implementations.
+ *
+ * These utilities help with PCM audio processing for voice AI integrations.
+ */
+/** Sample rate for audio sent to Persona (matches Gemini output) */
+export declare const SAMPLE_RATE = 24000;
+/**
+ * Convert base64-encoded audio to Uint8Array.
+ */
+export declare function base64ToBytes(base64: string): Uint8Array;
+/**
+ * Convert Uint8Array to base64 string.
+ */
+export declare function bytesToBase64(bytes: Uint8Array): string;
+/**
+ * Resample PCM audio from one sample rate to another.
+ * Simple linear interpolation - not high quality but sufficient for real-time.
+ */
+export declare function resamplePcm(input: Uint8Array, fromRate: number, toRate: number): Uint8Array;
+/**
+ * Create a simple typed event emitter.
+ */
+export declare function createEventEmitter<T extends Record<string, any>>(): {
+    on<K extends keyof T>(event: K, handler: (data: T[K]) => void): void;
+    off<K extends keyof T>(event: K, handler: (data: T[K]) => void): void;
+    emit<K extends keyof T>(event: K, data: T[K]): void;
+    removeAllListeners(): void;
+};
+export declare function floatTo16BitPCM(float32: Float32Array): Uint8Array;

package/dist/agents/base.d.ts

@@ -0,0 +1,72 @@
+import { Agent, AgentConfig, AgentEventMap, AgentState } from './types';
+/** Default input sample rate (16kHz mono PCM) */
+export declare const DEFAULT_INPUT_SAMPLE_RATE = 16000;
+/**
+ * Abstract base class for voice AI agents.
+ *
+ * Provides:
+ * - Event emitter with typed events
+ * - State management with automatic stateChange emission
+ * - WebSocket message handling with JSON parsing
+ * - Standard lifecycle methods (on, off, close)
+ */
+export declare abstract class BaseAgent implements Agent {
+    protected ws: WebSocket | null;
+    protected _state: AgentState;
+    protected events: {
+        on<K extends keyof AgentEventMap>(event: K, handler: (data: AgentEventMap[K]) => void): void;
+        off<K extends keyof AgentEventMap>(event: K, handler: (data: AgentEventMap[K]) => void): void;
+        emit<K extends keyof AgentEventMap>(event: K, data: AgentEventMap[K]): void;
+        removeAllListeners(): void;
+    };
+    protected inputSampleRate: number;
+    /** Agent name for logging (override in subclass) */
+    protected abstract readonly agentName: string;
+    /** Current agent state */
+    get state(): AgentState;
+    /**
+     * Update state and emit stateChange event.
+     */
+    protected setState(state: AgentState): void;
+    /**
+     * Connect to the agent service.
+     * Must be implemented by subclasses.
+     */
+    abstract connect(config: AgentConfig): Promise<void>;
+    /**
+     * Send audio input to the agent.
+     * Must be implemented by subclasses.
+     */
+    abstract sendAudio(pcmData: Uint8Array): void;
+    /**
+     * Handle parsed JSON message from WebSocket.
+     * Must be implemented by subclasses.
+     */
+    protected abstract handleParsedMessage(message: unknown): void;
+    /**
+     * Handle WebSocket message (string or Blob).
+     * Converts to string and parses JSON before calling handleParsedMessage.
+     */
+    protected handleMessage(data: string | Blob): void;
+    /**
+     * Parse JSON and call handleParsedMessage if valid.
+     */
+    private parseAndHandle;
+    /**
+     * Close the WebSocket connection and clean up resources.
+     * Subclasses can override to add custom cleanup, but should call super.close().
+     */
+    close(): void;
+    /**
+     * Register an event handler.
+     */
+    on<K extends keyof AgentEventMap>(event: K, handler: (data: AgentEventMap[K]) => void): void;
+    /**
+     * Remove an event handler.
+     */
+    off<K extends keyof AgentEventMap>(event: K, handler: (data: AgentEventMap[K]) => void): void;
+    /**
+     * Helper to emit the closed event with code and reason.
+     */
+    protected emitClosed(code?: number, reason?: string): void;
+}

package/dist/agents/cartesia.d.ts

@@ -0,0 +1,32 @@
+import { AgentConfig } from './types';
+import { BaseAgent } from './base';
+/** Cartesia specific configuration */
+export interface CartesiaConfig extends AgentConfig {
+    /** Agent ID from Cartesia dashboard */
+    agentId: string;
+}
+/**
+ * Cartesia Agent implementation.
+ */
+export declare class CartesiaAgent extends BaseAgent {
+    protected readonly agentName = "Cartesia";
+    private cartesiaInputFormat;
+    private cartesiaOutputRate;
+    private streamId;
+    private isReady;
+    private pingInterval;
+    connect(config: CartesiaConfig): Promise<void>;
+    private sendStartEvent;
+    /**
+     * Keep connection alive with periodic custom events.
+     * Cartesia requires activity every 30s.
+     */
+    private startHeartbeat;
+    private stopHeartbeat;
+    protected handleParsedMessage(message: unknown): void;
+    private handleAck;
+    private handleMediaOutput;
+    private handleClear;
+    sendAudio(pcmData: Uint8Array): void;
+    close(): void;
+}

package/dist/agents/elevenlabs.d.ts

@@ -0,0 +1,41 @@
+import { AgentConfig } from './types';
+import { BaseAgent } from './base';
+/** ElevenLabs specific configuration */
+export interface ElevenLabsConfig extends AgentConfig {
+    /** Agent ID from ElevenLabs dashboard */
+    agentId: string;
+    /** Signed URL for authentication (alternative to API key) */
+    signedUrl?: string;
+}
+/**
+ * ElevenLabs Conversational AI agent implementation.
+ *
+ * Handles WebSocket connection to ElevenLabs and converts
+ * audio responses to events that Persona SDK can consume.
+ */
+export declare class ElevenLabsAgent extends BaseAgent {
+    protected readonly agentName = "ElevenLabs";
+    private outputSampleRate;
+    private expectedInputSampleRate;
+    private sourceInputSampleRate;
+    private initialized;
+    private lastInterruptId;
+    connect(config: ElevenLabsConfig): Promise<void>;
+    protected handleParsedMessage(message: unknown): void;
+    private handleInitMetadata;
+    private handlePing;
+    private handleAudio;
+    private handleUserTranscript;
+    private handleAgentResponse;
+    private handleInterruption;
+    sendAudio(pcmData: Uint8Array): void;
+    /**
+     * Send a text message as if the user spoke it.
+     */
+    sendText(text: string): void;
+    /**
+     * Send contextual information to the agent without interrupting.
+     */
+    sendContext(text: string): void;
+    close(): void;
+}

package/dist/agents/gemini-live.d.ts

@@ -0,0 +1,25 @@
+import { AgentConfig } from './types';
+import { BaseAgent } from './base';
+/** Gemini Live specific configuration */
+export interface GeminiLiveConfig extends AgentConfig {
+    /** Model to use (defaults to gemini-2.5-flash-native-audio-preview) */
+    model?: string;
+    /**
+     * Auth mechanism for websocket connection.
+     * - `api_key`: use `?key=<apiKey>`
+     * - `ephemeral_token`: use `?access_token=<apiKey>` where apiKey is the ephemeral token value
+     */
+    authType?: 'api_key' | 'ephemeral_token';
+}
+/**
+ * Gemini Live agent implementation.
+ *
+ * Handles WebSocket connection to Gemini Live API and converts
+ * audio responses to events that Persona SDK can consume.
+ */
+export declare class GeminiLiveAgent extends BaseAgent {
+    protected readonly agentName = "GeminiLive";
+    connect(config: GeminiLiveConfig): Promise<void>;
+    protected handleParsedMessage(message: unknown): void;
+    sendAudio(pcmData: Uint8Array): void;
+}

package/dist/agents/index.d.ts

@@ -0,0 +1,50 @@
+import { GeminiLiveAgent, GeminiLiveConfig } from './gemini-live';
+import { ElevenLabsAgent, ElevenLabsConfig } from './elevenlabs';
+import { CartesiaAgent, CartesiaConfig } from './cartesia';
+/**
+ * Agent implementations for voice AI platforms.
+ *
+ * These are reference implementations demonstrating how to connect
+ * various voice AI backends to the Persona SDK.
+ */
+export { BaseAgent, DEFAULT_INPUT_SAMPLE_RATE } from './base';
+export type { Agent, AgentConfig, AgentEventMap, AgentState } from './types';
+export { GeminiLiveAgent, type GeminiLiveConfig };
+export { ElevenLabsAgent, type ElevenLabsConfig };
+export { CartesiaAgent, type CartesiaConfig };
+export { SAMPLE_RATE, base64ToBytes, bytesToBase64, resamplePcm, createEventEmitter, floatTo16BitPCM } from './audio-utils';
+/** Supported agent types */
+export type AgentType = 'gemini' | 'elevenlabs' | 'cartesia';
+/** Agent type metadata */
+export interface AgentTypeInfo {
+    id: AgentType;
+    name: string;
+    description: string;
+}
+/** Registry of available agents */
+export declare const AGENT_REGISTRY: AgentTypeInfo[];
+/** Configuration types by agent type */
+export interface AgentConfigMap {
+    gemini: GeminiLiveConfig;
+    elevenlabs: ElevenLabsConfig;
+    cartesia: CartesiaConfig;
+}
+/** Union type of all agent instances */
+export type AnyAgent = GeminiLiveAgent | ElevenLabsAgent | CartesiaAgent;
+/**
+ * Create an agent instance by type.
+ *
+ * @example
+ * ```ts
+ * const agent = createAgent('gemini');
+ * await agent.connect({ apiKey: 'YOUR_KEY' });
+ * ```
+ */
+export declare function createAgent(type: 'gemini'): GeminiLiveAgent;
+export declare function createAgent(type: 'elevenlabs'): ElevenLabsAgent;
+export declare function createAgent(type: 'cartesia'): CartesiaAgent;
+export declare function createAgent(type: AgentType): AnyAgent;
+/**
+ * Get agent type metadata by ID.
+ */
+export declare function getAgentInfo(type: AgentType): AgentTypeInfo | undefined;

package/dist/agents/types.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Agent interface for voice AI backends.
+ *
+ * Agents handle the conversation logic (STT, LLM, TTS) and emit audio
+ * that gets sent to the Persona avatar for video synthesis.
+ *
+ * This is a reference implementation - copy and adapt for your own use.
+ */
+/** Agent state for UI */
+export type AgentState = 'idle' | 'listening' | 'thinking' | 'speaking';
+/** Agent configuration */
+export interface AgentConfig {
+    /** System prompt for the agent */
+    systemPrompt?: string;
+    /** Voice ID (agent-specific) */
+    voice?: string;
+    /** API key for the agent service */
+    apiKey?: string;
+    /** Sample rate of audio being sent via sendAudio (from microphone). Default: 16000 */
+    inputSampleRate?: number;
+}
+/** Events emitted by agents */
+export interface AgentEventMap {
+    /** Audio data from agent (PCM bytes) */
+    audio: Uint8Array;
+    /** Agent started speaking */
+    turnStart: void;
+    /** Agent finished speaking */
+    turnEnd: void;
+    /** Agent was interrupted */
+    interrupted: void;
+    /** Agent state changed */
+    stateChange: AgentState;
+    /** Transcript update */
+    transcript: {
+        role: 'user' | 'assistant';
+        text: string;
+        isFinal: boolean;
+    };
+    /** Agent connection closed (unexpected disconnect) */
+    closed: {
+        code?: number;
+        reason?: string;
+    };
+}
+/**
+ * Abstract agent interface.
+ *
+ * Implement this for each voice AI platform (Gemini, ElevenLabs, Cartesia, etc.)
+ */
+export interface Agent {
+    /** Current agent state */
+    readonly state: AgentState;
+    /**
+     * Connect to the agent service.
+     * @param config - Agent configuration
+     */
+    connect(config: AgentConfig): Promise<void>;
+    /**
+     * Send audio input to the agent (from microphone).
+     * @param pcmData - 16-bit PCM audio at 16kHz
+     */
+    sendAudio(pcmData: Uint8Array): void;
+    /**
+     * Disconnect and clean up.
+     */
+    close(): void;
+    /**
+     * Register an event handler.
+     */
+    on<K extends keyof AgentEventMap>(event: K, handler: (data: AgentEventMap[K]) => void): void;
+    /**
+     * Remove an event handler.
+     */
+    off<K extends keyof AgentEventMap>(event: K, handler: (data: AgentEventMap[K]) => void): void;
+}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+export { PersonaEmbed } from './PersonaEmbed';
+export type { PersonaEmbedOptions, EmbedStatus, VideoFit } from './PersonaEmbed';
+export { createAgent, GeminiLiveAgent, ElevenLabsAgent, CartesiaAgent, BaseAgent, AGENT_REGISTRY, getAgentInfo, } from './agents';
+export type { AgentType, AgentState, AgentConfig, AgentEventMap, Agent, AnyAgent, AgentTypeInfo, GeminiLiveConfig, ElevenLabsConfig, CartesiaConfig, } from './agents';
+export { floatTo16BitPCM, resamplePcm, base64ToBytes, bytesToBase64, SAMPLE_RATE, createEventEmitter, } from './agents';

package/dist/index.js

@@ -0,0 +1 @@
+"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const A=require("@keyframelabs/sdk"),h=24e3;function c(i){const e=atob(i),t=new Uint8Array(e.length);for(let s=0;s<e.length;s++)t[s]=e.charCodeAt(s);return t}function u(i){let e="";for(let t=0;t<i.length;t++)e+=String.fromCharCode(i[t]);return btoa(e)}function d(i,e,t){if(e===t)return i;const s=new Int16Array(i.buffer,i.byteOffset,i.length/2),n=e/t,a=Math.floor(s.length/n),o=new Int16Array(a);for(let r=0;r<a;r++){const g=r*n,m=Math.floor(g),E=Math.min(m+1,s.length-1),_=g-m;o[r]=Math.round(s[m]*(1-_)+s[E]*_)}return new Uint8Array(o.buffer)}function S(){const i=new Map;return{on(e,t){i.has(e)||i.set(e,new Set),i.get(e).add(t)},off(e,t){i.get(e)?.delete(t)},emit(e,t){i.get(e)?.forEach(s=>s(t))},removeAllListeners(){i.clear()}}}function v(i){const e=new Int16Array(i.length);for(let t=0;t<i.length;t++){const s=Math.max(-1,Math.min(1,i[t]));e[t]=s<0?s*32768:s*32767}return new Uint8Array(e.buffer)}const C=16e3;class p{ws=null;_state="idle";events=S();inputSampleRate=C;get state(){return this._state}setState(e){this._state!==e&&(this._state=e,this.events.emit("stateChange",e))}handleMessage(e){if(e instanceof Blob){e.text().then(t=>this.parseAndHandle(t));return}this.parseAndHandle(e)}parseAndHandle(e){try{const t=JSON.parse(e);this.handleParsedMessage(t)}catch{console.warn(`[${this.agentName}] Failed to parse message:`,e.slice(0,200))}}close(){this.ws&&(this.ws.close(),this.ws=null),this.events.removeAllListeners(),this.setState("idle")}on(e,t){this.events.on(e,t)}off(e,t){this.events.off(e,t)}emitClosed(e,t){this.events.emit("closed",{code:e,reason:t})}}const k="gemini-2.5-flash-native-audio-preview-12-2025",R="wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent",M="wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";class f extends p{agentName="GeminiLive";async connect(e){if(this.ws)throw new Error("Already connected");if(!e.apiKey)throw new Error("Gemini API key is required");e.inputSampleRate&&(this.inputSampleRate=e.inputSampleRate);const t=e.model??k,n=(e.authType??"api_key")==="ephemeral_token"?`${M}?access_token=${encodeURIComponent(e.apiKey)}`:`${R}?key=${encodeURIComponent(e.apiKey)}`;return new Promise((a,o)=>{this.ws=new WebSocket(n),this.ws.onopen=()=>{const r={setup:{model:`models/${t}`,generationConfig:{responseModalities:["AUDIO"]},systemInstruction:e.systemPrompt?{parts:[{text:e.systemPrompt}]}:void 0}};this.ws.send(JSON.stringify(r)),this.setState("listening"),a()},this.ws.onerror=()=>{o(new Error("Failed to connect to Gemini Live"))},this.ws.onclose=r=>{this.ws=null,this.setState("idle"),this.emitClosed(r.code,r.reason)},this.ws.onmessage=r=>{this.handleMessage(r.data)}})}handleParsedMessage(e){const s=e.serverContent;if(s){if(s.interrupted){this.events.emit("interrupted",void 0),this.setState("listening");return}if(s.turnComplete){this.events.emit("turnEnd",void 0),this.setState("listening");return}if(s.modelTurn?.parts){this._state!=="speaking"&&(this.events.emit("turnStart",void 0),this.setState("speaking"));for(const n of s.modelTurn.parts){if(n.inlineData?.data){const a=c(n.inlineData.data);this.events.emit("audio",a)}n.text&&this.events.emit("transcript",{role:"assistant",text:n.text,isFinal:!0})}}}}sendAudio(e){if(!this.ws||this.ws.readyState!==WebSocket.OPEN){console.warn("[GeminiLive] Cannot send audio: not connected");return}const t={realtimeInput:{mediaChunks:[{mimeType:`audio/pcm;rate=${this.inputSampleRate}`,data:u(e)}]}};this.ws.send(JSON.stringify(t))}}const P="wss://api.elevenlabs.io/v1/convai/conversation";class w extends p{agentName="ElevenLabs";outputSampleRate=24e3;expectedInputSampleRate=16e3;sourceInputSampleRate=16e3;initialized=!1;lastInterruptId=0;async connect(e){if(this.ws)throw new Error("Already connected");if(!e.agentId&&!e.signedUrl)throw new Error("ElevenLabs agent ID or signed URL is required");e.inputSampleRate&&(this.sourceInputSampleRate=e.inputSampleRate);let t;return e.signedUrl?t=e.signedUrl:(t=`${P}?agent_id=${e.agentId}`,e.apiKey&&(t+=`&xi-api-key=${e.apiKey}`)),new Promise((s,n)=>{this.ws=new WebSocket(t),this.ws.onopen=()=>{this.setState("listening"),s()},this.ws.onerror=()=>{n(new Error("Failed to connect to ElevenLabs"))},this.ws.onclose=a=>{this.ws=null,this.setState("idle"),this.emitClosed(a.code,a.reason)},this.ws.onmessage=a=>{this.handleMessage(a.data)}})}handleParsedMessage(e){const t=e;switch(t.type){case"conversation_initiation_metadata":this.handleInitMetadata(t);break;case"ping":this.handlePing(t);break;case"audio":this.handleAudio(t);break;case"user_transcript":this.handleUserTranscript(t);break;case"agent_response":this.handleAgentResponse(t);break;case"interruption":this.handleInterruption(t);break;case"agent_response_correction":this.setState("listening");break}}handleInitMetadata(e){const t=e.conversation_initiation_metadata_event;if(t){if(t.agent_output_audio_format){const s=t.agent_output_audio_format.match(/pcm_(\d+)/);s&&(this.outputSampleRate=parseInt(s[1],10))}if(t.user_input_audio_format){const s=t.user_input_audio_format.match(/pcm_(\d+)/);s&&(this.expectedInputSampleRate=parseInt(s[1],10))}this.initialized=!0}}handlePing(e){if(this.ws&&this.ws.readyState===WebSocket.OPEN){const t=e.ping_event?.event_id;this.ws.send(JSON.stringify({type:"pong",event_id:t}))}}handleAudio(e){const t=e.audio_event;if(!t?.audio_base_64||(t.event_id??0)<=this.lastInterruptId)return;this._state!=="speaking"&&(this.events.emit("turnStart",void 0),this.setState("speaking"));let n=c(t.audio_base_64);this.outputSampleRate!==h&&(n=d(n,this.outputSampleRate,h));const a=4800;if(n.length<=a)this.events.emit("audio",n);else for(let o=0;o<n.length;o+=a){const r=n.slice(o,Math.min(o+a,n.length));this.events.emit("audio",r)}}handleUserTranscript(e){const t=e.user_transcription_event;t?.user_transcript&&this.events.emit("transcript",{role:"user",text:t.user_transcript,isFinal:!0})}handleAgentResponse(e){const t=e.agent_response_event;t?.agent_response&&this.events.emit("transcript",{role:"assistant",text:t.agent_response,isFinal:!0})}handleInterruption(e){const t=e.interruption_event;t?.event_id&&(this.lastInterruptId=t.event_id),this.events.emit("interrupted",void 0),this.setState("listening")}sendAudio(e){if(!this.ws||this.ws.readyState!==WebSocket.OPEN||!this.initialized)return;let t=e;this.sourceInputSampleRate!==this.expectedInputSampleRate&&(t=d(e,this.sourceInputSampleRate,this.expectedInputSampleRate)),this.ws.send(JSON.stringify({user_audio_chunk:u(t)}))}sendText(e){if(!this.ws||this.ws.readyState!==WebSocket.OPEN){console.warn("[ElevenLabs] Cannot send text: not connected");return}this.ws.send(JSON.stringify({type:"user_message",text:e}))}sendContext(e){if(!this.ws||this.ws.readyState!==WebSocket.OPEN){console.warn("[ElevenLabs] Cannot send context: not connected");return}this.ws.send(JSON.stringify({type:"contextual_update",text:e}))}close(){this.initialized=!1,this.lastInterruptId=0,super.close()}}const T="wss://api.cartesia.ai/agents/stream",N="2025-04-16";class y extends p{agentName="Cartesia";cartesiaInputFormat="pcm_16000";cartesiaOutputRate=16e3;streamId=null;isReady=!1;pingInterval=null;async connect(e){if(this.ws)throw new Error("Already connected");if(!e.agentId)throw new Error("Cartesia Agent ID is required");if(!e.apiKey)throw new Error("Cartesia API Key is required");e.inputSampleRate&&(this.inputSampleRate=e.inputSampleRate),this.inputSampleRate===16e3?this.cartesiaInputFormat="pcm_16000":this.inputSampleRate===24e3?this.cartesiaInputFormat="pcm_24000":this.inputSampleRate===44100?this.cartesiaInputFormat="pcm_44100":this.cartesiaInputFormat="pcm_16000";const t=`${T}/${e.agentId}?api_key=${e.apiKey}&cartesia_version=${N}`;return new Promise((s,n)=>{this.ws=new WebSocket(t),this.ws.onopen=()=>{this.sendStartEvent(),this.startHeartbeat(),s()},this.ws.onerror=()=>{n(new Error("Failed to connect to Cartesia"))},this.ws.onclose=a=>{this.stopHeartbeat(),this.ws=null,this.isReady=!1,this.streamId=null,this.setState("idle"),this.emitClosed(a.code,a.reason)},this.ws.onmessage=a=>{this.handleMessage(a.data)}})}sendStartEvent(){if(!this.ws)return;const e={event:"start",config:{input_format:this.cartesiaInputFormat}};this.ws.send(JSON.stringify(e))}startHeartbeat(){this.pingInterval=window.setInterval(()=>{this.ws?.readyState===WebSocket.OPEN&&this.streamId&&this.ws.send(JSON.stringify({event:"custom",stream_id:this.streamId,metadata:{keepalive:!0}}))},2e4)}stopHeartbeat(){this.pingInterval&&(clearInterval(this.pingInterval),this.pingInterval=null)}handleParsedMessage(e){const t=e;switch(t.event){case"ack":this.handleAck(t);break;case"media_output":this.handleMediaOutput(t);break;case"clear":this.handleClear();break;case"error":console.error("[Cartesia] Server error:",t);break}}handleAck(e){this.streamId=e.stream_id||null,this.isReady=!0,this.setState("listening")}handleMediaOutput(e){if(!e.media?.payload)return;this._state!=="speaking"&&(this.events.emit("turnStart",void 0),this.setState("speaking"));let t=c(e.media.payload);this.cartesiaOutputRate!==h&&(t=d(t,this.cartesiaOutputRate,h)),this.events.emit("audio",t)}handleClear(){this.events.emit("interrupted",void 0),this.setState("listening")}sendAudio(e){if(!this.ws||this.ws.readyState!==WebSocket.OPEN||!this.isReady||!this.streamId)return;let t=e;const s=parseInt(this.cartesiaInputFormat.split("_")[1]);this.inputSampleRate!==s&&(t=d(e,this.inputSampleRate,s)),this.ws.send(JSON.stringify({event:"media_input",stream_id:this.streamId,media:{payload:u(t)}}))}close(){this.stopHeartbeat(),this.isReady=!1,this.streamId=null,super.close()}}const b=[{id:"gemini",name:"Gemini Live",description:"Google Gemini Live API"},{id:"elevenlabs",name:"ElevenLabs",description:"ElevenLabs Conversational AI"},{id:"cartesia",name:"Cartesia",description:"Cartesia Agents API"}];function I(i){switch(i){case"gemini":return new f;case"elevenlabs":return new w;case"cartesia":return new y;default:throw new Error(`Unknown agent type: ${i}`)}}function O(i){return b.find(e=>e.id===i)}const l=new Set;class L{apiBaseUrl;publishableKey;callbacks;_video;_audio;session=null;agent=null;audioContext=null;processor=null;stream=null;abortController=null;_status="disconnected";_agentState="idle";_isMuted=!1;mounted=!0;constructor(e){this.apiBaseUrl=e.apiBaseUrl??"https://api.keyframelabs.com",this.publishableKey=e.publishableKey,this.callbacks={onDisconnect:e.onDisconnect,onError:e.onError,onStateChange:e.onStateChange,onAgentStateChange:e.onAgentStateChange},this._video=document.createElement("video"),this._video.style.position="absolute",this._video.style.inset="0",this._video.style.width="100%",this._video.style.height="100%",this._video.style.objectFit=e.videoFit??"cover",this._video.autoplay=!0,this._video.playsInline=!0,this._video.muted=!0,e.container.style.position="relative",e.container.style.backgroundColor="#000",this._audio=document.createElement("audio"),this._audio.autoplay=!0,e.container.appendChild(this._video),e.container.appendChild(this._audio)}get status(){return this._status}get agentState(){return this._agentState}get isMuted(){return this._isMuted}get videoElement(){return this._video}get audioElement(){return this._audio}async connect(){if(l.has(this.publishableKey)){console.log("[PersonaEmbed] Connection already in progress, skipping");return}l.add(this.publishableKey),this.mounted=!0,this.abortController=new AbortController,this.setStatus("connecting");try{const e=await this.fetchSession(this.abortController.signal);if(!this.mounted){l.delete(this.publishableKey);return}if(await this.initSession(e),await this.initMicrophone(),await this.connectAgent(e.voice_agent_details),!this.mounted){this.cleanup(),l.delete(this.publishableKey);return}this.setStatus("connected")}catch(e){if(l.delete(this.publishableKey),e instanceof Error&&e.name==="AbortError")return;console.error("[PersonaEmbed]",e),this.mounted&&(this.setStatus("error"),this.callbacks.onError?.(e))}}disconnect(){this.mounted=!1,this.abortController?.abort(),this.abortController=null,l.delete(this.publishableKey),this.cleanup(),this.setStatus("disconnected")}toggleMute(){this._isMuted=!this._isMuted}setStatus(e){this._status!==e&&(this._status=e,this.callbacks.onStateChange?.(e))}setAgentState(e){this._agentState!==e&&(this._agentState=e,this.callbacks.onAgentStateChange?.(e))}async fetchSession(e){const t=await fetch(`${this.apiBaseUrl}/v1/embed/create_session`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({publishable_key:this.publishableKey}),signal:e});if(!t.ok){const s=await t.json().catch(()=>null);throw new Error(`create_session failed: ${t.status} ${JSON.stringify(s)}`)}return t.json()}async initSession(e){this.session=A.createClient({serverUrl:e.session_details.server_url,participantToken:e.session_details.participant_token,agentIdentity:e.session_details.agent_identity,onVideoTrack:t=>{console.log("[PersonaEmbed] Setting video track",t.readyState,t.enabled),this._video.srcObject=new MediaStream([t]),this._video.play().catch(s=>console.warn("[PersonaEmbed] Video play failed:",s))},onAudioTrack:t=>{this._audio.srcObject=new MediaStream([t]),this._audio.play().catch(()=>{})},onStateChange:t=>{this.mounted&&t==="disconnected"&&(this.setStatus("disconnected"),this.callbacks.onDisconnect?.())},onError:t=>{this.mounted&&this.callbacks.onError?.(t)},onClose:()=>{this.mounted&&this.callbacks.onDisconnect?.()}}),this.agent=I(e.voice_agent_details.type),this.agent.on("audio",t=>this.session?.sendAudio(t)),this.agent.on("interrupted",()=>this.session?.interrupt()),this.agent.on("stateChange",t=>this.setAgentState(t)),this.agent.on("closed",()=>{this.mounted&&this.callbacks.onDisconnect?.()}),await this.session.connect()}async initMicrophone(){this.stream=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:16e3,echoCancellation:!0,noiseSuppression:!0}}),this.audioContext=new AudioContext({sampleRate:16e3});const e=this.audioContext.createMediaStreamSource(this.stream);this.processor=this.audioContext.createScriptProcessor(4096,1,1),this.processor.onaudioprocess=t=>{if(!this._isMuted){const s=v(t.inputBuffer.getChannelData(0));this.agent?.sendAudio(s)}},e.connect(this.processor),this.processor.connect(this.audioContext.destination)}async connectAgent(e){if(!this.agent)return;const t={inputSampleRate:16e3};e.type==="gemini"?await this.agent.connect({...t,apiKey:e.token,authType:"ephemeral_token"}):e.type==="elevenlabs"?await this.agent.connect({...t,agentId:e.agent_id,signedUrl:e.signed_url}):e.type==="cartesia"&&await this.agent.connect({...t,agentId:e.agent_id,apiKey:e.token})}cleanup(){this.stream?.getTracks().forEach(e=>e.stop()),this.processor?.disconnect(),this.audioContext?.close(),this.agent?.close(),this.session?.close(),this.stream=null,this.processor=null,this.audioContext=null,this.agent=null,this.session=null}}exports.AGENT_REGISTRY=b;exports.BaseAgent=p;exports.CartesiaAgent=y;exports.ElevenLabsAgent=w;exports.GeminiLiveAgent=f;exports.PersonaEmbed=L;exports.SAMPLE_RATE=h;exports.base64ToBytes=c;exports.bytesToBase64=u;exports.createAgent=I;exports.createEventEmitter=S;exports.floatTo16BitPCM=v;exports.getAgentInfo=O;exports.resamplePcm=d;

package/dist/index.mjs

@@ -0,0 +1,636 @@
+import { createClient as v } from "@keyframelabs/sdk";
+const h = 24e3;
+function u(i) {
+  const e = atob(i), t = new Uint8Array(e.length);
+  for (let s = 0; s < e.length; s++)
+    t[s] = e.charCodeAt(s);
+  return t;
+}
+function p(i) {
+  let e = "";
+  for (let t = 0; t < i.length; t++)
+    e += String.fromCharCode(i[t]);
+  return btoa(e);
+}
+function d(i, e, t) {
+  if (e === t)
+    return i;
+  const s = new Int16Array(i.buffer, i.byteOffset, i.length / 2), n = e / t, a = Math.floor(s.length / n), o = new Int16Array(a);
+  for (let r = 0; r < a; r++) {
+    const g = r * n, c = Math.floor(g), S = Math.min(c + 1, s.length - 1), _ = g - c;
+    o[r] = Math.round(
+      s[c] * (1 - _) + s[S] * _
+    );
+  }
+  return new Uint8Array(o.buffer);
+}
+function f() {
+  const i = /* @__PURE__ */ new Map();
+  return {
+    on(e, t) {
+      i.has(e) || i.set(e, /* @__PURE__ */ new Set()), i.get(e).add(t);
+    },
+    off(e, t) {
+      i.get(e)?.delete(t);
+    },
+    emit(e, t) {
+      i.get(e)?.forEach((s) => s(t));
+    },
+    removeAllListeners() {
+      i.clear();
+    }
+  };
+}
+function w(i) {
+  const e = new Int16Array(i.length);
+  for (let t = 0; t < i.length; t++) {
+    const s = Math.max(-1, Math.min(1, i[t]));
+    e[t] = s < 0 ? s * 32768 : s * 32767;
+  }
+  return new Uint8Array(e.buffer);
+}
+const y = 16e3;
+class m {
+  ws = null;
+  _state = "idle";
+  events = f();
+  inputSampleRate = y;
+  /** Current agent state */
+  get state() {
+    return this._state;
+  }
+  /**
+   * Update state and emit stateChange event.
+   */
+  setState(e) {
+    this._state !== e && (this._state = e, this.events.emit("stateChange", e));
+  }
+  /**
+   * Handle WebSocket message (string or Blob).
+   * Converts to string and parses JSON before calling handleParsedMessage.
+   */
+  handleMessage(e) {
+    if (e instanceof Blob) {
+      e.text().then((t) => this.parseAndHandle(t));
+      return;
+    }
+    this.parseAndHandle(e);
+  }
+  /**
+   * Parse JSON and call handleParsedMessage if valid.
+   */
+  parseAndHandle(e) {
+    try {
+      const t = JSON.parse(e);
+      this.handleParsedMessage(t);
+    } catch {
+      console.warn(`[${this.agentName}] Failed to parse message:`, e.slice(0, 200));
+    }
+  }
+  /**
+   * Close the WebSocket connection and clean up resources.
+   * Subclasses can override to add custom cleanup, but should call super.close().
+   */
+  close() {
+    this.ws && (this.ws.close(), this.ws = null), this.events.removeAllListeners(), this.setState("idle");
+  }
+  /**
+   * Register an event handler.
+   */
+  on(e, t) {
+    this.events.on(e, t);
+  }
+  /**
+   * Remove an event handler.
+   */
+  off(e, t) {
+    this.events.off(e, t);
+  }
+  /**
+   * Helper to emit the closed event with code and reason.
+   */
+  emitClosed(e, t) {
+    this.events.emit("closed", { code: e, reason: t });
+  }
+}
+const b = "gemini-2.5-flash-native-audio-preview-12-2025", I = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", E = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
+class C extends m {
+  agentName = "GeminiLive";
+  async connect(e) {
+    if (this.ws)
+      throw new Error("Already connected");
+    if (!e.apiKey)
+      throw new Error("Gemini API key is required");
+    e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate);
+    const t = e.model ?? b, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${E}?access_token=${encodeURIComponent(e.apiKey)}` : `${I}?key=${encodeURIComponent(e.apiKey)}`;
+    return new Promise((a, o) => {
+      this.ws = new WebSocket(n), this.ws.onopen = () => {
+        const r = {
+          setup: {
+            model: `models/${t}`,
+            generationConfig: {
+              responseModalities: ["AUDIO"]
+            },
+            systemInstruction: e.systemPrompt ? { parts: [{ text: e.systemPrompt }] } : void 0
+          }
+        };
+        this.ws.send(JSON.stringify(r)), this.setState("listening"), a();
+      }, this.ws.onerror = () => {
+        o(new Error("Failed to connect to Gemini Live"));
+      }, this.ws.onclose = (r) => {
+        this.ws = null, this.setState("idle"), this.emitClosed(r.code, r.reason);
+      }, this.ws.onmessage = (r) => {
+        this.handleMessage(r.data);
+      };
+    });
+  }
+  handleParsedMessage(e) {
+    const s = e.serverContent;
+    if (s) {
+      if (s.interrupted) {
+        this.events.emit("interrupted", void 0), this.setState("listening");
+        return;
+      }
+      if (s.turnComplete) {
+        this.events.emit("turnEnd", void 0), this.setState("listening");
+        return;
+      }
+      if (s.modelTurn?.parts) {
+        this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
+        for (const n of s.modelTurn.parts) {
+          if (n.inlineData?.data) {
+            const a = u(n.inlineData.data);
+            this.events.emit("audio", a);
+          }
+          n.text && this.events.emit("transcript", {
+            role: "assistant",
+            text: n.text,
+            isFinal: !0
+          });
+        }
+      }
+    }
+  }
+  sendAudio(e) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      console.warn("[GeminiLive] Cannot send audio: not connected");
+      return;
+    }
+    const t = {
+      realtimeInput: {
+        mediaChunks: [
+          {
+            mimeType: `audio/pcm;rate=${this.inputSampleRate}`,
+            data: p(e)
+          }
+        ]
+      }
+    };
+    this.ws.send(JSON.stringify(t));
+  }
+}
+const A = "wss://api.elevenlabs.io/v1/convai/conversation";
+class k extends m {
+  agentName = "ElevenLabs";
+  outputSampleRate = 24e3;
+  // Default, updated from metadata
+  expectedInputSampleRate = 16e3;
+  // What ElevenLabs expects, updated from metadata
+  sourceInputSampleRate = 16e3;
+  // What caller sends via sendAudio, from config
+  initialized = !1;
+  // True after conversation_initiation_metadata received
+  lastInterruptId = 0;
+  // Track interruptions to filter stale audio
+  async connect(e) {
+    if (this.ws)
+      throw new Error("Already connected");
+    if (!e.agentId && !e.signedUrl)
+      throw new Error("ElevenLabs agent ID or signed URL is required");
+    e.inputSampleRate && (this.sourceInputSampleRate = e.inputSampleRate);
+    let t;
+    return e.signedUrl ? t = e.signedUrl : (t = `${A}?agent_id=${e.agentId}`, e.apiKey && (t += `&xi-api-key=${e.apiKey}`)), new Promise((s, n) => {
+      this.ws = new WebSocket(t), this.ws.onopen = () => {
+        this.setState("listening"), s();
+      }, this.ws.onerror = () => {
+        n(new Error("Failed to connect to ElevenLabs"));
+      }, this.ws.onclose = (a) => {
+        this.ws = null, this.setState("idle"), this.emitClosed(a.code, a.reason);
+      }, this.ws.onmessage = (a) => {
+        this.handleMessage(a.data);
+      };
+    });
+  }
+  handleParsedMessage(e) {
+    const t = e;
+    switch (t.type) {
+      case "conversation_initiation_metadata":
+        this.handleInitMetadata(t);
+        break;
+      case "ping":
+        this.handlePing(t);
+        break;
+      case "audio":
+        this.handleAudio(t);
+        break;
+      case "user_transcript":
+        this.handleUserTranscript(t);
+        break;
+      case "agent_response":
+        this.handleAgentResponse(t);
+        break;
+      case "interruption":
+        this.handleInterruption(t);
+        break;
+      case "agent_response_correction":
+        this.setState("listening");
+        break;
+    }
+  }
+  handleInitMetadata(e) {
+    const t = e.conversation_initiation_metadata_event;
+    if (t) {
+      if (t.agent_output_audio_format) {
+        const s = t.agent_output_audio_format.match(/pcm_(\d+)/);
+        s && (this.outputSampleRate = parseInt(s[1], 10));
+      }
+      if (t.user_input_audio_format) {
+        const s = t.user_input_audio_format.match(/pcm_(\d+)/);
+        s && (this.expectedInputSampleRate = parseInt(s[1], 10));
+      }
+      this.initialized = !0;
+    }
+  }
+  handlePing(e) {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+      const t = e.ping_event?.event_id;
+      this.ws.send(JSON.stringify({ type: "pong", event_id: t }));
+    }
+  }
+  handleAudio(e) {
+    const t = e.audio_event;
+    if (!t?.audio_base_64 || (t.event_id ?? 0) <= this.lastInterruptId)
+      return;
+    this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
+    let n = u(t.audio_base_64);
+    this.outputSampleRate !== h && (n = d(n, this.outputSampleRate, h));
+    const a = 4800;
+    if (n.length <= a)
+      this.events.emit("audio", n);
+    else
+      for (let o = 0; o < n.length; o += a) {
+        const r = n.slice(o, Math.min(o + a, n.length));
+        this.events.emit("audio", r);
+      }
+  }
+  handleUserTranscript(e) {
+    const t = e.user_transcription_event;
+    t?.user_transcript && this.events.emit("transcript", {
+      role: "user",
+      text: t.user_transcript,
+      isFinal: !0
+    });
+  }
+  handleAgentResponse(e) {
+    const t = e.agent_response_event;
+    t?.agent_response && this.events.emit("transcript", {
+      role: "assistant",
+      text: t.agent_response,
+      isFinal: !0
+    });
+  }
+  handleInterruption(e) {
+    const t = e.interruption_event;
+    t?.event_id && (this.lastInterruptId = t.event_id), this.events.emit("interrupted", void 0), this.setState("listening");
+  }
+  sendAudio(e) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN || !this.initialized)
+      return;
+    let t = e;
+    this.sourceInputSampleRate !== this.expectedInputSampleRate && (t = d(e, this.sourceInputSampleRate, this.expectedInputSampleRate)), this.ws.send(JSON.stringify({
+      user_audio_chunk: p(t)
+    }));
+  }
+  /**
+   * Send a text message as if the user spoke it.
+   */
+  sendText(e) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      console.warn("[ElevenLabs] Cannot send text: not connected");
+      return;
+    }
+    this.ws.send(JSON.stringify({
+      type: "user_message",
+      text: e
+    }));
+  }
+  /**
+   * Send contextual information to the agent without interrupting.
+   */
+  sendContext(e) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      console.warn("[ElevenLabs] Cannot send context: not connected");
+      return;
+    }
+    this.ws.send(JSON.stringify({
+      type: "contextual_update",
+      text: e
+    }));
+  }
+  close() {
+    this.initialized = !1, this.lastInterruptId = 0, super.close();
+  }
+}
+const R = "wss://api.cartesia.ai/agents/stream", M = "2025-04-16";
+class P extends m {
+  agentName = "Cartesia";
+  // Audio configuration
+  cartesiaInputFormat = "pcm_16000";
+  // Format we tell Cartesia we are sending
+  cartesiaOutputRate = 16e3;
+  // Cartesia defaults to 16kHz for web
+  // Connection state
+  streamId = null;
+  isReady = !1;
+  pingInterval = null;
+  async connect(e) {
+    if (this.ws)
+      throw new Error("Already connected");
+    if (!e.agentId)
+      throw new Error("Cartesia Agent ID is required");
+    if (!e.apiKey)
+      throw new Error("Cartesia API Key is required");
+    e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate), this.inputSampleRate === 16e3 ? this.cartesiaInputFormat = "pcm_16000" : this.inputSampleRate === 24e3 ? this.cartesiaInputFormat = "pcm_24000" : this.inputSampleRate === 44100 ? this.cartesiaInputFormat = "pcm_44100" : this.cartesiaInputFormat = "pcm_16000";
+    const t = `${R}/${e.agentId}?api_key=${e.apiKey}&cartesia_version=${M}`;
+    return new Promise((s, n) => {
+      this.ws = new WebSocket(t), this.ws.onopen = () => {
+        this.sendStartEvent(), this.startHeartbeat(), s();
+      }, this.ws.onerror = () => {
+        n(new Error("Failed to connect to Cartesia"));
+      }, this.ws.onclose = (a) => {
+        this.stopHeartbeat(), this.ws = null, this.isReady = !1, this.streamId = null, this.setState("idle"), this.emitClosed(a.code, a.reason);
+      }, this.ws.onmessage = (a) => {
+        this.handleMessage(a.data);
+      };
+    });
+  }
+  sendStartEvent() {
+    if (!this.ws) return;
+    const e = {
+      event: "start",
+      config: {
+        input_format: this.cartesiaInputFormat
+      }
+    };
+    this.ws.send(JSON.stringify(e));
+  }
+  /**
+   * Keep connection alive with periodic custom events.
+   * Cartesia requires activity every 30s.
+   */
+  startHeartbeat() {
+    this.pingInterval = window.setInterval(() => {
+      this.ws?.readyState === WebSocket.OPEN && this.streamId && this.ws.send(JSON.stringify({
+        event: "custom",
+        stream_id: this.streamId,
+        metadata: { keepalive: !0 }
+      }));
+    }, 2e4);
+  }
+  stopHeartbeat() {
+    this.pingInterval && (clearInterval(this.pingInterval), this.pingInterval = null);
+  }
+  handleParsedMessage(e) {
+    const t = e;
+    switch (t.event) {
+      case "ack":
+        this.handleAck(t);
+        break;
+      case "media_output":
+        this.handleMediaOutput(t);
+        break;
+      case "clear":
+        this.handleClear();
+        break;
+      case "error":
+        console.error("[Cartesia] Server error:", t);
+        break;
+    }
+  }
+  handleAck(e) {
+    this.streamId = e.stream_id || null, this.isReady = !0, this.setState("listening");
+  }
+  handleMediaOutput(e) {
+    if (!e.media?.payload) return;
+    this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
+    let t = u(e.media.payload);
+    this.cartesiaOutputRate !== h && (t = d(t, this.cartesiaOutputRate, h)), this.events.emit("audio", t);
+  }
+  handleClear() {
+    this.events.emit("interrupted", void 0), this.setState("listening");
+  }
+  sendAudio(e) {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN || !this.isReady || !this.streamId)
+      return;
+    let t = e;
+    const s = parseInt(this.cartesiaInputFormat.split("_")[1]);
+    this.inputSampleRate !== s && (t = d(e, this.inputSampleRate, s)), this.ws.send(JSON.stringify({
+      event: "media_input",
+      stream_id: this.streamId,
+      media: {
+        payload: p(t)
+      }
+    }));
+  }
+  close() {
+    this.stopHeartbeat(), this.isReady = !1, this.streamId = null, super.close();
+  }
+}
+const x = [
+  { id: "gemini", name: "Gemini Live", description: "Google Gemini Live API" },
+  { id: "elevenlabs", name: "ElevenLabs", description: "ElevenLabs Conversational AI" },
+  { id: "cartesia", name: "Cartesia", description: "Cartesia Agents API" }
+];
+function N(i) {
+  switch (i) {
+    case "gemini":
+      return new C();
+    case "elevenlabs":
+      return new k();
+    case "cartesia":
+      return new P();
+    default:
+      throw new Error(`Unknown agent type: ${i}`);
+  }
+}
+function T(i) {
+  return x.find((e) => e.id === i);
+}
+const l = /* @__PURE__ */ new Set();
+class L {
+  apiBaseUrl;
+  publishableKey;
+  callbacks;
+  // DOM
+  _video;
+  _audio;
+  // Session
+  session = null;
+  agent = null;
+  audioContext = null;
+  processor = null;
+  stream = null;
+  abortController = null;
+  _status = "disconnected";
+  _agentState = "idle";
+  _isMuted = !1;
+  mounted = !0;
+  constructor(e) {
+    this.apiBaseUrl = e.apiBaseUrl ?? "https://api.keyframelabs.com", this.publishableKey = e.publishableKey, this.callbacks = {
+      onDisconnect: e.onDisconnect,
+      onError: e.onError,
+      onStateChange: e.onStateChange,
+      onAgentStateChange: e.onAgentStateChange
+    }, this._video = document.createElement("video"), this._video.style.position = "absolute", this._video.style.inset = "0", this._video.style.width = "100%", this._video.style.height = "100%", this._video.style.objectFit = e.videoFit ?? "cover", this._video.autoplay = !0, this._video.playsInline = !0, this._video.muted = !0, e.container.style.position = "relative", e.container.style.backgroundColor = "#000", this._audio = document.createElement("audio"), this._audio.autoplay = !0, e.container.appendChild(this._video), e.container.appendChild(this._audio);
+  }
+  // Read-only state
+  get status() {
+    return this._status;
+  }
+  get agentState() {
+    return this._agentState;
+  }
+  get isMuted() {
+    return this._isMuted;
+  }
+  get videoElement() {
+    return this._video;
+  }
+  get audioElement() {
+    return this._audio;
+  }
+  /** Connect to the embed session */
+  async connect() {
+    if (l.has(this.publishableKey)) {
+      console.log("[PersonaEmbed] Connection already in progress, skipping");
+      return;
+    }
+    l.add(this.publishableKey), this.mounted = !0, this.abortController = new AbortController(), this.setStatus("connecting");
+    try {
+      const e = await this.fetchSession(this.abortController.signal);
+      if (!this.mounted) {
+        l.delete(this.publishableKey);
+        return;
+      }
+      if (await this.initSession(e), await this.initMicrophone(), await this.connectAgent(e.voice_agent_details), !this.mounted) {
+        this.cleanup(), l.delete(this.publishableKey);
+        return;
+      }
+      this.setStatus("connected");
+    } catch (e) {
+      if (l.delete(this.publishableKey), e instanceof Error && e.name === "AbortError")
+        return;
+      console.error("[PersonaEmbed]", e), this.mounted && (this.setStatus("error"), this.callbacks.onError?.(e));
+    }
+  }
+  /** Disconnect and cleanup */
+  disconnect() {
+    this.mounted = !1, this.abortController?.abort(), this.abortController = null, l.delete(this.publishableKey), this.cleanup(), this.setStatus("disconnected");
+  }
+  /** Toggle microphone mute */
+  toggleMute() {
+    this._isMuted = !this._isMuted;
+  }
+  setStatus(e) {
+    this._status !== e && (this._status = e, this.callbacks.onStateChange?.(e));
+  }
+  setAgentState(e) {
+    this._agentState !== e && (this._agentState = e, this.callbacks.onAgentStateChange?.(e));
+  }
+  async fetchSession(e) {
+    const t = await fetch(`${this.apiBaseUrl}/v1/embed/create_session`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ publishable_key: this.publishableKey }),
+      signal: e
+    });
+    if (!t.ok) {
+      const s = await t.json().catch(() => null);
+      throw new Error(`create_session failed: ${t.status} ${JSON.stringify(s)}`);
+    }
+    return t.json();
+  }
+  async initSession(e) {
+    this.session = v({
+      serverUrl: e.session_details.server_url,
+      participantToken: e.session_details.participant_token,
+      agentIdentity: e.session_details.agent_identity,
+      onVideoTrack: (t) => {
+        console.log("[PersonaEmbed] Setting video track", t.readyState, t.enabled), this._video.srcObject = new MediaStream([t]), this._video.play().catch((s) => console.warn("[PersonaEmbed] Video play failed:", s));
+      },
+      onAudioTrack: (t) => {
+        this._audio.srcObject = new MediaStream([t]), this._audio.play().catch(() => {
+        });
+      },
+      onStateChange: (t) => {
+        this.mounted && t === "disconnected" && (this.setStatus("disconnected"), this.callbacks.onDisconnect?.());
+      },
+      onError: (t) => {
+        this.mounted && this.callbacks.onError?.(t);
+      },
+      onClose: () => {
+        this.mounted && this.callbacks.onDisconnect?.();
+      }
+    }), this.agent = N(e.voice_agent_details.type), this.agent.on("audio", (t) => this.session?.sendAudio(t)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (t) => this.setAgentState(t)), this.agent.on("closed", () => {
+      this.mounted && this.callbacks.onDisconnect?.();
+    }), await this.session.connect();
+  }
+  async initMicrophone() {
+    this.stream = await navigator.mediaDevices.getUserMedia({
+      audio: { sampleRate: 16e3, echoCancellation: !0, noiseSuppression: !0 }
+    }), this.audioContext = new AudioContext({ sampleRate: 16e3 });
+    const e = this.audioContext.createMediaStreamSource(this.stream);
+    this.processor = this.audioContext.createScriptProcessor(4096, 1, 1), this.processor.onaudioprocess = (t) => {
+      if (!this._isMuted) {
+        const s = w(t.inputBuffer.getChannelData(0));
+        this.agent?.sendAudio(s);
+      }
+    }, e.connect(this.processor), this.processor.connect(this.audioContext.destination);
+  }
+  async connectAgent(e) {
+    if (!this.agent) return;
+    const t = { inputSampleRate: 16e3 };
+    e.type === "gemini" ? await this.agent.connect({
+      ...t,
+      apiKey: e.token,
+      authType: "ephemeral_token"
+    }) : e.type === "elevenlabs" ? await this.agent.connect({
+      ...t,
+      agentId: e.agent_id,
+      signedUrl: e.signed_url
+    }) : e.type === "cartesia" && await this.agent.connect({
+      ...t,
+      agentId: e.agent_id,
+      apiKey: e.token
+    });
+  }
+  cleanup() {
+    this.stream?.getTracks().forEach((e) => e.stop()), this.processor?.disconnect(), this.audioContext?.close(), this.agent?.close(), this.session?.close(), this.stream = null, this.processor = null, this.audioContext = null, this.agent = null, this.session = null;
+  }
+}
+export {
+  x as AGENT_REGISTRY,
+  m as BaseAgent,
+  P as CartesiaAgent,
+  k as ElevenLabsAgent,
+  C as GeminiLiveAgent,
+  L as PersonaEmbed,
+  h as SAMPLE_RATE,
+  u as base64ToBytes,
+  p as bytesToBase64,
+  N as createAgent,
+  f as createEventEmitter,
+  w as floatTo16BitPCM,
+  T as getAgentInfo,
+  d as resamplePcm
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@keyframelabs/elements",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
   "types": "dist/index.d.ts",