@keyframelabs/elements 0.2.1 → 0.3.0

package/README.md

@@ -71,7 +71,7 @@ view.disconnect();
 
 ## Supported agents and real-time LLMs
 
-Supports Cartesia, ElevenLabs, Vapi, Gemini Live (closed alpha), OpenAI Realtime (closed alpha).
+Supports ElevenLabs and OpenAI Realtime.
 
 For `PersonaEmbed`, this is determined by the values you set in the Keyframe platform dashboard.
 
@@ -79,27 +79,35 @@ For `PersonaView`, this is determined by `voiceAgentDetails`.
 
 ## Emotion Controls
 
-The avatar can display emotional expressions (`neutral`, `angry`, `sad`, `happy`) that affect its facial expression and demeanor.
+The avatar can display emotional expressions (`neutral`, `angry`, `sad`, `happy`) that affect its facial expression and demeanor. All supported voice agents can drive emotions automatically via tool/function calling.
 
-### ElevenLabs: `set_emotion` Tool Call
+### Agent Events
+
+The `emotion` event is emitted when any agent triggers a `set_emotion` tool call:
+
+```typescript
+agent.on('emotion', (emotion) => {
+  console.log('Emotion changed:', emotion); // 'neutral' | 'angry' | 'sad' | 'happy'
+});
+```
 
-When using ElevenLabs as the voice agent, emotions are driven by a **client tool call** named `set_emotion`. The ElevenLabs agent parses incoming `client_tool_call` WebSocket messages and, when the tool name is `set_emotion`, updates the avatar's expression accordingly.
+When using `PersonaEmbed` or `PersonaView`, emotion events are automatically wired to the avatar session -- no extra code is needed.
 
-> **Important:** Transcripts from the ElevenLabs agent are **not** automatically consumed. The `transcript` event is emitted, but it is up to you to subscribe to it if you need transcript data.
+### ElevenLabs
 
-#### Setup
+Emotions are driven by a **client tool call** named `set_emotion`. The agent parses incoming `client_tool_call` WebSocket messages and sends a `client_tool_result` back.
 
-You must create a `set_emotion` tool in the [ElevenLabs API](https://elevenlabs.io/docs) for your agent. The tool should accept a single parameter:
+**Setup:** Create a `set_emotion` [client tool](https://elevenlabs.io/docs/conversational-ai/customization/tools/client-tools) in the ElevenLabs dashboard for your agent with a single `emotion` parameter (enum: `neutral`, `angry`, `sad`, `happy`). Then instruct your agent (via its system prompt) to call `set_emotion` on each turn.
 
-| Parameter | Type     | Description                                              |
-| --------- | -------- | -------------------------------------------------------- |
-| `emotion` | `enum` | One of `neutral`, `angry`, `sad`, `happy`. |
+### OpenAI Realtime
 
-Then instruct your agent (via its system prompt) to call `set_emotion` on each turn with the appropriate emotion. The client library handles the rest — it validates the emotion, emits an `emotion` event, and sends a `client_tool_result` back to ElevenLabs.
+The `set_emotion` function is **automatically declared** in the OpenAI Realtime session setup. The model calls it via Realtime [function calling](https://developers.openai.com/api/docs/guides/realtime-conversations#function-calling) (`response.done` with `function_call` output items), and the client responds by creating a `function_call_output` conversation item before asking the model to continue.
+
+**Setup:** No additional dashboard configuration is needed. Instruct the model via its system prompt to call `set_emotion` on each turn to reflect the tone of its response.
 
 ### Manual Emotion Control
 
-For other agents or custom emotion logic, you can access the underlying session to set emotions manually:
+For custom emotion logic outside of tool calling, you can access the underlying session directly:
 
 ```typescript
 import { createClient } from '@keyframelabs/sdk';
@@ -108,18 +116,6 @@ const session = createClient({ ... });
 await session.setEmotion('happy');
 ```
 
-### Agent Events
-
-The `emotion` event is emitted when the agent triggers a `set_emotion` tool call:
-
-```typescript
-agent.on('emotion', (emotion) => {
-  console.log('Emotion changed:', emotion); // 'neutral' | 'angry' | 'sad' | 'happy'
-});
-```
-
-Currently, only the ElevenLabs agent emits emotion events via tool calls.
-
 ## API
 
 ### `PersonaEmbed`
@@ -188,10 +184,12 @@ type SessionDetails = {
 };
 
 type VoiceAgentDetails = {
-  type: 'cartesia' | 'elevenlabs' | 'vapi' | 'gemini' | 'openai';
-  token?: string;       // For gemini, cartesia
-  agent_id?: string;    // For elevenlabs, cartesia
-  signed_url?: string;  // For elevenlabs, vapi
+  type: 'elevenlabs' | 'openai';
+  token?: string;        // For openai (ephemeral client secret)
+  agent_id?: string;     // For elevenlabs
+  signed_url?: string;   // For elevenlabs
+  system_prompt?: string; // For openai
+  voice?: string;         // For openai
 };
 
 type Emotion = 'neutral' | 'angry' | 'sad' | 'happy';

package/dist/agents/audio-utils.d.ts

@@ -3,7 +3,7 @@
  *
  * These utilities help with PCM audio processing for voice AI integrations.
  */
-/** Sample rate for audio sent to Persona (matches Gemini output) */
+/** Standard output sample rate for audio sent to Persona */
 export declare const SAMPLE_RATE = 24000;
 /**
  * Convert base64-encoded audio to Uint8Array.

package/dist/agents/index.d.ts

@@ -1,7 +1,5 @@
-import { GeminiLiveAgent, GeminiLiveConfig } from './gemini-live';
 import { ElevenLabsAgent, ElevenLabsConfig } from './elevenlabs';
-import { CartesiaAgent, CartesiaConfig } from './cartesia';
-import { VapiAgent, VapiConfig } from './vapi';
+import { OpenAIRealtimeAgent, OpenAIRealtimeConfig, TurnDetection } from './openai-realtime';
 /**
  * Agent implementations for voice AI platforms.
  *
@@ -10,13 +8,11 @@ import { VapiAgent, VapiConfig } from './vapi';
  */
 export { BaseAgent, DEFAULT_INPUT_SAMPLE_RATE } from './base';
 export type { Agent, AgentConfig, AgentEventMap, AgentState, Emotion } from './types';
-export { GeminiLiveAgent, type GeminiLiveConfig };
 export { ElevenLabsAgent, type ElevenLabsConfig };
-export { CartesiaAgent, type CartesiaConfig };
-export { VapiAgent, type VapiConfig };
+export { OpenAIRealtimeAgent, type OpenAIRealtimeConfig, type TurnDetection };
 export { SAMPLE_RATE, base64ToBytes, bytesToBase64, resamplePcm, createEventEmitter, floatTo16BitPCM } from './audio-utils';
 /** Supported agent types */
-export type AgentType = 'gemini' | 'elevenlabs' | 'cartesia' | 'vapi';
+export type AgentType = 'elevenlabs' | 'openai';
 /** Agent type metadata */
 export interface AgentTypeInfo {
     id: AgentType;
@@ -27,26 +23,22 @@ export interface AgentTypeInfo {
 export declare const AGENT_REGISTRY: AgentTypeInfo[];
 /** Configuration types by agent type */
 export interface AgentConfigMap {
-    gemini: GeminiLiveConfig;
     elevenlabs: ElevenLabsConfig;
-    cartesia: CartesiaConfig;
-    vapi: VapiConfig;
+    openai: OpenAIRealtimeConfig;
 }
 /** Union type of all agent instances */
-export type AnyAgent = GeminiLiveAgent | ElevenLabsAgent | CartesiaAgent | VapiAgent;
+export type AnyAgent = ElevenLabsAgent | OpenAIRealtimeAgent;
 /**
  * Create an agent instance by type.
  *
  * @example
  * ```ts
- * const agent = createAgent('gemini');
- * await agent.connect({ apiKey: 'YOUR_KEY' });
+ * const agent = createAgent('elevenlabs');
+ * await agent.connect({ agentId: '...', signedUrl: '...' });
  * ```
  */
-export declare function createAgent(type: 'gemini'): GeminiLiveAgent;
 export declare function createAgent(type: 'elevenlabs'): ElevenLabsAgent;
-export declare function createAgent(type: 'cartesia'): CartesiaAgent;
-export declare function createAgent(type: 'vapi'): VapiAgent;
+export declare function createAgent(type: 'openai'): OpenAIRealtimeAgent;
 export declare function createAgent(type: AgentType): AnyAgent;
 /**
  * Get agent type metadata by ID.

package/dist/agents/openai-realtime.d.ts

@@ -0,0 +1,60 @@
+import { AgentConfig } from './types';
+import { BaseAgent } from './base';
+/**
+ * Turn detection configuration for OpenAI Realtime.
+ * @see https://developers.openai.com/api/docs/guides/realtime-vad
+ */
+export type TurnDetection = {
+    type: 'server_vad';
+    /** Activation threshold 0-1. Higher = requires louder audio. */
+    threshold?: number;
+    /** Audio (ms) to include before detected speech. */
+    prefix_padding_ms?: number;
+    /** Silence duration (ms) before speech stop is detected. */
+    silence_duration_ms?: number;
+} | {
+    type: 'semantic_vad';
+    /** How eager the model is to consider a turn finished. Default: 'auto'. */
+    eagerness?: 'low' | 'medium' | 'high' | 'auto';
+};
+/** OpenAI Realtime specific configuration */
+export interface OpenAIRealtimeConfig extends AgentConfig {
+    /** Model to use (defaults to gpt-realtime) */
+    model?: string;
+    /** Turn detection / VAD settings. Defaults to semantic_vad with eagerness 'high'. */
+    turnDetection?: TurnDetection;
+}
+/**
+ * OpenAI Realtime agent implementation.
+ *
+ * Handles WebSocket connection to OpenAI Realtime and converts
+ * audio responses to events that Persona SDK can consume.
+ */
+export declare class OpenAIRealtimeAgent extends BaseAgent {
+    protected readonly agentName = "OpenAIRealtime";
+    private connectResolve;
+    private connectReject;
+    private connectTimeout;
+    private initialSessionUpdate;
+    private currentResponseHasAudio;
+    private currentTranscript;
+    private readonly handledFunctionCallIds;
+    private sourceInputSampleRate;
+    private pendingFunctionCallStartedAtMs;
+    private pendingFunctionCallNames;
+    connect(config: OpenAIRealtimeConfig): Promise<void>;
+    protected handleParsedMessage(message: unknown): void;
+    sendAudio(pcmData: Uint8Array): void;
+    close(): void;
+    private buildSessionUpdate;
+    private sendInitialSessionUpdate;
+    private handleResponseDone;
+    private handleFunctionCalls;
+    private handleFunctionCall;
+    private finishAudioTurn;
+    private resetTurnState;
+    private sendEvent;
+    private resolvePendingConnect;
+    private rejectPendingConnect;
+    private clearConnectTimeout;
+}

package/dist/agents/types.d.ts

@@ -39,7 +39,7 @@ export interface AgentEventMap {
         text: string;
         isFinal: boolean;
     };
-    /** Emotion change (currently only supported by ElevenLabs agent) */
+    /** Emotion change (supported by all agents: ElevenLabs, OpenAI Realtime) */
     emotion: Emotion;
     /** Agent connection closed (unexpected disconnect) */
     closed: {
@@ -50,7 +50,7 @@ export interface AgentEventMap {
 /**
  * Abstract agent interface.
  *
- * Implement this for each voice AI platform (Gemini, ElevenLabs, Cartesia, etc.)
+ * Implement this for each voice AI platform (ElevenLabs, OpenAI Realtime, etc.)
  */
 export interface Agent {
     /** Current agent state */

package/dist/index.d.ts

@@ -3,8 +3,8 @@ export type { PersonaEmbedOptions } from './PersonaEmbed';
 export { PersonaView } from './PersonaView';
 export type { PersonaViewOptions } from './PersonaView';
 export type { EmbedStatus, VideoFit, VoiceAgentDetails, SessionDetails, BaseCallbacks, } from './types';
-export { createAgent, GeminiLiveAgent, ElevenLabsAgent, CartesiaAgent, BaseAgent, AGENT_REGISTRY, getAgentInfo, } from './agents';
-export type { AgentType, AgentConfig, AgentEventMap, Agent, AnyAgent, AgentTypeInfo, GeminiLiveConfig, ElevenLabsConfig, CartesiaConfig, } from './agents';
+export { createAgent, ElevenLabsAgent, OpenAIRealtimeAgent, BaseAgent, AGENT_REGISTRY, getAgentInfo, } from './agents';
+export type { AgentType, AgentConfig, AgentEventMap, Agent, AnyAgent, AgentTypeInfo, ElevenLabsConfig, OpenAIRealtimeConfig, TurnDetection, } from './agents';
 export type { AgentState } from '@keyframelabs/sdk';
 export { floatTo16BitPCM, resamplePcm, base64ToBytes, bytesToBase64, SAMPLE_RATE, createEventEmitter, } from './agents';
 export { ApiError as KeyframeApiError } from './ApiError';