@keyframelabs/elements 0.2.1 → 0.4.0

package/README.md

@@ -18,9 +18,10 @@ pnpm add @keyframelabs/elements
 
 ## Usage
 
-This package provides two primary classes depending on your integration strategy:
+This package provides three integration options depending on your strategy:
 1. **`PersonaEmbed`**: Fully managed. Uses a publishable key. Best for rapid frontend integration.
 2. **`PersonaView`**: Bring your own backend. Uses session tokens generated by your server.
+3. **`<kfl-embed>`**: Drop-in web component. Wraps `PersonaEmbed` with a complete widget UI (states, controls, positioning). Zero framework dependencies.
 
 ### Option A: PersonaEmbed (managed)
 
@@ -69,48 +70,108 @@ await view.connect();
 view.disconnect();
 ```
 
-## Supported agents and real-time LLMs
+### Option C: `<kfl-embed>` Web Component
+
+A self-registering custom element that wraps `PersonaEmbed` with a widget UI shell: minimized/active/hidden states, "Join call" button, minimize/expand toggle, corner positioning, and button color styling. Drop it into any page with zero framework dependencies.
+
+```html
+<kfl-embed
+  publishable-key="kfl_pk_live_..."
+  initial-state="minimized"
+  corner="bottom-right"
+  minimized-width="144"
+  minimized-height="216"
+  active-width="252"
+  active-height="377"
+  button-color="#919191"
+  button-color-opacity="0.3"
+  video-fit="cover"
+  preview-image="https://example.com/avatar.png"></kfl-embed>
+<script type="module" src="https://unpkg.com/@keyframelabs/elements/dist/kfl-embed.js"></script>
+```
 
-Supports Cartesia, ElevenLabs, Vapi, Gemini Live (closed alpha), OpenAI Realtime (closed alpha).
+#### Attributes
+
+| Attribute                        | Type                                                       | Default          | Description                                                        |
+| -------------------------------- | ---------------------------------------------------------- | ---------------- | ------------------------------------------------------------------ |
+| `publishable-key`                | `string`                                                   | Required         | Your publishable embed key.                                        |
+| `api-base-url`                   | `string`                                                   | Keyframe default | Base URL for the Keyframe API.                                     |
+| `initial-state`                  | `'minimized' \| 'active' \| 'hidden'`                      | `'minimized'`    | Widget state on first load.                                        |
+| `controlled-widget-state`        | `'minimized' \| 'active' \| 'hidden'`                      | —                | Externally control the widget state (overrides internal state).     |
+| `corner`                         | `'bottom-right' \| 'bottom-left' \| 'top-right' \| 'top-left'` | `'bottom-right'` | Which corner to anchor the widget (fixed mode only).               |
+| `inline`                         | boolean attribute                                          | —                | Use `position: relative` instead of `fixed`.                       |
+| `minimized-width`                | `number`                                                   | `144`            | Width in px when minimized.                                        |
+| `minimized-height`               | `number`                                                   | `216`            | Height in px when minimized.                                       |
+| `active-width`                   | `number`                                                   | `252`            | Width in px when active (in-call).                                 |
+| `active-height`                  | `number`                                                   | `377`            | Height in px when active (in-call).                                |
+| `hide-ui`                        | boolean attribute                                          | —                | Hides all overlay controls. Useful for building your own UI shell. |
+| `show-minimize-button`           | `'true' \| 'false'`                                        | `'true'`         | Show the X/minimize button on hover.                               |
+| `controlled-show-minimize-button`| `'true' \| 'false'`                                        | —                | Externally control the minimize button visibility.                 |
+| `button-color`                   | hex color string                                           | `'#919191'`      | Color of the "Join call" button.                                   |
+| `button-color-opacity`           | `number` (0–1)                                             | `0.3`            | Opacity of the "Join call" button background.                      |
+| `video-fit`                      | `'cover' \| 'contain'`                                     | `'cover'`        | Video scaling mode.                                                |
+| `preview-image`                  | URL string                                                 | —                | Image shown in the widget before a call starts.                    |
+
+#### Events
+
+| Event              | `detail`                                | Description                            |
+| ------------------ | --------------------------------------- | -------------------------------------- |
+| `statechange`      | `{ status: EmbedStatus }`               | Connection status changed.             |
+| `agentstatechange` | `{ state: AgentState }`                 | Avatar playback state changed.         |
+| `widgetstatechange`| `{ state: 'minimized' \| 'active' \| 'hidden' }` | Widget UI state changed.      |
+| `disconnected`     | —                                       | Session disconnected.                  |
+| `error`            | `{ error: Error }`                      | A fatal error occurred.                |
+
+#### JavaScript API
 
-For `PersonaEmbed`, this is determined by the values you set in the Keyframe platform dashboard.
+```ts
+const el = document.querySelector('kfl-embed');
 
-For `PersonaView`, this is determined by `voiceAgentDetails`.
+await el.micOn();     // Start session if needed, then unmute mic
+await el.micOff();    // Mute mic
+el.isMicOn();         // boolean
 
-## Emotion Controls
+await el.mute();      // Mute speaker audio
+await el.unmute();    // Unmute speaker audio
+el.isMuted();         // boolean
+```
 
-The avatar can display emotional expressions (`neutral`, `angry`, `sad`, `happy`) that affect its facial expression and demeanor.
+#### Build
 
-### ElevenLabs: `set_emotion` Tool Call
+The web component is built as a self-contained ES module via `vite.config.embed.ts`:
 
-When using ElevenLabs as the voice agent, emotions are driven by a **client tool call** named `set_emotion`. The ElevenLabs agent parses incoming `client_tool_call` WebSocket messages and, when the tool name is `set_emotion`, updates the avatar's expression accordingly.
+```bash
+pnpm build:embed   # → dist/kfl-embed.js
+```
 
-> **Important:** Transcripts from the ElevenLabs agent are **not** automatically consumed. The `transcript` event is emitted, but it is up to you to subscribe to it if you need transcript data.
+Import the auto-registering entry point (registers `<kfl-embed>` on `customElements`):
 
-#### Setup
+```ts
+import '@keyframelabs/elements/kfl-embed';
+```
 
-You must create a `set_emotion` tool in the [ElevenLabs API](https://elevenlabs.io/docs) for your agent. The tool should accept a single parameter:
+Or import the class directly for manual registration:
 
-| Parameter | Type     | Description                                              |
-| --------- | -------- | -------------------------------------------------------- |
-| `emotion` | `enum` | One of `neutral`, `angry`, `sad`, `happy`. |
+```ts
+import { KflEmbedElement } from '@keyframelabs/elements';
+customElements.define('my-embed', KflEmbedElement);
+```
 
-Then instruct your agent (via its system prompt) to call `set_emotion` on each turn with the appropriate emotion. The client library handles the rest — it validates the emotion, emits an `emotion` event, and sends a `client_tool_result` back to ElevenLabs.
+## Supported agents and real-time LLMs
 
-### Manual Emotion Control
+Supports ElevenLabs and OpenAI Realtime.
 
-For other agents or custom emotion logic, you can access the underlying session to set emotions manually:
+For `PersonaEmbed`, this is determined by the values you set in the Keyframe platform dashboard.
 
-```typescript
-import { createClient } from '@keyframelabs/sdk';
+For `PersonaView`, this is determined by `voiceAgentDetails`.
 
-const session = createClient({ ... });
-await session.setEmotion('happy');
-```
+## Emotion Controls
+
+The avatar can display emotional expressions (`neutral`, `angry`, `sad`, `happy`) that affect its facial expression and demeanor. All supported voice agents can drive emotions automatically via tool/function calling.
 
 ### Agent Events
 
-The `emotion` event is emitted when the agent triggers a `set_emotion` tool call:
+The `emotion` event is emitted when any agent triggers a `set_emotion` tool call:
 
 ```typescript
 agent.on('emotion', (emotion) => {
@@ -118,7 +179,30 @@ agent.on('emotion', (emotion) => {
 });
 ```
 
-Currently, only the ElevenLabs agent emits emotion events via tool calls.
+When using `PersonaEmbed` or `PersonaView`, emotion events are automatically wired to the avatar session -- no extra code is needed.
+
+### ElevenLabs
+
+Emotions are driven by a **client tool call** named `set_emotion`. The agent parses incoming `client_tool_call` WebSocket messages and sends a `client_tool_result` back.
+
+**Setup:** Create a `set_emotion` [client tool](https://elevenlabs.io/docs/conversational-ai/customization/tools/client-tools) in the ElevenLabs dashboard for your agent with a single `emotion` parameter (enum: `neutral`, `angry`, `sad`, `happy`). Then instruct your agent (via its system prompt) to call `set_emotion` on each turn.
+
+### OpenAI Realtime
+
+The `set_emotion` function is **automatically declared** in the OpenAI Realtime session setup. The model calls it via Realtime [function calling](https://developers.openai.com/api/docs/guides/realtime-conversations#function-calling) (`response.done` with `function_call` output items), and the client responds by creating a `function_call_output` conversation item before asking the model to continue.
+
+**Setup:** No additional dashboard configuration is needed. Instruct the model via its system prompt to call `set_emotion` on each turn to reflect the tone of its response.
+
+### Manual Emotion Control
+
+For custom emotion logic outside of tool calling, you can access the underlying session directly:
+
+```typescript
+import { createClient } from '@keyframelabs/sdk';
+
+const session = createClient({ ... });
+await session.setEmotion('happy');
+```
 
 ## API
 
@@ -188,10 +272,12 @@ type SessionDetails = {
 };
 
 type VoiceAgentDetails = {
-  type: 'cartesia' | 'elevenlabs' | 'vapi' | 'gemini' | 'openai';
-  token?: string;       // For gemini, cartesia
-  agent_id?: string;    // For elevenlabs, cartesia
-  signed_url?: string;  // For elevenlabs, vapi
+  type: 'elevenlabs' | 'openai';
+  token?: string;        // For openai (ephemeral client secret)
+  agent_id?: string;     // For elevenlabs
+  signed_url?: string;   // For elevenlabs
+  system_prompt?: string; // For openai
+  voice?: string;         // For openai
 };
 
 type Emotion = 'neutral' | 'angry' | 'sad' | 'happy';

package/dist/KflEmbedElement.d.ts

@@ -0,0 +1,55 @@
+/**
+ * <kfl-embed> Web Component
+ *
+ * Composes PersonaEmbed internally -- does NOT reimplement session/video/audio logic.
+ * Adds the widget UI shell: minimized/active/hidden states, "Join call" button,
+ * minimize/expand toggle, corner positioning CSS, button color styling.
+ */
+export declare class KflEmbedElement extends HTMLElement {
+    static get observedAttributes(): ("publishable-key" | "api-base-url" | "initial-state" | "controlled-widget-state" | "active-width" | "active-height" | "minimized-width" | "minimized-height" | "inline" | "corner" | "hide-ui" | "show-minimize-button" | "controlled-show-minimize-button" | "button-color" | "button-color-opacity" | "video-fit" | "preview-image")[];
+    private shadow;
+    private embed;
+    private _widgetState;
+    private _connected;
+    private _connecting;
+    private widgetEl;
+    private innerEl;
+    private containerEl;
+    private previewImg;
+    private spinnerEl;
+    private errorToast;
+    private errorTimer;
+    private joinBtn;
+    private endCallBtn;
+    private toggleBtn;
+    private revealBtn;
+    private toolbarEl;
+    private micBtn;
+    constructor();
+    connectedCallback(): void;
+    disconnectedCallback(): void;
+    attributeChangedCallback(name: string, _old: string | null, value: string | null): void;
+    mute(): Promise<void>;
+    unmute(): Promise<void>;
+    isMuted(): boolean;
+    canUnmute(): boolean;
+    micOn(): Promise<void>;
+    micOff(): Promise<void>;
+    isMicOn(): boolean;
+    canTurnOnMic(): boolean;
+    setEmotion(_emotion: 'neutral' | 'angry' | 'sad' | 'happy'): void;
+    private _getAttrNum;
+    private _getCorner;
+    private _applyLayout;
+    private _applyState;
+    private _shouldShowMinimizeButton;
+    private _handleJoinCall;
+    private _handleToggle;
+    private _handleEndCall;
+    private _handleReveal;
+    private _handleMicToggle;
+    private _updateMicIcon;
+    private _resetToMinimized;
+    private _showError;
+    private _connect;
+}

package/dist/agents/audio-utils.d.ts

@@ -3,7 +3,7 @@
  *
  * These utilities help with PCM audio processing for voice AI integrations.
  */
-/** Sample rate for audio sent to Persona (matches Gemini output) */
+/** Standard output sample rate for audio sent to Persona */
 export declare const SAMPLE_RATE = 24000;
 /**
  * Convert base64-encoded audio to Uint8Array.

package/dist/agents/index.d.ts

@@ -1,7 +1,5 @@
-import { GeminiLiveAgent, GeminiLiveConfig } from './gemini-live';
 import { ElevenLabsAgent, ElevenLabsConfig } from './elevenlabs';
-import { CartesiaAgent, CartesiaConfig } from './cartesia';
-import { VapiAgent, VapiConfig } from './vapi';
+import { OpenAIRealtimeAgent, OpenAIRealtimeConfig, TurnDetection } from './openai-realtime';
 /**
  * Agent implementations for voice AI platforms.
  *
@@ -10,13 +8,11 @@ import { VapiAgent, VapiConfig } from './vapi';
  */
 export { BaseAgent, DEFAULT_INPUT_SAMPLE_RATE } from './base';
 export type { Agent, AgentConfig, AgentEventMap, AgentState, Emotion } from './types';
-export { GeminiLiveAgent, type GeminiLiveConfig };
 export { ElevenLabsAgent, type ElevenLabsConfig };
-export { CartesiaAgent, type CartesiaConfig };
-export { VapiAgent, type VapiConfig };
+export { OpenAIRealtimeAgent, type OpenAIRealtimeConfig, type TurnDetection };
 export { SAMPLE_RATE, base64ToBytes, bytesToBase64, resamplePcm, createEventEmitter, floatTo16BitPCM } from './audio-utils';
 /** Supported agent types */
-export type AgentType = 'gemini' | 'elevenlabs' | 'cartesia' | 'vapi';
+export type AgentType = 'elevenlabs' | 'openai';
 /** Agent type metadata */
 export interface AgentTypeInfo {
     id: AgentType;
@@ -27,26 +23,22 @@ export interface AgentTypeInfo {
 export declare const AGENT_REGISTRY: AgentTypeInfo[];
 /** Configuration types by agent type */
 export interface AgentConfigMap {
-    gemini: GeminiLiveConfig;
     elevenlabs: ElevenLabsConfig;
-    cartesia: CartesiaConfig;
-    vapi: VapiConfig;
+    openai: OpenAIRealtimeConfig;
 }
 /** Union type of all agent instances */
-export type AnyAgent = GeminiLiveAgent | ElevenLabsAgent | CartesiaAgent | VapiAgent;
+export type AnyAgent = ElevenLabsAgent | OpenAIRealtimeAgent;
 /**
  * Create an agent instance by type.
  *
  * @example
  * ```ts
- * const agent = createAgent('gemini');
- * await agent.connect({ apiKey: 'YOUR_KEY' });
+ * const agent = createAgent('elevenlabs');
+ * await agent.connect({ agentId: '...', signedUrl: '...' });
  * ```
  */
-export declare function createAgent(type: 'gemini'): GeminiLiveAgent;
 export declare function createAgent(type: 'elevenlabs'): ElevenLabsAgent;
-export declare function createAgent(type: 'cartesia'): CartesiaAgent;
-export declare function createAgent(type: 'vapi'): VapiAgent;
+export declare function createAgent(type: 'openai'): OpenAIRealtimeAgent;
 export declare function createAgent(type: AgentType): AnyAgent;
 /**
  * Get agent type metadata by ID.

package/dist/agents/openai-realtime.d.ts

@@ -0,0 +1,60 @@
+import { AgentConfig } from './types';
+import { BaseAgent } from './base';
+/**
+ * Turn detection configuration for OpenAI Realtime.
+ * @see https://developers.openai.com/api/docs/guides/realtime-vad
+ */
+export type TurnDetection = {
+    type: 'server_vad';
+    /** Activation threshold 0-1. Higher = requires louder audio. */
+    threshold?: number;
+    /** Audio (ms) to include before detected speech. */
+    prefix_padding_ms?: number;
+    /** Silence duration (ms) before speech stop is detected. */
+    silence_duration_ms?: number;
+} | {
+    type: 'semantic_vad';
+    /** How eager the model is to consider a turn finished. Default: 'auto'. */
+    eagerness?: 'low' | 'medium' | 'high' | 'auto';
+};
+/** OpenAI Realtime specific configuration */
+export interface OpenAIRealtimeConfig extends AgentConfig {
+    /** Model to use (defaults to gpt-realtime) */
+    model?: string;
+    /** Turn detection / VAD settings. Defaults to semantic_vad with eagerness 'high'. */
+    turnDetection?: TurnDetection;
+}
+/**
+ * OpenAI Realtime agent implementation.
+ *
+ * Handles WebSocket connection to OpenAI Realtime and converts
+ * audio responses to events that Persona SDK can consume.
+ */
+export declare class OpenAIRealtimeAgent extends BaseAgent {
+    protected readonly agentName = "OpenAIRealtime";
+    private connectResolve;
+    private connectReject;
+    private connectTimeout;
+    private initialSessionUpdate;
+    private currentResponseHasAudio;
+    private currentTranscript;
+    private readonly handledFunctionCallIds;
+    private sourceInputSampleRate;
+    private pendingFunctionCallStartedAtMs;
+    private pendingFunctionCallNames;
+    connect(config: OpenAIRealtimeConfig): Promise<void>;
+    protected handleParsedMessage(message: unknown): void;
+    sendAudio(pcmData: Uint8Array): void;
+    close(): void;
+    private buildSessionUpdate;
+    private sendInitialSessionUpdate;
+    private handleResponseDone;
+    private handleFunctionCalls;
+    private handleFunctionCall;
+    private finishAudioTurn;
+    private resetTurnState;
+    private sendEvent;
+    private resolvePendingConnect;
+    private rejectPendingConnect;
+    private clearConnectTimeout;
+}

package/dist/agents/types.d.ts

@@ -39,7 +39,7 @@ export interface AgentEventMap {
         text: string;
         isFinal: boolean;
     };
-    /** Emotion change (currently only supported by ElevenLabs agent) */
+    /** Emotion change (supported by all agents: ElevenLabs, OpenAI Realtime) */
     emotion: Emotion;
     /** Agent connection closed (unexpected disconnect) */
     closed: {
@@ -50,7 +50,7 @@ export interface AgentEventMap {
 /**
  * Abstract agent interface.
  *
- * Implement this for each voice AI platform (Gemini, ElevenLabs, Cartesia, etc.)
+ * Implement this for each voice AI platform (ElevenLabs, OpenAI Realtime, etc.)
  */
 export interface Agent {
     /** Current agent state */

package/dist/index.d.ts

@@ -3,9 +3,10 @@ export type { PersonaEmbedOptions } from './PersonaEmbed';
 export { PersonaView } from './PersonaView';
 export type { PersonaViewOptions } from './PersonaView';
 export type { EmbedStatus, VideoFit, VoiceAgentDetails, SessionDetails, BaseCallbacks, } from './types';
-export { createAgent, GeminiLiveAgent, ElevenLabsAgent, CartesiaAgent, BaseAgent, AGENT_REGISTRY, getAgentInfo, } from './agents';
-export type { AgentType, AgentConfig, AgentEventMap, Agent, AnyAgent, AgentTypeInfo, GeminiLiveConfig, ElevenLabsConfig, CartesiaConfig, } from './agents';
+export { createAgent, ElevenLabsAgent, OpenAIRealtimeAgent, BaseAgent, AGENT_REGISTRY, getAgentInfo, } from './agents';
+export type { AgentType, AgentConfig, AgentEventMap, Agent, AnyAgent, AgentTypeInfo, ElevenLabsConfig, OpenAIRealtimeConfig, TurnDetection, } from './agents';
 export type { AgentState } from '@keyframelabs/sdk';
 export { floatTo16BitPCM, resamplePcm, base64ToBytes, bytesToBase64, SAMPLE_RATE, createEventEmitter, } from './agents';
+export { KflEmbedElement } from './KflEmbedElement';
 export { ApiError as KeyframeApiError } from './ApiError';
 export type { ApiErrorPayload as KeyframeApiErrorPayload } from './ApiError';