@keyframelabs/elements 0.0.7 → 0.0.9

package/README.md

@@ -77,6 +77,41 @@ For `PersonaEmbed`, this is determined by the values you set in the Keyframe pla
 
 For `PersonaView`, this is determined by `voiceAgentDetails`.
 
+## Emotion Controls
+
+The avatar can display emotional expressions (`neutral`, `angry`, `sad`, `happy`) that affect its facial expression and demeanor.
+
+### Automatic Emotion Detection (ElevenLabs)
+
+When using ElevenLabs as the voice agent, emotions are automatically detected from the agent's speech. The ElevenLabs agent parses emotion tags from audio alignment data (e.g., `[angry]`, `[happy]`) and the avatar expression updates in real-time.
+
+This requires no additional configuration—just configure your ElevenLabs agent to include emotion tags in its responses.
+
+### Manual Emotion Control
+
+For other agents or custom emotion logic, you can access the underlying session to set emotions manually:
+
+```typescript
+// Access the underlying SDK session for manual control
+// (Available when using @keyframelabs/sdk directly)
+import { createClient } from '@keyframelabs/sdk';
+
+const session = createClient({ ... });
+await session.setEmotion('happy');
+```
+
+### Agent Events
+
+The `emotion` event is emitted when an agent detects an emotion change:
+
+```typescript
+agent.on('emotion', (emotion) => {
+  console.log('Emotion detected:', emotion); // 'neutral' | 'angry' | 'sad' | 'happy'
+});
+```
+
+Currently, only the ElevenLabs agent emits emotion events.
+
 ## API
 
 ### `PersonaEmbed`
@@ -150,6 +185,8 @@ type VoiceAgentDetails = {
   agent_id?: string;    // For elevenlabs, cartesia
   signed_url?: string;  // For elevenlabs, vapi
 };
+
+type Emotion = 'neutral' | 'angry' | 'sad' | 'happy';
 ```
 
 ## License

package/dist/agents/elevenlabs.d.ts

@@ -20,6 +20,7 @@ export declare class ElevenLabsAgent extends BaseAgent {
     private sourceInputSampleRate;
     private initialized;
     private lastInterruptId;
+    private emotionEmittedForEventId;
     connect(config: ElevenLabsConfig): Promise<void>;
     protected handleParsedMessage(message: unknown): void;
     private handleInitMetadata;

package/dist/agents/index.d.ts

@@ -9,7 +9,7 @@ import { VapiAgent, VapiConfig } from './vapi';
  * various voice AI backends to the Persona SDK.
  */
 export { BaseAgent, DEFAULT_INPUT_SAMPLE_RATE } from './base';
-export type { Agent, AgentConfig, AgentEventMap, AgentState } from './types';
+export type { Agent, AgentConfig, AgentEventMap, AgentState, Emotion } from './types';
 export { GeminiLiveAgent, type GeminiLiveConfig };
 export { ElevenLabsAgent, type ElevenLabsConfig };
 export { CartesiaAgent, type CartesiaConfig };

package/dist/agents/types.d.ts

@@ -8,6 +8,8 @@
  */
 /** Agent state for UI */
 export type AgentState = 'idle' | 'listening' | 'thinking' | 'speaking';
+/** Emotion states for avatar expression */
+export type Emotion = 'neutral' | 'angry' | 'sad' | 'happy';
 /** Agent configuration */
 export interface AgentConfig {
     /** System prompt for the agent */
@@ -37,6 +39,8 @@ export interface AgentEventMap {
         text: string;
         isFinal: boolean;
     };
+    /** Emotion change (currently only supported by ElevenLabs agent) */
+    emotion: Emotion;
     /** Agent connection closed (unexpected disconnect) */
     closed: {
         code?: number;

package/dist/index.js

@@ -1,5 +1,5 @@
-import { createClient as v } from "@keyframelabs/sdk";
-const h = 24e3;
+import { createClient as w } from "@keyframelabs/sdk";
+const l = 24e3;
 function g(i) {
   const e = atob(i), t = new Uint8Array(e.length);
   for (let s = 0; s < e.length; s++)
@@ -15,16 +15,16 @@ function m(i) {
 function c(i, e, t) {
   if (e === t)
     return i;
-  const s = new Int16Array(i.buffer, i.byteOffset, i.length / 2), n = e / t, a = Math.floor(s.length / n), o = new Int16Array(a);
-  for (let r = 0; r < a; r++) {
-    const S = r * n, p = Math.floor(S), f = Math.min(p + 1, s.length - 1), _ = S - p;
-    o[r] = Math.round(
-      s[p] * (1 - _) + s[f] * _
+  const s = new Int16Array(i.buffer, i.byteOffset, i.length / 2), n = e / t, a = Math.floor(s.length / n), r = new Int16Array(a);
+  for (let o = 0; o < a; o++) {
+    const _ = o * n, p = Math.floor(_), b = Math.min(p + 1, s.length - 1), v = _ - p;
+    r[o] = Math.round(
+      s[p] * (1 - v) + s[b] * v
     );
   }
-  return new Uint8Array(o.buffer);
+  return new Uint8Array(r.buffer);
 }
-function b() {
+function E() {
   const i = /* @__PURE__ */ new Map();
   return {
     on(e, t) {
@@ -41,7 +41,7 @@ function b() {
     }
   };
 }
-function w(i) {
+function f(i) {
   const e = new Int16Array(i.length);
   for (let t = 0; t < i.length; t++) {
     const s = Math.max(-1, Math.min(1, i[t]));
@@ -53,7 +53,7 @@ const I = 16e3;
 class u {
   ws = null;
   _state = "idle";
-  events = b();
+  events = E();
   inputSampleRate = I;
   /** Current agent state */
   get state() {
@@ -113,8 +113,8 @@ class u {
     this.events.emit("closed", { code: e, reason: t });
   }
 }
-const E = "gemini-2.5-flash-native-audio-preview-12-2025", C = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", k = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
-class A extends u {
+const A = "gemini-2.5-flash-native-audio-preview-12-2025", C = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", k = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
+class R extends u {
   agentName = "GeminiLive";
   async connect(e) {
     if (this.ws)
@@ -122,10 +122,10 @@ class A extends u {
     if (!e.apiKey)
       throw new Error("Gemini API key is required");
     e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate);
-    const t = e.model ?? E, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${k}?access_token=${encodeURIComponent(e.apiKey)}` : `${C}?key=${encodeURIComponent(e.apiKey)}`;
-    return new Promise((a, o) => {
+    const t = e.model ?? A, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${k}?access_token=${encodeURIComponent(e.apiKey)}` : `${C}?key=${encodeURIComponent(e.apiKey)}`;
+    return new Promise((a, r) => {
       this.ws = new WebSocket(n), this.ws.onopen = () => {
-        const r = {
+        const o = {
           setup: {
             model: `models/${t}`,
             generationConfig: {
@@ -134,13 +134,13 @@ class A extends u {
             systemInstruction: e.systemPrompt ? { parts: [{ text: e.systemPrompt }] } : void 0
           }
         };
-        this.ws.send(JSON.stringify(r)), this.setState("listening"), a();
+        this.ws.send(JSON.stringify(o)), this.setState("listening"), a();
       }, this.ws.onerror = () => {
-        o(new Error("Failed to connect to Gemini Live"));
-      }, this.ws.onclose = (r) => {
-        this.ws = null, this.setState("idle"), this.emitClosed(r.code, r.reason);
-      }, this.ws.onmessage = (r) => {
-        this.handleMessage(r.data);
+        r(new Error("Failed to connect to Gemini Live"));
+      }, this.ws.onclose = (o) => {
+        this.ws = null, this.setState("idle"), this.emitClosed(o.code, o.reason);
+      }, this.ws.onmessage = (o) => {
+        this.handleMessage(o.data);
       };
     });
   }
@@ -189,8 +189,8 @@ class A extends u {
     this.ws.send(JSON.stringify(t));
   }
 }
-const R = "wss://api.elevenlabs.io/v1/convai/conversation";
-class M extends u {
+const M = ["neutral", "angry", "sad", "happy"], x = "wss://api.elevenlabs.io/v1/convai/conversation";
+class S extends u {
   agentName = "ElevenLabs";
   outputSampleRate = 24e3;
   // Default, updated from metadata
@@ -202,6 +202,8 @@ class M extends u {
   // True after conversation_initiation_metadata received
   lastInterruptId = 0;
   // Track interruptions to filter stale audio
+  emotionEmittedForEventId = -1;
+  // Track which turn's emotion we've already emitted
   async connect(e) {
     if (this.ws)
       throw new Error("Already connected");
@@ -209,7 +211,7 @@ class M extends u {
       throw new Error("ElevenLabs agent ID or signed URL is required");
     e.inputSampleRate && (this.sourceInputSampleRate = e.inputSampleRate);
     let t;
-    return e.signedUrl ? t = e.signedUrl : (t = `${R}?agent_id=${e.agentId}`, e.apiKey && (t += `&xi-api-key=${e.apiKey}`)), new Promise((s, n) => {
+    return e.signedUrl ? t = e.signedUrl : (t = `${x}?agent_id=${e.agentId}`, e.apiKey && (t += `&xi-api-key=${e.apiKey}`)), new Promise((s, n) => {
       this.ws = new WebSocket(t), this.ws.onopen = () => {
         this.setState("listening"), s();
       }, this.ws.onerror = () => {
@@ -269,19 +271,19 @@ class M extends u {
   }
   handleAudio(e) {
     const t = e.audio_event;
-    if (!t?.audio_base_64 || (t.event_id ?? 0) <= this.lastInterruptId)
+    if (!t?.audio_base_64) return;
+    const s = t.event_id ?? 0;
+    if (s <= this.lastInterruptId)
       return;
-    this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    let n = g(t.audio_base_64);
-    this.outputSampleRate !== h && (n = c(n, this.outputSampleRate, h));
-    const a = 4800;
-    if (n.length <= a)
-      this.events.emit("audio", n);
-    else
-      for (let o = 0; o < n.length; o += a) {
-        const r = n.slice(o, Math.min(o + a, n.length));
-        this.events.emit("audio", r);
+    if (this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking")), this.emotionEmittedForEventId !== s && t.alignment?.chars) {
+      const r = t.alignment.chars.join("").match(/\[(\w+)\]/);
+      if (r) {
+        const o = r[1].toLowerCase();
+        M.includes(o) && (this.events.emit("emotion", o), this.emotionEmittedForEventId = s);
       }
+    }
+    let n = g(t.audio_base_64);
+    this.outputSampleRate !== l && (n = c(n, this.outputSampleRate, l)), this.events.emit("audio", n);
   }
   handleUserTranscript(e) {
     const t = e.user_transcription_event;
@@ -293,11 +295,11 @@ class M extends u {
   }
   handleAgentResponse(e) {
     const t = e.agent_response_event;
-    t?.agent_response && this.events.emit("transcript", {
+    t?.agent_response && (this.events.emit("turnEnd", void 0), this.setState("listening"), this.events.emit("transcript", {
       role: "assistant",
       text: t.agent_response,
       isFinal: !0
-    });
+    }));
   }
   handleInterruption(e) {
     const t = e.interruption_event;
@@ -341,8 +343,8 @@ class M extends u {
     this.initialized = !1, this.lastInterruptId = 0, super.close();
   }
 }
-const x = "wss://api.cartesia.ai/agents/stream", P = "2025-04-16";
-class T extends u {
+const P = "wss://api.cartesia.ai/agents/stream", T = "2025-04-16";
+class O extends u {
   agentName = "Cartesia";
   // Audio configuration
   cartesiaInputFormat = "pcm_16000";
@@ -361,7 +363,7 @@ class T extends u {
     if (!e.apiKey)
       throw new Error("Cartesia API Key is required");
     e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate), this.inputSampleRate === 16e3 ? this.cartesiaInputFormat = "pcm_16000" : this.inputSampleRate === 24e3 ? this.cartesiaInputFormat = "pcm_24000" : this.inputSampleRate === 44100 ? this.cartesiaInputFormat = "pcm_44100" : this.cartesiaInputFormat = "pcm_16000";
-    const t = `${x}/${e.agentId}?api_key=${e.apiKey}&cartesia_version=${P}`;
+    const t = `${P}/${e.agentId}?api_key=${e.apiKey}&cartesia_version=${T}`;
     return new Promise((s, n) => {
       this.ws = new WebSocket(t), this.ws.onopen = () => {
         this.sendStartEvent(), this.startHeartbeat(), s();
@@ -424,7 +426,7 @@ class T extends u {
     if (!e.media?.payload) return;
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
     let t = g(e.media.payload);
-    this.cartesiaOutputRate !== h && (t = c(t, this.cartesiaOutputRate, h)), this.events.emit("audio", t);
+    this.cartesiaOutputRate !== l && (t = c(t, this.cartesiaOutputRate, l)), this.events.emit("audio", t);
   }
   handleClear() {
     this.events.emit("interrupted", void 0), this.setState("listening");
@@ -446,7 +448,7 @@ class T extends u {
     this.stopHeartbeat(), this.isReady = !1, this.streamId = null, super.close();
   }
 }
-class O extends u {
+class N extends u {
   agentName = "Vapi";
   // Audio configuration - Vapi uses 16kHz PCM by default
   vapiSampleRate = 16e3;
@@ -473,7 +475,7 @@ class O extends u {
    */
   handleBinaryAudio(e) {
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    const t = new Uint8Array(e), s = this.vapiSampleRate !== h ? c(t, this.vapiSampleRate, h) : t;
+    const t = new Uint8Array(e), s = this.vapiSampleRate !== l ? c(t, this.vapiSampleRate, l) : t;
     this.events.emit("audio", s);
   }
   handleParsedMessage(e) {
@@ -525,7 +527,7 @@ class O extends u {
     this.hangup(), super.close();
   }
 }
-const N = [
+const U = [
   { id: "gemini", name: "Gemini Live", description: "Google Gemini Live API" },
   { id: "elevenlabs", name: "ElevenLabs", description: "ElevenLabs Conversational AI" },
   { id: "cartesia", name: "Cartesia", description: "Cartesia Agents API" },
@@ -534,21 +536,21 @@ const N = [
 function y(i) {
   switch (i) {
     case "gemini":
-      return new A();
+      return new R();
     case "elevenlabs":
-      return new M();
+      return new S();
     case "cartesia":
-      return new T();
-    case "vapi":
       return new O();
+    case "vapi":
+      return new N();
     default:
       throw new Error(`Unknown agent type: ${i}`);
   }
 }
-function L(i) {
-  return N.find((e) => e.id === i);
+function F(i) {
+  return U.find((e) => e.id === i);
 }
-class U extends Error {
+class D extends Error {
   status;
   payload;
   url;
@@ -556,8 +558,8 @@ class U extends Error {
     super(e.message), this.name = "ApiError", this.status = e.status, this.payload = e.payload, this.url = e.url;
   }
 }
-const l = /* @__PURE__ */ new Set();
-class F {
+const h = /* @__PURE__ */ new Set();
+class K {
   apiBaseUrl;
   publishableKey;
   callbacks;
@@ -601,31 +603,31 @@ class F {
   }
   /** Connect to the embed session */
   async connect() {
-    if (l.has(this.publishableKey)) {
+    if (h.has(this.publishableKey)) {
       console.log("[PersonaEmbed] Connection already in progress, skipping");
       return;
     }
-    l.add(this.publishableKey), this.mounted = !0, this.abortController = new AbortController(), this.setStatus("connecting");
+    h.add(this.publishableKey), this.mounted = !0, this.abortController = new AbortController(), this.setStatus("connecting");
     try {
       const e = await this.fetchSession(this.abortController.signal);
       if (!this.mounted) {
-        l.delete(this.publishableKey);
+        h.delete(this.publishableKey);
         return;
       }
       if (await this.initSession(e), await this.initMicrophone(), await this.connectAgent(e.voice_agent_details), !this.mounted) {
-        this.cleanup(), l.delete(this.publishableKey);
+        this.cleanup(), h.delete(this.publishableKey);
         return;
       }
       this.setStatus("connected");
     } catch (e) {
-      if (l.delete(this.publishableKey), e instanceof Error && e.name === "AbortError")
+      if (h.delete(this.publishableKey), e instanceof Error && e.name === "AbortError")
         return;
       console.error("[PersonaEmbed]", e), this.mounted && (this.setStatus("error"), this.callbacks.onError?.(e));
     }
   }
   /** Disconnect and cleanup */
   disconnect() {
-    this.mounted = !1, this.abortController?.abort(), this.abortController = null, l.delete(this.publishableKey), this.cleanup(), this.setStatus("disconnected");
+    this.mounted = !1, this.abortController?.abort(), this.abortController = null, h.delete(this.publishableKey), this.cleanup(), this.setStatus("disconnected");
   }
   /** Toggle microphone mute */
   toggleMute() {
@@ -650,7 +652,7 @@ class F {
         s = await t.json();
       } catch {
       }
-      throw new U({
+      throw new D({
         message: s?.message ?? "create_session failed",
         status: t.status,
         payload: s,
@@ -664,7 +666,7 @@ class F {
     return t.json();
   }
   async initSession(e) {
-    this.session = v({
+    this.session = w({
       serverUrl: e.session_details.server_url,
       participantToken: e.session_details.participant_token,
       agentIdentity: e.session_details.agent_identity,
@@ -684,9 +686,11 @@ class F {
       onClose: () => {
         this.mounted && this.callbacks.onDisconnect?.();
       }
-    }), this.agent = y(e.voice_agent_details.type), this.agent.on("audio", (t) => this.session?.sendAudio(t)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (t) => this.setAgentState(t)), this.agent.on("closed", () => {
+    }), this.agent = y(e.voice_agent_details.type), this.agent.on("audio", (t) => this.session?.sendAudio(t)), this.agent.on("turnEnd", () => this.session?.endAudioTurn()), this.agent.on("interrupted", () => {
+      this.session?.endAudioTurn(), this.session?.interrupt();
+    }), this.agent.on("stateChange", (t) => this.setAgentState(t)), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
-    }), await this.session.connect();
+    }), this.agent instanceof S && this.agent.on("emotion", (t) => this.session?.setEmotion(t)), await this.session.connect();
   }
   async initMicrophone() {
     this.stream = await navigator.mediaDevices.getUserMedia({
@@ -695,7 +699,7 @@ class F {
     const e = this.audioContext.createMediaStreamSource(this.stream);
     this.processor = this.audioContext.createScriptProcessor(4096, 1, 1), this.processor.onaudioprocess = (t) => {
       if (!this._isMuted) {
-        const s = w(t.inputBuffer.getChannelData(0));
+        const s = f(t.inputBuffer.getChannelData(0));
         this.agent?.sendAudio(s);
       }
     }, e.connect(this.processor), this.processor.connect(this.audioContext.destination);
@@ -725,7 +729,7 @@ class F {
   }
 }
 const d = /* @__PURE__ */ new Set();
-class K {
+class B {
   voiceAgentDetails;
   sessionDetails;
   callbacks;
@@ -799,7 +803,7 @@ class K {
     this._agentState !== e && (this._agentState = e, this.callbacks.onAgentStateChange?.(e));
   }
   async initSession() {
-    this.session = v({
+    this.session = w({
       serverUrl: this.sessionDetails.server_url,
       participantToken: this.sessionDetails.participant_token,
       agentIdentity: this.sessionDetails.agent_identity,
@@ -819,9 +823,11 @@ class K {
       onClose: () => {
         this.mounted && this.callbacks.onDisconnect?.();
       }
-    }), this.agent = y(this.voiceAgentDetails.type), this.agent.on("audio", (e) => this.session?.sendAudio(e)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (e) => this.setAgentState(e)), this.agent.on("closed", () => {
+    }), this.agent = y(this.voiceAgentDetails.type), this.agent.on("audio", (e) => this.session?.sendAudio(e)), this.agent.on("turnEnd", () => this.session?.endAudioTurn()), this.agent.on("interrupted", () => {
+      this.session?.endAudioTurn(), this.session?.interrupt();
+    }), this.agent.on("stateChange", (e) => this.setAgentState(e)), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
-    }), await this.session.connect();
+    }), this.agent instanceof S && this.agent.on("emotion", (e) => this.session?.setEmotion(e)), await this.session.connect();
   }
   async initMicrophone() {
     this.stream = await navigator.mediaDevices.getUserMedia({
@@ -830,7 +836,7 @@ class K {
     const e = this.audioContext.createMediaStreamSource(this.stream);
     this.processor = this.audioContext.createScriptProcessor(4096, 1, 1), this.processor.onaudioprocess = (t) => {
       if (!this._isMuted) {
-        const s = w(t.inputBuffer.getChannelData(0));
+        const s = f(t.inputBuffer.getChannelData(0));
         this.agent?.sendAudio(s);
       }
     }, e.connect(this.processor), this.processor.connect(this.audioContext.destination);
@@ -860,20 +866,20 @@ class K {
   }
 }
 export {
-  N as AGENT_REGISTRY,
+  U as AGENT_REGISTRY,
   u as BaseAgent,
-  T as CartesiaAgent,
-  M as ElevenLabsAgent,
-  A as GeminiLiveAgent,
-  U as KeyframeApiError,
-  F as PersonaEmbed,
-  K as PersonaView,
-  h as SAMPLE_RATE,
+  O as CartesiaAgent,
+  S as ElevenLabsAgent,
+  R as GeminiLiveAgent,
+  D as KeyframeApiError,
+  K as PersonaEmbed,
+  B as PersonaView,
+  l as SAMPLE_RATE,
   g as base64ToBytes,
   m as bytesToBase64,
   y as createAgent,
-  b as createEventEmitter,
-  w as floatTo16BitPCM,
-  L as getAgentInfo,
+  E as createEventEmitter,
+  f as floatTo16BitPCM,
+  F as getAgentInfo,
   c as resamplePcm
 };

package/package.json

@@ -4,7 +4,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.7",
+  "version": "0.0.9",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -19,7 +19,7 @@
   ],
   "sideEffects": false,
   "dependencies": {
-    "@keyframelabs/sdk": "0.1.5"
+    "@keyframelabs/sdk": "0.1.7"
   },
   "devDependencies": {
     "@types/node": "^25.0.9",