@keyframelabs/elements 0.0.8 → 0.1.0

package/README.md

@@ -125,7 +125,7 @@ Currently, only the ElevenLabs agent emits emotion events.
 | `apiBaseUrl`         | `string`                        | `'https://api.keyframelabs.com'` | Base URL for the Keyframe API.                 |
 | `videoFit`           | `'cover' \| 'contain'`          | `'cover'`                        | Video scaling mode (`object-fit`).             |
 | `onStateChange`      | `(status: EmbedStatus) => void` | —                                | Fired when connection status changes.          |
-| `onAgentStateChange` | `(state: AgentState) => void`   | —                                | Fired when agent state changes.                |
+| `onAgentStateChange` | `(state: AgentState) => void`   | —                                | Fired when avatar playback state changes. Signaled by the GPU node via RPC, not the voice agent. |
 | `onDisconnect`       | `() => void`                    | —                                | Fired when the session disconnects.            |
 | `onError`            | `(err: Error) => void`          | —                                | Fired on fatal errors.                         |
 
@@ -142,7 +142,7 @@ Currently, only the ElevenLabs agent emits emotion events.
 | Property       | Type               | Description                                                                            |
 | -------------- | ------------------ | -------------------------------------------------------------------------------------- |
 | `status`       | `EmbedStatus`      | Current connection status: `'connecting' \| 'connected' \| 'disconnected' \| 'error'`. |
-| `agentState`   | `AgentState`       | Current agent state: `'idle' \| 'listening' \| 'thinking' \| 'speaking'`.              |
+| `agentState`   | `AgentState`       | Avatar playback state: `'listening' \| 'speaking'`. Set by the GPU node, not the voice agent. |
 | `isMuted`      | `boolean`          | Whether the microphone is currently muted.                                             |
 | `videoElement` | `HTMLVideoElement` | The underlying video element used for rendering.                                       |
 | `audioElement` | `HTMLAudioElement` | The underlying audio element used for playback.                                        |

package/dist/PersonaEmbed.d.ts

@@ -1,4 +1,4 @@
-import { AgentState } from './agents';
+import { AgentState } from '@keyframelabs/sdk';
 import { EmbedStatus, VideoFit, BaseCallbacks } from './types';
 export type { EmbedStatus, VideoFit } from './types';
 export interface PersonaEmbedOptions extends BaseCallbacks {

package/dist/PersonaView.d.ts

@@ -1,4 +1,4 @@
-import { AgentState } from './agents';
+import { AgentState } from '@keyframelabs/sdk';
 import { EmbedStatus, VideoFit, VoiceAgentDetails, SessionDetails, BaseCallbacks } from './types';
 export interface PersonaViewOptions extends BaseCallbacks {
     /** Target container element */

package/dist/index.d.ts

@@ -4,7 +4,8 @@ export { PersonaView } from './PersonaView';
 export type { PersonaViewOptions } from './PersonaView';
 export type { EmbedStatus, VideoFit, VoiceAgentDetails, SessionDetails, BaseCallbacks, } from './types';
 export { createAgent, GeminiLiveAgent, ElevenLabsAgent, CartesiaAgent, BaseAgent, AGENT_REGISTRY, getAgentInfo, } from './agents';
-export type { AgentType, AgentState, AgentConfig, AgentEventMap, Agent, AnyAgent, AgentTypeInfo, GeminiLiveConfig, ElevenLabsConfig, CartesiaConfig, } from './agents';
+export type { AgentType, AgentConfig, AgentEventMap, Agent, AnyAgent, AgentTypeInfo, GeminiLiveConfig, ElevenLabsConfig, CartesiaConfig, } from './agents';
+export type { AgentState } from '@keyframelabs/sdk';
 export { floatTo16BitPCM, resamplePcm, base64ToBytes, bytesToBase64, SAMPLE_RATE, createEventEmitter, } from './agents';
 export { ApiError as KeyframeApiError } from './ApiError';
 export type { ApiErrorPayload as KeyframeApiErrorPayload } from './ApiError';

package/dist/index.js

@@ -1,12 +1,12 @@
-import { createClient as w } from "@keyframelabs/sdk";
+import { createClient as f } from "@keyframelabs/sdk";
 const l = 24e3;
-function m(i) {
+function g(i) {
   const e = atob(i), t = new Uint8Array(e.length);
   for (let s = 0; s < e.length; s++)
     t[s] = e.charCodeAt(s);
   return t;
 }
-function g(i) {
+function m(i) {
   let e = "";
   for (let t = 0; t < i.length; t++)
     e += String.fromCharCode(i[t]);
@@ -41,7 +41,7 @@ function E() {
     }
   };
 }
-function f(i) {
+function w(i) {
   const e = new Int16Array(i.length);
   for (let t = 0; t < i.length; t++) {
     const s = Math.max(-1, Math.min(1, i[t]));
@@ -113,7 +113,7 @@ class u {
     this.events.emit("closed", { code: e, reason: t });
   }
 }
-const C = "gemini-2.5-flash-native-audio-preview-12-2025", k = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", A = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
+const A = "gemini-2.5-flash-native-audio-preview-12-2025", C = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent", k = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained";
 class R extends u {
   agentName = "GeminiLive";
   async connect(e) {
@@ -122,7 +122,7 @@ class R extends u {
     if (!e.apiKey)
       throw new Error("Gemini API key is required");
     e.inputSampleRate && (this.inputSampleRate = e.inputSampleRate);
-    const t = e.model ?? C, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${A}?access_token=${encodeURIComponent(e.apiKey)}` : `${k}?key=${encodeURIComponent(e.apiKey)}`;
+    const t = e.model ?? A, n = (e.authType ?? "api_key") === "ephemeral_token" ? `${k}?access_token=${encodeURIComponent(e.apiKey)}` : `${C}?key=${encodeURIComponent(e.apiKey)}`;
     return new Promise((a, r) => {
       this.ws = new WebSocket(n), this.ws.onopen = () => {
         const o = {
@@ -159,7 +159,7 @@ class R extends u {
         this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
         for (const n of s.modelTurn.parts) {
           if (n.inlineData?.data) {
-            const a = m(n.inlineData.data);
+            const a = g(n.inlineData.data);
             this.events.emit("audio", a);
           }
           n.text && this.events.emit("transcript", {
@@ -181,7 +181,7 @@ class R extends u {
         mediaChunks: [
           {
             mimeType: `audio/pcm;rate=${this.inputSampleRate}`,
-            data: g(e)
+            data: m(e)
           }
         ]
       }
@@ -282,7 +282,7 @@ class S extends u {
         M.includes(o) && (this.events.emit("emotion", o), this.emotionEmittedForEventId = s);
       }
     }
-    let n = m(t.audio_base_64);
+    let n = g(t.audio_base_64);
     this.outputSampleRate !== l && (n = c(n, this.outputSampleRate, l)), this.events.emit("audio", n);
   }
   handleUserTranscript(e) {
@@ -295,11 +295,11 @@ class S extends u {
   }
   handleAgentResponse(e) {
     const t = e.agent_response_event;
-    t?.agent_response && this.events.emit("transcript", {
+    t?.agent_response && (this.events.emit("turnEnd", void 0), this.setState("listening"), this.events.emit("transcript", {
       role: "assistant",
       text: t.agent_response,
       isFinal: !0
-    });
+    }));
   }
   handleInterruption(e) {
     const t = e.interruption_event;
@@ -310,7 +310,7 @@ class S extends u {
       return;
     let t = e;
     this.sourceInputSampleRate !== this.expectedInputSampleRate && (t = c(e, this.sourceInputSampleRate, this.expectedInputSampleRate)), this.ws.send(JSON.stringify({
-      user_audio_chunk: g(t)
+      user_audio_chunk: m(t)
     }));
   }
   /**
@@ -425,7 +425,7 @@ class O extends u {
   handleMediaOutput(e) {
     if (!e.media?.payload) return;
     this._state !== "speaking" && (this.events.emit("turnStart", void 0), this.setState("speaking"));
-    let t = m(e.media.payload);
+    let t = g(e.media.payload);
     this.cartesiaOutputRate !== l && (t = c(t, this.cartesiaOutputRate, l)), this.events.emit("audio", t);
   }
   handleClear() {
@@ -440,7 +440,7 @@ class O extends u {
       event: "media_input",
       stream_id: this.streamId,
       media: {
-        payload: g(t)
+        payload: m(t)
       }
     }));
   }
@@ -574,7 +574,7 @@ class K {
   stream = null;
   abortController = null;
   _status = "disconnected";
-  _agentState = "idle";
+  _agentState = "listening";
   _isMuted = !1;
   mounted = !0;
   constructor(e) {
@@ -666,7 +666,7 @@ class K {
     return t.json();
   }
   async initSession(e) {
-    this.session = w({
+    this.session = f({
       serverUrl: e.session_details.server_url,
       participantToken: e.session_details.participant_token,
       agentIdentity: e.session_details.agent_identity,
@@ -680,13 +680,18 @@ class K {
       onStateChange: (t) => {
         this.mounted && t === "disconnected" && (this.setStatus("disconnected"), this.callbacks.onDisconnect?.());
       },
+      onAgentStateChange: (t) => {
+        this.mounted && this.setAgentState(t);
+      },
       onError: (t) => {
         this.mounted && this.callbacks.onError?.(t);
       },
       onClose: () => {
         this.mounted && this.callbacks.onDisconnect?.();
       }
-    }), this.agent = y(e.voice_agent_details.type), this.agent.on("audio", (t) => this.session?.sendAudio(t)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (t) => this.setAgentState(t)), this.agent.on("closed", () => {
+    }), this.agent = y(e.voice_agent_details.type), this.agent.on("audio", (t) => this.session?.sendAudio(t)), this.agent.on("turnEnd", () => this.session?.endAudioTurn()), this.agent.on("interrupted", () => {
+      this.session?.endAudioTurn(), this.session?.interrupt();
+    }), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
     }), this.agent instanceof S && this.agent.on("emotion", (t) => this.session?.setEmotion(t)), await this.session.connect();
   }
@@ -697,7 +702,7 @@ class K {
     const e = this.audioContext.createMediaStreamSource(this.stream);
     this.processor = this.audioContext.createScriptProcessor(4096, 1, 1), this.processor.onaudioprocess = (t) => {
       if (!this._isMuted) {
-        const s = f(t.inputBuffer.getChannelData(0));
+        const s = w(t.inputBuffer.getChannelData(0));
         this.agent?.sendAudio(s);
       }
     }, e.connect(this.processor), this.processor.connect(this.audioContext.destination);
@@ -742,7 +747,7 @@ class B {
   processor = null;
   stream = null;
   _status = "disconnected";
-  _agentState = "idle";
+  _agentState = "listening";
   _isMuted = !1;
   mounted = !0;
   constructor(e) {
@@ -801,7 +806,7 @@ class B {
     this._agentState !== e && (this._agentState = e, this.callbacks.onAgentStateChange?.(e));
   }
   async initSession() {
-    this.session = w({
+    this.session = f({
       serverUrl: this.sessionDetails.server_url,
       participantToken: this.sessionDetails.participant_token,
       agentIdentity: this.sessionDetails.agent_identity,
@@ -815,13 +820,18 @@ class B {
       onStateChange: (e) => {
         this.mounted && e === "disconnected" && (this.setStatus("disconnected"), this.callbacks.onDisconnect?.());
       },
+      onAgentStateChange: (e) => {
+        this.mounted && this.setAgentState(e);
+      },
       onError: (e) => {
         this.mounted && this.callbacks.onError?.(e);
       },
       onClose: () => {
         this.mounted && this.callbacks.onDisconnect?.();
       }
-    }), this.agent = y(this.voiceAgentDetails.type), this.agent.on("audio", (e) => this.session?.sendAudio(e)), this.agent.on("interrupted", () => this.session?.interrupt()), this.agent.on("stateChange", (e) => this.setAgentState(e)), this.agent.on("closed", () => {
+    }), this.agent = y(this.voiceAgentDetails.type), this.agent.on("audio", (e) => this.session?.sendAudio(e)), this.agent.on("turnEnd", () => this.session?.endAudioTurn()), this.agent.on("interrupted", () => {
+      this.session?.endAudioTurn(), this.session?.interrupt();
+    }), this.agent.on("closed", () => {
       this.mounted && this.callbacks.onDisconnect?.();
     }), this.agent instanceof S && this.agent.on("emotion", (e) => this.session?.setEmotion(e)), await this.session.connect();
   }
@@ -832,7 +842,7 @@ class B {
     const e = this.audioContext.createMediaStreamSource(this.stream);
     this.processor = this.audioContext.createScriptProcessor(4096, 1, 1), this.processor.onaudioprocess = (t) => {
       if (!this._isMuted) {
-        const s = f(t.inputBuffer.getChannelData(0));
+        const s = w(t.inputBuffer.getChannelData(0));
         this.agent?.sendAudio(s);
       }
     }, e.connect(this.processor), this.processor.connect(this.audioContext.destination);
@@ -871,11 +881,11 @@ export {
   K as PersonaEmbed,
   B as PersonaView,
   l as SAMPLE_RATE,
-  m as base64ToBytes,
-  g as bytesToBase64,
+  g as base64ToBytes,
+  m as bytesToBase64,
   y as createAgent,
   E as createEventEmitter,
-  f as floatTo16BitPCM,
+  w as floatTo16BitPCM,
   F as getAgentInfo,
   c as resamplePcm
 };

package/dist/types.d.ts

@@ -1,4 +1,5 @@
-import { AgentType, AgentState } from './agents';
+import { AgentType } from './agents';
+import { AgentState } from '@keyframelabs/sdk';
 export type EmbedStatus = 'connecting' | 'connected' | 'error' | 'disconnected';
 export type VideoFit = 'cover' | 'contain';
 export type VoiceAgentDetails = {

package/package.json

@@ -4,7 +4,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.8",
+  "version": "0.1.0",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -19,7 +19,7 @@
   ],
   "sideEffects": false,
   "dependencies": {
-    "@keyframelabs/sdk": "0.1.6"
+    "@keyframelabs/sdk": "0.1.8"
   },
   "devDependencies": {
     "@types/node": "^25.0.9",