@alxxsck/ai-assistant 1.0.0 → 1.1.0

package/README.md

@@ -3,7 +3,7 @@
 Embeddable React + TypeScript chat widget for Bridge AI agents, with:
 
 - text chat
-- optional real-time voice mode (WebRTC)
+- optional realtime voice mode backed by Lola WebRTC
 - optional animated 3D avatar (three.js)
 - WebSocket live message updates (AnyCable)
 - Shadow DOM isolation for host-page style safety
@@ -25,7 +25,7 @@ The instance mounts a React app into a host element's Shadow DOM, fetches static
 - Zustand (state stores)
 - three.js + FBX assets (avatar rendering/animation)
 - AnyCable Web client (live chat transport)
-- WebRTC + data channels (voice mode)
+- Lola + OpenAI Realtime WebRTC (voice mode)
 
 ## Repository Layout
 
@@ -37,7 +37,7 @@ High-level structure:
 - `src/react/mount.tsx` - React root mount helper
 - `src/ui/components` - UI components (chat widget, routes, controls)
 - `src/domain/message` - message store + websocket channel integration
-- `src/domain/voice` - voice lifecycle + WebRTC state
+- `src/domain/voice` - Lola voice session, WebRTC media, and realtime events
 - `src/avatar` - three.js avatar scene, animation manager, blend shapes
 - `src/api` - HTTP clients for messages/voice/session-related calls
 - `src/context/ChatWidgetContext.tsx` - widget-level UI state/context
@@ -81,15 +81,14 @@ Live updates arrive via AnyCable channel events and are pushed into the store.
 
 ### 4) Voice Domain
 
-`useVoiceStore` manages full voice call lifecycle:
+`useVoiceStore` manages the Lola voice turn lifecycle:
 
-- starts voice session via API (`StartVoiceSession`)
-- builds RTCPeerConnection using TURN credentials from session payload
-- sends SDP offer / receives answer
-- buffers + sends ICE candidates
-- consumes voice data channels (`voice-events`, `voice-input`)
-- streams remote audio to an `Audio` element
-- handles mic enable/disable, device switching, and teardown (`EndSession` + `EndVoiceSession`)
+- creates a canonical `POST /voice/sessions` session
+- captures the microphone with `getUserMedia`
+- exchanges SDP through the backend-provided `/voice/sessions/:id/webrtc` path
+- plays the OpenAI Realtime remote audio track
+- renders native realtime transcripts and keeps avatar talking state in sync
+- closes media resources and calls `/voice/sessions/:id/end`
 
 ### 5) Avatar Domain
 
@@ -131,6 +130,7 @@ Actual widget config
 - `apiUrl: string`
 - `AIAvatar?: boolean` - is 3D avatar be used
 - `voiceEnabled?: boolean` - is voice mode enebled
+- `voiceConfig?: { voice: string }` - optional Lola Realtime voice
 - `background?: boolean` - clear or visible background
 - `footer?: boolean` - is footer rendered
 - `debug?: boolean` (avatar debug GUI)
@@ -340,6 +340,6 @@ This allows cache-friendly hashed asset delivery without changing code reference
 - No live updates:
   - verify `wsUrl`, websocket reachability, and token validity
 - Voice fails to start:
-  - verify mic permission, TURN reachability, and voice API endpoints
+  - verify mic permission, Lola `apiUrl`, backend `Project.settings.voiceEnabled`, and OpenAI Realtime configuration
 - Avatar never becomes ready:
   - verify manifest URL + asset availability under static host

package/dist/{Avatar-XISRbl7X.js → Avatar-B6ktAPlH.js}

@@ -29156,6 +29156,7 @@ class nS {
   animations = [];
   zeroMorphingInfluences = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
   visemeDataIndex = 0;
+  previousSpeechTime = 0;
   visemeData;
   mixer;
   actions = /* @__PURE__ */ new Map();
@@ -29205,9 +29206,12 @@ class nS {
     this.actions.set(t.name, {
       animation: i,
       blendShapeScenario: r
-    }), this.loadedAnimations++, this.totalAnimations === this.loadedAnimations && this.avatar.onAllAssetsLoaded();
+    }), this.avatar.onAnimationLoaded(t.name), this.loadedAnimations++, this.totalAnimations === this.loadedAnimations && this.avatar.onAllAssetsLoaded();
   }
   fadeDuration = 0.2;
+  hasAction(e) {
+    return this.actions.has(e);
+  }
   fadeToAction(e) {
     const t = this.actions.get(this.activeActionName);
     this.activeActionName = e;
@@ -29218,7 +29222,7 @@ class nS {
     this.actions.get(this.activeActionName).blendShapeScenario?.play();
   }
   resetVisemeData() {
-    this.visemeDataIndex = 0;
+    this.visemeDataIndex = 0, this.previousSpeechTime = 0;
   }
   setVisemeData() {
     this.visemeData = Jy.trim().split(`
@@ -29233,7 +29237,7 @@ class nS {
   convertVisemeDataToAnimationScenario() {
   }
   update(e, t = 0) {
-    this.mixer && this.mixer.update(e), !(this.visemeDataIndex >= this.visemeData.length) && t && t >= this.visemeData[this.visemeDataIndex].time - this.blendShapeResetAnimationLength * 0.7 && (this.playVisemeAnimation(
+    this.mixer && this.mixer.update(e), t < this.previousSpeechTime && (this.visemeDataIndex = 0), this.previousSpeechTime = t, !(this.visemeDataIndex >= this.visemeData.length) && t && t >= this.visemeData[this.visemeDataIndex].time - this.blendShapeResetAnimationLength * 0.7 && (this.playVisemeAnimation(
       t,
       this.visemeData[this.visemeDataIndex],
       this.visemeData[this.visemeDataIndex + 1]
@@ -29531,8 +29535,22 @@ class iS extends EventTarget {
   IDLE_ANIMATION_NAME_2 = "waiting_idle_2";
   speakingAnimationName = this.SPEAKING_ANIMATION_NAME_1;
   idleAnimationName = this.IDLE_ANIMATION_NAME_1;
+  talkingRequested = !1;
+  talkingApplied = !1;
   setTalking(e) {
-    e ? (this.sound.play(), this.animationManager.fadeToAction(this.speakingAnimationName)) : (this.sound.stop(), this.sound.seek(0), this.animationManager.resetMorphing(), this.animationManager.resetVisemeData(), this.speakingAnimationName = this.speakingAnimationName === this.SPEAKING_ANIMATION_NAME_1 ? this.SPEAKING_ANIMATION_NAME_2 : this.SPEAKING_ANIMATION_NAME_1, this.idleAnimationName = this.idleAnimationName === this.IDLE_ANIMATION_NAME_1 ? this.IDLE_ANIMATION_NAME_2 : this.IDLE_ANIMATION_NAME_1, this.animationManager.activeActionName === this.speakingAnimationName && this.animationManager.fadeToAction(this.idleAnimationName));
+    if (this.talkingRequested = e, e) {
+      this.applyTalkingWhenReady();
+      return;
+    }
+    if (!this.talkingApplied || !this.sound || !this.animationManager) return;
+    const t = this.animationManager.activeActionName === this.speakingAnimationName;
+    this.sound.loop(!1), this.sound.stop(), this.sound.seek(0), this.animationManager.resetMorphing(), this.animationManager.resetVisemeData(), this.talkingApplied = !1, this.speakingAnimationName = this.speakingAnimationName === this.SPEAKING_ANIMATION_NAME_1 ? this.SPEAKING_ANIMATION_NAME_2 : this.SPEAKING_ANIMATION_NAME_1, this.idleAnimationName = this.idleAnimationName === this.IDLE_ANIMATION_NAME_1 ? this.IDLE_ANIMATION_NAME_2 : this.IDLE_ANIMATION_NAME_1, t && this.animationManager.hasAction(this.idleAnimationName) && this.animationManager.fadeToAction(this.idleAnimationName);
+  }
+  applyTalkingWhenReady() {
+    !this.talkingRequested || this.talkingApplied || !this.sound || !this.animationManager?.hasAction(this.speakingAnimationName) || (this.sound.loop(!0), this.sound.play(), this.animationManager.fadeToAction(this.speakingAnimationName), this.talkingApplied = !0);
+  }
+  onAnimationLoaded(e) {
+    e === this.speakingAnimationName && this.applyTalkingWhenReady();
   }
   playReaction(e, t) {
     t === "win_big" && (t = "win_small"), t === "welcome_game" && (t = "spin"), this.animationManager.fadeToAction(t), this.animationManager.resetVisemeData(), this.speakPhrase(e);
@@ -29574,7 +29592,7 @@ class iS extends EventTarget {
     this.api.onAvatarReady(this.externalLaunchAnimationAction);
   };
   async initDebugControls(e) {
-    const { OrbitControls: t } = await import("./OrbitControls-XelJ40xM.js");
+    const { OrbitControls: t } = await import("./OrbitControls-BE3BANZi.js");
     this.controls = new t(e, this.renderer.domElement);
   }
   async addDebugGui() {

package/dist/{OrbitControls-XelJ40xM.js → OrbitControls-BE3BANZi.js}

@@ -1,4 +1,4 @@
-import { C as S, V as p, M as u, T as _, Q as E, S as P, a as c, R as w, P as O, b as R } from "./Avatar-XISRbl7X.js";
+import { C as S, V as p, M as u, T as _, Q as E, S as P, a as c, R as w, P as O, b as R } from "./Avatar-B6ktAPlH.js";
 const D = { type: "change" }, b = { type: "start" }, M = { type: "end" }, f = new w(), T = new O(), j = Math.cos(70 * R.DEG2RAD), n = new p(), r = 2 * Math.PI, o = {
   NONE: -1,
   ROTATE: 0,

package/dist/api/audio.api.d.ts

@@ -0,0 +1,20 @@
+import type { AppApi } from "@/App";
+import type { ProtectedAudioMimeType, StreamingSpeechMimeType } from "@/domain/speech/speech.constants";
+import type { WidgetConfig } from "@/index.types";
+export type ProtectedAudioRequest = {
+    audioPath: string;
+    method: "POST";
+    mimeType: ProtectedAudioMimeType;
+};
+export type StreamingAudioRequest = ProtectedAudioRequest & {
+    mimeType: StreamingSpeechMimeType;
+    format: "pcm16";
+    sampleRate: 24_000;
+    channels: 1;
+};
+export declare class AudioApiError extends Error {
+    readonly status: number;
+    constructor(status: number, message: string);
+}
+export declare const fetchProtectedAudio: (config: WidgetConfig, api: AppApi, request: ProtectedAudioRequest) => Promise<Blob>;
+export declare const fetchProtectedAudioStream: (config: WidgetConfig, api: AppApi, request: StreamingAudioRequest, signal: AbortSignal) => Promise<ReadableStream<Uint8Array>>;

package/dist/api/voice.api.d.ts

@@ -1,65 +1,10 @@
-import type { ChatParticipant } from "./messages.api.types";
 import type { AppApi } from "@/App";
-import type { GetTurnBootstrapPayload, StartVoiceSessionResponse, VoiceSessionData, VoiceTurn } from "./voice.api.types";
 import type { WidgetConfig } from "@/index.types";
-export declare const startVoiceSession: (config: WidgetConfig, api: AppApi) => Promise<StartVoiceSessionResponse>;
-export declare function getVoiceSession(config: WidgetConfig, api: AppApi, id: string): Promise<VoiceSessionData>;
-type VoiceSessionStatus = "VOICE_SESSION_STATUS_ENDED" | "VOICE_SESSION_STATUS_ACTIVE";
-type VoiceConfig = {
-    voice: string;
-};
-type EndVoiceSessionResponse = {
-    agentId: string;
-    chatId: string;
-    companyId: string;
-    createdAt: string;
-    endedAt: string;
-    id: string;
-    modelId: string;
-    participant: ChatParticipant;
-    signalingUrl: string;
-    status: VoiceSessionStatus;
-    tokenExpiresAt: string;
-    updatedAt: string;
-    voiceConfig: VoiceConfig;
-};
-export declare function endVoiceSession(config: WidgetConfig, api: AppApi, id: string, reason?: string): Promise<EndVoiceSessionResponse>;
-type SendOfferPayload = {
-    voiceBaseUrl: string;
-    voiceSessionId: string;
-    sessionToken: string;
-    clientInstanceId: string;
-    protocolVersion: number;
-    expectedConnectionGeneration: number;
-    offerSdp: string;
-};
-type SendOfferResponse = {
-    answerSdp: string;
-    protocolVersion: number;
-    voiceSessionId: string;
-    connectionId: string;
-    connectionGeneration: number;
-};
-export declare function sendOffer({ voiceBaseUrl, voiceSessionId, sessionToken, clientInstanceId, protocolVersion, expectedConnectionGeneration, offerSdp, }: SendOfferPayload): Promise<SendOfferResponse>;
-type SendCandidatesPayload = {
-    voiceBaseUrl: string;
-    voiceSessionId: string;
-    sessionToken: string;
-    clientInstanceId: string;
-    connectionId: string;
-    connectionGeneration: number;
-    candidates: RTCIceCandidateInit[];
-};
-export declare function sendCandidates({ voiceBaseUrl, voiceSessionId, sessionToken, clientInstanceId, connectionId, connectionGeneration, candidates, }: SendCandidatesPayload): Promise<void>;
-type EndWebRtcSessionPayload = {
-    voiceBaseUrl: string;
-    voiceSessionId: string;
-    sessionToken: string;
-    clientInstanceId: string;
-    connectionId: string;
-    connectionGeneration: number;
-    reason?: string;
-};
-export declare function endWebRtcSession({ voiceBaseUrl, voiceSessionId, sessionToken, clientInstanceId, connectionId, connectionGeneration, reason, }: EndWebRtcSessionPayload): Promise<void>;
-export declare function getTurnBootstrap({ voiceBaseUrl, voiceSessionId, sessionToken, clientInstanceId, protocolVersion, }: GetTurnBootstrapPayload): Promise<VoiceTurn>;
-export {};
+import type { ConnectVoiceSessionData, VoiceSessionData } from "./voice.api.types";
+export declare class VoiceApiError extends Error {
+    readonly status: number;
+    constructor(status: number, message: string);
+}
+export declare const startVoiceSession: (config: WidgetConfig, api: AppApi, conversationId?: string) => Promise<VoiceSessionData>;
+export declare const connectVoiceSession: (config: WidgetConfig, api: AppApi, offerPath: string, offerSdp: string) => Promise<ConnectVoiceSessionData>;
+export declare const endVoiceSession: (config: WidgetConfig, api: AppApi, voiceSessionId: string, reason?: string) => Promise<VoiceSessionData>;

package/dist/api/voice.api.types.d.ts

@@ -1,37 +1,21 @@
-import type { ChatParticipant } from "./messages.api.types";
-export type StartVoiceSessionResponse = {
-    status: number;
-    json: VoiceSessionData | ApiErrorResponse;
-};
-export type ApiErrorResponse = {
-    code: string;
-    message: string;
-};
 export type VoiceSessionData = {
-    session: {
-        id: string;
-        companyId: string;
-        agentId: string;
-        chatId: string;
-        participant: ChatParticipant;
+    voiceSessionId: string;
+    conversationId: string;
+    status: string;
+    voiceConfig: {
+        voice: string;
     };
-    sessionToken: string;
     voiceProtocolVersion: number;
-    voiceBaseUrl: string;
-    voiceTurn?: VoiceTurn;
+    webrtc: {
+        offerPath: string;
+        dataChannel: string;
+        eventProtocol: string;
+    };
 };
-export type GetTurnBootstrapPayload = {
-    voiceBaseUrl: string;
+export type ConnectVoiceSessionData = {
     voiceSessionId: string;
-    sessionToken: string;
-    clientInstanceId: string;
-    protocolVersion: number;
-};
-export type VoiceTurn = {
-    urls: string[];
-    username: string;
-    credential: string;
-    credentialExpiresAt: string;
-    realm: string;
-    useSessionTokenCredential: boolean;
+    answerSdp: string;
+    model: string;
+    voice: string;
+    dataChannel: string;
 };

package/dist/avatar/AnimationManager.d.ts

@@ -1,6 +1,6 @@
 import { Object3D } from "three";
 import type { VisemeData } from "./Avatar";
-import { Avatar } from "./Avatar";
+import type { Avatar } from "./Avatar";
 export declare const BLEND_SHAPE_KEYS: readonly ["blendShape1.AE", "blendShape1.ChJ", "blendShape1.FV", "blendShape1.Oh", "blendShape1.Ah", "blendShape1.Ih", "blendShape1.L", "blendShape1.SZ", "blendShape1.Woo", "blendShape1.MBP", "blendShape1.Smile_Closed", "blendShape1.Smile_Open", "blendShape1.Kiss"];
 export type BlendShapeKey = (typeof BLEND_SHAPE_KEYS)[number];
 export default class AnimationManager {
@@ -10,6 +10,7 @@ export default class AnimationManager {
     private animations;
     private zeroMorphingInfluences;
     private visemeDataIndex;
+    private previousSpeechTime;
     private visemeData;
     private mixer;
     private actions;
@@ -23,6 +24,7 @@ export default class AnimationManager {
     private loadAnimationLazy;
     private onLoadAnimation;
     private fadeDuration;
+    hasAction(name: string): boolean;
     fadeToAction(name: string): void;
     applyActiveActionBlendShapeScenario(): void;
     resetVisemeData(): void;

package/dist/avatar/Avatar.d.ts

@@ -53,7 +53,11 @@ export declare class Avatar extends EventTarget {
     private IDLE_ANIMATION_NAME_2;
     private speakingAnimationName;
     private idleAnimationName;
+    private talkingRequested;
+    private talkingApplied;
     setTalking(talking: boolean): void;
+    private applyTalkingWhenReady;
+    onAnimationLoaded(name: string): void;
     playReaction(speech: string, animation: string): void;
     speakPhrase(phrase: string): void;
     externalLaunchAnimationAction: (name: ExternallyAvailableActions) => void;