@alxxsck/ai-assistant 1.5.0 → 1.8.0

package/README.md

@@ -3,7 +3,7 @@
 Embeddable React + TypeScript chat widget for Bridge AI agents, with:
 
 - text chat
-- optional realtime voice mode backed by Lola WebRTC
+- optional realtime voice mode backed by the Lola Socket.IO proxy and xAI
 - optional animated 3D avatar (three.js)
 - WebSocket live message updates (AnyCable)
 - Shadow DOM isolation for host-page style safety
@@ -25,7 +25,7 @@ The instance mounts a React app into a host element's Shadow DOM, fetches static
 - Zustand (state stores)
 - three.js + FBX assets (avatar rendering/animation)
 - AnyCable Web client (live chat transport)
-- Lola + OpenAI Realtime WebRTC (voice mode)
+- Socket.IO + AudioWorklet (realtime voice mode through Lola Backend)
 
 ## Repository Layout
 
@@ -37,7 +37,7 @@ High-level structure:
 - `src/react/mount.tsx` - React root mount helper
 - `src/ui/components` - UI components (chat widget, routes, controls)
 - `src/domain/message` - message store + websocket channel integration
-- `src/domain/voice` - Lola voice session, WebRTC media, and realtime events
+- `src/domain/voice` - Lola voice session, PCM media, and xAI realtime events
 - `src/avatar` - three.js avatar scene, animation manager, blend shapes
 - `src/api` - HTTP clients for messages/voice/session-related calls
 - `src/context/ChatWidgetContext.tsx` - widget-level UI state/context
@@ -78,17 +78,27 @@ High-level structure:
 - agentic steps merge/update and rendering support
 
 Live updates arrive via AnyCable channel events and are pushed into the store.
+Versioned `messageDelta` updates are applied by `messageId` and increasing
+`sequence`; each frame contains the current accumulated text, so duplicate or
+stale frames cannot append content twice. The UI renders every accepted frame
+immediately and animates only the newly arrived suffix. See
+[`docs/chat-streaming.ru.md`](docs/chat-streaming.ru.md) for the wire contract,
+lifecycle, rendering, and scroll behavior.
 
 ### 4) Voice Domain
 
 `useVoiceStore` manages the Lola voice turn lifecycle:
 
 - creates a canonical `POST /voice/sessions` session
-- captures the microphone with `getUserMedia`
-- exchanges SDP through the backend-provided `/voice/sessions/:id/webrtc` path
-- plays the OpenAI Realtime remote audio track
-- renders native realtime transcripts and keeps avatar talking state in sync
-- closes media resources and calls `/voice/sessions/:id/end`
+- captures mono microphone input and converts it to PCM16 24 kHz in an AudioWorklet
+- connects to the authenticated `/assistant` Socket.IO namespace, activates the
+  provider session with `voice.connect`, and sends sequenced Lola protocol v4
+  `voice.audio.append` events with acknowledgements
+- plays `response.output_audio.delta` through a continuous AudioWorklet queue
+- unwraps canonical `voice.event` envelopes, renders Grok realtime transcripts,
+  and keeps avatar talking state in sync
+- creates a fresh VoiceSession after an unexpected voice socket disconnect
+- closes the session through `voice.disconnect` with the REST endpoint as a fallback
 
 ### 5) Avatar Domain
 
@@ -130,7 +140,7 @@ Actual widget config
 - `apiUrl: string`
 - `AIAvatar?: boolean` - is 3D avatar be used
 - `voiceEnabled?: boolean` - is voice mode enebled
-- `voiceConfig?: { voice: string }` - optional Lola Realtime voice
+- `voiceConfig?: { voice: "ara" | "eve" | "leo" | "rex" | "sal" }` - optional xAI voice
 - `background?: boolean` - clear or visible background
 - `footer?: boolean` - is footer rendered
 - `debug?: boolean` (avatar debug GUI)
@@ -226,6 +236,10 @@ Outputs include:
 
 ### Other project scripts
 
+The demo app reads its service credential from
+`VITE_SERVICE_ACCOUNT_API_KEY`; credentials must not be committed to source or
+static config files.
+
 - `npm run lint`
 - `npm run lint-fix`
 - `npm run format`
@@ -346,6 +360,6 @@ This allows cache-friendly hashed asset delivery without changing code reference
 - No live updates:
   - verify `wsUrl`, websocket reachability, and token validity
 - Voice fails to start:
-  - verify mic permission, Lola `apiUrl`, backend `Project.settings.voiceEnabled`, and OpenAI Realtime configuration
+  - verify mic permission, Lola `apiUrl`, Socket.IO reachability, and backend `Project.settings.voiceEnabled`
 - Avatar never becomes ready:
   - verify manifest URL + asset availability under static host

package/dist/{Avatar-_GvrPzkn.js → Avatar-BAlmduYw.js}

@@ -29337,6 +29337,8 @@ class iS extends EventTarget {
   };
   controls;
   locale;
+  animationFrame = null;
+  renderingEnabled = !0;
   lightAmbient;
   lightDirectional;
   loadingManager;
@@ -29526,10 +29528,19 @@ class iS extends EventTarget {
     !e || !t || e == this.width && t == this.height || (this.camera.aspect = e / t, this.camera.updateProjectionMatrix(), this.renderer.setSize(e, t, !1), this.render(), this.width = e, this.height = t);
   };
   animate = () => {
-    requestAnimationFrame(this.animate);
+    if (this.animationFrame = null, !this.renderingEnabled) return;
     const e = this.clock.getDelta(), t = this.sound?.seek();
-    this.animationManager.update(e, t), this.controls?.update(), this.render();
+    this.animationManager.update(e, t), this.controls?.update(), this.render(), this.animationFrame = requestAnimationFrame(this.animate);
   };
+  setRenderingEnabled(e) {
+    if (this.renderingEnabled !== e) {
+      if (this.renderingEnabled = e, !e) {
+        this.animationFrame !== null && (cancelAnimationFrame(this.animationFrame), this.animationFrame = null), this.clock.stop();
+        return;
+      }
+      this.clock.start(), this.animationFrame === null && this.animationManager && (this.animationFrame = requestAnimationFrame(this.animate));
+    }
+  }
   SPEAKING_ANIMATION_NAME_1 = "generic_speaking_1";
   SPEAKING_ANIMATION_NAME_2 = "generic_speaking_2";
   IDLE_ANIMATION_NAME_1 = "idle_1";
@@ -29593,7 +29604,7 @@ class iS extends EventTarget {
     this.api.onAvatarReady(this.externalLaunchAnimationAction);
   };
   async initDebugControls(e) {
-    const { OrbitControls: t } = await import("./OrbitControls-BCIW0o5s.js");
+    const { OrbitControls: t } = await import("./OrbitControls-BhEr8WlG.js");
     this.controls = new t(e, this.renderer.domElement);
   }
   async addDebugGui() {

package/dist/{OrbitControls-BCIW0o5s.js → OrbitControls-BhEr8WlG.js}

@@ -1,4 +1,4 @@
-import { C as S, V as p, M as u, T as _, Q as E, S as P, a as c, R as w, P as O, b as R } from "./Avatar-_GvrPzkn.js";
+import { C as S, V as p, M as u, T as _, Q as E, S as P, a as c, R as w, P as O, b as R } from "./Avatar-BAlmduYw.js";
 const D = { type: "change" }, b = { type: "start" }, M = { type: "end" }, f = new w(), T = new O(), j = Math.cos(70 * R.DEG2RAD), n = new p(), r = 2 * Math.PI, o = {
   NONE: -1,
   ROTATE: 0,

package/dist/api/messages.api.d.ts

@@ -16,5 +16,5 @@ export declare const listConversations: (config: WidgetConfig, api: AppApi) => P
 export declare const listConversationMessages: (config: WidgetConfig, api: AppApi, conversationId: string) => Promise<MessageData[]>;
 export declare const createConversation: (config: WidgetConfig, api: AppApi) => Promise<ConversationData>;
 export declare const selectConversation: (config: WidgetConfig, api: AppApi, conversationId: string) => Promise<ConversationData>;
-export declare function sendMessage(config: WidgetConfig, api: AppApi, messageText: string, conversationId?: string): Promise<MessageData>;
+export declare function sendMessage(config: WidgetConfig, api: AppApi, messageText: string, conversationId?: string, clientMessageId?: `${string}-${string}-${string}-${string}-${string}`): Promise<MessageData>;
 export {};

package/dist/api/messages.api.types.d.ts

@@ -85,4 +85,10 @@ export type AgenticStepsAppendedData = {
     threadId: string;
     agenticSteps: AgenticStep[];
 };
+export type ChatMessageDeltaData = {
+    messageId: string;
+    sequence: number;
+    text: string;
+    done: boolean;
+};
 export {};

package/dist/api/voice.api.d.ts

@@ -1,10 +1,11 @@
 import type { AppApi } from "@/App";
 import type { WidgetConfig } from "@/index.types";
-import type { ConnectVoiceSessionData, VoiceSessionData } from "./voice.api.types";
+import type { VoiceSessionData } from "./voice.api.types";
 export declare class VoiceApiError extends Error {
     readonly status: number;
     constructor(status: number, message: string);
 }
+export declare const parseVoiceSession: (value: unknown) => VoiceSessionData;
 export declare const startVoiceSession: (config: WidgetConfig, api: AppApi, conversationId?: string) => Promise<VoiceSessionData>;
-export declare const connectVoiceSession: (config: WidgetConfig, api: AppApi, offerPath: string, offerSdp: string) => Promise<ConnectVoiceSessionData>;
+export declare const getVoiceSession: (config: WidgetConfig, api: AppApi, voiceSessionId: string) => Promise<VoiceSessionData>;
 export declare const endVoiceSession: (config: WidgetConfig, api: AppApi, voiceSessionId: string, reason?: string) => Promise<VoiceSessionData>;

package/dist/api/voice.api.types.d.ts

@@ -1,21 +1,19 @@
+export type VoiceSessionStatus = "CREATED" | "CONNECTING" | "ACTIVE" | "ENDED" | "FAILED" | "EXPIRED";
+export type VoiceWebsocketContract = {
+    namespace: "/assistant";
+    connectEvent: "voice.connect";
+    audioAppendEvent: "voice.audio.append";
+    disconnectEvent: "voice.disconnect";
+    serverEvent: "voice.event";
+    eventProtocol: "xai-realtime-v1";
+};
 export type VoiceSessionData = {
     voiceSessionId: string;
     conversationId: string;
-    status: string;
+    status: VoiceSessionStatus;
     voiceConfig: {
         voice: string;
     };
     voiceProtocolVersion: number;
-    webrtc: {
-        offerPath: string;
-        dataChannel: string;
-        eventProtocol: string;
-    };
-};
-export type ConnectVoiceSessionData = {
-    voiceSessionId: string;
-    answerSdp: string;
-    model: string;
-    voice: string;
-    dataChannel: string;
+    websocket: VoiceWebsocketContract;
 };

package/dist/avatar/Avatar.d.ts

@@ -34,6 +34,8 @@ export declare class Avatar extends EventTarget {
     private cameraParams;
     private controls?;
     private locale?;
+    private animationFrame;
+    private renderingEnabled;
     constructor(canvas: HTMLCanvasElement, videoContainer: HTMLDivElement, config: WidgetConfig, api: AppApi);
     private lightAmbient;
     private lightDirectional;
@@ -47,6 +49,7 @@ export declare class Avatar extends EventTarget {
     private height;
     onResize: (width?: number, height?: number) => void;
     animate: () => void;
+    setRenderingEnabled(enabled: boolean): void;
     private SPEAKING_ANIMATION_NAME_1;
     private SPEAKING_ANIMATION_NAME_2;
     private IDLE_ANIMATION_NAME_1;