@alxxsck/ai-assistant 1.5.0 → 1.7.0

package/README.md

@@ -3,7 +3,7 @@
 Embeddable React + TypeScript chat widget for Bridge AI agents, with:
 
 - text chat
-- optional realtime voice mode backed by Lola WebRTC
+- optional realtime voice mode backed by the Lola Socket.IO proxy and xAI
 - optional animated 3D avatar (three.js)
 - WebSocket live message updates (AnyCable)
 - Shadow DOM isolation for host-page style safety
@@ -25,7 +25,7 @@ The instance mounts a React app into a host element's Shadow DOM, fetches static
 - Zustand (state stores)
 - three.js + FBX assets (avatar rendering/animation)
 - AnyCable Web client (live chat transport)
-- Lola + OpenAI Realtime WebRTC (voice mode)
+- Socket.IO + AudioWorklet (realtime voice mode through Lola Backend)
 
 ## Repository Layout
 
@@ -37,7 +37,7 @@ High-level structure:
 - `src/react/mount.tsx` - React root mount helper
 - `src/ui/components` - UI components (chat widget, routes, controls)
 - `src/domain/message` - message store + websocket channel integration
-- `src/domain/voice` - Lola voice session, WebRTC media, and realtime events
+- `src/domain/voice` - Lola voice session, PCM media, and xAI realtime events
 - `src/avatar` - three.js avatar scene, animation manager, blend shapes
 - `src/api` - HTTP clients for messages/voice/session-related calls
 - `src/context/ChatWidgetContext.tsx` - widget-level UI state/context
@@ -78,17 +78,27 @@ High-level structure:
 - agentic steps merge/update and rendering support
 
 Live updates arrive via AnyCable channel events and are pushed into the store.
+Versioned `messageDelta` updates are applied by `messageId` and increasing
+`sequence`; each frame contains the current accumulated text, so duplicate or
+stale frames cannot append content twice. The UI renders every accepted frame
+immediately and animates only the newly arrived suffix. See
+[`docs/chat-streaming.ru.md`](docs/chat-streaming.ru.md) for the wire contract,
+lifecycle, rendering, and scroll behavior.
 
 ### 4) Voice Domain
 
 `useVoiceStore` manages the Lola voice turn lifecycle:
 
 - creates a canonical `POST /voice/sessions` session
-- captures the microphone with `getUserMedia`
-- exchanges SDP through the backend-provided `/voice/sessions/:id/webrtc` path
-- plays the OpenAI Realtime remote audio track
-- renders native realtime transcripts and keeps avatar talking state in sync
-- closes media resources and calls `/voice/sessions/:id/end`
+- captures mono microphone input and converts it to PCM16 24 kHz in an AudioWorklet
+- connects to the authenticated `/assistant` Socket.IO namespace, activates the
+  provider session with `voice.connect`, and sends sequenced Lola protocol v4
+  `voice.audio.append` events with acknowledgements
+- plays `response.output_audio.delta` through a continuous AudioWorklet queue
+- unwraps canonical `voice.event` envelopes, renders Grok realtime transcripts,
+  and keeps avatar talking state in sync
+- creates a fresh VoiceSession after an unexpected voice socket disconnect
+- closes the session through `voice.disconnect` with the REST endpoint as a fallback
 
 ### 5) Avatar Domain
 
@@ -130,7 +140,7 @@ Actual widget config
 - `apiUrl: string`
 - `AIAvatar?: boolean` - is 3D avatar be used
 - `voiceEnabled?: boolean` - is voice mode enebled
-- `voiceConfig?: { voice: string }` - optional Lola Realtime voice
+- `voiceConfig?: { voice: "ara" | "eve" | "leo" | "rex" | "sal" }` - optional xAI voice
 - `background?: boolean` - clear or visible background
 - `footer?: boolean` - is footer rendered
 - `debug?: boolean` (avatar debug GUI)
@@ -226,6 +236,10 @@ Outputs include:
 
 ### Other project scripts
 
+The demo app reads its service credential from
+`VITE_SERVICE_ACCOUNT_API_KEY`; credentials must not be committed to source or
+static config files.
+
 - `npm run lint`
 - `npm run lint-fix`
 - `npm run format`
@@ -346,6 +360,6 @@ This allows cache-friendly hashed asset delivery without changing code reference
 - No live updates:
   - verify `wsUrl`, websocket reachability, and token validity
 - Voice fails to start:
-  - verify mic permission, Lola `apiUrl`, backend `Project.settings.voiceEnabled`, and OpenAI Realtime configuration
+  - verify mic permission, Lola `apiUrl`, Socket.IO reachability, and backend `Project.settings.voiceEnabled`
 - Avatar never becomes ready:
   - verify manifest URL + asset availability under static host

package/dist/api/messages.api.d.ts

@@ -16,5 +16,5 @@ export declare const listConversations: (config: WidgetConfig, api: AppApi) => P
 export declare const listConversationMessages: (config: WidgetConfig, api: AppApi, conversationId: string) => Promise<MessageData[]>;
 export declare const createConversation: (config: WidgetConfig, api: AppApi) => Promise<ConversationData>;
 export declare const selectConversation: (config: WidgetConfig, api: AppApi, conversationId: string) => Promise<ConversationData>;
-export declare function sendMessage(config: WidgetConfig, api: AppApi, messageText: string, conversationId?: string): Promise<MessageData>;
+export declare function sendMessage(config: WidgetConfig, api: AppApi, messageText: string, conversationId?: string, clientMessageId?: `${string}-${string}-${string}-${string}-${string}`): Promise<MessageData>;
 export {};

package/dist/api/messages.api.types.d.ts

@@ -85,4 +85,10 @@ export type AgenticStepsAppendedData = {
     threadId: string;
     agenticSteps: AgenticStep[];
 };
+export type ChatMessageDeltaData = {
+    messageId: string;
+    sequence: number;
+    text: string;
+    done: boolean;
+};
 export {};

package/dist/api/voice.api.d.ts

@@ -1,10 +1,11 @@
 import type { AppApi } from "@/App";
 import type { WidgetConfig } from "@/index.types";
-import type { ConnectVoiceSessionData, VoiceSessionData } from "./voice.api.types";
+import type { VoiceSessionData } from "./voice.api.types";
 export declare class VoiceApiError extends Error {
     readonly status: number;
     constructor(status: number, message: string);
 }
+export declare const parseVoiceSession: (value: unknown) => VoiceSessionData;
 export declare const startVoiceSession: (config: WidgetConfig, api: AppApi, conversationId?: string) => Promise<VoiceSessionData>;
-export declare const connectVoiceSession: (config: WidgetConfig, api: AppApi, offerPath: string, offerSdp: string) => Promise<ConnectVoiceSessionData>;
+export declare const getVoiceSession: (config: WidgetConfig, api: AppApi, voiceSessionId: string) => Promise<VoiceSessionData>;
 export declare const endVoiceSession: (config: WidgetConfig, api: AppApi, voiceSessionId: string, reason?: string) => Promise<VoiceSessionData>;

package/dist/api/voice.api.types.d.ts

@@ -1,21 +1,19 @@
+export type VoiceSessionStatus = "CREATED" | "CONNECTING" | "ACTIVE" | "ENDED" | "FAILED" | "EXPIRED";
+export type VoiceWebsocketContract = {
+    namespace: "/assistant";
+    connectEvent: "voice.connect";
+    audioAppendEvent: "voice.audio.append";
+    disconnectEvent: "voice.disconnect";
+    serverEvent: "voice.event";
+    eventProtocol: "xai-realtime-v1";
+};
 export type VoiceSessionData = {
     voiceSessionId: string;
     conversationId: string;
-    status: string;
+    status: VoiceSessionStatus;
     voiceConfig: {
         voice: string;
     };
     voiceProtocolVersion: number;
-    webrtc: {
-        offerPath: string;
-        dataChannel: string;
-        eventProtocol: string;
-    };
-};
-export type ConnectVoiceSessionData = {
-    voiceSessionId: string;
-    answerSdp: string;
-    model: string;
-    voice: string;
-    dataChannel: string;
+    websocket: VoiceWebsocketContract;
 };