@convai/web-sdk 1.3.2-beta.0 → 1.4.0-beta.1

package/dist/core/ConvaiClient.js

@@ -1,4 +1,5 @@
-import { Room, RoomEvent, DefaultReconnectPolicy, } from "livekit-client";
+import { Room, RoomEvent, DefaultReconnectPolicy, setLogLevel, } from "livekit-client";
+setLogLevel("warn");
 import { AudioManager } from "./AudioManager";
 import { VideoManager } from "./VideoManager";
 import { ScreenShareManager } from "./ScreenShareManager";
@@ -6,6 +7,7 @@ import { MessageHandler } from "./MessageHandler";
 import { EventEmitter } from "./EventEmitter";
 import { MemoryManager } from "./MemoryManager";
 import { ConnectionStateHandler } from "./ConnectionStateHandler";
+import { WebSocketSession } from "./WebSocketSession";
 /**
  * Custom reconnect policy that disables automatic reconnection
  */
@@ -64,6 +66,13 @@ export class ConvaiClient extends EventEmitter {
         this._storedConfig = null;
         this._endUserId = null;
         this._endUserMetadata = null;
+        this._wsSession = null;
+        this._activeTransport = null;
+        this._logRtviMessages = true;
+        // RTVI client-ready handshake (race-proof bot-ready delivery)
+        this._clientReadyRetryTimer = null;
+        this._clientReadyTimeoutTimer = null;
+        this._clientReadyMsgId = null;
         // Conversation session tracking
         this._conversationSessionId = 0;
         this._conversationStartTime = 0;
@@ -72,6 +81,7 @@ export class ConvaiClient extends EventEmitter {
         // Store config if provided
         if (config) {
             this._storedConfig = config;
+            this._logRtviMessages = config.logRtviMessages !== false;
         }
         // Initialize room with no reconnect policy
         this._room = new Room({
@@ -104,6 +114,7 @@ export class ConvaiClient extends EventEmitter {
         this._videoManager = new VideoManager(this._room);
         this._screenShareManager = new ScreenShareManager(this._room);
         this._messageHandler = new MessageHandler(this._room);
+        this._messageHandler.setRtviMessageLogging(this._logRtviMessages);
         this._connectionStateHandler = new ConnectionStateHandler(this.updateState.bind(this), (ready) => (this._isBotReady = ready), () => {
             this._messageHandler.reset();
             this._endUserId = null;
@@ -177,7 +188,13 @@ export class ConvaiClient extends EventEmitter {
         this._room.on(RoomEvent.ConnectionStateChanged, this._connectionStateHandler.handleConnectionStateChanged.bind(this._connectionStateHandler));
         // Message handler events
         this._messageHandler.on("botReady", () => {
+            // Server may send multiple bot-ready frames (one per client-ready
+            // we retried). Only surface the first to consumers per connect.
+            if (this._isBotReady) {
+                return;
+            }
             this._isBotReady = true;
+            this._stopClientReadyHandshake();
             this.updateState({ isConnected: true });
             this.emit("botReady");
         });
@@ -280,6 +297,15 @@ export class ConvaiClient extends EventEmitter {
         this._messageHandler.on("userMuteStopped", () => {
             this.emit("userMuteStopped");
         });
+        this._messageHandler.on("actionResponse", (data) => {
+            this.emit("actionResponse", data);
+        });
+        this._messageHandler.on("serverResponse", (response) => {
+            this.emit("serverResponse", response);
+        });
+        this._messageHandler.on("interactionCreated", (data) => {
+            this.emit("interactionCreated", data);
+        });
     }
     /**
      * Setup MessageHandler to respond to conversation start events
@@ -318,6 +344,31 @@ export class ConvaiClient extends EventEmitter {
             this.emit("stateChange", this._state);
         }
     }
+    /**
+     * Send a data message via the active transport.
+     * For LiveKit: publishes via the data channel.
+     * For WebSocket: sends via PipecatClient.sendClientMessage (wrapped in {t,d} envelope).
+     */
+    publishMessage(type, data) {
+        const message = { type, data };
+        if (this._activeTransport === "websocket" && this._wsSession?.isConnected) {
+            this._wsSession.sendMessage(type, data);
+        }
+        else if (this._room?.localParticipant &&
+            this._room.state !== "disconnected") {
+            const encodedData = new TextEncoder().encode(JSON.stringify(message));
+            this._room.localParticipant.publishData(encodedData, { reliable: true });
+        }
+    }
+    /**
+     * Returns true when a message can be sent on the active transport.
+     */
+    isTransportReady() {
+        if (this._activeTransport === "websocket") {
+            return this._wsSession?.isConnected ?? false;
+        }
+        return (!!this._room?.localParticipant && this._room.state !== "disconnected");
+    }
     /**
      * Connect to a Convai character
      */
@@ -336,9 +387,11 @@ export class ConvaiClient extends EventEmitter {
         }
         // Add default URL if not provided
         const configWithDefaults = {
-            url: "https://realtime-api-preview.convai.com",
+            url: "https://realtime-api.convai.com",
             ...finalConfig,
         };
+        this._logRtviMessages = configWithDefaults.logRtviMessages !== false;
+        this._messageHandler.setRtviMessageLogging(this._logRtviMessages);
         const hasApiKey = Boolean(configWithDefaults.apiKey);
         const hasAuthToken = Boolean(configWithDefaults.authToken);
         if ((!hasApiKey && !hasAuthToken) || !configWithDefaults.characterId) {
@@ -354,6 +407,8 @@ export class ConvaiClient extends EventEmitter {
             // Determine connection type based on enableVideo
             const connType = configWithDefaults.enableVideo ? "video" : "audio";
             this._connectionType = connType;
+            const transportType = configWithDefaults.transport ?? "livekit";
+            this._activeTransport = transportType;
             // Prepare request body with required parameters
             const characterSessionIdToSend = configWithDefaults.characterSessionId ?? this._characterSessionId;
             const requestBody = {
@@ -364,7 +419,7 @@ export class ConvaiClient extends EventEmitter {
                 ...(configWithDefaults.endUserMetadata != null && {
                     end_user_metadata: configWithDefaults.endUserMetadata,
                 }),
-                transport: "livekit",
+                transport: transportType,
                 connection_type: connType,
                 blendshape_provider: configWithDefaults.enableLipsync
                     ? "neurosync"
@@ -377,7 +432,7 @@ export class ConvaiClient extends EventEmitter {
                     output_fps: 90,
                 },
                 llm_provider: "dynamic",
-                ...(configWithDefaults.enableEmotion !== false && {
+                ...(configWithDefaults.enableEmotion === true && {
                     emotion_config: {
                         provider: configWithDefaults.emotionConfig?.provider ?? "llm",
                         min_word_threshold: configWithDefaults.emotionConfig?.min_word_threshold ?? 3,
@@ -395,19 +450,14 @@ export class ConvaiClient extends EventEmitter {
                     character_session_id: characterSessionIdToSend,
                 }),
                 ...(configWithDefaults.actionConfig && {
-                    action_config: configWithDefaults.actionConfig,
-                }),
-                ...(configWithDefaults.characters && {
-                    characters: configWithDefaults.characters,
-                }),
-                ...(configWithDefaults.objects && {
-                    objects: configWithDefaults.objects,
-                }),
-                ...(configWithDefaults.currentAttentionObject && {
-                    current_attention_object: configWithDefaults.currentAttentionObject,
-                }),
-                ...(configWithDefaults.sceneDescription && {
-                    scene_description: configWithDefaults.sceneDescription,
+                    action_config: {
+                        actions: configWithDefaults.actionConfig.actions,
+                        objects: configWithDefaults.actionConfig.objects,
+                        characters: configWithDefaults.actionConfig.characters,
+                        ...(configWithDefaults.actionConfig.current_attention_object && {
+                            current_attention_object: configWithDefaults.actionConfig.current_attention_object,
+                        }),
+                    },
                 }),
                 ...(configWithDefaults.dynamicInfo && {
                     dynamic_info: {
@@ -429,18 +479,19 @@ export class ConvaiClient extends EventEmitter {
                     }),
                 },
             };
-            const headers = {
+            const connectHeaders = {
                 "Content-Type": "application/json",
             };
             if (configWithDefaults.authToken) {
-                headers["API-AUTH-TOKEN"] = configWithDefaults.authToken;
+                connectHeaders["API-AUTH-TOKEN"] = configWithDefaults.authToken;
             }
             else if (configWithDefaults.apiKey) {
-                headers["X-API-Key"] = configWithDefaults.apiKey;
+                connectHeaders["X-API-Key"] = configWithDefaults.apiKey;
             }
+            // HTTP POST /connect — same request for both transports
             const response = await fetch(`${configWithDefaults.url}/connect`, {
                 method: "POST",
-                headers,
+                headers: connectHeaders,
                 body: JSON.stringify(requestBody),
             });
             if (!response.ok) {
@@ -466,7 +517,7 @@ export class ConvaiClient extends EventEmitter {
                 throw new Error(errorMessage);
             }
             const connectionData = await response.json();
-            // Capture character_session_id from connection response and persist in stored config for reconnection
+            // Capture session identifiers from /connect response
             if (connectionData.character_session_id) {
                 this._characterSessionId = connectionData.character_session_id;
                 this._storedConfig = {
@@ -474,39 +525,87 @@ export class ConvaiClient extends EventEmitter {
                     characterSessionId: connectionData.character_session_id,
                 };
             }
-            // Capture end_user_id and end_user_metadata from connection response
             if (connectionData.end_user_id) {
                 this._endUserId = connectionData.end_user_id;
             }
             if (connectionData.end_user_metadata) {
                 this._endUserMetadata = connectionData.end_user_metadata;
             }
-            // Connect to LiveKit room
-            await this._room.connect(connectionData.room_url, connectionData.token, {
-                rtcConfig: {
-                    iceTransportPolicy: "relay",
-                },
-            });
-            // Enable microphone only if startWithAudioOn is true (default: false)
-            // If false, microphone stays off until user enables it via audioControls
-            if (configWithDefaults.startWithAudioOn) {
-                await this._room.localParticipant.setMicrophoneEnabled(true, {
-                    echoCancellation: this._audioSettings.echoCancellation,
-                    noiseSuppression: this._audioSettings.noiseSuppression,
-                    autoGainControl: this._audioSettings.autoGainControl,
-                    sampleRate: this._audioSettings.sampleRate,
-                    channelCount: this._audioSettings.channelCount,
+            if (transportType === "websocket") {
+                // ── WebSocket (Pipecat) transport path ────────────────────────────
+                // Server returns the WebSocket URL in room_url (same field as LiveKit).
+                const wsUrl = connectionData.room_url;
+                if (!wsUrl) {
+                    throw new Error("No WebSocket URL returned from /connect endpoint (expected room_url)");
+                }
+                // Always enable mic at construction so WavMediaManager initializes the
+                // audio stream during connect(). If startWithAudioOn is false, we mute
+                // after the connection is established (enableMic(false) works once the
+                // stream exists, but not before it is initialized).
+                this._wsSession = new WebSocketSession((payload) => this._messageHandler.handleDataReceivedPublic(payload), true);
+                this._wsSession.on("botAudioTrack", (track) => {
+                    this.emit("botAudioTrack", track);
                 });
+                this._wsSession.on("disconnected", () => {
+                    this._connectionStateHandler.handleDisconnected();
+                });
+                this._audioManager.setWebSocketSession(this._wsSession);
+                // Wait for WebSocket transport to connect (fires before bot-ready)
+                await new Promise((resolve, reject) => {
+                    const onConnected = () => {
+                        this._wsSession.off("connected", onConnected);
+                        this._wsSession.off("error", onError);
+                        resolve();
+                    };
+                    const onError = (err) => {
+                        this._wsSession.off("connected", onConnected);
+                        this._wsSession.off("error", onError);
+                        reject(err instanceof Error ? err : new Error(String(err)));
+                    };
+                    this._wsSession.on("connected", onConnected);
+                    this._wsSession.on("error", onError);
+                    // connectWithUrl calls initDevices() then connect({ wsUrl }) — matches sandbox flow
+                    this._wsSession.connectWithUrl(wsUrl).catch((err) => {
+                        this._wsSession.off("connected", onConnected);
+                        this._wsSession.off("error", onError);
+                        reject(err instanceof Error ? err : new Error(String(err)));
+                    });
+                });
+                // For WebSocket, mic is already streaming after connect.
+                // Only mute if the caller explicitly sets startWithAudioOn: false.
+                // Defaulting to on matches the sandbox behavior and ensures the bot
+                // sees audio when it initializes (avoids server "tap mic to talk" prompt).
+                if (configWithDefaults.startWithAudioOn === false) {
+                    this._wsSession.enableMic(false);
+                    this._audioManager.syncWsAudioState(false);
+                }
+                else {
+                    this._audioManager.syncWsAudioState(true);
+                }
             }
-            // Enable camera only if enableVideo is true AND startWithVideoOn is true
-            if (configWithDefaults.enableVideo &&
-                configWithDefaults.startWithVideoOn) {
-                await this._room.localParticipant.setCameraEnabled(true);
+            else {
+                // ── LiveKit transport path (default) ─────────────────────────────
+                await this._room.connect(connectionData.room_url, connectionData.token, {
+                    rtcConfig: {
+                        iceTransportPolicy: "relay",
+                    },
+                });
+                if (configWithDefaults.startWithAudioOn) {
+                    await this._room.localParticipant.setMicrophoneEnabled(true, {
+                        echoCancellation: this._audioSettings.echoCancellation,
+                        noiseSuppression: this._audioSettings.noiseSuppression,
+                        autoGainControl: this._audioSettings.autoGainControl,
+                        sampleRate: this._audioSettings.sampleRate,
+                        channelCount: this._audioSettings.channelCount,
+                    });
+                }
+                if (configWithDefaults.enableVideo &&
+                    configWithDefaults.startWithVideoOn) {
+                    await this._room.localParticipant.setCameraEnabled(true);
+                }
+                this._audioManager.syncStateFromRoom({ emit: true });
+                this._participantSid = this._room.localParticipant.sid;
             }
-            // Ensure audio manager mirrors the actual microphone permission state
-            this._audioManager.syncStateFromRoom({ emit: true });
-            // Capture participant SID
-            this._participantSid = this._room.localParticipant.sid;
             // Apply custom mapper to blendshape queue if provided
             if (configWithDefaults.blendshapeConfig?.customMapper) {
                 this.blendshapeQueue.setMapper(configWithDefaults.blendshapeConfig.customMapper);
@@ -531,6 +630,7 @@ export class ConvaiClient extends EventEmitter {
                 }
             }
             this.emit("connect");
+            this._startClientReadyHandshake();
         }
         catch (error) {
             this.updateState({
@@ -541,11 +641,82 @@ export class ConvaiClient extends EventEmitter {
             throw error;
         }
     }
+    /**
+     * RTVI client-ready handshake. Server-side gates bot-ready emission on
+     * receipt of client-ready (and re-emits bot-ready on every client-ready),
+     * so we can retry until we hear bot-ready back. Closes the race where the
+     * bot publishes bot-ready before the client data-channel is subscribed.
+     */
+    _startClientReadyHandshake() {
+        // RTVI client-ready handshake is LiveKit-only. WebSocket transport
+        // uses its own pipecat ready signalling.
+        if (this._activeTransport !== "livekit") {
+            return;
+        }
+        this._stopClientReadyHandshake();
+        this._clientReadyMsgId =
+            typeof crypto !== "undefined" && "randomUUID" in crypto
+                ? crypto.randomUUID()
+                : `client-ready-${Date.now()}-${Math.random()}`;
+        const send = () => {
+            if (this._isBotReady)
+                return;
+            if (!this._room || this._room.state !== "connected")
+                return;
+            try {
+                const msg = {
+                    id: this._clientReadyMsgId,
+                    label: "rtvi-ai",
+                    type: "client-ready",
+                    data: {
+                        version: "1.0.0",
+                        about: { library: "convai-web-sdk" },
+                    },
+                };
+                const encoded = new TextEncoder().encode(JSON.stringify(msg));
+                this._room.localParticipant.publishData(encoded, { reliable: true });
+            }
+            catch {
+                // swallow; next retry will try again
+            }
+            this._clientReadyRetryTimer = setTimeout(send, ConvaiClient.CLIENT_READY_RETRY_MS);
+        };
+        send();
+        this._clientReadyTimeoutTimer = setTimeout(() => {
+            if (this._isBotReady)
+                return;
+            this._stopClientReadyHandshake();
+            const err = new Error(`bot-ready not received within ${ConvaiClient.CLIENT_READY_TIMEOUT_MS}ms`);
+            this.emit("error", err);
+        }, ConvaiClient.CLIENT_READY_TIMEOUT_MS);
+    }
+    _stopClientReadyHandshake() {
+        if (this._clientReadyRetryTimer) {
+            clearTimeout(this._clientReadyRetryTimer);
+            this._clientReadyRetryTimer = null;
+        }
+        if (this._clientReadyTimeoutTimer) {
+            clearTimeout(this._clientReadyTimeoutTimer);
+            this._clientReadyTimeoutTimer = null;
+        }
+    }
     /**
      * Disconnect from the current character session
      */
     async disconnect() {
-        if (this._room && this._room.state !== "disconnected") {
+        this._stopClientReadyHandshake();
+        if (this._activeTransport === "websocket" && this._wsSession) {
+            try {
+                await this._wsSession.disconnect();
+            }
+            catch {
+                // ignore disconnect errors
+            }
+            this._wsSession = null;
+            this._connectionStateHandler.resetConnectionState();
+            this.clearAllManagers();
+        }
+        else if (this._room && this._room.state !== "disconnected") {
             try {
                 await this._room.disconnect();
                 this._connectionStateHandler.resetConnectionState();
@@ -563,6 +734,8 @@ export class ConvaiClient extends EventEmitter {
      */
     clearAllManagers() {
         this._connectionType = null;
+        this._activeTransport = null;
+        this._wsSession = null;
         this._apiKey = null;
         this._authToken = null;
         this._characterId = null;
@@ -570,6 +743,7 @@ export class ConvaiClient extends EventEmitter {
         this._endUserId = null;
         this._endUserMetadata = null;
         this._memoryManager = null; // Clear memory manager on disconnect
+        this._audioManager.setWebSocketSession(null);
         this._audioManager.reset();
         this._videoManager.reset();
         this._screenShareManager.reset();
@@ -596,14 +770,10 @@ export class ConvaiClient extends EventEmitter {
      * Send a text message to the character
      */
     sendUserTextMessage(text) {
-        if (!this._room ||
-            this._room.state === "disconnected" ||
-            !this._room.localParticipant) {
+        if (!this.isTransportReady())
             return;
-        }
-        if (!text || !text.trim()) {
+        if (!text || !text.trim())
             return;
-        }
         try {
             // Reset blendshape queue if mid-conversation, but don't interrupt the server
             if (this.blendshapeQueue.isConversationActive() ||
@@ -619,16 +789,9 @@ export class ConvaiClient extends EventEmitter {
                 userMessage: text.trim(),
                 timestamp: this._conversationStartTime,
             });
-            const message = {
-                type: "user_text_message",
-                data: {
-                    text: text.trim(),
-                    participant_sid: this._participantSid || this._room.localParticipant.sid,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
+            this.publishMessage("user_text_message", {
+                text: text.trim(),
+                participant_sid: this._participantSid || this._room.localParticipant?.sid,
             });
         }
         catch (error) {
@@ -639,32 +802,24 @@ export class ConvaiClient extends EventEmitter {
      * Send a trigger message to invoke specific character actions
      */
     sendTriggerMessage(triggerName, triggerMessage) {
-        if (this._room && this._room.localParticipant) {
-            // Reset blendshape queue if mid-conversation, but don't interrupt the server
-            if (this.blendshapeQueue.isConversationActive() ||
-                this._state.isSpeaking) {
-                this.blendshapeQueue.interrupt();
-            }
-            this._conversationSessionId++;
-            this._conversationStartTime = Date.now();
-            this.emit("conversationStart", {
-                sessionId: this._conversationSessionId,
-                userMessage: `[trigger:${triggerName ?? ""}]`,
-                timestamp: this._conversationStartTime,
-            });
-            const message = {
-                type: "trigger-message",
-                data: {
-                    ...(triggerName && { trigger_name: triggerName }),
-                    ...(triggerMessage && { trigger_message: triggerMessage }),
-                    participant_sid: this._participantSid || this._room.localParticipant.sid,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
-            });
-        }
+        if (!this.isTransportReady())
+            return;
+        // Reset blendshape queue if mid-conversation, but don't interrupt the server
+        if (this.blendshapeQueue.isConversationActive() || this._state.isSpeaking) {
+            this.blendshapeQueue.interrupt();
+        }
+        this._conversationSessionId++;
+        this._conversationStartTime = Date.now();
+        this.emit("conversationStart", {
+            sessionId: this._conversationSessionId,
+            userMessage: `[trigger:${triggerName ?? ""}]`,
+            timestamp: this._conversationStartTime,
+        });
+        this.publishMessage("trigger-message", {
+            ...(triggerName && { trigger_name: triggerName }),
+            ...(triggerMessage && { trigger_message: triggerMessage }),
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
+        });
     }
     /**
      * If the bot is currently speaking or a conversation is active, send an
@@ -676,17 +831,10 @@ export class ConvaiClient extends EventEmitter {
         const botSpeaking = this._state.isSpeaking;
         if (!queueActive && !botSpeaking)
             return;
-        // Tell the server to stop the current turn
-        if (this._room &&
-            this._room.state !== "disconnected" &&
-            this._room.localParticipant) {
-            const message = {
-                type: "interrupt-bot",
-                data: {
-                    participant_sid: this._participantSid || this._room.localParticipant.sid,
-                },
-            };
-            this._room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { reliable: true });
+        if (this.isTransportReady()) {
+            this.publishMessage("interrupt-bot", {
+                participant_sid: this._participantSid || this._room.localParticipant?.sid,
+            });
         }
         // End the blendshape queue so the lipsync player finishes cleanly
         this.blendshapeQueue.interrupt();
@@ -695,26 +843,20 @@ export class ConvaiClient extends EventEmitter {
      * Send an interrupt message to stop the bot's current response
      */
     sendInterruptMessage() {
-        if (!this._room ||
-            this._room.state === "disconnected" ||
-            !this._room.localParticipant) {
+        if (!this.isTransportReady())
             return;
-        }
         try {
-            const message = {
-                type: "interrupt-bot",
-                data: {
-                    participant_sid: this._participantSid || this._room.localParticipant.sid,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
+            this.publishMessage("interrupt-bot", {
+                participant_sid: this._participantSid || this._room.localParticipant?.sid,
             });
-            // Trigger fade out for interruption - keeps last 10 frames
-            if (this.blendshapeQueue.isConversationActive() ||
-                this._state.isSpeaking) {
-                this.blendshapeQueue.interrupt();
+            // Blendshape fade-out only for LiveKit (WebSocket transport has no lipsync;
+            // calling interrupt() would set conversationEnded=true and block the next
+            // bot-started-speaking from ever firing isSpeaking).
+            if (this._activeTransport !== "websocket") {
+                if (this.blendshapeQueue.isConversationActive() ||
+                    this._state.isSpeaking) {
+                    this.blendshapeQueue.interrupt();
+                }
             }
         }
         catch (error) {
@@ -725,39 +867,23 @@ export class ConvaiClient extends EventEmitter {
      * Update template keys in the character's context
      */
     updateTemplateKeys(templateKeys) {
-        if (this._room &&
-            this._room.localParticipant &&
-            Object.keys(templateKeys).length > 0) {
-            const message = {
-                type: "update-template-keys",
-                data: {
-                    template_keys: templateKeys,
-                    participant_sid: this._participantSid || this._room.localParticipant.sid,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
-            });
-        }
+        if (!this.isTransportReady() || Object.keys(templateKeys).length === 0)
+            return;
+        this.publishMessage("update-template-keys", {
+            template_keys: templateKeys,
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
+        });
     }
     /**
      * Update dynamic information about the current context
      */
     updateDynamicInfo(dynamicInfo) {
-        if (this._room && this._room.localParticipant && dynamicInfo?.trim()) {
-            const message = {
-                type: "update-dynamic-info",
-                data: {
-                    dynamic_info: { text: dynamicInfo },
-                    participant_sid: this._participantSid || this._room.localParticipant.sid,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
-            });
-        }
+        if (!this.isTransportReady() || !dynamicInfo?.trim())
+            return;
+        this.publishMessage("update-dynamic-info", {
+            dynamic_info: { text: dynamicInfo },
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
+        });
     }
     /**
      * Update the bot's temporary runtime context in a unified way.
@@ -786,19 +912,15 @@ export class ConvaiClient extends EventEmitter {
      * ```
      */
     updateContext(options) {
-        if (!this._room || !this._room.localParticipant) {
+        if (!this.isTransportReady())
             return;
-        }
-        // Validate: text is required unless mode is "reset"
-        if (options.mode !== "reset" && !options.text?.trim()) {
-            console.warn("[ConvaiClient] updateContext: text is required unless mode is 'reset'");
+        const hasText = !!options.text?.trim();
+        const hasAttention = options.current_attention_object !== undefined;
+        // text is required unless mode is "reset" or only updating attention
+        if (options.mode !== "reset" && !hasText && !hasAttention) {
+            console.warn("[ConvaiClient] updateContext: text or current_attention_object is required unless mode is 'reset'");
             return;
         }
-        // When run_llm is "false" the server won't respond, so no conversation
-        // lifecycle is needed. For "true", "auto", or unset the server may respond,
-        // so we treat it the same as sendUserTextMessage: interrupt any active turn
-        // and start a new conversation so the blendshape queue is ready to accept
-        // the incoming frames.
         const willTriggerLlm = options.run_llm !== "false";
         if (willTriggerLlm) {
             if (this.blendshapeQueue.isConversationActive() ||
@@ -813,70 +935,38 @@ export class ConvaiClient extends EventEmitter {
                 timestamp: this._conversationStartTime,
             });
         }
-        const message = {
-            type: "context-update",
-            data: {
-                ...(options.text && { text: options.text.trim() }),
-                ...(options.mode && { mode: options.mode }),
-                ...(options.run_llm && { run_llm: options.run_llm }),
-                participant_sid: this._participantSid || this._room.localParticipant.sid,
-            },
-        };
-        const encodedData = new TextEncoder().encode(JSON.stringify(message));
-        this._room.localParticipant.publishData(encodedData, {
-            reliable: true,
+        this.publishMessage("context-update", {
+            ...(hasText && { text: options.text.trim() }),
+            ...(options.mode && { mode: options.mode }),
+            ...(options.run_llm && { run_llm: options.run_llm }),
+            ...(hasAttention && { current_attention_object: options.current_attention_object }),
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
         });
     }
+    /**
+     * Send descriptive scene updates. Use when environment description changes
+     * and the bot should know what is visible or nearby. Does not modify action_config.
+     */
+    updateSceneMetadata(items) {
+        if (!this.isTransportReady())
+            return;
+        this.publishMessage("update-scene-metadata", { scene_metadata: items });
+    }
     /**
      * Toggle text-to-speech on or off
      */
     toggleTts(enabled) {
-        if (!this._room ||
-            this._room.state === "disconnected" ||
-            !this._room.localParticipant) {
+        if (!this.isTransportReady())
             return;
-        }
-        try {
-            const message = {
-                type: "tts-toggle",
-                data: {
-                    enabled: enabled,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
-            });
-        }
-        catch (error) {
-            throw error;
-        }
+        this.publishMessage("tts-toggle", { enabled });
     }
     /**
      * Toggle speech-to-text on or off
      */
     toggleStt(enabled) {
-        if (!this._room ||
-            this._room.state === "disconnected" ||
-            !this._room.localParticipant) {
+        if (!this.isTransportReady())
             return;
-        }
-        try {
-            const message = {
-                lable: "rtvi-ai",
-                type: "stt-toggle",
-                data: {
-                    muted: !enabled,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
-            });
-        }
-        catch (error) {
-            throw error;
-        }
+        this.publishMessage("stt-toggle", { muted: !enabled });
     }
     /**
      * Reset the server-side idle timer.
@@ -884,26 +974,13 @@ export class ConvaiClient extends EventEmitter {
      * the session from being disconnected due to inactivity.
      */
     resetIdleTimer() {
-        if (!this._room ||
-            this._room.state === "disconnected" ||
-            !this._room.localParticipant) {
+        if (!this.isTransportReady())
             return;
-        }
-        try {
-            const message = {
-                type: "reset-idle-timer",
-                data: {
-                    participant_sid: this._participantSid || this._room.localParticipant.sid,
-                },
-            };
-            const encodedData = new TextEncoder().encode(JSON.stringify(message));
-            this._room.localParticipant.publishData(encodedData, {
-                reliable: true,
-            });
-        }
-        catch (error) {
-            throw error;
-        }
+        this.publishMessage("reset-idle-timer", {
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
+        });
     }
 }
+ConvaiClient.CLIENT_READY_RETRY_MS = 500;
+ConvaiClient.CLIENT_READY_TIMEOUT_MS = 45000;
 //# sourceMappingURL=ConvaiClient.js.map