@convai/web-sdk 1.5.0 → 1.6.0-beta.1

package/dist/core/ConvaiClient.d.ts

@@ -1,5 +1,5 @@
 import { Room } from "livekit-client";
-import { ConvaiConfig, ConvaiClientState, ChatMessage, IConvaiClient, AudioControls, VideoControls, ScreenShareControls, DynamicInfo, ContextUpdateOptions, UploadFileOptions, WebSocketSessionFactory } from "./types";
+import { ConvaiConfig, ConvaiClientState, ChatMessage, IConvaiClient, AudioControls, VideoControls, ScreenShareControls, DynamicInfo, ContextUpdateOptions, UploadFileOptions, WebSocketSessionFactory, RespondModeUpdateOptions, UpdateSceneMetadataOptions, VisionStatusOptions, VisionTriggerOptions } from "./types";
 import { EventEmitter } from "./EventEmitter";
 import { BlendshapeQueue } from "./BlendshapeQueue";
 import { MemoryManager } from "./MemoryManager";
@@ -57,6 +57,7 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
     private _wsSession;
     private _activeTransport;
     private _logRtviMessages;
+    private _isDisconnecting;
     private _clientReadyRetryTimer;
     private _clientReadyTimeoutTimer;
     private _clientReadyMsgId;
@@ -122,6 +123,7 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
      * Connect to a Convai character
      */
     connect(config?: ConvaiConfig): Promise<void>;
+    private cleanupWebSocketSessionAfterConnectFailure;
     /**
      * RTVI client-ready handshake. Server-side gates bot-ready emission on
      * receipt of client-ready (and re-emits bot-ready on every client-ready),
@@ -160,6 +162,7 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
      * is clean before a new response starts arriving.
      */
     private endCurrentConversationIfActive;
+    private startTrackedConversation;
     /**
      * Send an interrupt message to stop the bot's current response
      */
@@ -201,6 +204,9 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
      * ```
      */
     updateContext(options: ContextUpdateOptions): void;
+    visionStatus(options?: VisionStatusOptions): string | null;
+    visionTrigger(options?: VisionTriggerOptions): string | null;
+    respondModeUpdate(options: RespondModeUpdateOptions): string | null;
     /**
      * Send descriptive scene updates. Use when environment description changes
      * and the bot should know what is visible or nearby. Does not modify action_config.
@@ -208,7 +214,9 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
     updateSceneMetadata(items: Array<{
         name: string;
         description: string;
-    }>): void;
+    }>, options?: UpdateSceneMetadataOptions): void;
+    private createUpdateId;
+    private serializeRespondModes;
     /**
      * Upload a file to the character via the LiveKit byte stream transport.
      * Only works when connected with the LiveKit transport.

package/dist/core/ConvaiClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAML,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,WAAW,EAEX,aAAa,EACb,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,oBAAoB,EACpB,iBAAiB,EAIjB,uBAAuB,EACxB,MAAM,SAAS,CAAC;AAKjB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAehD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;IACrE,OAAO,CAAC,KAAK,CAAO;IACpB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,gBAAgB,CAAiB;IAGzC,OAAO,CAAC,sBAAsB,CAA8C;IAC5E,OAAO,CAAC,wBAAwB,CAA8C;IAC9E,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAO;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAU;IAEzD;;;;OAIG;IACH,MAAM,CAAC,0BAA0B,CAAC,OAAO,EAAE,uBAAuB,GAAG,IAAI;IAKzE,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,mBAAmB,CAAqB;IAChD,OAAO,CAAC,eAAe,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAyB;IAGxD,OAAO,CAAC,sBAAsB,CAAa;IAC3C,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,cAAc,CAA8B;gBAExC,MAAM,CAAC,EAAE,YAAY;IA+DjC,IAAI,KAAK,IAAI,iBAAiB,CAE7B;IAED,IAAI,cAAc,IAAI,OAAO,GAAG,OAAO,GAAG,IAAI,CAE7C;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,IAAI,CAE1B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAE/B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,IAAI,IAAI,IAAI,CAEf;IAED,IAAI,YAAY,IAAI,WAAW,EAAE,CAEhC;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAEtC;IAED,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,mBAAmB,IAAI,mBAAmB,CAE7C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAED,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAExC;IAED;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA+K3B;;;;OAIG;IACH,OAAO,CAAC,yBAAyB;IASjC;;OAEG;IACH,OAAO,CAAC,WAAW;IAuBnB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IActB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IASxB;;OAEG;IACG,OAAO,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA0VnD;;;;;OAKG;IACH,OAAO,CAAC,0BAA0B;IAgDlC,OAAO,CAAC,yBAAyB;IAWjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAwBjC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAOhC;;OAEG;IACH,YAAY,IAAI,IAAI;IAOpB;;OAEG;IACH,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAkCvC;;OAEG;IACH,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;IAuBvE;;;;OAIG;IACH,OAAO,CAAC,8BAA8B;IAiBtC;;OAEG;IACH,oBAAoB,IAAI,IAAI;IAyB5B;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,IAAI;IAUjE;;OAEG;IACH,iBAAiB,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IASjD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAyClD;;;OAGG;IACH,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI;IAK9E;;;;;OAKG;IACG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB5E;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKjC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKjC;;;;OAIG;IACH,cAAc,IAAI,IAAI;CAMvB"}
+{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAML,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,WAAW,EAEX,aAAa,EACb,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,oBAAoB,EACpB,iBAAiB,EAIjB,uBAAuB,EACvB,wBAAwB,EAExB,0BAA0B,EAC1B,mBAAmB,EACnB,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAKjB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAwBhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;IACrE,OAAO,CAAC,KAAK,CAAO;IACpB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,gBAAgB,CAAwC;IAChE,OAAO,CAAC,gBAAgB,CAAiB;IACzC,OAAO,CAAC,gBAAgB,CAAkB;IAG1C,OAAO,CAAC,sBAAsB,CAA8C;IAC5E,OAAO,CAAC,wBAAwB,CAA8C;IAC9E,OAAO,CAAC,iBAAiB,CAAuB;IAChD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAO;IACpD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAU;IAEzD;;;;OAIG;IACH,MAAM,CAAC,0BAA0B,CAAC,OAAO,EAAE,uBAAuB,GAAG,IAAI;IAKzE,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,mBAAmB,CAAqB;IAChD,OAAO,CAAC,eAAe,CAAiB;IACxC,OAAO,CAAC,uBAAuB,CAAyB;IAGxD,OAAO,CAAC,sBAAsB,CAAa;IAC3C,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,cAAc,CAA8B;gBAExC,MAAM,CAAC,EAAE,YAAY;IA+DjC,IAAI,KAAK,IAAI,iBAAiB,CAE7B;IAED,IAAI,cAAc,IAAI,OAAO,GAAG,OAAO,GAAG,IAAI,CAE7C;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,IAAI,CAE1B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAE/B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,IAAI,IAAI,IAAI,CAEf;IAED,IAAI,YAAY,IAAI,WAAW,EAAE,CAEhC;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAEtC;IAED,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,mBAAmB,IAAI,mBAAmB,CAE7C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAED,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAExC;IAED;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqL3B;;;;OAIG;IACH,OAAO,CAAC,yBAAyB;IASjC;;OAEG;IACH,OAAO,CAAC,WAAW;IAuBnB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IActB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IASxB;;OAEG;IACG,OAAO,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;YA8YrC,0CAA0C;IAgBxD;;;;;OAKG;IACH,OAAO,CAAC,0BAA0B;IAgDlC,OAAO,CAAC,yBAAyB;IAWjC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA6BjC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAkBxB;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAOhC;;OAEG;IACH,YAAY,IAAI,IAAI;IAOpB;;OAEG;IACH,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAmCvC;;OAEG;IACH,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;IA2BvE;;;;OAIG;IACH,OAAO,CAAC,8BAA8B;IAmBtC,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,oBAAoB,IAAI,IAAI;IAwB5B;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,IAAI;IAUjE;;OAEG;IACH,iBAAiB,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IASjD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI;IAmClD,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,MAAM,GAAG,IAAI;IAU9D,aAAa,CAAC,OAAO,GAAE,oBAAyB,GAAG,MAAM,GAAG,IAAI;IAkBhE,iBAAiB,CAAC,OAAO,EAAE,wBAAwB,GAAG,MAAM,GAAG,IAAI;IAYnE;;;OAGG;IACH,mBAAmB,CACjB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,EACnD,OAAO,GAAE,0BAA+B,GACvC,IAAI;IAaP,OAAO,CAAC,cAAc;IAOtB,OAAO,CAAC,qBAAqB;IAe7B;;;;;OAKG;IACG,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB5E;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKjC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKjC;;;;OAIG;IACH,cAAc,IAAI,IAAI;CAMvB"}

package/dist/core/ConvaiClient.js

@@ -1,5 +1,6 @@
 import { Room, RoomEvent, DefaultReconnectPolicy, setLogLevel, } from "livekit-client";
 setLogLevel("warn");
+import { DisconnectReasonEnum, } from "./types";
 import { AudioManager } from "./AudioManager";
 import { VideoManager } from "./VideoManager";
 import { ScreenShareManager } from "./ScreenShareManager";
@@ -7,6 +8,9 @@ import { MessageHandler } from "./MessageHandler";
 import { EventEmitter } from "./EventEmitter";
 import { MemoryManager } from "./MemoryManager";
 import { ConnectionStateHandler } from "./ConnectionStateHandler";
+import { buildBlendshapeConnectConfig } from "./connectRequest";
+import { shouldPreemptForContextUpdateResponse, shouldPreemptForExplicitRespondMode, } from "./contextUpdateRequest";
+import { resolveVisionInputConfig, serializeVisionInputConfig, } from "./visionRequest";
 // Module-level registry — populated by importing @convai/web-sdk/vanilla/websocket
 let _wsFactory = null;
 /**
@@ -78,6 +82,7 @@ export class ConvaiClient extends EventEmitter {
         this._wsSession = null;
         this._activeTransport = null;
         this._logRtviMessages = true;
+        this._isDisconnecting = false;
         // RTVI client-ready handshake (race-proof bot-ready delivery)
         this._clientReadyRetryTimer = null;
         this._clientReadyTimeoutTimer = null;
@@ -244,8 +249,9 @@ export class ConvaiClient extends EventEmitter {
             if (!wasListening && isListening) {
                 // Reset blendshape queue if mid-conversation, but don't interrupt the server
                 if (this.blendshapeQueue.isConversationActive() ||
-                    this._state.isSpeaking) {
-                    this.blendshapeQueue.interrupt();
+                    this._state.isSpeaking ||
+                    this.blendshapeQueue.hasBufferedFrames()) {
+                    this._messageHandler.interruptLipsyncPlayback();
                 }
                 this._conversationSessionId++;
                 this._conversationStartTime = Date.now();
@@ -277,6 +283,10 @@ export class ConvaiClient extends EventEmitter {
         this._messageHandler.on("blendshapeStatsReceived", (stats) => {
             this.emit("blendshapeStatsReceived", stats);
         });
+        // Forward blendshape cancel event (owner-scoped visual hard stop)
+        this._messageHandler.on("blendshapeCancel", (data) => {
+            this.emit("blendshapeCancel", data);
+        });
         // Forward idle warning event
         this._messageHandler.on("idleWarning", (data) => {
             this.emit("idleWarning", data);
@@ -419,6 +429,7 @@ export class ConvaiClient extends EventEmitter {
             this._connectionType = connType;
             const transportType = configWithDefaults.transport ?? "livekit";
             this._activeTransport = transportType;
+            const effectiveVisionInputConfig = resolveVisionInputConfig(configWithDefaults);
             // Prepare request body with required parameters
             const characterSessionIdToSend = configWithDefaults.characterSessionId ?? this._characterSessionId;
             const requestBody = {
@@ -431,16 +442,7 @@ export class ConvaiClient extends EventEmitter {
                 }),
                 transport: transportType,
                 connection_type: connType,
-                blendshape_provider: configWithDefaults.enableLipsync
-                    ? "neurosync"
-                    : "not_provided",
-                blendshape_config: {
-                    enable_chunking: true,
-                    chunk_size: 10,
-                    format: configWithDefaults.blendshapeConfig?.format || "mha",
-                    frames_buffer_duration: configWithDefaults.blendshapeConfig?.frames_buffer_duration ?? 0.1,
-                    output_fps: 90,
-                },
+                ...buildBlendshapeConnectConfig(configWithDefaults),
                 llm_provider: "dynamic",
                 ...(configWithDefaults.enableEmotion === true && {
                     emotion_config: (() => {
@@ -461,6 +463,12 @@ export class ConvaiClient extends EventEmitter {
                     ? configWithDefaults.ttsEnabled
                     : true,
                 default_stt_enabled: true,
+                ...(effectiveVisionInputConfig && {
+                    vision_input_config: serializeVisionInputConfig(effectiveVisionInputConfig),
+                }),
+                ...(configWithDefaults.respondModes && {
+                    respond_modes: this.serializeRespondModes(configWithDefaults.respondModes),
+                }),
                 ...(characterSessionIdToSend &&
                     characterSessionIdToSend !== "-1" && {
                     character_session_id: characterSessionIdToSend,
@@ -489,7 +497,7 @@ export class ConvaiClient extends EventEmitter {
                 invocation_metadata: {
                     source: configWithDefaults.invocationMetadata?.source ?? "web_sdk",
                     client_version: configWithDefaults.invocationMetadata?.clientVersion ??
-                        "1.3.2-beta.0",
+                        "1.6.0-beta.0",
                     ...(configWithDefaults.invocationMetadata?.extraMetadata && {
                         extra_metadata: configWithDefaults.invocationMetadata.extraMetadata,
                     }),
@@ -569,32 +577,69 @@ export class ConvaiClient extends EventEmitter {
                 // after the connection is established (enableMic(false) works once the
                 // stream exists, but not before it is initialized).
                 this._wsSession = _wsFactory((payload) => this._messageHandler.handleDataReceivedPublic(payload), true);
-                this._wsSession.on("botAudioTrack", (track) => {
+                const webSocketVisionEnabled = configWithDefaults.enableVideo === true &&
+                    effectiveVisionInputConfig?.enabled !== false;
+                if (webSocketVisionEnabled) {
+                    this._videoManager.setWebSocketVisionMessageSender((type, data) => {
+                        if (this._activeTransport !== "websocket" ||
+                            !this._wsSession?.isConnected) {
+                            return;
+                        }
+                        this._wsSession.sendMessage(type, data);
+                    });
+                }
+                else {
+                    this._videoManager.setWebSocketVisionDisabled();
+                }
+                const wsSession = this._wsSession;
+                wsSession.on("botAudioTrack", (track) => {
                     this.emit("botAudioTrack", track);
                 });
-                this._wsSession.on("disconnected", () => {
+                wsSession.on("disconnected", () => {
+                    if (this._wsSession !== wsSession || this._isDisconnecting) {
+                        return;
+                    }
                     this._connectionStateHandler.handleDisconnected();
+                    this.clearAllManagers();
                 });
                 this._audioManager.setWebSocketSession(this._wsSession);
                 // Wait for WebSocket transport to connect (fires before bot-ready)
                 await new Promise((resolve, reject) => {
+                    let settled = false;
+                    const cleanupConnectListeners = () => {
+                        wsSession.off("connected", onConnected);
+                        wsSession.off("error", onError);
+                        wsSession.off("disconnected", onDisconnected);
+                    };
+                    const settle = (callback) => {
+                        if (settled) {
+                            return;
+                        }
+                        settled = true;
+                        cleanupConnectListeners();
+                        callback();
+                    };
                     const onConnected = () => {
-                        this._wsSession.off("connected", onConnected);
-                        this._wsSession.off("error", onError);
-                        resolve();
+                        settle(resolve);
                     };
                     const onError = (err) => {
-                        this._wsSession.off("connected", onConnected);
-                        this._wsSession.off("error", onError);
-                        reject(err instanceof Error ? err : new Error(String(err)));
+                        settle(() => {
+                            reject(err instanceof Error ? err : new Error(String(err)));
+                        });
+                    };
+                    const onDisconnected = () => {
+                        settle(() => {
+                            reject(new Error("WebSocket disconnected before connect completed"));
+                        });
                     };
-                    this._wsSession.on("connected", onConnected);
-                    this._wsSession.on("error", onError);
+                    wsSession.on("connected", onConnected);
+                    wsSession.on("error", onError);
+                    wsSession.on("disconnected", onDisconnected);
                     // connectWithUrl calls initDevices() then connect({ wsUrl }) — matches sandbox flow
-                    this._wsSession.connectWithUrl(wsUrl).catch((err) => {
-                        this._wsSession.off("connected", onConnected);
-                        this._wsSession.off("error", onError);
-                        reject(err instanceof Error ? err : new Error(String(err)));
+                    wsSession.connectWithUrl(wsUrl).catch((err) => {
+                        settle(() => {
+                            reject(err instanceof Error ? err : new Error(String(err)));
+                        });
                     });
                 });
                 // For WebSocket, mic is already streaming after connect.
@@ -608,8 +653,14 @@ export class ConvaiClient extends EventEmitter {
                 else {
                     this._audioManager.syncWsAudioState(true);
                 }
+                if (webSocketVisionEnabled &&
+                    configWithDefaults.enableVideo &&
+                    configWithDefaults.startWithVideoOn) {
+                    await this._videoManager.enableVideo();
+                }
             }
             else {
+                this._videoManager.setWebSocketVisionMessageSender(null);
                 // ── LiveKit transport path (default) ─────────────────────────────
                 await this._room.connect(connectionData.room_url, connectionData.token, {
                     rtcConfig: {
@@ -659,6 +710,7 @@ export class ConvaiClient extends EventEmitter {
             this._startClientReadyHandshake();
         }
         catch (error) {
+            await this.cleanupWebSocketSessionAfterConnectFailure();
             this.updateState({
                 isConnected: false,
                 isConnecting: false,
@@ -667,6 +719,22 @@ export class ConvaiClient extends EventEmitter {
             throw error;
         }
     }
+    async cleanupWebSocketSessionAfterConnectFailure() {
+        if (this._activeTransport !== "websocket" || !this._wsSession) {
+            return;
+        }
+        this._isDisconnecting = true;
+        try {
+            await this._wsSession.disconnect();
+        }
+        catch {
+            // ignore cleanup errors while preserving the original connect failure
+        }
+        finally {
+            this._isDisconnecting = false;
+        }
+        this.clearAllManagers();
+    }
     /**
      * RTVI client-ready handshake. Server-side gates bot-ready emission on
      * receipt of client-ready (and re-emits bot-ready on every client-ready),
@@ -732,14 +800,18 @@ export class ConvaiClient extends EventEmitter {
     async disconnect() {
         this._stopClientReadyHandshake();
         if (this._activeTransport === "websocket" && this._wsSession) {
+            this._isDisconnecting = true;
             try {
                 await this._wsSession.disconnect();
             }
             catch {
                 // ignore disconnect errors
             }
+            finally {
+                this._isDisconnecting = false;
+            }
             this._wsSession = null;
-            this._connectionStateHandler.resetConnectionState();
+            this._connectionStateHandler.handleDisconnected(DisconnectReasonEnum.CLIENT_INITIATED);
             this.clearAllManagers();
         }
         else if (this._room && this._room.state !== "disconnected") {
@@ -771,6 +843,7 @@ export class ConvaiClient extends EventEmitter {
         this._memoryManager = null; // Clear memory manager on disconnect
         this._audioManager.setWebSocketSession(null);
         this._audioManager.reset();
+        this._videoManager.setWebSocketVisionMessageSender(null);
         this._videoManager.reset();
         this._screenShareManager.reset();
     }
@@ -803,8 +876,9 @@ export class ConvaiClient extends EventEmitter {
         try {
             // Reset blendshape queue if mid-conversation, but don't interrupt the server
             if (this.blendshapeQueue.isConversationActive() ||
-                this._state.isSpeaking) {
-                this.blendshapeQueue.interrupt();
+                this._state.isSpeaking ||
+                this.blendshapeQueue.hasBufferedFrames()) {
+                this._messageHandler.interruptLipsyncPlayback();
             }
             // Start new conversation session
             this._conversationSessionId++;
@@ -831,8 +905,10 @@ export class ConvaiClient extends EventEmitter {
         if (!this.isTransportReady())
             return;
         // Reset blendshape queue if mid-conversation, but don't interrupt the server
-        if (this.blendshapeQueue.isConversationActive() || this._state.isSpeaking) {
-            this.blendshapeQueue.interrupt();
+        if (this.blendshapeQueue.isConversationActive() ||
+            this._state.isSpeaking ||
+            this.blendshapeQueue.hasBufferedFrames()) {
+            this._messageHandler.interruptLipsyncPlayback();
         }
         this._conversationSessionId++;
         this._conversationStartTime = Date.now();
@@ -853,7 +929,8 @@ export class ConvaiClient extends EventEmitter {
      * is clean before a new response starts arriving.
      */
     endCurrentConversationIfActive() {
-        const queueActive = this.blendshapeQueue.isConversationActive();
+        const queueActive = this.blendshapeQueue.isConversationActive() ||
+            this.blendshapeQueue.hasBufferedFrames();
         const botSpeaking = this._state.isSpeaking;
         if (!queueActive && !botSpeaking)
             return;
@@ -863,7 +940,21 @@ export class ConvaiClient extends EventEmitter {
             });
         }
         // End the blendshape queue so the lipsync player finishes cleanly
-        this.blendshapeQueue.interrupt();
+        this._messageHandler.interruptLipsyncPlayback();
+    }
+    startTrackedConversation(userMessage) {
+        if (this.blendshapeQueue.isConversationActive() ||
+            this._state.isSpeaking ||
+            this.blendshapeQueue.hasBufferedFrames()) {
+            this._messageHandler.interruptLipsyncPlayback();
+        }
+        this._conversationSessionId++;
+        this._conversationStartTime = Date.now();
+        this.emit("conversationStart", {
+            sessionId: this._conversationSessionId,
+            userMessage,
+            timestamp: this._conversationStartTime,
+        });
     }
     /**
      * Send an interrupt message to stop the bot's current response
@@ -875,14 +966,13 @@ export class ConvaiClient extends EventEmitter {
             this.publishMessage("interrupt-bot", {
                 participant_sid: this._participantSid || this._room.localParticipant?.sid,
             });
-            // Blendshape fade-out only for LiveKit (WebSocket transport has no lipsync;
-            // calling interrupt() would set conversationEnded=true and block the next
-            // bot-started-speaking from ever firing isSpeaking).
-            if (this._activeTransport !== "websocket") {
-                if (this.blendshapeQueue.isConversationActive() ||
-                    this._state.isSpeaking) {
-                    this.blendshapeQueue.interrupt();
-                }
+            // Stop local lipsync immediately for every transport. WebSocket can also
+            // buffer ahead-delivered frames, and waiting for a server cancel leaves
+            // stale mouth animation visible after the audio stop.
+            if (this.blendshapeQueue.isConversationActive() ||
+                this._state.isSpeaking ||
+                this.blendshapeQueue.hasBufferedFrames()) {
+                this._messageHandler.interruptLipsyncPlayback();
             }
         }
         catch (error) {
@@ -947,36 +1037,97 @@ export class ConvaiClient extends EventEmitter {
             console.warn("[ConvaiClient] updateContext: text or current_attention_object is required unless mode is 'reset'");
             return;
         }
-        const willTriggerLlm = options.run_llm !== "false";
-        if (willTriggerLlm) {
-            if (this.blendshapeQueue.isConversationActive() ||
-                this._state.isSpeaking) {
-                this.blendshapeQueue.interrupt();
-            }
-            this._conversationSessionId++;
-            this._conversationStartTime = Date.now();
-            this.emit("conversationStart", {
-                sessionId: this._conversationSessionId,
-                userMessage: `[context-update:${options.mode ?? "append"}]`,
-                timestamp: this._conversationStartTime,
-            });
+        // Only guaranteed-response updates should preempt active local lipsync.
+        // Server-scheduled auto/silent updates may be downgraded while the bot is
+        // speaking, so clearing the local queue here can stop lipsync while audio
+        // continues.
+        const shouldPreemptLipsync = shouldPreemptForContextUpdateResponse(options);
+        if (shouldPreemptLipsync) {
+            this.startTrackedConversation(`[context-update:${options.mode ?? "append"}]`);
         }
         this.publishMessage("context-update", {
             ...(hasText && { text: options.text.trim() }),
             ...(options.mode && { mode: options.mode }),
             ...(options.run_llm && { run_llm: options.run_llm }),
+            ...(options.respond_mode && { respond_mode: options.respond_mode }),
             ...(hasAttention && { current_attention_object: options.current_attention_object }),
             participant_sid: this._participantSid || this._room.localParticipant?.sid,
         });
     }
+    visionStatus(options = {}) {
+        if (!this.isTransportReady())
+            return null;
+        const updateId = options.updateId ?? this.createUpdateId("vision-status");
+        this.publishMessage("vision-status", {
+            update_id: updateId,
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
+        });
+        return updateId;
+    }
+    visionTrigger(options = {}) {
+        if (!this.isTransportReady())
+            return null;
+        const updateId = options.updateId ?? this.createUpdateId("vision-trigger");
+        const respondMode = options.respondMode;
+        if (shouldPreemptForExplicitRespondMode(respondMode)) {
+            this.startTrackedConversation(`[vision-trigger:${respondMode}]`);
+        }
+        this.publishMessage("vision-trigger", {
+            update_id: updateId,
+            ...(options.text && { text: options.text }),
+            ...(respondMode && { respond_mode: respondMode }),
+            ...(options.frameIndices && { frame_indices: options.frameIndices }),
+            ...(options.frameIds && { frame_ids: options.frameIds }),
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
+        });
+        return updateId;
+    }
+    respondModeUpdate(options) {
+        if (!this.isTransportReady())
+            return null;
+        const updateId = options.updateId ?? this.createUpdateId("respond-mode");
+        this.publishMessage("respond-mode-update", {
+            update_id: updateId,
+            modality: options.modality,
+            mode: options.mode,
+            participant_sid: this._participantSid || this._room.localParticipant?.sid,
+        });
+        return updateId;
+    }
     /**
      * Send descriptive scene updates. Use when environment description changes
      * and the bot should know what is visible or nearby. Does not modify action_config.
      */
-    updateSceneMetadata(items) {
+    updateSceneMetadata(items, options = {}) {
         if (!this.isTransportReady())
             return;
-        this.publishMessage("update-scene-metadata", { scene_metadata: items });
+        if (shouldPreemptForExplicitRespondMode(options.respondMode)) {
+            this.startTrackedConversation(`[scene-metadata:${options.respondMode}]`);
+        }
+        this.publishMessage("update-scene-metadata", {
+            scene_metadata: items,
+            ...(options.respondMode && { respond_mode: options.respondMode }),
+        });
+    }
+    createUpdateId(prefix) {
+        if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
+            return `${prefix}:${crypto.randomUUID()}`;
+        }
+        return `${prefix}:${Date.now()}:${Math.random().toString(36).slice(2)}`;
+    }
+    serializeRespondModes(config) {
+        return {
+            ...(config.text !== undefined && { text: config.text }),
+            ...(config.audio !== undefined && { audio: config.audio }),
+            ...(config.vision !== undefined && { vision: config.vision }),
+            ...(config.contextUpdate !== undefined && {
+                context_update: config.contextUpdate,
+            }),
+            ...(config.trigger !== undefined && { trigger: config.trigger }),
+            ...(config.sceneMetadata !== undefined && {
+                scene_metadata: config.sceneMetadata,
+            }),
+        };
     }
     /**
      * Upload a file to the character via the LiveKit byte stream transport.