@sanseng/livekit-ws-sdk 0.1.20 → 1.0.0

package/dist/index.d.ts

@@ -79,6 +79,7 @@ interface AudioEvents {
   'inner:audio:input:started': void;
   'inner:audio:input:stopped': void;
   'inner:audio:frame:received': { data: Float32Array; sampleRate: number };
+  'inner:audio:no_speech:reported': { silenceDurationMs: number; timestamp: number };
   /**
    * Raw audio payload event
    * Contains PCM audio data and metadata (without protocol header)
@@ -134,6 +135,7 @@ interface AudioEvents {
   'inner:audio:track:added': { trackId: string; participantId: string };
   'inner:audio:track:removed': { trackId: string };
   'inner:audio:volume:changed': { volume: number };
+  'inner:audio:speaking:changed': { participantId: string; isSpeaking: boolean };
   'inner:audio:muted': void;
   'inner:audio:unmuted': void;
 }
@@ -160,8 +162,11 @@ interface ConversationEvents {
   'inner:conversation:answer:chunk': { questionId: string; chunk: string; isComplete: boolean };
   'inner:conversation:answer:completed': { questionId: string; fullAnswer: string };
   'inner:conversation:server:message': { questionId: string; message: string; type: string };
+  'inner:conversation:asr:chunk': { questionId: string; text: string; isComplete: boolean };
   'inner:conversation:asr:received': { questionId: string; text: string };
   'inner:conversation:server:command': { code: number; command: string };
+  'inner:conversation:voice:start': { requestId: string };
+  'inner:conversation:voice:finish': { requestId: string };
 }
 
 /**
@@ -171,14 +176,14 @@ interface ConversationEvents {
  * @interface ConnectionEvents
  */
 interface ConnectionEvents {
-  'inner:sdk:connected': { source: 'livekit' | 'ws' | 'http' };
-  'inner:sdk:disconnected': { source: 'livekit' | 'ws' | 'http'; reason?: string };
+  'inner:sdk:connected': { source: 'livekit' | 'http' };
+  'inner:sdk:disconnected': { source: 'livekit' | 'http'; reason?: string };
   'inner:sdk:error': { error: Error };
 }
 
 /**
- * Internal connection fact events (rtc/ws only).
- * Emitted by rtc and transport modules; coordinator aggregates and drives FSM.
+ * Internal connection fact events (rtc).
+ * Emitted by rtc; coordinator aggregates and drives FSM.
  * Must NOT emit sdk:connected / sdk:disconnected.
  *
  * @author sansen
@@ -188,8 +193,6 @@ interface InternalConnectionFactEvents {
   'inner:rtc:disconnected': { reason?: string };
   'inner:rtc:video:available': void;
   'inner:rtc:video:unavailable': void;
-  'inner:ws:connected': void;
-  'inner:ws:disconnected': { reason?: string };
 }
 
 /**
@@ -201,7 +204,8 @@ interface InternalConnectionFactEvents {
  */
 interface PublicSDKEvents {
   // Connection status
-  'sdk:connected': { livekit: boolean; ws: boolean; http: boolean; all: boolean };
+  /** `all` = livekit && http */
+  'sdk:connected': { livekit: boolean; http: boolean; all: boolean };
   'sdk:disconnected': { reason?: string };
   'sdk:error': { message: string; code: string };
 
@@ -227,6 +231,7 @@ interface PublicSDKEvents {
   'conversation:question:sent': { questionId: string; text: string };
   'conversation:answer:waiting': { questionId: string };
   'conversation:server:message': { questionId: string; message: string; type: string };
+  'conversation:asr:chunk': { questionId: string; text: string; isComplete: boolean };
   'conversation:asr:received': { questionId: string; text: string };
   'conversation:answer:chunk': { questionId: string; chunk: string; isComplete: boolean };
   'conversation:answer:completed': { questionId: string; fullAnswer: string };
@@ -257,6 +262,11 @@ type EventListener<T = unknown> = (data: T) => void;
  * Event Bus
  * Implements observer pattern for decoupled communication between modules
  *
+ * @remarks Internal vs public events:
+ * - Subsystems emit **factual** events on this bus using `inner:*` keys (and occasional `sdk:*` aggregates from {@link import('@/client/ConnectionLifecycleCoordinator').ConnectionLifecycleCoordinator}).
+ * - Application code must subscribe via {@link import('./publicEvents').PublicEventEmitter} (`SDKClient.events`), which maps a whitelist of `inner:*` → stable `media:*` / `conversation:*` / `sdk:*` names and strips sensitive shapes.
+ * - Do not treat every `inner:*` emission as a stable API; only forwarded public names in {@link import('./publicEvents').ALLOWED_EVENTS} are supported for external listeners.
+ *
  * @author sansen
  * @class EventBus
  * @extends {Disposable}
@@ -352,7 +362,7 @@ declare enum SessionStatus {
      */
     CONNECTED = "connected",
     /**
-     * Partially connected (e.g. only RTC or only WS)
+     * Partially connected (e.g. only HTTP session or only LiveKit)
      *
      * @author sansen
      */
@@ -407,7 +417,9 @@ declare class SessionState {
     status: SessionStatus;
     state: SessionStatus;
     /**
-     * Indicates if currently connected to server (both rtc and ws when fully connected)
+     * True when FSM is `connected`.
+     * - direct mode: LiveKit RTC connected
+     * - auth mode: HTTP config/session + LiveKit RTC connected
      *
      * @type {boolean}
      * @default false
@@ -584,39 +596,15 @@ interface VideoOptions extends BaseOptions {
   fitMode?: RenderFitMode;
 }
 
-/**
- * Audio Input Configuration Options
- *
- * @author sansen
- * @interface AudioInputOptions
- */
 interface AudioInputOptions {
-  /**
-   * Audio sample rate in Hz (default 24000)
-   *
-   * @type {number}
-   */
+  deviceId?: string;
   sampleRate?: number;
-
-  /**
-   * Audio bit depth in bits (default 16)
-   *
-   * @type {number}
-   */
-  bitDepth?: number;
-
-  /**
-   * Number of audio channels, default 1 (mono)
-   *
-   * @type {number}
-   */
+  channelCount: number;
   channels?: number;
-
-  /**
-   * Media track constraints for audio input
-   *
-   * @type {MediaTrackConstraints}
-   */
+  sampleSize?: number;
+  noiseSuppression?: boolean;
+  voiceIsolation?: boolean;
+  bitDepth?: number;
   constraints?: MediaTrackConstraints;
 }
 
@@ -672,6 +660,47 @@ interface AudioOptions extends BaseOptions {
   output?: AudioOutputOptions;
 }
 
+/**
+ * Performance metric names emitted by SDK monitoring module.
+ */
+type PerformanceMetricName =
+  | 'connect_to_first_frame_ms'
+  | 'text_send_to_text_response_ms'
+  | 'text_send_to_audio_response_ms'
+  | 'no_speech_report_to_audio_response_ms';
+
+/**
+ * Normalized metric payload for external observability systems.
+ */
+interface PerformanceMetricRecord {
+  /** Stable metric name */
+  metric: PerformanceMetricName;
+  /** Duration in milliseconds */
+  durationMs: number;
+  /** Start timestamp (ms since epoch) */
+  startedAt: number;
+  /** End timestamp (ms since epoch) */
+  endedAt: number;
+  /** Optional correlation id (for question scoped metrics) */
+  questionId?: string;
+}
+
+/**
+ * Reporter callback for forwarding performance metrics to external systems (e.g. SkyWalking).
+ * If omitted, SDK uses the built-in reporter (default logger output).
+ */
+type PerformanceMetricReporter = (metric: PerformanceMetricRecord) => void;
+
+/**
+ * Performance monitor options for client initialization.
+ */
+interface PerformanceMonitorOptions {
+  /** Enable performance monitoring. Defaults to true. */
+  enabled?: boolean;
+  /** Optional external metric reporter callback. */
+  reporter?: PerformanceMetricReporter;
+}
+
 /**
  * Client Configuration Options
  *
@@ -680,32 +709,6 @@ interface AudioOptions extends BaseOptions {
  * @extends {BaseOptions}
  */
 interface ClientOptions extends BaseOptions {
-  /**
-   * WebSocket server URL prefix (for upstream communication)
-   *
-   * @type {string}
-   * @default ''
-   * @author sansen
-   */
-  wsUrlPrefix?: string;
-
-  /**
-   * Avatar name identifier
-   *
-   * @type {string}
-   * @default ''
-   * @author sansen
-   */
-  avatarName: string;
-
-  /**
-   * Avatar voice identifier
-   *
-   * @type {string}
-   * @default ''
-   * @author sansen
-   */
-  avatarVoice?: string;
   /**
    * Audio configuration options
    *
@@ -763,6 +766,13 @@ interface ClientOptions extends BaseOptions {
      */
     headers?: Record<string, string>;
   };
+  /**
+   * Performance monitoring options for external observability integration.
+   * Built-in monitoring is enabled by default even when this field is omitted.
+   *
+   * @type {PerformanceMonitorOptions}
+   */
+  performanceMonitor?: PerformanceMonitorOptions;
   /**
    * Indicates if sandbox is enabled
    *
@@ -771,6 +781,29 @@ interface ClientOptions extends BaseOptions {
    * @author sansen
    */
   sandbox?: boolean;
+
+  /**
+   * Connection configuration
+   * direct: direct connection to sfu server
+   * @type {Object}
+   * @author sansen
+   */
+  connectConfig:
+    | {
+        type: 'direct';
+        config: {
+          sfuUrl: string;
+          clientToken: string;
+        };
+      }
+    | {
+        type: 'auth';
+        config: {
+          authToken?: string;
+          avatarId: string;
+          avatarVoice?: string;
+        };
+      };
 }
 
 /**
@@ -799,17 +832,17 @@ declare class SDKError extends Error {
     /**
      * Creates an SDKError instance
      *
+     * @param {string} code - Error code for categorization; defaults to {@link ErrorCode.SDK_ERROR} when using {@link SDKError.fromError}
      * @param {string} message - Error description message
-     * @param {string} [code='SDK_ERROR'] - Error code for categorization
      * @param {unknown} [cause] - Original error that caused this SDKError
      * @author sansen
      */
-    constructor(message: string, code?: string, cause?: unknown);
+    constructor(code: string, message: string, cause?: unknown);
     /**
      * Creates an SDKError from any error type
      *
      * @param {unknown} error - Original error to convert
-     * @param {string} [code] - Optional error code, defaults to 'SDK_ERROR'
+     * @param {string} [code] - Optional error code; defaults to {@link ErrorCode.SDK_ERROR}
      * @returns {SDKError} SDKError instance
      * @author sansen
      */
@@ -825,11 +858,11 @@ declare class SDKError extends Error {
 type SDKConnectionState = 'idle' | 'connecting' | 'connected' | 'partial' | 'reconnecting:auto' | 'reconnecting:manual' | 'disconnecting' | 'disconnected' | 'disposed' | 'error';
 /**
  * Read-only connection snapshot.
- * Derived from aggregated internal facts (rtc:* / ws:* events), not from LiveKit Room or Track objects.
+ * Derived from aggregated internal facts (rtc:* and HTTP session facts), not from LiveKit Room or Track objects.
  * Used for user-driven reconnect UX: available synchronously when user triggers manual reconnect.
  *
  * @remarks
- * - ws.connected: whether WebSocket (conversation) is currently connected.
+ * - http.connected: whether the HTTP stream session is active (after /stream/v1/start success).
  * - rtc.connected: whether RTC (LiveKit) is currently connected.
  * - rtc.hasVideoTrack: whether at least one video track is available (from rtc:video:available / rtc:video:unavailable).
  * - overall.state: current SDK lifecycle state from SDKConnectionFSM.
@@ -837,8 +870,8 @@ type SDKConnectionState = 'idle' | 'connecting' | 'connected' | 'partial' | 'rec
  * @author sansen
  */
 interface ConnectionSnapshot {
-    readonly ws: {
-        /** True when WebSocket connection is open. */
+    readonly http: {
+        /** True after HTTP start/stream session is established (room config applied). */
         readonly connected: boolean;
     };
     readonly rtc: {
@@ -853,6 +886,29 @@ interface ConnectionSnapshot {
     };
 }
 
+/**
+ * Records which microphone capture path was last claimed (`rtc` = LiveKit; `ws` label unused in LiveKit-only SDK but kept for audio module compatibility).
+ *
+ * @author sansen
+ */
+declare class MediaAudioCaptureState {
+    private _active;
+    /**
+     * Currently active capture label, or null if none.
+     */
+    get active(): 'ws' | 'rtc' | null;
+    /**
+     * @param mode - Capture label to claim
+     * @returns false if another label is already active
+     */
+    tryBegin(mode: 'ws' | 'rtc'): boolean;
+    /**
+     * @param mode - Capture label to release
+     */
+    end(mode: 'ws' | 'rtc'): void;
+    reset(): void;
+}
+
 /**
  * SDK Context
  * Contains all shared states and event bus
@@ -875,6 +931,10 @@ declare class SDKContext {
      * @memberof SDKContext
      */
     readonly sessionState: SessionState;
+    /**
+     * Microphone capture mode coordination (see {@link MediaAudioCaptureState}).
+     */
+    readonly mediaAudioCaptureState: MediaAudioCaptureState;
     /**
      * Client configuration options
      *
@@ -911,7 +971,7 @@ declare class SDKContext {
      * Emits an error event and creates SDKError instance
      *
      * @param {unknown} error - Original error
-     * @param {string} code - Error code
+     * @param {string} code - Error code; use a value from {@link import('@/errors/ErrorCodes').ErrorCode}
      * @returns {SDKError} Created SDKError instance
      * @author sansen
      */
@@ -923,14 +983,7 @@ declare class SDKContext {
      */
     setVideoOptions(options: VideoOptions): void;
     /**
-     * Sets WebSocket URL
-     *
-     * @param {string} wsUrl - WebSocket server URL
-     * @author sansen
-     */
-    setWsUrl(wsUrl: string): void;
-    /**
-     * Gets WebSocket URL
+     * Opaque URL from preConnect HTTP response.
      *
      * @type {string}
      * @author sansen
@@ -996,10 +1049,26 @@ interface PublicEmitterAPI {
     once<K extends keyof PublicSDKEvents>(event: K, listener: (data: PublicSDKEvents[K]) => void): () => void;
 }
 
+interface DirectConnectionConfig {
+    sfuUrl: string;
+    clientToken: string;
+}
+
 /**
  * SDK client
  * Provides a unified API surface.
  * Orchestrates the lifecycles of all subsystems.
+ *
+ * @remarks Upstream transport: session text, microphone, and camera uplink use {@link LiveKitService} (publish / publishData); {@link HttpController} is the control plane.
+ *
+ * @remarks SDK lifecycle (ordering is contract for {@link ConnectionLifecycleCoordinator} and session state):
+ * 1. **Constructor** — builds {@link SDKContext} only; no I/O.
+ * 2. **preConnect** — idempotent/cached: {@link HttpController}, auth token, LiveKit config; may be skipped by callers but is always awaited inside {@link connect}.
+ * 3. **connect** — creates coordinator → binds snapshot → `startConnecting` → bootstraps domain controllers → {@link LiveKitService} → wires protocol dispatch and RTC uplinks → awaits LiveKit connect.
+ * 4. **Runtime** — public methods guard with {@link _ensureConnected}; media/conversation use controllers; connection facts emit on internal bus (`inner:*`).
+ * 5. **disconnect** — coordinator `startDisconnecting` → stop audio/LiveKit/camera → HTTP conversation end → `completeDisconnecting`.
+ * 6. **reconnect** — snapshot before transition → stop capture paths → optionally full LiveKit teardown when video missing → preConnect + restore LiveKit.
+ * 7. **dispose** — triggers {@link disconnect} then disposes controllers, services, coordinator, and context.
  * @author sansen
  */
 declare class SDKClient extends Disposable {
@@ -1031,29 +1100,28 @@ declare class SDKClient extends Disposable {
      */
     private _cameraController;
     /**
-     * LiveKit service
+     * LiveKit service; downstream RTC and session uplink (media + data).
      * @author sansen
      */
     private _liveKitService;
-    /**
-     * Conversation transport service
-     * @author sansen
-     */
-    private _conversationTransport;
     /**
      * Central coordinator for connection lifecycle (snapshot + FSM).
-     * Created in connect(); drives sessionState from rtc/ws fact events.
+     * Created in connect(); drives sessionState from rtc/http fact aggregation.
      * @author sansen
      */
     private _connectionCoordinator;
     /**
-     * HTTP controller
+     * HTTP controller (auth, stream start/stop, avatar control).
      * @author sansen
      */
     private _httpController;
-    private _preConnectPromise;
-    /** Timestamp (ms) of last successful preConnect */
-    private _lastPreConnectSuccessAt;
+    private _configManager;
+    private _performanceMonitor;
+    private _pendingDirectConfig;
+    private _isReconnectInProgress;
+    private _reconnectPromise;
+    /** Inbound session protocol: {@link ProtocolMessageDispatcher.dispatch}. */
+    private _liveKitProtocolDispatcher;
     /**
      * Create SDK client
      * @param options Client configuration options
@@ -1065,23 +1133,6 @@ declare class SDKClient extends Disposable {
      * @author sansen
      */
     get events(): PublicEmitterAPI;
-    /**
-     * Push actions to the server
-     * @param onceActions One-time actions
-     * @param loopActions Repeating/loop actions
-     * @returns Promise<boolean> resolves to true on success, or false on failure
-     * @author sansen
-     */
-    pushActions({ onceActions, loopActions }: {
-        onceActions?: string[];
-        loopActions?: string[];
-    }): Promise<boolean> | undefined;
-    /**
-     * Interrupt the current conversation connection
-     * @returns Promise<boolean> resolves to true on success, or false on failure
-     * @author sansen
-     */
-    interruptConnection(): Promise<boolean> | undefined;
     /**
      * Send a text question/message
      * @param text The question text to send
@@ -1089,6 +1140,13 @@ declare class SDKClient extends Disposable {
      * @author sansen
      */
     sendTextQuestion(text: string): Promise<string>;
+    interrupt(): Promise<void>;
+    /**
+     * Set external performance metric reporter.
+     * @param reporter Reporter callback
+     * @author sansen
+     */
+    setPerformanceMetricReporter(reporter: PerformanceMetricReporter | undefined): void;
     /**
      * Set render fit mode
      * @param mode Render fit mode
@@ -1097,7 +1155,7 @@ declare class SDKClient extends Disposable {
     setRenderFitMode(mode: RenderFitMode): void;
     /**
      * Start audio capture
-     * Begins microphone capture and starts sending audio frames over WebSocket
+     * Begins microphone capture and publishes audio via LiveKit
      * @returns Promise<void>
      * @throws SDKError if the SDK is not initialized or capture fails to start
      * @author sansen
@@ -1105,7 +1163,7 @@ declare class SDKClient extends Disposable {
     startAudioCapture(): Promise<void>;
     /**
      * Stop audio capture
-     * Stops microphone capture and stops sending audio frames over WebSocket
+     * Stops microphone capture and LiveKit microphone publish
      * @throws SDKError if the SDK is not initialized
      * @author sansen
      */
@@ -1155,7 +1213,12 @@ declare class SDKClient extends Disposable {
      */
     setAuthToken(token: string): void;
     /**
-     * Start camera capture and periodic screenshot sending over WebSocket
+     * Update direct-mode connection config for next reconnect.
+     * Does not affect current active connection.
+     */
+    updateConnectionConfig(config: DirectConnectionConfig): void;
+    /**
+     * Start camera capture and publish via LiveKit
      * @returns Promise<void>
      * @throws SDKError if not connected or camera controller unavailable
      * @author sansen
@@ -1190,6 +1253,9 @@ declare class SDKClient extends Disposable {
      * @returns Promise<boolean> resolves to true on success
      * @author sansen
      */
+    /**
+     * @remarks Pre-connect path: dedupe in-flight work, short TTL cache on success, then HTTP auth + LiveKit config into context.
+     */
     preConnect(): Promise<boolean>;
     /**
      * Connect to the server
@@ -1197,12 +1263,18 @@ declare class SDKClient extends Disposable {
      * @returns Promise<void>
      * @author sansen
      */
+    /**
+     * @remarks Connect sequence: preConnect → coordinator + domain bootstrap → LiveKit connect. Errors transition coordinator idle and run {@link _cleanup}.
+     */
     connect(): Promise<void>;
     /**
      * Disconnect
      * Tears down active connections but retains controller instances to allow reconnect
      * @author sansen
      */
+    /**
+     * @remarks User-initiated teardown: stop local capture hooks, LiveKit, camera controller, HTTP conversation; clears preConnect in-flight handle.
+     */
     disconnect(): Promise<void>;
     /**
      * Manual reconnect.
@@ -1211,9 +1283,12 @@ declare class SDKClient extends Disposable {
      * @returns Promise<ConnectionSnapshot> Snapshot captured at the moment reconnect was triggered (before transition).
      * @author sansen
      */
+    /**
+     * @remarks Reconnect uses coordinator gating; transport restoration mirrors connect (preConnect + LiveKit) without re-running full client constructor.
+     */
     reconnect(): Promise<ConnectionSnapshot>;
     /**
-     * Current connection snapshot (ws/rtc/overall state).
+     * Current connection snapshot (rtc/http/overall state).
      * Derived from internal facts; available synchronously for user-driven reconnect UX.
      *
      * @returns Read-only ConnectionSnapshot
@@ -1225,8 +1300,35 @@ declare class SDKClient extends Disposable {
      * @author sansen
      */
     private _cleanup;
+    private _disposeConnectionRuntimeBeforeConnect;
+    /**
+     * Instantiate domain controllers and bind context (video/audio/camera/conversation).
+     */
+    private _bootstrapDomainControllers;
+    /**
+     * Create LiveKit service; conversation uses RTC text sender only.
+     */
+    private _bootstrapLiveKitService;
+    /**
+     * Attach LiveKit event adapter to video/audio controllers for track routing.
+     */
+    private _wireLiveKitEventAdapterToControllers;
+    /**
+     * Configure LiveKit data path dispatcher and RTC text sender for conversation.
+     */
+    private _setupRtcProtocolDispatcher;
+    /**
+     * Connect LiveKit (downstream + uplink).
+     */
+    private _establishMediaAndSessionTransports;
+    private _refreshConnectionConfigForReconnect;
+    private _ensureConfigManagerReady;
+    private _applyConnectionConfig;
+    private _stageDirectConfigForNextReconnect;
+    private _validateDirectConnectionConfig;
+    private _ensurePerformanceMonitorReady;
     /**
-     * Check whether the SDK is connected (both rtc and ws; synced by ConnectionLifecycleCoordinator).
+     * Whether the SDK is fully connected (HTTP session + LiveKit RTC; synced by ConnectionLifecycleCoordinator).
      * @author sansen
      */
     get isConnected(): boolean;
@@ -1236,6 +1338,10 @@ declare class SDKClient extends Disposable {
      * @author sansen
      */
     private _ensureConnected;
+    private _startAudioCaptureRtc;
+    private _stopAudioCaptureRtc;
+    private _requireAudioController;
+    private _requireLiveKitService;
     /**
      * Release resources
      * @author sansen
@@ -1720,4 +1826,4 @@ declare class StandaloneVideoDebugger extends Disposable {
 }
 
 export { ConversationStateType, DEFAULT_AUDIO_BUFFER_SIZE, DEFAULT_AUDIO_CHANNELS, DEFAULT_AUDIO_SAMPLE_RATE, DEFAULT_RECONNECT_ATTEMPTS, DEFAULT_RECONNECT_DELAY, DEFAULT_WS_TIMEOUT, Disposable, EventBus, LogLevel, Logger, SDKClient, SDKContext, SDKError, SessionState, SessionStatus, StandaloneVideoDebugger, createClient };
-export type { AsyncResult, AudioEvents, AudioOptions, BaseOptions, Callback, ClientOptions, ConnectionEvents, ConnectionSnapshot, ConversationEvents, ConversationOptions, ErrorCallback, GreenScreenOptions, IDisposable, InputMode, SDKConnectionState, SDKEvents, VideoEvents, VideoOptions, VideoRenderMode };
+export type { AsyncResult, AudioEvents, AudioOptions, BaseOptions, Callback, ClientOptions, ConnectionEvents, ConnectionSnapshot, ConversationEvents, ConversationOptions, ErrorCallback, GreenScreenOptions, IDisposable, InputMode, PerformanceMetricName, PerformanceMetricRecord, PerformanceMetricReporter, PerformanceMonitorOptions, SDKConnectionState, SDKEvents, VideoEvents, VideoOptions, VideoRenderMode };