@kaltura-sdk/avatars 1.14.0

package/src/lib/avatar-client-manager/types.d.ts

@@ -0,0 +1,4 @@
+import { Notifications } from '@unisphere/notifications-core';
+export type AvatarClientManagerActions = {
+    showNotification: (notification: Notifications) => void;
+};

package/src/lib/avatar-client-manager/utils/actor-manager.d.ts

@@ -0,0 +1,54 @@
+import type { AnyActorRef, SnapshotFrom } from 'xstate';
+import { TinyDataStore } from '@unisphere/core';
+export type ActorSnapshot<TActor extends AnyActorRef> = SnapshotFrom<TActor>;
+interface StoreBinding<TSnapshot> {
+    skip?: (snapshot: TSnapshot) => boolean;
+    apply: (snapshot: TSnapshot) => void;
+    reset: () => void;
+}
+/**
+ * Generic engine bridging one xstate actor to a set of `TinyDataStore`s:
+ * owns the actor's lifecycle (start/subscribe/stop), replays snapshots into
+ * bound stores, and resets/cleans up on disconnect. Knows nothing about
+ * avatars specifically — domain state and terminal-state detection are
+ * supplied by subclasses via the hooks below.
+ */
+export declare abstract class ActorManager<TActor extends AnyActorRef> {
+    protected _actor: TActor | null;
+    private _subscriptions;
+    /**
+     * Bindings registered by `_select()` below. Rebuilding all of them to
+     * their initial value (`_resetConnectionState`) is what replaces "throw
+     * away the old proxy instance and create a fresh one" on every
+     * disconnect/reconnect cycle.
+     */
+    protected readonly _storeBindings: StoreBinding<ActorSnapshot<TActor>>[];
+    /**
+     * Registers a piece of connection state derived from the xstate snapshot.
+     * Cuts the old "TinyDataStore field + StoreBinding entry + getter" trio
+     * down to a single field declaration per property.
+     */
+    protected _select<T>(initialValue: T, extract: (snapshot: ActorSnapshot<TActor>) => T, options?: {
+        equals?: (a: T, b: T) => boolean;
+        skip?: (snapshot: ActorSnapshot<TActor>) => boolean;
+    }): TinyDataStore<T>;
+    protected _setupActorSync(actor: TActor): void;
+    private _sync;
+    protected _resetConnectionState(): void;
+    protected _cleanupConnection(): void;
+    /**
+     * Domain state derived from the snapshot that isn't a plain `_select`
+     * binding (cross-references external state, combines multiple checks,
+     * etc). Runs after all bindings have been applied.
+     */
+    protected abstract _syncDerivedState(snapshot: ActorSnapshot<TActor>): void;
+    /** Whether this snapshot means the connection is done and should be torn down. */
+    protected abstract _isTerminalSnapshot(snapshot: ActorSnapshot<TActor>): boolean;
+    /** Called once, right after the first sync, before subscribing to further updates. */
+    protected abstract _onSetupActorSync(actor: TActor): void;
+    /** Called after all bindings have been reset back to their initial values. */
+    protected abstract _onReset(): void;
+    /** Called once, right before the actor is stopped. */
+    protected abstract _onCleanup(): void;
+}
+export {};

package/src/lib/avatar-client-manager/utils/set-if-changed.d.ts

@@ -0,0 +1,2 @@
+import { TinyDataStore } from '@unisphere/core';
+export declare function setIfChanged<T>(store: TinyDataStore<T>, next: T, eq?: (a: T, b: T) => boolean): void;

package/src/lib/server/avatar-rtc-client-manager.d.ts

@@ -0,0 +1,104 @@
+import { IceCandidateData } from '@kaltura-sdk/rtc-avatar';
+import { DeviceType, GetUserMediaEveObj } from '@kaltura-sdk/rtc-core';
+import { UserDevicesList } from '@unisphere/genie-types';
+export interface RTCClientConfig {
+    turnServerUrl: string;
+    noiseReductionEnabled?: boolean;
+}
+export interface SignalingCallbacks {
+    sendOfferCallback: (connectionId: string, offer: RTCSessionDescriptionInit, isReconnect: boolean) => Promise<RTCSessionDescriptionInit>;
+    sendICECandidateCallback: (connectionId: string, candidate: IceCandidateData) => Promise<void>;
+}
+export interface RTCClientStatus {
+    isInitialized: boolean;
+    hasSession: boolean;
+    asrConnected: boolean;
+    stvConnected: boolean;
+}
+export type AudioState = {
+    muted: boolean;
+    isSpeaking: boolean;
+};
+declare class AvatarRTCClientManager {
+    private _session;
+    private _isInitialized;
+    private _asrConnection;
+    private _stvConnection;
+    private _onStvConnectedCallback;
+    private _devicesList;
+    private _audioState;
+    private _audioVolumeLevel;
+    private _lastVolumeUpdateAt;
+    private _subscriptions;
+    private _unsubscribeConnectionState;
+    private _unsubscribeDevicesListChange;
+    private _unsubscribeDeviceChanged;
+    private _setupWSEventListeners;
+    private _setupAudioStateListeners;
+    private _updateAudioState;
+    get devicesList(): import("@unisphere/core").TinyDataStoreConsumer<UserDevicesList | null>;
+    get audioState(): import("@unisphere/core").TinyDataStoreConsumer<AudioState>;
+    get audioVolumeLevel(): import("@unisphere/core").TinyDataStoreConsumer<number>;
+    /**
+     * Initialize the eSelf SDK session
+     * Must be called before getUserMedia or joinASR
+     */
+    initSession(config: RTCClientConfig): Promise<void>;
+    parseNoiseReductionParams(configEnabled?: boolean): {
+        nrEnabled: boolean;
+        nrMode: number;
+    };
+    /**
+     * Get user media (microphone) via the SDK
+     * The SDK handles audio constraints internally
+     */
+    getUserMedia(): Promise<{
+        gumResult: GetUserMediaEveObj;
+        stream: MediaStream | null;
+    }>;
+    /**
+     * Join ASR (Automatic Speech Recognition) connection with WebRTC
+     *
+     * @param signalingCallbacks - Callbacks for WebRTC signaling (offer/answer/ICE)
+     * @param sessionId - Session identifier (typically the socket peerId)
+     */
+    joinASR(signalingCallbacks: SignalingCallbacks, sessionId: string): Promise<void>;
+    /**
+     * Leave ASR connection
+     */
+    leaveASR(): void;
+    /**
+     * Join STV (Streaming Video) connection for avatar playback
+     * The SDK injects a <video> element into the container identified by videoContainerId
+     *
+     * @param videoContainerId - DOM element ID for the video container
+     * @param sessionId - STV session/stream identifier
+     * @param stvUrl - STV WHEP endpoint URL
+     * @param onConnected - Callback fired when the WebRTC peer connection is established
+     */
+    joinSTV(videoContainerId: string, sessionId: string, stvUrl: string, onConnected: () => void): Promise<void>;
+    /**
+     * Leave STV connection
+     */
+    leaveSTV(): void;
+    /**
+     * Get current connection status
+     */
+    getStatus(): RTCClientStatus;
+    /**
+     * Leave ASR/STV peer connections without destroying the underlying session.
+     * Use on WS reconnect: the EselfSession, user media, and device listeners stay
+     * alive so connectToASR can rejoin without re-initializing from scratch.
+     */
+    leaveConnections(): void;
+    /**
+     * Cleanup and destroy the SDK session
+     * Call this when disconnecting or on error
+     */
+    cleanup(): void;
+    getDevices(): Promise<UserDevicesList>;
+    changeDevice(deviceType: DeviceType, deviceId: string): Promise<boolean>;
+    toggleMuteAudio(muted: boolean): Promise<void>;
+}
+export declare const avatarRTCClientManager: AvatarRTCClientManager;
+export {};

package/src/lib/server/index.d.ts

@@ -0,0 +1,2 @@
+export * from './websocket-client-manager';
+export * from './avatar-rtc-client-manager';

package/src/lib/server/websocket-client-manager.d.ts

@@ -0,0 +1,44 @@
+import { Socket } from '../vendor/socket-io-esm';
+import { TinyDataStore, TinyPubSub } from '@unisphere/core';
+export declare const websocketRecoveryState: TinyDataStore<{
+    status: 'connected' | 'recovering' | 'disconnected';
+}>;
+export declare function resetWebsocketClientTeardownState(): void;
+export declare const websocketMessage: TinyPubSub<{
+    name: string;
+    payload: unknown;
+}>;
+export declare const websocketSystemMessage: TinyPubSub<{
+    name: string;
+    payload: unknown;
+}>;
+export declare const websocketNonRecoveredReconnect: TinyPubSub<{
+    newSocketId: string;
+}>;
+export declare const websocketDebugInfo: TinyDataStore<{
+    hostName: string;
+    socketId: string;
+    partnerId: string;
+}>;
+interface WebsocketClientManager {
+    createClient(serverUrl: string, query: Record<string, string>, params: {
+        ks: string;
+    }): Socket;
+    getClient(serverUrl: string): Socket | null;
+    destroyClient(serverUrl: string): void;
+    destroyAllClients(): void;
+}
+declare class DefaultWebsocketClientManager implements WebsocketClientManager {
+    private _instances;
+    constructor();
+    private _handleMessage;
+    createClient(serverUrl: string, query: Record<string, any>, params: {
+        ks: string;
+    }): Socket;
+    getClient(serverUrl: string): Socket | null;
+    destroyClient(serverUrl: string): void;
+    destroyAllClients(): void;
+}
+export declare const websocketClientManager: DefaultWebsocketClientManager;
+export declare function initializeLogsCollectionIfNeeded(conversationManagerUrl: string, partnerId?: string, flowId?: string, ks?: string): void;
+export {};

package/src/lib/timing-tracker.d.ts

@@ -0,0 +1,158 @@
+/**
+ * Avatar Timing Tracker Module
+ *
+ * Centralized module for tracking performance timing throughout the avatar connection session.
+ *
+ * TIMING FLOW OVERVIEW:
+ * ======================
+ * STEP 1: User clicks "Talk to Agent" → Runtime connects
+ *         Measures: Initial button click to runtime connection
+ *         Parameters: buttonClickTime → runtimeConnectTime
+ *
+ * STEP 2: Runtime connect → WebSocket connected
+ *         Measures: WebSocket establishment
+ *         Parameters: runtimeConnectTime → websocketConnectedTime
+ *
+ * STEP 3: WebSocket connected → Session created
+ *         Measures: Backend session creation (join room + create session)
+ *         Parameters: websocketConnectedTime → sessionCreatedTime
+ *
+ * STEP 4: Session created → User media granted
+ *         Measures: User microphone permission and getUserMedia
+ *         Parameters: sessionCreatedTime → userMediaGrantedTime
+ *
+ * STEP 5: User media granted → ASR WebRTC connected
+ *         Measures: ASR peer connection establishment
+ *         Parameters: userMediaGrantedTime → asrConnectedTime
+ *         Sub-steps:
+ *           5a: ASR init signal sent
+ *           5b: ASR backend ready
+ *           5c: ASR peer connection established
+ *
+ * STEP 6: Session created → Video player URL available
+ *         Measures: Time from session to video player component initialization
+ *         Parameters: sessionCreatedTime → playerUrlAvailableTime
+ *
+ * STEP 7: Player URL available → SRS WebRTC connected
+ *         Measures: SRS WHEP video connection establishment
+ *         Parameters: playerUrlAvailableTime → srsWebrtcConnectedTime
+ *         Sub-steps:
+ *           7a: SRS player manager start
+ *           7b: ICE gathering
+ *           7c: WHEP request
+ *           7d: Peer connection established
+ *
+ * STEP 8: SRS connected → Video playing
+ *         Measures: Time from connection to first video frame
+ *         Parameters: srsWebrtcConnectedTime → videoPlayingTime
+ *
+ * FILES INVOLVED:
+ * ===============
+ * - avatar-launcher-button.tsx: Sets buttonClickTime
+ * - runtime.tsx: Sets runtimeConnectTime
+ * - connecting-state.ts: Sets websocketConnectedTime, sessionCreatedTime
+ * - init-rtc-session.ts: Sets userMediaGrantedTime
+ * - connect-to-asr.ts: Sets asrInitSignalTime, asrBackendReadyTime, asrConnectedTime
+ * - avatar.tsx: Sets playerUrlAvailableTime
+ * - SRSWebRTCPlayerManager.ts: Sets srsWebrtcStartTime, srsWebrtcConnectedTime, videoPlayingTime
+ *
+ * WHEN TO RESET:
+ * ==============
+ * Call timingTracker.reset() at:
+ * - Start of new avatar connection (avatar-launcher-button.tsx before connection)
+ * - After connection is closed (avatar-client-manager.ts disconnect)
+ *
+ * Usage:
+ * ```ts
+ * import { timingTracker } from './timing-tracker';
+ *
+ * // Reset for new session
+ * timingTracker.reset();
+ *
+ * // Set a timing value
+ * timingTracker.buttonClickTime = performance.now();
+ *
+ * // Get a timing value
+ * const startTime = timingTracker.buttonClickTime;
+ *
+ * // Log summary
+ * timingTracker.logSummary();
+ * ```
+ */
+declare class AvatarTimingTracker {
+    /**
+     * When "Talk to Agent" button is clicked by user
+     * Set in: avatar-launcher-button.tsx - handleLaunchAvatar()
+     */
+    buttonClickTime: DOMHighResTimeStamp | null;
+    /**
+     * When runtime.connect() is called
+     * Set in: runtime.tsx - connect() method
+     */
+    runtimeConnectTime: DOMHighResTimeStamp | null;
+    /**
+     * When WebSocket connection is established
+     * Set in: connecting-state.ts - connectingWebsocket onDone transition
+     */
+    websocketConnectedTime: DOMHighResTimeStamp | null;
+    /**
+     * When stvNewSession request is sent to backend
+     * Set in: send-create-session.ts - when stvNewSession is emitted
+     */
+    sessionRequestSentTime: DOMHighResTimeStamp | null;
+    /**
+     * When session is created (has sessionId and webrtcUrl)
+     * Set in: connecting-state.ts - creatingSession onDone transition
+     */
+    sessionCreatedTime: DOMHighResTimeStamp | null;
+    /**
+     * When user grants microphone permission and getUserMedia completes
+     * Set in: init-rtc-session.ts - after getUserMedia() succeeds
+     */
+    userMediaGrantedTime: DOMHighResTimeStamp | null;
+    /**
+     * When ASR init signal is sent to backend
+     * Set in: connect-to-asr.ts - after asr-webrtc-init emit
+     */
+    asrInitSignalTime: DOMHighResTimeStamp | null;
+    /**
+     * When backend signals ASR is ready
+     * Set in: connect-to-asr.ts - on asr-webrtc-ready event
+     */
+    asrBackendReadyTime: DOMHighResTimeStamp | null;
+    /**
+     * When ASR WebRTC peer connection is fully established
+     * Set in: connect-to-asr.ts - after joinASR() completes
+     */
+    asrConnectedTime: DOMHighResTimeStamp | null;
+    /**
+     * When video player component receives URL and starts initialization
+     * Set in: avatar.tsx - when SRSWebRTCPlayer is rendered
+     */
+    playerUrlAvailableTime: DOMHighResTimeStamp | null;
+    /**
+     * When SRS player manager starts connection process
+     * Set in: SRSWebRTCPlayerManager.ts - start() method begin
+     */
+    srsWebrtcStartTime: DOMHighResTimeStamp | null;
+    /**
+     * When SRS WHEP WebRTC peer connection is established
+     * Set in: SRSWebRTCPlayerManager.ts - onconnectionstatechange when state='connected'
+     */
+    srsWebrtcConnectedTime: DOMHighResTimeStamp | null;
+    /**
+     * When video element starts playing (first frame visible)
+     * Set in: SRSWebRTCPlayerManager.ts - video.play().then()
+     */
+    videoPlayingTime: DOMHighResTimeStamp | null;
+    /**
+     * Reset all timing values (useful for new sessions)
+     */
+    reset(): void;
+    /**
+     * Log complete timing summary
+     */
+    logSummary(): void;
+}
+export declare const timingTracker: AvatarTimingTracker;
+export {};

package/src/lib/utils/create-tiny-pubsub-events-handler.d.ts

@@ -0,0 +1,10 @@
+import { EventPayloads, EventsData, TinyPubsubEventsSource } from './create-tiny-pubsub-events-source';
+import { UnisphereLoggerType } from '@unisphere/core';
+export type EventsHandlersMap<TEvents extends EventsData> = {
+    [K in keyof TEvents]?: (payload: EventPayloads<TEvents>[K]) => void;
+};
+export declare const createTinyPubsubEventsHandler: <TEvents extends EventsData>(data: {
+    source: TinyPubsubEventsSource<TEvents> | null | undefined;
+    handlers: EventsHandlersMap<TEvents>;
+    logger: UnisphereLoggerType;
+}) => () => void;

package/src/lib/utils/create-tiny-pubsub-events-source.d.ts

@@ -0,0 +1,27 @@
+import { TinyPubSub, UnisphereLoggerType } from '@unisphere/core';
+import z from 'zod/mini';
+export type EventData<TSchema extends z.ZodMiniType | null = z.ZodMiniType | null> = {
+    schema: TSchema;
+    once?: boolean;
+    abortOnSchemaValidationError?: boolean;
+};
+export type EventsData = Record<string, EventData>;
+export type EventPayloads<TEvents extends EventsData> = {
+    [K in keyof TEvents]: TEvents[K]['schema'] extends z.ZodMiniType ? z.infer<TEvents[K]['schema']> : unknown;
+};
+export type Handler<TEvents extends EventsData> = <TEventName extends keyof TEvents & string>(eventName: TEventName, payload: EventPayloads<TEvents>[TEventName]) => void;
+export interface TinyPubsubEventsSource<TEvents extends EventsData> {
+    setLogger: (logger: UnisphereLoggerType) => void;
+    setHandler: (handler: Handler<TEvents>) => void;
+    removeHandler: () => void;
+    destroy: () => void;
+    resetQueue: () => void;
+}
+export declare const createTinyPubsubEventsSource: <TEvents extends EventsData>(sourceEvent: TinyPubSub<{
+    name: string;
+    payload: unknown;
+}>, eventsMap: TEvents, options?: {
+    logger?: UnisphereLoggerType;
+    queueWhilePending?: boolean;
+    abortOnSchemaValidationError?: boolean;
+}) => TinyPubsubEventsSource<TEvents>;

package/src/lib/utils/get-query-param.d.ts

@@ -0,0 +1,4 @@
+export declare function getQueryParam(paramName: string, options?: {
+    removeAfterRead?: boolean;
+    trim?: boolean;
+}): string;

package/src/lib/utils/index.d.ts

@@ -0,0 +1,5 @@
+export * from './logger-factory';
+export * from './create-tiny-pubsub-events-handler';
+export * from './create-tiny-pubsub-events-source';
+export * from './get-query-param';
+export * from './is-mobile-device';

package/src/lib/utils/is-mobile-device.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This function is direct copy from eself client to ensure compatibility.
+ * It detects if the current device is a mobile device based on the user agent string.
+ * Improve if needed to cover more edge cases.
+ *
+ * @return {boolean} True if the device is mobile, false otherwise.
+ */
+export declare function isMobileDevice(): boolean;

package/src/lib/utils/logger-factory.d.ts

@@ -0,0 +1,2 @@
+import { UnisphereLoggerFactory } from '@unisphere/core';
+export declare const avatarClientLoggerFactory: UnisphereLoggerFactory;