@unisphere/models-sdk-js 1.2.0

package/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@unisphere/models-sdk-js",
+  "version": "1.2.0",
+  "author": "kaltura",
+  "private": false,
+  "license": "AGPL-3.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/kaltura/models"
+  },
+  "types": "./src/index.d.ts",
+  "dependencies": {},
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org"
+  },
+  "module": "./index.esm.js",
+  "type": "module"
+}

package/src/KalturaAvatarSession.d.ts

@@ -0,0 +1,123 @@
+/**
+ * Kaltura Avatar SDK - Main Public API
+ *
+ * This is the main entry point for the Kaltura Avatar SDK. It provides a simple,
+ * high-level API for creating avatar sessions, controlling avatar behavior, and
+ * managing video streams.
+ *
+ * @example
+ * ```typescript
+ * const session = new KalturaAvatarSession('your-api-key');
+ *
+ * await session.createSession({ avatarId: 'avatar-123' });
+ * session.attachAvatar(videoElement);
+ * await session.sayText('Hello from Kaltura Avatar!');
+ * await session.endSession();
+ * ```
+ */
+import { AvatarConfig, ConnectionState, CreateSessionOptions, SessionState, SessionEvent } from './types';
+/**
+ * Kaltura Avatar Session - Main SDK class
+ */
+export declare class KalturaAvatarSession {
+    private readonly controlSDK;
+    private readonly rtcSDK;
+    private readonly logger;
+    private state;
+    private sessionId;
+    private eventHandlers;
+    /**
+     * Create a new Kaltura Avatar Session
+     *
+     * @param apiKey - API key for authentication
+     * @param config - Optional configuration
+     */
+    constructor(apiKey: string, config?: AvatarConfig);
+    /**
+     * Create a new avatar session
+     *
+     * This method:
+     * 1. Creates a session with the backend
+     * 2. Initializes the client
+     * 3. Establishes WebRTC connection
+     *
+     * @param options - Session creation options
+     */
+    createSession(options: CreateSessionOptions): Promise<void>;
+    /**
+     * Attach avatar video to a video element
+     *
+     * @param videoElement - HTML video element to display avatar
+     */
+    attachAvatar(videoElement: HTMLVideoElement): void;
+    /**
+     * Send audio file for the avatar to speak
+     *
+     * @param mp3File - MP3 file or Blob to speak
+     */
+    say(mp3File: File | Blob): Promise<void>;
+    /**
+     * Send text for the avatar to speak
+     *
+     * The text will be automatically chunked into 3-word segments for optimal processing.
+     *
+     * @param text - Text for the avatar to speak
+     */
+    sayText(text: string): Promise<void>;
+    /**
+     * Interrupt the avatar's current speech
+     */
+    interrupt(): Promise<void>;
+    /**
+     * End the avatar session and cleanup resources
+     */
+    endSession(): Promise<void>;
+    /**
+     * Get current session state
+     */
+    getSessionState(): SessionState;
+    /**
+     * Get current connection state
+     */
+    getConnectionState(): ConnectionState;
+    /**
+     * Get current session ID
+     */
+    getSessionId(): string | null;
+    /**
+     * Register event handler
+     *
+     * @param event - Event name
+     * @param callback - Callback function
+     */
+    on(event: SessionEvent, callback: (data?: any) => void): void;
+    /**
+     * Unregister event handler
+     *
+     * @param event - Event name
+     * @param callback - Callback function
+     */
+    off(event: SessionEvent, callback: (data?: unknown) => void): void;
+    /**
+     * Setup event handlers for RTC SDK
+     */
+    private setupRTCEventHandlers;
+    /**
+     * Ensure session is in READY state
+     * @throws AvatarError if not ready
+     */
+    private ensureReady;
+    /**
+     * Set internal session state
+     *
+     * @param state - New session state
+     */
+    private setState;
+    /**
+     * Emit event to registered handlers
+     *
+     * @param event - Event name
+     * @param data - Event data
+     */
+    private emit;
+}

package/src/control-sdk/AvatarControlSDK.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Avatar Control SDK - HTTP API Client
+ *
+ * Handles all HTTP communication with the avatar backend including:
+ * - Session lifecycle (create, init, end)
+ * - Avatar control (say, sayText, interrupt)
+ * - JWT token management
+ * - Request retry logic
+ */
+import { RetryConfig } from '../types';
+import { VisualVoiceConfig, CreateSessionResponse, InitClientResponse } from './types';
+/**
+ * Configuration for the Avatar Control SDK
+ */
+export interface ControlSDKConfig {
+    /** Base URL for the avatar backend API */
+    baseUrl: string;
+    /** API key for authentication */
+    apiKey: string;
+    /** Retry configuration */
+    retryConfig?: RetryConfig;
+    /** Log level */
+    logLevel?: string;
+    /** Request timeout in milliseconds */
+    timeoutMs?: number;
+}
+/**
+ * Avatar Control SDK - Manages HTTP communication with avatar backend
+ */
+export declare class AvatarControlSDK {
+    private readonly baseUrl;
+    private readonly apiKey;
+    private readonly retryConfig;
+    private readonly logger;
+    private readonly timeoutMs;
+    private sessionToken;
+    constructor(config: ControlSDKConfig);
+    /**
+     * Create a new avatar session
+     *
+     * @param config - Visual and voice configuration
+     * @returns Session ID and JWT token
+     */
+    createSession(config: VisualVoiceConfig): Promise<CreateSessionResponse>;
+    /**
+     * Initialize client and get WHEP URL for WebRTC connection
+     *
+     * @param sessionId - Session identifier
+     * @returns WHEP URL for WebRTC connection
+     */
+    initClient(sessionId: string): Promise<InitClientResponse>;
+    /**
+     * Send text for the avatar to speak
+     *
+     * @param sessionId - Session identifier
+     * @param text - Text chunk to speak
+     * @param turnId - Turn ID for grouping related chunks
+     * @param isFinal - Whether this is the final chunk in the turn
+     */
+    sayText(sessionId: string, text: string, turnId: string, isFinal: boolean): Promise<void>;
+    /**
+     * Send audio file for the avatar to speak
+     *
+     * @param sessionId - Session identifier
+     * @param audioFile - MP3 file to speak
+     */
+    say(sessionId: string, audioFile: File | Blob): Promise<void>;
+    /**
+     * Interrupt the avatar's current speech
+     *
+     * @param sessionId - Session identifier
+     */
+    interrupt(sessionId: string): Promise<void>;
+    /**
+     * End the avatar session
+     *
+     * @param sessionId - Session identifier
+     */
+    endSession(sessionId: string): Promise<void>;
+    /**
+     * Make an HTTP request with retry logic and timeout
+     *
+     * @param endpoint - API endpoint (relative to baseUrl)
+     * @param options - Fetch options
+     * @returns Response data
+     */
+    private makeRequest;
+    /**
+     * Ensure JWT token is available
+     * @throws AvatarError if not authenticated
+     */
+    private ensureAuthenticated;
+}

package/src/control-sdk/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Avatar Control SDK exports
+ */
+export * from './AvatarControlSDK';
+export * from './types';

package/src/control-sdk/types.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Types for the Avatar Control SDK (HTTP API)
+ */
+/**
+ * Configuration for visual and voice settings
+ */
+export interface VisualVoiceConfig {
+    /** Avatar visual ID */
+    avatarId: string;
+    /** Optional voice ID for the avatar */
+    voiceId?: string;
+}
+/**
+ * Response from session creation endpoint
+ */
+export interface CreateSessionResponse {
+    /** Whether the session was created successfully */
+    success: boolean;
+    /** Unique session identifier */
+    sessionId: string;
+    /** JWT token for authenticating subsequent requests */
+    token: string;
+    /** Optional error message if creation failed */
+    error?: string;
+}
+/**
+ * Response from client initialization endpoint
+ */
+export interface InitClientResponse {
+    /** Whether initialization was successful */
+    success: boolean;
+    /** WHEP URL for WebRTC connection */
+    whepUrl: string;
+    /** Optional error message if initialization failed */
+    error?: string;
+}
+/**
+ * Internal request body for say-text endpoint
+ */
+export interface SayTextRequest {
+    /** Text chunk to speak */
+    text: string;
+    /** Turn ID for grouping related chunks */
+    turnId: string;
+    /** Whether this is the final chunk in the turn */
+    isFinal: boolean;
+}

package/src/index.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Kaltura Avatar SDK
+ *
+ * A TypeScript SDK for integrating with Kaltura's speech-to-video avatar service.
+ *
+ * @example
+ * ```typescript
+ * import { KalturaAvatarSession } from '@kaltura-corp/unisphere-rtc-avatar';
+ *
+ * const session = new KalturaAvatarSession('your-api-key');
+ *
+ * // Create session
+ * await session.createSession({ avatarId: 'avatar-123' });
+ *
+ * // Attach to video element
+ * const videoElement = document.getElementById('avatar-video') as HTMLVideoElement;
+ * session.attachAvatar(videoElement);
+ *
+ * // Send text for avatar to speak
+ * await session.sayText('Hello from Kaltura Avatar!');
+ *
+ * // Interrupt speech
+ * await session.interrupt();
+ *
+ * // End session
+ * await session.endSession();
+ * ```
+ *
+ * @packageDocumentation
+ */
+export { KalturaAvatarSession } from './KalturaAvatarSession';
+export type { AvatarConfig, CreateSessionOptions, SessionEvent, RetryConfig, } from './types';
+export { SessionState, ConnectionState, AvatarError, AvatarErrorCode, DEFAULT_RETRY_CONFIG, } from './types';
+export { AvatarControlSDK } from './control-sdk';
+export type { ControlSDKConfig, VisualVoiceConfig, CreateSessionResponse, InitClientResponse, } from './control-sdk';
+export { AvatarRTCSDK, PeerConnectionManager, SignalingManager, DEFAULT_ICE_SERVERS, } from './rtc-sdk';
+export type { RTCConfig, PeerConnectionConfig, SignalingConfig, } from './rtc-sdk';
+export { chunkText, generateTurnId, } from './utils/text-chunker';
+export { retryWithBackoff, waitForCondition, } from './utils/retry';
+export { createLogger, parseLogLevel, } from './utils/logger';
+export type { Logger, LogLevel, } from './utils/logger';
+export { createAvatarError, isNetworkError, isRetryableStatusCode, shouldRetry, } from './utils/errors';

package/src/rtc-sdk/AvatarRTCSDK.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Avatar RTC SDK - High-level WebRTC management
+ *
+ * This SDK orchestrates the PeerConnectionManager and SignalingManager to provide
+ * a simple interface for WebRTC connections. It handles:
+ * - Connection lifecycle
+ * - Video stream attachment
+ * - Event emission
+ * - Error handling
+ */
+import { ConnectionState } from '../types';
+import { RTCConfig } from './types';
+/**
+ * RTC events that can be subscribed to
+ */
+export type RTCEvent = 'connected' | 'disconnected' | 'failed' | 'track';
+/**
+ * Avatar RTC SDK - Orchestrates WebRTC connection
+ */
+export declare class AvatarRTCSDK {
+    private pcManager;
+    private signalingManager;
+    private videoElement;
+    private state;
+    private readonly logger;
+    private readonly iceServers;
+    private readonly iceTransportPolicy;
+    private readonly retryConfig;
+    private whepUrl;
+    private reconnectAttempts;
+    private isReconnecting;
+    private eventHandlers;
+    constructor(config?: RTCConfig);
+    /**
+     * Connect to WebRTC endpoint using WHEP URL with retry logic
+     *
+     * @param whepUrl - WHEP URL for WebRTC signaling
+     */
+    connect(whepUrl: string): Promise<void>;
+    /**
+     * Internal connection logic (without retry)
+     */
+    private connectInternal;
+    /**
+     * Reconnect after connection failure
+     */
+    private reconnect;
+    /**
+     * Cleanup connection resources without changing state
+     */
+    private cleanup;
+    /**
+     * Attach video element to receive avatar video
+     *
+     * @param videoElement - HTML video element to attach streams to
+     */
+    attachVideo(videoElement: HTMLVideoElement): void;
+    /**
+     * Disconnect and cleanup resources
+     */
+    disconnect(): void;
+    /**
+     * Get current connection state
+     */
+    getConnectionState(): ConnectionState;
+    /**
+     * Register event handler
+     *
+     * @param event - Event name
+     * @param callback - Callback function
+     */
+    on(event: RTCEvent, callback: (data?: unknown) => void): void;
+    /**
+     * Unregister event handler
+     *
+     * @param event - Event name
+     * @param callback - Callback function
+     */
+    off(event: RTCEvent, callback: (data?: unknown) => void): void;
+    /**
+     * Setup event handlers for peer connection
+     */
+    private setupPeerConnectionEventHandlers;
+    /**
+     * Attach media stream to video element
+     *
+     * @param stream - Media stream to attach
+     */
+    private attachStreamToVideo;
+    /**
+     * Wait for connection to be established
+     *
+     * @param timeoutMs - Timeout in milliseconds
+     */
+    private waitForConnection;
+    /**
+     * Set internal connection state
+     *
+     * @param state - New connection state
+     */
+    private setState;
+    /**
+     * Emit event to registered handlers
+     *
+     * @param event - Event name
+     * @param data - Event data
+     */
+    private emit;
+}

package/src/rtc-sdk/PeerConnectionManager.d.ts

@@ -0,0 +1,106 @@
+/**
+ * PeerConnectionManager - Manages RTCPeerConnection lifecycle
+ *
+ * This class wraps RTCPeerConnection and provides a clean API for:
+ * - Creating and configuring peer connections
+ * - Managing transceivers (sendonly, recvonly, sendrecv)
+ * - Creating offers and answers
+ * - Setting local and remote descriptions
+ * - Monitoring connection state
+ * - Handling tracks
+ */
+import { PeerConnectionConfig } from './types';
+/**
+ * PeerConnectionManager - Low-level RTCPeerConnection wrapper
+ */
+export declare class PeerConnectionManager {
+    private pc;
+    private readonly iceServers;
+    private readonly iceTransportPolicy;
+    private readonly logger;
+    private trackHandlers;
+    private connectionStateChangeHandlers;
+    private iceGatheringStateChangeHandlers;
+    constructor(config: PeerConnectionConfig);
+    /**
+     * Create the RTCPeerConnection
+     */
+    createPeerConnection(): void;
+    /**
+     * Add transceivers for audio and video
+     *
+     * @param direction - Direction of the transceivers (sendonly, recvonly, sendrecv)
+     */
+    addTransceivers(direction: RTCRtpTransceiverDirection): void;
+    /**
+     * Create an SDP offer
+     *
+     * @returns SDP offer
+     */
+    createOffer(): Promise<RTCSessionDescriptionInit>;
+    /**
+     * Create an SDP answer
+     *
+     * @returns SDP answer
+     */
+    createAnswer(): Promise<RTCSessionDescriptionInit>;
+    /**
+     * Set local description
+     *
+     * @param desc - SDP description
+     */
+    setLocalDescription(desc: RTCSessionDescriptionInit): Promise<void>;
+    /**
+     * Set remote description
+     *
+     * @param desc - SDP description
+     */
+    setRemoteDescription(desc: RTCSessionDescriptionInit): Promise<void>;
+    /**
+     * Get the peer connection (read-only)
+     */
+    getPeerConnection(): RTCPeerConnection | null;
+    /**
+     * Get current connection state
+     */
+    getConnectionState(): RTCPeerConnectionState;
+    /**
+     * Get current ICE gathering state
+     */
+    getIceGatheringState(): RTCIceGatheringState;
+    /**
+     * Get current signaling state
+     */
+    getSignalingState(): RTCSignalingState;
+    /**
+     * Register handler for track events
+     *
+     * @param callback - Callback to invoke when track is received
+     */
+    onTrack(callback: (event: RTCTrackEvent) => void): void;
+    /**
+     * Register handler for connection state changes
+     *
+     * @param callback - Callback to invoke when connection state changes
+     */
+    onConnectionStateChange(callback: (state: RTCPeerConnectionState) => void): void;
+    /**
+     * Register handler for ICE gathering state changes
+     *
+     * @param callback - Callback to invoke when ICE gathering state changes
+     */
+    onIceGatheringStateChange(callback: (state: RTCIceGatheringState) => void): void;
+    /**
+     * Close the peer connection and clean up resources
+     */
+    close(): void;
+    /**
+     * Setup event handlers for the peer connection
+     */
+    private setupEventHandlers;
+    /**
+     * Ensure peer connection exists
+     * @throws AvatarError if peer connection doesn't exist
+     */
+    private ensurePeerConnection;
+}

package/src/rtc-sdk/SignalingManager.d.ts

@@ -0,0 +1,57 @@
+/**
+ * SignalingManager - Handles SDP offer/answer exchange using WHEP protocol
+ *
+ * WHEP (WebRTC-HTTP Egress Protocol) is an HTTP-based signaling protocol
+ * for WebRTC streaming. This manager handles:
+ * - SDP offer creation
+ * - ICE gathering
+ * - HTTP-based SDP exchange
+ * - SDP answer processing
+ */
+import { SignalingConfig } from './types';
+import { PeerConnectionManager } from './PeerConnectionManager';
+/**
+ * SignalingManager - Handles WHEP protocol signaling
+ */
+export declare class SignalingManager {
+    private readonly whepUrl;
+    private readonly timeout;
+    private readonly retryConfig;
+    private readonly logger;
+    constructor(config: SignalingConfig);
+    /**
+     * Negotiate WebRTC connection using WHEP protocol
+     *
+     * This method orchestrates the full WHEP handshake:
+     * 1. Add receive-only transceivers
+     * 2. Create SDP offer
+     * 3. Wait for ICE gathering
+     * 4. Send offer to WHEP endpoint
+     * 5. Receive SDP answer
+     * 6. Set remote description
+     *
+     * @param pcManager - PeerConnectionManager instance
+     */
+    negotiate(pcManager: PeerConnectionManager): Promise<void>;
+    /**
+     * Wait for ICE gathering to complete
+     *
+     * @param pcManager - PeerConnectionManager instance
+     * @param timeoutMs - Timeout in milliseconds (default: 5000)
+     */
+    private waitForIceGathering;
+    /**
+     * Send SDP offer to WHEP endpoint and receive answer with retry logic
+     *
+     * @param offerSdp - SDP offer string
+     * @returns SDP answer string
+     */
+    private sendOfferToServer;
+    /**
+     * Internal method to send SDP offer (without retry)
+     *
+     * @param offerSdp - SDP offer string
+     * @returns SDP answer string
+     */
+    private sendOfferToServerInternal;
+}

package/src/rtc-sdk/index.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Avatar RTC SDK exports
+ */
+export * from './AvatarRTCSDK';
+export * from './PeerConnectionManager';
+export * from './SignalingManager';
+export * from './types';

package/src/rtc-sdk/types.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Types for the Avatar RTC SDK (WebRTC)
+ */
+import { Logger } from '../utils/logger';
+import { RetryConfig } from '../types';
+/**
+ * Configuration for the RTC SDK
+ */
+export interface RTCConfig {
+    /** ICE servers for WebRTC connection */
+    iceServers?: RTCIceServer[];
+    /** ICE transport policy */
+    iceTransportPolicy?: 'all' | 'relay';
+    /** Retry configuration for connection and reconnection */
+    retryConfig?: RetryConfig;
+    /** Logger instance */
+    logger?: Logger;
+    /** Log level */
+    logLevel?: string;
+}
+/**
+ * Configuration for PeerConnectionManager
+ */
+export interface PeerConnectionConfig {
+    /** ICE servers for WebRTC connection */
+    iceServers: RTCIceServer[];
+    /** ICE transport policy */
+    iceTransportPolicy?: 'all' | 'relay';
+    /** Logger instance */
+    logger?: Logger;
+}
+/**
+ * Configuration for SignalingManager
+ */
+export interface SignalingConfig {
+    /** WHEP URL for signaling */
+    whepUrl: string;
+    /** Request timeout in milliseconds */
+    timeout?: number;
+    /** Retry configuration for WHEP requests */
+    retryConfig?: RetryConfig;
+    /** Logger instance */
+    logger?: Logger;
+}
+/**
+ * Default ICE servers (Google STUN)
+ */
+export declare const DEFAULT_ICE_SERVERS: RTCIceServer[];