@d-id/client-sdk 1.0.29-beta.55 → 1.1.0-beta.2

package/dist/src/types/entities/agents/manager.d.ts

@@ -1,8 +1,9 @@
-import { SupportedStreamScipt } from '../../StreamScript';
+import { STTTokenResponse } from '../..';
+import { SupportedStreamScipt } from '../../stream-script';
 import { Auth } from '../../auth';
-import { SendStreamPayloadResponse, StreamingState } from '../../stream';
+import { CompatibilityMode, ConnectionState, SendStreamPayloadResponse, StreamEvents, StreamingState } from '../../stream';
 import { Agent } from './agent';
-import { ChatResponse, Message, RatingEntity } from './chat';
+import { ChatMode, ChatResponse, Message, RatingEntity } from './chat';
 /**
  * Types of events provided in Chat Progress Callback
  */
@@ -28,20 +29,20 @@ export declare enum ChatProgress {
      */
     Complete = "done"
 }
-export type ChatProgressCallback = (progress: ChatProgress, data: string) => void;
-export type ConnectionStateChangeCallback = (state: RTCIceConnectionState) => void;
+export type ChatProgressCallback = (progress: ChatProgress | StreamEvents, data: any) => void;
+export type ConnectionStateChangeCallback = (state: ConnectionState) => void;
 export type VideoStateChangeCallback = (state: StreamingState, data: any) => void;
 interface ManagerCallbacks {
     /**
      * Optional callback will be triggered each time the RTC connection changes state
      * @param state
      */
-    onConnectionStateChange?(state: RTCIceConnectionState): void;
+    onConnectionStateChange?(state: ConnectionState): void;
     /**
      * Optional callback function that will be triggered each time video events happen
      * @param state
      */
-    onVideoStateChange?(state: StreamingState, data?: any): void;
+    onVideoStateChange?(state: StreamingState): void;
     /**
      * Callback function that will be triggered each time the video stream starts or stops to update html element on webpage
      * Required callback for SDK
@@ -52,22 +53,76 @@ interface ManagerCallbacks {
      */
     onSrcObjectReady(srcObject: MediaStream): void;
     /**
-     * Optional callback function that will be triggered each time any changes happen in the chat
-     * @param progress
+     * Optional callback function that will be triggered each time new message is received
+     * @param messages - array of messages
      */
-    onChatEvents?(progress: ChatProgress, data: any): void;
+    onNewMessage?(messages: Message[], type: 'answer' | 'partial' | 'user'): void;
     /**
-     * Optional callback function that will be triggered when the agent is ready
-     * @param agent - Agent instance you are working with
+     * Optional callback function that will be triggered each time new chat is created
+     * @param chatId - id of the new chat
      */
-    onAgentReady?(agent: Agent): void;
+    onNewChat?(chatId: string): void;
+    /**
+     * Optional callback function that will be triggered each time the chat mode changes
+     * @param mode - ChatMode
+     */
+    onModeChange?(mode: ChatMode): void;
+    /**
+     * Optional callback function that will be triggered on fetch request errors
+     */
+    onError?: (error: Error, errorData?: object) => void;
+}
+interface StreamOptions {
+    /**
+     * Defines the video codec to be used in the stream.
+     * When set to on: VP8 will be used.
+     * When set to off: H264 will be used
+     * When set to auto the codec will be selected according to the browser.
+     * @default auto
+     */
+    compatibilityMode?: CompatibilityMode;
+    /**
+     * Whether to stream wamrup video on the connection.
+     * If set to true, will stream a warmup video when connection is established.
+     * At the end of the warmup video, a message containing "stream/ready" will be sent on the data channel.
+     * @default false
+     */
+    streamWarmup?: boolean;
+    /**
+     * Maximum duration (in seconds) between messages before session times out.
+     * Can only be used with proper permissions
+     * @maximum 300
+     * @example 180
+     */
+    sessionTimeout?: number;
+    /**
+     * Desired stream resolution for the session
+     * @minimum 150
+     * @maximum 1080
+     */
+    outputResolution?: number;
+    /**
+     * Whether to stream greeting video on the connection.
+     * Not supported for premium presenters.
+     */
+    streamGreeting?: boolean;
 }
 export interface AgentManagerOptions {
+    auth: Auth;
     callbacks: ManagerCallbacks;
+    mode: ChatMode;
     baseURL?: string;
     wsURL?: string;
     debug?: boolean;
-    auth: Auth;
+    enableAnalitics?: boolean;
+    mixpanelKey?: string;
+    /**
+     * Unique ID of agent user used in analytics. Pass it to override the default way to get distinctId
+     */
+    distinctId?: string;
+    streamOptions?: StreamOptions;
+    initialMessages?: Message[];
+    persistentChat?: boolean;
 }
 export interface AgentManager {
     /**
@@ -80,30 +135,34 @@ export interface AgentManager {
      */
     starterMessages: string[];
     /**
-     * Method to be reconnected to chat
-     * Since chat uses an RTC connection to communicate with the agent, it could be dropped and to continue to chat you need to reconnect
+     * Get a token for the Speech to Text service
+     * Only available after a chat has started and the agent has been connected
+     */
+    getSTTToken: () => Promise<STTTokenResponse | undefined>;
+    /**
+     * Method to connect to stream and chat
+     */
+    connect: () => Promise<void>;
+    /**
+     * Method to reconnect to stream and continue chat
      */
     reconnect: () => Promise<void>;
     /**
      * Method to close all connections with agent, stream and web socket
      */
     disconnect: () => Promise<void>;
-    /**
-     * ID of chat you are working on now
-     */
-    chatId: string;
     /**
      * Method to send a chat message to existing chat with the agent
      * @param messages
      */
-    chat: (messages: Message[]) => Promise<ChatResponse>;
+    chat: (userMessage: string) => Promise<ChatResponse>;
     /**
      * Method to rate the answer in chat
      * @param score: 1 | -1 - score of the answer. 1 for positive, -1 for negative
      * @param matches - array of matches that were used to find the answer
      * @param id - id of Rating entity. Leave it empty to create a new, one or pass it to work with the existing one
      */
-    rate: (score: 1 | -1, Message: Message, id?: string) => Promise<RatingEntity>;
+    rate: (messageId: string, score: 1 | -1, rateId?: string) => Promise<RatingEntity>;
     /**
      * Method to delete rating from answer in chat
      * @param id - id of Rating entity.
@@ -113,6 +172,16 @@ export interface AgentManager {
      * Method to make your agent read the text you provide or reproduce sound
      * @param payload
      */
-    speak: (payload: SupportedStreamScipt) => Promise<SendStreamPayloadResponse>;
+    speak: (payload: SupportedStreamScipt | string) => Promise<SendStreamPayloadResponse>;
+    /**
+     * Method to change the mode of the chat
+     * @param mode - ChatMode
+     */
+    changeMode(mode: ChatMode): void;
+    /**
+     * Method to enrich analytics properties
+     * @param properties flat json object with properties that will be added to analytics events fired from the sdk
+     */
+    enrichAnalytics: (properties: Record<string, any>) => void;
 }
 export {};

package/dist/src/types/entities/agents/presenter.d.ts

@@ -1,9 +1,12 @@
-import { ExtendedTextToSpeechProviders } from '../../tts';
+import { ExtendedTextToSpeechProviders } from '../../voice/tts';
+import { Rect } from '../../face-rect';
 export type videoType = 'talk' | 'clip';
 export type Presenter = TalkPresenter | ClipPresenter;
 export interface BasePresenter {
     type: videoType;
-    voice?: ExtendedTextToSpeechProviders;
+    voice?: ExtendedTextToSpeechProviders & {
+        language?: string;
+    };
     idle_video?: string;
     thumbnail?: string;
 }
@@ -11,10 +14,13 @@ export interface TalkPresenter extends BasePresenter {
     type: 'talk';
     source_url: string;
     driver_url?: string;
+    stitch?: boolean;
+    face?: Rect;
 }
 export interface ClipPresenter extends BasePresenter {
     type: 'clip';
     driver_id: string;
     background?: string;
     presenter_id: string;
+    is_greenscreen?: boolean;
 }

package/dist/src/types/entities/video.d.ts

@@ -2,3 +2,4 @@ export declare enum VideoType {
     Clip = "clip",
     Talk = "talk"
 }
+export declare const mapVideoType: (type: string) => VideoType;

package/dist/src/types/face-rect.d.ts

@@ -0,0 +1,6 @@
+export interface Rect {
+    top: number;
+    left: number;
+    bottom: number;
+    right: number;
+}

package/dist/src/types/index.d.ts

@@ -1,6 +1,6 @@
-export * from './StreamScript';
-export * from './tts';
-export * from './stream';
+export * from './stream-script';
 export * from './auth';
 export * from './entities';
-export * from '..';
+export * from './stream';
+export * from './voice/stt';
+export * from './voice/tts';

package/dist/src/types/stream/api/clip.d.ts

@@ -25,16 +25,14 @@ interface ClipConfig {
 }
 export interface CreateClipStreamRequest {
     /**
-     * ID of the selected presenter.
-     * @example "rian-lZC6MmWfC1"
+     * The output resolution sets the maximum height or width of the streamed video.
+     * The aspect ratio is preserved from the source video.
+     * When resolution is not configured, it defaults to the agent output resolution.
+     * @minimum 150
+     * @maximum 1080
+     * @example 512
      */
-    presenter_id: string;
-    /**
-     * ID of the selected driver.
-     * If not provided a driver video will be selected for you from the predefined drivers bank.
-     * @example "mXra4jY38i"
-     */
-    driver_id?: string;
+    output_resolution?: number;
     /**
      * Defines the video codec to be used in the stream.
      * When set to on: VP8 will be used.
@@ -43,6 +41,19 @@ export interface CreateClipStreamRequest {
      * @default auto
      */
     compatibility_mode?: CompatibilityMode;
+    /**
+     * Whether to stream wamrup video on the connection.
+     * If set to true, will stream a warmup video when connection is established.
+     * At the end of the warmup video, a message containing "stream/ready" will be sent on the data channel.
+     */
+    stream_warmup?: boolean;
+    /**
+     * Maximum duration (in seconds) between messages before session times out.
+     * Can only be used with proper permissions
+     * @maximum 300
+     * @example 180
+     */
+    session_timeout?: number;
 }
 export interface SendClipStreamPayload extends StickyRequest {
     script: StreamScript;
@@ -106,5 +117,6 @@ export interface SendClipStreamPayload extends StickyRequest {
      * @example "https://path.to.directory/"
      */
     raw_result_url?: string;
+    metadata: Record<string, any>;
 }
 export {};

package/dist/src/types/stream/api/talk.d.ts

@@ -1,6 +1,6 @@
 import { StreamScript } from '../..';
+import { CompatibilityMode } from '../stream';
 export interface CreateTalkStreamRequest {
-    source_url: string;
     driver_url?: string;
     face?: {
         size: number;
@@ -20,6 +20,36 @@ export interface CreateTalkStreamRequest {
         align_expand_factor?: number;
         stitch?: boolean;
     };
+    /**
+     * The output resolution sets the maximum height or width of the streamed video.
+     * The aspect ratio is preserved from the source image.
+     * When resolution is not configured, it defaults to the agent output resolution.
+     * @minimum 150
+     * @maximum 1080
+     * @example 512
+     */
+    output_resolution?: number;
+    /**
+     * Defines the video codec to be used in the stream.
+     * When set to on: VP8 will be used.
+     * When set to off: H264 will be used
+     * When set to auto the codec will be selected according to the browser.
+     * @default auto
+     */
+    compatibility_mode?: CompatibilityMode;
+    /**
+     * Whether to stream wamrup video on the connection.
+     * If set to true, will stream a warmup video when connection is established.
+     * At the end of the warmup video, a message containing "stream/ready" will be sent on the data channel.
+     */
+    stream_warmup?: boolean;
+    /**
+     * Maximum duration (in seconds) between messages before session times out.
+     * Can only be used with proper permissions
+     * @maximum 300
+     * @example 180
+     */
+    session_timeout?: number;
 }
 export interface SendTalkStreamPayload {
     script: StreamScript;

package/dist/src/types/stream/rtc.d.ts

@@ -40,17 +40,17 @@ export interface IceCandidate {
      * The format of this address is a candidate-attribute as defined in RFC 5245. This string is empty ("") if the
      * RTCIceCandidate is an "end of candidates" indicator.
      */
-    candidate: string;
+    candidate: string | null;
     /**
      * A string specifying the candidate's media stream identification tag which uniquely identifies the media stream
      * within the component with which the candidate is associated, or null if no such association exists.
      */
-    sdpMid: string;
+    sdpMid?: string;
     /**
      * If not null, sdpMLineIndex indicates the zero-based index number of the media description (as defined in RFC
      * 4566) in the SDP with which the candidate is associated.
      */
-    sdpMLineIndex: number;
+    sdpMLineIndex?: number;
 }
 export interface Status {
     status: string;

package/dist/src/types/stream/stream.d.ts

@@ -1,3 +1,5 @@
+import { Analytics } from '../../services/analytics/mixpanel';
+import { VideoRTCStatsReport } from '../../services/streaming-manager/stats/report';
 import { Auth } from '../auth';
 import { VideoType } from '../entities';
 import { CreateClipStreamRequest, CreateTalkStreamRequest, SendClipStreamPayload, SendTalkStreamPayload } from './api';
@@ -12,21 +14,38 @@ export declare enum StreamEvents {
     ChatPartial = "chat/partial",
     StreamDone = "stream/done",
     StreamStarted = "stream/started",
-    StreamFailed = "stream/error"
+    StreamFailed = "stream/error",
+    StreamReady = "stream/ready",
+    StreamCreated = "stream/created",
+    StreamVideoCreated = "stream-video/started",
+    StreamVideoDone = "stream-video/done",
+    StreamVideoError = "stream-video/error",
+    StreamVideoRejected = "stream-video/rejected"
+}
+export declare enum ConnectionState {
+    New = "new",
+    Fail = "fail",
+    Connected = "connected",
+    Connecting = "connecting",
+    Closed = "closed",
+    Completed = "completed",
+    Disconnected = "disconnected"
 }
 export interface ManagerCallbacks {
     onMessage?: (event: string, data: string) => void;
-    onConnectionStateChange?: (state: RTCIceConnectionState) => void;
-    onVideoStateChange?: (state: StreamingState, data?: any) => void;
+    onConnectionStateChange?: (state: ConnectionState) => void;
+    onVideoStateChange?: (state: StreamingState, report?: VideoRTCStatsReport) => void;
     onSrcObjectReady?: (value: MediaStream) => void;
+    onError?: (error: Error, errorData: object) => void;
 }
 export type ManagerCallbackKeys = keyof ManagerCallbacks;
 export interface TalkStreamOptions extends CreateTalkStreamRequest {
     videoType: VideoType.Talk;
+    stream_greeting?: string;
 }
 export interface ClipStreamOptions extends CreateClipStreamRequest {
     videoType: VideoType.Clip;
-    idle_video_url?: string;
+    stream_greeting?: string;
 }
 export type CreateStreamOptions = TalkStreamOptions | ClipStreamOptions;
 export type PayloadType<T> = T extends TalkStreamOptions ? SendTalkStreamPayload : T extends ClipStreamOptions ? SendClipStreamPayload : never;
@@ -41,16 +60,43 @@ export interface StreamingManagerOptions {
     callbacks: ManagerCallbacks;
     baseURL?: string;
     debug?: boolean;
+    warmup?: boolean;
     auth: Auth;
+    analytics: Analytics;
 }
 export interface SlimRTCStatsReport {
     index: number;
+    codec: string;
+    duration?: number;
+    bitrate?: number;
     timestamp: any;
     bytesReceived: any;
     packetsReceived: any;
     packetsLost: any;
+    framesDropped: any;
+    framesDecoded: any;
     jitter: any;
+    jitterBufferDelay: number;
     frameWidth: any;
     frameHeight: any;
     framesPerSecond: any;
+    freezeCount: number;
+    freezeDuration: number;
+}
+export interface AnalyticsRTCStatsReport {
+    timestamp?: number;
+    duration: number;
+    bytesReceived: number;
+    bitrate: number;
+    packetsReceived: number;
+    packetsLost: number;
+    framesDropped: number;
+    framesDecoded: number;
+    jitter: number;
+    jitterBufferDelay: number;
+    framesPerSecond: number;
+    freezeCount: number;
+    freezeDuration: number;
+    lowFpsCount?: number;
+    causes?: string[];
 }

package/dist/src/types/{StreamScript.d.ts → stream-script.d.ts}

@@ -1,5 +1,5 @@
 import { Message } from './entities';
-import { StreamTextToSpeechProviders } from './tts';
+import { StreamTextToSpeechProviders } from './voice/tts';
 export type StreamScriptType = 'text' | 'audio';
 export interface BaseStreamScript {
     type: StreamScriptType;

package/dist/src/types/voice/stt.d.ts

@@ -0,0 +1,4 @@
+export interface STTTokenResponse {
+    token: string;
+    region: string;
+}

package/dist/src/utils/agent.d.ts

@@ -0,0 +1,4 @@
+import { Agent } from '../types';
+type AgentType = 'clip_v2' | Agent['presenter']['type'];
+export declare const getAgentType: (presenter: Agent['presenter']) => AgentType;
+export {};

package/dist/src/utils/analytics.d.ts

@@ -0,0 +1,13 @@
+import { Agent } from '../types/index';
+export declare function getAnalyticsInfo(agent: Agent): {
+    $os: string;
+    isMobile: string;
+    browser: string;
+    origin: string;
+    agentType: "talk" | "clip" | "clip_v2";
+    agentVoice: {
+        voiceId: string | undefined;
+        provider: import('../types/index').Providers | undefined;
+    };
+};
+export declare function getStreamAnalyticsProps(data: any, agent: Agent, additionalProps: Record<string, any>): any;

package/dist/src/utils/index.d.ts

@@ -0,0 +1,2 @@
+export declare const sleep: (ms: number) => Promise<unknown>;
+export declare const getRandom: () => string;

package/dist/src/utils/retry-operation.d.ts

@@ -0,0 +1,38 @@
+interface RetryOptions {
+    limit?: number;
+    delayMs?: number;
+    timeout?: number;
+    timeoutErrorMessage?: string;
+    shouldRetryFn?: (error: any) => boolean;
+    onRetry?: (error: any) => void;
+}
+/**
+ * Utility function to retry a promise operation with configurable retry logic
+ *
+ * @param operation - A function that returns a Promise to be retried
+ * @param userOptions - Configuration options for retry behavior
+ * @param userOptions.limit - Maximum number of retry attempts (default: 3)
+ * @param userOptions.delayMs - Delay between retries in milliseconds (default: 0, no delay)
+ * @param userOptions.timeout - Timeout for each attempt in milliseconds (default: 30000, set 0 to disable)
+ * @param userOptions.timeoutErrorMessage - Custom timeout error message
+ * @param userOptions.shouldRetryFn - Function to determine if retry should occur based on error, that will force throw even if limit is not reached when returns "false"
+ *
+ * @returns Promise that resolves with the operation result or rejects with the last error
+ *
+ * @throws {Error} Last error encountered after all retries are exhausted
+ * @throws {Error} Timeout error if operation exceeds specified timeout
+ *
+ * @example
+ * // With custom options
+ * const result = await retryOperation(
+ *   async () => await fetch(url),
+ *   {
+ *     limit: 5,
+ *     delayMs: 1000,
+ *     timeout: 5000,
+ *     shouldRetryFn: (error) => error.status === 429
+ *   }
+ * );
+ */
+export declare function retryOperation<T>(operation: () => Promise<T>, userOptions?: RetryOptions): Promise<T>;
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@d-id/client-sdk",
     "private": false,
-    "version": "1.0.29-beta.55",
+    "version": "1.1.0-beta.2",
     "type": "module",
     "description": "d-id client sdk",
     "repository": {
@@ -20,11 +20,12 @@
     ],
     "main": "./dist/index.umd.cjs",
     "module": "./dist/index.js",
-    "types": "./dist/src/types/index.d.ts",
+    "types": "./dist/src/index.d.ts",
     "scripts": {
         "dev": "vite",
         "build": "node ./infra/build.js -m production",
         "build:dev": "node ./infra/build.js -m development",
+        "dev:prod": "export NODE_ENV=production && vite --mode production",
         "deploy:prod": "node ./infra/deploy.js --version beta",
         "preview": "vite preview",
         "test-build": "node .infra/build.js -m development",
@@ -44,4 +45,4 @@
         "vite": "^5.1.4",
         "vite-plugin-dts": "^3.7.3"
     }
-}
+}

package/dist/src/api/clipStream.d.ts

@@ -1,2 +0,0 @@
-import { Auth, RtcApi } from '../types/index';
-export declare function createApi(auth: Auth, host: string): RtcApi;

package/dist/src/api/getClient.d.ts

@@ -1,7 +0,0 @@
-import { Auth } from '../types/auth';
-export declare function createClient(auth: Auth, host?: string): {
-    get<T = any>(url: string, options?: RequestInit): Promise<T>;
-    post<T_1 = any>(url: string, body?: any, options?: RequestInit): Promise<T_1>;
-    delete<T_2 = any>(url: string, body?: any, options?: RequestInit): Promise<T_2>;
-    patch<T_3 = any>(url: string, body?: any, options?: RequestInit): Promise<T_3>;
-};

package/dist/src/api/ratings.d.ts

@@ -1,7 +0,0 @@
-import { Auth, RatingEntity, RatingPayload } from '../index';
-export declare function createRatingsApi(auth: Auth, host?: string): {
-    create(payload: RatingPayload, options?: RequestInit): Promise<RatingEntity>;
-    getByKnowledge(knowledgeId?: string, options?: RequestInit): Promise<RatingEntity[]>;
-    update(id: string, payload: Partial<RatingPayload>, options?: RequestInit): Promise<RatingEntity>;
-    delete(id: string, options?: RequestInit): Promise<RatingEntity>;
-};

package/dist/src/api/talkStream.d.ts

@@ -1,2 +0,0 @@
-import { Auth, RtcApi } from '../types/index';
-export declare function createApi(auth: Auth, host: string): RtcApi;

package/dist/src/connectToSocket.d.ts

@@ -1,9 +0,0 @@
-import { Auth } from './types/auth';
-import { ChatProgressCallback } from '.';
-interface SocketManager {
-    socket?: WebSocket;
-    disconnect: () => void;
-    subscribeToEvents: (data: any) => void;
-}
-export declare function SocketManager(auth: Auth, host: string, onMessage?: ChatProgressCallback): Promise<SocketManager>;
-export {};

package/dist/src/utils/webrtc.d.ts

@@ -1,2 +0,0 @@
-import { SlimRTCStatsReport } from '../types';
-export declare function createVideoStatsReport(stats: SlimRTCStatsReport[], previousStats?: SlimRTCStatsReport): SlimRTCStatsReport[];