@d-id/client-sdk 1.0.18-beta.10 → 1.0.18-beta.12

package/dist/index.js

@@ -189,7 +189,12 @@ async function SocketManager(auth, host = didSocketApiUrl) {
     host,
     callbacks: {
       onMessage: (event) => {
-        messageCallbacks.forEach((callback) => callback(event));
+        const data = JSON.parse(event.data);
+        let payload;
+        if (data.event === "answer") {
+          payload = data.content;
+        }
+        messageCallbacks.forEach((callback) => callback(data.event, payload));
       }
     }
   });
@@ -230,6 +235,14 @@ var RateState = /* @__PURE__ */ ((RateState2) => {
   RateState2["Negative"] = "Negative";
   return RateState2;
 })(RateState || {});
+var ChatProgress = /* @__PURE__ */ ((ChatProgress2) => {
+  ChatProgress2["Embed"] = "embed";
+  ChatProgress2["Query"] = "query";
+  ChatProgress2["Partial"] = "chat/partial";
+  ChatProgress2["Answer"] = "answer";
+  ChatProgress2["Complete"] = "done";
+  return ChatProgress2;
+})(ChatProgress || {});
 var Subject = /* @__PURE__ */ ((Subject2) => {
   Subject2["KnowledgeProcessing"] = "knowledge/processing";
   Subject2["KnowledgeIndexing"] = "knowledge/indexing";
@@ -297,6 +310,12 @@ function initializeStreamAndChat(agent, options, agentsApi, chat) {
             reject(new Error("Cannot create connection"));
           }
           (_b = (_a = options.callbacks).onConnectionStateChange) == null ? void 0 : _b.call(_a, state);
+        },
+        onMessage: (event, data) => {
+          var _a, _b;
+          if (event === StreamEvents.ChatPartial) {
+            (_b = (_a = options.callbacks).onChatEvents) == null ? void 0 : _b.call(_a, ChatProgress.Partial, data);
+          }
         }
       }
     });
@@ -307,6 +326,7 @@ async function createAgentManager(agentId, options) {
   const abortController = new AbortController();
   const agentsApi = createAgentsApi(options.auth, baseURL);
   const ratingsAPI = createRatingsApi(options.auth, baseURL);
+  const knowledgeApi = createKnowledgeApi(options.auth, baseURL);
   const agent = await agentsApi.getById(agentId);
   const socketManager = await SocketManager(options.auth);
   let {
@@ -327,16 +347,14 @@ async function createAgentManager(agentId, options) {
       return streamingManager.terminate();
     },
     chatId: chat.id,
-    async chat(messages) {
-      const result = await agentsApi.chat(agentId, chat.id, {
+    chat(messages) {
+      return agentsApi.chat(agentId, chat.id, {
         sessionId: streamingManager.sessionId,
         streamId: streamingManager.streamId,
         messages
       }, {
         signal: abortController.signal
       });
-      console.log(result);
-      return result;
     },
     rate(payload, id) {
       if (id) {
@@ -376,6 +394,12 @@ async function createAgentManager(agentId, options) {
     },
     onVideoEvents(callback) {
       streamingManager.onCallback("onVideoStateChange", callback);
+    },
+    getStarterMessages() {
+      var _a, _b;
+      if (!((_a = agent.knowledge) == null ? void 0 : _a.id))
+        return Promise.resolve([]);
+      return knowledgeApi.getKnowledge((_b = agent.knowledge) == null ? void 0 : _b.id).then((knowledge) => (knowledge == null ? void 0 : knowledge.starter_message) || []);
     }
   };
 }
@@ -450,6 +474,44 @@ function createApi(auth, host) {
     }
   };
 }
+function createVideoStatsReport(stats, previousStats) {
+  return stats.map((report, index) => {
+    if (index === 0) {
+      return !previousStats ? {
+        index,
+        timestamp: report.timestamp,
+        bytesReceived: report.bytesReceived,
+        packetsReceived: report.packetsReceived,
+        packetsLost: report.packetsLost,
+        jitter: report.jitter,
+        frameWidth: report.frameWidth,
+        frameHeight: report.frameHeight,
+        frameRate: report.frameRate
+      } : {
+        index,
+        timestamp: report.timestamp,
+        bytesReceived: report.bytesReceived - previousStats.bytesReceived,
+        packetsReceived: report.packetsReceived - previousStats.packetsReceived,
+        packetsLost: report.packetsLost - previousStats.packetsLost,
+        jitter: report.jitter,
+        frameWidth: report.frameWidth,
+        frameHeight: report.frameHeight,
+        frameRate: report.frameRate
+      };
+    } else
+      return {
+        index,
+        timestamp: report.timestamp,
+        bytesReceived: report.bytesReceived - stats[index - 1].bytesReceived,
+        packetsReceived: report.packetsReceived - stats[index - 1].packetsReceived,
+        packetsLost: report.packetsLost - stats[index - 1].packetsLost,
+        jitter: report.jitter,
+        frameWidth: report.frameWidth,
+        frameHeight: report.frameHeight,
+        frameRate: report.frameRate
+      };
+  });
+}
 let _debug = false;
 const log = (message, extra) => _debug && console.log(message, extra);
 const actualRTCPC = (window.RTCPeerConnection || window.webkitRTCPeerConnection || window.mozRTCPeerConnection).bind(window);
@@ -480,6 +542,9 @@ async function createStreamingManager(agent, {
     iceServers: ice_servers
   });
   const pcDataChannel = peerConnection.createDataChannel("JanusDataChannel");
+  const videoStats = [];
+  let videoStatsStartIndex = 0;
+  let videoStatsInterval;
   if (!session_id) {
     throw new Error("Could not create session_id");
   }
@@ -507,10 +572,28 @@ async function createStreamingManager(agent, {
     var _a, _b, _c;
     if (pcDataChannel.readyState === "open") {
       const [event, data] = message.data.split(":");
-      if (event === StreamEvents.StreamDone) {
-        (_a = callbacksObj.onVideoStateChange) == null ? void 0 : _a.call(callbacksObj, StreamingState.Stop);
-      } else if (event === StreamEvents.StreamStarted) {
-        (_b = callbacksObj.onVideoStateChange) == null ? void 0 : _b.call(callbacksObj, StreamingState.Start);
+      if (event === StreamEvents.StreamStarted) {
+        videoStatsStartIndex = videoStats.length;
+        videoStatsInterval = setInterval(() => {
+          const stats = peerConnection.getStats();
+          stats.then((result) => {
+            result.forEach((report) => {
+              if (report.type === "inbound-rtp" && report.kind === "video") {
+                videoStats.push(report);
+              }
+            });
+          });
+        }, 1e3);
+        (_a = callbacksObj.onVideoStateChange) == null ? void 0 : _a.call(callbacksObj, StreamingState.Start);
+      } else if (event === StreamEvents.StreamDone) {
+        clearInterval(videoStatsInterval);
+        const stats = videoStats.slice(videoStatsStartIndex);
+        if (stats) {
+          const previousStats = videoStatsStartIndex === 0 ? void 0 : videoStats[videoStatsStartIndex - 1];
+          const videoStatsReport = createVideoStatsReport(stats, previousStats);
+          videoStatsStartIndex = videoStats.length;
+          (_b = callbacksObj.onVideoStateChange) == null ? void 0 : _b.call(callbacksObj, StreamingState.Stop, videoStatsReport.sort((a, b) => b.packetsLost - a.packetsLost).slice(0, 5));
+        }
       } else {
         (_c = callbacksObj.onMessage) == null ? void 0 : _c.call(callbacksObj, event, decodeURIComponent(data));
       }
@@ -570,11 +653,13 @@ async function createStreamingManager(agent, {
   };
 }
 export {
+  ChatProgress,
   DocumentType,
   KnowledgeType,
   Providers,
   RateState,
   SocketManager,
+  StreamEvents,
   Subject,
   VoiceAccess,
   createAgentManager,

package/dist/index.umd.cjs

@@ -193,7 +193,12 @@
       host,
       callbacks: {
         onMessage: (event) => {
-          messageCallbacks.forEach((callback) => callback(event));
+          const data = JSON.parse(event.data);
+          let payload;
+          if (data.event === "answer") {
+            payload = data.content;
+          }
+          messageCallbacks.forEach((callback) => callback(data.event, payload));
         }
       }
     });
@@ -234,6 +239,14 @@
     RateState2["Negative"] = "Negative";
     return RateState2;
   })(RateState || {});
+  var ChatProgress = /* @__PURE__ */ ((ChatProgress2) => {
+    ChatProgress2["Embed"] = "embed";
+    ChatProgress2["Query"] = "query";
+    ChatProgress2["Partial"] = "chat/partial";
+    ChatProgress2["Answer"] = "answer";
+    ChatProgress2["Complete"] = "done";
+    return ChatProgress2;
+  })(ChatProgress || {});
   var Subject = /* @__PURE__ */ ((Subject2) => {
     Subject2["KnowledgeProcessing"] = "knowledge/processing";
     Subject2["KnowledgeIndexing"] = "knowledge/indexing";
@@ -301,6 +314,12 @@
               reject(new Error("Cannot create connection"));
             }
             (_b = (_a = options.callbacks).onConnectionStateChange) == null ? void 0 : _b.call(_a, state);
+          },
+          onMessage: (event, data) => {
+            var _a, _b;
+            if (event === StreamEvents.ChatPartial) {
+              (_b = (_a = options.callbacks).onChatEvents) == null ? void 0 : _b.call(_a, ChatProgress.Partial, data);
+            }
           }
         }
       });
@@ -311,6 +330,7 @@
     const abortController = new AbortController();
     const agentsApi = createAgentsApi(options.auth, baseURL);
     const ratingsAPI = createRatingsApi(options.auth, baseURL);
+    const knowledgeApi = createKnowledgeApi(options.auth, baseURL);
     const agent = await agentsApi.getById(agentId);
     const socketManager = await SocketManager(options.auth);
     let {
@@ -331,16 +351,14 @@
         return streamingManager.terminate();
       },
       chatId: chat.id,
-      async chat(messages) {
-        const result = await agentsApi.chat(agentId, chat.id, {
+      chat(messages) {
+        return agentsApi.chat(agentId, chat.id, {
           sessionId: streamingManager.sessionId,
           streamId: streamingManager.streamId,
           messages
         }, {
           signal: abortController.signal
         });
-        console.log(result);
-        return result;
       },
       rate(payload, id) {
         if (id) {
@@ -380,6 +398,12 @@
       },
       onVideoEvents(callback) {
         streamingManager.onCallback("onVideoStateChange", callback);
+      },
+      getStarterMessages() {
+        var _a, _b;
+        if (!((_a = agent.knowledge) == null ? void 0 : _a.id))
+          return Promise.resolve([]);
+        return knowledgeApi.getKnowledge((_b = agent.knowledge) == null ? void 0 : _b.id).then((knowledge) => (knowledge == null ? void 0 : knowledge.starter_message) || []);
       }
     };
   }
@@ -454,6 +478,44 @@
       }
     };
   }
+  function createVideoStatsReport(stats, previousStats) {
+    return stats.map((report, index) => {
+      if (index === 0) {
+        return !previousStats ? {
+          index,
+          timestamp: report.timestamp,
+          bytesReceived: report.bytesReceived,
+          packetsReceived: report.packetsReceived,
+          packetsLost: report.packetsLost,
+          jitter: report.jitter,
+          frameWidth: report.frameWidth,
+          frameHeight: report.frameHeight,
+          frameRate: report.frameRate
+        } : {
+          index,
+          timestamp: report.timestamp,
+          bytesReceived: report.bytesReceived - previousStats.bytesReceived,
+          packetsReceived: report.packetsReceived - previousStats.packetsReceived,
+          packetsLost: report.packetsLost - previousStats.packetsLost,
+          jitter: report.jitter,
+          frameWidth: report.frameWidth,
+          frameHeight: report.frameHeight,
+          frameRate: report.frameRate
+        };
+      } else
+        return {
+          index,
+          timestamp: report.timestamp,
+          bytesReceived: report.bytesReceived - stats[index - 1].bytesReceived,
+          packetsReceived: report.packetsReceived - stats[index - 1].packetsReceived,
+          packetsLost: report.packetsLost - stats[index - 1].packetsLost,
+          jitter: report.jitter,
+          frameWidth: report.frameWidth,
+          frameHeight: report.frameHeight,
+          frameRate: report.frameRate
+        };
+    });
+  }
   let _debug = false;
   const log = (message, extra) => _debug && console.log(message, extra);
   const actualRTCPC = (window.RTCPeerConnection || window.webkitRTCPeerConnection || window.mozRTCPeerConnection).bind(window);
@@ -484,6 +546,9 @@
       iceServers: ice_servers
     });
     const pcDataChannel = peerConnection.createDataChannel("JanusDataChannel");
+    const videoStats = [];
+    let videoStatsStartIndex = 0;
+    let videoStatsInterval;
     if (!session_id) {
       throw new Error("Could not create session_id");
     }
@@ -511,10 +576,28 @@
       var _a, _b, _c;
       if (pcDataChannel.readyState === "open") {
         const [event, data] = message.data.split(":");
-        if (event === StreamEvents.StreamDone) {
-          (_a = callbacksObj.onVideoStateChange) == null ? void 0 : _a.call(callbacksObj, StreamingState.Stop);
-        } else if (event === StreamEvents.StreamStarted) {
-          (_b = callbacksObj.onVideoStateChange) == null ? void 0 : _b.call(callbacksObj, StreamingState.Start);
+        if (event === StreamEvents.StreamStarted) {
+          videoStatsStartIndex = videoStats.length;
+          videoStatsInterval = setInterval(() => {
+            const stats = peerConnection.getStats();
+            stats.then((result) => {
+              result.forEach((report) => {
+                if (report.type === "inbound-rtp" && report.kind === "video") {
+                  videoStats.push(report);
+                }
+              });
+            });
+          }, 1e3);
+          (_a = callbacksObj.onVideoStateChange) == null ? void 0 : _a.call(callbacksObj, StreamingState.Start);
+        } else if (event === StreamEvents.StreamDone) {
+          clearInterval(videoStatsInterval);
+          const stats = videoStats.slice(videoStatsStartIndex);
+          if (stats) {
+            const previousStats = videoStatsStartIndex === 0 ? void 0 : videoStats[videoStatsStartIndex - 1];
+            const videoStatsReport = createVideoStatsReport(stats, previousStats);
+            videoStatsStartIndex = videoStats.length;
+            (_b = callbacksObj.onVideoStateChange) == null ? void 0 : _b.call(callbacksObj, StreamingState.Stop, videoStatsReport.sort((a, b) => b.packetsLost - a.packetsLost).slice(0, 5));
+          }
         } else {
           (_c = callbacksObj.onMessage) == null ? void 0 : _c.call(callbacksObj, event, decodeURIComponent(data));
         }
@@ -573,11 +656,13 @@
       }
     };
   }
+  exports2.ChatProgress = ChatProgress;
   exports2.DocumentType = DocumentType;
   exports2.KnowledgeType = KnowledgeType;
   exports2.Providers = Providers;
   exports2.RateState = RateState;
   exports2.SocketManager = SocketManager;
+  exports2.StreamEvents = StreamEvents;
   exports2.Subject = Subject;
   exports2.VoiceAccess = VoiceAccess;
   exports2.createAgentManager = createAgentManager;

package/dist/src/lib/createAgentManager.d.ts

@@ -1,15 +1,15 @@
-import { Agent, AgentManagerOptions, AgentsManager, CreateStreamOptions } from '../types/index';
+import { Agent, AgentManager, AgentManagerOptions, CreateStreamOptions } from '../types/index';
 export declare function getAgentStreamArgs(agent: Agent): CreateStreamOptions;
 /**
  * Creates a new Agent Manager instance for interacting with an agent, chat, and related connections.
  *
  * @param {string} agentId - The ID of the agent to chat with.
  * @param {AgentManagerOptions} options - Configurations for the Agent Manager API.
- * * @returns {Promise<AgentsManager>} - A promise that resolves to an instance of the AgentsAPI interface.
+ * * @returns {Promise<AgentManager>} - A promise that resolves to an instance of the AgentsAPI interface.
  *
  * @throws {Error} Throws an error if the agent is not initialized.
  *
  * @example
  * const agentManager = await createAgentManager('id-agent123', { auth: { type: 'key', clientKey: '123', externalId: '123' } });
  */
-export declare function createAgentManager(agentId: string, options: AgentManagerOptions): Promise<AgentsManager>;
+export declare function createAgentManager(agentId: string, options: AgentManagerOptions): Promise<AgentManager>;

package/dist/src/lib/utils/webrtc.d.ts

@@ -0,0 +1,2 @@
+import { SlimRTCStatsReport } from "../../types";
+export declare function createVideoStatsReport(stats: SlimRTCStatsReport[], previousStats?: SlimRTCStatsReport): SlimRTCStatsReport[];

package/dist/src/types/entities/agents/manager.d.ts

@@ -1,32 +1,36 @@
 import { SupportedStreamScipt } from '../../../types/StreamScript';
 import { Auth } from '../../auth';
-import { SendStreamPayloadResponse, StreamingState } from '../../stream';
+import { SendStreamPayloadResponse, SlimRTCStatsReport, StreamingState } from '../../stream';
 import { Agent } from './agent';
 import { ChatResponse, Message, RatingEntity, RatingPayload } from './chat';
 /**
  * Types of events provided in Chat Progress Callback
  */
-declare enum ChatProgress {
+export declare enum ChatProgress {
     /**
      * Chat was successfully embedded
      */
-    Embed = 0,
+    Embed = "embed",
     /**
      * Server processing chat message
      */
-    Query = 1,
+    Query = "query",
+    /**
+     * Server returns a part of the message
+     */
+    Partial = "chat/partial",
     /**
      * Server processed message and returned response
      */
-    Answer = 2,
+    Answer = "answer",
     /**
      * Chat was closed
      */
-    Complete = 3
+    Complete = "done"
 }
-export type ChatProgressCallback = (progress: ChatProgress) => void;
+export type ChatProgressCallback = (progress: ChatProgress, data: string) => void;
 export type ConnectionStateChangeCallback = (state: RTCIceConnectionState) => void;
-export type VideoStateChangeCallback = (state: StreamingState) => void;
+export type VideoStateChangeCallback = (state: StreamingState, stats?: SlimRTCStatsReport[]) => void;
 interface ManagerCallbacks {
     /**
      * This callback will be triggered each time the RTC connection changes state
@@ -51,7 +55,7 @@ interface ManagerCallbacks {
      * Optional callback function that will be triggered each time any changes happen in the chat
      * @param progress
      */
-    onChatEvents?(progress: ChatProgress): void;
+    onChatEvents?(progress: ChatProgress, data: any): void;
 }
 export interface AgentManagerOptions {
     callbacks: ManagerCallbacks;
@@ -59,7 +63,7 @@ export interface AgentManagerOptions {
     debug?: boolean;
     auth: Auth;
 }
-export interface AgentsManager {
+export interface AgentManager {
     /**
      * Agent instance you are working with.
      * To know more about agents go to https://docs.d-id.com/reference/agents
@@ -99,7 +103,7 @@ export interface AgentsManager {
      * Optional callback function that will be triggered each time any changes happen in the chat
      * @param callback
      */
-    onChatEvents: (callback: ChatProgressCallback) => void;
+    onChatEvents: (callback: (ChatProgressCallback)) => void;
     /**
      * Optional callback function that will be triggered each time the RTC connection gets new status
      * @param callback
@@ -110,5 +114,11 @@ export interface AgentsManager {
      * @param callback
      */
     onVideoEvents: (callback: VideoStateChangeCallback) => void;
+    /**
+     * Get Starter messages from agent knowledge
+     * Used as a proposal for a user to ask Agent these questions
+     * @param agent
+     */
+    getStarterMessages: () => Promise<string[]>;
 }
 export {};

package/dist/src/types/stream/stream.d.ts

@@ -16,7 +16,7 @@ export declare enum StreamEvents {
 export interface ManagerCallbacks {
     onMessage?: (event: string, data: string) => void;
     onConnectionStateChange?: (state: RTCIceConnectionState) => void;
-    onVideoStateChange?: (state: StreamingState) => void;
+    onVideoStateChange?: (state: StreamingState, stats?: SlimRTCStatsReport[]) => void;
     onSrcObjectReady?: (value: MediaStream) => void;
 }
 export type ManagerCallbackKeys = keyof ManagerCallbacks;
@@ -41,3 +41,14 @@ export interface StreamingManagerOptions {
     debug?: boolean;
     auth: Auth;
 }
+export interface SlimRTCStatsReport {
+    index: number;
+    timestamp: any;
+    bytesReceived: any;
+    packetsReceived: any;
+    packetsLost: any;
+    jitter: any;
+    frameWidth: any;
+    frameHeight: any;
+    frameRate: any;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@d-id/client-sdk",
     "private": false,
-    "version": "1.0.18-beta.10",
+    "version": "1.0.18-beta.12",
     "type": "module",
     "description": "d-id client sdk",
     "repository": {