@d-id/client-sdk 1.1.22-staging.103 → 1.1.22-staging.104

package/dist/livekit-manager-Cv9nty3S.js

@@ -0,0 +1,256 @@
+import { c as oe, a as ae, S as se, d as ce, T as de, C as S, b as E, A as _, e as U, f as R, g as j } from "./index-sABN87aB.js";
+function ue(m, T, h, i) {
+  const y = oe(m, `${T}/v2/agents/${h}`, i);
+  return {
+    async createStream(s) {
+      return y.post("/sessions", s);
+    }
+  };
+}
+async function F() {
+  try {
+    return await import("./livekit-client.esm-C1GRQdZY.js");
+  } catch {
+    throw new Error(
+      "LiveKit client is required for this streaming manager. Please install it using: npm install livekit-client"
+    );
+  }
+}
+const he = {
+  excellent: E.Strong,
+  good: E.Strong,
+  poor: E.Weak,
+  lost: E.Unknown,
+  unknown: E.Unknown
+}, K = JSON.stringify({
+  kind: "InternalServerError",
+  description: "Stream Error"
+});
+function V(m, T, h, i) {
+  var y, s;
+  throw T("Failed to connect to LiveKit room:", m), i(), (y = h.onConnectionStateChange) == null || y.call(h, S.Fail), (s = h.onError) == null || s.call(h, m, { sessionId: "" }), m;
+}
+async function fe(m, T, h) {
+  var N, x;
+  const i = ae(h.debug || !1, "LiveKitStreamingManager"), { Room: y, RoomEvent: s, ConnectionState: l } = await F(), { callbacks: e, auth: O, baseURL: J, analytics: Q, microphoneStream: ge } = h;
+  let c = null, w = !1;
+  const $ = se.Fluent;
+  let A = !0, C = null, g = null;
+  c = new y({
+    adaptiveStream: !1,
+    // Must be false to use mediaStreamTrack directly
+    dynacast: !0
+  });
+  const q = ue(O, J || ce, m, e.onError);
+  let p, k, P;
+  try {
+    const n = await q.createStream({
+      transport_provider: de.Livekit,
+      chat_persist: T.chat_persist ?? !0
+    }), { id: t, session_token: r, session_url: o } = n;
+    (N = e.onStreamCreated) == null || N.call(e, { session_id: t, stream_id: t, agent_id: m }), p = t, k = r, P = o, await c.prepareConnection(P, k);
+  } catch (n) {
+    V(n, i, e, () => {
+      A = !1;
+    });
+  }
+  if (!P || !k || !p)
+    return Promise.reject(new Error("Failed to initialize LiveKit stream"));
+  c.on(s.ConnectionStateChanged, b).on(s.ConnectionQualityChanged, z).on(s.ActiveSpeakersChanged, W).on(s.ParticipantConnected, B).on(s.TrackSubscribed, G).on(s.TrackUnsubscribed, H).on(s.DataReceived, X).on(s.MediaDevicesError, Y).on(s.EncryptionError, Z).on(s.TrackSubscriptionFailed, ee), (x = e.onConnectionStateChange) == null || x.call(e, S.New);
+  try {
+    await c.connect(P, k), i("LiveKit room joined successfully"), A = !1;
+  } catch (n) {
+    V(n, i, e, () => {
+      A = !1;
+    });
+  }
+  Q.enrich({
+    "stream-type": $
+  });
+  function b(n) {
+    var t, r, o, a, u;
+    switch (i("Connection state changed:", n), n) {
+      case l.Connecting:
+        (t = e.onConnectionStateChange) == null || t.call(e, S.Connecting);
+        break;
+      case l.Connected:
+        i("LiveKit room connected successfully"), w = !0, A ? queueMicrotask(() => {
+          var d;
+          (d = e.onConnectionStateChange) == null || d.call(e, S.Connected);
+        }) : (r = e.onConnectionStateChange) == null || r.call(e, S.Connected);
+        break;
+      case l.Disconnected:
+        i("LiveKit room disconnected"), w = !1, (o = e.onConnectionStateChange) == null || o.call(e, S.Disconnected);
+        break;
+      case l.Reconnecting:
+        i("LiveKit room reconnecting..."), (a = e.onConnectionStateChange) == null || a.call(e, S.Connecting);
+        break;
+      case l.SignalReconnecting:
+        i("LiveKit room signal reconnecting..."), (u = e.onConnectionStateChange) == null || u.call(e, S.Connecting);
+        break;
+    }
+  }
+  function z(n, t) {
+    var r;
+    i("Connection quality:", n), t != null && t.isLocal && ((r = e.onConnectivityStateChange) == null || r.call(e, he[n]));
+  }
+  function W(n) {
+    var r, o;
+    i("Active speakers changed:", n), n[0] ? (r = e.onAgentActivityStateChange) == null || r.call(e, _.Talking) : (o = e.onAgentActivityStateChange) == null || o.call(e, _.Idle);
+  }
+  function B(n) {
+    i("Participant connected:", n.identity);
+  }
+  function G(n, t, r) {
+    var a, u, d;
+    i(`Track subscribed: ${n.kind} from ${r.identity}`);
+    const o = n.mediaStreamTrack;
+    if (!o) {
+      i(`No mediaStreamTrack available for ${n.kind}`);
+      return;
+    }
+    C ? (C.addTrack(o), i(`Added ${n.kind} track to shared MediaStream`)) : (C = new MediaStream([o]), i(`Created shared MediaStream with ${n.kind} track`)), n.kind === "video" && ((a = e.onStreamReady) == null || a.call(e), (u = e.onSrcObjectReady) == null || u.call(e, C), (d = e.onVideoStateChange) == null || d.call(e, U.Start));
+  }
+  function H(n, t, r) {
+    var o;
+    i(`Track unsubscribed: ${n.kind} from ${r.identity}`), n.kind === "video" && ((o = e.onVideoStateChange) == null || o.call(e, U.Stop));
+  }
+  function X(n, t, r, o) {
+    var u, d, M;
+    const a = new TextDecoder().decode(n);
+    i("Data received:", a);
+    try {
+      const f = JSON.parse(a), L = o || f.subject;
+      if (L === R.ChatAnswer) {
+        const v = j.Answer;
+        (u = e.onMessage) == null || u.call(e, v, {
+          event: v,
+          ...f
+        });
+      } else if ([R.StreamVideoCreated, R.StreamVideoDone].includes(L)) {
+        const v = (f == null ? void 0 : f.role) || (t == null ? void 0 : t.identity) || "datachannel";
+        (d = e.onMessage) == null || d.call(e, L, {
+          [v]: f
+        });
+      } else if (L === R.ChatAudioTranscribed) {
+        const v = j.Transcribe;
+        (M = e.onMessage) == null || M.call(e, v, {
+          event: v,
+          ...f
+        });
+      }
+    } catch (f) {
+      i("Failed to parse data channel message:", f);
+    }
+  }
+  function Y(n) {
+    var t;
+    i("Media devices error:", n), (t = e.onError) == null || t.call(e, new Error(K), { sessionId: p });
+  }
+  function Z(n) {
+    var t;
+    i("Encryption error:", n), (t = e.onError) == null || t.call(e, new Error(K), { sessionId: p });
+  }
+  function ee(n, t, r) {
+    i("Track subscription failed:", { trackSid: n, participant: t, reason: r });
+  }
+  async function ne(n) {
+    if (!c) return null;
+    const { Track: t } = await F(), r = c.localParticipant.audioTrackPublications;
+    if (r) {
+      for (const [o, a] of r)
+        if (a.source === t.Source.Microphone && a.track) {
+          const d = a.track.mediaStreamTrack;
+          if (d === n || d && d.id === n.id)
+            return a;
+        }
+    }
+    return null;
+  }
+  function te(n) {
+    if (!g || !g.track)
+      return !1;
+    const t = g.track.mediaStreamTrack;
+    return t !== n && (t == null ? void 0 : t.id) !== n.id;
+  }
+  async function ie(n) {
+    var u, d;
+    if (!w || !c)
+      throw i("Room is not connected, cannot publish microphone stream"), new Error("Room is not connected");
+    const t = n.getAudioTracks();
+    if (t.length === 0) {
+      i("No audio track found in the provided MediaStream");
+      return;
+    }
+    const r = t[0], { Track: o } = await F(), a = await ne(r);
+    if (a) {
+      i("Microphone track is already published, skipping", {
+        trackId: r.id,
+        publishedTrackId: (d = (u = a.track) == null ? void 0 : u.mediaStreamTrack) == null ? void 0 : d.id
+      }), g = a;
+      return;
+    }
+    te(r) && (i("Unpublishing existing microphone track before publishing new one"), await D()), i("Publishing microphone track from provided MediaStream", { trackId: r.id });
+    try {
+      g = await c.localParticipant.publishTrack(r, {
+        source: o.Source.Microphone
+      }), i("Microphone track published successfully", { trackSid: g.trackSid });
+    } catch (M) {
+      throw i("Failed to publish microphone track:", M), M;
+    }
+  }
+  async function D() {
+    if (!(!g || !g.track))
+      try {
+        c && (await c.localParticipant.unpublishTrack(g.track), i("Microphone track unpublished"));
+      } catch (n) {
+        i("Error unpublishing microphone track:", n);
+      } finally {
+        g = null;
+      }
+  }
+  function re() {
+    C && (C.getTracks().forEach((n) => n.stop()), C = null);
+  }
+  async function I(n, t = "lk.chat") {
+    var r, o;
+    if (!w || !c) {
+      i("Room is not connected for sending messages"), (r = e.onError) == null || r.call(e, new Error(K), {
+        sessionId: p
+      });
+      return;
+    }
+    try {
+      await c.localParticipant.sendText(n, { topic: t }), i("Message sent successfully:", n);
+    } catch (a) {
+      i("Failed to send message:", a), (o = e.onError) == null || o.call(e, new Error(K), { sessionId: p });
+    }
+  }
+  return {
+    speak(n) {
+      const t = typeof n == "string" ? n : JSON.stringify(n);
+      return I(
+        t,
+        "did.speak"
+        /* Speak */
+      );
+    },
+    async disconnect() {
+      var n, t;
+      c && (await D(), await c.disconnect(), c = null), re(), w = !1, (n = e.onConnectionStateChange) == null || n.call(e, S.Completed), (t = e.onAgentActivityStateChange) == null || t.call(e, _.Idle);
+    },
+    sendDataChannelMessage: I,
+    sendTextMessage: I,
+    publishMicrophoneStream: ie,
+    unpublishMicrophoneStream: D,
+    sessionId: p,
+    streamId: p,
+    streamType: $,
+    interruptAvailable: !0,
+    triggersAvailable: !1
+  };
+}
+export {
+  fe as createLiveKitStreamingManager,
+  V as handleInitError
+};

package/dist/src/services/streaming-manager/common.d.ts

@@ -26,6 +26,19 @@ export type StreamingManager<T extends CreateStreamOptions | CreateSessionV2Opti
      * supported only for livekit manager
      */
     sendTextMessage?(payload: string): Promise<void>;
+    /**
+     * Publish a microphone stream to the DataChannel
+     * Can be called after connection to add microphone input
+     * @param stream The MediaStream containing the microphone audio track
+     * supported only for livekit manager
+     */
+    publishMicrophoneStream?(stream: MediaStream): Promise<void>;
+    /**
+     * Unpublish the currently published microphone stream
+     * Can be called after connection to remove microphone input
+     * supported only for livekit manager
+     */
+    unpublishMicrophoneStream?(): Promise<void>;
     /**
      * Session identifier information, should be returned in the body of all streaming requests
      */

package/dist/src/services/streaming-manager/livekit-manager.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/types/entities/agents/chat.d.ts

@@ -27,6 +27,7 @@ export interface Message {
     context?: string;
     videoId?: string;
     interrupted?: boolean;
+    transcribed?: boolean;
 }
 export interface ChatPayload {
     messages: Message[];

package/dist/src/types/entities/agents/manager.d.ts

@@ -26,6 +26,10 @@ export declare enum ChatProgress {
      * Server processed message and returned response
      */
     Answer = "answer",
+    /**
+     * Audio-transcribed user message
+     */
+    Transcribe = "transcribe",
     /**
      * Chat was closed
      */
@@ -138,6 +142,7 @@ export interface AgentManagerOptions {
     streamOptions?: StreamOptions;
     initialMessages?: Message[];
     persistentChat?: boolean;
+    microphoneStream?: MediaStream;
 }
 export interface AgentManager {
     /**
@@ -178,6 +183,19 @@ export interface AgentManager {
      * Method to close all connections with agent, stream and web socket
      */
     disconnect: () => Promise<void>;
+    /**
+     * Publish a microphone stream to the data channel
+     * Can be called after connection to add microphone input
+     * @param stream The MediaStream containing the microphone audio track
+     * supported only for livekit manager
+     */
+    publishMicrophoneStream?: (stream: MediaStream) => Promise<void>;
+    /**
+     * Unpublish the currently published microphone stream
+     * Can be called after connection to remove microphone input
+     * supported only for livekit manager
+     */
+    unpublishMicrophoneStream?: () => Promise<void>;
     /**
      * Method to send a chat message to existing chat with the agent
      * @param messages

package/dist/src/types/stream/stream.d.ts

@@ -22,6 +22,7 @@ export declare enum AgentActivityState {
 export declare enum StreamEvents {
     ChatAnswer = "chat/answer",
     ChatPartial = "chat/partial",
+    ChatAudioTranscribed = "chat/audio-transcribed",
     StreamDone = "stream/done",
     StreamStarted = "stream/started",
     StreamFailed = "stream/error",
@@ -89,6 +90,12 @@ export interface StreamingManagerOptions {
     debug?: boolean;
     auth: Auth;
     analytics: Analytics;
+    /**
+     * Optional MediaStream to use for microphone input.
+     * If provided, the audio track from this stream will be published to the data channel.
+     * Supported by LiveKit streaming managers.
+     */
+    microphoneStream?: MediaStream;
 }
 export interface SlimRTCStatsReport {
     index: number;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@d-id/client-sdk",
   "private": false,
-  "version": "1.1.22-staging.103",
+  "version": "1.1.22-staging.104",
   "type": "module",
   "description": "d-id client sdk",
   "repository": {

package/dist/livekit-manager-BKZAOT9G.js

@@ -1,193 +0,0 @@
-import { c as l, a as k, S as b, d as ee, T as ne, C as u, b as w, A as _, e as N, f as $, g as te } from "./index-Cx4MD7mS.js";
-function oe(c, y, s, o) {
-  const v = l(c, `${y}/v2/agents/${s}`, o);
-  return {
-    async createStream(a) {
-      return v.post("/sessions", a);
-    }
-  };
-}
-async function ie() {
-  try {
-    return await import("./livekit-client.esm-C1GRQdZY.js");
-  } catch {
-    throw new Error(
-      "LiveKit client is required for this streaming manager. Please install it using: npm install livekit-client"
-    );
-  }
-}
-const re = {
-  excellent: w.Strong,
-  good: w.Strong,
-  poor: w.Weak,
-  lost: w.Unknown,
-  unknown: w.Unknown
-}, K = JSON.stringify({
-  kind: "InternalServerError",
-  description: "Stream Error"
-});
-function U(c, y, s, o) {
-  var v, a;
-  throw y("Failed to connect to LiveKit room:", c), o(), (v = s.onConnectionStateChange) == null || v.call(s, u.Fail), (a = s.onError) == null || a.call(s, c, { sessionId: "" }), c;
-}
-async function se(c, y, s) {
-  var P, I;
-  const o = k(s.debug || !1, "LiveKitStreamingManager"), { Room: v, RoomEvent: a, ConnectionState: p } = await ie(), { callbacks: e, auth: j, baseURL: x, analytics: V } = s;
-  let g = null, E = !1;
-  const F = b.Fluent;
-  let T = !0, f = null;
-  g = new v({
-    adaptiveStream: !1,
-    // Must be false to use mediaStreamTrack directly
-    dynacast: !0
-  });
-  const O = oe(j, x || ee, c, e.onError);
-  let S, A, M;
-  try {
-    const n = await O.createStream({
-      transport_provider: ne.Livekit,
-      chat_persist: y.chat_persist ?? !0
-    }), { id: t, session_token: i, session_url: r } = n;
-    (P = e.onStreamCreated) == null || P.call(e, { session_id: t, stream_id: t, agent_id: c }), S = t, A = i, M = r, await g.prepareConnection(M, A);
-  } catch (n) {
-    U(n, o, e, () => {
-      T = !1;
-    });
-  }
-  if (!M || !A || !S)
-    return Promise.reject(new Error("Failed to initialize LiveKit stream"));
-  g.on(a.ConnectionStateChanged, J).on(a.ConnectionQualityChanged, Q).on(a.ActiveSpeakersChanged, q).on(a.ParticipantConnected, z).on(a.TrackSubscribed, W).on(a.TrackUnsubscribed, B).on(a.DataReceived, G).on(a.MediaDevicesError, H).on(a.EncryptionError, X).on(a.TrackSubscriptionFailed, Y), (I = e.onConnectionStateChange) == null || I.call(e, u.New);
-  try {
-    await g.connect(M, A), o("LiveKit room joined successfully"), T = !1;
-  } catch (n) {
-    U(n, o, e, () => {
-      T = !1;
-    });
-  }
-  V.enrich({
-    "stream-type": F
-  });
-  function J(n) {
-    var t, i, r, d, C;
-    switch (o("Connection state changed:", n), n) {
-      case p.Connecting:
-        (t = e.onConnectionStateChange) == null || t.call(e, u.Connecting);
-        break;
-      case p.Connected:
-        o("LiveKit room connected successfully"), E = !0, T ? queueMicrotask(() => {
-          var m;
-          (m = e.onConnectionStateChange) == null || m.call(e, u.Connected);
-        }) : (i = e.onConnectionStateChange) == null || i.call(e, u.Connected);
-        break;
-      case p.Disconnected:
-        o("LiveKit room disconnected"), E = !1, (r = e.onConnectionStateChange) == null || r.call(e, u.Disconnected);
-        break;
-      case p.Reconnecting:
-        o("LiveKit room reconnecting..."), (d = e.onConnectionStateChange) == null || d.call(e, u.Connecting);
-        break;
-      case p.SignalReconnecting:
-        o("LiveKit room signal reconnecting..."), (C = e.onConnectionStateChange) == null || C.call(e, u.Connecting);
-        break;
-    }
-  }
-  function Q(n, t) {
-    var i;
-    o("Connection quality:", n), t != null && t.isLocal && ((i = e.onConnectivityStateChange) == null || i.call(e, re[n]));
-  }
-  function q(n) {
-    var i, r;
-    o("Active speakers changed:", n), n[0] ? (i = e.onAgentActivityStateChange) == null || i.call(e, _.Talking) : (r = e.onAgentActivityStateChange) == null || r.call(e, _.Idle);
-  }
-  function z(n) {
-    o("Participant connected:", n.identity);
-  }
-  function W(n, t, i) {
-    var d, C, m;
-    o(`Track subscribed: ${n.kind} from ${i.identity}`);
-    const r = n.mediaStreamTrack;
-    if (!r) {
-      o(`No mediaStreamTrack available for ${n.kind}`);
-      return;
-    }
-    f ? (f.addTrack(r), o(`Added ${n.kind} track to shared MediaStream`)) : (f = new MediaStream([r]), o(`Created shared MediaStream with ${n.kind} track`)), n.kind === "video" && ((d = e.onStreamReady) == null || d.call(e), (C = e.onSrcObjectReady) == null || C.call(e, f), (m = e.onVideoStateChange) == null || m.call(e, N.Start));
-  }
-  function B(n, t, i) {
-    var r;
-    o(`Track unsubscribed: ${n.kind} from ${i.identity}`), n.kind === "video" && ((r = e.onVideoStateChange) == null || r.call(e, N.Stop));
-  }
-  function G(n, t, i, r) {
-    var C, m;
-    const d = new TextDecoder().decode(n);
-    o("Data received:", d);
-    try {
-      const h = JSON.parse(d), D = r || h.subject;
-      if (D === $.ChatAnswer) {
-        const L = te.Answer;
-        (C = e.onMessage) == null || C.call(e, L, {
-          event: L,
-          ...h
-        });
-      } else if ([$.StreamVideoCreated, $.StreamVideoDone].includes(D)) {
-        const L = (h == null ? void 0 : h.role) || (t == null ? void 0 : t.identity) || "datachannel";
-        (m = e.onMessage) == null || m.call(e, D, {
-          [L]: h
-        });
-      }
-    } catch (h) {
-      o("Failed to parse data channel message:", h);
-    }
-  }
-  function H(n) {
-    var t;
-    o("Media devices error:", n), (t = e.onError) == null || t.call(e, new Error(K), { sessionId: S });
-  }
-  function X(n) {
-    var t;
-    o("Encryption error:", n), (t = e.onError) == null || t.call(e, new Error(K), { sessionId: S });
-  }
-  function Y(n, t, i) {
-    o("Track subscription failed:", { trackSid: n, participant: t, reason: i });
-  }
-  function Z() {
-    f && (f.getTracks().forEach((n) => n.stop()), f = null);
-  }
-  async function R(n, t = "lk.chat") {
-    var i, r;
-    if (!E || !g) {
-      o("Room is not connected for sending messages"), (i = e.onError) == null || i.call(e, new Error(K), {
-        sessionId: S
-      });
-      return;
-    }
-    try {
-      await g.localParticipant.sendText(n, { topic: t }), o("Message sent successfully:", n);
-    } catch (d) {
-      o("Failed to send message:", d), (r = e.onError) == null || r.call(e, new Error(K), { sessionId: S });
-    }
-  }
-  return {
-    speak(n) {
-      const t = typeof n == "string" ? n : JSON.stringify(n);
-      return R(
-        t,
-        "did.speak"
-        /* Speak */
-      );
-    },
-    async disconnect() {
-      var n, t;
-      g && (await g.disconnect(), g = null), Z(), E = !1, (n = e.onConnectionStateChange) == null || n.call(e, u.Completed), (t = e.onAgentActivityStateChange) == null || t.call(e, _.Idle);
-    },
-    sendDataChannelMessage: R,
-    sendTextMessage: R,
-    sessionId: S,
-    streamId: S,
-    streamType: F,
-    interruptAvailable: !0,
-    triggersAvailable: !1
-  };
-}
-export {
-  se as createLiveKitStreamingManager,
-  U as handleInitError
-};