@defcomm/meetingsdk 0.5.5 → 0.5.7

package/dist/index.js

@@ -24,10 +24,12 @@ __export(index_exports, {
   MeetingProvider: () => MeetingProvider,
   MeetingState: () => MeetingState,
   VideoSDKCore: () => VideoSDKCore,
+  useLiveTranscript: () => useLiveTranscript,
   useLocalParticipant: () => useLocalParticipant,
   useMeeting: () => useMeeting,
   useMeetingContext: () => useMeetingContext,
   useMeetingPreview: () => useMeetingPreview,
+  useNetworkQuality: () => useNetworkQuality,
   useParticipants: () => useParticipants,
   usePubSub: () => usePubSub,
   useRemoteMedia: () => useRemoteMedia
@@ -227,6 +229,8 @@ var MeetingState = class {
     this.presenterId = null;
     this.activeSpeakerIds = /* @__PURE__ */ new Set();
     this.recording = null;
+    this.liveTranscripts = [];
+    this.networkQuality = null;
     this.listeners = /* @__PURE__ */ new Map();
   }
   // ---- reactive system ----
@@ -264,6 +268,21 @@ var MeetingState = class {
     this.recording = null;
     this.notify("recording");
   }
+  addLiveTranscript(transcript) {
+    this.liveTranscripts = [...this.liveTranscripts.slice(-499), transcript];
+    this.notify("transcript");
+  }
+  getLiveTranscripts() {
+    return this.liveTranscripts;
+  }
+  clearLiveTranscripts() {
+    this.liveTranscripts = [];
+    this.notify("transcript");
+  }
+  setNetworkQuality(quality) {
+    this.networkQuality = quality;
+    this.notify("networkQuality");
+  }
   // ---- participants ----
   addParticipant(p) {
     if (this.participants.has(p.id)) return false;
@@ -384,10 +403,14 @@ var MeetingState = class {
     this.chatMessages.clear();
     this.presenterId = null;
     this.activeSpeakerIds = /* @__PURE__ */ new Set();
+    this.liveTranscripts = [];
+    this.networkQuality = null;
     this.notify("participants");
     this.notify("chat");
     this.notify("presenter");
     this.notify("activeSpeakers");
+    this.notify("transcript");
+    this.notify("networkQuality");
   }
 };
 
@@ -449,6 +472,12 @@ var SFUClient = class {
     this.trackToSource = /* @__PURE__ */ new Map();
     this.localTrackIds = /* @__PURE__ */ new Map();
     this.subscriberStatsTimer = null;
+    this.liveTranscriptionActive = false;
+    this.liveTranscriptionSequence = 0;
+    this.liveTranscriptionStartedAt = 0;
+    this.liveTranscriptionRecorder = null;
+    this.liveTranscriptionLoopPromise = null;
+    this.networkProfile = null;
     this.heartbeatTimer = null;
     this.apiBase = options.apiBase || "https://api.example.com/api";
     this.wsBase = options.wsBase || "wss://api.example.com/ws";
@@ -505,6 +534,10 @@ var SFUClient = class {
     });
     this.onRoomUpdate = options.onRoomUpdate || (() => {
     });
+    this.onLiveTranscript = options.onLiveTranscript || (() => {
+    });
+    this.onNetworkQuality = options.onNetworkQuality || (() => {
+    });
     this.onParticipantMediaState = options.onParticipantMediaState || (() => {
     });
     this.autoSubscribe = options.autoSubscribe !== false;
@@ -1099,6 +1132,40 @@ var SFUClient = class {
         if (reaction?.id && reaction?.emoji) this.onReaction(reaction);
         break;
       }
+      case "live_transcript": {
+        const transcript = {
+          transcript_id: String(payload.transcript_id),
+          participant_id: String(payload.participant_id),
+          speaker_name: payload.speaker_name ?? null,
+          sequence: Number(payload.sequence ?? 0),
+          text: String(payload.text ?? ""),
+          offset_ms: payload.offset_ms == null ? null : Number(payload.offset_ms),
+          start_ms: payload.start_ms == null ? null : Number(payload.start_ms),
+          end_ms: payload.end_ms == null ? null : Number(payload.end_ms),
+          timestamp: String(payload.timestamp ?? (/* @__PURE__ */ new Date()).toISOString())
+        };
+        if (transcript.text.trim()) this.onLiveTranscript(transcript);
+        break;
+      }
+      case "network_quality": {
+        const raw = payload.quality || payload;
+        const quality = {
+          participant_id: String(raw.participant_id ?? this.currentParticipantId ?? this.participantId),
+          quality: String(raw.quality ?? "unknown"),
+          score: Number(raw.score ?? 0),
+          recommended_action: String(raw.recommended_action ?? "maintain"),
+          packet_loss_percent: Number(raw.packet_loss_percent ?? 0),
+          upload_kbps: Number(raw.upload_kbps ?? 0),
+          forwarded_drop_percent: Number(raw.forwarded_drop_percent ?? 0),
+          queued_packets: Number(raw.queued_packets ?? 0),
+          packets_received: Number(raw.packets_received ?? 0),
+          packets_lost: Number(raw.packets_lost ?? 0),
+          observed_at: String(raw.observed_at ?? (/* @__PURE__ */ new Date()).toISOString())
+        };
+        this.onNetworkQuality(quality);
+        void this.applyNetworkRecommendation(quality.recommended_action);
+        break;
+      }
       case "error": {
         const message2 = payload.message || "SFU signaling error";
         if (this.state === "reconnecting" && message2.includes("participant session is stale")) {
@@ -1758,6 +1825,208 @@ var SFUClient = class {
       }
     });
   }
+  getLiveTranscriptionMimeType() {
+    if (typeof MediaRecorder === "undefined") {
+      throw new Error("MediaRecorder is not supported by this browser");
+    }
+    const candidates = [
+      "audio/webm;codecs=opus",
+      "audio/webm",
+      "audio/ogg;codecs=opus"
+    ];
+    return candidates.find(
+      (type) => MediaRecorder.isTypeSupported(type)
+    ) || "";
+  }
+  async startLiveTranscription() {
+    if (this.liveTranscriptionActive) {
+      return;
+    }
+    if (!this.room) {
+      throw new Error("Cannot start live transcription without a room");
+    }
+    if (!this.authToken) {
+      throw new Error(
+        "Cannot start live transcription without an authenticated token"
+      );
+    }
+    const audioTrack = this.localStream?.getAudioTracks()[0];
+    if (!audioTrack || audioTrack.readyState !== "live") {
+      throw new Error(
+        "Cannot start live transcription without a live microphone track"
+      );
+    }
+    const mimeType = this.getLiveTranscriptionMimeType();
+    const stream = new MediaStream([audioTrack]);
+    this.liveTranscriptionActive = true;
+    this.liveTranscriptionSequence = 0;
+    this.liveTranscriptionStartedAt = Date.now();
+    console.log("[Transcript] starting", {
+      room: this.room,
+      trackId: audioTrack.id,
+      readyState: audioTrack.readyState,
+      enabled: audioTrack.enabled,
+      mimeType
+    });
+    this.liveTranscriptionLoopPromise = this.runLiveTranscriptionLoop(stream, mimeType).finally(() => {
+      this.liveTranscriptionRecorder = null;
+      this.liveTranscriptionLoopPromise = null;
+    });
+    await Promise.resolve();
+  }
+  async captureLiveChunk(stream, mimeType) {
+    return new Promise((resolve, reject) => {
+      const chunks = [];
+      const recorder = mimeType ? new MediaRecorder(stream, { mimeType }) : new MediaRecorder(stream);
+      this.liveTranscriptionRecorder = recorder;
+      const timeout = window.setTimeout(() => {
+        if (recorder.state !== "inactive") {
+          recorder.stop();
+        }
+      }, 1500);
+      recorder.onstart = () => {
+        console.log("[Transcript] MediaRecorder started", {
+          mimeType: recorder.mimeType
+        });
+      };
+      recorder.ondataavailable = (event) => {
+        if (event.data.size > 0) {
+          chunks.push(event.data);
+        }
+      };
+      recorder.onerror = () => {
+        window.clearTimeout(timeout);
+        reject(new Error("Live transcription recorder failed"));
+      };
+      recorder.onstop = () => {
+        window.clearTimeout(timeout);
+        this.liveTranscriptionRecorder = null;
+        resolve(
+          chunks.length ? new Blob(chunks, {
+            type: recorder.mimeType || mimeType || "audio/webm"
+          }) : null
+        );
+      };
+      recorder.start();
+    });
+  }
+  async runLiveTranscriptionLoop(stream, mimeType) {
+    while (this.liveTranscriptionActive && stream.getAudioTracks()[0]?.readyState === "live") {
+      try {
+        const blob = await this.captureLiveChunk(stream, mimeType);
+        if (!this.liveTranscriptionActive) {
+          break;
+        }
+        if (!blob || blob.size === 0) {
+          continue;
+        }
+        const sequence = this.liveTranscriptionSequence++;
+        const offsetMs = Math.max(
+          0,
+          Date.now() - this.liveTranscriptionStartedAt
+        );
+        await this.uploadLiveTranscriptionChunk(
+          blob,
+          sequence,
+          offsetMs,
+          blob.type || mimeType
+        );
+      } catch (error) {
+        console.error(
+          "[Transcript] chunk processing failed",
+          error
+        );
+        this.onError(error);
+        await new Promise(
+          (resolve) => setTimeout(resolve, 750)
+        );
+      }
+    }
+  }
+  async uploadLiveTranscriptionChunk(blob, sequence, offsetMs, mimeType) {
+    if (!this.room || !this.authToken) {
+      return;
+    }
+    const form = new FormData();
+    form.append("sequence", String(sequence));
+    form.append("offset_ms", String(offsetMs));
+    const extension = mimeType.includes("ogg") || blob.type.includes("ogg") ? "ogg" : "webm";
+    form.append(
+      "audio",
+      blob,
+      `live-${sequence}.${extension}`
+    );
+    const url = `${this.apiBase}/rooms/${encodeURIComponent(this.room)}/live-transcription`;
+    console.log("[Transcript] POST", {
+      url,
+      sequence,
+      size: blob.size,
+      mimeType
+    });
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${this.authToken}`,
+        Accept: "application/json"
+      },
+      body: form
+    });
+    const responseText = await response.text();
+    console.log("[Transcript] response", {
+      status: response.status,
+      body: responseText
+    });
+    if (!response.ok) {
+      throw new Error(
+        `Live transcription request failed: ${response.status} ${responseText}`
+      );
+    }
+  }
+  stopLiveTranscription() {
+    console.log("[Transcript] stopping");
+    this.liveTranscriptionActive = false;
+    try {
+      if (this.liveTranscriptionRecorder && this.liveTranscriptionRecorder.state !== "inactive") {
+        this.liveTranscriptionRecorder.stop();
+      }
+    } catch (error) {
+      console.warn(
+        "[Transcript] recorder shutdown failed",
+        error
+      );
+    }
+    this.liveTranscriptionRecorder = null;
+  }
+  isLiveTranscriptionActive() {
+    return this.liveTranscriptionActive;
+  }
+  async applyNetworkRecommendation(action) {
+    if (!this.videoSender || this.networkProfile === action) return;
+    const params = this.videoSender.getParameters();
+    const encodings = params.encodings?.length ? params.encodings : [{}];
+    const encoding = encodings[0];
+    switch (action) {
+      case "reduce_video":
+        encoding.maxBitrate = 7e5;
+        encoding.maxFramerate = 20;
+        break;
+      case "audio_priority":
+        encoding.maxBitrate = 35e4;
+        encoding.maxFramerate = 10;
+        break;
+      default:
+        encoding.maxBitrate = 15e5;
+        encoding.maxFramerate = 30;
+        break;
+    }
+    params.encodings = encodings;
+    try {
+      await this.videoSender.setParameters(params);
+      this.networkProfile = action;
+    } catch (error) {
+      this.onError(error);
+    }
+  }
   async startRecording() {
     if (!this.send({ type: "start_recording" })) {
       throw new Error("SFU signaling is not connected");
@@ -1890,6 +2159,7 @@ var SFUClient = class {
     }
   }
   async leave() {
+    this.stopLiveTranscription();
     this.stopHeartbeat();
     this.intentionalClose = true;
     this.setState("leaving");
@@ -2298,6 +2568,14 @@ var VideoSDKCore = class {
         console.warn(`[SFU] Reconnecting, attempt ${attempt}`);
       },
       // ADMISSION CONTROL
+      onLiveTranscript: (transcript) => {
+        this.state.addLiveTranscript(transcript);
+        this.events.onLiveTranscript?.(transcript);
+      },
+      onNetworkQuality: (quality) => {
+        this.state.setNetworkQuality(quality);
+        this.events.onNetworkQuality?.(quality);
+      },
       onRoomUpdate: (payload) => {
         const participants = payload?.participants || [];
         const local = participants.find(
@@ -2942,6 +3220,25 @@ var VideoSDKCore = class {
       }
     }
   }
+  async startLiveTranscription() {
+    if (!this.sfuClient) {
+      throw new Error("SFU client is not connected");
+    }
+    const audioTrack = this.localStream?.getAudioTracks()[0];
+    if (!audioTrack || audioTrack.readyState !== "live") {
+      throw new Error("Microphone audio track is not available");
+    }
+    await this.sfuClient.startLiveTranscription();
+  }
+  stopLiveTranscription() {
+    this.sfuClient?.stopLiveTranscription();
+  }
+  isLiveTranscriptionActive() {
+    return this.sfuClient?.isLiveTranscriptionActive() ?? false;
+  }
+  getNetworkQuality() {
+    return this.state.networkQuality;
+  }
   // CLEANUP
   cleanupRemoteParticipant(participantId) {
     this.activeSpeakerDetector.removeParticipant(participantId);
@@ -2990,6 +3287,8 @@ var VideoSDKCore = class {
     this.room.id = null;
     this.room.name = null;
     this.state.localParticipant = null;
+    this.state.clearLiveTranscripts();
+    this.state.setNetworkQuality(null);
     this.state.notify("localParticipant");
     this.state.resetRemoteState();
     this.events.onMeetingLeft?.();
@@ -3022,9 +3321,15 @@ var MeetingProvider = ({
   const entryResponseListeners = (0, import_react2.useRef)(
     /* @__PURE__ */ new Set()
   );
-  const joinRequestCanceledListeners = (0, import_react2.useRef)(/* @__PURE__ */ new Set());
-  const participantRemovedListeners = (0, import_react2.useRef)(/* @__PURE__ */ new Set());
-  const participantPermissionsChangedListeners = (0, import_react2.useRef)(/* @__PURE__ */ new Set());
+  const joinRequestCanceledListeners = (0, import_react2.useRef)(
+    /* @__PURE__ */ new Set()
+  );
+  const participantRemovedListeners = (0, import_react2.useRef)(
+    /* @__PURE__ */ new Set()
+  );
+  const participantPermissionsChangedListeners = (0, import_react2.useRef)(
+    /* @__PURE__ */ new Set()
+  );
   const meetingLeftListeners = (0, import_react2.useRef)(/* @__PURE__ */ new Set());
   const trackStateListeners = (0, import_react2.useRef)(/* @__PURE__ */ new Set());
   const recordingStartedListeners = (0, import_react2.useRef)(/* @__PURE__ */ new Set());
@@ -3037,6 +3342,12 @@ var MeetingProvider = ({
   const activeSpeakerListeners = (0, import_react2.useRef)(
     /* @__PURE__ */ new Set()
   );
+  const liveTranscriptListeners = (0, import_react2.useRef)(
+    /* @__PURE__ */ new Set()
+  );
+  const networkQualityListeners = (0, import_react2.useRef)(
+    /* @__PURE__ */ new Set()
+  );
   const connectionStateListeners = (0, import_react2.useRef)(
     /* @__PURE__ */ new Set()
   );
@@ -3060,10 +3371,14 @@ var MeetingProvider = ({
           joinRequestCanceledListeners.current.forEach((fn) => fn(requestId));
         },
         onParticipantRemoved: (participantId, removedBy) => {
-          participantRemovedListeners.current.forEach((fn) => fn(participantId, removedBy));
+          participantRemovedListeners.current.forEach(
+            (fn) => fn(participantId, removedBy)
+          );
         },
         onParticipantPermissionsChanged: (participantId, permissions) => {
-          participantPermissionsChangedListeners.current.forEach((fn) => fn(participantId, permissions));
+          participantPermissionsChangedListeners.current.forEach(
+            (fn) => fn(participantId, permissions)
+          );
         },
         onMeetingLeft: () => {
           meetingLeftListeners.current.forEach((fn) => fn());
@@ -3116,6 +3431,12 @@ var MeetingProvider = ({
           );
           pubSubListeners.current.get("REACTION")?.forEach((fn) => fn(message));
         },
+        onLiveTranscript: (transcript) => {
+          liveTranscriptListeners.current.forEach((fn) => fn(transcript));
+        },
+        onNetworkQuality: (quality) => {
+          networkQualityListeners.current.forEach((fn) => fn(quality));
+        },
         onConnectionStateChanged: (state) => {
           connectionStateListeners.current.forEach(
             (fn) => fn(state)
@@ -3151,6 +3472,16 @@ var MeetingProvider = ({
     "chat",
     (s) => s.getChatMessages()
   );
+  const liveTranscripts = useMeetingStore(
+    sdk.state,
+    "transcript",
+    (s) => s.getLiveTranscripts()
+  );
+  const networkQuality = useMeetingStore(
+    sdk.state,
+    "networkQuality",
+    (s) => s.networkQuality
+  );
   const value = (0, import_react2.useMemo)(() => {
     return {
       sdk,
@@ -3175,6 +3506,11 @@ var MeetingProvider = ({
       messages,
       presenterId,
       activeSpeakerIds,
+      liveTranscripts,
+      networkQuality,
+      startLiveTranscription: sdk.startLiveTranscription.bind(sdk),
+      stopLiveTranscription: sdk.stopLiveTranscription.bind(sdk),
+      isLiveTranscriptionActive: sdk.isLiveTranscriptionActive.bind(sdk),
       recording,
       usePubSub: (topic) => {
         if (topic === "SECURE_CHAT") {
@@ -3293,6 +3629,14 @@ var MeetingProvider = ({
         reactionListeners.current.add(cb);
         return () => reactionListeners.current.delete(cb);
       },
+      onLiveTranscript: (cb) => {
+        liveTranscriptListeners.current.add(cb);
+        return () => liveTranscriptListeners.current.delete(cb);
+      },
+      onNetworkQuality: (cb) => {
+        networkQualityListeners.current.add(cb);
+        return () => networkQualityListeners.current.delete(cb);
+      },
       onConnectionStateChanged: (cb) => {
         connectionStateListeners.current.add(cb);
         return () => {
@@ -3315,6 +3659,8 @@ var MeetingProvider = ({
     presenterId,
     recording,
     activeSpeakerIds,
+    liveTranscripts,
+    networkQuality,
     reactionPubSubMessages
   ]);
   return /* @__PURE__ */ (0, import_jsx_runtime.jsx)(MeetingContext.Provider, { value, children });
@@ -3399,7 +3745,9 @@ var useMeeting = (handlers) => {
   }, [handlers?.onParticipantRemoved]);
   (0, import_react4.useEffect)(() => {
     if (!handlers?.onParticipantPermissionsChanged) return;
-    return ctx.onParticipantPermissionsChanged(handlers.onParticipantPermissionsChanged);
+    return ctx.onParticipantPermissionsChanged(
+      handlers.onParticipantPermissionsChanged
+    );
   }, [handlers?.onParticipantPermissionsChanged]);
   (0, import_react4.useEffect)(() => {
     if (!handlers?.onMeetingLeft) return;
@@ -3421,6 +3769,14 @@ var useMeeting = (handlers) => {
     if (!handlers?.onConnectionStateChanged) return;
     return ctx.onConnectionStateChanged(handlers.onConnectionStateChanged);
   }, [handlers?.onConnectionStateChanged]);
+  (0, import_react4.useEffect)(() => {
+    if (!handlers?.onLiveTranscript) return;
+    return ctx.onLiveTranscript(handlers.onLiveTranscript);
+  }, [handlers?.onLiveTranscript]);
+  (0, import_react4.useEffect)(() => {
+    if (!handlers?.onNetworkQuality) return;
+    return ctx.onNetworkQuality(handlers.onNetworkQuality);
+  }, [handlers?.onNetworkQuality]);
   (0, import_react4.useEffect)(() => {
     if (!handlers?.onReaction) return;
     return ctx.onReaction(handlers.onReaction);
@@ -3558,6 +3914,26 @@ function useMeetingPreview(roomId, participantId) {
   };
 }
 
+// src/react/useLiveTranscript.ts
+var useLiveTranscript = () => {
+  const { sdk } = useMeetingContext();
+  return useMeetingStore(
+    sdk.state,
+    "transcript",
+    (state) => state.getLiveTranscripts()
+  );
+};
+
+// src/react/useNetworkQuality.ts
+var useNetworkQuality = () => {
+  const { sdk } = useMeetingContext();
+  return useMeetingStore(
+    sdk.state,
+    "networkQuality",
+    (state) => state.networkQuality
+  );
+};
+
 // src/react/usePubSub.ts
 var import_react8 = require("react");
 function usePubSub(topic, options) {
@@ -3577,10 +3953,12 @@ function usePubSub(topic, options) {
   MeetingProvider,
   MeetingState,
   VideoSDKCore,
+  useLiveTranscript,
   useLocalParticipant,
   useMeeting,
   useMeetingContext,
   useMeetingPreview,
+  useNetworkQuality,
   useParticipants,
   usePubSub,
   useRemoteMedia