mediasfu-shared 1.2.7 → 1.3.0

package/dist/{updateParticipantAudioDecibels-BJ2PxJJD.cjs → updateParticipantAudioDecibels-Dni8i6AY.cjs}

@@ -1,6 +1,6 @@
 "use strict";
 const io = require("socket.io-client");
-const audioProcessing = require("./audioProcessing-BpbrdCk-.cjs");
+const audioProcessing = require("./audioProcessing-VCbu70Jt.cjs");
 const createFrameworkConsumerContractError = (consumerName) => {
   return new Error(
     `${consumerName} is a render-coupled consumer helper and has no shared runtime implementation. Inject the framework-specific ${consumerName} implementation instead of calling the shared fallback export.`
@@ -507,6 +507,7 @@ const closeAndResize = async ({ producerId, kind, parameters }) => {
   parameters = getUpdatedAllParams();
   let {
     allAudioStreams,
+    audioOnlyStreams = [],
     allVideoStreams,
     activeNames,
     participants,
@@ -531,6 +532,7 @@ const closeAndResize = async ({ producerId, kind, parameters }) => {
     updateMainWindow,
     updateActiveNames,
     updateAllAudioStreams,
+    updateAudioOnlyStreams,
     updateAllVideoStreams,
     updateShareScreenStarted,
     updateUpdateMainWindow,
@@ -554,6 +556,11 @@ const closeAndResize = async ({ producerId, kind, parameters }) => {
       return audioStream.producerId !== producerId;
     });
     updateAllAudioStreams(allAudioStreams);
+    const playerProducerId = (element) => String(element?.props?.remoteProducerId ?? element?.inputs?.remoteProducerId ?? element?.key ?? "");
+    if (audioOnlyStreams.some((element) => playerProducerId(element) === producerId)) {
+      audioOnlyStreams = audioOnlyStreams.filter((element) => playerProducerId(element) !== producerId);
+      updateAudioOnlyStreams?.(audioOnlyStreams);
+    }
     if (recordingDisplayType == "video" && recordingVideoOptimized == true) ;
     else {
       participant = participants.find(
@@ -812,12 +819,26 @@ async function connectSocket({ apiUserName, apiKey, apiToken, link }) {
     }
     let settled = false;
     let timeoutHandle;
+    const startedAt = Date.now();
+    let transportOpened = false;
+    let rejection = null;
     const cleanupHandshake = () => {
       if (timeoutHandle) clearTimeout(timeoutHandle);
       socket.off("connection-success", handleConnectionSuccess);
       socket.off("connect_error", handleConnectError);
       socket.off("disconnect", handleDisconnectBeforeReady);
+      socket.off("connect", handleTransportOpen);
+      socket.off("connection-rejected", handleConnectionRejected);
     };
+    const failure = (phase, transportError) => new audioProcessing.MediaSocketConnectError({
+      phase: rejection ? "rejected" : phase,
+      reason: rejection?.reason,
+      stage: rejection?.stage,
+      elapsedMs: Date.now() - startedAt,
+      online: audioProcessing.currentOnlineState(),
+      transportError,
+      link
+    });
     const settle = (action) => {
       if (settled) return;
       settled = true;
@@ -838,28 +859,40 @@ async function connectSocket({ apiUserName, apiKey, apiToken, link }) {
     const handleConnectError = (error) => {
       settle(() => {
         socket.disconnect();
-        reject(new Error("Error connecting to media socket: " + error.message));
+        reject(failure(transportOpened ? "after-open" : "before-open", error?.message));
+      });
+    };
+    const handleTransportOpen = () => {
+      transportOpened = true;
+    };
+    const handleConnectionRejected = (payload) => {
+      rejection = {
+        reason: audioProcessing.normalizeRejectionReason(payload?.reason),
+        stage: typeof payload?.stage === "string" ? payload.stage : void 0
+      };
+      settle(() => {
+        try {
+          socket.disconnect();
+        } catch {
+        }
+        reject(failure("rejected"));
       });
     };
     const handleDisconnectBeforeReady = (reason) => {
-      settle(
-        () => reject(
-          new Error(
-            "Media socket closed before the room was ready (" + reason + ")."
-          )
-        )
-      );
+      settle(() => reject(failure("after-open", reason)));
     };
     socket.on("connection-success", handleConnectionSuccess);
     socket.on("connect_error", handleConnectError);
     socket.on("disconnect", handleDisconnectBeforeReady);
+    socket.on("connect", handleTransportOpen);
+    socket.on("connection-rejected", handleConnectionRejected);
     timeoutHandle = setTimeout(() => {
       settle(() => {
         try {
           socket.disconnect();
         } catch {
         }
-        reject(new Error("Timed out connecting to media socket."));
+        reject(failure("timeout"));
       });
     }, CONNECT_TIMEOUT_MS);
   });
@@ -873,11 +906,24 @@ async function connectLocalSocket({ link }) {
     socket = io(`${link}/media`, {
       transports: ["websocket"]
     });
+    const startedAt = Date.now();
+    let transportOpened = false;
+    socket.on("connect", () => {
+      transportOpened = true;
+    });
     socket.on("connection-success", (data) => {
       resolve({ socket, data });
     });
     socket.on("connect_error", (error) => {
-      reject(new Error("Error connecting to media socket: " + error.message));
+      reject(
+        new audioProcessing.MediaSocketConnectError({
+          phase: transportOpened ? "after-open" : "before-open",
+          elapsedMs: Date.now() - startedAt,
+          online: audioProcessing.currentOnlineState(),
+          transportError: error?.message,
+          link
+        })
+      );
     });
   });
 }
@@ -1246,11 +1292,72 @@ const newPipeProducer = async ({
     updateFirst_round(updatedFirstRound);
   }
 };
+const SCREEN_AUDIO_MEDIA_TAG = "screen-audio";
+const SCREEN_AUDIO_CODEC_OPTIONS = {
+  opusStereo: true,
+  opusDtx: false,
+  opusFec: true,
+  opusMaxAverageBitrate: 128e3
+};
+const screenAudioProducerIds = /* @__PURE__ */ new Set();
+const markScreenAudioProducer = (producerId, mediaTag) => {
+  if (producerId && mediaTag === SCREEN_AUDIO_MEDIA_TAG) screenAudioProducerIds.add(producerId);
+};
+const forgetScreenAudioProducer = (producerId) => {
+  if (producerId) screenAudioProducerIds.delete(producerId);
+};
+const isScreenAudioProducer = ({ producerId, participants = [], appData }) => {
+  if (!producerId) return false;
+  if (screenAudioProducerIds.has(producerId)) return true;
+  if (appData?.mediaTag === SCREEN_AUDIO_MEDIA_TAG) return true;
+  return participants.some((participant) => participant?.ScreenAudioID === producerId);
+};
+const shouldShareScreenAudio = ({ supportsScreenAudio, screenAudioEnabled, screenAudioMode }) => supportsScreenAudio === true && screenAudioEnabled !== false && screenAudioMode !== "never";
+const canCaptureDisplayAudio = (nav = typeof navigator !== "undefined" ? navigator : void 0) => {
+  if (!nav) return false;
+  const ua = String(nav.userAgent || "");
+  if (/Mobi|Android|iPhone|iPad|iPod/i.test(ua)) return false;
+  const brands = (nav.userAgentData?.brands || []).map((entry) => String(entry?.brand || ""));
+  if (brands.some((brand) => /Chromium|Google Chrome|Microsoft Edge|Opera|Brave/i.test(brand))) return true;
+  return /(Chrome|Chromium|Edg)\//.test(ua) && !/Firefox|FxiOS|OPiOS/.test(ua);
+};
+const setScreenAudioPaused = (producer, paused) => {
+  if (!producer || producer.closed) return false;
+  if (paused && !producer.paused) producer.pause();
+  if (!paused && producer.paused) producer.resume();
+  return producer.paused;
+};
+const buildDisplayMediaOptions = ({ targetWidth, targetHeight, withAudio }) => {
+  const options = {
+    video: { width: targetWidth, height: targetHeight, frameRate: 30 },
+    audio: withAudio ? { echoCancellation: false, noiseSuppression: false, autoGainControl: false } : false
+  };
+  if (withAudio) {
+    options.systemAudio = "include";
+    options.suppressLocalAudioPlayback = false;
+  }
+  return options;
+};
+const producersBySocket = /* @__PURE__ */ new WeakMap();
+const rememberScreenAudioProducer = (socket, producer) => {
+  if (!socket) return;
+  if (producer) producersBySocket.set(socket, producer);
+  else producersBySocket.delete(socket);
+};
+const recallScreenAudioProducer = (socket) => socket && producersBySocket.get(socket) || null;
+const setScreenAudioVolume = (volume, root = typeof document !== "undefined" ? document : null) => {
+  if (!root) return;
+  const value = Math.min(1, Math.max(0, Number.isFinite(volume) ? volume : 1));
+  root.querySelectorAll('audio[data-screen-audio="true"]').forEach((element) => {
+    element.volume = value;
+  });
+};
 const producerClosed = async ({
   remoteProducerId,
   parameters
 }) => {
   let { consumerTransports, closeAndResize: closeAndResize2, screenId, updateConsumerTransports } = parameters;
+  forgetScreenAudioProducer(remoteProducerId);
   const activeTranslationProducerIds = parameters.activeTranslationProducerIds;
   const isTranslationProducer = activeTranslationProducerIds?.has?.(remoteProducerId);
   if (isTranslationProducer) {
@@ -1401,6 +1508,20 @@ const joinConsumeRoom = async ({
 };
 const pendingConsumeConnections = /* @__PURE__ */ new WeakMap();
 const normalizeConsumeEndpoint = (ip) => ip.trim().toLowerCase().replace(/\.$/, "");
+const CONSUME_REJECTION_COOLDOWN_MS = 6e4;
+const rejectedConsumeEndpoints = /* @__PURE__ */ new WeakMap();
+const consumeEndpointRecentlyRejected = (owner, endpoint) => {
+  const at = rejectedConsumeEndpoints.get(owner)?.get(endpoint);
+  return at !== void 0 && Date.now() - at < CONSUME_REJECTION_COOLDOWN_MS;
+};
+const rememberConsumeEndpointRejection = (owner, endpoint) => {
+  let byEndpoint = rejectedConsumeEndpoints.get(owner);
+  if (!byEndpoint) {
+    byEndpoint = /* @__PURE__ */ new Map();
+    rejectedConsumeEndpoints.set(owner, byEndpoint);
+  }
+  byEndpoint.set(endpoint, Date.now());
+};
 const hasConsumeEndpoint = (consumeSockets, endpoint) => {
   for (let index = consumeSockets.length - 1; index >= 0; index -= 1) {
     const socketObj = consumeSockets[index];
@@ -1468,7 +1589,9 @@ const connectIps = async ({
       });
       return [consume_sockets, roomRecvIPs];
     }
+    const rejectionOwner = parameters?.socket || consume_sockets;
     for (const ip of remIP) {
+      if (consumeEndpointRecentlyRejected(rejectionOwner, normalizeConsumeEndpoint(ip))) continue;
       const releaseReservation = await reserveConsumeEndpoint(
         consume_sockets,
         ip,
@@ -1480,11 +1603,25 @@ const connectIps = async ({
       try {
         remote_sock = await connectSocket({ apiUserName, apiKey, apiToken, link: `https://${ip}.mediasfu.com` });
         if (remote_sock.id) {
+          let consumeRoomReady = false;
+          const pendingProducers = [];
           if (!roomRecvIPs.includes(ip)) {
             roomRecvIPs.push(ip);
             updateRoomRecvIPs(roomRecvIPs);
           }
-          remote_sock.on("new-pipe-producer", async ({ producerId, islevel, isTranslation, translationMeta }) => {
+          const consumeProducer = async ({ producerId, islevel, isTranslation, translationMeta, mediaTag }) => {
+            markScreenAudioProducer(producerId, mediaTag);
+            if (!consumeRoomReady) {
+              pendingProducers.push({ producerId, islevel, isTranslation, translationMeta });
+              return;
+            }
+            const preferDelay = isTranslation ? 0 : audioProcessing.consumerBackupPreferDelay(
+              ip,
+              parameters.getUpdatedAllParams?.()?.consume_sockets
+            );
+            if (preferDelay > 0) {
+              await new Promise((resolve) => setTimeout(resolve, preferDelay));
+            }
             if (newProducerMethod) {
               await newProducerMethod({
                 producerId,
@@ -1495,7 +1632,8 @@ const connectIps = async ({
                 translationMeta
               });
             }
-          });
+          };
+          remote_sock.on("new-pipe-producer", consumeProducer);
           remote_sock.on("producer-closed", async ({ remoteProducerId }) => {
             if (closedProducerMethod) {
               await closedProducerMethod({ remoteProducerId, parameters });
@@ -1512,6 +1650,11 @@ const connectIps = async ({
               throw new Error(`Consume room ${ip} did not return RTP capabilities.`);
             }
           }
+          const queuedProducers = pendingProducers.splice(0);
+          consumeRoomReady = true;
+          for (const producer of queuedProducers) {
+            await consumeProducer(producer);
+          }
           consume_sockets.push({ [ip]: remote_sock });
           updateConsume_sockets(consume_sockets);
           connected = true;
@@ -1533,7 +1676,12 @@ const connectIps = async ({
           });
         }
       } catch (error) {
-        console.log("connectIps error", error);
+        if (audioProcessing.isMediaSocketConnectError(error) && !error.retryable) {
+          rememberConsumeEndpointRejection(rejectionOwner, normalizeConsumeEndpoint(ip));
+          console.warn(`connectIps: ${ip} refused this room (${error.reason}); not retrying for ${CONSUME_REJECTION_COOLDOWN_MS / 1e3} s.`, error.message);
+        } else {
+          console.log("connectIps error", error);
+        }
       } finally {
         if (!connected && remote_sock) {
           remote_sock.removeAllListeners?.();
@@ -2232,6 +2380,99 @@ const disconnectSendTransportAudio = async ({
     console.log("Error disconnecting primary audio send transport:", error);
   }
 };
+const currentScreenAudioProducer = (parameters) => parameters.screenAudioProducer || recallScreenAudioProducer(parameters.socket);
+const storeScreenAudioProducer = (parameters, producer) => {
+  rememberScreenAudioProducer(parameters.socket, producer);
+  parameters.updateScreenAudioProducer?.(producer);
+};
+const disconnectSendTransportScreenAudio = async ({
+  parameters,
+  notifyServer = false
+}) => {
+  const params = parameters.getUpdatedAllParams ? parameters.getUpdatedAllParams() : parameters;
+  const producer = currentScreenAudioProducer(params);
+  if (!producer) return;
+  try {
+    if (!producer.closed) producer.close();
+  } catch {
+  }
+  storeScreenAudioProducer(params, null);
+  if (notifyServer) {
+    try {
+      params.socket?.emit("closeScreenAudioProducer");
+    } catch {
+    }
+  }
+};
+const toggleScreenAudio = ({ parameters, paused }) => {
+  const params = parameters.getUpdatedAllParams ? parameters.getUpdatedAllParams() : parameters;
+  const producer = currentScreenAudioProducer(params);
+  if (!producer || producer.closed) return null;
+  const next = paused ?? !producer.paused;
+  if (next && !producer.paused) producer.pause();
+  if (!next && producer.paused) producer.resume();
+  params.updateScreenAudioProducer?.(producer);
+  return producer.paused;
+};
+const produceOnce = async (transport, track) => {
+  const producer = await transport.produce({
+    track,
+    codecOptions: SCREEN_AUDIO_CODEC_OPTIONS,
+    appData: { mediaTag: SCREEN_AUDIO_MEDIA_TAG },
+    // The track belongs to the screen share stream, which stops it with the share.
+    stopTracks: false
+  });
+  if (!producer?.id) {
+    try {
+      producer?.close();
+    } catch {
+    }
+    return null;
+  }
+  return producer;
+};
+const connectSendTransportScreenAudio = async ({
+  stream,
+  parameters,
+  retryDelayMs = 600
+}) => {
+  const params = parameters.getUpdatedAllParams ? parameters.getUpdatedAllParams() : parameters;
+  const track = stream?.getAudioTracks?.()[0];
+  if (!track || track.readyState === "ended") return null;
+  if (!shouldShareScreenAudio(params)) return null;
+  const transport = params.producerTransport;
+  if (!transport || transport.closed) return null;
+  const existing = currentScreenAudioProducer(params);
+  if (existing && !existing.closed) return existing;
+  try {
+    let producer = await produceOnce(transport, track);
+    const trackEnded = () => track.readyState === "ended";
+    if (!producer && retryDelayMs > 0 && !trackEnded()) {
+      await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
+      producer = await produceOnce(transport, track);
+    }
+    if (!producer) return null;
+    storeScreenAudioProducer(params, producer);
+    const screenProducer = params.screenProducer;
+    if (screenProducer?.paused) producer.pause();
+    screenProducer?.observer?.on("pause", () => {
+      if (!producer.closed) producer.pause();
+    });
+    screenProducer?.observer?.on("resume", () => {
+      if (!producer.closed) producer.resume();
+    });
+    screenProducer?.observer?.on("close", () => {
+      void disconnectSendTransportScreenAudio({ parameters: params });
+    });
+    track.addEventListener("ended", () => {
+      void disconnectSendTransportScreenAudio({ parameters: params, notifyServer: true });
+    });
+    return producer;
+  } catch (error) {
+    console.log("connectSendTransportScreenAudio error", error);
+    return null;
+  }
+};
 const disconnectLocalSendTransportScreen = async ({ parameters }) => {
   try {
     let {
@@ -2253,6 +2494,10 @@ const disconnectLocalSendTransportScreen = async ({ parameters }) => {
 const disconnectSendTransportScreen = async ({ parameters }) => {
   let { getUpdatedAllParams } = parameters;
   parameters = getUpdatedAllParams();
+  try {
+    await disconnectSendTransportScreenAudio({ parameters });
+  } catch {
+  }
   try {
     let {
       screenProducer,
@@ -3654,6 +3899,11 @@ async function startShareScreen({ parameters }) {
     onWeb,
     targetWidth = 1280,
     targetHeight = 720,
+    supportsScreenAudio,
+    screenAudioEnabled,
+    screenAudioMode = "ask",
+    screenAudioPrompt,
+    displayAudioCapable,
     streamSuccessScreen: streamSuccessScreen2
   } = parameters;
   try {
@@ -3666,15 +3916,23 @@ async function startShareScreen({ parameters }) {
       return;
     }
     if (mediaDevices && mediaDevices.getDisplayMedia) {
+      let withAudio = shouldShareScreenAudio({ supportsScreenAudio, screenAudioEnabled, screenAudioMode }) && (displayAudioCapable ?? canCaptureDisplayAudio());
+      if (withAudio && screenAudioMode === "ask" && screenAudioPrompt) {
+        const choice = await screenAudioPrompt();
+        if (choice === null) {
+          updateShared(false);
+          return;
+        }
+        withAudio = choice === "audio";
+      }
       shared = true;
-      await mediaDevices.getDisplayMedia({
-        video: {
-          width: targetWidth,
-          height: targetHeight,
-          frameRate: 30
-        },
-        audio: false
-      }).then(async (stream) => {
+      await mediaDevices.getDisplayMedia(
+        buildDisplayMediaOptions({
+          targetWidth,
+          targetHeight,
+          withAudio
+        })
+      ).then(async (stream) => {
         await streamSuccessScreen2({ stream, parameters });
       }).catch(async () => {
         shared = false;
@@ -4102,6 +4360,14 @@ const streamSuccessScreen = async ({
       });
     }
     socket.emit("startScreenShare");
+    try {
+      await connectSendTransportScreenAudio({
+        stream: localStreamScreen,
+        parameters: getUpdatedAllParams()
+      });
+    } catch (error) {
+      console.log("Screen audio not shared:", error);
+    }
   } catch (error) {
     showAlert?.({
       message: error.message,
@@ -5373,6 +5639,7 @@ const getPipedProducersAlt = async ({
             } else {
               producerId = producer.id;
               translationMeta = producer.translationMeta || null;
+              markScreenAudioProducer(producerId, producer.mediaTag);
             }
             if (translationMeta) {
               const shouldConsume = shouldConsumeTranslationProducer(
@@ -5591,8 +5858,13 @@ function updateParticipantAudioDecibels({
   }
   updateAudioDecibels(audioDecibels);
 }
+exports.CONSUME_REJECTION_COOLDOWN_MS = CONSUME_REJECTION_COOLDOWN_MS;
+exports.SCREEN_AUDIO_CODEC_OPTIONS = SCREEN_AUDIO_CODEC_OPTIONS;
+exports.SCREEN_AUDIO_MEDIA_TAG = SCREEN_AUDIO_MEDIA_TAG;
 exports.autoAdjust = autoAdjust;
+exports.buildDisplayMediaOptions = buildDisplayMediaOptions;
 exports.calculateRowsAndColumns = calculateRowsAndColumns;
+exports.canCaptureDisplayAudio = canCaptureDisplayAudio;
 exports.changeVids = changeVids;
 exports.checkGrid = checkGrid;
 exports.checkPermission = checkPermission;
@@ -5608,6 +5880,7 @@ exports.connectRecvTransport = connectRecvTransport;
 exports.connectSendTransport = connectSendTransport;
 exports.connectSendTransportAudio = connectSendTransportAudio;
 exports.connectSendTransportScreen = connectSendTransportScreen;
+exports.connectSendTransportScreenAudio = connectSendTransportScreenAudio;
 exports.connectSendTransportVideo = connectSendTransportVideo;
 exports.connectSocket = connectSocket;
 exports.consumerResume = consumerResume;
@@ -5616,16 +5889,20 @@ exports.createFrameworkConsumerContractError = createFrameworkConsumerContractEr
 exports.createSendTransport = createSendTransport;
 exports.disconnectSendTransportAudio = disconnectSendTransportAudio;
 exports.disconnectSendTransportScreen = disconnectSendTransportScreen;
+exports.disconnectSendTransportScreenAudio = disconnectSendTransportScreenAudio;
 exports.disconnectSendTransportVideo = disconnectSendTransportVideo;
 exports.disconnectSocket = disconnectSocket;
 exports.dispStreams = dispStreams;
+exports.forgetScreenAudioProducer = forgetScreenAudioProducer;
 exports.generatePageContent = generatePageContent;
 exports.getEstimate = getEstimate;
 exports.getPipedProducersAlt = getPipedProducersAlt;
 exports.getProducersPiped = getProducersPiped;
 exports.getVideos = getVideos;
+exports.isScreenAudioProducer = isScreenAudioProducer;
 exports.joinConRoom = joinConRoom;
 exports.joinConsumeRoom = joinConsumeRoom;
+exports.markScreenAudioProducer = markScreenAudioProducer;
 exports.mixStreams = mixStreams;
 exports.newPipeProducer = newPipeProducer;
 exports.onScreenChanges = onScreenChanges;
@@ -5635,14 +5912,19 @@ exports.producerClosed = producerClosed;
 exports.rePort = rePort;
 exports.reUpdateInter = reUpdateInter;
 exports.readjust = readjust;
+exports.recallScreenAudioProducer = recallScreenAudioProducer;
 exports.receiveAllPipedTransports = receiveAllPipedTransports;
 exports.receiveRoomMessages = receiveRoomMessages;
+exports.rememberScreenAudioProducer = rememberScreenAudioProducer;
 exports.reorderStreams = reorderStreams;
 exports.requestScreenShare = requestScreenShare;
 exports.resumePauseAudioStreams = resumePauseAudioStreams;
 exports.resumePauseStreams = resumePauseStreams;
 exports.resumeSendTransportAudio = resumeSendTransportAudio;
 exports.selectVideoProducerCodec = selectVideoProducerCodec;
+exports.setScreenAudioPaused = setScreenAudioPaused;
+exports.setScreenAudioVolume = setScreenAudioVolume;
+exports.shouldShareScreenAudio = shouldShareScreenAudio;
 exports.signalNewConsumerTransport = signalNewConsumerTransport;
 exports.startShareScreen = startShareScreen;
 exports.stopShareScreen = stopShareScreen;
@@ -5653,8 +5935,9 @@ exports.streamSuccessVideo = streamSuccessVideo;
 exports.switchUserAudio = switchUserAudio;
 exports.switchUserVideo = switchUserVideo;
 exports.switchUserVideoAlt = switchUserVideoAlt;
+exports.toggleScreenAudio = toggleScreenAudio;
 exports.trigger = trigger;
 exports.updateMicLevel = updateMicLevel;
 exports.updateMiniCardsGrid = updateMiniCardsGrid;
 exports.updateParticipantAudioDecibels = updateParticipantAudioDecibels;
-//# sourceMappingURL=updateParticipantAudioDecibels-BJ2PxJJD.cjs.map
+//# sourceMappingURL=updateParticipantAudioDecibels-Dni8i6AY.cjs.map