mediasfu-shared 1.2.2 → 1.2.4

package/src/methods/utils/headless/virtualBackground.ts

@@ -3,7 +3,11 @@ import { HeadlessOptions } from './headlessTypes';
 import { getCurrentParams } from './getCurrentParams';
 import { getLocalVideoStream } from './getMediaStreams';
 import { HeadlessActionResult } from './roomActions';
-import { compositeVirtualBackgroundFrame } from '../virtualBackgroundCompositor';
+import {
+  compositeVirtualBackgroundFrame,
+  DEFAULT_BACKGROUND_BLUR_PIXELS,
+} from '../virtualBackgroundCompositor';
+import { startVirtualBackgroundFrameLoop } from '../virtualBackgroundFrameLoop';
 
 const MEDIAPIPE_CDN = 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation';
 
@@ -57,6 +61,13 @@ export interface ApplyVirtualBackgroundOptions extends HeadlessOptions {
    * A trailing slash is added if missing.
    */
   assetPath?: string;
+  /**
+   * Blur the area behind the segmented person by this many pixels. When this
+   * is greater than zero it takes precedence over `image`. Defaults to 0.
+   */
+  blurPixels?: number;
+  /** Best-effort hidden-tab processing for blur and image backgrounds. Defaults to true. */
+  keepProcessingWhenHidden?: boolean;
 }
 
 export type ApplyVirtualBackgroundType = (
@@ -98,6 +109,8 @@ export async function applyVirtualBackground({
   modelSelection = 1,
   publish = true,
   assetPath = MEDIAPIPE_CDN,
+  blurPixels = 0,
+  keepProcessingWhenHidden = true,
 }: ApplyVirtualBackgroundOptions): Promise<HeadlessActionResult & { stream: MediaStream | null }> {
   const live = getCurrentParams({ parameters });
   if (live.audioOnlyRoom) {
@@ -124,7 +137,10 @@ export async function applyVirtualBackground({
     const width = Number(settings.width) || 640;
     const height = Number(settings.height) || 360;
 
-    const backgroundImage = typeof image === 'string' ? await loadImage(image) : image;
+    const resolvedBlurPixels = Math.max(0, Number(blurPixels) || 0);
+    const backgroundImage = resolvedBlurPixels > 0
+      ? null
+      : (typeof image === 'string' ? await loadImage(image) : image);
     const producer = publish ? (live.videoProducer || live.localVideoProducer) : null;
 
     // mediasoup Producers normally own their track (`stopTracks: true`). Their
@@ -169,6 +185,7 @@ export async function applyVirtualBackground({
           backgroundImage: backgroundImage as CanvasImageSource | null,
           width: canvas.width,
           height: canvas.height,
+          blurFallbackPixels: resolvedBlurPixels,
         });
       } catch {
         // A dropped frame must never kill the loop.
@@ -176,27 +193,31 @@ export async function applyVirtualBackground({
     });
 
     let stopped = false;
-    let frameHandle = 0;
-    const pump = async () => {
-      if (stopped) return;
-      try {
-        if (video.readyState >= 2) await segmentation.send({ image: video });
-      } catch {
-        // Ignore a single failed frame; the next one is already queued.
-      }
-      frameHandle = window.requestAnimationFrame(() => { void pump(); });
-    };
-    void pump();
+    const stopFrameLoop = startVirtualBackgroundFrameLoop({
+      owner: segmentation,
+      keepProcessingWhenHidden,
+      shouldContinue: () => !stopped && processingTrack.readyState === 'live',
+      processFrame: () => video.readyState >= 2
+        ? segmentation.send({ image: video })
+        : undefined,
+    });
 
-    const processed: MediaStream = (canvas as any).captureStream(frameRate);
+    let processed: MediaStream;
+    try {
+      processed = (canvas as any).captureStream(frameRate);
+    } catch (error) {
+      stopFrameLoop();
+      throw error;
+    }
     if (!processed?.getVideoTracks?.().length) {
+      stopFrameLoop();
       throw new Error('The processed background stream produced no video track.');
     }
 
     const stop = () => {
       if (stopped) return;
       stopped = true;
-      window.cancelAnimationFrame(frameHandle);
+      stopFrameLoop();
       try { segmentation.close(); } catch { /* already closed */ }
       processed.getTracks().forEach((track) => track.stop());
       processingTrack.stop();
@@ -236,6 +257,25 @@ export async function applyVirtualBackground({
   }
 }
 
+export interface ApplyBackgroundBlurOptions extends Omit<ApplyVirtualBackgroundOptions, 'image'> {
+  /** Blur strength in CSS pixels. Defaults to 16. */
+  blurPixels?: number;
+}
+
+/** Apply person-aware background blur through the standard processed-track lifecycle. */
+export function applyBackgroundBlur({
+  blurPixels = DEFAULT_BACKGROUND_BLUR_PIXELS,
+  ...options
+}: ApplyBackgroundBlurOptions): ReturnType<ApplyVirtualBackgroundType> {
+  return applyVirtualBackground({
+    ...options,
+    image: null,
+    blurPixels,
+  });
+}
+
+export type ApplyBackgroundBlurType = typeof applyBackgroundBlur;
+
 export type ClearVirtualBackgroundType = (
   options: HeadlessOptions
 ) => Promise<HeadlessActionResult>;

package/src/methods/utils/virtualBackgroundCompositor.ts

@@ -10,6 +10,14 @@ export interface CompositeVirtualBackgroundFrameOptions {
   blurFallbackPixels?: number;
 }
 
+/** Stable room-state value used by every web UI surface for background blur. */
+export const VIRTUAL_BACKGROUND_BLUR = 'blur';
+export const DEFAULT_BACKGROUND_BLUR_PIXELS = 16;
+
+export function isVirtualBackgroundBlur(value: unknown): boolean {
+  return value === VIRTUAL_BACKGROUND_BLUR;
+}
+
 /**
  * Paint the segmented person over a replacement background.
  *
@@ -43,7 +51,16 @@ export function compositeVirtualBackgroundFrame({
       ctx.fillRect(0, 0, width, height);
     } else if (blurFallbackPixels > 0) {
       ctx.filter = `blur(${blurFallbackPixels}px)`;
-      ctx.drawImage(sourceImage, 0, 0, width, height);
+      // Draw beyond the visible bounds so the blur kernel does not expose a
+      // transparent/black fringe around the processed frame.
+      const bleed = Math.max(2, blurFallbackPixels * 2);
+      ctx.drawImage(
+        sourceImage,
+        -bleed,
+        -bleed,
+        width + bleed * 2,
+        height + bleed * 2,
+      );
     }
   } finally {
     ctx.filter = previousFilter || 'none';

package/src/methods/utils/virtualBackgroundFrameLoop.ts

@@ -0,0 +1,69 @@
+export interface VirtualBackgroundFrameLoopOptions {
+  owner: object;
+  processFrame: () => Promise<unknown> | unknown;
+  shouldContinue: () => boolean;
+  /** Best effort: background tabs may still throttle timers and camera capture. */
+  keepProcessingWhenHidden?: boolean;
+}
+
+const activeLoops = new WeakMap<object, () => void>();
+const activeFrames = new WeakMap<object, Promise<unknown>>();
+
+/** Run at most one segmentation frame at a time, including in hidden tabs. */
+export function startVirtualBackgroundFrameLoop({
+  owner,
+  processFrame,
+  shouldContinue,
+  keepProcessingWhenHidden = true,
+}: VirtualBackgroundFrameLoopOptions): () => void {
+  activeLoops.get(owner)?.();
+  let stopped = false;
+  let inFlight = false;
+  let animationFrameId: number | null = null;
+  let timeoutId: ReturnType<typeof setTimeout> | null = null;
+
+  const cancelScheduledFrame = () => {
+    if (animationFrameId !== null) cancelAnimationFrame(animationFrameId);
+    if (timeoutId !== null) clearTimeout(timeoutId);
+    animationFrameId = null;
+    timeoutId = null;
+  };
+  const stop = () => {
+    if (stopped) return;
+    stopped = true;
+    cancelScheduledFrame();
+    document.removeEventListener('visibilitychange', onVisibilityChange);
+    if (activeLoops.get(owner) === stop) activeLoops.delete(owner);
+  };
+  const schedule = () => {
+    if (stopped || inFlight) return;
+    if (!shouldContinue()) { stop(); return; }
+    if (document.hidden && keepProcessingWhenHidden) {
+      timeoutId = setTimeout(() => { void runFrame(); }, 1000);
+    } else {
+      animationFrameId = requestAnimationFrame(() => { void runFrame(); });
+    }
+  };
+  const runFrame = async () => {
+    animationFrameId = null;
+    timeoutId = null;
+    if (stopped || inFlight) return;
+    if (!shouldContinue()) { stop(); return; }
+    inFlight = true;
+    try {
+      const previousFrame = activeFrames.get(owner);
+      if (previousFrame) await previousFrame.catch(() => undefined);
+      if (stopped || !shouldContinue()) { stop(); return; }
+      const currentFrame = Promise.resolve(processFrame());
+      activeFrames.set(owner, currentFrame);
+      try { await currentFrame; }
+      finally { if (activeFrames.get(owner) === currentFrame) activeFrames.delete(owner); }
+    } catch { /* A dropped frame is recoverable. */ }
+    finally { inFlight = false; schedule(); }
+  };
+  const onVisibilityChange = () => { cancelScheduledFrame(); schedule(); };
+  activeLoops.set(owner, stop);
+  document.addEventListener('visibilitychange', onVisibilityChange);
+  void runFrame();
+  return stop;
+}

package/src/sockets/SocketManager.ts

@@ -80,6 +80,9 @@ export type ConnectLocalSocketType = (options: ConnectLocalSocketOptions) => Pro
  * ```
  */
 
+/** How long to wait for `connection-success` before giving up. */
+const CONNECT_TIMEOUT_MS = 12000;
+
 async function connectSocket(
   { apiUserName, apiKey, apiToken, link }: ConnectSocketOptions,
 ): Promise<Socket> {
@@ -132,8 +135,24 @@ async function connectSocket(
       });
     }
 
-    // Handle socket connection events
-    socket.on('connection-success', ({ socketId }: { socketId: string }) => {
+    // Dispose failed handshakes immediately. Otherwise Socket.IO continues
+    // reconnecting a client that was never handed to the room lifecycle.
+    let settled = false;
+    let timeoutHandle: ReturnType<typeof setTimeout> | undefined;
+    const cleanupHandshake = () => {
+      if (timeoutHandle) clearTimeout(timeoutHandle);
+      socket.off('connection-success', handleConnectionSuccess);
+      socket.off('connect_error', handleConnectError);
+      socket.off('disconnect', handleDisconnectBeforeReady);
+    };
+    const settle = (action: () => void) => {
+      if (settled) return;
+      settled = true;
+      cleanupHandshake();
+      action();
+    };
+
+    const handleConnectionSuccess = ({ socketId }: { socketId: string }) => {
       //check if link contains mediasfu.com and contains more than one c
       let conn = 'media';
       try {
@@ -145,12 +164,40 @@ async function connectSocket(
       }
 
       console.log(`Connected to ${conn} socket with ID: ${socketId}`);
-      resolve(socket);
-    });
+      settle(() => resolve(socket));
+    };
 
-    socket.on('connect_error', (error: Error) => {
-      reject(new Error('Error connecting to media socket: ' + error.message));
-    });
+    const handleConnectError = (error: Error) => {
+      settle(() => {
+        socket.disconnect();
+        reject(new Error('Error connecting to media socket: ' + error.message));
+      });
+    };
+
+    const handleDisconnectBeforeReady = (reason: string) => {
+      settle(() =>
+        reject(
+          new Error(
+            'Media socket closed before the room was ready (' + reason + ').'
+          )
+        )
+      );
+    };
+
+    socket.on('connection-success', handleConnectionSuccess);
+    socket.on('connect_error', handleConnectError);
+    socket.on('disconnect', handleDisconnectBeforeReady);
+
+    timeoutHandle = setTimeout(() => {
+      settle(() => {
+        try {
+          socket.disconnect();
+        } catch {
+          // Nothing useful to do here.
+        }
+        reject(new Error('Timed out connecting to media socket.'));
+      });
+    }, CONNECT_TIMEOUT_MS);
   });
 }
 

package/src/types/shared-base-types.ts

@@ -240,6 +240,7 @@ export type SeedData = {
 };
 
 export interface MeetingRoomParams {
+  backendAudioDenoise?: { enabled: boolean; profile?: 'arnndn' }; // Optional; omitted uses the account preset.
   itemPageLimit: number;
   mediaType: 'audio' | 'video';
   addCoHost: boolean;
@@ -316,6 +317,7 @@ export type ScreenParamsType = {
 };
 
 export interface CreateRoomOptions {
+  backendAudioDenoise?: { enabled: boolean; profile?: 'arnndn' };
   action: 'create' | 'join';
   meetingID: string;
   duration: number;
@@ -339,6 +341,7 @@ export interface CreateRoomOptions {
 }
 
 export interface CreateMediaSFURoomOptions {
+  backendAudioDenoise?: { enabled: boolean; profile?: 'arnndn' };
   action: 'create';
   duration: number;
   capacity: number;