@stream-io/video-client 1.50.0 → 1.51.0

package/src/devices/VirtualDevice.ts

@@ -0,0 +1,69 @@
+/**
+ * A MediaStream produced for a virtual device session, along with an optional
+ * cleanup callback. Returned by {@link VirtualDevice.getUserMedia}.
+ */
+export interface VirtualDeviceSession {
+  readonly stream: MediaStream;
+  readonly stop?: () => void | Promise<void>;
+}
+
+/**
+ * A virtual camera or microphone definition supplied by the integrator.
+ *
+ * Pass this to `camera.registerVirtualDevice()` /
+ * `microphone.registerVirtualDevice()` to make it appear in the device list
+ * and become selectable.
+ */
+export interface VirtualDevice<C = MediaTrackConstraints> {
+  /**
+   * Human-readable label shown in device dropdowns.
+   */
+  label: string;
+
+  /**
+   * Called when the virtual device is selected and the SDK needs media.
+   * Returns the MediaStream to publish along with an optional `stop`
+   * callback that runs when the session is replaced, the tracks end, or
+   * the device is unregistered.
+   *
+   * `constraints` is the resolved set the SDK would otherwise pass to
+   * `getUserMedia` for a real device.
+   */
+  getUserMedia: (
+    constraints: C,
+  ) => VirtualDeviceSession | Promise<VirtualDeviceSession>;
+}
+
+/**
+ * @internal Internal entry stored in the device manager's registry.
+ */
+export interface VirtualDeviceEntry<
+  C = MediaTrackConstraints,
+> extends VirtualDevice<C> {
+  readonly deviceId: string;
+  readonly kind: 'audioinput' | 'videoinput';
+}
+
+/**
+ * @internal Tracks the currently active virtual device session inside the
+ * device manager so its `stop` callback can be invoked when the session is
+ * replaced or torn down.
+ */
+export interface ActiveVirtualSession {
+  deviceId: string;
+  stop?: () => void | Promise<void>;
+}
+
+export interface VirtualDeviceHandle {
+  /**
+   * The device id under which the virtual device was registered. Pass this
+   * to `camera.select()` / `microphone.select()` to switch to it.
+   */
+  readonly deviceId: string;
+
+  /**
+   * Removes the virtual device from the manager. If it is currently selected,
+   * the selection is reset so the SDK falls back to the default device.
+   */
+  unregister: () => Promise<void>;
+}

package/src/devices/__tests__/CameraManager.test.ts

@@ -210,6 +210,25 @@ describe('CameraManager', () => {
     });
   });
 
+  it('should pass resolved camera constraints to virtual devices', async () => {
+    const virtualStream = mockVideoStream();
+    const getUserMedia = vi.fn(() => ({ stream: virtualStream }));
+
+    const { deviceId } = manager.registerVirtualDevice({
+      label: 'Virtual camera',
+      getUserMedia,
+    });
+
+    await manager.select(deviceId);
+    await manager.enable();
+
+    expect(getUserMedia).toHaveBeenCalledWith({
+      deviceId: { exact: deviceId },
+      width: 1280,
+      height: 720,
+    });
+  });
+
   it(`should set target resolution, but shouldn't change device status`, async () => {
     manager['targetResolution'] = { width: 640, height: 480 };
 

package/src/devices/__tests__/DeviceManager.test.ts

@@ -17,7 +17,7 @@ import {
 } from './mocks';
 import { DeviceManager } from '../DeviceManager';
 import { DeviceManagerState } from '../DeviceManagerState';
-import { of } from 'rxjs';
+import { firstValueFrom, of } from 'rxjs';
 import { TrackType } from '../../gen/video/sfu/models/models';
 import { PermissionsContext } from '../../permissions';
 import { readPreferences } from '../devicePersistence';
@@ -221,6 +221,126 @@ describe('Device Manager', () => {
     expect(spy.mock.calls.length).toBe(1);
   });
 
+  it('should use a virtual device stream factory instead of requesting a real device stream', async () => {
+    const virtualStream = mockVideoStream();
+    const getUserMedia = vi.fn(() => ({ stream: virtualStream }));
+
+    const { deviceId } = manager.registerVirtualDevice({
+      label: 'Virtual camera',
+      getUserMedia,
+    });
+
+    await manager.select(deviceId);
+    await manager.enable();
+
+    expect(getUserMedia).toHaveBeenCalledOnce();
+    expect(getUserMedia).toHaveBeenCalledWith({
+      deviceId: { exact: deviceId },
+    });
+    expect(manager.getStream).not.toHaveBeenCalled();
+    expect(manager.state.mediaStream).toBe(virtualStream);
+    expect(manager.state.selectedDevice).toBe(deviceId);
+  });
+
+  it('should call virtual device stop when switching away from it', async () => {
+    const stop = vi.fn();
+    const virtualStream = mockVideoStream();
+
+    const { deviceId } = manager.registerVirtualDevice({
+      label: 'Virtual camera',
+      getUserMedia: vi.fn(() => ({ stream: virtualStream, stop })),
+    });
+
+    await manager.select(deviceId);
+    await manager.enable();
+    await manager.select(mockVideoDevices[1].deviceId);
+
+    expect(stop).toHaveBeenCalledTimes(1);
+  });
+
+  it('should support an async getUserMedia returning a Promise', async () => {
+    const virtualStream = mockVideoStream();
+    const getUserMedia = vi.fn(() =>
+      Promise.resolve({ stream: virtualStream }),
+    );
+
+    const { deviceId } = manager.registerVirtualDevice({
+      label: 'Async virtual camera',
+      getUserMedia,
+    });
+
+    await manager.select(deviceId);
+    await manager.enable();
+
+    expect(getUserMedia).toHaveBeenCalledOnce();
+    expect(manager.state.mediaStream).toBe(virtualStream);
+    expect(manager.state.selectedDevice).toBe(deviceId);
+  });
+
+  it('should roll back selection when getUserMedia rejects', async () => {
+    const failure = new Error('factory boom');
+    const getUserMedia = vi.fn(() => Promise.reject(failure));
+
+    await manager.enable();
+    const previousDevice = manager.state.selectedDevice;
+
+    const { deviceId } = manager.registerVirtualDevice({
+      label: 'Failing camera',
+      getUserMedia,
+    });
+
+    await expect(manager.select(deviceId)).rejects.toThrow(failure);
+
+    expect(manager.state.selectedDevice).toBe(previousDevice);
+  });
+
+  it('should stop the active session and clear selection on unregister', async () => {
+    const stop = vi.fn();
+    const virtualStream = mockVideoStream();
+
+    const { deviceId, unregister } = manager.registerVirtualDevice({
+      label: 'Virtual camera',
+      getUserMedia: vi.fn(() => ({ stream: virtualStream, stop })),
+    });
+
+    await manager.select(deviceId);
+    await manager.enable();
+
+    await unregister();
+
+    expect(stop).toHaveBeenCalledTimes(1);
+    expect(manager.state.selectedDevice).not.toBe(deviceId);
+  });
+
+  it('should remove the entry on unregister without stopping when not selected', async () => {
+    const stop = vi.fn();
+    const getUserMedia = vi.fn(() => ({ stream: mockVideoStream(), stop }));
+
+    const { unregister } = manager.registerVirtualDevice({
+      label: 'Unused virtual camera',
+      getUserMedia,
+    });
+
+    await unregister();
+
+    expect(stop).not.toHaveBeenCalled();
+    expect(getUserMedia).not.toHaveBeenCalled();
+  });
+
+  it('should expose virtual devices via listDevices() with the provided label', async () => {
+    manager.registerVirtualDevice({
+      label: 'My virtual camera',
+      getUserMedia: vi.fn(() => ({ stream: mockVideoStream() })),
+    });
+
+    const devices = await firstValueFrom(manager.listDevices());
+
+    expect(devices.length).toBe(mockVideoDevices.length + 1);
+    const virtual = devices.find((d) => d.label === 'My virtual camera');
+    expect(virtual).toBeDefined();
+    expect(virtual?.kind).toBe('videoinput');
+  });
+
   it('should resume previously enabled state', async () => {
     vi.spyOn(manager, 'enable');
 

package/src/devices/devicePersistence.ts

@@ -48,8 +48,9 @@ export const normalize = (
 export const createSyntheticDevice = (
   deviceId: string,
   kind: MediaDeviceKind,
+  label = '',
 ): MediaDeviceInfo => {
-  return { deviceId, kind, label: '', groupId: '' } as MediaDeviceInfo;
+  return { deviceId, kind, label, groupId: '' } as MediaDeviceInfo;
 };
 
 export const readPreferences = (storageKey: string): LocalDevicePreferences => {

package/src/devices/index.ts

@@ -10,3 +10,4 @@ export * from './ScreenShareManager';
 export * from './ScreenShareState';
 export * from './SpeakerManager';
 export * from './SpeakerState';
+export * from './VirtualDevice';

package/src/rtc/BasePeerConnection.ts

@@ -42,7 +42,7 @@ export abstract class BasePeerConnection {
   private iceRestartTimeout?: NodeJS.Timeout;
   private preConnectStuckTimeout?: NodeJS.Timeout;
   protected isIceRestarting = false;
-  private isDisposed = false;
+  protected isDisposed = false;
 
   protected trackIdToTrackType = new Map<string, TrackType>();
 
@@ -115,7 +115,7 @@ export abstract class BasePeerConnection {
   /**
    * Disposes the `RTCPeerConnection` instance.
    */
-  dispose() {
+  async dispose(): Promise<void> {
     clearTimeout(this.iceRestartTimeout);
     this.iceRestartTimeout = undefined;
     clearTimeout(this.preConnectStuckTimeout);
@@ -141,6 +141,10 @@ export abstract class BasePeerConnection {
       this.onIceConnectionStateChange,
     );
     pc.removeEventListener('icegatheringstatechange', this.onIceGatherChange);
+    pc.removeEventListener(
+      'connectionstatechange',
+      this.onConnectionStateChange,
+    );
     this.unsubscribeIceTrickle?.();
     this.subscriptions.forEach((unsubscribe) => unsubscribe());
     this.subscriptions = [];
@@ -183,7 +187,7 @@ export abstract class BasePeerConnection {
     const getTag = () => this.tag;
     this.subscriptions.push(
       this.dispatcher.on(event, getTag, (e) => {
-        const lockKey = `pc.${this.lock}.${event}`;
+        const lockKey = this.eventLockKey(event);
         withoutConcurrency(lockKey, async () => fn(e)).catch((err) => {
           if (this.isDisposed) return;
           this.logger.warn(`Error handling ${event}`, err);
@@ -192,6 +196,14 @@ export abstract class BasePeerConnection {
     );
   };
 
+  /**
+   * Returns the per-event `withoutConcurrency` tag used to serialize the
+   * dispatcher handler for `event` on this peer connection.
+   */
+  protected eventLockKey = (event: keyof AllSfuEvents): string => {
+    return `pc.${this.lock}.${event}`;
+  };
+
   /**
    * Appends the trickled ICE candidates to the `RTCPeerConnection`.
    */

package/src/rtc/Publisher.ts

@@ -20,11 +20,15 @@ import {
   toVideoLayers,
 } from './layers';
 import { isSvcCodec } from './codecs';
-import { toRTCDegradationPreference } from './helpers/degradationPreference';
+import {
+  fromRTCDegradationPreference,
+  toRTCDegradationPreference,
+} from './helpers/degradationPreference';
 import { isAudioTrackType } from './helpers/tracks';
 import { extractMid, removeCodecsExcept, setStartBitrate } from './helpers/sdp';
 import { withoutConcurrency } from '../helpers/concurrency';
 import { isReactNative } from '../helpers/platforms';
+import { isFirefox } from '../helpers/browsers';
 
 /**
  * The `Publisher` is responsible for publishing/unpublishing media streams to/from the SFU
@@ -53,7 +57,16 @@ export class Publisher extends BasePeerConnection {
 
     this.on('changePublishQuality', async (event) => {
       for (const videoSender of event.videoSenders) {
-        await this.changePublishQuality(videoSender);
+        // if not publishing, update the encodingConfigCache and don't modify the state.
+        // we'll apply this config on the next publish/unmute.
+        const { trackType, publishOptionId } = videoSender;
+        const bundle = this.transceiverCache.getBy(publishOptionId, trackType);
+        if (bundle) {
+          this.transceiverCache.update(bundle.publishOption, { videoSender });
+        }
+        if (isFirefox() && !this.isPublishing(trackType)) continue;
+
+        await this.changePublishQuality(videoSender, bundle);
       }
     });
 
@@ -66,9 +79,13 @@ export class Publisher extends BasePeerConnection {
   /**
    * Disposes this Publisher instance.
    */
-  dispose() {
-    super.dispose();
-    this.stopAllTracks();
+  async dispose(): Promise<void> {
+    await super.dispose();
+    try {
+      await this.stopAllTracks();
+    } catch (err) {
+      this.logger.warn('Failed to stop tracks during dispose', err);
+    }
     this.clonedTracks.clear();
   }
 
@@ -98,17 +115,12 @@ export class Publisher extends BasePeerConnection {
       // appear in the SDP in multiple transceivers
       const trackToPublish = this.cloneTrack(track);
 
-      const { transceiver } = this.transceiverCache.get(publishOption) || {};
-      if (!transceiver) {
+      const bundle = this.transceiverCache.get(publishOption);
+      if (!bundle) {
         await this.addTransceiver(trackToPublish, publishOption, options);
       } else {
-        const previousTrack = transceiver.sender.track;
-        await this.updateTransceiver(
-          transceiver,
-          trackToPublish,
-          trackType,
-          options,
-        );
+        const previousTrack = bundle.transceiver.sender.track;
+        await this.updateTransceiver(bundle, trackToPublish, options);
         if (!isReactNative()) {
           this.stopTrack(previousTrack);
         }
@@ -153,15 +165,22 @@ export class Publisher extends BasePeerConnection {
    * Updates the transceiver with the given track and track type.
    */
   private updateTransceiver = async (
-    transceiver: RTCRtpTransceiver,
+    bundle: PublishBundle,
     track: MediaStreamTrack | null,
-    trackType: TrackType,
     options: TrackPublishOptions = {},
   ) => {
+    const { transceiver, publishOption } = bundle;
+    const trackType = publishOption.trackType;
     const sender = transceiver.sender;
     if (sender.track) this.trackIdToTrackType.delete(sender.track.id);
     await sender.replaceTrack(track);
-    if (track) this.trackIdToTrackType.set(track.id, trackType);
+    if (track) {
+      this.trackIdToTrackType.set(track.id, trackType);
+      if (isFirefox() && bundle.videoSender) {
+        // restore the encoding config from the cache, if any
+        await this.changePublishQuality(bundle.videoSender, bundle);
+      }
+    }
     if (isAudioTrackType(trackType)) {
       await this.updateAudioPublishOptions(trackType, options);
     }
@@ -230,7 +249,7 @@ export class Publisher extends BasePeerConnection {
       if (hasPublishOption) continue;
       // it is safe to stop the track here, it is a clone
       this.stopTrack(transceiver.sender.track);
-      await this.updateTransceiver(transceiver, null, publishOption.trackType);
+      await this.updateTransceiver(item, null);
     }
   };
 
@@ -286,39 +305,50 @@ export class Publisher extends BasePeerConnection {
   /**
    * Stops the cloned track that is being published to the SFU.
    */
-  stopTracks = (...trackTypes: TrackType[]) => {
-    for (const item of this.transceiverCache.items()) {
-      const { publishOption, transceiver } = item;
-      if (!trackTypes.includes(publishOption.trackType)) continue;
-      this.stopTrack(transceiver.sender.track);
-    }
+  stopTracks = async (...trackTypes: TrackType[]) => {
+    return withoutConcurrency(
+      this.eventLockKey('changePublishQuality'),
+      async () => {
+        for (const item of this.transceiverCache.items()) {
+          const { publishOption, transceiver } = item;
+          if (!trackTypes.includes(publishOption.trackType)) continue;
+          const track = transceiver.sender.track;
+          await this.silenceSenderOnFirefox(item);
+          this.stopTrack(track);
+        }
+      },
+    );
   };
 
   /**
    * Stops all the cloned tracks that are being published to the SFU.
    */
-  stopAllTracks = () => {
-    for (const { transceiver } of this.transceiverCache.items()) {
-      this.stopTrack(transceiver.sender.track);
-    }
-    for (const track of this.clonedTracks) {
-      this.stopTrack(track);
-    }
+  stopAllTracks = async () => {
+    return withoutConcurrency(
+      this.eventLockKey('changePublishQuality'),
+      async () => {
+        for (const item of this.transceiverCache.items()) {
+          const track = item.transceiver.sender.track;
+          await this.silenceSenderOnFirefox(item);
+          this.stopTrack(track);
+        }
+        for (const track of this.clonedTracks) {
+          this.stopTrack(track);
+        }
+      },
+    );
   };
 
-  private changePublishQuality = async (videoSender: VideoSender) => {
-    const { trackType, layers, publishOptionId } = videoSender;
-    const enabledLayers = layers.filter((l) => l.active);
+  private changePublishQuality = async (
+    videoSender: VideoSender,
+    bundle: PublishBundle | undefined,
+  ) => {
+    const enabledLayers = videoSender.layers.filter((l) => l.active);
 
     const tag = 'Update publish quality:';
     this.logger.info(`${tag} requested layers by SFU:`, enabledLayers);
 
-    const transceiverId = this.transceiverCache.find(
-      (t) =>
-        t.publishOption.id === publishOptionId &&
-        t.publishOption.trackType === trackType,
-    );
-    const sender = transceiverId?.transceiver.sender;
+    const sender = bundle?.transceiver.sender;
     if (!sender) {
       return this.logger.warn(`${tag} no video sender found.`);
     }
@@ -328,7 +358,7 @@ export class Publisher extends BasePeerConnection {
       return this.logger.warn(`${tag} there are no encodings set.`);
     }
 
-    const codecInUse = transceiverId?.publishOption.codec?.name;
+    const codecInUse = bundle?.publishOption.codec?.name;
     const usesSvcCodec = codecInUse && isSvcCodec(codecInUse);
 
     let changed = false;
@@ -560,4 +590,73 @@ export class Publisher extends BasePeerConnection {
     track.stop();
     this.clonedTracks.delete(track);
   };
+
+  /**
+   * Silences a Firefox sender on the wire during unpublish.
+   *
+   * Firefox keeps emitting RTP after track.stop(), but the right lever
+   * differs by track type:
+   *   - audio: `replaceTrack(null)` is the only reliable silencer;
+   *     `setParameters({encodings:[...active:false]})` does NOT stop
+   *     the Opus encoder.
+   *   - video: `setParameters({encodings:[...active:false]})` pauses
+   *     the encoder; `replaceTrack(null)` does NOT reliably stop the
+   *     video encoder. The prior active=true configuration is captured
+   *     onto `bundle.videoSender` so `updateTransceiver` can restore
+   *     it on the next publish.
+   *
+   * No-op on non-Firefox browsers and during teardown.
+   */
+  private silenceSenderOnFirefox = async (bundle: PublishBundle) => {
+    if (this.isDisposed || !isFirefox()) return;
+    const { transceiver, publishOption } = bundle;
+    if (isAudioTrackType(publishOption.trackType)) {
+      await transceiver.sender.replaceTrack(null).catch((err) => {
+        this.logger.warn('Failed to clear audio sender track', err);
+      });
+      return;
+    }
+    await this.disableAllEncodings(bundle);
+  };
+
+  private disableAllEncodings = async (bundle: PublishBundle) => {
+    const { transceiver, publishOption } = bundle;
+    const sender = transceiver.sender;
+    const params = sender.getParameters();
+    if (!params.encodings || params.encodings.length === 0) return;
+
+    if (!bundle.videoSender) {
+      this.transceiverCache.update(publishOption, {
+        videoSender: {
+          trackType: publishOption.trackType,
+          publishOptionId: publishOption.id,
+          codec: publishOption.codec,
+          degradationPreference: fromRTCDegradationPreference(
+            params.degradationPreference,
+          ),
+          layers: params.encodings.map((e) => ({
+            name: e.rid ?? 'q',
+            active: e.active ?? true,
+            maxBitrate: e.maxBitrate ?? 0,
+            scaleResolutionDownBy: e.scaleResolutionDownBy ?? 0,
+            maxFramerate: e.maxFramerate ?? 0,
+            // @ts-expect-error scalabilityMode is not in the typedefs yet
+            scalabilityMode: e.scalabilityMode ?? '',
+          })),
+        },
+      });
+    }
+
+    let changed = false;
+    for (const encoding of params.encodings) {
+      if (encoding.active !== false) {
+        encoding.active = false;
+        changed = true;
+      }
+    }
+    if (!changed) return;
+    await sender.setParameters(params).catch((err) => {
+      this.logger.error('Failed to disable video sender encodings:', err);
+    });
+  };
 }

package/src/rtc/TransceiverCache.ts

@@ -1,4 +1,4 @@
-import { PublishOption } from '../gen/video/sfu/models/models';
+import { PublishOption, TrackType } from '../gen/video/sfu/models/models';
 import type { OptimalVideoLayer } from './layers';
 import type { PublishBundle, TrackLayersCache } from './types';
 
@@ -25,10 +25,17 @@ export class TransceiverCache {
    * Gets the transceiver for the given publish option.
    */
   get = (publishOption: PublishOption): PublishBundle | undefined => {
+    return this.getBy(publishOption.id, publishOption.trackType);
+  };
+
+  /**
+   * Gets the transceiver for the given publish option id and track type.
+   */
+  getBy = (publishOptionId: number, trackType: TrackType) => {
     return this.cache.find(
       (bundle) =>
-        bundle.publishOption.id === publishOption.id &&
-        bundle.publishOption.trackType === publishOption.trackType,
+        bundle.publishOption.id === publishOptionId &&
+        bundle.publishOption.trackType === trackType,
     );
   };