@stream-io/video-client 1.61.2 → 1.62.0

package/src/devices/__tests__/MicrophoneManager.test.ts

@@ -327,6 +327,241 @@ describe('MicrophoneManager', () => {
       await manager.disable();
       expect(manager['startSpeakingWhileMutedDetection']).toHaveBeenCalled();
     });
+
+    describe('mute hand-off', () => {
+      let micTrack: MediaStreamTrack;
+      let clonedTrack: MediaStreamTrack;
+
+      beforeEach(() => {
+        vi.stubGlobal(
+          'MediaStream',
+          vi.fn(function (this: object, tracks: MediaStreamTrack[]) {
+            Object.assign(this, { getTracks: () => tracks });
+          }),
+        );
+        const stream = mockAudioStream();
+        [micTrack] = stream.getAudioTracks();
+        [clonedTrack] = mockAudioStream().getAudioTracks();
+        micTrack.clone = vi.fn(() => clonedTrack);
+        vi.mocked(getAudioStream).mockResolvedValueOnce(stream);
+        manager.dispose();
+        const devicePersistence = { enabled: false, storageKey: '' };
+        manager = new MicrophoneManager(call, devicePersistence);
+      });
+
+      afterEach(() => {
+        vi.unstubAllGlobals();
+      });
+
+      const enableAndSettle = async () => {
+        await manager.enable();
+        await sleep(25);
+        vi.mocked(micTrack.clone).mockClear();
+        vi.mocked(getAudioStream).mockClear();
+        vi.mocked(createSoundDetector).mockClear();
+      };
+
+      it('keeps the mic captured by handing a clone to the detector', async () => {
+        await enableAndSettle();
+        await manager.disable();
+        await sleep(25);
+
+        expect(micTrack.readyState).toBe('ended');
+        expect(clonedTrack.readyState).toBe('live');
+        expect(createSoundDetector).toHaveBeenCalledTimes(1);
+        const [detectorStream] = vi.mocked(createSoundDetector).mock.calls[0];
+        expect(detectorStream.getTracks()).toEqual([clonedTrack]);
+        expect(getAudioStream).not.toHaveBeenCalled();
+      });
+
+      it('releases the clone after unmute opened the new capture', async () => {
+        await enableAndSettle();
+        const cleanup = vi.fn(async () => {});
+        vi.mocked(createSoundDetector).mockImplementationOnce(() => cleanup);
+        await manager.disable();
+        await sleep(25);
+        await manager.enable();
+        await sleep(25);
+
+        expect(cleanup).toHaveBeenCalledTimes(1);
+        expect(getAudioStream).toHaveBeenCalledTimes(1);
+        const [captureOrder] =
+          vi.mocked(getAudioStream).mock.invocationCallOrder;
+        expect(captureOrder).toBeLessThan(cleanup.mock.invocationCallOrder[0]);
+      });
+
+      it('releases the clone and keeps the mic live when muting fails', async () => {
+        await enableAndSettle();
+        vi.mocked(call.stopPublish).mockRejectedValueOnce(
+          new Error('SFU unavailable'),
+        );
+
+        await expect(manager.disable()).rejects.toThrow('SFU unavailable');
+        await sleep(25);
+
+        expect(manager.state.status).toBe('enabled');
+        expect(micTrack.readyState).toBe('live');
+        expect(clonedTrack.readyState).toBe('ended');
+        expect(createSoundDetector).not.toHaveBeenCalled();
+      });
+
+      it('releases the clone and falls back when the detector cannot start', async () => {
+        await enableAndSettle();
+        vi.mocked(createSoundDetector).mockImplementationOnce(() => {
+          throw new Error('AudioContext unavailable');
+        });
+
+        await manager.disable();
+        await sleep(25);
+
+        expect(manager.state.status).toBe('disabled');
+        expect(clonedTrack.readyState).toBe('ended');
+        expect(getAudioStream).toHaveBeenCalledTimes(1);
+      });
+
+      it('unpublishes without waiting for the detector queue', async () => {
+        await enableAndSettle();
+        let unblock!: () => void;
+        const blocker = new Promise<void>((resolve) => (unblock = resolve));
+        withoutConcurrency(
+          manager['soundDetectorConcurrencyTag'],
+          () => blocker,
+        );
+
+        const muted = manager.disable();
+        await sleep(25);
+
+        expect(call.stopPublish).toHaveBeenCalledWith(TrackType.AUDIO);
+        expect(micTrack.readyState).toBe('ended');
+        expect(clonedTrack.readyState).toBe('live');
+        unblock();
+        await muted;
+      });
+
+      it('does not open a capture before muting an ended mic track', async () => {
+        await enableAndSettle();
+        micTrack.stop();
+
+        await manager.disable();
+        await sleep(25);
+
+        expect(micTrack.clone).not.toHaveBeenCalled();
+        const [unpublished] = vi.mocked(call.stopPublish).mock
+          .invocationCallOrder;
+        vi.mocked(getAudioStream).mock.invocationCallOrder.forEach((order) => {
+          expect(order).toBeGreaterThan(unpublished);
+        });
+      });
+
+      it('releases the handed-off track when a detector already runs', async () => {
+        await manager['startSpeakingWhileMutedDetection']('device-1');
+        await manager['startSpeakingWhileMutedDetection'](
+          'device-1',
+          clonedTrack,
+        );
+
+        expect(clonedTrack.readyState).toBe('ended');
+      });
+
+      const blockUnpublish = () => {
+        let unblock!: () => void;
+        vi.mocked(call.stopPublish).mockImplementationOnce(
+          () => new Promise<void>((resolve) => (unblock = resolve)),
+        );
+        return () => unblock();
+      };
+
+      it('falls back to a new capture when the mic ends while unpublishing', async () => {
+        await enableAndSettle();
+        const unblock = blockUnpublish();
+
+        const muted = manager.disable();
+        await sleep(25);
+        clonedTrack.stop();
+        unblock();
+        await muted;
+        await sleep(25);
+
+        expect(createSoundDetector).toHaveBeenCalledTimes(1);
+        const [detectorStream] = vi.mocked(createSoundDetector).mock.calls[0];
+        expect(detectorStream.getTracks()).not.toContain(clonedTrack);
+        expect(getAudioStream).toHaveBeenCalledTimes(1);
+      });
+
+      it.each([
+        [
+          'the call is left',
+          () => call.state.setCallingState(CallingState.LEFT),
+        ],
+        ['sending audio is revoked', () => call.state.setOwnCapabilities([])],
+        [
+          'the notification is disabled',
+          () => manager.disableSpeakingWhileMutedNotification(),
+        ],
+      ])('releases the clone when %s while unpublishing', async (_, change) => {
+        await enableAndSettle();
+        const unblock = blockUnpublish();
+
+        const muted = manager.disable();
+        await sleep(25);
+        await change();
+        unblock();
+        await muted;
+        await sleep(25);
+
+        expect(clonedTrack.readyState).toBe('ended');
+        expect(createSoundDetector).not.toHaveBeenCalled();
+      });
+
+      it('releases the clone when another mic is selected while unpublishing', async () => {
+        await enableAndSettle();
+        const unblock = blockUnpublish();
+
+        const muted = manager.disable();
+        await sleep(25);
+        const selected = manager.select('other-mic');
+        unblock();
+        await Promise.all([muted, selected]);
+        await sleep(25);
+
+        expect(clonedTrack.readyState).toBe('ended');
+        expect(getAudioStream).toHaveBeenCalledWith(
+          expect.objectContaining({ deviceId: { exact: 'other-mic' } }),
+          expect.any(Tracer),
+        );
+      });
+
+      it('keeps the detector when a stale unmute is handled after muting again', async () => {
+        await enableAndSettle();
+        const cleanup = vi.fn(async () => {});
+        vi.mocked(createSoundDetector).mockImplementationOnce(() => cleanup);
+        let unblock!: () => void;
+        const blocker = new Promise<void>((resolve) => (unblock = resolve));
+        withoutConcurrency(
+          manager['soundDetectorConcurrencyTag'],
+          () => blocker,
+        );
+
+        await manager.disable();
+        await manager.enable();
+        await manager.disable();
+        unblock();
+        await sleep(25);
+
+        expect(manager.state.status).toBe('disabled');
+        expect(cleanup).not.toHaveBeenCalled();
+        expect(clonedTrack.readyState).toBe('live');
+      });
+
+      it('does not hand off the mic when leaving the call', async () => {
+        await enableAndSettle();
+        call.state.setCallingState(CallingState.LEFT);
+        await manager.disable(true);
+
+        expect(micTrack.clone).not.toHaveBeenCalled();
+        expect(micTrack.readyState).toBe('ended');
+      });
+    });
   });
 
   describe('Noise Cancellation', () => {

package/src/devices/__tests__/SpeakerManager.test.ts

@@ -234,6 +234,33 @@ describe('SpeakerManager.test', () => {
       expect(persistedManager.state.selectedDevice).toBe('speaker-1');
     });
 
+    it('does not restore a speaker when output selection is not supported', async () => {
+      const persistedManager = createPersistedManager();
+      Object.defineProperty(
+        persistedManager.state,
+        'isDeviceSelectionSupported',
+        { value: false },
+      );
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of([
+          {
+            deviceId: 'speaker-1',
+            kind: 'audiooutput',
+            label: 'Speaker 1',
+            groupId: 'speaker-group',
+          } as MediaDeviceInfo,
+        ]),
+      );
+      persist('speaker-1', 'Speaker 1');
+
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(selectSpy).not.toHaveBeenCalled();
+      expect(persistedManager.state.selectedDevice).toBe('');
+    });
+
     it('does not select a missing persisted speaker device', async () => {
       const persistedManager = createPersistedManager();
       vi.spyOn(persistedManager, 'listDevices').mockReturnValue(

package/src/devices/__tests__/devices.test.ts

@@ -23,6 +23,7 @@ vi.mock('../BrowserPermission', () => {
 vi.mock('../../helpers/browsers', () => ({
   isSafari: vi.fn(),
   isFirefox: vi.fn(),
+  isWebKit: vi.fn(),
 }));
 
 type MediaDevicesMock = MediaDevices &
@@ -66,11 +67,11 @@ describe('devices', () => {
     vi.resetModules();
   });
 
-  it('checkIfAudioOutputChangeSupported uses audio element when not Safari', async () => {
+  it('checkIfAudioOutputChangeSupported uses audio element when not WebKit', async () => {
     setupMediaDevices();
     const { checkIfAudioOutputChangeSupported } = await loadDevicesModule();
     const browsers = await import('../../helpers/browsers');
-    vi.mocked(browsers.isSafari).mockReturnValue(false);
+    vi.mocked(browsers.isWebKit).mockReturnValue(false);
     const createElementSpy = vi
       .spyOn(document, 'createElement')
       .mockReturnValue({ setSinkId: vi.fn() } as unknown as HTMLMediaElement);
@@ -79,21 +80,16 @@ describe('devices', () => {
     expect(createElementSpy).toHaveBeenCalledWith('audio');
   });
 
-  it('checkIfAudioOutputChangeSupported uses AudioContext in Safari', async () => {
+  it('checkIfAudioOutputChangeSupported is false on WebKit', async () => {
     setupMediaDevices();
-    class AudioContextStub {}
-    (
-      AudioContextStub.prototype as { setSinkId?: () => Promise<void> }
-    ).setSinkId = vi.fn();
-    Object.defineProperty(globalThis, 'AudioContext', {
-      configurable: true,
-      value: AudioContextStub as typeof AudioContext,
-    });
-
     const { checkIfAudioOutputChangeSupported } = await loadDevicesModule();
     const browsers = await import('../../helpers/browsers');
-    vi.mocked(browsers.isSafari).mockReturnValue(true);
-    expect(checkIfAudioOutputChangeSupported()).toBe(true);
+    vi.mocked(browsers.isWebKit).mockReturnValue(true);
+    vi.spyOn(document, 'createElement').mockReturnValue({
+      setSinkId: vi.fn(),
+    } as unknown as HTMLMediaElement);
+
+    expect(checkIfAudioOutputChangeSupported()).toBe(false);
   });
 
   it('getAudioDevices prompts and filters devices', async () => {

package/src/devices/__tests__/mediaStreamTestHelpers.ts

@@ -10,6 +10,7 @@ const createTrack = (
     getSettings: () => settings,
     enabled: true,
     readyState: 'live',
+    clone: () => createTrack(settings, extras),
     stop: () => {
       // @ts-expect-error read-only property
       track.readyState = 'ended';

package/src/devices/__tests__/mocks.ts

@@ -157,6 +157,7 @@ export const mockAudioStream = () => {
     label: mockAudioDevices[0].label,
     enabled: true,
     readyState: 'live',
+    clone: () => mockAudioStream().getAudioTracks()[0],
     stop: () => {
       track.readyState = 'ended';
     },

package/src/devices/devices.ts

@@ -11,7 +11,7 @@ import {
 } from 'rxjs';
 import { BrowserPermission } from './BrowserPermission';
 import { lazy } from '../helpers/lazy';
-import { isFirefox } from '../helpers/browsers';
+import { isFirefox, isWebKit } from '../helpers/browsers';
 import { dumpStream, Tracer } from '../stats';
 import { getCurrentValue } from '../store/rxUtils';
 import { videoLoggerSystem } from '../logger';
@@ -58,9 +58,15 @@ const getDevices = (
 /**
  * Tells if the browser supports audio output change on 'audio' elements,
  * see https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/setSinkId.
+ *
+ * WebKit (Safari, iOS browsers, WKWebView) is reported as unsupported:
+ * setSinkId requires a user gesture, a per-device output can run at a
+ * different sample rate than the remote audio mixer (pitched/slowed audio),
+ * and audio goes silent when the selected device disconnects.
  */
 export const checkIfAudioOutputChangeSupported = () => {
   if (typeof document === 'undefined') return false;
+  if (isWebKit()) return false;
   const element = document.createElement('audio');
   return 'setSinkId' in element;
 };

package/src/helpers/DynascaleManager.ts

@@ -323,7 +323,7 @@ export class DynascaleManager {
       deviceId: string,
       audioContext: AudioContext | undefined,
     ) => {
-      if (!deviceId) return;
+      if (!deviceId || !this.speaker.state.isDeviceSelectionSupported) return;
       if ('setSinkId' in audioElement) {
         audioElement.setSinkId(deviceId).catch((e) => {
           this.logger.warn(`Can't to set AudioElement sinkId`, e);
@@ -430,7 +430,7 @@ export class DynascaleManager {
         });
       });
 
-    const sinkIdSubscription = !('setSinkId' in audioElement)
+    const sinkIdSubscription = !this.speaker.state.isDeviceSelectionSupported
       ? null
       : this.speaker.state.selectedDevice$.subscribe((deviceId) => {
           const audioContext = this.getOrCreateAudioContext();

package/src/helpers/__tests__/DynascaleManager.test.ts

@@ -29,6 +29,7 @@ vi.mock(import('../browsers'), async (importOriginal) => {
   return {
     ...module,
     isSafari: () => globalThis._isSafari ?? false,
+    isWebKit: () => false,
   };
 });
 
@@ -191,6 +192,9 @@ describe('DynascaleManager', () => {
     it('audio: Safari should use AudioContext for audio playback', () => {
       globalThis._isSafari = true;
       dynascaleManager.setUseWebAudio(true); // enabled by default on Safari
+      Object.defineProperty(call.speaker.state, 'isDeviceSelectionSupported', {
+        value: false,
+      });
 
       vi.useFakeTimers();
       const audioElement = document.createElement('audio');
@@ -257,11 +261,9 @@ describe('DynascaleManager', () => {
       expect(gainNode.connect).toHaveBeenCalledWith(audioContext.destination);
 
       call.speaker.select('different-device-id');
-      expect(audioElement.setSinkId).toHaveBeenCalledWith(
-        'different-device-id',
-      );
+      expect(audioElement.setSinkId).not.toHaveBeenCalled();
       // @ts-expect-error sinkId isn't available in the TS definition
-      expect(audioContext.sinkId).toBe('different-device-id');
+      expect(audioContext.sinkId).not.toBe('different-device-id');
 
       call.speaker.setVolume(0.5);
       expect(audioElement.volume).toBe(0.5);

package/src/rtc/BasePeerConnection.ts

@@ -19,6 +19,7 @@ import { toJSON } from './helpers/iceCandiates';
 import {
   BasePeerConnectionOpts,
   OnIceConnected,
+  OnNoLocalCandidates,
   OnPeerConnectionStateChange,
   OnReconnectionNeeded,
   OnRemoteTrackUnmute,
@@ -43,10 +44,12 @@ export abstract class BasePeerConnection {
 
   protected onReconnectionNeeded?: OnReconnectionNeeded;
   private onIceConnected?: OnIceConnected;
+  private onNoLocalCandidates?: OnNoLocalCandidates;
   private onPeerConnectionStateChange?: OnPeerConnectionStateChange;
   protected onRemoteTrackUnmute?: OnRemoteTrackUnmute;
   private readonly iceRestartDelay: number;
   private iceHasEverConnected = false;
+  private localCandidatesGathered = 0;
   private iceRestartTimeout?: NodeJS.Timeout;
   private preConnectStuckTimeout?: NodeJS.Timeout;
   protected isIceRestarting = false;
@@ -73,6 +76,7 @@ export abstract class BasePeerConnection {
       dispatcher,
       onReconnectionNeeded,
       onIceConnected,
+      onNoLocalCandidates,
       onPeerConnectionStateChange,
       onRemoteTrackUnmute,
       tag,
@@ -93,6 +97,7 @@ export abstract class BasePeerConnection {
     this.tag = tag;
     this.onReconnectionNeeded = onReconnectionNeeded;
     this.onIceConnected = onIceConnected;
+    this.onNoLocalCandidates = onNoLocalCandidates;
     this.onPeerConnectionStateChange = onPeerConnectionStateChange;
     this.onRemoteTrackUnmute = onRemoteTrackUnmute;
     this.logger = videoLoggerSystem.getLogger(
@@ -148,6 +153,7 @@ export abstract class BasePeerConnection {
     this.preConnectStuckTimeout = undefined;
     this.onReconnectionNeeded = undefined;
     this.onIceConnected = undefined;
+    this.onNoLocalCandidates = undefined;
     this.onPeerConnectionStateChange = undefined;
     this.onRemoteTrackUnmute = undefined;
     this.isDisposed = true;
@@ -285,6 +291,20 @@ export abstract class BasePeerConnection {
     return this.trackIdToTrackType.get(trackId);
   };
 
+  /**
+   * Returns `true` once ICE has reached `connected`/`completed` at least
+   * once during the lifetime of this peer connection.
+   */
+  hasIceEverConnected = () => this.iceHasEverConnected;
+
+  /**
+   * Returns `true` when the latest ICE gathering round completed without
+   * producing a single local candidate.
+   */
+  hasNoLocalCandidates = () =>
+    this.pc.iceGatheringState === 'complete' &&
+    this.localCandidatesGathered === 0;
+
   /**
    * Checks if the `RTCPeerConnection` is healthy.
    * It checks the ICE connection state and the peer connection state.
@@ -311,6 +331,7 @@ export abstract class BasePeerConnection {
       this.logger.debug('null ice candidate');
       return;
     }
+    if (candidate.candidate) this.localCandidatesGathered++;
 
     const iceCandidate = toJSON(candidate);
     this.sfuClient
@@ -389,6 +410,20 @@ export abstract class BasePeerConnection {
   private handleConnectionStateUpdate = (
     state: RTCIceConnectionState | RTCPeerConnectionState,
   ) => {
+    // Fire `onIceConnected` exactly once per peer-connection lifetime, the
+    // first time ICE reaches `connected`/`completed` end-to-end. Recorded
+    // ahead of the guards below so a connection established while offline,
+    // reconnecting or restarting ICE is not missed. Used by `Call` to reset
+    // the unsupported-network failure counter only after WebRTC has actually
+    // recovered, not merely on SFU join.
+    if (
+      (state === 'connected' || state === 'completed') &&
+      !this.iceHasEverConnected
+    ) {
+      this.iceHasEverConnected = true;
+      this.onIceConnected?.(this.peerType);
+    }
+
     const { callingState } = this.state;
     if (callingState === CallingState.OFFLINE) return;
     if (callingState === CallingState.RECONNECTING) return;
@@ -466,14 +501,6 @@ export abstract class BasePeerConnection {
 
       case 'connected':
       case 'completed':
-        // Fire `onIceConnected` exactly once per peer-connection lifetime —
-        // the first time ICE reaches `connected`/`completed` end-to-end.
-        // Used by `Call` to reset the unsupported-network failure counter
-        // only after WebRTC has actually recovered, not merely on SFU join.
-        if (!this.iceHasEverConnected) {
-          this.iceHasEverConnected = true;
-          this.onIceConnected?.(this.peerType);
-        }
         // clear any scheduled restartICE since the connection is healthy
         if (this.iceRestartTimeout) {
           this.logger.info('connected connection, canceling restartICE');
@@ -502,7 +529,18 @@ export abstract class BasePeerConnection {
    * Handles the ICE gathering state change event.
    */
   private onIceGatherChange = () => {
-    this.logger.debug(`ICE Gathering State`, this.pc.iceGatheringState);
+    const state = this.pc.iceGatheringState;
+    this.logger.debug(`ICE Gathering State`, state);
+    if (state === 'gathering') {
+      this.localCandidatesGathered = 0;
+    } else if (
+      state === 'complete' &&
+      this.localCandidatesGathered === 0 &&
+      !this.iceHasEverConnected
+    ) {
+      this.logger.info('ICE gathering completed without local candidates');
+      this.onNoLocalCandidates?.(this.peerType);
+    }
   };
 
   /**