@stream-io/video-client 1.57.0 → 1.58.0

package/src/devices/SpeakerManager.ts

@@ -1,4 +1,4 @@
-import { combineLatest } from 'rxjs';
+import { combineLatest, firstValueFrom, pairwise } from 'rxjs';
 import { Call } from '../Call';
 import { isReactNative } from '../helpers/platforms';
 import { SpeakerState } from './SpeakerState';
@@ -39,11 +39,15 @@ export class SpeakerManager {
     this.setup();
   }
 
-  apply(settings: CallSettingsResponse) {
-    return isReactNative() ? this.applyRN(settings) : this.applyWeb();
+  async apply(settings: CallSettingsResponse): Promise<void> {
+    if (isReactNative()) {
+      this.applyRN(settings);
+      return;
+    }
+    await this.applyWeb();
   }
 
-  private applyWeb() {
+  private async applyWeb() {
     const { enabled, storageKey } = this.devicePersistence;
     if (!enabled) return;
 
@@ -56,8 +60,26 @@ export class SpeakerManager {
       preference.selectedDeviceId === defaultDeviceId
         ? ''
         : preference.selectedDeviceId;
-    if (this.state.selectedDevice !== nextDeviceId) {
-      this.select(nextDeviceId);
+    if (!nextDeviceId) {
+      if (this.state.selectedDevice !== nextDeviceId) {
+        this.select(nextDeviceId);
+      }
+      return;
+    }
+
+    const permissionState = await firstValueFrom(
+      getAudioBrowserPermission(this.call.tracer).asStateObservable(),
+    );
+    if (permissionState !== 'granted') return;
+
+    const devices = await firstValueFrom(this.listDevices());
+    const device =
+      this.findDevice(devices, nextDeviceId) ??
+      (preference.selectedDeviceLabel
+        ? devices.find((d) => d.label === preference.selectedDeviceLabel)
+        : undefined);
+    if (device && this.state.selectedDevice !== device.deviceId) {
+      this.select(device.deviceId);
     }
   }
 
@@ -90,6 +112,7 @@ export class SpeakerManager {
       globalThis.streamRNVideoSDK?.callManager.setup({
         defaultDevice,
         isRingingTypeCall: this.call.ringing,
+        cid: this.call.cid,
       });
     }
   }
@@ -101,13 +124,16 @@ export class SpeakerManager {
     if (deviceIds$ && !isReactNative()) {
       this.subscriptions.push(
         createSubscription(
-          combineLatest([deviceIds$, this.state.selectedDevice$]),
-          ([devices, deviceId]) => {
+          combineLatest([
+            deviceIds$!.pipe(pairwise()),
+            this.state.selectedDevice$,
+          ]),
+          ([[prevDevices, currentDevices], deviceId]) => {
             if (!deviceId) return;
-            const device = devices.find(
-              (d) => d.deviceId === deviceId && d.kind === 'audiooutput',
-            );
-            if (!device) this.select('');
+            const isDisconnected =
+              this.findDevice(prevDevices, deviceId) &&
+              !this.findDevice(currentDevices, deviceId);
+            if (isDisconnected) this.select('');
           },
         ),
       );
@@ -164,6 +190,7 @@ export class SpeakerManager {
     this.subscriptions.forEach((unsubscribe) => unsubscribe());
     this.subscriptions = [];
     this.areSubscriptionsSetUp = false;
+    this.defaultDevice = undefined;
   };
 
   /**
@@ -201,6 +228,9 @@ export class SpeakerManager {
     });
   }
 
+  private findDevice = (devices: MediaDeviceInfo[], deviceId: string) =>
+    devices.find((d) => d.deviceId === deviceId && d.kind === 'audiooutput');
+
   private persistSpeakerDevicePreference(selectedDevice: string) {
     const { storageKey } = this.devicePersistence;
     const devices = getCurrentValue(this.listDevices()) || [];

package/src/devices/__tests__/MicrophoneManagerRN.test.ts

@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { MicrophoneManager } from '../MicrophoneManager';
 import { Call } from '../../Call';
 import { StreamClient } from '../../coordinator/connection/client';
-import { CallingState, StreamVideoWriteableStateStore } from '../../store';
+import { StreamVideoWriteableStateStore } from '../../store';
 import {
   mockAudioDevices,
   mockAudioStream,
@@ -263,25 +263,6 @@ describe('MicrophoneManager React Native', () => {
     expect(setMutedRecordingPreparedMock).not.toHaveBeenCalledWith(true);
   });
 
-  it('should release prepared muted recording when the call is left', async () => {
-    await manager.disable();
-    await vi.waitUntil(
-      () =>
-        setMutedRecordingPreparedMock.mock.calls.some(([arg]) => arg === true),
-      { timeout: 100 },
-    );
-    setMutedRecordingPreparedMock.mockClear();
-
-    manager['call'].state.setCallingState(CallingState.LEFT);
-
-    await vi.waitUntil(
-      () =>
-        setMutedRecordingPreparedMock.mock.calls.some(([arg]) => arg === false),
-      { timeout: 100 },
-    );
-    expect(setMutedRecordingPreparedMock).toHaveBeenCalledWith(false);
-  });
-
   afterEach(() => {
     globalThis.streamRNVideoSDK = undefined;
     vi.clearAllMocks();

package/src/devices/__tests__/SpeakerManager.test.ts

@@ -6,8 +6,10 @@ import {
   emitDeviceIds,
   LocalStorageMock,
   mockAudioDevices,
+  mockAudioOutputDevices,
   mockBrowserPermission,
   mockDeviceIds$,
+  mockDevicesWithoutAudioPermission,
 } from './mocks';
 import { of } from 'rxjs';
 import { SpeakerManager } from '../SpeakerManager';
@@ -122,15 +124,36 @@ describe('SpeakerManager.test', () => {
   });
 
   it('should disable device if selected device is disconnected', () => {
-    emitDeviceIds(mockAudioDevices);
-    const deviceId = mockAudioDevices[1].deviceId;
+    const deviceId = mockAudioOutputDevices[0].deviceId;
+    emitDeviceIds([...mockAudioDevices, ...mockAudioOutputDevices]);
     manager.select(deviceId);
 
-    emitDeviceIds(mockAudioDevices.slice(2));
+    emitDeviceIds([...mockAudioDevices, ...mockAudioOutputDevices.slice(1)]);
 
     expect(manager.state.selectedDevice).toBe('');
   });
 
+  it('should keep the selection when the device list never exposed audio ids', () => {
+    const deviceId = mockAudioOutputDevices[0].deviceId;
+    emitDeviceIds(mockDevicesWithoutAudioPermission);
+
+    manager.select(deviceId);
+    expect(manager.state.selectedDevice).toBe(deviceId);
+
+    emitDeviceIds(mockDevicesWithoutAudioPermission);
+    expect(manager.state.selectedDevice).toBe(deviceId);
+  });
+
+  it('should keep the selection when the device appears in a later enumeration', () => {
+    const deviceId = mockAudioOutputDevices[0].deviceId;
+    emitDeviceIds(mockDevicesWithoutAudioPermission);
+    manager.select(deviceId);
+
+    emitDeviceIds([...mockAudioDevices, ...mockAudioOutputDevices]);
+
+    expect(manager.state.selectedDevice).toBe(deviceId);
+  });
+
   it('persists speaker selection when permission is granted', async () => {
     const streamClient = new StreamClient('abc123');
     const persistedManager = new SpeakerManager(
@@ -159,16 +182,9 @@ describe('SpeakerManager.test', () => {
   });
 
   describe('apply (web)', () => {
-    it('does nothing when persistence is disabled', () => {
-      const selectSpy = vi.spyOn(manager, 'select');
-      // @ts-expect-error - partial data
-      manager.apply({});
-      expect(selectSpy).not.toHaveBeenCalled();
-    });
-
-    it('selects the persisted speaker device', () => {
+    const createPersistedManager = () => {
       const streamClient = new StreamClient('abc123');
-      const persistedManager = new SpeakerManager(
+      return new SpeakerManager(
         new Call({
           id: '',
           type: '',
@@ -178,56 +194,136 @@ describe('SpeakerManager.test', () => {
         }),
         { enabled: true, storageKey },
       );
+    };
 
+    const persist = (selectedDeviceId: string, selectedDeviceLabel: string) => {
       localStorageMock.setItem(
         storageKey,
         JSON.stringify({
-          speaker: [
-            {
-              selectedDeviceId: 'speaker-1',
-              selectedDeviceLabel: 'Speaker 1',
-            },
-          ],
+          speaker: [{ selectedDeviceId, selectedDeviceLabel }],
         }),
       );
+    };
+
+    it('does nothing when persistence is disabled', async () => {
+      const selectSpy = vi.spyOn(manager, 'select');
+      // @ts-expect-error - partial data
+      await manager.apply({});
+      expect(selectSpy).not.toHaveBeenCalled();
+    });
+
+    it('selects the persisted speaker device', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of([
+          {
+            deviceId: 'speaker-1',
+            kind: 'audiooutput',
+            label: 'Speaker 1',
+            groupId: 'speaker-group',
+          } as MediaDeviceInfo,
+        ]),
+      );
+      persist('speaker-1', 'Speaker 1');
 
       const selectSpy = vi.spyOn(persistedManager, 'select');
       // @ts-expect-error - partial data
-      persistedManager.apply({});
+      await persistedManager.apply({});
 
       expect(selectSpy).toHaveBeenCalledWith('speaker-1');
       expect(persistedManager.state.selectedDevice).toBe('speaker-1');
     });
 
-    it('selects system default when persisted device is default', () => {
-      const streamClient = new StreamClient('abc123');
-      const persistedManager = new SpeakerManager(
-        new Call({
-          id: '',
-          type: '',
-          streamClient,
-          clientEventReporter: new ClientEventReporter({ streamClient }),
-          clientStore: new StreamVideoWriteableStateStore(),
-        }),
-        { enabled: true, storageKey },
+    it('does not select a missing persisted speaker device', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of(mockAudioOutputDevices),
       );
-      persistedManager.select('previous-device');
+      persist('speaker-1', 'Speaker 1');
 
-      localStorageMock.setItem(
-        storageKey,
-        JSON.stringify({
-          speaker: [
-            {
-              selectedDeviceId: defaultDeviceId,
-              selectedDeviceLabel: '',
-            },
-          ],
-        }),
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(selectSpy).not.toHaveBeenCalled();
+      expect(persistedManager.state.selectedDevice).toBe('');
+    });
+
+    it('does not restore a speaker preference when audio permission is not granted', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(mockBrowserPermission, 'asStateObservable').mockReturnValue(
+        of('prompt'),
+      );
+      const listDevicesSpy = vi.spyOn(persistedManager, 'listDevices');
+      persist('speaker-1', 'Speaker 1');
+
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(listDevicesSpy).not.toHaveBeenCalled();
+      expect(selectSpy).not.toHaveBeenCalled();
+      expect(persistedManager.state.selectedDevice).toBe('');
+    });
+
+    it('does not restore a speaker preference from a non-output device with the same label', async () => {
+      const persistedManager = createPersistedManager();
+      const videoDevice = {
+        deviceId: 'video-device-id',
+        kind: 'videoinput',
+        label: 'RODECaster Video (19f7:006b)',
+        groupId: 'video-group',
+      } as MediaDeviceInfo;
+      const outputDevice = {
+        deviceId: 'speaker-device-id',
+        kind: 'audiooutput',
+        label: 'RODECaster Video (19f7:006b)',
+        groupId: 'speaker-group',
+      } as MediaDeviceInfo;
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of([outputDevice]),
       );
+      persist('missing-speaker-device-id', outputDevice.label);
+
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(selectSpy).toHaveBeenCalledWith(outputDevice.deviceId);
+      expect(selectSpy).not.toHaveBeenCalledWith(videoDevice.deviceId);
+      expect(persistedManager.state.selectedDevice).toBe(outputDevice.deviceId);
+    });
+
+    it('does not restore a speaker preference from an empty label placeholder', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of([
+          {
+            deviceId: '',
+            kind: 'audiooutput',
+            label: '',
+            groupId: '',
+          } as MediaDeviceInfo,
+        ]),
+      );
+      persist('missing-speaker-device-id', '');
+
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(selectSpy).not.toHaveBeenCalled();
+      expect(persistedManager.state.selectedDevice).toBe('');
+    });
+
+    it('selects system default when persisted device is default', async () => {
+      const persistedManager = createPersistedManager();
+      persistedManager.select('previous-device');
+      persist(defaultDeviceId, '');
 
       const selectSpy = vi.spyOn(persistedManager, 'select');
       // @ts-expect-error - partial data
-      persistedManager.apply({});
+      await persistedManager.apply({});
 
       expect(selectSpy).toHaveBeenCalledWith('');
       expect(persistedManager.state.selectedDevice).toBe('');

package/src/devices/__tests__/mocks.ts

@@ -6,7 +6,7 @@ import {
   OwnCapability,
 } from '../../gen/coordinator';
 import { Call } from '../../Call';
-import { of, Subject } from 'rxjs';
+import { of, ReplaySubject } from 'rxjs';
 import { BrowserPermission } from '../BrowserPermission';
 import { Tracer } from '../../stats';
 
@@ -71,6 +71,35 @@ export const mockAudioDevices = [
   },
 ] as MediaDeviceInfo[];
 
+export const mockAudioOutputDevices = [
+  {
+    deviceId:
+      'f1602a62d9d363c0711de07182751b646b87cfc0edd9e89d2849289e49304158',
+    kind: 'audiooutput',
+    label: 'RODECaster Video (19f7:006b)',
+    groupId: '27f43ef815ab224429532d6e8ebbeb00d7a895e0cbb0c2e4fcbe5b370f234259',
+  },
+  {
+    deviceId:
+      'd542885e6a47d7a38837aee0323bd2c50e825971690f47295cb338e1f1061283',
+    kind: 'audiooutput',
+    label: 'MacBook Pro Speakers (Built-in)',
+    groupId: 'a8708545cb3969c0b7e3712d2ffdbc381992aeb5577b06cd71ba7a94ee20cf3e',
+  },
+] as MediaDeviceInfo[];
+
+export const mockDevicesWithoutAudioPermission = [
+  { deviceId: '', kind: 'audioinput', label: '', groupId: '' },
+  {
+    deviceId:
+      'f10561a0bbd3effbd25645e7f06fc2e261255afeb05f6bc4fdaba3ed7bd9c2b7',
+    kind: 'videoinput',
+    label: 'RODECaster Video (19f7:006b)',
+    groupId: '27f43ef815ab224429532d6e8ebbeb00d7a895e0cbb0c2e4fcbe5b370f234259',
+  },
+  { deviceId: '', kind: 'audiooutput', label: '', groupId: '' },
+] as MediaDeviceInfo[];
+
 export const mockCall = (): Partial<Call> => {
   const callState = new CallState();
   callState.setCallingState(CallingState.JOINED);
@@ -97,6 +126,7 @@ export const mockCall = (): Partial<Call> => {
   return {
     cid: 'default:test-call',
     state: callState,
+    hasMediaEngine: true,
     publish: vi.fn(),
     stopPublish: vi.fn(),
     streamClient: fromPartial({
@@ -106,6 +136,9 @@ export const mockCall = (): Partial<Call> => {
     notifyNoiseCancellationStopped: vi.fn().mockResolvedValue(undefined),
     notifyTrackMuteState: vi.fn().mockResolvedValue(undefined),
     refreshPublishedTrack: vi.fn().mockResolvedValue(undefined),
+    ensureMediaFactory: vi.fn().mockResolvedValue({
+      dispose: vi.fn().mockResolvedValue(undefined),
+    }),
     tracer: new Tracer('tests'),
   };
 };
@@ -252,9 +285,9 @@ export const createLocalStorageMock = (): LocalStorageMock => {
   };
 };
 
-let deviceIds: Subject<MediaDeviceInfo[]>;
+let deviceIds: ReplaySubject<MediaDeviceInfo[]>;
 export const mockDeviceIds$ = () => {
-  deviceIds = new Subject();
+  deviceIds = new ReplaySubject(1);
   return deviceIds;
 };
 

package/src/events/__tests__/participant.test.ts

@@ -1,5 +1,5 @@
 import '../../rtc/__tests__/mocks/webrtc.mocks';
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 import { CallState } from '../../store';
 import { VisibilityState } from '../../types';
 import { TrackType } from '../../gen/video/sfu/models/models';
@@ -192,6 +192,139 @@ describe('Participant events', () => {
       expect(p?.screenShareStream).toBe(mediaStream);
       expect(state.takeOrphanedTracks('track-lookup-prefix')).toHaveLength(0);
     });
+
+    it('participantJoined should attach E2EE decryptor to orphaned track receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.VIDEO,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      // @ts-expect-error partial mock
+      const onParticipantJoined = watchParticipantJoined(state, () => e2ee);
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).toHaveBeenCalledWith(receiver, 'user-id', 'VIDEO');
+    });
+
+    it('trackPublished should attach E2EE decryptor to orphaned track receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.AUDIO,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      // @ts-expect-error partial mock
+      const onTrackPublished = watchTrackPublished(state, () => e2ee);
+      onTrackPublished({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).toHaveBeenCalledWith(receiver, 'user-id', 'AUDIO');
+    });
+
+    it('trackUnpublished should attach E2EE decryptor to orphaned track receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.SCREEN_SHARE,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      // @ts-expect-error partial mock
+      const onTrackUnPublished = watchTrackUnpublished(state, () => e2ee);
+      onTrackUnPublished({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).toHaveBeenCalledWith(
+        receiver,
+        'user-id',
+        'SCREEN_SHARE',
+      );
+    });
+
+    it('should not call decrypt when no E2EE manager is provided', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.VIDEO,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      const onParticipantJoined = watchParticipantJoined(state);
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      const p = state.findParticipantBySessionId('session-id');
+      expect(p?.videoStream).toBe(mediaStream);
+    });
+
+    it('should not call decrypt when orphaned track has no receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.VIDEO,
+        track: mediaStream,
+        id: mediaStream.id,
+      });
+      // @ts-expect-error partial mock
+      const onParticipantJoined = watchParticipantJoined(state, () => e2ee);
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).not.toHaveBeenCalled();
+      const p = state.findParticipantBySessionId('session-id');
+      expect(p?.videoStream).toBe(mediaStream);
+    });
   });
 
   describe('trackPublished', () => {

package/src/events/callEventHandlers.ts

@@ -40,6 +40,12 @@ type RingCallEvents = Extract<
  */
 export const registerEventHandlers = (call: Call, dispatcher: Dispatcher) => {
   const state = call.state;
+  // Read lazily on each event: setE2EEManager can run after setup() (an app that
+  // inspects call settings via get()/getOrCreate() before deciding to encrypt has
+  // already triggered it), so a value captured here would be a stale undefined
+  // for the whole call and orphaned tracks would never get a decryptor.
+  const e2ee = () => call.e2eeManager;
+
   const eventHandlers = [
     call.on('call.ended', watchCallEnded(call)),
     watchSfuCallEnded(call),
@@ -49,12 +55,12 @@ export const registerEventHandlers = (call: Call, dispatcher: Dispatcher) => {
     watchConnectionQualityChanged(dispatcher, state),
     watchParticipantCountChanged(dispatcher, state),
 
-    call.on('participantJoined', watchParticipantJoined(state)),
+    call.on('participantJoined', watchParticipantJoined(state, e2ee)),
     call.on('participantLeft', watchParticipantLeft(state)),
     call.on('participantUpdated', watchParticipantUpdated(state)),
 
-    call.on('trackPublished', watchTrackPublished(state)),
-    call.on('trackUnpublished', watchTrackUnpublished(state)),
+    call.on('trackPublished', watchTrackPublished(state, e2ee)),
+    call.on('trackUnpublished', watchTrackUnpublished(state, e2ee)),
 
     watchAudioLevelChanged(dispatcher, state),
     watchDominantSpeakerChanged(dispatcher, state),