@stream-io/video-client 1.57.0 → 1.59.0

package/src/devices/__tests__/SpeakerManager.test.ts

@@ -6,8 +6,10 @@ import {
   emitDeviceIds,
   LocalStorageMock,
   mockAudioDevices,
+  mockAudioOutputDevices,
   mockBrowserPermission,
   mockDeviceIds$,
+  mockDevicesWithoutAudioPermission,
 } from './mocks';
 import { of } from 'rxjs';
 import { SpeakerManager } from '../SpeakerManager';
@@ -122,15 +124,36 @@ describe('SpeakerManager.test', () => {
   });
 
   it('should disable device if selected device is disconnected', () => {
-    emitDeviceIds(mockAudioDevices);
-    const deviceId = mockAudioDevices[1].deviceId;
+    const deviceId = mockAudioOutputDevices[0].deviceId;
+    emitDeviceIds([...mockAudioDevices, ...mockAudioOutputDevices]);
     manager.select(deviceId);
 
-    emitDeviceIds(mockAudioDevices.slice(2));
+    emitDeviceIds([...mockAudioDevices, ...mockAudioOutputDevices.slice(1)]);
 
     expect(manager.state.selectedDevice).toBe('');
   });
 
+  it('should keep the selection when the device list never exposed audio ids', () => {
+    const deviceId = mockAudioOutputDevices[0].deviceId;
+    emitDeviceIds(mockDevicesWithoutAudioPermission);
+
+    manager.select(deviceId);
+    expect(manager.state.selectedDevice).toBe(deviceId);
+
+    emitDeviceIds(mockDevicesWithoutAudioPermission);
+    expect(manager.state.selectedDevice).toBe(deviceId);
+  });
+
+  it('should keep the selection when the device appears in a later enumeration', () => {
+    const deviceId = mockAudioOutputDevices[0].deviceId;
+    emitDeviceIds(mockDevicesWithoutAudioPermission);
+    manager.select(deviceId);
+
+    emitDeviceIds([...mockAudioDevices, ...mockAudioOutputDevices]);
+
+    expect(manager.state.selectedDevice).toBe(deviceId);
+  });
+
   it('persists speaker selection when permission is granted', async () => {
     const streamClient = new StreamClient('abc123');
     const persistedManager = new SpeakerManager(
@@ -159,16 +182,9 @@ describe('SpeakerManager.test', () => {
   });
 
   describe('apply (web)', () => {
-    it('does nothing when persistence is disabled', () => {
-      const selectSpy = vi.spyOn(manager, 'select');
-      // @ts-expect-error - partial data
-      manager.apply({});
-      expect(selectSpy).not.toHaveBeenCalled();
-    });
-
-    it('selects the persisted speaker device', () => {
+    const createPersistedManager = () => {
       const streamClient = new StreamClient('abc123');
-      const persistedManager = new SpeakerManager(
+      return new SpeakerManager(
         new Call({
           id: '',
           type: '',
@@ -178,56 +194,136 @@ describe('SpeakerManager.test', () => {
         }),
         { enabled: true, storageKey },
       );
+    };
 
+    const persist = (selectedDeviceId: string, selectedDeviceLabel: string) => {
       localStorageMock.setItem(
         storageKey,
         JSON.stringify({
-          speaker: [
-            {
-              selectedDeviceId: 'speaker-1',
-              selectedDeviceLabel: 'Speaker 1',
-            },
-          ],
+          speaker: [{ selectedDeviceId, selectedDeviceLabel }],
         }),
       );
+    };
+
+    it('does nothing when persistence is disabled', async () => {
+      const selectSpy = vi.spyOn(manager, 'select');
+      // @ts-expect-error - partial data
+      await manager.apply({});
+      expect(selectSpy).not.toHaveBeenCalled();
+    });
+
+    it('selects the persisted speaker device', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of([
+          {
+            deviceId: 'speaker-1',
+            kind: 'audiooutput',
+            label: 'Speaker 1',
+            groupId: 'speaker-group',
+          } as MediaDeviceInfo,
+        ]),
+      );
+      persist('speaker-1', 'Speaker 1');
 
       const selectSpy = vi.spyOn(persistedManager, 'select');
       // @ts-expect-error - partial data
-      persistedManager.apply({});
+      await persistedManager.apply({});
 
       expect(selectSpy).toHaveBeenCalledWith('speaker-1');
       expect(persistedManager.state.selectedDevice).toBe('speaker-1');
     });
 
-    it('selects system default when persisted device is default', () => {
-      const streamClient = new StreamClient('abc123');
-      const persistedManager = new SpeakerManager(
-        new Call({
-          id: '',
-          type: '',
-          streamClient,
-          clientEventReporter: new ClientEventReporter({ streamClient }),
-          clientStore: new StreamVideoWriteableStateStore(),
-        }),
-        { enabled: true, storageKey },
+    it('does not select a missing persisted speaker device', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of(mockAudioOutputDevices),
       );
-      persistedManager.select('previous-device');
+      persist('speaker-1', 'Speaker 1');
 
-      localStorageMock.setItem(
-        storageKey,
-        JSON.stringify({
-          speaker: [
-            {
-              selectedDeviceId: defaultDeviceId,
-              selectedDeviceLabel: '',
-            },
-          ],
-        }),
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(selectSpy).not.toHaveBeenCalled();
+      expect(persistedManager.state.selectedDevice).toBe('');
+    });
+
+    it('does not restore a speaker preference when audio permission is not granted', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(mockBrowserPermission, 'asStateObservable').mockReturnValue(
+        of('prompt'),
+      );
+      const listDevicesSpy = vi.spyOn(persistedManager, 'listDevices');
+      persist('speaker-1', 'Speaker 1');
+
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(listDevicesSpy).not.toHaveBeenCalled();
+      expect(selectSpy).not.toHaveBeenCalled();
+      expect(persistedManager.state.selectedDevice).toBe('');
+    });
+
+    it('does not restore a speaker preference from a non-output device with the same label', async () => {
+      const persistedManager = createPersistedManager();
+      const videoDevice = {
+        deviceId: 'video-device-id',
+        kind: 'videoinput',
+        label: 'RODECaster Video (19f7:006b)',
+        groupId: 'video-group',
+      } as MediaDeviceInfo;
+      const outputDevice = {
+        deviceId: 'speaker-device-id',
+        kind: 'audiooutput',
+        label: 'RODECaster Video (19f7:006b)',
+        groupId: 'speaker-group',
+      } as MediaDeviceInfo;
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of([outputDevice]),
       );
+      persist('missing-speaker-device-id', outputDevice.label);
+
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(selectSpy).toHaveBeenCalledWith(outputDevice.deviceId);
+      expect(selectSpy).not.toHaveBeenCalledWith(videoDevice.deviceId);
+      expect(persistedManager.state.selectedDevice).toBe(outputDevice.deviceId);
+    });
+
+    it('does not restore a speaker preference from an empty label placeholder', async () => {
+      const persistedManager = createPersistedManager();
+      vi.spyOn(persistedManager, 'listDevices').mockReturnValue(
+        of([
+          {
+            deviceId: '',
+            kind: 'audiooutput',
+            label: '',
+            groupId: '',
+          } as MediaDeviceInfo,
+        ]),
+      );
+      persist('missing-speaker-device-id', '');
+
+      const selectSpy = vi.spyOn(persistedManager, 'select');
+      // @ts-expect-error - partial data
+      await persistedManager.apply({});
+
+      expect(selectSpy).not.toHaveBeenCalled();
+      expect(persistedManager.state.selectedDevice).toBe('');
+    });
+
+    it('selects system default when persisted device is default', async () => {
+      const persistedManager = createPersistedManager();
+      persistedManager.select('previous-device');
+      persist(defaultDeviceId, '');
 
       const selectSpy = vi.spyOn(persistedManager, 'select');
       // @ts-expect-error - partial data
-      persistedManager.apply({});
+      await persistedManager.apply({});
 
       expect(selectSpy).toHaveBeenCalledWith('');
       expect(persistedManager.state.selectedDevice).toBe('');

package/src/devices/__tests__/mocks.ts

@@ -6,7 +6,7 @@ import {
   OwnCapability,
 } from '../../gen/coordinator';
 import { Call } from '../../Call';
-import { of, Subject } from 'rxjs';
+import { of, ReplaySubject } from 'rxjs';
 import { BrowserPermission } from '../BrowserPermission';
 import { Tracer } from '../../stats';
 
@@ -71,6 +71,35 @@ export const mockAudioDevices = [
   },
 ] as MediaDeviceInfo[];
 
+export const mockAudioOutputDevices = [
+  {
+    deviceId:
+      'f1602a62d9d363c0711de07182751b646b87cfc0edd9e89d2849289e49304158',
+    kind: 'audiooutput',
+    label: 'RODECaster Video (19f7:006b)',
+    groupId: '27f43ef815ab224429532d6e8ebbeb00d7a895e0cbb0c2e4fcbe5b370f234259',
+  },
+  {
+    deviceId:
+      'd542885e6a47d7a38837aee0323bd2c50e825971690f47295cb338e1f1061283',
+    kind: 'audiooutput',
+    label: 'MacBook Pro Speakers (Built-in)',
+    groupId: 'a8708545cb3969c0b7e3712d2ffdbc381992aeb5577b06cd71ba7a94ee20cf3e',
+  },
+] as MediaDeviceInfo[];
+
+export const mockDevicesWithoutAudioPermission = [
+  { deviceId: '', kind: 'audioinput', label: '', groupId: '' },
+  {
+    deviceId:
+      'f10561a0bbd3effbd25645e7f06fc2e261255afeb05f6bc4fdaba3ed7bd9c2b7',
+    kind: 'videoinput',
+    label: 'RODECaster Video (19f7:006b)',
+    groupId: '27f43ef815ab224429532d6e8ebbeb00d7a895e0cbb0c2e4fcbe5b370f234259',
+  },
+  { deviceId: '', kind: 'audiooutput', label: '', groupId: '' },
+] as MediaDeviceInfo[];
+
 export const mockCall = (): Partial<Call> => {
   const callState = new CallState();
   callState.setCallingState(CallingState.JOINED);
@@ -97,6 +126,7 @@ export const mockCall = (): Partial<Call> => {
   return {
     cid: 'default:test-call',
     state: callState,
+    hasMediaEngine: true,
     publish: vi.fn(),
     stopPublish: vi.fn(),
     streamClient: fromPartial({
@@ -106,6 +136,9 @@ export const mockCall = (): Partial<Call> => {
     notifyNoiseCancellationStopped: vi.fn().mockResolvedValue(undefined),
     notifyTrackMuteState: vi.fn().mockResolvedValue(undefined),
     refreshPublishedTrack: vi.fn().mockResolvedValue(undefined),
+    ensureMediaFactory: vi.fn().mockResolvedValue({
+      dispose: vi.fn().mockResolvedValue(undefined),
+    }),
     tracer: new Tracer('tests'),
   };
 };
@@ -252,9 +285,9 @@ export const createLocalStorageMock = (): LocalStorageMock => {
   };
 };
 
-let deviceIds: Subject<MediaDeviceInfo[]>;
+let deviceIds: ReplaySubject<MediaDeviceInfo[]>;
 export const mockDeviceIds$ = () => {
-  deviceIds = new Subject();
+  deviceIds = new ReplaySubject(1);
   return deviceIds;
 };
 

package/src/events/__tests__/participant.test.ts

@@ -1,5 +1,5 @@
 import '../../rtc/__tests__/mocks/webrtc.mocks';
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 import { CallState } from '../../store';
 import { VisibilityState } from '../../types';
 import { TrackType } from '../../gen/video/sfu/models/models';
@@ -192,6 +192,139 @@ describe('Participant events', () => {
       expect(p?.screenShareStream).toBe(mediaStream);
       expect(state.takeOrphanedTracks('track-lookup-prefix')).toHaveLength(0);
     });
+
+    it('participantJoined should attach E2EE decryptor to orphaned track receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.VIDEO,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      // @ts-expect-error partial mock
+      const onParticipantJoined = watchParticipantJoined(state, () => e2ee);
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).toHaveBeenCalledWith(receiver, 'user-id', 'VIDEO');
+    });
+
+    it('trackPublished should attach E2EE decryptor to orphaned track receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.AUDIO,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      // @ts-expect-error partial mock
+      const onTrackPublished = watchTrackPublished(state, () => e2ee);
+      onTrackPublished({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).toHaveBeenCalledWith(receiver, 'user-id', 'AUDIO');
+    });
+
+    it('trackUnpublished should attach E2EE decryptor to orphaned track receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.SCREEN_SHARE,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      // @ts-expect-error partial mock
+      const onTrackUnPublished = watchTrackUnpublished(state, () => e2ee);
+      onTrackUnPublished({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).toHaveBeenCalledWith(
+        receiver,
+        'user-id',
+        'SCREEN_SHARE',
+      );
+    });
+
+    it('should not call decrypt when no E2EE manager is provided', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const receiver = {} as RTCRtpReceiver;
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.VIDEO,
+        track: mediaStream,
+        id: mediaStream.id,
+        receiver,
+      });
+      const onParticipantJoined = watchParticipantJoined(state);
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      const p = state.findParticipantBySessionId('session-id');
+      expect(p?.videoStream).toBe(mediaStream);
+    });
+
+    it('should not call decrypt when orphaned track has no receiver', () => {
+      const state = new CallState();
+      const mediaStream = new MediaStream();
+      const e2ee = { decrypt: vi.fn() };
+      state.registerOrphanedTrack({
+        trackLookupPrefix: 'track-lookup-prefix',
+        trackType: TrackType.VIDEO,
+        track: mediaStream,
+        id: mediaStream.id,
+      });
+      // @ts-expect-error partial mock
+      const onParticipantJoined = watchParticipantJoined(state, () => e2ee);
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          trackLookupPrefix: 'track-lookup-prefix',
+        },
+      });
+
+      expect(e2ee.decrypt).not.toHaveBeenCalled();
+      const p = state.findParticipantBySessionId('session-id');
+      expect(p?.videoStream).toBe(mediaStream);
+    });
   });
 
   describe('trackPublished', () => {

package/src/events/call.ts

@@ -81,7 +81,6 @@ export const watchCallRejected = (call: Call) => {
  */
 export const watchCallEnded = (call: Call) => {
   return function onCallEnded() {
-    globalThis.streamRNVideoSDK?.callingX?.endCall(call, 'remote');
     const { callingState } = call.state;
     if (
       callingState !== CallingState.IDLE &&
@@ -92,9 +91,13 @@ export const watchCallEnded = (call: Call) => {
         reason: 'call.ended event received',
       });
       call
-        .leave({ message: 'call.ended event received', reject: false })
+        .leave({
+          message: 'call.ended event received',
+          reject: false,
+          reason: 'ended',
+        })
         .catch((err) => {
-          call.logger.error('Failed to leave call after call.ended ', err);
+          call.logger.error('Failed to leave call after call.ended', err);
         });
     }
   };

package/src/events/callEventHandlers.ts

@@ -40,6 +40,12 @@ type RingCallEvents = Extract<
  */
 export const registerEventHandlers = (call: Call, dispatcher: Dispatcher) => {
   const state = call.state;
+  // Read lazily on each event: setE2EEManager can run after setup() (an app that
+  // inspects call settings via get()/getOrCreate() before deciding to encrypt has
+  // already triggered it), so a value captured here would be a stale undefined
+  // for the whole call and orphaned tracks would never get a decryptor.
+  const e2ee = () => call.e2eeManager;
+
   const eventHandlers = [
     call.on('call.ended', watchCallEnded(call)),
     watchSfuCallEnded(call),
@@ -49,12 +55,12 @@ export const registerEventHandlers = (call: Call, dispatcher: Dispatcher) => {
     watchConnectionQualityChanged(dispatcher, state),
     watchParticipantCountChanged(dispatcher, state),
 
-    call.on('participantJoined', watchParticipantJoined(state)),
+    call.on('participantJoined', watchParticipantJoined(state, e2ee)),
     call.on('participantLeft', watchParticipantLeft(state)),
     call.on('participantUpdated', watchParticipantUpdated(state)),
 
-    call.on('trackPublished', watchTrackPublished(state)),
-    call.on('trackUnpublished', watchTrackUnpublished(state)),
+    call.on('trackPublished', watchTrackPublished(state, e2ee)),
+    call.on('trackUnpublished', watchTrackUnpublished(state, e2ee)),
 
     watchAudioLevelChanged(dispatcher, state),
     watchDominantSpeakerChanged(dispatcher, state),

package/src/events/participant.ts

@@ -5,7 +5,7 @@ import type {
   TrackPublished,
   TrackUnpublished,
 } from '../gen/video/sfu/event/events';
-import type { Participant } from '../gen/video/sfu/models/models';
+import { type Participant, TrackType } from '../gen/video/sfu/models/models';
 import {
   StreamVideoParticipant,
   StreamVideoParticipantPatch,
@@ -14,11 +14,21 @@ import {
 import { CallState } from '../store';
 import { trackTypeToParticipantStreamKey } from '../rtc';
 import { pushToIfMissing } from '../helpers/array';
+import type { E2EEManager } from '../rtc/e2ee/E2EEManager';
+
+/**
+ * Reads the call's current encryption manager. Deliberately not the manager
+ * itself: `Call.setE2EEManager` may be called after these handlers are wired up.
+ */
+type GetE2EEManager = () => E2EEManager | undefined;
 
 /**
  * An event responder which handles the `participantJoined` event.
  */
-export const watchParticipantJoined = (state: CallState) => {
+export const watchParticipantJoined = (
+  state: CallState,
+  e2ee: GetE2EEManager | undefined = undefined,
+) => {
   return function onParticipantJoined(e: ParticipantJoined) {
     const { participant } = e;
     if (!participant) return;
@@ -30,7 +40,7 @@ export const watchParticipantJoined = (state: CallState) => {
     // The SFU would send participant info as part of the `join`
     // response and then follow up with a `participantJoined` event for
     // already announced participants.
-    const orphanedTracks = reconcileOrphanedTracks(state, participant);
+    const orphanedTracks = reconcileOrphanedTracks(state, participant, e2ee);
     state.updateOrAddParticipant(
       participant.sessionId,
       Object.assign<
@@ -77,7 +87,10 @@ export const watchParticipantUpdated = (state: CallState) => {
  * An event responder which handles the `trackPublished` event.
  * The SFU will send this event when a participant publishes a track.
  */
-export const watchTrackPublished = (state: CallState) => {
+export const watchTrackPublished = (
+  state: CallState,
+  e2ee: GetE2EEManager | undefined = undefined,
+) => {
   return function onTrackPublished(e: TrackPublished) {
     const { type, sessionId } = e;
     // An optimization for large calls.
@@ -85,7 +98,11 @@ export const watchTrackPublished = (state: CallState) => {
     // events, and instead, it would only provide the participant's information
     // once they start publishing a track.
     if (e.participant) {
-      const orphanedTracks = reconcileOrphanedTracks(state, e.participant);
+      const orphanedTracks = reconcileOrphanedTracks(
+        state,
+        e.participant,
+        e2ee,
+      );
       const participant = Object.assign(e.participant, orphanedTracks);
       state.updateOrAddParticipant(sessionId, participant);
     } else {
@@ -100,12 +117,19 @@ export const watchTrackPublished = (state: CallState) => {
  * An event responder which handles the `trackUnpublished` event.
  * The SFU will send this event when a participant unpublishes a track.
  */
-export const watchTrackUnpublished = (state: CallState) => {
+export const watchTrackUnpublished = (
+  state: CallState,
+  e2ee: GetE2EEManager | undefined = undefined,
+) => {
   return function onTrackUnpublished(e: TrackUnpublished) {
     const { type, sessionId } = e;
     // An optimization for large calls. See `watchTrackPublished`.
     if (e.participant) {
-      const orphanedTracks = reconcileOrphanedTracks(state, e.participant);
+      const orphanedTracks = reconcileOrphanedTracks(
+        state,
+        e.participant,
+        e2ee,
+      );
       const participant = Object.assign(e.participant, orphanedTracks);
       state.updateOrAddParticipant(sessionId, participant, (p) => ({
         pausedTracks: p.pausedTracks?.filter((t) => t !== type),
@@ -124,18 +148,29 @@ export const watchTrackUnpublished = (state: CallState) => {
  *
  * @param state the call state.
  * @param participant the participant.
+ * @param e2ee accessor for the call's encryption manager, if any.
  */
 const reconcileOrphanedTracks = (
   state: CallState,
   participant: Participant,
+  e2ee: GetE2EEManager | undefined,
 ): StreamVideoParticipantPatch | undefined => {
   const orphanTracks = state.takeOrphanedTracks(participant.trackLookupPrefix);
   if (!orphanTracks.length) return;
+  const manager = e2ee?.();
   const reconciledTracks: StreamVideoParticipantPatch = {};
   for (const orphan of orphanTracks) {
     const key = trackTypeToParticipantStreamKey(orphan.trackType);
     if (!key) continue;
     reconciledTracks[key] = orphan.track;
+
+    if (manager && orphan.receiver) {
+      manager.decrypt(
+        orphan.receiver,
+        participant.userId,
+        TrackType[orphan.trackType],
+      );
+    }
   }
   return reconciledTracks;
 };