@stream-io/video-client 0.3.15 → 0.3.17

package/src/devices/__tests__/InputMediaDeviceManager.test.ts

@@ -2,7 +2,7 @@ import { Call } from '../../Call';
 import { StreamClient } from '../../coordinator/connection/client';
 import { CallingState, StreamVideoWriteableStateStore } from '../../store';
 
-import { afterEach, beforeEach, describe, vi, it, expect } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { mockCall, mockVideoDevices, mockVideoStream } from './mocks';
 import { InputMediaDeviceManager } from '../InputMediaDeviceManager';
 import { InputMediaDeviceManagerState } from '../InputMediaDeviceManagerState';
@@ -67,8 +67,7 @@ describe('InputMediaDeviceManager.test', () => {
   });
 
   it('enable device - after joined to call', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
 
     await manager.enable();
 
@@ -93,8 +92,7 @@ describe('InputMediaDeviceManager.test', () => {
   });
 
   it('disable device - after joined to call', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
     await manager.enable();
 
     await manager.disable();
@@ -136,8 +134,7 @@ describe('InputMediaDeviceManager.test', () => {
   });
 
   it('select device when status is enabled and in call', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
     await manager.enable();
 
     const deviceId = mockVideoDevices[1].deviceId;

package/src/devices/__tests__/MicrophoneManager.test.ts

@@ -2,12 +2,17 @@ import { Call } from '../../Call';
 import { StreamClient } from '../../coordinator/connection/client';
 import { CallingState, StreamVideoWriteableStateStore } from '../../store';
 
-import { afterEach, beforeEach, describe, vi, it, expect } from 'vitest';
-import { mockCall, mockAudioDevices, mockAudioStream } from './mocks';
+import { afterEach, beforeEach, describe, expect, it, Mock, vi } from 'vitest';
+import { mockAudioDevices, mockAudioStream, mockCall } from './mocks';
 import { getAudioStream } from '../devices';
 import { TrackType } from '../../gen/video/sfu/models/models';
 import { MicrophoneManager } from '../MicrophoneManager';
 import { of } from 'rxjs';
+import {
+  createSoundDetector,
+  SoundStateChangeHandler,
+} from '../../helpers/sound-detector';
+import { OwnCapability } from '../../gen/coordinator';
 
 vi.mock('../devices.ts', () => {
   console.log('MOCKING devices API');
@@ -20,6 +25,13 @@ vi.mock('../devices.ts', () => {
   };
 });
 
+vi.mock('../../helpers/sound-detector.ts', () => {
+  console.log('MOCKING sound detector');
+  return {
+    createSoundDetector: vi.fn(),
+  };
+});
+
 vi.mock('../../Call.ts', () => {
   console.log('MOCKING Call');
   return {
@@ -64,8 +76,7 @@ describe('MicrophoneManager', () => {
   });
 
   it('publish stream', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
 
     await manager.enable();
 
@@ -75,8 +86,7 @@ describe('MicrophoneManager', () => {
   });
 
   it('stop publish stream', async () => {
-    // @ts-expect-error
-    manager['call'].state.callingState = CallingState.JOINED;
+    manager['call'].state.setCallingState(CallingState.JOINED);
     await manager.enable();
 
     await manager.disable();
@@ -97,6 +107,67 @@ describe('MicrophoneManager', () => {
     expect(manager.state.mediaStream!.getAudioTracks()[0].enabled).toBe(false);
   });
 
+  it(`should start sound detection if mic is disabled`, async () => {
+    await manager.enable();
+    // @ts-expect-error
+    vi.spyOn(manager, 'startSpeakingWhileMutedDetection');
+    await manager.disable();
+
+    expect(manager['startSpeakingWhileMutedDetection']).toHaveBeenCalled();
+  });
+
+  it(`should stop sound detection if mic is enabled`, async () => {
+    manager.state.setSpeakingWhileMuted(true);
+    manager['soundDetectorCleanup'] = () => {};
+
+    await manager.enable();
+
+    expect(manager.state.speakingWhileMuted).toBe(false);
+  });
+
+  it('should update speaking while muted state', async () => {
+    const mock = createSoundDetector as Mock;
+    let handler: SoundStateChangeHandler;
+    mock.mockImplementation((_: MediaStream, h: SoundStateChangeHandler) => {
+      handler = h;
+    });
+    await manager['startSpeakingWhileMutedDetection']();
+
+    expect(manager.state.speakingWhileMuted).toBe(false);
+
+    handler!({ isSoundDetected: true, audioLevel: 2 });
+
+    expect(manager.state.speakingWhileMuted).toBe(true);
+
+    handler!({ isSoundDetected: false, audioLevel: 0 });
+
+    expect(manager.state.speakingWhileMuted).toBe(false);
+  });
+
+  it('should stop speaking while muted notifications if user loses permission to send audio', async () => {
+    await manager.enable();
+    await manager.disable();
+
+    // @ts-expect-error
+    vi.spyOn(manager, 'stopSpeakingWhileMutedDetection');
+    manager['call'].state.setOwnCapabilities([]);
+
+    expect(manager['stopSpeakingWhileMutedDetection']).toHaveBeenCalled();
+  });
+
+  it('should start speaking while muted notifications if user gains permission to send audio', async () => {
+    await manager.enable();
+    await manager.disable();
+
+    manager['call'].state.setOwnCapabilities([]);
+
+    // @ts-expect-error
+    vi.spyOn(manager, 'stopSpeakingWhileMutedDetection');
+    manager['call'].state.setOwnCapabilities([OwnCapability.SEND_AUDIO]);
+
+    expect(manager['stopSpeakingWhileMutedDetection']).toHaveBeenCalled();
+  });
+
   afterEach(() => {
     vi.clearAllMocks();
     vi.resetModules();

package/src/devices/__tests__/mocks.ts

@@ -1,5 +1,6 @@
 import { vi } from 'vitest';
-import { CallingState } from '../../store';
+import { CallingState, CallState } from '../../store';
+import { OwnCapability } from '../../gen/coordinator';
 
 export const mockVideoDevices = [
   {
@@ -63,10 +64,14 @@ export const mockAudioDevices = [
 ] as MediaDeviceInfo[];
 
 export const mockCall = () => {
+  const callState = new CallState();
+  callState.setCallingState(CallingState.JOINED);
+  callState.setOwnCapabilities([
+    OwnCapability.SEND_AUDIO,
+    OwnCapability.SEND_VIDEO,
+  ]);
   return {
-    state: {
-      callingState: CallingState.IDLE,
-    },
+    state: callState,
     publishVideoStream: vi.fn(),
     publishAudioStream: vi.fn(),
     stopPublish: vi.fn(),
@@ -79,6 +84,10 @@ export const mockAudioStream = () => {
       deviceId: mockAudioDevices[0].deviceId,
     }),
     enabled: true,
+    readyState: 'live',
+    stop: () => {
+      track.readyState = 'ended';
+    },
   };
   return {
     getAudioTracks: () => [track],
@@ -95,7 +104,7 @@ export const mockVideoStream = () => {
     enabled: true,
     readyState: 'live',
     stop: () => {
-      track.readyState = 'eneded';
+      track.readyState = 'ended';
     },
   };
   return {

package/src/devices/index.ts

@@ -5,3 +5,5 @@ export * from './CameraManager';
 export * from './CameraManagerState';
 export * from './MicrophoneManager';
 export * from './MicrophoneManagerState';
+export * from './SpeakerManager';
+export * from './SpeakerState';

package/src/events/__tests__/mutes.test.ts

@@ -0,0 +1,133 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { Call } from '../../Call';
+import {
+  TrackType,
+  TrackUnpublishReason,
+} from '../../gen/video/sfu/models/models';
+import { SfuEvent } from '../../gen/video/sfu/event/events';
+import { StreamClient } from '../../coordinator/connection/client';
+import { handleRemoteSoftMute } from '../mutes';
+import { SfuEventListener } from '../../rtc';
+
+describe('mutes', () => {
+  describe('soft mute', () => {
+    let handler: SfuEventListener;
+    let call: Call;
+
+    beforeEach(() => {
+      // @ts-expect-error
+      call = new Call({
+        type: 'test',
+        id: 'test',
+        streamClient: new StreamClient('api-key'),
+      });
+      // disable all event handlers
+      call['dispatcher'].offAll();
+
+      // @ts-expect-error partial data
+      call.publisher = vi.fn();
+      // @ts-expect-error partial data
+      call.publisher.isPublishing = vi.fn().mockReturnValue(true);
+
+      vi.spyOn(call, 'stopPublish').mockResolvedValue(undefined);
+      vi.spyOn(call.camera, 'disable').mockResolvedValue(undefined);
+      vi.spyOn(call.microphone, 'disable').mockResolvedValue(undefined);
+
+      // @ts-ignore
+      call.on = (event: string, h) => {
+        if (event === 'trackUnpublished') {
+          // @ts-ignore
+          handler = h;
+        }
+      };
+
+      handleRemoteSoftMute(call);
+
+      // @ts-expect-error partial data
+      call.state.updateOrAddParticipant('session-id', {
+        userId: 'user-id',
+        sessionId: 'session-id',
+        isLocalParticipant: true,
+        publishedTracks: [
+          TrackType.VIDEO,
+          TrackType.AUDIO,
+          TrackType.SCREEN_SHARE,
+          TrackType.SCREEN_SHARE_AUDIO,
+        ],
+      });
+    });
+
+    it('should automatically mute only when cause is moderation', async () => {
+      const event: SfuEvent = {
+        eventPayload: {
+          oneofKind: 'trackUnpublished',
+          trackUnpublished: {
+            cause: TrackUnpublishReason.PERMISSION_REVOKED,
+            type: TrackType.VIDEO,
+            sessionId: 'session-id',
+            userId: 'user-id',
+          },
+        },
+      };
+
+      await handler!(event);
+      expect(call.camera.disable).not.toHaveBeenCalled();
+      expect(call.stopPublish).not.toHaveBeenCalledWith(TrackType.VIDEO);
+    });
+
+    it('should handle remote soft video mute', async () => {
+      const event: SfuEvent = {
+        eventPayload: {
+          oneofKind: 'trackUnpublished',
+          trackUnpublished: {
+            cause: TrackUnpublishReason.MODERATION,
+            type: TrackType.VIDEO,
+            sessionId: 'session-id',
+            userId: 'user-id',
+          },
+        },
+      };
+
+      await handler!(event);
+      expect(call.camera.disable).toHaveBeenCalled();
+      expect(call.stopPublish).toHaveBeenCalledWith(TrackType.VIDEO);
+    });
+
+    it('should handle remote soft audio mute', async () => {
+      const event: SfuEvent = {
+        eventPayload: {
+          oneofKind: 'trackUnpublished',
+          trackUnpublished: {
+            cause: TrackUnpublishReason.MODERATION,
+            type: TrackType.AUDIO,
+            sessionId: 'session-id',
+            userId: 'user-id',
+          },
+        },
+      };
+
+      await handler!(event);
+      expect(call.microphone.disable).toHaveBeenCalled();
+      expect(call.stopPublish).toHaveBeenCalledWith(TrackType.AUDIO);
+    });
+
+    it('should handle remote soft screenshare mute', async () => {
+      const event: SfuEvent = {
+        eventPayload: {
+          oneofKind: 'trackUnpublished',
+          trackUnpublished: {
+            cause: TrackUnpublishReason.MODERATION,
+            type: TrackType.SCREEN_SHARE,
+            sessionId: 'session-id',
+            userId: 'user-id',
+          },
+        },
+      };
+
+      await handler!(event);
+      expect(call.camera.disable).not.toHaveBeenCalled();
+      expect(call.microphone.disable).not.toHaveBeenCalled();
+      expect(call.stopPublish).toHaveBeenCalledWith(TrackType.SCREEN_SHARE);
+    });
+  });
+});

package/src/events/callEventHandlers.ts

@@ -2,6 +2,7 @@ import { Call } from '../Call';
 import { Dispatcher } from '../rtc';
 import { CallState } from '../store';
 import {
+  handleRemoteSoftMute,
   watchAudioLevelChanged,
   watchCallAccepted,
   watchCallEnded,
@@ -61,6 +62,8 @@ export const registerEventHandlers = (
 
     call.on('callGrantsUpdated', watchCallGrantsUpdated(state)),
     call.on('pinsUpdated', watchPinsUpdated(state)),
+
+    handleRemoteSoftMute(call),
   ];
 
   if (call.ringing) {

package/src/events/index.ts

@@ -1,5 +1,6 @@
 export * from './call';
 export * from './call-permissions';
 export * from './internal';
+export * from './mutes';
 export * from './participant';
 export * from './speaker';

package/src/events/mutes.ts

@@ -0,0 +1,48 @@
+import { Call } from '../Call';
+import {
+  TrackType,
+  TrackUnpublishReason,
+} from '../gen/video/sfu/models/models';
+
+/**
+ * An event handler that handles soft mutes.
+ *
+ * @param call the call.
+ */
+export const handleRemoteSoftMute = (call: Call) => {
+  return call.on('trackUnpublished', async (event) => {
+    if (event.eventPayload.oneofKind !== 'trackUnpublished') return;
+    const {
+      trackUnpublished: { cause, type, sessionId },
+    } = event.eventPayload;
+    const { localParticipant } = call.state;
+    if (
+      cause === TrackUnpublishReason.MODERATION &&
+      sessionId === localParticipant?.sessionId
+    ) {
+      const logger = call.logger;
+      logger(
+        'info',
+        `Local participant's ${TrackType[type]} track is muted remotely`,
+      );
+      try {
+        if (type === TrackType.VIDEO) {
+          await call.camera.disable();
+        } else if (type === TrackType.AUDIO) {
+          await call.microphone.disable();
+        } else {
+          logger(
+            'warn',
+            'Unsupported track type to soft mute',
+            TrackType[type],
+          );
+        }
+        if (call.publisher?.isPublishing(type)) {
+          await call.stopPublish(type);
+        }
+      } catch (error) {
+        logger('error', 'Failed to stop publishing', error);
+      }
+    }
+  });
+};

package/src/helpers/sound-detector.ts

@@ -82,7 +82,13 @@ export const createSoundDetector = (
         ? 100
         : Math.round((averagedDataValue / audioLevelThreshold) * 100);
 
-    onSoundDetectedStateChanged({ isSoundDetected, audioLevel: percentage });
+    // When the track is disabled, it takes time for the buffer to empty
+    // This check will ensure that we don't send anything if the track is disabled
+    if (audioStream.getAudioTracks()[0]?.enabled) {
+      onSoundDetectedStateChanged({ isSoundDetected, audioLevel: percentage });
+    } else {
+      onSoundDetectedStateChanged({ isSoundDetected: false, audioLevel: 0 });
+    }
   }, detectionFrequencyInMs);
 
   return async function stop() {

package/src/rtc/Publisher.ts

@@ -29,14 +29,13 @@ import { getOSInfo } from '../client-details';
 
 const logger: Logger = getLogger(['Publisher']);
 
-export type PublisherOpts = {
+export type PublisherConstructorOpts = {
   sfuClient: StreamSfuClient;
   state: CallState;
   dispatcher: Dispatcher;
   connectionConfig?: RTCConfiguration;
   isDtxEnabled: boolean;
   isRedEnabled: boolean;
-  preferredVideoCodec?: string;
   iceRestartDelay?: number;
 };
 
@@ -86,18 +85,8 @@ export class Publisher {
     [TrackType.UNSPECIFIED]: undefined,
   };
 
-  /**
-   * A map keeping track of track types that were published to the SFU.
-   * This map shouldn't be cleared when unpublishing a track, as it is used
-   * to determine whether a track was published before.
-   *
-   * @private
-   */
-  private readonly trackTypePublishHistory = new Map<TrackType, boolean>();
-
   private readonly isDtxEnabled: boolean;
   private readonly isRedEnabled: boolean;
-  private readonly preferredVideoCodec?: string;
 
   private readonly unsubscribeOnIceRestart: () => void;
 
@@ -118,7 +107,6 @@ export class Publisher {
    * @param dispatcher the dispatcher to use.
    * @param isDtxEnabled whether DTX is enabled.
    * @param isRedEnabled whether RED is enabled.
-   * @param preferredVideoCodec the preferred video codec.
    * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE once connection goes to `disconnected` state.
    */
   constructor({
@@ -128,15 +116,13 @@ export class Publisher {
     state,
     isDtxEnabled,
     isRedEnabled,
-    preferredVideoCodec,
     iceRestartDelay = 2500,
-  }: PublisherOpts) {
+  }: PublisherConstructorOpts) {
     this.pc = this.createPeerConnection(connectionConfig);
     this.sfuClient = sfuClient;
     this.state = state;
     this.isDtxEnabled = isDtxEnabled;
     this.isRedEnabled = isRedEnabled;
-    this.preferredVideoCodec = preferredVideoCodec;
     this.iceRestartDelay = iceRestartDelay;
 
     this.unsubscribeOnIceRestart = dispatcher.on(
@@ -278,7 +264,6 @@ export class Publisher {
       logger('debug', `Added ${TrackType[trackType]} transceiver`);
       this.transceiverInitOrder.push(trackType);
       this.transceiverRegistry[trackType] = transceiver;
-      this.trackTypePublishHistory.set(trackType, true);
 
       if ('setCodecPreferences' in transceiver && codecPreferences) {
         logger(
@@ -362,17 +347,6 @@ export class Publisher {
     return false;
   };
 
-  /**
-   * Returns true if the given track type was ever published to the SFU.
-   * Contrary to `isPublishing`, this method returns true if a certain
-   * track type was published before, even if it is currently unpublished.
-   *
-   * @param trackType the track type to check.
-   */
-  hasEverPublished = (trackType: TrackType): boolean => {
-    return this.trackTypePublishHistory.get(trackType) ?? false;
-  };
-
   /**
    * Returns true if the given track type is currently live
    *