@stream-io/video-client 1.20.2 → 1.22.0

package/src/devices/__tests__/CameraManager.test.ts

@@ -14,6 +14,7 @@ import { TrackType } from '../../gen/video/sfu/models/models';
 import { CameraManager } from '../CameraManager';
 import { of } from 'rxjs';
 import { PermissionsContext } from '../../permissions';
+import { Tracer } from '../../stats';
 
 const getVideoStream = vi.hoisted(() =>
   vi.fn(() => Promise.resolve(mockVideoStream())),
@@ -78,11 +79,14 @@ describe('CameraManager', () => {
   it('get stream', async () => {
     await manager.enable();
 
-    expect(getVideoStream).toHaveBeenCalledWith({
-      deviceId: undefined,
-      width: 1280,
-      height: 720,
-    });
+    expect(getVideoStream).toHaveBeenCalledWith(
+      {
+        deviceId: undefined,
+        width: 1280,
+        height: 720,
+      },
+      expect.any(Tracer),
+    );
   });
 
   it('should get device id from stream', async () => {
@@ -133,29 +137,38 @@ describe('CameraManager', () => {
 
     await manager.enable();
 
-    expect(getVideoStream).toHaveBeenCalledWith({
-      deviceId: undefined,
-      width: 1280,
-      height: 720,
-    });
+    expect(getVideoStream).toHaveBeenCalledWith(
+      {
+        deviceId: undefined,
+        width: 1280,
+        height: 720,
+      },
+      expect.any(Tracer),
+    );
 
     await manager.selectDirection('front');
 
-    expect(getVideoStream).toHaveBeenCalledWith({
-      deviceId: undefined,
-      width: 1280,
-      height: 720,
-      facingMode: 'user',
-    });
+    expect(getVideoStream).toHaveBeenCalledWith(
+      {
+        deviceId: undefined,
+        width: 1280,
+        height: 720,
+        facingMode: 'user',
+      },
+      expect.any(Tracer),
+    );
 
     await manager.selectDirection('back');
 
-    expect(getVideoStream).toHaveBeenCalledWith({
-      deviceId: undefined,
-      facingMode: 'environment',
-      width: 1280,
-      height: 720,
-    });
+    expect(getVideoStream).toHaveBeenCalledWith(
+      {
+        deviceId: undefined,
+        facingMode: 'environment',
+        width: 1280,
+        height: 720,
+      },
+      expect.any(Tracer),
+    );
   });
 
   it(`shouldn't set deviceId and facingMode at the same time`, async () => {
@@ -163,11 +176,14 @@ describe('CameraManager', () => {
 
     await manager.flip();
 
-    expect(getVideoStream).toHaveBeenCalledWith({
-      facingMode: 'environment',
-      width: 1280,
-      height: 720,
-    });
+    expect(getVideoStream).toHaveBeenCalledWith(
+      {
+        facingMode: 'environment',
+        width: 1280,
+        height: 720,
+      },
+      expect.any(Tracer),
+    );
 
     const deviceId = mockVideoDevices[1].deviceId;
     await manager.select(deviceId);

package/src/devices/__tests__/MicrophoneManager.test.ts

@@ -24,6 +24,7 @@ import {
   SoundStateChangeHandler,
 } from '../../helpers/sound-detector';
 import { PermissionsContext } from '../../permissions';
+import { Tracer } from '../../stats';
 
 vi.mock('../devices.ts', () => {
   console.log('MOCKING devices API');
@@ -92,9 +93,10 @@ describe('MicrophoneManager', () => {
   it('get stream', async () => {
     await manager.enable();
 
-    expect(getAudioStream).toHaveBeenCalledWith({
-      deviceId: undefined,
-    });
+    expect(getAudioStream).toHaveBeenCalledWith(
+      { deviceId: undefined },
+      expect.any(Tracer),
+    );
   });
 
   it('should get device id from stream', async () => {

package/src/devices/__tests__/ScreenShareManager.test.ts

@@ -7,6 +7,7 @@ import * as RxUtils from '../../store/rxUtils';
 import { mockCall, mockDeviceIds$, mockScreenShareStream } from './mocks';
 import { getScreenShareStream } from '../devices';
 import { TrackType } from '../../gen/video/sfu/models/models';
+import { Tracer } from '../../stats';
 
 vi.mock('../devices.ts', () => {
   console.log('MOCKING devices API');
@@ -50,7 +51,7 @@ describe('ScreenShareManager', () => {
   });
 
   it('select device', async () => {
-    await expect(manager.select('any-device-id')).rejects.toThrowError();
+    await expect(manager.select()).rejects.toThrowError();
   });
 
   it('get stream', async () => {
@@ -62,6 +63,7 @@ describe('ScreenShareManager', () => {
       expect.objectContaining({
         deviceId: undefined,
       }),
+      expect.any(Tracer),
     );
   });
 
@@ -75,6 +77,7 @@ describe('ScreenShareManager', () => {
         deviceId: undefined,
         audio: false,
       }),
+      expect.any(Tracer),
     );
   });
 
@@ -106,6 +109,7 @@ describe('ScreenShareManager', () => {
           height: 600,
         },
       }),
+      expect.any(Tracer),
     );
   });
 

package/src/devices/__tests__/mocks.ts

@@ -7,6 +7,7 @@ import {
 import { Call } from '../../Call';
 import { of, Subject } from 'rxjs';
 import { BrowserPermission } from '../BrowserPermission';
+import { Tracer } from '../../stats';
 
 export const mockVideoDevices = [
   {
@@ -93,13 +94,11 @@ export const mockCall = (): Partial<Call> => {
   });
   return {
     state: callState,
-    publishVideoStream: vi.fn(),
-    publishAudioStream: vi.fn(),
-    publishScreenShareStream: vi.fn(),
     publish: vi.fn(),
     stopPublish: vi.fn(),
     notifyNoiseCancellationStarting: vi.fn().mockResolvedValue(undefined),
     notifyNoiseCancellationStopped: vi.fn().mockResolvedValue(undefined),
+    tracer: new Tracer('tests'),
   };
 };
 

package/src/devices/devices.ts

@@ -12,6 +12,7 @@ import { getLogger } from '../logger';
 import { BrowserPermission } from './BrowserPermission';
 import { lazy } from '../helpers/lazy';
 import { isFirefox } from '../helpers/browsers';
+import { dumpStream, Tracer } from '../stats';
 
 /**
  * Returns an Observable that emits the list of available devices
@@ -158,15 +159,27 @@ export const getAudioOutputDevices = lazy(() => {
   );
 });
 
-const getStream = async (constraints: MediaStreamConstraints) => {
-  const stream = await navigator.mediaDevices.getUserMedia(constraints);
-  if (isFirefox()) {
-    // When enumerating devices, Firefox will hide device labels unless there's been
-    // an active user media stream on the page. So we force device list updates after
-    // every successful getUserMedia call.
-    navigator.mediaDevices.dispatchEvent(new Event('devicechange'));
+let getUserMediaExecId = 0;
+const getStream = async (
+  constraints: MediaStreamConstraints,
+  tracer: Tracer | undefined,
+) => {
+  const tag = `navigator.mediaDevices.getUserMedia.${getUserMediaExecId++}.`;
+  try {
+    tracer?.trace(tag, constraints);
+    const stream = await navigator.mediaDevices.getUserMedia(constraints);
+    tracer?.trace(`${tag}OnSuccess`, dumpStream(stream));
+    if (isFirefox()) {
+      // When enumerating devices, Firefox will hide device labels unless there's been
+      // an active user media stream on the page. So we force device list updates after
+      // every successful getUserMedia call.
+      navigator.mediaDevices.dispatchEvent(new Event('devicechange'));
+    }
+    return stream;
+  } catch (error) {
+    tracer?.trace(`${tag}OnFailure`, (error as Error).name);
+    throw error;
   }
-  return stream;
 };
 
 function isNotFoundOrOverconstrainedError(error: unknown) {
@@ -195,12 +208,13 @@ function isNotFoundOrOverconstrainedError(error: unknown) {
  * Returns an audio media stream that fulfills the given constraints.
  * If no constraints are provided, it uses the browser's default ones.
  *
- * @angular It's recommended to use the [`DeviceManagerService`](./DeviceManagerService.md) for a higher level API, use this low-level method only if the `DeviceManagerService` doesn't suit your requirements.
  * @param trackConstraints the constraints to use when requesting the stream.
- * @returns the new `MediaStream` fulfilling the given constraints.
+ * @param tracer the tracer to use for tracing the stream creation.
+ * @returns a new `MediaStream` fulfilling the given constraints.
  */
 export const getAudioStream = async (
   trackConstraints?: MediaTrackConstraints,
+  tracer?: Tracer,
 ): Promise<MediaStream> => {
   const constraints: MediaStreamConstraints = {
     audio: {
@@ -214,7 +228,7 @@ export const getAudioStream = async (
       throwOnNotAllowed: true,
       forcePrompt: true,
     });
-    return await getStream(constraints);
+    return await getStream(constraints, tracer);
   } catch (error) {
     if (isNotFoundOrOverconstrainedError(error) && trackConstraints?.deviceId) {
       // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -239,12 +253,13 @@ export const getAudioStream = async (
  * Returns a video media stream that fulfills the given constraints.
  * If no constraints are provided, it uses the browser's default ones.
  *
- * @angular It's recommended to use the [`DeviceManagerService`](./DeviceManagerService.md) for a higher level API, use this low-level method only if the `DeviceManagerService` doesn't suit your requirements.
  * @param trackConstraints the constraints to use when requesting the stream.
+ * @param tracer the tracer to use for tracing the stream creation.
  * @returns a new `MediaStream` fulfilling the given constraints.
  */
 export const getVideoStream = async (
   trackConstraints?: MediaTrackConstraints,
+  tracer?: Tracer,
 ): Promise<MediaStream> => {
   const constraints: MediaStreamConstraints = {
     video: {
@@ -257,7 +272,7 @@ export const getVideoStream = async (
       throwOnNotAllowed: true,
       forcePrompt: true,
     });
-    return await getStream(constraints);
+    return await getStream(constraints, tracer);
   } catch (error) {
     if (isNotFoundOrOverconstrainedError(error) && trackConstraints?.deviceId) {
       // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -278,21 +293,25 @@ export const getVideoStream = async (
   }
 };
 
+let getDisplayMediaExecId = 0;
+
 /**
  * Prompts the user for a permission to share a screen.
  * If the user grants the permission, a screen sharing stream is returned. Throws otherwise.
  *
  * The callers of this API are responsible to handle the possible errors.
  *
- * @angular It's recommended to use the [`DeviceManagerService`](./DeviceManagerService.md) for a higher level API, use this low-level method only if the `DeviceManagerService` doesn't suit your requirements.
- *
  * @param options any additional options to pass to the [`getDisplayMedia`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia) API.
+ * @param tracer the tracer to use for tracing the stream creation.
  */
 export const getScreenShareStream = async (
   options?: DisplayMediaStreamOptions,
+  tracer?: Tracer | undefined,
 ) => {
+  const tag = `navigator.mediaDevices.getDisplayMedia.${getDisplayMediaExecId++}.`;
   try {
-    return await navigator.mediaDevices.getDisplayMedia({
+    tracer?.trace(tag, options);
+    const stream = await navigator.mediaDevices.getDisplayMedia({
       video: true,
       audio: {
         channelCount: {
@@ -306,7 +325,10 @@ export const getScreenShareStream = async (
       systemAudio: 'include',
       ...options,
     });
+    tracer?.trace(`${tag}OnSuccess`, dumpStream(stream));
+    return stream;
   } catch (e) {
+    tracer?.trace(`${tag}OnFailure`, (e as Error).name);
     getLogger(['devices'])('error', 'Failed to get screen share stream', e);
     throw e;
   }

package/src/events/__tests__/call.test.ts

@@ -10,6 +10,7 @@ import {
   CallAcceptedEvent,
   CallEndedEvent,
   CallResponse,
+  OwnCapability,
   RejectCallResponse,
 } from '../../gen/coordinator';
 import { Call } from '../../Call';
@@ -295,6 +296,28 @@ describe('Call ringing events', () => {
 
       expect(call.leave).not.toHaveBeenCalled();
     });
+
+    it('will stay in backstage if live ended and has permission', async () => {
+      const call = fakeCall();
+      call.state.setBackstage(false);
+      call.permissionsContext.setPermissions([OwnCapability.JOIN_BACKSTAGE]);
+      vi.spyOn(call, 'leave').mockImplementation(async () => {
+        console.log(`TEST: leave() called`);
+      });
+
+      watchSfuCallEnded(call);
+      const event: SfuEvent = {
+        eventPayload: {
+          oneofKind: 'callEnded',
+          callEnded: { reason: CallEndedReason.LIVE_ENDED },
+        },
+      };
+      // @ts-expect-error type issue
+      call['dispatcher'].dispatch(event);
+
+      expect(call.leave).not.toHaveBeenCalled();
+      expect(call.state.backstage).toBe(true);
+    });
   });
 
   describe('call.leave', () => {

package/src/events/call.ts

@@ -1,6 +1,10 @@
 import { CallingState } from '../store';
 import { Call } from '../Call';
-import type { CallAcceptedEvent, CallRejectedEvent } from '../gen/coordinator';
+import {
+  CallAcceptedEvent,
+  CallRejectedEvent,
+  OwnCapability,
+} from '../gen/coordinator';
 import { CallEnded } from '../gen/video/sfu/event/events';
 import { CallEndedReason } from '../gen/video/sfu/models/models';
 
@@ -99,6 +103,13 @@ export const watchSfuCallEnded = (call: Call) => {
   return call.on('callEnded', async (e: CallEnded) => {
     if (call.state.callingState === CallingState.LEFT) return;
     try {
+      if (e.reason === CallEndedReason.LIVE_ENDED) {
+        call.state.setBackstage(true);
+
+        // don't leave the call if the user has permission to join backstage
+        const { hasPermission } = call.permissionsContext;
+        if (hasPermission(OwnCapability.JOIN_BACKSTAGE)) return;
+      }
       // `call.ended` event arrived after the call is already left
       // and all event handlers are detached. We need to manually
       // update the call state to reflect the call has ended.

package/src/gen/video/sfu/models/models.ts

@@ -626,6 +626,49 @@ export interface AppleState {
    */
   isLowPowerModeEnabled: boolean;
 }
+/**
+ * PerformanceStats represents the encoding/decoding statistics for a track.
+ *
+ * @generated from protobuf message stream.video.sfu.models.PerformanceStats
+ */
+export interface PerformanceStats {
+  /**
+   * the type of the track (e.g., video, audio, screen share)
+   *
+   * @generated from protobuf field: stream.video.sfu.models.TrackType track_type = 1;
+   */
+  trackType: TrackType;
+  /**
+   * the codec used for the track
+   *
+   * @generated from protobuf field: stream.video.sfu.models.Codec codec = 2;
+   */
+  codec?: Codec;
+  /**
+   * the average encode/decode time in ms
+   *
+   * @generated from protobuf field: float avg_frame_time_ms = 3;
+   */
+  avgFrameTimeMs: number;
+  /**
+   * the average fps for the track
+   *
+   * @generated from protobuf field: float avg_fps = 4;
+   */
+  avgFps: number;
+  /**
+   * the track dimensions
+   *
+   * @generated from protobuf field: stream.video.sfu.models.VideoDimension video_dimension = 5;
+   */
+  videoDimension?: VideoDimension;
+  /**
+   * the target bitrate for the track, only for published tracks
+   *
+   * @generated from protobuf field: int32 target_bitrate = 6;
+   */
+  targetBitrate: number;
+}
 /**
  * @generated from protobuf enum stream.video.sfu.models.PeerType
  */
@@ -1636,3 +1679,44 @@ class AppleState$Type extends MessageType<AppleState> {
  * @generated MessageType for protobuf message stream.video.sfu.models.AppleState
  */
 export const AppleState = new AppleState$Type();
+// @generated message type with reflection information, may provide speed optimized methods
+class PerformanceStats$Type extends MessageType<PerformanceStats> {
+  constructor() {
+    super('stream.video.sfu.models.PerformanceStats', [
+      {
+        no: 1,
+        name: 'track_type',
+        kind: 'enum',
+        T: () => [
+          'stream.video.sfu.models.TrackType',
+          TrackType,
+          'TRACK_TYPE_',
+        ],
+      },
+      { no: 2, name: 'codec', kind: 'message', T: () => Codec },
+      {
+        no: 3,
+        name: 'avg_frame_time_ms',
+        kind: 'scalar',
+        T: 2 /*ScalarType.FLOAT*/,
+      },
+      { no: 4, name: 'avg_fps', kind: 'scalar', T: 2 /*ScalarType.FLOAT*/ },
+      {
+        no: 5,
+        name: 'video_dimension',
+        kind: 'message',
+        T: () => VideoDimension,
+      },
+      {
+        no: 6,
+        name: 'target_bitrate',
+        kind: 'scalar',
+        T: 5 /*ScalarType.INT32*/,
+      },
+    ]);
+  }
+}
+/**
+ * @generated MessageType for protobuf message stream.video.sfu.models.PerformanceStats
+ */
+export const PerformanceStats = new PerformanceStats$Type();

package/src/gen/video/sfu/signal_rpc/signal.ts

@@ -8,6 +8,7 @@ import {
   ICETrickle,
   InputDevices,
   PeerType,
+  PerformanceStats,
   RTMPIngress,
   TrackInfo,
   TrackType,
@@ -158,13 +159,39 @@ export interface SendStatsRequest {
    */
   rtmp?: RTMPIngress;
   /**
-   * @generated from protobuf field: string subscriber_rtc_stats = 13;
+   * @deprecated
+   * @generated from protobuf field: string subscriber_rtc_stats = 13 [deprecated = true];
    */
   subscriberRtcStats: string;
   /**
-   * @generated from protobuf field: string publisher_rtc_stats = 14;
+   * @deprecated
+   * @generated from protobuf field: string publisher_rtc_stats = 14 [deprecated = true];
    */
   publisherRtcStats: string;
+  /**
+   * @generated from protobuf field: string rtc_stats = 15;
+   */
+  rtcStats: string;
+  /**
+   * Encode stats for the publisher
+   *
+   * @generated from protobuf field: repeated stream.video.sfu.models.PerformanceStats encode_stats = 16;
+   */
+  encodeStats: PerformanceStats[];
+  /**
+   * Decode stats for the subscriber
+   *
+   * @generated from protobuf field: repeated stream.video.sfu.models.PerformanceStats decode_stats = 17;
+   */
+  decodeStats: PerformanceStats[];
+  /**
+   * user_session id can change during reconnects, this helps us to
+   * identify the user across reconnects and should remain consistent until the user explicitly
+   * disconnects, is kicked or the call is ended.
+   *
+   * @generated from protobuf field: string unified_session_id = 18;
+   */
+  unifiedSessionId: string;
 }
 /**
  * @generated from protobuf message stream.video.sfu.signal.SendStatsResponse
@@ -534,6 +561,27 @@ class SendStatsRequest$Type extends MessageType<SendStatsRequest> {
         kind: 'scalar',
         T: 9 /*ScalarType.STRING*/,
       },
+      { no: 15, name: 'rtc_stats', kind: 'scalar', T: 9 /*ScalarType.STRING*/ },
+      {
+        no: 16,
+        name: 'encode_stats',
+        kind: 'message',
+        repeat: 1 /*RepeatType.PACKED*/,
+        T: () => PerformanceStats,
+      },
+      {
+        no: 17,
+        name: 'decode_stats',
+        kind: 'message',
+        repeat: 1 /*RepeatType.PACKED*/,
+        T: () => PerformanceStats,
+      },
+      {
+        no: 18,
+        name: 'unified_session_id',
+        kind: 'scalar',
+        T: 9 /*ScalarType.STRING*/,
+      },
     ]);
   }
 }

package/src/helpers/DynascaleManager.ts

@@ -28,7 +28,6 @@ import type { CallState } from '../store';
 import type { StreamSfuClient } from '../StreamSfuClient';
 import { SpeakerManager } from '../devices';
 import { getCurrentValue, setCurrentValue } from '../store/rxUtils';
-import { tracer as mediaStatsTracer } from '../stats/rtc/mediaDevices';
 
 const DEFAULT_VIEWPORT_VISIBILITY_STATE: Record<
   VideoTrackType,
@@ -536,10 +535,6 @@ export class DynascaleManager {
             const { selectedDevice } = this.speaker.state;
             if (selectedDevice && 'setSinkId' in audioElement) {
               audioElement.setSinkId(selectedDevice);
-              mediaStatsTracer.trace(
-                'navigator.mediaDevices.setSinkId',
-                selectedDevice,
-              );
             }
           }
         });
@@ -550,10 +545,6 @@ export class DynascaleManager {
       : this.speaker.state.selectedDevice$.subscribe((deviceId) => {
           if (deviceId) {
             audioElement.setSinkId(deviceId);
-            mediaStatsTracer.trace(
-              'navigator.mediaDevices.setSinkId',
-              deviceId,
-            );
           }
         });
 

package/src/rpc/createClient.ts

@@ -11,7 +11,7 @@ import {
 } from '@protobuf-ts/twirp-transport';
 import { SignalServerClient } from '../gen/video/sfu/signal_rpc/signal.client';
 import { Logger, LogLevel } from '../coordinator/connection/types';
-import type { Trace } from '../stats/rtc/types';
+import type { Trace } from '../stats';
 
 const defaultOptions: TwirpOptions = {
   baseUrl: '',