@stream-io/video-client 1.18.9 → 1.19.1

package/src/rtc/BasePeerConnection.ts

@@ -9,14 +9,16 @@ import { PeerType } from '../gen/video/sfu/models/models';
 import { StreamSfuClient } from '../StreamSfuClient';
 import { AllSfuEvents, Dispatcher } from './Dispatcher';
 import { withoutConcurrency } from '../helpers/concurrency';
+import { Tracer, traceRTCPeerConnection, TraceSlice } from '../stats';
 
 export type BasePeerConnectionOpts = {
   sfuClient: StreamSfuClient;
   state: CallState;
   connectionConfig?: RTCConfiguration;
   dispatcher: Dispatcher;
-  onUnrecoverableError?: () => void;
+  onUnrecoverableError?: (reason: string) => void;
   logTag: string;
+  enableTracing: boolean;
 };
 
 /**
@@ -31,10 +33,11 @@ export abstract class BasePeerConnection {
   protected readonly dispatcher: Dispatcher;
   protected sfuClient: StreamSfuClient;
 
-  protected onUnrecoverableError?: () => void;
+  protected onUnrecoverableError?: (reason: string) => void;
   protected isIceRestarting = false;
   private isDisposed = false;
 
+  private readonly tracer?: Tracer;
   private readonly subscriptions: (() => void)[] = [];
   private unsubscribeIceTrickle?: () => void;
 
@@ -50,6 +53,7 @@ export abstract class BasePeerConnection {
       dispatcher,
       onUnrecoverableError,
       logTag,
+      enableTracing,
     }: BasePeerConnectionOpts,
   ) {
     this.peerType = peerType;
@@ -62,6 +66,11 @@ export abstract class BasePeerConnection {
       logTag,
     ]);
     this.pc = new RTCPeerConnection(connectionConfig);
+    if (enableTracing) {
+      this.tracer = new Tracer(logTag);
+      this.tracer.trace('create', connectionConfig);
+      traceRTCPeerConnection(this.pc, this.tracer.trace);
+    }
     this.pc.addEventListener('icecandidate', this.onIceCandidate);
     this.pc.addEventListener('icecandidateerror', this.onIceCandidateError);
     this.pc.addEventListener(
@@ -80,6 +89,7 @@ export abstract class BasePeerConnection {
     this.isDisposed = true;
     this.detachEventHandlers();
     this.pc.close();
+    this.tracer?.dispose();
   }
 
   /**
@@ -163,6 +173,13 @@ export abstract class BasePeerConnection {
     return this.pc.getStats(selector);
   };
 
+  /**
+   * Returns the current tracing buffer.
+   */
+  getTrace = (): TraceSlice | undefined => {
+    return this.tracer?.take();
+  };
+
   /**
    * Handles the ICECandidate event and
    * Initiates an ICE Trickle process with the SFU.
@@ -214,8 +231,9 @@ export abstract class BasePeerConnection {
       this.logger('debug', `Attempting to restart ICE`);
       this.restartIce().catch((e) => {
         if (this.isDisposed) return;
-        this.logger('error', `ICE restart failed`, e);
-        this.onUnrecoverableError?.();
+        const reason = `ICE restart failed`;
+        this.logger('error', reason, e);
+        this.onUnrecoverableError?.(`${reason}: ${e}`);
       });
     }
   };

package/src/rtc/Publisher.ts

@@ -45,8 +45,9 @@ export class Publisher extends BasePeerConnection {
     this.on('iceRestart', (iceRestart) => {
       if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED) return;
       this.restartIce().catch((err) => {
-        this.logger('warn', `ICERestart failed`, err);
-        this.onUnrecoverableError?.();
+        const reason = `ICE restart failed`;
+        this.logger('warn', reason, err);
+        this.onUnrecoverableError?.(`${reason}: ${err}`);
       });
     });
 

package/src/rtc/Subscriber.ts

@@ -144,8 +144,6 @@ export class Subscriber extends BasePeerConnection {
   };
 
   private negotiate = async (subscriberOffer: SubscriberOffer) => {
-    this.logger('info', `Received subscriberOffer`, subscriberOffer);
-
     await this.pc.setRemoteDescription({
       type: 'offer',
       sdp: subscriberOffer.sdp,

package/src/rtc/__tests__/Subscriber.test.ts

@@ -190,6 +190,7 @@ describe('Subscriber', () => {
       subscriber['pc'].createAnswer = vi
         .fn()
         .mockResolvedValue({ sdp: 'answer-sdp' });
+      vi.spyOn(subscriber['pc'], 'setRemoteDescription').mockResolvedValue();
 
       const offer = SubscriberOffer.create({ sdp: 'offer-sdp' });
       // @ts-expect-error - private method

package/src/rtc/helpers/__tests__/rtcConfiguration.test.ts

@@ -17,6 +17,7 @@ describe('rtcConfiguration', () => {
       },
     ];
     expect(toRtcConfiguration(config)).toEqual({
+      bundlePolicy: 'max-bundle',
       iceServers: [
         {
           urls: ['stun:stun.l.google.com:19302'],

package/src/rtc/helpers/rtcConfiguration.ts

@@ -2,6 +2,7 @@ import { ICEServer } from '../../gen/coordinator';
 
 export const toRtcConfiguration = (config: ICEServer[]): RTCConfiguration => {
   return {
+    bundlePolicy: 'max-bundle',
     iceServers: config.map((ice) => ({
       urls: ice.urls,
       username: ice.username,

package/src/stats/SfuStatsReporter.ts

@@ -3,6 +3,7 @@ import { StreamSfuClient } from '../StreamSfuClient';
 import { OwnCapability, StatsOptions } from '../gen/coordinator';
 import { getLogger } from '../logger';
 import { Publisher, Subscriber } from '../rtc';
+import { tracer as mediaStatsTracer } from './rtc/mediaDevices';
 import { flatten, getSdkName, getSdkVersion } from './utils';
 import { getDeviceState, getWebRTCInfo } from '../helpers/client-details';
 import {
@@ -147,23 +148,46 @@ export class SfuStatsReporter {
     });
   };
 
-  private run = async (telemetryData?: Telemetry) => {
+  private run = async (telemetry?: Telemetry) => {
     const [subscriberStats, publisherStats] = await Promise.all([
       this.subscriber.getStats().then(flatten).then(JSON.stringify),
       this.publisher?.getStats().then(flatten).then(JSON.stringify) ?? '[]',
     ]);
 
-    await this.sfuClient.sendStats({
-      sdk: this.sdkName,
-      sdkVersion: this.sdkVersion,
-      webrtcVersion: this.webRTCVersion,
-      subscriberStats,
-      publisherStats,
-      audioDevices: this.inputDevices.get('mic'),
-      videoDevices: this.inputDevices.get('camera'),
-      deviceState: getDeviceState(),
-      telemetry: telemetryData,
-    });
+    const subscriberTrace = this.subscriber.getTrace();
+    const publisherTrace = this.publisher?.getTrace();
+    const mediaTrace = mediaStatsTracer.take();
+    const sfuTrace = this.sfuClient.getTrace();
+    const publisherTraces = [
+      ...mediaTrace.snapshot,
+      ...(sfuTrace?.snapshot ?? []),
+      ...(publisherTrace?.snapshot ?? []),
+    ];
+
+    try {
+      await this.sfuClient.sendStats({
+        sdk: this.sdkName,
+        sdkVersion: this.sdkVersion,
+        webrtcVersion: this.webRTCVersion,
+        subscriberStats,
+        subscriberRtcStats: subscriberTrace
+          ? JSON.stringify(subscriberTrace.snapshot)
+          : '',
+        publisherStats,
+        publisherRtcStats:
+          publisherTraces.length > 0 ? JSON.stringify(publisherTraces) : '',
+        audioDevices: this.inputDevices.get('mic'),
+        videoDevices: this.inputDevices.get('camera'),
+        deviceState: getDeviceState(),
+        telemetry,
+      });
+    } catch (err) {
+      publisherTrace?.rollback();
+      subscriberTrace?.rollback();
+      mediaTrace.rollback();
+      sfuTrace?.rollback();
+      throw err;
+    }
   };
 
   start = () => {

package/src/stats/index.ts

@@ -1,3 +1,5 @@
 export * from './CallStateStatsReporter';
 export * from './SfuStatsReporter';
 export * from './types';
+export * from './utils';
+export * from './rtc';

package/src/stats/rtc/Tracer.ts

@@ -0,0 +1,42 @@
+import type { Trace, TraceRecord } from './types';
+
+export type TraceSlice = {
+  snapshot: TraceRecord[];
+  rollback: () => void;
+};
+
+export class Tracer {
+  private buffer: TraceRecord[] = [];
+  private enabled = true;
+  private readonly id: string | null;
+
+  constructor(id: string | null) {
+    this.id = id;
+  }
+
+  setEnabled = (enabled: boolean) => {
+    if (this.enabled === enabled) return;
+    this.enabled = enabled;
+    this.buffer = [];
+  };
+
+  trace: Trace = (tag, data) => {
+    if (!this.enabled) return;
+    this.buffer.push([tag, this.id, data, Date.now()]);
+  };
+
+  take = (): TraceSlice => {
+    const snapshot = this.buffer;
+    this.buffer = [];
+    return {
+      snapshot,
+      rollback: () => {
+        this.buffer.unshift(...snapshot);
+      },
+    };
+  };
+
+  dispose = () => {
+    this.buffer = [];
+  };
+}

package/src/stats/rtc/index.ts

@@ -0,0 +1,5 @@
+// Based on the original code from:
+// https://github.com/fippo/rtcstats
+
+export * from './pc';
+export * from './Tracer';

package/src/stats/rtc/mediaDevices.ts

@@ -0,0 +1,42 @@
+import { Tracer } from './Tracer';
+
+export const tracer = new Tracer(null);
+
+if (
+  typeof navigator !== 'undefined' &&
+  typeof navigator.mediaDevices !== 'undefined'
+) {
+  const dumpStream = (stream: MediaStream) => ({
+    id: stream.id,
+    tracks: stream.getTracks().map((track) => ({
+      id: track.id,
+      kind: track.kind,
+      label: track.label,
+      enabled: track.enabled,
+      muted: track.muted,
+      readyState: track.readyState,
+    })),
+  });
+
+  const trace = tracer.trace;
+  const target = navigator.mediaDevices;
+  for (const method of ['getUserMedia', 'getDisplayMedia'] as const) {
+    const original = target[method];
+    if (!original) continue;
+
+    target[method] = async function tracedMethod(
+      constraints: MediaStreamConstraints,
+    ) {
+      const tag = `navigator.mediaDevices.${method}`;
+      trace(tag, constraints);
+      try {
+        const stream = await original.call(target, constraints);
+        trace(`${tag}OnSuccess`, dumpStream(stream));
+        return stream;
+      } catch (err) {
+        trace(`${tag}OnFailure`, (err as Error).name);
+        throw err;
+      }
+    };
+  }
+}

package/src/stats/rtc/pc.ts

@@ -0,0 +1,130 @@
+import type { RTCStatsDataType, Trace } from './types';
+
+export const traceRTCPeerConnection = (pc: RTCPeerConnection, trace: Trace) => {
+  pc.addEventListener('icecandidate', (e) => {
+    trace('onicecandidate', e.candidate);
+  });
+  pc.addEventListener('track', (e) => {
+    const streams = e.streams.map((stream) => `stream:${stream.id}`);
+    trace('ontrack', `${e.track.kind}:${e.track.id} ${streams}`);
+  });
+  pc.addEventListener('signalingstatechange', () => {
+    trace('onsignalingstatechange', pc.signalingState);
+  });
+  pc.addEventListener('iceconnectionstatechange', () => {
+    trace('oniceconnectionstatechange', pc.iceConnectionState);
+  });
+  pc.addEventListener('icegatheringstatechange', () => {
+    trace('onicegatheringstatechange', pc.iceGatheringState);
+  });
+  pc.addEventListener('connectionstatechange', () => {
+    trace('onconnectionstatechange', pc.connectionState);
+  });
+  pc.addEventListener('negotiationneeded', () => {
+    trace('onnegotiationneeded', undefined);
+  });
+  pc.addEventListener('datachannel', ({ channel }) => {
+    trace('ondatachannel', [channel.id, channel.label]);
+  });
+
+  let prev: Record<string, RTCStats> = {};
+  const getStats = () => {
+    pc.getStats(null)
+      .then((stats) => {
+        const now = toObject(stats);
+        trace('getstats', deltaCompression(prev, now));
+        prev = now;
+      })
+      .catch((err) => {
+        trace('getstatsOnFailure', (err as Error).toString());
+      });
+  };
+
+  const interval = setInterval(() => {
+    getStats();
+  }, 8000);
+
+  pc.addEventListener('connectionstatechange', () => {
+    const state = pc.connectionState;
+    if (state === 'connected' || state === 'failed') {
+      getStats();
+    }
+  });
+
+  const origClose = pc.close;
+  pc.close = function tracedClose() {
+    clearInterval(interval);
+    trace('close', undefined);
+    return origClose.call(this);
+  };
+
+  for (const method of [
+    'createOffer',
+    'createAnswer',
+    'setLocalDescription',
+    'setRemoteDescription',
+    'addIceCandidate',
+  ] as const) {
+    const original = pc[method];
+    if (!original) continue;
+
+    // @ts-expect-error we don't use deprecated APIs
+    pc[method] = async function tracedMethod(...args: any[]) {
+      try {
+        trace(method, args);
+        // @ts-expect-error improper types
+        const result = await original.apply(this, args);
+        trace(`${method}OnSuccess`, result as RTCStatsDataType);
+        return result;
+      } catch (err) {
+        trace(`${method}OnFailure`, (err as Error).toString());
+        throw err;
+      }
+    };
+  }
+};
+
+const toObject = (s: RTCStatsReport): Record<string, RTCStats> => {
+  const obj: Record<string, RTCStats> = {};
+  s.forEach((v, k) => {
+    obj[k] = v;
+  });
+  return obj;
+};
+
+/**
+ * Apply delta compression to the stats report.
+ * Reduces size by ~90%.
+ * To reduce further, report keys could be compressed.
+ */
+const deltaCompression = (
+  oldStats: Record<any, any>,
+  newStats: Record<any, any>,
+): Record<any, any> => {
+  newStats = JSON.parse(JSON.stringify(newStats));
+
+  for (const [id, report] of Object.entries(newStats)) {
+    delete report.id;
+    if (!oldStats[id]) continue;
+
+    for (const [name, value] of Object.entries(report)) {
+      if (value === oldStats[id][name]) {
+        delete report[name];
+      }
+    }
+  }
+
+  let timestamp = -Infinity;
+  for (const report of Object.values(newStats)) {
+    if (report.timestamp > timestamp) {
+      timestamp = report.timestamp;
+    }
+  }
+  for (const report of Object.values(newStats)) {
+    if (report.timestamp === timestamp) {
+      report.timestamp = 0;
+    }
+  }
+  newStats.timestamp = timestamp;
+  return newStats;
+};

package/src/stats/rtc/types.ts

@@ -0,0 +1,26 @@
+export type RTCStatsDataType =
+  | RTCConfiguration
+  | RTCIceCandidate
+  | RTCSignalingState
+  | RTCIceConnectionState
+  | RTCIceGatheringState
+  | RTCPeerConnectionState
+  | [number | null | string] // RTCDataChannelEvent
+  | string
+  | RTCOfferOptions
+  | [string | RTCDataChannelInit | undefined] // createDataChannel
+  | (RTCOfferOptions | undefined) // createOffer | createAnswer
+  | RTCSessionDescriptionInit
+  | (RTCIceCandidateInit | RTCIceCandidate) // addIceCandidate
+  | object
+  | null
+  | undefined;
+
+export type Trace = (tag: string, data: RTCStatsDataType) => void;
+
+export type TraceRecord = [
+  tag: string,
+  id: string | null,
+  data: RTCStatsDataType,
+  timestamp: number,
+];

package/src/store/CallState.ts

@@ -413,6 +413,7 @@ export class CallState {
 
     this.eventHandlers = {
       // these events are not updating the call state:
+      'call.frame_recording_ready': undefined,
       'call.permission_request': undefined,
       'call.recording_ready': undefined,
       'call.rtmp_broadcast_failed': undefined,
@@ -445,6 +446,15 @@ export class CallState {
         this.updateFromCallResponse(e.call);
         this.setCurrentValue(this.endedBySubject, e.user);
       },
+      'call.frame_recording_failed': (e) => {
+        this.updateFromCallResponse(e.call);
+      },
+      'call.frame_recording_started': (e) => {
+        this.updateFromCallResponse(e.call);
+      },
+      'call.frame_recording_stopped': (e) => {
+        this.updateFromCallResponse(e.call);
+      },
       'call.hls_broadcasting_failed': this.updateFromHLSBroadcastingFailed,
       'call.hls_broadcasting_started': (e) => {
         this.updateFromCallResponse(e.call);