@stream-io/video-client 1.15.6 → 1.16.0

package/src/helpers/RNSpeechDetector.ts

@@ -1,19 +1,17 @@
 import { BaseStats } from '../stats';
 import { SoundStateChangeHandler } from './sound-detector';
 import { flatten } from '../stats/utils';
-
-const AUDIO_LEVEL_THRESHOLD = 0.2;
+import { getLogger } from '../logger';
 
 export class RNSpeechDetector {
   private pc1 = new RTCPeerConnection({});
   private pc2 = new RTCPeerConnection({});
-  private intervalId: NodeJS.Timeout | undefined;
   private audioStream: MediaStream | undefined;
 
   /**
    * Starts the speech detection.
    */
-  public async start() {
+  public async start(onSoundDetectedStateChanged: SoundStateChangeHandler) {
     try {
       this.cleanupAudioStream();
       const audioStream = await navigator.mediaDevices.getUserMedia({
@@ -31,6 +29,14 @@ export class RNSpeechDetector {
           e.candidate as RTCIceCandidateInit | undefined,
         );
       });
+      this.pc2.addEventListener('track', (e) => {
+        e.streams[0].getTracks().forEach((track) => {
+          // In RN, the remote track is automatically added to the audio output device
+          // so we need to mute it to avoid hearing the audio back
+          // @ts-ignore _setVolume is a private method in react-native-webrtc
+          track._setVolume(0);
+        });
+      });
 
       audioStream
         .getTracks()
@@ -41,64 +47,122 @@ export class RNSpeechDetector {
       const answer = await this.pc2.createAnswer();
       await this.pc1.setRemoteDescription(answer);
       await this.pc2.setLocalDescription(answer);
-      const audioTracks = audioStream.getAudioTracks();
-      // We need to mute the audio track for this temporary stream, or else you will hear yourself twice while in the call.
-      audioTracks.forEach((track) => (track.enabled = false));
-    } catch (error) {
-      console.error(
-        'Error connecting and negotiating between PeerConnections:',
-        error,
+      const unsub = this.onSpeakingDetectedStateChange(
+        onSoundDetectedStateChanged,
       );
+      return () => {
+        unsub();
+        this.stop();
+      };
+    } catch (error) {
+      const logger = getLogger(['RNSpeechDetector']);
+      logger('error', 'error handling permissions: ', error);
+      return () => {};
     }
   }
 
   /**
    * Stops the speech detection and releases all allocated resources.
    */
-  public stop() {
+  private stop() {
     this.pc1.close();
     this.pc2.close();
     this.cleanupAudioStream();
-    if (this.intervalId) {
-      clearInterval(this.intervalId);
-    }
   }
 
   /**
    * Public method that detects the audio levels and returns the status.
    */
-  public onSpeakingDetectedStateChange(
+  private onSpeakingDetectedStateChange(
     onSoundDetectedStateChanged: SoundStateChangeHandler,
   ) {
-    this.intervalId = setInterval(async () => {
-      const stats = (await this.pc1.getStats()) as RTCStatsReport;
-      const report = flatten(stats);
-      // Audio levels are present inside stats of type `media-source` and of kind `audio`
-      const audioMediaSourceStats = report.find(
-        (stat) =>
-          stat.type === 'media-source' &&
-          (stat as RTCRtpStreamStats).kind === 'audio',
-      ) as BaseStats;
-      if (audioMediaSourceStats) {
-        const { audioLevel } = audioMediaSourceStats;
-        if (audioLevel) {
-          if (audioLevel >= AUDIO_LEVEL_THRESHOLD) {
-            onSoundDetectedStateChanged({
-              isSoundDetected: true,
-              audioLevel,
-            });
-          } else {
-            onSoundDetectedStateChanged({
-              isSoundDetected: false,
-              audioLevel: 0,
-            });
+    const initialBaselineNoiseLevel = 0.13;
+    let baselineNoiseLevel = initialBaselineNoiseLevel;
+    let speechDetected = false;
+    let intervalId: NodeJS.Timeout | undefined;
+    let speechTimer: NodeJS.Timeout | undefined;
+    let silenceTimer: NodeJS.Timeout | undefined;
+    let audioLevelHistory = []; // Store recent audio levels for smoother detection
+    const historyLength = 10;
+    const silenceThreshold = 1.1;
+    const resetThreshold = 0.9;
+    const speechTimeout = 500; // Speech is set to true after 500ms of audio detection
+    const silenceTimeout = 5000; // Reset baseline after 5 seconds of silence
+
+    const checkAudioLevel = async () => {
+      try {
+        const stats = (await this.pc1.getStats()) as RTCStatsReport;
+        const report = flatten(stats);
+        // Audio levels are present inside stats of type `media-source` and of kind `audio`
+        const audioMediaSourceStats = report.find(
+          (stat) =>
+            stat.type === 'media-source' &&
+            (stat as RTCRtpStreamStats).kind === 'audio',
+        ) as BaseStats;
+        if (audioMediaSourceStats) {
+          const { audioLevel } = audioMediaSourceStats;
+          if (audioLevel) {
+            // Update audio level history (with max historyLength sized array)
+            audioLevelHistory.push(audioLevel);
+            if (audioLevelHistory.length > historyLength) {
+              audioLevelHistory.shift();
+            }
+
+            // Calculate average audio level
+            const avgAudioLevel =
+              audioLevelHistory.reduce((a, b) => a + b, 0) /
+              audioLevelHistory.length;
+
+            // Update baseline (if necessary) based on silence detection
+            if (avgAudioLevel < baselineNoiseLevel * silenceThreshold) {
+              if (!silenceTimer) {
+                silenceTimer = setTimeout(() => {
+                  baselineNoiseLevel = Math.min(
+                    avgAudioLevel * resetThreshold,
+                    initialBaselineNoiseLevel,
+                  );
+                }, silenceTimeout);
+              }
+            } else {
+              clearTimeout(silenceTimer);
+              silenceTimer = undefined;
+            }
+
+            // Speech detection with hysteresis
+            if (avgAudioLevel > baselineNoiseLevel * 1.5) {
+              if (!speechDetected) {
+                speechDetected = true;
+                onSoundDetectedStateChanged({
+                  isSoundDetected: true,
+                  audioLevel,
+                });
+              }
+
+              clearTimeout(speechTimer);
+
+              speechTimer = setTimeout(() => {
+                speechDetected = false;
+                onSoundDetectedStateChanged({
+                  isSoundDetected: false,
+                  audioLevel: 0,
+                });
+              }, speechTimeout);
+            }
           }
         }
+      } catch (error) {
+        const logger = getLogger(['RNSpeechDetector']);
+        logger('error', 'error checking audio level from stats', error);
       }
-    }, 1000);
+    };
+
+    // Call checkAudioLevel periodically (every 100ms)
+    intervalId = setInterval(checkAudioLevel, 100);
 
     return () => {
-      clearInterval(this.intervalId);
+      clearInterval(intervalId);
+      clearTimeout(speechTimer);
+      clearTimeout(silenceTimer);
     };
   }
 

package/src/store/CallState.ts

@@ -21,7 +21,6 @@ import { CallStatsReport } from '../stats';
 import {
   BlockedUserEvent,
   CallClosedCaption,
-  CallHLSBroadcastingStartedEvent,
   CallIngressResponse,
   CallMemberAddedEvent,
   CallMemberRemovedEvent,
@@ -42,7 +41,7 @@ import {
   UnblockedUserEvent,
   UpdatedCallPermissionsEvent,
   UserResponse,
-  WSEvent,
+  VideoEvent,
 } from '../gen/coordinator';
 import { Timestamp } from '../gen/google/protobuf/timestamp';
 import { ReconnectDetails } from '../gen/video/sfu/event/events';
@@ -60,7 +59,7 @@ import { hasScreenShare } from '../helpers/participantUtils';
  */
 const defaultEgress: EgressResponse = {
   broadcasting: false,
-  hls: { playlist_url: '' },
+  hls: { playlist_url: '', status: '' },
   rtmps: [],
 };
 
@@ -314,8 +313,8 @@ export class CallState {
   private closedCaptionsTasks = new Map<string, NodeJS.Timeout>();
 
   private readonly eventHandlers: {
-    [EventType in WSEvent['type']]:
-      | ((event: Extract<WSEvent, { type: EventType }>) => void)
+    [EventType in VideoEvent['type']]:
+      | ((event: Extract<VideoEvent, { type: EventType }>) => void)
       | undefined;
   };
 
@@ -414,21 +413,16 @@ export class CallState {
 
     this.eventHandlers = {
       // these events are not updating the call state:
-      'call.deleted': undefined,
       'call.permission_request': undefined,
       'call.recording_ready': undefined,
+      'call.rtmp_broadcast_failed': undefined,
+      'call.rtmp_broadcast_started': undefined,
+      'call.rtmp_broadcast_stopped': undefined,
       'call.transcription_ready': undefined,
       'call.user_muted': undefined,
       'connection.error': undefined,
       'connection.ok': undefined,
       'health.check': undefined,
-      'user.banned': undefined,
-      'user.deactivated': undefined,
-      'user.deleted': undefined,
-      'user.muted': undefined,
-      'user.presence.changed': undefined,
-      'user.reactivated': undefined,
-      'user.unbanned': undefined,
       'user.updated': undefined,
       custom: undefined,
 
@@ -446,12 +440,15 @@ export class CallState {
         this.setCurrentValue(this.captioningSubject, false);
       },
       'call.created': (e) => this.updateFromCallResponse(e.call),
+      'call.deleted': (e) => this.updateFromCallResponse(e.call),
       'call.ended': (e) => {
         this.updateFromCallResponse(e.call);
         this.setCurrentValue(this.endedBySubject, e.user);
       },
       'call.hls_broadcasting_failed': this.updateFromHLSBroadcastingFailed,
-      'call.hls_broadcasting_started': this.updateFromHLSBroadcastStarted,
+      'call.hls_broadcasting_started': (e) => {
+        this.updateFromCallResponse(e.call);
+      },
       'call.hls_broadcasting_stopped': this.updateFromHLSBroadcastStopped,
       'call.live_started': (e) => this.updateFromCallResponse(e.call),
       'call.member_added': this.updateFromMemberAdded,
@@ -1012,7 +1009,7 @@ export class CallState {
    *
    * @param event the video event that our backend sent us.
    */
-  updateFromEvent = (event: WSEvent) => {
+  updateFromEvent = (event: VideoEvent) => {
     const update = this.eventHandlers[event.type];
     if (update) {
       update(event as any);
@@ -1205,6 +1202,10 @@ export class CallState {
     this.setCurrentValue(this.egressSubject, (egress = defaultEgress) => ({
       ...egress,
       broadcasting: false,
+      hls: {
+        ...egress.hls!,
+        status: '',
+      },
     }));
   };
 
@@ -1212,18 +1213,9 @@ export class CallState {
     this.setCurrentValue(this.egressSubject, (egress = defaultEgress) => ({
       ...egress,
       broadcasting: false,
-    }));
-  };
-
-  private updateFromHLSBroadcastStarted = (
-    event: CallHLSBroadcastingStartedEvent,
-  ) => {
-    this.setCurrentValue(this.egressSubject, (egress = defaultEgress) => ({
-      ...egress,
-      broadcasting: true,
       hls: {
-        ...egress.hls,
-        playlist_url: event.hls_playlist_url,
+        ...egress.hls!,
+        status: '',
       },
     }));
   };

package/src/store/__tests__/CallState.test.ts

@@ -669,18 +669,30 @@ describe('CallState', () => {
             broadcasting: false,
             hls: {
               playlist_url: '',
+              status: 'starting',
             },
           },
         });
         // @ts-ignore
         state.updateFromEvent({
           type: 'call.hls_broadcasting_started',
-          hls_playlist_url: 'https://example.com/playlist.m3u8',
+          // @ts-expect-error incomplete data
+          call: {
+            egress: {
+              broadcasting: true,
+              hls: {
+                playlist_url: 'https://example.com/playlist.m3u8',
+                status: 'running',
+              },
+              rtmps: [],
+            },
+          },
         });
         expect(state.egress?.broadcasting).toBe(true);
         expect(state.egress?.hls?.playlist_url).toBe(
           'https://example.com/playlist.m3u8',
         );
+        expect(state.egress?.hls?.status).toBe('running');
       });
 
       it('handles call.hls_broadcasting_stopped events', () => {

package/dist/src/gen/shims.d.ts

@@ -1,41 +0,0 @@
-import type { TargetResolution } from './coordinator';
-export type TargetResolutionRequest = TargetResolution;
-export type TargetResolutionResponse = Required<TargetResolution>;
-/**
- *
- * @export
- * @interface Bound
- */
-export interface Bound {
-    /**
-     *
-     * @type {boolean}
-     * @memberof Bound
-     */
-    inclusive: boolean;
-    /**
-     *
-     * @type {number}
-     * @memberof Bound
-     */
-    value: number;
-}
-/**
- *
- * @export
- * @interface DailyAggregateCallStatsResponse
- */
-export interface DailyAggregateCallStatsResponse<T = any> {
-    /**
-     *
-     * @type {string}
-     * @memberof DailyAggregateCallStatsResponse<T>
-     */
-    date: string;
-    /**
-     *
-     * @type {T}
-     * @memberof DailyAggregateCallStatsResponse<T>
-     */
-    report: T;
-}

package/src/gen/shims.ts

@@ -1,44 +0,0 @@
-import type { TargetResolution } from './coordinator';
-
-export type TargetResolutionRequest = TargetResolution;
-export type TargetResolutionResponse = Required<TargetResolution>;
-
-/**
- *
- * @export
- * @interface Bound
- */
-export interface Bound {
-  /**
-   *
-   * @type {boolean}
-   * @memberof Bound
-   */
-  inclusive: boolean;
-  /**
-   *
-   * @type {number}
-   * @memberof Bound
-   */
-  value: number;
-}
-
-/**
- *
- * @export
- * @interface DailyAggregateCallStatsResponse
- */
-export interface DailyAggregateCallStatsResponse<T = any> {
-  /**
-   *
-   * @type {string}
-   * @memberof DailyAggregateCallStatsResponse<T>
-   */
-  date: string;
-  /**
-   *
-   * @type {T}
-   * @memberof DailyAggregateCallStatsResponse<T>
-   */
-  report: T;
-}