@stream-io/video-client 1.41.2 → 1.42.0

package/dist/index.cjs.js

@@ -4809,7 +4809,7 @@ const hasVideo = (p) => p.publishedTracks.includes(TrackType.VIDEO);
  *
  * @param p the participant to check.
  */
-const hasAudio = (p) => p.publishedTracks.includes(TrackType.AUDIO);
+const hasAudio$1 = (p) => p.publishedTracks.includes(TrackType.AUDIO);
 /**
  * Check if a participant is screen sharing.
  *
@@ -4910,8 +4910,8 @@ const publishingVideo = (a, b) => {
  * @param b the second participant.
  */
 const publishingAudio = (a, b) => {
-    const hasA = hasAudio(a);
-    const hasB = hasAudio(b);
+    const hasA = hasAudio$1(a);
+    const hasB = hasAudio$1(b);
     if (hasA && !hasB)
         return -1;
     if (!hasA && hasB)
@@ -6208,7 +6208,7 @@ const getSdkVersion = (sdk) => {
     return sdk ? `${sdk.major}.${sdk.minor}.${sdk.patch}` : '0.0.0-development';
 };
 
-const version = "1.41.2";
+const version = "1.42.0";
 const [major, minor, patch] = version.split('.');
 let sdkInfo = {
     type: SdkType.PLAIN_JAVASCRIPT,
@@ -6531,6 +6531,16 @@ const createStatsReporter = ({ subscriber, publisher, state, datacenter, polling
         stop,
     };
 };
+/**
+ * Extracts camera statistics from a media source.
+ *
+ * @param mediaSource the media source stats to extract camera info from.
+ */
+const getCameraStats = (mediaSource) => ({
+    frameRate: mediaSource.framesPerSecond,
+    frameWidth: mediaSource.width,
+    frameHeight: mediaSource.height,
+});
 /**
  * Transforms raw RTC stats into a slimmer and uniform across browsers format.
  *
@@ -6556,6 +6566,7 @@ const transform = (report, opts) => {
         }
         let trackType;
         let audioLevel;
+        let camera;
         let concealedSamples;
         let concealmentEvents;
         let packetsReceived;
@@ -6571,6 +6582,9 @@ const transform = (report, opts) => {
                     typeof mediaSource.audioLevel === 'number') {
                     audioLevel = mediaSource.audioLevel;
                 }
+                if (trackKind === 'video') {
+                    camera = getCameraStats(mediaSource);
+                }
             }
         }
         else if (kind === 'subscriber' && trackKind === 'audio') {
@@ -6604,6 +6618,7 @@ const transform = (report, opts) => {
             concealmentEvents,
             packetsReceived,
             packetsLost,
+            camera,
         };
     });
     return {
@@ -6623,6 +6638,7 @@ const getEmptyVideoStats = (stats) => {
         highestFrameWidth: 0,
         highestFrameHeight: 0,
         highestFramesPerSecond: 0,
+        camera: {},
         codec: '',
         codecPerTrackType: {},
         timestamp: Date.now(),
@@ -6667,6 +6683,9 @@ const aggregate = (stats) => {
             acc.highestFramesPerSecond = stream.framesPerSecond || 0;
             maxArea = streamArea;
         }
+        if (stream.trackType === TrackType.VIDEO) {
+            acc.camera = stream.camera;
+        }
         qualityLimitationReasons.add(stream.qualityLimitationReason || '');
         return acc;
     }, aggregatedStats);
@@ -6731,6 +6750,7 @@ const aggregateAudio = (stats) => {
 class SfuStatsReporter {
     constructor(sfuClient, { options, clientDetails, subscriber, publisher, microphone, camera, state, tracer, unifiedSessionId, }) {
         this.logger = videoLoggerSystem.getLogger('SfuStatsReporter');
+        this.reportCount = 0;
         this.inputDevices = new Map();
         this.observeDevice = (device, kind) => {
             const { browserPermissionState$ } = device.state;
@@ -6832,17 +6852,31 @@ class SfuStatsReporter {
                 throw err;
             }
         };
+        this.scheduleNextReport = () => {
+            const intervals = [1500, 3000, 3000, 5000];
+            if (this.reportCount < intervals.length) {
+                this.timeoutId = setTimeout(() => {
+                    this.flush();
+                    this.reportCount++;
+                    this.scheduleNextReport();
+                }, intervals[this.reportCount]);
+            }
+            else {
+                clearInterval(this.intervalId);
+                this.intervalId = setInterval(() => {
+                    this.flush();
+                }, this.options.reporting_interval_ms);
+            }
+        };
         this.start = () => {
             if (this.options.reporting_interval_ms <= 0)
                 return;
             this.observeDevice(this.microphone, 'mic');
             this.observeDevice(this.camera, 'camera');
+            this.reportCount = 0;
             clearInterval(this.intervalId);
-            this.intervalId = setInterval(() => {
-                this.run().catch((err) => {
-                    this.logger.warn('Failed to report stats', err);
-                });
-            }, this.options.reporting_interval_ms);
+            clearTimeout(this.timeoutId);
+            this.scheduleNextReport();
         };
         this.stop = () => {
             this.unsubscribeDevicePermissionsSubscription?.();
@@ -6854,20 +6888,13 @@ class SfuStatsReporter {
             this.intervalId = undefined;
             clearTimeout(this.timeoutId);
             this.timeoutId = undefined;
+            this.reportCount = 0;
         };
         this.flush = () => {
             this.run().catch((err) => {
                 this.logger.warn('Failed to flush report stats', err);
             });
         };
-        this.scheduleOne = (timeout) => {
-            clearTimeout(this.timeoutId);
-            this.timeoutId = setTimeout(() => {
-                this.run().catch((err) => {
-                    this.logger.warn('Failed to report stats', err);
-                });
-            }, timeout);
-        };
         this.sfuClient = sfuClient;
         this.options = options;
         this.subscriber = subscriber;
@@ -10372,7 +10399,7 @@ const getAudioStream = async (trackConstraints, tracer) => {
             videoLoggerSystem
                 .getLogger('devices')
                 .warn('Failed to get audio stream, will try again with relaxed constraints', { error, constraints, relaxedConstraints });
-            return getAudioStream(relaxedConstraints);
+            return getAudioStream(relaxedConstraints, tracer);
         }
         videoLoggerSystem.getLogger('devices').error('Failed to get audio stream', {
             error,
@@ -11369,9 +11396,6 @@ class MicrophoneManagerState extends AudioDeviceManagerState {
     }
 }
 
-const DETECTION_FREQUENCY_IN_MS = 500;
-const AUDIO_LEVEL_THRESHOLD = 150;
-const FFT_SIZE = 128;
 /**
  * Creates a new sound detector.
  *
@@ -11381,7 +11405,7 @@ const FFT_SIZE = 128;
  * @returns a clean-up function which once invoked stops the sound detector.
  */
 const createSoundDetector = (audioStream, onSoundDetectedStateChanged, options = {}) => {
-    const { detectionFrequencyInMs = DETECTION_FREQUENCY_IN_MS, audioLevelThreshold = AUDIO_LEVEL_THRESHOLD, fftSize = FFT_SIZE, destroyStreamOnStop = true, } = options;
+    const { detectionFrequencyInMs = 500, audioLevelThreshold = 150, fftSize = 128, destroyStreamOnStop = true, } = options;
     const audioContext = new AudioContext();
     const analyser = audioContext.createAnalyser();
     analyser.fftSize = fftSize;
@@ -11422,6 +11446,101 @@ const createSoundDetector = (audioStream, onSoundDetectedStateChanged, options =
     };
 };
 
+/**
+ * Analyzes frequency data to determine if audio is being captured.
+ */
+const hasAudio = (analyser, threshold) => {
+    const data = new Uint8Array(analyser.frequencyBinCount);
+    analyser.getByteFrequencyData(data);
+    return data.some((value) => value > threshold);
+};
+/** Helper for "no event" transitions */
+const noEmit = (nextState) => ({
+    shouldEmit: false,
+    nextState,
+});
+/** Helper for event-emitting transitions */
+const emit = (capturesAudio, nextState) => ({ shouldEmit: true, nextState, capturesAudio });
+/**
+ * State transition function - computes next state and whether to emit an event.
+ */
+const transitionState = (state, audioDetected, options) => {
+    if (audioDetected) {
+        return state.kind === 'IDLE' || state.kind === 'EMITTING'
+            ? emit(true, state)
+            : noEmit(state);
+    }
+    const { noAudioThresholdMs, emitIntervalMs } = options;
+    const now = Date.now();
+    switch (state.kind) {
+        case 'IDLE':
+            return noEmit({ kind: 'DETECTING', noAudioStartTime: now });
+        case 'DETECTING': {
+            const { noAudioStartTime } = state;
+            const elapsed = now - noAudioStartTime;
+            return elapsed >= noAudioThresholdMs
+                ? emit(false, { kind: 'EMITTING', noAudioStartTime, lastEmitTime: now })
+                : noEmit(state);
+        }
+        case 'EMITTING': {
+            const timeSinceLastEmit = now - state.lastEmitTime;
+            return timeSinceLastEmit >= emitIntervalMs
+                ? emit(false, { ...state, lastEmitTime: now })
+                : noEmit(state);
+        }
+    }
+};
+/**
+ * Creates and configures an audio analyzer for the given stream.
+ */
+const createAudioAnalyzer = (audioStream, fftSize) => {
+    const audioContext = new AudioContext();
+    const analyser = audioContext.createAnalyser();
+    analyser.fftSize = fftSize;
+    const microphone = audioContext.createMediaStreamSource(audioStream);
+    microphone.connect(analyser);
+    return { audioContext, analyser };
+};
+/**
+ * Creates a new no-audio detector that monitors continuous absence of audio on an audio stream.
+ *
+ * @param audioStream the audio stream to observe.
+ * @param options custom options for the no-audio detector.
+ * @returns a cleanup function which once invoked stops the no-audio detector.
+ */
+const createNoAudioDetector = (audioStream, options) => {
+    const { detectionFrequencyInMs = 350, audioLevelThreshold = 0, fftSize = 256, onCaptureStatusChange, } = options;
+    let state = { kind: 'IDLE' };
+    const { audioContext, analyser } = createAudioAnalyzer(audioStream, fftSize);
+    const detectionIntervalId = setInterval(() => {
+        const [audioTrack] = audioStream.getAudioTracks();
+        if (!audioTrack?.enabled || audioTrack.readyState === 'ended') {
+            state = { kind: 'IDLE' };
+            return;
+        }
+        const audioDetected = hasAudio(analyser, audioLevelThreshold);
+        const transition = transitionState(state, audioDetected, options);
+        state = transition.nextState;
+        if (!transition.shouldEmit)
+            return;
+        const { capturesAudio } = transition;
+        onCaptureStatusChange(capturesAudio);
+        if (capturesAudio) {
+            stop().catch((err) => {
+                const logger = videoLoggerSystem.getLogger('NoAudioDetector');
+                logger.error('Error stopping no-audio detector', err);
+            });
+        }
+    }, detectionFrequencyInMs);
+    async function stop() {
+        clearInterval(detectionIntervalId);
+        if (audioContext.state !== 'closed') {
+            await audioContext.close();
+        }
+    }
+    return stop;
+};
+
 class RNSpeechDetector {
     constructor(externalAudioStream) {
         this.pc1 = new RTCPeerConnection({});
@@ -11433,16 +11552,10 @@ class RNSpeechDetector {
      */
     async start(onSoundDetectedStateChanged) {
         try {
-            let audioStream;
-            if (this.externalAudioStream != null) {
-                audioStream = this.externalAudioStream;
-            }
-            else {
-                audioStream = await navigator.mediaDevices.getUserMedia({
-                    audio: true,
-                });
-                this.audioStream = audioStream;
-            }
+            const audioStream = this.externalAudioStream != null
+                ? this.externalAudioStream
+                : await navigator.mediaDevices.getUserMedia({ audio: true });
+            this.audioStream = audioStream;
             this.pc1.addEventListener('icecandidate', async (e) => {
                 await this.pc2.addIceCandidate(e.candidate);
             });
@@ -11466,9 +11579,9 @@ class RNSpeechDetector {
             const answer = await this.pc2.createAnswer();
             await this.pc1.setRemoteDescription(answer);
             await this.pc2.setLocalDescription(answer);
-            const unsub = this.onSpeakingDetectedStateChange(onSoundDetectedStateChanged);
+            const unsubscribe = this.onSpeakingDetectedStateChange(onSoundDetectedStateChanged);
             return () => {
-                unsub();
+                unsubscribe();
                 this.stop();
             };
         }
@@ -11508,7 +11621,7 @@ class RNSpeechDetector {
         const silenceTimeout = 5000; // Reset baseline after 5 seconds of silence
         const checkAudioLevel = async () => {
             try {
-                const stats = (await this.pc1.getStats());
+                const stats = await this.pc1.getStats();
                 const report = flatten(stats);
                 // Audio levels are present inside stats of type `media-source` and of kind `audio`
                 const audioMediaSourceStats = report.find((stat) => stat.type === 'media-source' &&
@@ -11562,8 +11675,7 @@ class RNSpeechDetector {
                 logger.error('error checking audio level from stats', error);
             }
         };
-        // Call checkAudioLevel periodically (every 100ms)
-        const intervalId = setInterval(checkAudioLevel, 100);
+        const intervalId = setInterval(checkAudioLevel, 250);
         return () => {
             clearInterval(intervalId);
             clearTimeout(speechTimer);
@@ -11589,8 +11701,11 @@ class MicrophoneManager extends AudioDeviceManager {
         super(call, new MicrophoneManagerState(disableMode), TrackType.AUDIO);
         this.speakingWhileMutedNotificationEnabled = true;
         this.soundDetectorConcurrencyTag = Symbol('soundDetectorConcurrencyTag');
+        this.silenceThresholdMs = 5000;
     }
     setup() {
+        if (this.areSubscriptionsSetUp)
+            return;
         super.setup();
         this.subscriptions.push(createSafeAsyncSubscription(rxjs.combineLatest([
             this.call.state.callingState$,
@@ -11607,7 +11722,7 @@ class MicrophoneManager extends AudioDeviceManager {
                 if (!this.speakingWhileMutedNotificationEnabled)
                     return;
                 if (ownCapabilities.includes(OwnCapability.SEND_AUDIO)) {
-                    if (status === 'disabled') {
+                    if (status !== 'enabled') {
                         await this.startSpeakingWhileMutedDetection(deviceId);
                     }
                     else {
@@ -11656,6 +11771,40 @@ class MicrophoneManager extends AudioDeviceManager {
                 });
             }
         }));
+        if (!isReactNative()) {
+            const unsubscribe = createSafeAsyncSubscription(rxjs.combineLatest([this.state.status$, this.state.mediaStream$]), async ([status, mediaStream]) => {
+                if (this.noAudioDetectorCleanup) {
+                    const cleanup = this.noAudioDetectorCleanup;
+                    this.noAudioDetectorCleanup = undefined;
+                    await cleanup().catch((err) => {
+                        this.logger.warn('Failed to stop no-audio detector', err);
+                    });
+                }
+                if (status !== 'enabled' || !mediaStream)
+                    return;
+                if (this.silenceThresholdMs <= 0)
+                    return;
+                const deviceId = this.state.selectedDevice;
+                const devices = getCurrentValue(this.listDevices());
+                const label = devices.find((d) => d.deviceId === deviceId)?.label;
+                this.noAudioDetectorCleanup = createNoAudioDetector(mediaStream, {
+                    noAudioThresholdMs: this.silenceThresholdMs,
+                    emitIntervalMs: this.silenceThresholdMs,
+                    onCaptureStatusChange: (capturesAudio) => {
+                        const event = {
+                            type: 'mic.capture_report',
+                            call_cid: this.call.cid,
+                            capturesAudio,
+                            deviceId,
+                            label,
+                        };
+                        this.call.tracer.trace('mic.capture_report', event);
+                        this.call.streamClient.dispatchEvent(event);
+                    },
+                });
+            });
+            this.subscriptions.push(unsubscribe);
+        }
     }
     /**
      * Enables noise cancellation for the microphone.
@@ -11753,6 +11902,45 @@ class MicrophoneManager extends AudioDeviceManager {
         this.speakingWhileMutedNotificationEnabled = false;
         await this.stopSpeakingWhileMutedDetection();
     }
+    /**
+     * Sets the silence threshold in milliseconds for no-audio detection.
+     * When the microphone is enabled but produces no audio for this duration,
+     * a 'mic.capture_report' event will be emitted.
+     *
+     * @param thresholdMs the threshold in milliseconds (default: 5000).
+     *   Set to 0 or a negative value to disable no-audio detection.
+     */
+    setSilenceThreshold(thresholdMs) {
+        this.silenceThresholdMs = thresholdMs;
+    }
+    /**
+     * Performs audio capture test on a specific microphone.
+     *
+     * This method is only available in browser environments (not React Native).
+     *
+     * @param deviceId The device ID to test.
+     * @param options Optional test configuration.
+     * @returns Promise that resolves with the test result (true or false).
+     */
+    async performTest(deviceId, options) {
+        if (isReactNative())
+            throw new Error('Not available in React Native');
+        const stream = await this.getStream({ deviceId: { exact: deviceId } });
+        const { testDurationMs = 3000 } = options || {};
+        const { promise, resolve } = promiseWithResolvers();
+        const cleanup = createNoAudioDetector(stream, {
+            noAudioThresholdMs: testDurationMs,
+            emitIntervalMs: testDurationMs,
+            onCaptureStatusChange: async (capturesAudio) => {
+                resolve(capturesAudio);
+                await cleanup().catch((err) => {
+                    this.logger.warn('Failed to stop detector during test', err);
+                });
+                disposeOfMediaStream(stream);
+            },
+        });
+        return promise;
+    }
     /**
      * Applies the audio settings to the microphone.
      * @param settings the audio settings to apply.
@@ -11801,13 +11989,12 @@ class MicrophoneManager extends AudioDeviceManager {
     }
     async startSpeakingWhileMutedDetection(deviceId) {
         await withoutConcurrency(this.soundDetectorConcurrencyTag, async () => {
-            await this.stopSpeakingWhileMutedDetection();
             if (isReactNative()) {
                 this.rnSpeechDetector = new RNSpeechDetector();
                 const unsubscribe = await this.rnSpeechDetector.start((event) => {
                     this.state.setSpeakingWhileMuted(event.isSoundDetected);
                 });
-                this.soundDetectorCleanup = () => {
+                this.soundDetectorCleanup = async () => {
                     unsubscribe();
                     this.rnSpeechDetector = undefined;
                 };
@@ -11815,6 +12002,7 @@ class MicrophoneManager extends AudioDeviceManager {
             else {
                 // Need to start a new stream that's not connected to publisher
                 const stream = await this.getStream({
+                    ...this.state.defaultConstraints,
                     deviceId: { exact: deviceId },
                 });
                 this.soundDetectorCleanup = createSoundDetector(stream, (event) => {
@@ -11894,6 +12082,8 @@ class ScreenShareManager extends AudioDeviceManager {
         super(call, new ScreenShareState(), TrackType.SCREEN_SHARE);
     }
     setup() {
+        if (this.areSubscriptionsSetUp)
+            return;
         super.setup();
         this.subscriptions.push(createSubscription(this.call.state.settings$, (settings) => {
             const maybeTargetResolution = settings?.screensharing.target_resolution;
@@ -12240,6 +12430,9 @@ class Call {
                 this.leaveCallHooks.add(registerEventHandlers(this, this.dispatcher));
                 this.registerEffects();
                 this.registerReconnectHandlers();
+                // Set up the device managers again. Although this is already done
+                // in the DeviceManager's constructor, they'll need to be re-set up
+                // in the cases where a call instance is recycled (join -> leave -> join).
                 this.camera.setup();
                 this.microphone.setup();
                 this.screenShare.setup();
@@ -12467,6 +12660,8 @@ class Call {
                 }
                 this.statsReporter?.stop();
                 this.statsReporter = undefined;
+                const leaveReason = message ?? reason ?? 'user is leaving the call';
+                this.tracer.trace('call.leaveReason', leaveReason);
                 this.sfuStatsReporter?.flush();
                 this.sfuStatsReporter?.stop();
                 this.sfuStatsReporter = undefined;
@@ -12474,7 +12669,7 @@ class Call {
                 this.subscriber = undefined;
                 this.publisher?.dispose();
                 this.publisher = undefined;
-                await this.sfuClient?.leaveAndClose(message ?? reason ?? 'user is leaving the call');
+                await this.sfuClient?.leaveAndClose(leaveReason);
                 this.sfuClient = undefined;
                 this.dynascaleManager.setSfuClient(undefined);
                 await this.dynascaleManager.dispose();
@@ -12616,6 +12811,7 @@ class Call {
          * Unless you are implementing a custom "ringing" flow, you should not use this method.
          */
         this.accept = async () => {
+            this.tracer.trace('call.accept', '');
             return this.streamClient.post(`${this.streamClientBasePath}/accept`);
         };
         /**
@@ -12628,6 +12824,7 @@ class Call {
          * @param reason the reason for rejecting the call.
          */
         this.reject = async (reason = 'decline') => {
+            this.tracer.trace('call.reject', reason);
             return this.streamClient.post(`${this.streamClientBasePath}/reject`, { reason: reason });
         };
         /**
@@ -13391,11 +13588,6 @@ class Call {
                     trackTypes.push(screenShareAudio);
                 }
             }
-            if (track.kind === 'video') {
-                // schedules calibration report - the SFU will use the performance stats
-                // to adjust the quality thresholds as early as possible
-                this.sfuStatsReporter?.scheduleOne(3000);
-            }
             await this.updateLocalStreamState(mediaStream, ...trackTypes);
         };
         /**
@@ -15338,7 +15530,7 @@ class StreamClient {
         this.getUserAgent = () => {
             if (!this.cachedUserAgent) {
                 const { clientAppIdentifier = {} } = this.options;
-                const { sdkName = 'js', sdkVersion = "1.41.2", ...extras } = clientAppIdentifier;
+                const { sdkName = 'js', sdkVersion = "1.42.0", ...extras } = clientAppIdentifier;
                 this.cachedUserAgent = [
                     `stream-video-${sdkName}-v${sdkVersion}`,
                     ...Object.entries(extras).map(([key, value]) => `${key}=${value}`),
@@ -16064,7 +16256,7 @@ exports.getVideoBrowserPermission = getVideoBrowserPermission;
 exports.getVideoDevices = getVideoDevices;
 exports.getVideoStream = getVideoStream;
 exports.getWebRTCInfo = getWebRTCInfo;
-exports.hasAudio = hasAudio;
+exports.hasAudio = hasAudio$1;
 exports.hasPausedTrack = hasPausedTrack;
 exports.hasScreenShare = hasScreenShare;
 exports.hasScreenShareAudio = hasScreenShareAudio;