@stream-io/video-client 1.23.2 → 1.23.4

package/dist/index.es.js

@@ -4,7 +4,7 @@ import { ServiceType, stackIntercept, RpcError } from '@protobuf-ts/runtime-rpc'
 import axios from 'axios';
 export { AxiosError } from 'axios';
 import { TwirpFetchTransport, TwirpErrorCode } from '@protobuf-ts/twirp-transport';
-import { ReplaySubject, combineLatest, BehaviorSubject, map, shareReplay, distinctUntilChanged, takeWhile, distinctUntilKeyChanged, fromEventPattern, startWith, concatMap, merge, from, fromEvent, debounceTime, pairwise, of } from 'rxjs';
+import { ReplaySubject, combineLatest, BehaviorSubject, shareReplay, map, distinctUntilChanged, takeWhile, distinctUntilKeyChanged, fromEventPattern, startWith, concatMap, merge, from, fromEvent, debounceTime, pairwise, of } from 'rxjs';
 import { UAParser } from 'ua-parser-js';
 import { parse } from 'sdp-transform';
 import https from 'https';
@@ -5032,6 +5032,9 @@ class CallState {
                 return nextQueue.slice(-maxVisibleCaptions);
             });
         };
+        this.rawParticipants$ = this.participantsSubject
+            .asObservable()
+            .pipe(shareReplay({ bufferSize: 1, refCount: true }));
         this.participants$ = this.participantsSubject.asObservable().pipe(
         // maintain stable-sort by mutating the participants stored
         // in the original subject
@@ -5205,6 +5208,12 @@ class CallState {
     get participants() {
         return this.getCurrentValue(this.participants$);
     }
+    /**
+     * The stable list of participants in the current call, unsorted.
+     */
+    get rawParticipants() {
+        return this.getCurrentValue(this.rawParticipants$);
+    }
     /**
      * The local participant in the current call.
      */
@@ -5677,7 +5686,7 @@ const aggregate = (stats) => {
     return report;
 };
 
-const version = "1.23.2";
+const version = "1.23.4";
 const [major, minor, patch] = version.split('.');
 let sdkInfo = {
     type: SdkType.PLAIN_JAVASCRIPT,
@@ -6750,17 +6759,19 @@ const withSimulcastConstraints = (settings, optimalVideoLayers, useSingleLayer)
     let layers;
     const size = Math.max(settings.width || 0, settings.height || 0);
     if (size <= 320) {
-        // provide only one layer 320x240 (q), the one with the highest quality
+        // provide only one layer 320x240 (f), the one with the highest quality
         layers = optimalVideoLayers.filter((layer) => layer.rid === 'f');
     }
     else if (size <= 640) {
-        // provide two layers, 160x120 (q) and 640x480 (h)
-        layers = optimalVideoLayers.filter((layer) => layer.rid !== 'h');
+        // provide two layers, 320x240 (h) and 640x480 (f)
+        layers = optimalVideoLayers.filter((layer) => layer.rid !== 'q');
     }
     else {
         // provide three layers for sizes > 640x480
         layers = optimalVideoLayers;
     }
+    // we might have removed some layers, so we need to reassign the rid
+    // to match the expected order of [q, h, f] for simulcast
     const ridMapping = ['q', 'h', 'f'];
     return layers.map((layer, index, arr) => ({
         ...layer,
@@ -7561,6 +7572,8 @@ class StreamSfuClient {
             const eventsToTrace = {
                 callEnded: true,
                 changePublishQuality: true,
+                changePublishOptions: true,
+                connectionQualityChanged: true,
                 error: true,
                 goAway: true,
             };
@@ -8434,6 +8447,20 @@ class DynascaleManager {
                     true,
             };
         }), shareReplay(1));
+        /**
+         * Disposes the allocated resources and closes the audio context if it was created.
+         */
+        this.dispose = async () => {
+            if (this.pendingSubscriptionsUpdate) {
+                clearTimeout(this.pendingSubscriptionsUpdate);
+            }
+            const context = this.getOrCreateAudioContext();
+            if (context && context.state !== 'closed') {
+                document.removeEventListener('click', this.resumeAudioContext);
+                await context.close();
+                this.audioContext = undefined;
+            }
+        };
         this.setVideoTrackSubscriptionOverrides = (override, sessionIds) => {
             if (!sessionIds) {
                 return setCurrentValue(this.videoTrackSubscriptionOverridesSubject, override ? { [globalOverrideKey]: override } : {});
@@ -8549,7 +8576,7 @@ class DynascaleManager {
                 });
                 this.applyTrackSubscriptions(debounceType);
             };
-            const participant$ = this.callState.participants$.pipe(map((participants) => participants.find((participant) => participant.sessionId === sessionId)), takeWhile((participant) => !!participant), distinctUntilChanged(), shareReplay({ bufferSize: 1, refCount: true }));
+            const participant$ = this.callState.participants$.pipe(map((ps) => ps.find((p) => p.sessionId === sessionId)), takeWhile((participant) => !!participant), distinctUntilChanged(), shareReplay({ bufferSize: 1, refCount: true }));
             /**
              * Since the video elements are now being removed from the DOM (React SDK) upon
              * visibility change, this subscription is not in use an stays here only for the
@@ -8677,7 +8704,24 @@ class DynascaleManager {
             const participant = this.callState.findParticipantBySessionId(sessionId);
             if (!participant || participant.isLocalParticipant)
                 return;
-            const participant$ = this.callState.participants$.pipe(map((participants) => participants.find((p) => p.sessionId === sessionId)), takeWhile((p) => !!p), distinctUntilChanged(), shareReplay({ bufferSize: 1, refCount: true }));
+            const participant$ = this.callState.participants$.pipe(map((ps) => ps.find((p) => p.sessionId === sessionId)), takeWhile((p) => !!p), distinctUntilChanged(), shareReplay({ bufferSize: 1, refCount: true }));
+            const updateSinkId = (deviceId, audioContext) => {
+                if (!deviceId)
+                    return;
+                if ('setSinkId' in audioElement) {
+                    audioElement.setSinkId(deviceId).catch((e) => {
+                        this.logger('warn', `Can't to set AudioElement sinkId`, e);
+                    });
+                }
+                if (audioContext && 'setSinkId' in audioContext) {
+                    // @ts-expect-error setSinkId is not available in all browsers
+                    audioContext.setSinkId(deviceId).catch((e) => {
+                        this.logger('warn', `Can't to set AudioContext sinkId`, e);
+                    });
+                }
+            };
+            let sourceNode = undefined;
+            let gainNode = undefined;
             const updateMediaStreamSubscription = participant$
                 .pipe(distinctUntilKeyChanged(trackType === 'screenShareAudioTrack'
                 ? 'screenShareAudioStream'
@@ -8690,40 +8734,82 @@ class DynascaleManager {
                     return;
                 setTimeout(() => {
                     audioElement.srcObject = source ?? null;
-                    if (audioElement.srcObject) {
+                    if (!source)
+                        return;
+                    // Safari has a special quirk that prevents playing audio until the user
+                    // interacts with the page or focuses on the tab where the call happens.
+                    // This is a workaround for the issue where:
+                    // - A and B are in a call
+                    // - A switches to another tab
+                    // - B mutes their microphone and unmutes it
+                    // - A does not hear B's unmuted audio until they focus the tab
+                    const audioContext = this.getOrCreateAudioContext();
+                    if (audioContext) {
+                        // we will play audio through the audio context in Safari
+                        audioElement.muted = true;
+                        sourceNode?.disconnect();
+                        sourceNode = audioContext.createMediaStreamSource(source);
+                        gainNode ?? (gainNode = audioContext.createGain());
+                        gainNode.gain.value = p.audioVolume ?? this.speaker.state.volume;
+                        sourceNode.connect(gainNode).connect(audioContext.destination);
+                        this.resumeAudioContext();
+                    }
+                    else {
+                        // we will play audio directly through the audio element in other browsers
+                        audioElement.muted = false;
                         audioElement.play().catch((e) => {
-                            this.logger('warn', `Failed to play stream`, e);
+                            this.logger('warn', `Failed to play audio stream`, e);
                         });
-                        // audio output device shall be set after the audio element is played
-                        // otherwise, the browser will not pick it up, and will always
-                        // play audio through the system's default device
-                        const { selectedDevice } = this.speaker.state;
-                        if (selectedDevice && 'setSinkId' in audioElement) {
-                            audioElement.setSinkId(selectedDevice);
-                        }
                     }
+                    const { selectedDevice } = this.speaker.state;
+                    if (selectedDevice)
+                        updateSinkId(selectedDevice, audioContext);
                 });
             });
             const sinkIdSubscription = !('setSinkId' in audioElement)
                 ? null
                 : this.speaker.state.selectedDevice$.subscribe((deviceId) => {
-                    if (deviceId) {
-                        audioElement.setSinkId(deviceId);
-                    }
+                    const audioContext = this.getOrCreateAudioContext();
+                    updateSinkId(deviceId, audioContext);
                 });
             const volumeSubscription = combineLatest([
                 this.speaker.state.volume$,
                 participant$.pipe(distinctUntilKeyChanged('audioVolume')),
             ]).subscribe(([volume, p]) => {
-                audioElement.volume = p.audioVolume ?? volume;
+                const participantVolume = p.audioVolume ?? volume;
+                audioElement.volume = participantVolume;
+                if (gainNode)
+                    gainNode.gain.value = participantVolume;
             });
             audioElement.autoplay = true;
             return () => {
                 sinkIdSubscription?.unsubscribe();
                 volumeSubscription.unsubscribe();
                 updateMediaStreamSubscription.unsubscribe();
+                audioElement.srcObject = null;
+                sourceNode?.disconnect();
+                gainNode?.disconnect();
             };
         };
+        this.getOrCreateAudioContext = () => {
+            if (this.audioContext || !isSafari())
+                return this.audioContext;
+            const context = new AudioContext();
+            if (context.state === 'suspended') {
+                document.addEventListener('click', this.resumeAudioContext);
+            }
+            return (this.audioContext = context);
+        };
+        this.resumeAudioContext = () => {
+            if (this.audioContext?.state === 'suspended') {
+                this.audioContext
+                    .resume()
+                    .catch((err) => this.logger('warn', `Can't resume audio context`, err))
+                    .then(() => {
+                    document.removeEventListener('click', this.resumeAudioContext);
+                });
+            }
+        };
         this.callState = callState;
         this.speaker = speaker;
     }
@@ -9163,6 +9249,16 @@ const getStream = async (constraints, tracer) => {
             // every successful getUserMedia call.
             navigator.mediaDevices.dispatchEvent(new Event('devicechange'));
         }
+        if (constraints.video) {
+            const [videoTrack] = stream.getVideoTracks();
+            if (videoTrack) {
+                const { width, height } = videoTrack.getSettings();
+                const target = constraints.video;
+                if (width !== target.width || height !== target.height) {
+                    tracer?.trace(`${tag}Warn`, `Requested resolution ${target.width}x${target.height} but got ${width}x${height}`);
+                }
+            }
+        }
         return stream;
     }
     catch (error) {
@@ -9506,10 +9602,22 @@ class InputMediaDeviceManager {
         }
     }
     async applySettingsToStream() {
-        await withCancellation(this.statusChangeConcurrencyTag, async () => {
+        await withCancellation(this.statusChangeConcurrencyTag, async (signal) => {
             if (this.enabled) {
-                await this.muteStream();
-                await this.unmuteStream();
+                try {
+                    await this.muteStream();
+                    this.state.setStatus('disabled');
+                    if (signal.aborted) {
+                        return;
+                    }
+                    await this.unmuteStream();
+                    this.state.setStatus('enabled');
+                }
+                finally {
+                    if (!signal.aborted) {
+                        this.state.setPendingStatus(this.state.status);
+                    }
+                }
             }
         });
     }
@@ -9575,130 +9683,122 @@ class InputMediaDeviceManager {
         this.logger('debug', 'Starting stream');
         let stream;
         let rootStream;
-        try {
-            if (this.state.mediaStream &&
-                this.getTracks().every((t) => t.readyState === 'live')) {
-                stream = this.state.mediaStream;
-                this.enableTracks();
-            }
-            else {
-                const defaultConstraints = this.state.defaultConstraints;
-                const constraints = {
-                    ...defaultConstraints,
-                    deviceId: this.state.selectedDevice
-                        ? { exact: this.state.selectedDevice }
-                        : undefined,
-                };
-                /**
-                 * Chains two media streams together.
-                 *
-                 * In our case, filters MediaStreams are derived from their parent MediaStream.
-                 * However, once a child filter's track is stopped,
-                 * the tracks of the parent MediaStream aren't automatically stopped.
-                 * This leads to a situation where the camera indicator light is still on
-                 * even though the user stopped publishing video.
-                 *
-                 * This function works around this issue by stopping the parent MediaStream's tracks
-                 * as well once the child filter's tracks are stopped.
-                 *
-                 * It works by patching the stop() method of the child filter's tracks to also stop
-                 * the parent MediaStream's tracks of the same type. Here we assume that
-                 * the parent MediaStream has only one track of each type.
-                 *
-                 * @param parentStream the parent MediaStream. Omit for the root stream.
-                 */
-                const chainWith = (parentStream) => async (filterStream) => {
-                    if (!parentStream)
-                        return filterStream;
-                    // TODO OL: take care of track.enabled property as well
-                    const parent = await parentStream;
-                    filterStream.getTracks().forEach((track) => {
-                        const originalStop = track.stop;
-                        track.stop = function stop() {
-                            originalStop.call(track);
-                            parent.getTracks().forEach((parentTrack) => {
-                                if (parentTrack.kind === track.kind) {
-                                    parentTrack.stop();
-                                }
-                            });
-                        };
-                    });
-                    parent.getTracks().forEach((parentTrack) => {
-                        // When the parent stream abruptly ends, we propagate the event
-                        // to the filter stream.
-                        // This usually happens when the camera/microphone permissions
-                        // are revoked or when the device is disconnected.
-                        const handleParentTrackEnded = () => {
-                            filterStream.getTracks().forEach((track) => {
-                                if (parentTrack.kind !== track.kind)
-                                    return;
-                                track.stop();
-                                track.dispatchEvent(new Event('ended')); // propagate the event
-                            });
-                        };
-                        parentTrack.addEventListener('ended', handleParentTrackEnded);
-                        this.subscriptions.push(() => {
-                            parentTrack.removeEventListener('ended', handleParentTrackEnded);
-                        });
-                    });
+        if (this.state.mediaStream &&
+            this.getTracks().every((t) => t.readyState === 'live')) {
+            stream = this.state.mediaStream;
+            this.enableTracks();
+        }
+        else {
+            const defaultConstraints = this.state.defaultConstraints;
+            const constraints = {
+                ...defaultConstraints,
+                deviceId: this.state.selectedDevice
+                    ? { exact: this.state.selectedDevice }
+                    : undefined,
+            };
+            /**
+             * Chains two media streams together.
+             *
+             * In our case, filters MediaStreams are derived from their parent MediaStream.
+             * However, once a child filter's track is stopped,
+             * the tracks of the parent MediaStream aren't automatically stopped.
+             * This leads to a situation where the camera indicator light is still on
+             * even though the user stopped publishing video.
+             *
+             * This function works around this issue by stopping the parent MediaStream's tracks
+             * as well once the child filter's tracks are stopped.
+             *
+             * It works by patching the stop() method of the child filter's tracks to also stop
+             * the parent MediaStream's tracks of the same type. Here we assume that
+             * the parent MediaStream has only one track of each type.
+             *
+             * @param parentStream the parent MediaStream. Omit for the root stream.
+             */
+            const chainWith = (parentStream) => async (filterStream) => {
+                if (!parentStream)
                     return filterStream;
-                };
-                // the rootStream represents the stream coming from the actual device
-                // e.g. camera or microphone stream
-                rootStream = this.getStream(constraints);
-                // we publish the last MediaStream of the chain
-                stream = await this.filters.reduce((parent, entry) => parent
-                    .then((inputStream) => {
-                    const { stop, output } = entry.start(inputStream);
-                    entry.stop = stop;
-                    return output;
-                })
-                    .then(chainWith(parent), (error) => {
-                    this.logger('warn', 'Filter failed to start and will be ignored', error);
-                    return parent;
-                }), rootStream);
-            }
-            if (this.call.state.callingState === CallingState.JOINED) {
-                await this.publishStream(stream);
-            }
-            if (this.state.mediaStream !== stream) {
-                this.state.setMediaStream(stream, await rootStream);
-                const handleTrackEnded = async () => {
-                    await this.statusChangeSettled();
-                    if (this.enabled) {
-                        this.isTrackStoppedDueToTrackEnd = true;
-                        setTimeout(() => {
-                            this.isTrackStoppedDueToTrackEnd = false;
-                        }, 2000);
-                        await this.disable();
-                    }
-                };
-                const createTrackMuteHandler = (muted) => () => {
-                    if (!isMobile() || this.trackType !== TrackType.VIDEO)
-                        return;
-                    this.call.notifyTrackMuteState(muted, this.trackType).catch((err) => {
-                        this.logger('warn', 'Error while notifying track mute state', err);
-                    });
-                };
-                stream.getTracks().forEach((track) => {
-                    const muteHandler = createTrackMuteHandler(true);
-                    const unmuteHandler = createTrackMuteHandler(false);
-                    track.addEventListener('mute', muteHandler);
-                    track.addEventListener('unmute', unmuteHandler);
-                    track.addEventListener('ended', handleTrackEnded);
+                // TODO OL: take care of track.enabled property as well
+                const parent = await parentStream;
+                filterStream.getTracks().forEach((track) => {
+                    const originalStop = track.stop;
+                    track.stop = function stop() {
+                        originalStop.call(track);
+                        parent.getTracks().forEach((parentTrack) => {
+                            if (parentTrack.kind === track.kind) {
+                                parentTrack.stop();
+                            }
+                        });
+                    };
+                });
+                parent.getTracks().forEach((parentTrack) => {
+                    // When the parent stream abruptly ends, we propagate the event
+                    // to the filter stream.
+                    // This usually happens when the camera/microphone permissions
+                    // are revoked or when the device is disconnected.
+                    const handleParentTrackEnded = () => {
+                        filterStream.getTracks().forEach((track) => {
+                            if (parentTrack.kind !== track.kind)
+                                return;
+                            track.stop();
+                            track.dispatchEvent(new Event('ended')); // propagate the event
+                        });
+                    };
+                    parentTrack.addEventListener('ended', handleParentTrackEnded);
                     this.subscriptions.push(() => {
-                        track.removeEventListener('mute', muteHandler);
-                        track.removeEventListener('unmute', unmuteHandler);
-                        track.removeEventListener('ended', handleTrackEnded);
+                        parentTrack.removeEventListener('ended', handleParentTrackEnded);
                     });
                 });
-            }
+                return filterStream;
+            };
+            // the rootStream represents the stream coming from the actual device
+            // e.g. camera or microphone stream
+            rootStream = this.getStream(constraints);
+            // we publish the last MediaStream of the chain
+            stream = await this.filters.reduce((parent, entry) => parent
+                .then((inputStream) => {
+                const { stop, output } = entry.start(inputStream);
+                entry.stop = stop;
+                return output;
+            })
+                .then(chainWith(parent), (error) => {
+                this.logger('warn', 'Filter failed to start and will be ignored', error);
+                return parent;
+            }), rootStream);
         }
-        catch (err) {
-            if (rootStream) {
-                disposeOfMediaStream(await rootStream);
-            }
-            throw err;
+        if (this.call.state.callingState === CallingState.JOINED) {
+            await this.publishStream(stream);
+        }
+        if (this.state.mediaStream !== stream) {
+            this.state.setMediaStream(stream, await rootStream);
+            const handleTrackEnded = async () => {
+                await this.statusChangeSettled();
+                if (this.enabled) {
+                    this.isTrackStoppedDueToTrackEnd = true;
+                    setTimeout(() => {
+                        this.isTrackStoppedDueToTrackEnd = false;
+                    }, 2000);
+                    await this.disable();
+                }
+            };
+            const createTrackMuteHandler = (muted) => () => {
+                if (!isMobile() || this.trackType !== TrackType.VIDEO)
+                    return;
+                this.call.notifyTrackMuteState(muted, this.trackType).catch((err) => {
+                    this.logger('warn', 'Error while notifying track mute state', err);
+                });
+            };
+            stream.getTracks().forEach((track) => {
+                const muteHandler = createTrackMuteHandler(true);
+                const unmuteHandler = createTrackMuteHandler(false);
+                track.addEventListener('mute', muteHandler);
+                track.addEventListener('unmute', unmuteHandler);
+                track.addEventListener('ended', handleTrackEnded);
+                this.subscriptions.push(() => {
+                    track.removeEventListener('mute', muteHandler);
+                    track.removeEventListener('unmute', unmuteHandler);
+                    track.removeEventListener('ended', handleTrackEnded);
+                });
+            });
         }
     }
     get mediaDeviceKind() {
@@ -10028,7 +10128,14 @@ class CameraManager extends InputMediaDeviceManager {
         // Wait for any in progress camera operation
         await this.statusChangeSettled();
         const { target_resolution, camera_facing, camera_default_on } = settings;
-        await this.selectTargetResolution(target_resolution);
+        // normalize target resolution to landscape format.
+        // on mobile devices, the device itself adjusts the resolution to portrait or landscape
+        // depending on the orientation of the device. using portrait resolution
+        // will result in falling back to the default resolution (640x480).
+        let { width, height } = target_resolution;
+        if (width < height)
+            [width, height] = [height, width];
+        await this.selectTargetResolution({ width, height });
         // Set camera direction if it's not yet set
         if (!this.state.direction && !this.state.selectedDevice) {
             this.state.setDirection(camera_facing === 'front' ? 'front' : 'back');
@@ -10425,6 +10532,7 @@ class MicrophoneManager extends InputMediaDeviceManager {
             await this.disableNoiseCancellation().catch((err) => {
                 this.logger('warn', 'Failed to disable noise cancellation', err);
             });
+            throw e;
         }
     }
     /**
@@ -11061,6 +11169,7 @@ class Call {
                 await this.sfuClient?.leaveAndClose(message ?? reason ?? 'user is leaving the call');
                 this.sfuClient = undefined;
                 this.dynascaleManager.setSfuClient(undefined);
+                await this.dynascaleManager.dispose();
                 this.state.setCallingState(CallingState.LEFT);
                 this.state.setParticipants([]);
                 this.state.dispose();
@@ -13365,7 +13474,9 @@ class TokenManager {
                         this.token = token;
                     }
                     catch (e) {
-                        return reject(new Error(`Call to tokenProvider failed with message: ${e}`));
+                        return reject(new Error(`Call to tokenProvider failed with message: ${e}`, {
+                            cause: e,
+                        }));
                     }
                     resolve(this.token);
                 }
@@ -13782,7 +13893,7 @@ class StreamClient {
         this.getUserAgent = () => {
             if (!this.cachedUserAgent) {
                 const { clientAppIdentifier = {} } = this.options;
-                const { sdkName = 'js', sdkVersion = "1.23.2", ...extras } = clientAppIdentifier;
+                const { sdkName = 'js', sdkVersion = "1.23.4", ...extras } = clientAppIdentifier;
                 this.cachedUserAgent = [
                     `stream-video-${sdkName}-v${sdkVersion}`,
                     ...Object.entries(extras).map(([key, value]) => `${key}=${value}`),