@stream-io/video-client 1.55.2 → 1.56.0

package/dist/index.cjs.js

@@ -4946,7 +4946,7 @@ const toIceCandidate = (iceTrickle) => {
     }
 };
 
-const version = "1.55.2";
+const version = "1.56.0";
 const [major, minor, patch] = version.split('.');
 let sdkInfo = {
     type: SdkType.PLAIN_JAVASCRIPT,
@@ -5275,6 +5275,23 @@ function wrapWithCancellation(tag, cb) {
     return { cb: wrapped, onContinued };
 }
 
+/**
+ * Performs shallow comparison of two arrays.
+ * Expects primitive values: [1, 2, 3] is equal to [2, 1, 3].
+ */
+const isShallowArrayEqual = (a, b) => {
+    if (a.length !== b.length)
+        return false;
+    for (const item of a) {
+        if (!b.includes(item))
+            return false;
+    }
+    for (const item of b) {
+        if (!a.includes(item))
+            return false;
+    }
+    return true;
+};
 /**
  * Checks if the provided update is a function patch.
  *
@@ -5387,6 +5404,7 @@ var rxUtils = /*#__PURE__*/Object.freeze({
     createSafeAsyncSubscription: createSafeAsyncSubscription,
     createSubscription: createSubscription,
     getCurrentValue: getCurrentValue,
+    isShallowArrayEqual: isShallowArrayEqual,
     setCurrentValue: setCurrentValue,
     setCurrentValueAsync: setCurrentValueAsync,
     updateValue: updateValue
@@ -6127,23 +6145,6 @@ class CallState {
         this.endedBy$ = this.endedBySubject.asObservable();
         this.thumbnails$ = this.thumbnailsSubject.asObservable();
         this.closedCaptions$ = this.closedCaptionsSubject.asObservable();
-        /**
-         * Performs shallow comparison of two arrays.
-         * Expects primitive values: [1, 2, 3] is equal to [2, 1, 3].
-         */
-        const isShallowEqual = (a, b) => {
-            if (a.length !== b.length)
-                return false;
-            for (const item of a) {
-                if (!b.includes(item))
-                    return false;
-            }
-            for (const item of b) {
-                if (!a.includes(item))
-                    return false;
-            }
-            return true;
-        };
         /**
          * Creates an Observable from the given subject by piping to the
          * `distinctUntilChanged()` operator.
@@ -6151,7 +6152,7 @@ class CallState {
         const duc = (subject, comparator) => subject.pipe(rxjs.distinctUntilChanged(comparator));
         // primitive values should only emit once the value they hold changes
         this.anonymousParticipantCount$ = duc(this.anonymousParticipantCountSubject);
-        this.blockedUserIds$ = duc(this.blockedUserIdsSubject, isShallowEqual);
+        this.blockedUserIds$ = duc(this.blockedUserIdsSubject, isShallowArrayEqual);
         this.backstage$ = duc(this.backstageSubject);
         this.callingState$ = duc(this.callingStateSubject);
         this.ownCapabilities$ = rxjs.combineLatest([
@@ -6180,7 +6181,7 @@ class CallState {
                 }
             }
             return nextCapabilities;
-        }), rxjs.distinctUntilChanged(isShallowEqual), rxjs.shareReplay({ bufferSize: 1, refCount: true }));
+        }), rxjs.distinctUntilChanged(isShallowArrayEqual), rxjs.shareReplay({ bufferSize: 1, refCount: true }));
         this.participantCount$ = duc(this.participantCountSubject);
         this.recording$ = duc(this.recordingSubject);
         this.individualRecording$ = duc(this.individualRecordingSubject);
@@ -8491,6 +8492,7 @@ class BasePeerConnection {
         try {
             this.onPeerConnectionStateChange?.({
                 peerType: this.peerType,
+                iceConnectionState: this.pc.iceConnectionState,
                 ...event,
             });
         }
@@ -8945,6 +8947,7 @@ class Publisher extends BasePeerConnection {
     clonedTracks = new Set();
     publishOptions;
     selfSubEnabled;
+    transceiverLockKey = `pub.tx.${this.lock}`;
     /**
      * Constructs a new `Publisher` instance.
      */
@@ -9000,32 +9003,34 @@ class Publisher extends BasePeerConnection {
      * @param options the publish options to use.
      */
     publish = async (track, trackType, options = {}) => {
-        if (!this.publishOptions.some((o) => o.trackType === trackType)) {
-            throw new Error(`No publish options found for ${TrackType[trackType]}`);
-        }
-        for (const publishOption of this.publishOptions) {
-            if (publishOption.trackType !== trackType)
-                continue;
-            // create a clone of the track as otherwise the same trackId will
-            // appear in the SDP in multiple transceivers
-            const trackToPublish = this.cloneTrack(track);
-            const bundle = this.transceiverCache.get(publishOption);
-            if (!bundle) {
-                await this.addTransceiver(trackToPublish, publishOption, options);
+        return withoutConcurrency(this.transceiverLockKey, async () => {
+            const publishOptions = this.publishOptions.filter((o) => o.trackType === trackType);
+            if (!publishOptions.length) {
+                throw new Error(`No publish options found for ${TrackType[trackType]}`);
             }
-            else {
-                const previousTrack = bundle.transceiver.sender.track;
-                await this.updateTransceiver(bundle, trackToPublish, options);
-                if (!isReactNative()) {
-                    this.stopTrack(previousTrack);
+            for (const publishOption of publishOptions) {
+                // create a clone of the track as otherwise the same trackId will
+                // appear in the SDP in multiple transceivers
+                const trackToPublish = this.cloneTrack(track);
+                const bundle = this.transceiverCache.get(publishOption);
+                if (!bundle) {
+                    await this.addTransceiver(trackToPublish, publishOption, options);
+                }
+                else {
+                    const previousTrack = bundle.transceiver.sender.track;
+                    await this.updateTransceiver(bundle, trackToPublish, options);
+                    if (!isReactNative()) {
+                        this.stopTrack(previousTrack);
+                    }
                 }
             }
-        }
+        });
     };
     /**
      * Adds a new transceiver carrying the given track to the peer connection.
      */
     addTransceiver = async (track, publishOption, options) => {
+        const trackType = publishOption.trackType;
         const encodings = isAudioTrackType(publishOption.trackType)
             ? computeAudioLayers(publishOption, options)
             : computeVideoLayers(track, publishOption);
@@ -9036,15 +9041,17 @@ class Publisher extends BasePeerConnection {
             direction: 'sendonly',
             sendEncodings,
         });
+        // claim the cache entry in the same task that created the transceiver:
+        // an await in between would let a concurrent caller add a second one
+        // for the same publish option
+        this.transceiverCache.add({ publishOption, transceiver, options });
+        this.trackIdToTrackType.set(track.id, trackType);
+        this.logger.debug(`Added ${TrackType[trackType]} transceiver`);
         const params = transceiver.sender.getParameters();
         params.degradationPreference =
             toRTCDegradationPreference(publishOption.degradationPreference) ??
                 'maintain-framerate';
         await transceiver.sender.setParameters(params);
-        const trackType = publishOption.trackType;
-        this.logger.debug(`Added ${TrackType[trackType]} transceiver`);
-        this.transceiverCache.add({ publishOption, transceiver, options });
-        this.trackIdToTrackType.set(track.id, trackType);
         await this.negotiate();
     };
     /**
@@ -9101,33 +9108,35 @@ class Publisher extends BasePeerConnection {
      * Synchronizes the current Publisher state with the provided publish options.
      */
     syncPublishOptions = async () => {
-        // enable publishing with new options -> [av1, vp9]
-        for (const publishOption of this.publishOptions) {
-            const { trackType } = publishOption;
-            if (!this.isPublishing(trackType))
-                continue;
-            if (this.transceiverCache.has(publishOption))
-                continue;
-            const item = this.transceiverCache.find((i) => !!i.transceiver.sender.track &&
-                i.publishOption.trackType === trackType);
-            if (!item)
-                continue;
-            // take the track from the existing transceiver for the same track type,
-            // clone it and publish it with the new publish options
-            const track = this.cloneTrack(item.transceiver.sender.track);
-            await this.addTransceiver(track, publishOption, item.options);
-        }
-        // stop publishing with options not required anymore -> [vp9]
-        for (const item of this.transceiverCache.items()) {
-            const { publishOption, transceiver } = item;
-            const hasPublishOption = this.publishOptions.some((option) => option.id === publishOption.id &&
-                option.trackType === publishOption.trackType);
-            if (hasPublishOption)
-                continue;
-            // it is safe to stop the track here, it is a clone
-            this.stopTrack(transceiver.sender.track);
-            await this.updateTransceiver(item, null);
-        }
+        return withoutConcurrency(this.transceiverLockKey, async () => {
+            // enable publishing with new options -> [av1, vp9]
+            for (const publishOption of this.publishOptions) {
+                const { trackType } = publishOption;
+                if (!this.isPublishing(trackType))
+                    continue;
+                if (this.transceiverCache.has(publishOption))
+                    continue;
+                const item = this.transceiverCache.find((i) => !!i.transceiver.sender.track &&
+                    i.publishOption.trackType === trackType);
+                if (!item)
+                    continue;
+                // take the track from the existing transceiver for the same track type,
+                // clone it and publish it with the new publish options
+                const track = this.cloneTrack(item.transceiver.sender.track);
+                await this.addTransceiver(track, publishOption, item.options);
+            }
+            // stop publishing with options not required anymore -> [vp9]
+            for (const item of this.transceiverCache.items()) {
+                const { publishOption, transceiver } = item;
+                const hasPublishOption = this.publishOptions.some((option) => option.id === publishOption.id &&
+                    option.trackType === publishOption.trackType);
+                if (hasPublishOption)
+                    continue;
+                // it is safe to stop the track here, it is a clone
+                this.stopTrack(transceiver.sender.track);
+                await this.updateTransceiver(item, null);
+            }
+        });
     };
     /**
      * Returns true if the given track type is currently being published to the SFU.
@@ -9295,12 +9304,23 @@ class Publisher extends BasePeerConnection {
     negotiate = async (options) => {
         return withoutConcurrency(`publisher.negotiate.${this.lock}`, async () => {
             const offer = await this.pc.createOffer(options);
-            const tracks = this.getAnnouncedTracks(offer.sdp);
-            if (!tracks.length)
-                throw new Error(`Can't negotiate without any tracks`);
             try {
                 this.isIceRestarting = options?.iceRestart ?? false;
                 await this.pc.setLocalDescription(offer);
+                // the announced tracks have to be read after `setLocalDescription`.
+                // `transceiver.mid` is null until the offer is applied, and reading the
+                // tracks earlier makes `extractMid` guess the mid from the offer SDP.
+                // That guess goes stale after a rolled-back negotiation, and a mid that
+                // doesn't match the m-section carrying the track stops the SFU from
+                // correlating the two: it then answers with every offered codec instead
+                // of the announced one, and we publish the first codec of that list.
+                // Per https://www.w3.org/TR/webrtc/#set-description, applying a local
+                // offer associates the transceiver with its m-section and sets
+                // `[[Mid]]` to `[[JsepMid]]`; a rollback disassociates a transceiver
+                // that wasn't associated before and resets both slots to null.
+                const tracks = this.getAnnouncedTracks(offer.sdp);
+                if (!tracks.length)
+                    throw new Error(`Can't negotiate without any tracks`);
                 const { sdp: baseSdp = '' } = offer;
                 const { dangerouslyForceCodec, dangerouslySetStartBitrateFactor, fmtpLine, } = this.clientPublishOptions || {};
                 let sdp = dangerouslyForceCodec
@@ -10925,34 +10945,67 @@ class AudioBindingsWatchdog {
 class BlockedAudioTracker {
     logger = videoLoggerSystem.getLogger('BlockedAudioTracker');
     tracer;
-    blockedElementsSubject = new rxjs.BehaviorSubject(new Set());
+    blockedElementsSubject = new rxjs.BehaviorSubject([]);
     /**
      * Whether the browser's autoplay policy is blocking audio playback.
      * Will be `true` when at least one audio element is currently blocked.
      * Use {@link resumeAudio} within a user gesture to unblock.
      */
-    autoplayBlocked$ = this.blockedElementsSubject.pipe(rxjs.map((elements) => elements.size > 0), rxjs.distinctUntilChanged());
+    autoplayBlocked$ = this.blockedElementsSubject.pipe(rxjs.map((elements) => elements.length > 0), rxjs.distinctUntilChanged());
+    /**
+     * The list of participant `sessionId`s whose audio element is currently
+     * blocked by the browser's autoplay policy. Only some participants may be
+     * blocked (e.g. one joined while a gesture was active and a later one was
+     * not), so use this to render a per-participant affordance rather than a
+     * call-wide one. Call {@link resumeAudio} within a user gesture to unblock
+     * all of them.
+     *
+     * Session ids are registered together with the audio element.
+     */
+    blockedSessionIds$ = this.blockedElementsSubject.pipe(rxjs.map((elements) => {
+        const ids = [];
+        elements.forEach(({ sessionId }) => {
+            if (sessionId && !ids.includes(sessionId))
+                ids.push(sessionId);
+        });
+        return ids;
+    }), rxjs.distinctUntilChanged(isShallowArrayEqual), rxjs.shareReplay({ bufferSize: 1, refCount: true }));
     constructor(tracer) {
         this.tracer = tracer;
     }
     /**
      * Registers an audio element as blocked by the browser's autoplay policy.
      */
-    markBlocked = (audioElement, blocked) => {
+    markBlocked = (audioElement, blocked, sessionId) => {
         setCurrentValue(this.blockedElementsSubject, (elements) => {
-            if (blocked)
-                elements.add(audioElement);
-            else
-                elements.delete(audioElement);
-            return elements;
+            if (!blocked) {
+                return elements.filter(({ element }) => element !== audioElement);
+            }
+            const existing = elements.find(({ element }) => element === audioElement);
+            if (existing) {
+                return elements.map((entry) => entry.element === audioElement
+                    ? { element: audioElement, sessionId: sessionId ?? entry.sessionId }
+                    : entry);
+            }
+            return [...elements, { element: audioElement, sessionId }];
         });
     };
+    /**
+     * Clears all tracked elements. The tracker stays usable afterwards.
+     */
+    reset = () => {
+        if (this.blockedElementsSubject.getValue().length > 0) {
+            this.blockedElementsSubject.next([]);
+        }
+    };
     /**
      * Returns whether the given audio element is currently flagged as blocked
      * by the browser's autoplay policy.
      */
     isBlocked = (audioElement) => {
-        return this.blockedElementsSubject.getValue().has(audioElement);
+        return this.blockedElementsSubject
+            .getValue()
+            .some(({ element }) => element === audioElement);
     };
     /**
      * Plays all audio elements blocked by the browser's autoplay policy.
@@ -10961,17 +11014,18 @@ class BlockedAudioTracker {
     resumeAudio = async () => {
         this.tracer.trace('resumeAudio', null);
         await setCurrentValueAsync(this.blockedElementsSubject, async (elements) => {
-            await Promise.all(Array.from(elements, async (element) => {
+            let next = elements;
+            await Promise.all(elements.map(async ({ element }) => {
                 try {
                     if (element.srcObject)
                         await timeboxed([element.play()], 2000);
-                    elements.delete(element);
+                    next = next.filter((entry) => entry.element !== element);
                 }
                 catch (err) {
                     this.logger.warn(`Can't resume audio for element`, element, err);
                 }
             }));
-            return elements;
+            return next;
         });
     };
 }
@@ -11503,6 +11557,12 @@ class DynascaleManager {
         let sourceNode = undefined;
         let gainNode = undefined;
         let audioWatchdog = undefined;
+        const clearBlockedAudio = () => {
+            if (!this.blockedAudioTracker.isBlocked(audioElement))
+                return;
+            this.blockedAudioTracker.markBlocked(audioElement, false);
+        };
+        audioElement.addEventListener('playing', clearBlockedAudio);
         const isAudioTrack = trackType === 'audioTrack';
         const trackKey = isAudioTrack ? 'audioStream' : 'screenShareAudioStream';
         const updateMediaStreamSubscription = participant$
@@ -11516,7 +11576,7 @@ class DynascaleManager {
                 audioWatchdog?.dispose();
                 audioWatchdog = undefined;
                 if (!source) {
-                    this.blockedAudioTracker.markBlocked(audioElement, false);
+                    clearBlockedAudio();
                     return;
                 }
                 // Safari has a special quirk that prevents playing audio until the user
@@ -11544,7 +11604,7 @@ class DynascaleManager {
                         this.tracer.trace('audioPlaybackError', e.message);
                         if (e.name === 'NotAllowedError') {
                             this.tracer.trace('audioPlaybackBlocked', null);
-                            this.blockedAudioTracker.markBlocked(audioElement, true);
+                            this.blockedAudioTracker.markBlocked(audioElement, true, sessionId);
                         }
                         this.logger.warn(`Failed to play audio stream`, e);
                     });
@@ -11577,7 +11637,8 @@ class DynascaleManager {
         });
         audioElement.autoplay = true;
         return () => {
-            this.blockedAudioTracker.markBlocked(audioElement, false);
+            audioElement.removeEventListener('playing', clearBlockedAudio);
+            clearBlockedAudio();
             sinkIdSubscription?.unsubscribe();
             volumeSubscription.unsubscribe();
             updateMediaStreamSubscription.unsubscribe();
@@ -14752,6 +14813,7 @@ class Call {
         this.screenShare = new ScreenShareManager(this);
         this.trackSubscriptionManager = new TrackSubscriptionManager(this.state, this.tracer);
         this.blockedAudioTracker = new BlockedAudioTracker(this.tracer);
+        this.leaveCallHooks.add(this.blockedAudioTracker.reset);
         if (typeof document !== 'undefined') {
             this.audioBindingsWatchdog = new AudioBindingsWatchdog(this.state, this.tracer);
             this.viewportTracker = new ViewportTracker(this.state);
@@ -18448,11 +18510,11 @@ class StreamClient {
         return await this.wsConnection.connect(this.defaultWSTimeout);
     };
     getSdkVersion = () => this.options.clientAppIdentifier?.sdkVersion ||
-        "1.55.2";
+        "1.56.0";
     getUserAgent = () => {
         if (!this.cachedUserAgent) {
             const { clientAppIdentifier = {} } = this.options;
-            const { sdkName = 'js', sdkVersion = "1.55.2", ...extras } = clientAppIdentifier;
+            const { sdkName = 'js', sdkVersion = "1.56.0", ...extras } = clientAppIdentifier;
             this.cachedUserAgent = [
                 `stream-video-${sdkName}-v${sdkVersion}`,
                 ...Object.entries(extras).map(([key, value]) => `${key}=${value}`),
@@ -18789,8 +18851,8 @@ class ClientEventReporter {
             applyErrorIfAbsent(this.wsPairs.get(cid), stageError);
             this.failCoordinator(cid);
             this.failWs(cid);
-            this.emitPeerConnectionFailure(cid, 'publish', code, reason, 'NOT_CONNECTED');
-            this.emitPeerConnectionFailure(cid, 'subscribe', code, reason, 'NOT_CONNECTED');
+            this.emitPeerConnectionFailure(cid, 'publish', code, reason);
+            this.emitPeerConnectionFailure(cid, 'subscribe', code, reason);
         }
         catch (err) {
             this.logger.warn('Failed to report abort', err);
@@ -18802,7 +18864,7 @@ class ClientEventReporter {
         if (this.wsPairs.get(cid))
             this.failWs(cid);
         for (const role of ['publish', 'subscribe']) {
-            this.emitPeerConnectionFailure(cid, role, 'CLIENT_ABORTED', 'superseded by a new join attempt', 'NOT_CONNECTED');
+            this.emitPeerConnectionFailure(cid, role, 'CLIENT_ABORTED', 'superseded by a new join attempt');
         }
     };
     emitJoinInitiated = (cid) => {
@@ -18964,21 +19026,24 @@ class ClientEventReporter {
     };
     onPeerConnectionStateChange = (cid, event) => {
         const role = event.peerType === PeerType.SUBSCRIBER ? 'subscribe' : 'publish';
-        if (event.stateType === 'ice' && event.state === 'failed') {
-            this.emitPeerConnectionFailure(cid, role, 'ICE_CONNECTIVITY_FAILED', 'ICE connectivity checks failed', 'FAILED');
+        const pair = this.peerConnectionPairs.get(pcKey(cid, role));
+        if (pair)
+            pair.lastIceState = event.iceConnectionState;
+        if (event.stateType === 'ice') {
+            if (event.state === 'failed') {
+                this.emitPeerConnectionFailure(cid, role, 'ICE_CONNECTIVITY_FAILED', 'ICE connectivity checks failed');
+            }
             return;
         }
-        if (event.stateType === 'peerConnection' && event.state === 'failed') {
-            this.emitPeerConnectionFailure(cid, role, 'DTLS_CONNECTIVITY_FAILED', 'DTLS connectivity checks failed', 'CONNECTED');
+        if (event.state === 'failed') {
+            this.emitPeerConnectionFailure(cid, role, 'DTLS_CONNECTIVITY_FAILED', 'DTLS connectivity checks failed');
             return;
         }
-        if (event.stateType !== 'peerConnection')
-            return;
         switch (event.state) {
             case 'connecting':
-                if (this.peerConnectionPairs.has(pcKey(cid, role)))
+                if (pair)
                     return;
-                this.openPeerConnectionPair(cid, role);
+                this.openPeerConnectionPair(cid, role, event.iceConnectionState);
                 break;
             case 'connected':
                 this.emitPeerConnectionSuccess(cid, role);
@@ -18986,7 +19051,7 @@ class ClientEventReporter {
                 break;
         }
     };
-    openPeerConnectionPair = (cid, role) => {
+    openPeerConnectionPair = (cid, role, iceConnectionState) => {
         const key = pcKey(cid, role);
         const pair = {
             sid: generateUUIDv4(),
@@ -18996,6 +19061,7 @@ class ClientEventReporter {
             sfuId: this.getSfuId(cid),
             userSessionId: this.getUserSessionId(cid),
             wasPreviouslyConnected: this.pcEverConnected.get(key) === true,
+            lastIceState: iceConnectionState,
         };
         this.peerConnectionPairs.set(key, pair);
         this.sendForCall(cid, {
@@ -19031,7 +19097,7 @@ class ClientEventReporter {
         });
         this.peerConnectionPairs.delete(key);
     };
-    emitPeerConnectionFailure = (cid, role, code, reason, iceState) => {
+    emitPeerConnectionFailure = (cid, role, code, reason) => {
         const key = pcKey(cid, role);
         const pair = this.peerConnectionPairs.get(key);
         if (!pair)
@@ -19049,7 +19115,7 @@ class ClientEventReporter {
             outcome: 'failure',
             retry_count_attempt: 0,
             elapsed_time: Date.now() - pair.startedAt,
-            ice_state: iceState,
+            ice_state: toReportedIceState(pair.lastIceState),
             retry_failure_reason: reason,
             retry_failure_code: code,
         });
@@ -19115,6 +19181,17 @@ class ClientEventReporter {
         return false;
     };
 }
+const toReportedIceState = (state) => {
+    switch (state) {
+        case 'connected':
+        case 'completed':
+            return 'CONNECTED';
+        case 'failed':
+            return 'FAILED';
+        default:
+            return 'NOT_CONNECTED';
+    }
+};
 const readPermissionStatus = (permission) => {
     const state = getCurrentValue(permission.asStateObservable());
     switch (state) {