@stream-io/video-client 1.55.1 → 1.56.0

package/dist/src/Call.d.ts

@@ -482,6 +482,10 @@ export declare class Call {
      * @param criteria the list of criteria to sort the participants by.
      */
     setSortParticipantsBy: CallState['setSortParticipantsBy'];
+    /**
+     * Returns the comparator currently used to sort the participants.
+     */
+    getSortParticipantsBy: CallState['getSortParticipantsBy'];
     /**
      * Sends a reaction to the other call participants.
      *

package/dist/src/helpers/BlockedAudioTracker.d.ts

@@ -12,11 +12,26 @@ export declare class BlockedAudioTracker {
      * Use {@link resumeAudio} within a user gesture to unblock.
      */
     autoplayBlocked$: import("rxjs").Observable<boolean>;
+    /**
+     * The list of participant `sessionId`s whose audio element is currently
+     * blocked by the browser's autoplay policy. Only some participants may be
+     * blocked (e.g. one joined while a gesture was active and a later one was
+     * not), so use this to render a per-participant affordance rather than a
+     * call-wide one. Call {@link resumeAudio} within a user gesture to unblock
+     * all of them.
+     *
+     * Session ids are registered together with the audio element.
+     */
+    blockedSessionIds$: import("rxjs").Observable<string[]>;
     constructor(tracer: Tracer);
     /**
      * Registers an audio element as blocked by the browser's autoplay policy.
      */
-    markBlocked: (audioElement: HTMLAudioElement, blocked: boolean) => void;
+    markBlocked: (audioElement: HTMLAudioElement, blocked: boolean, sessionId?: string) => void;
+    /**
+     * Clears all tracked elements. The tracker stays usable afterwards.
+     */
+    reset: () => void;
     /**
      * Returns whether the given audio element is currently flagged as blocked
      * by the browser's autoplay policy.

package/dist/src/reporting/ClientEventReporter.d.ts

@@ -4,6 +4,7 @@ import type { PeerConnectionStateChangeEvent } from '../rtc';
 export type ClientEventPeerConnection = 'publish' | 'subscribe';
 export type ClientEventStage = 'JoinInitiated' | 'CoordinatorWS' | 'MediaDevicePermission' | 'CoordinatorJoin' | 'WSJoin' | 'PeerConnectionConnect' | 'FirstVideoFrame' | 'FirstAudioFrame';
 export type MediaPermissionState = 'INITIATED' | 'FAILED' | 'GRANTED' | 'NOT_INITIATED';
+export type ReportedIceState = 'CONNECTED' | 'FAILED' | 'NOT_CONNECTED';
 export type JoinReason = 'first-attempt' | 'network-available' | 'migration' | 'full-rejoin';
 export type ClientEventStandardCode = 'CLIENT_ABORTED' | 'BACKEND_LEAVE' | 'REQUEST_TIMEOUT' | 'NETWORK_ERROR' | 'SFU_ERROR' | 'SFU_GO_AWAY' | 'ICE_CONNECTIVITY_FAILED' | 'DTLS_CONNECTIVITY_FAILED';
 export type CallReportContext = {

package/dist/src/rtc/Publisher.d.ts

@@ -11,6 +11,7 @@ export declare class Publisher extends BasePeerConnection {
     private readonly clonedTracks;
     private publishOptions;
     private readonly selfSubEnabled;
+    private readonly transceiverLockKey;
     /**
      * Constructs a new `Publisher` instance.
      */

package/dist/src/rtc/types.d.ts

@@ -43,16 +43,23 @@ export type OnIceConnected = (peerType: PeerType) => void;
  * Snapshot of the peer connection's ICE and DTLS state surfaced to telemetry
  * consumers (e.g. `ClientEventReporter`). Fired on every transition of
  * either `iceConnectionState` or `peerConnectionState`.
+ *
+ * `iceConnectionState` is the live `RTCPeerConnection.iceConnectionState` read
+ * at the moment the event fires, regardless of which transition triggered it.
+ * Consumers must use it (rather than assuming a value from `stateType`) because
+ * the browser does not guarantee the relative ordering of the `ice` and
+ * `peerConnection` `failed` transitions.
  */
 export type PeerConnectionStateChangeEvent = {
     peerType: PeerType;
+    iceConnectionState: RTCIceConnectionState;
+} & ({
     stateType: 'ice';
     state: RTCIceConnectionState;
 } | {
-    peerType: PeerType;
     stateType: 'peerConnection';
     state: RTCPeerConnectionState;
-};
+});
 export type OnPeerConnectionStateChange = (event: PeerConnectionStateChangeEvent) => void;
 /**
  * Fired when a remote track starts receiving media (`unmute`). Used by

package/dist/src/store/CallState.d.ts

@@ -227,6 +227,10 @@ export declare class CallState {
      * @param comparator the comparator to use to sort the participants.
      */
     setSortParticipantsBy: (comparator: Comparator<StreamVideoParticipant>) => void;
+    /**
+     * Returns the comparator currently used to sort the participants.
+     */
+    getSortParticipantsBy: () => Comparator<StreamVideoParticipant>;
     /**
      * Gets the current value of an observable, or undefined if the observable has
      * not emitted a value yet.

package/dist/src/store/rxUtils.d.ts

@@ -5,6 +5,11 @@ type AsyncFunctionPatch<T> = (currentValue: T) => Promise<T>;
  * A value or a function which takes the current value and returns a new value.
  */
 export type Patch<T> = T | FunctionPatch<T>;
+/**
+ * Performs shallow comparison of two arrays.
+ * Expects primitive values: [1, 2, 3] is equal to [2, 1, 3].
+ */
+export declare const isShallowArrayEqual: <T>(a: Array<T>, b: Array<T>) => boolean;
 /**
  * Gets the current value of an observable, or undefined if the observable has
  * not emitted a value yet.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-client",
-  "version": "1.55.1",
+  "version": "1.56.0",
   "main": "dist/index.cjs.js",
   "module": "dist/index.es.js",
   "browser": "dist/index.browser.es.js",
@@ -46,7 +46,7 @@
     "@openapitools/openapi-generator-cli": "^2.34.0",
     "@rollup/plugin-replace": "^6.0.3",
     "@rollup/plugin-typescript": "^12.3.0",
-    "@stream-io/audio-filters-web": "^0.9.1",
+    "@stream-io/audio-filters-web": "^0.10.2",
     "@stream-io/node-sdk": "^0.7.59",
     "@stream-io/typescript-config": "^0.0.0",
     "@total-typescript/shoehorn": "^0.1.2",

package/src/Call.ts

@@ -404,6 +404,7 @@ export class Call {
       this.tracer,
     );
     this.blockedAudioTracker = new BlockedAudioTracker(this.tracer);
+    this.leaveCallHooks.add(this.blockedAudioTracker.reset);
 
     if (typeof document !== 'undefined') {
       this.audioBindingsWatchdog = new AudioBindingsWatchdog(
@@ -2439,6 +2440,13 @@ export class Call {
     return this.state.setSortParticipantsBy(criteria);
   };
 
+  /**
+   * Returns the comparator currently used to sort the participants.
+   */
+  getSortParticipantsBy: CallState['getSortParticipantsBy'] = () => {
+    return this.state.getSortParticipantsBy();
+  };
+
   /**
    * Sends a reaction to the other call participants.
    *

package/src/helpers/BlockedAudioTracker.ts

@@ -1,9 +1,18 @@
-import { BehaviorSubject, distinctUntilChanged, map } from 'rxjs';
+import { BehaviorSubject, distinctUntilChanged, map, shareReplay } from 'rxjs';
 import { videoLoggerSystem } from '../logger';
 import { Tracer } from '../stats';
-import { setCurrentValue, setCurrentValueAsync } from '../store/rxUtils';
+import {
+  isShallowArrayEqual,
+  setCurrentValue,
+  setCurrentValueAsync,
+} from '../store/rxUtils';
 import { timeboxed } from '../coordinator/connection/utils';
 
+type BlockedAudioElement = {
+  element: HTMLAudioElement;
+  sessionId?: string;
+};
+
 /**
  * Tracks audio elements that the browser's autoplay policy has blocked.
  */
@@ -11,8 +20,8 @@ export class BlockedAudioTracker {
   private logger = videoLoggerSystem.getLogger('BlockedAudioTracker');
   private tracer: Tracer;
 
-  private blockedElementsSubject = new BehaviorSubject(
-    new Set<HTMLAudioElement>(),
+  private blockedElementsSubject = new BehaviorSubject<BlockedAudioElement[]>(
+    [],
   );
 
   /**
@@ -21,10 +30,32 @@ export class BlockedAudioTracker {
    * Use {@link resumeAudio} within a user gesture to unblock.
    */
   autoplayBlocked$ = this.blockedElementsSubject.pipe(
-    map((elements) => elements.size > 0),
+    map((elements) => elements.length > 0),
     distinctUntilChanged(),
   );
 
+  /**
+   * The list of participant `sessionId`s whose audio element is currently
+   * blocked by the browser's autoplay policy. Only some participants may be
+   * blocked (e.g. one joined while a gesture was active and a later one was
+   * not), so use this to render a per-participant affordance rather than a
+   * call-wide one. Call {@link resumeAudio} within a user gesture to unblock
+   * all of them.
+   *
+   * Session ids are registered together with the audio element.
+   */
+  blockedSessionIds$ = this.blockedElementsSubject.pipe(
+    map((elements) => {
+      const ids: string[] = [];
+      elements.forEach(({ sessionId }) => {
+        if (sessionId && !ids.includes(sessionId)) ids.push(sessionId);
+      });
+      return ids;
+    }),
+    distinctUntilChanged(isShallowArrayEqual),
+    shareReplay({ bufferSize: 1, refCount: true }),
+  );
+
   constructor(tracer: Tracer) {
     this.tracer = tracer;
   }
@@ -32,20 +63,46 @@ export class BlockedAudioTracker {
   /**
    * Registers an audio element as blocked by the browser's autoplay policy.
    */
-  markBlocked = (audioElement: HTMLAudioElement, blocked: boolean) => {
+  markBlocked = (
+    audioElement: HTMLAudioElement,
+    blocked: boolean,
+    sessionId?: string,
+  ) => {
     setCurrentValue(this.blockedElementsSubject, (elements) => {
-      if (blocked) elements.add(audioElement);
-      else elements.delete(audioElement);
-      return elements;
+      if (!blocked) {
+        return elements.filter(({ element }) => element !== audioElement);
+      }
+
+      const existing = elements.find(({ element }) => element === audioElement);
+      if (existing) {
+        return elements.map((entry) =>
+          entry.element === audioElement
+            ? { element: audioElement, sessionId: sessionId ?? entry.sessionId }
+            : entry,
+        );
+      }
+
+      return [...elements, { element: audioElement, sessionId }];
     });
   };
 
+  /**
+   * Clears all tracked elements. The tracker stays usable afterwards.
+   */
+  reset = () => {
+    if (this.blockedElementsSubject.getValue().length > 0) {
+      this.blockedElementsSubject.next([]);
+    }
+  };
+
   /**
    * Returns whether the given audio element is currently flagged as blocked
    * by the browser's autoplay policy.
    */
   isBlocked = (audioElement: HTMLAudioElement): boolean => {
-    return this.blockedElementsSubject.getValue().has(audioElement);
+    return this.blockedElementsSubject
+      .getValue()
+      .some(({ element }) => element === audioElement);
   };
 
   /**
@@ -57,17 +114,18 @@ export class BlockedAudioTracker {
     await setCurrentValueAsync(
       this.blockedElementsSubject,
       async (elements) => {
+        let next = elements;
         await Promise.all(
-          Array.from(elements, async (element) => {
+          elements.map(async ({ element }) => {
             try {
               if (element.srcObject) await timeboxed([element.play()], 2000);
-              elements.delete(element);
+              next = next.filter((entry) => entry.element !== element);
             } catch (err) {
               this.logger.warn(`Can't resume audio for element`, element, err);
             }
           }),
         );
-        return elements;
+        return next;
       },
     );
   };

package/src/helpers/DynascaleManager.ts

@@ -328,6 +328,12 @@ export class DynascaleManager {
     let gainNode: GainNode | undefined = undefined;
     let audioWatchdog: MediaPlaybackWatchdog | undefined = undefined;
 
+    const clearBlockedAudio = () => {
+      if (!this.blockedAudioTracker.isBlocked(audioElement)) return;
+      this.blockedAudioTracker.markBlocked(audioElement, false);
+    };
+    audioElement.addEventListener('playing', clearBlockedAudio);
+
     const isAudioTrack = trackType === 'audioTrack';
     const trackKey = isAudioTrack ? 'audioStream' : 'screenShareAudioStream';
     const updateMediaStreamSubscription = participant$
@@ -341,7 +347,7 @@ export class DynascaleManager {
           audioWatchdog?.dispose();
           audioWatchdog = undefined;
           if (!source) {
-            this.blockedAudioTracker.markBlocked(audioElement, false);
+            clearBlockedAudio();
             return;
           }
 
@@ -369,7 +375,11 @@ export class DynascaleManager {
               this.tracer.trace('audioPlaybackError', e.message);
               if (e.name === 'NotAllowedError') {
                 this.tracer.trace('audioPlaybackBlocked', null);
-                this.blockedAudioTracker.markBlocked(audioElement, true);
+                this.blockedAudioTracker.markBlocked(
+                  audioElement,
+                  true,
+                  sessionId,
+                );
               }
               this.logger.warn(`Failed to play audio stream`, e);
             });
@@ -405,7 +415,8 @@ export class DynascaleManager {
     audioElement.autoplay = true;
 
     return () => {
-      this.blockedAudioTracker.markBlocked(audioElement, false);
+      audioElement.removeEventListener('playing', clearBlockedAudio);
+      clearBlockedAudio();
       sinkIdSubscription?.unsubscribe();
       volumeSubscription.unsubscribe();
       updateMediaStreamSubscription.unsubscribe();

package/src/helpers/__tests__/BlockedAudioTracker.test.ts

@@ -2,7 +2,7 @@
  * @vitest-environment happy-dom
  */
 
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { BlockedAudioTracker } from '../BlockedAudioTracker';
 import { getCurrentValue } from '../../store/rxUtils';
 import type { Tracer } from '../../stats';
@@ -30,6 +30,10 @@ describe('BlockedAudioTracker', () => {
     tracker = new BlockedAudioTracker(tracer);
   });
 
+  afterEach(() => {
+    tracker.reset();
+  });
+
   it('emits autoplayBlocked$ true after markBlocked(el, true)', () => {
     expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
     tracker.markBlocked(createAudioElement(), true);
@@ -58,14 +62,58 @@ describe('BlockedAudioTracker', () => {
     expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
   });
 
+  it('emits blocked participant session ids', () => {
+    const el1 = createAudioElement();
+    const el2 = createAudioElement();
+
+    tracker.markBlocked(el1, true, 'session-id-1');
+    tracker.markBlocked(el2, true, 'session-id-2');
+
+    expect(getCurrentValue(tracker.blockedSessionIds$)).toEqual([
+      'session-id-1',
+      'session-id-2',
+    ]);
+
+    tracker.markBlocked(el1, false);
+
+    expect(getCurrentValue(tracker.blockedSessionIds$)).toEqual([
+      'session-id-2',
+    ]);
+  });
+
+  it(`doesn't emit when blocked participant session ids don't change`, () => {
+    const el1 = createAudioElement();
+    const el2 = createAudioElement();
+    tracker.markBlocked(el1, true, 'session-id-1');
+
+    const subscriber = vi.fn();
+    const subscription = tracker.blockedSessionIds$.subscribe(subscriber);
+
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el1, true, 'session-id-1');
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el2, true, 'session-id-1');
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el1, false);
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el2, false);
+    expect(subscriber).toHaveBeenCalledTimes(2);
+
+    subscription.unsubscribe();
+  });
+
   it('resumeAudio plays each element with srcObject and clears successful ones', async () => {
     const el1 = createAudioElement();
     const el2 = createAudioElement();
     const play1 = vi.spyOn(el1, 'play').mockResolvedValue();
     const play2 = vi.spyOn(el2, 'play').mockResolvedValue();
 
-    tracker.markBlocked(el1, true);
-    tracker.markBlocked(el2, true);
+    tracker.markBlocked(el1, true, 'session-id-1');
+    tracker.markBlocked(el2, true, 'session-id-2');
 
     await tracker.resumeAudio();
 
@@ -82,8 +130,8 @@ describe('BlockedAudioTracker', () => {
       new DOMException('', 'NotAllowedError'),
     );
 
-    tracker.markBlocked(ok, true);
-    tracker.markBlocked(stillBlocked, true);
+    tracker.markBlocked(ok, true, 'session-id-1');
+    tracker.markBlocked(stillBlocked, true, 'session-id-2');
 
     await tracker.resumeAudio();
 
@@ -100,7 +148,7 @@ describe('BlockedAudioTracker', () => {
     const detached = createAudioElement(false);
     const play = vi.spyOn(detached, 'play').mockResolvedValue();
 
-    tracker.markBlocked(detached, true);
+    tracker.markBlocked(detached, true, 'session-id');
     await tracker.resumeAudio();
 
     expect(play).not.toHaveBeenCalled();

package/src/helpers/__tests__/DynascaleManager.test.ts

@@ -143,6 +143,51 @@ describe('DynascaleManager', () => {
       cleanup?.();
     });
 
+    it('audio: tracks the blocked session and clears it once the element plays', async () => {
+      vi.useFakeTimers();
+      let cleanup: ReturnType<typeof dynascaleManager.bindAudioElement>;
+      try {
+        const audioElement = document.createElement('audio');
+        Object.defineProperty(audioElement, 'srcObject', { writable: true });
+        vi.spyOn(audioElement, 'play').mockRejectedValue(
+          new DOMException('', 'NotAllowedError'),
+        );
+
+        // @ts-expect-error incomplete data
+        call.state.updateOrAddParticipant('session-id', {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          publishedTracks: [],
+        });
+
+        cleanup = dynascaleManager.bindAudioElement(
+          audioElement,
+          'session-id',
+          'audioTrack',
+        );
+
+        call.state.updateParticipant('session-id', {
+          audioStream: new MediaStream(),
+        });
+        await vi.advanceTimersByTimeAsync(0); // flush the rejected play()
+
+        expect(call.blockedAudioTracker.isBlocked(audioElement)).toBe(true);
+        expect(
+          getCurrentValue(call.blockedAudioTracker.blockedSessionIds$),
+        ).toEqual(['session-id']);
+
+        audioElement.dispatchEvent(new Event('playing'));
+
+        expect(call.blockedAudioTracker.isBlocked(audioElement)).toBe(false);
+        expect(
+          getCurrentValue(call.blockedAudioTracker.blockedSessionIds$),
+        ).toEqual([]);
+      } finally {
+        cleanup?.();
+        vi.useRealTimers();
+      }
+    });
+
     it('audio: Safari should use AudioContext for audio playback', () => {
       globalThis._isSafari = true;
       dynascaleManager.setUseWebAudio(true); // enabled by default on Safari

package/src/reporting/ClientEventReporter.ts

@@ -38,6 +38,8 @@ export type MediaPermissionState =
   | 'GRANTED'
   | 'NOT_INITIATED';
 
+export type ReportedIceState = 'CONNECTED' | 'FAILED' | 'NOT_CONNECTED';
+
 export type JoinReason =
   | 'first-attempt'
   | 'network-available'
@@ -86,6 +88,7 @@ type PeerConnectionPairState = StagePairState & {
   sfuId: string;
   userSessionId: string;
   wasPreviouslyConnected: boolean;
+  lastIceState?: RTCIceConnectionState;
 };
 
 const pcKey = (cid: string, role: ClientEventPeerConnection): string =>
@@ -356,20 +359,8 @@ export class ClientEventReporter {
       this.failCoordinator(cid);
       this.failWs(cid);
 
-      this.emitPeerConnectionFailure(
-        cid,
-        'publish',
-        code,
-        reason,
-        'NOT_CONNECTED',
-      );
-      this.emitPeerConnectionFailure(
-        cid,
-        'subscribe',
-        code,
-        reason,
-        'NOT_CONNECTED',
-      );
+      this.emitPeerConnectionFailure(cid, 'publish', code, reason);
+      this.emitPeerConnectionFailure(cid, 'subscribe', code, reason);
     } catch (err) {
       this.logger.warn('Failed to report abort', err);
     }
@@ -384,7 +375,6 @@ export class ClientEventReporter {
         role,
         'CLIENT_ABORTED',
         'superseded by a new join attempt',
-        'NOT_CONNECTED',
       );
     }
   };
@@ -573,34 +563,35 @@ export class ClientEventReporter {
     const role: ClientEventPeerConnection =
       event.peerType === PeerType.SUBSCRIBER ? 'subscribe' : 'publish';
 
-    if (event.stateType === 'ice' && event.state === 'failed') {
-      this.emitPeerConnectionFailure(
-        cid,
-        role,
-        'ICE_CONNECTIVITY_FAILED',
-        'ICE connectivity checks failed',
-        'FAILED',
-      );
+    const pair = this.peerConnectionPairs.get(pcKey(cid, role));
+    if (pair) pair.lastIceState = event.iceConnectionState;
+
+    if (event.stateType === 'ice') {
+      if (event.state === 'failed') {
+        this.emitPeerConnectionFailure(
+          cid,
+          role,
+          'ICE_CONNECTIVITY_FAILED',
+          'ICE connectivity checks failed',
+        );
+      }
       return;
     }
 
-    if (event.stateType === 'peerConnection' && event.state === 'failed') {
+    if (event.state === 'failed') {
       this.emitPeerConnectionFailure(
         cid,
         role,
         'DTLS_CONNECTIVITY_FAILED',
         'DTLS connectivity checks failed',
-        'CONNECTED',
       );
       return;
     }
 
-    if (event.stateType !== 'peerConnection') return;
-
     switch (event.state) {
       case 'connecting':
-        if (this.peerConnectionPairs.has(pcKey(cid, role))) return;
-        this.openPeerConnectionPair(cid, role);
+        if (pair) return;
+        this.openPeerConnectionPair(cid, role, event.iceConnectionState);
         break;
       case 'connected':
         this.emitPeerConnectionSuccess(cid, role);
@@ -614,6 +605,7 @@ export class ClientEventReporter {
   private openPeerConnectionPair = (
     cid: string,
     role: ClientEventPeerConnection,
+    iceConnectionState: RTCIceConnectionState,
   ) => {
     const key = pcKey(cid, role);
     const pair: PeerConnectionPairState = {
@@ -624,6 +616,7 @@ export class ClientEventReporter {
       sfuId: this.getSfuId(cid),
       userSessionId: this.getUserSessionId(cid),
       wasPreviouslyConnected: this.pcEverConnected.get(key) === true,
+      lastIceState: iceConnectionState,
     };
     this.peerConnectionPairs.set(key, pair);
 
@@ -670,7 +663,6 @@ export class ClientEventReporter {
     role: ClientEventPeerConnection,
     code: ClientEventStandardCode,
     reason: string,
-    iceState: 'CONNECTED' | 'FAILED' | 'NOT_CONNECTED',
   ) => {
     const key = pcKey(cid, role);
     const pair = this.peerConnectionPairs.get(key);
@@ -689,7 +681,7 @@ export class ClientEventReporter {
       outcome: 'failure',
       retry_count_attempt: 0,
       elapsed_time: Date.now() - pair.startedAt,
-      ice_state: iceState,
+      ice_state: toReportedIceState(pair.lastIceState),
       retry_failure_reason: reason,
       retry_failure_code: code,
     });
@@ -782,6 +774,20 @@ export class ClientEventReporter {
   };
 }
 
+const toReportedIceState = (
+  state: RTCIceConnectionState | undefined,
+): ReportedIceState => {
+  switch (state) {
+    case 'connected':
+    case 'completed':
+      return 'CONNECTED';
+    case 'failed':
+      return 'FAILED';
+    default:
+      return 'NOT_CONNECTED';
+  }
+};
+
 const readPermissionStatus = (
   permission: BrowserPermission,
 ): MediaPermissionState => {