@stream-io/video-client 1.55.2 → 1.57.0

package/dist/src/helpers/BlockedAudioTracker.d.ts

@@ -12,11 +12,26 @@ export declare class BlockedAudioTracker {
      * Use {@link resumeAudio} within a user gesture to unblock.
      */
     autoplayBlocked$: import("rxjs").Observable<boolean>;
+    /**
+     * The list of participant `sessionId`s whose audio element is currently
+     * blocked by the browser's autoplay policy. Only some participants may be
+     * blocked (e.g. one joined while a gesture was active and a later one was
+     * not), so use this to render a per-participant affordance rather than a
+     * call-wide one. Call {@link resumeAudio} within a user gesture to unblock
+     * all of them.
+     *
+     * Session ids are registered together with the audio element.
+     */
+    blockedSessionIds$: import("rxjs").Observable<string[]>;
     constructor(tracer: Tracer);
     /**
      * Registers an audio element as blocked by the browser's autoplay policy.
      */
-    markBlocked: (audioElement: HTMLAudioElement, blocked: boolean) => void;
+    markBlocked: (audioElement: HTMLAudioElement, blocked: boolean, sessionId?: string) => void;
+    /**
+     * Clears all tracked elements. The tracker stays usable afterwards.
+     */
+    reset: () => void;
     /**
      * Returns whether the given audio element is currently flagged as blocked
      * by the browser's autoplay policy.

package/dist/src/helpers/MediaPlaybackWatchdog.d.ts

@@ -18,12 +18,16 @@ export declare class MediaPlaybackWatchdog {
     private tracer;
     private controller;
     private pendingTimer;
+    private settleTimer;
     private attempt;
     private disposed;
+    private stopped;
     constructor(opts: MediaPlaybackWatchdogOptions);
     private attach;
     dispose: () => void;
+    private clearTimers;
     private onPlaying;
+    private onSettled;
     private onPauseOrSuspend;
     private scheduleRecovery;
     private computeSkipReason;

package/dist/src/reporting/ClientEventReporter.d.ts

@@ -4,6 +4,7 @@ import type { PeerConnectionStateChangeEvent } from '../rtc';
 export type ClientEventPeerConnection = 'publish' | 'subscribe';
 export type ClientEventStage = 'JoinInitiated' | 'CoordinatorWS' | 'MediaDevicePermission' | 'CoordinatorJoin' | 'WSJoin' | 'PeerConnectionConnect' | 'FirstVideoFrame' | 'FirstAudioFrame';
 export type MediaPermissionState = 'INITIATED' | 'FAILED' | 'GRANTED' | 'NOT_INITIATED';
+export type ReportedIceState = 'CONNECTED' | 'FAILED' | 'NOT_CONNECTED';
 export type JoinReason = 'first-attempt' | 'network-available' | 'migration' | 'full-rejoin';
 export type ClientEventStandardCode = 'CLIENT_ABORTED' | 'BACKEND_LEAVE' | 'REQUEST_TIMEOUT' | 'NETWORK_ERROR' | 'SFU_ERROR' | 'SFU_GO_AWAY' | 'ICE_CONNECTIVITY_FAILED' | 'DTLS_CONNECTIVITY_FAILED';
 export type CallReportContext = {

package/dist/src/rtc/Publisher.d.ts

@@ -11,6 +11,7 @@ export declare class Publisher extends BasePeerConnection {
     private readonly clonedTracks;
     private publishOptions;
     private readonly selfSubEnabled;
+    private readonly transceiverLockKey;
     /**
      * Constructs a new `Publisher` instance.
      */

package/dist/src/rtc/types.d.ts

@@ -43,16 +43,23 @@ export type OnIceConnected = (peerType: PeerType) => void;
  * Snapshot of the peer connection's ICE and DTLS state surfaced to telemetry
  * consumers (e.g. `ClientEventReporter`). Fired on every transition of
  * either `iceConnectionState` or `peerConnectionState`.
+ *
+ * `iceConnectionState` is the live `RTCPeerConnection.iceConnectionState` read
+ * at the moment the event fires, regardless of which transition triggered it.
+ * Consumers must use it (rather than assuming a value from `stateType`) because
+ * the browser does not guarantee the relative ordering of the `ice` and
+ * `peerConnection` `failed` transitions.
  */
 export type PeerConnectionStateChangeEvent = {
     peerType: PeerType;
+    iceConnectionState: RTCIceConnectionState;
+} & ({
     stateType: 'ice';
     state: RTCIceConnectionState;
 } | {
-    peerType: PeerType;
     stateType: 'peerConnection';
     state: RTCPeerConnectionState;
-};
+});
 export type OnPeerConnectionStateChange = (event: PeerConnectionStateChangeEvent) => void;
 /**
  * Fired when a remote track starts receiving media (`unmute`). Used by

package/dist/src/store/rxUtils.d.ts

@@ -5,6 +5,11 @@ type AsyncFunctionPatch<T> = (currentValue: T) => Promise<T>;
  * A value or a function which takes the current value and returns a new value.
  */
 export type Patch<T> = T | FunctionPatch<T>;
+/**
+ * Performs shallow comparison of two arrays.
+ * Expects primitive values: [1, 2, 3] is equal to [2, 1, 3].
+ */
+export declare const isShallowArrayEqual: <T>(a: Array<T>, b: Array<T>) => boolean;
 /**
  * Gets the current value of an observable, or undefined if the observable has
  * not emitted a value yet.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-client",
-  "version": "1.55.2",
+  "version": "1.57.0",
   "main": "dist/index.cjs.js",
   "module": "dist/index.es.js",
   "browser": "dist/index.browser.es.js",
@@ -36,31 +36,31 @@
     "@protobuf-ts/twirp-transport": "^2.11.1",
     "@stream-io/logger": "^2.0.0",
     "@stream-io/worker-timer": "^1.2.5",
-    "axios": "^1.16.1",
+    "axios": "^1.19.0",
     "rxjs": "~7.8.2",
     "sdp-transform": "^2.15.0",
     "ua-parser-js": "^1.0.41",
     "webrtc-adapter": "^8.2.4"
   },
   "devDependencies": {
-    "@openapitools/openapi-generator-cli": "^2.34.0",
+    "@openapitools/openapi-generator-cli": "^2.40.1",
     "@rollup/plugin-replace": "^6.0.3",
     "@rollup/plugin-typescript": "^12.3.0",
-    "@stream-io/audio-filters-web": "^0.10.1",
-    "@stream-io/node-sdk": "^0.7.59",
+    "@stream-io/audio-filters-web": "^0.10.4",
+    "@stream-io/node-sdk": "^0.7.63",
     "@stream-io/typescript-config": "^0.0.0",
     "@total-typescript/shoehorn": "^0.1.2",
-    "@types/node": "^25.9.3",
+    "@types/node": "^25.9.5",
     "@types/sdp-transform": "^2.15.0",
     "@types/ua-parser-js": "^0.7.39",
-    "@vitest/coverage-v8": "^4.1.7",
+    "@vitest/coverage-v8": "^4.1.10",
     "dotenv": "^16.6.1",
-    "happy-dom": "^20.9.0",
-    "prettier": "^3.8.3",
+    "happy-dom": "^20.11.1",
+    "prettier": "^3.9.6",
     "rimraf": "^6.1.3",
-    "rollup": "^4.60.4",
+    "rollup": "^4.62.4",
     "typescript": "^6.0.3",
-    "vitest": "^4.1.7",
+    "vitest": "^4.1.10",
     "vitest-mock-extended": "^4.0.0"
   }
 }

package/src/Call.ts

@@ -404,6 +404,7 @@ export class Call {
       this.tracer,
     );
     this.blockedAudioTracker = new BlockedAudioTracker(this.tracer);
+    this.leaveCallHooks.add(this.blockedAudioTracker.reset);
 
     if (typeof document !== 'undefined') {
       this.audioBindingsWatchdog = new AudioBindingsWatchdog(

package/src/devices/__tests__/MicrophoneManagerRN.test.ts

@@ -16,8 +16,7 @@ import { settled, withoutConcurrency } from '../../helpers/concurrency';
 import { ClientEventReporter } from '../../reporting';
 
 let speechActivityCallback:
-  | ((state: { isSoundDetected: boolean }) => void)
-  | null = null;
+  ((state: { isSoundDetected: boolean }) => void) | null = null;
 let unsubscribeMocks: ReturnType<typeof vi.fn>[] = [];
 
 vi.mock('../../helpers/platforms.ts', () => {

package/src/devices/devicePersistence.ts

@@ -24,8 +24,7 @@ export type LocalDevicePreference = {
 
 export type LocalDevicePreferences = {
   [type in DevicePreferenceKey]?:
-    | LocalDevicePreference
-    | LocalDevicePreference[];
+    LocalDevicePreference | LocalDevicePreference[];
 };
 
 export const defaultDeviceId = 'default';

package/src/helpers/BlockedAudioTracker.ts

@@ -1,9 +1,18 @@
-import { BehaviorSubject, distinctUntilChanged, map } from 'rxjs';
+import { BehaviorSubject, distinctUntilChanged, map, shareReplay } from 'rxjs';
 import { videoLoggerSystem } from '../logger';
 import { Tracer } from '../stats';
-import { setCurrentValue, setCurrentValueAsync } from '../store/rxUtils';
+import {
+  isShallowArrayEqual,
+  setCurrentValue,
+  setCurrentValueAsync,
+} from '../store/rxUtils';
 import { timeboxed } from '../coordinator/connection/utils';
 
+type BlockedAudioElement = {
+  element: HTMLAudioElement;
+  sessionId?: string;
+};
+
 /**
  * Tracks audio elements that the browser's autoplay policy has blocked.
  */
@@ -11,8 +20,8 @@ export class BlockedAudioTracker {
   private logger = videoLoggerSystem.getLogger('BlockedAudioTracker');
   private tracer: Tracer;
 
-  private blockedElementsSubject = new BehaviorSubject(
-    new Set<HTMLAudioElement>(),
+  private blockedElementsSubject = new BehaviorSubject<BlockedAudioElement[]>(
+    [],
   );
 
   /**
@@ -21,10 +30,32 @@ export class BlockedAudioTracker {
    * Use {@link resumeAudio} within a user gesture to unblock.
    */
   autoplayBlocked$ = this.blockedElementsSubject.pipe(
-    map((elements) => elements.size > 0),
+    map((elements) => elements.length > 0),
     distinctUntilChanged(),
   );
 
+  /**
+   * The list of participant `sessionId`s whose audio element is currently
+   * blocked by the browser's autoplay policy. Only some participants may be
+   * blocked (e.g. one joined while a gesture was active and a later one was
+   * not), so use this to render a per-participant affordance rather than a
+   * call-wide one. Call {@link resumeAudio} within a user gesture to unblock
+   * all of them.
+   *
+   * Session ids are registered together with the audio element.
+   */
+  blockedSessionIds$ = this.blockedElementsSubject.pipe(
+    map((elements) => {
+      const ids: string[] = [];
+      elements.forEach(({ sessionId }) => {
+        if (sessionId && !ids.includes(sessionId)) ids.push(sessionId);
+      });
+      return ids;
+    }),
+    distinctUntilChanged(isShallowArrayEqual),
+    shareReplay({ bufferSize: 1, refCount: true }),
+  );
+
   constructor(tracer: Tracer) {
     this.tracer = tracer;
   }
@@ -32,20 +63,46 @@ export class BlockedAudioTracker {
   /**
    * Registers an audio element as blocked by the browser's autoplay policy.
    */
-  markBlocked = (audioElement: HTMLAudioElement, blocked: boolean) => {
+  markBlocked = (
+    audioElement: HTMLAudioElement,
+    blocked: boolean,
+    sessionId?: string,
+  ) => {
     setCurrentValue(this.blockedElementsSubject, (elements) => {
-      if (blocked) elements.add(audioElement);
-      else elements.delete(audioElement);
-      return elements;
+      if (!blocked) {
+        return elements.filter(({ element }) => element !== audioElement);
+      }
+
+      const existing = elements.find(({ element }) => element === audioElement);
+      if (existing) {
+        return elements.map((entry) =>
+          entry.element === audioElement
+            ? { element: audioElement, sessionId: sessionId ?? entry.sessionId }
+            : entry,
+        );
+      }
+
+      return [...elements, { element: audioElement, sessionId }];
     });
   };
 
+  /**
+   * Clears all tracked elements. The tracker stays usable afterwards.
+   */
+  reset = () => {
+    if (this.blockedElementsSubject.getValue().length > 0) {
+      this.blockedElementsSubject.next([]);
+    }
+  };
+
   /**
    * Returns whether the given audio element is currently flagged as blocked
    * by the browser's autoplay policy.
    */
   isBlocked = (audioElement: HTMLAudioElement): boolean => {
-    return this.blockedElementsSubject.getValue().has(audioElement);
+    return this.blockedElementsSubject
+      .getValue()
+      .some(({ element }) => element === audioElement);
   };
 
   /**
@@ -57,17 +114,18 @@ export class BlockedAudioTracker {
     await setCurrentValueAsync(
       this.blockedElementsSubject,
       async (elements) => {
+        let next = elements;
         await Promise.all(
-          Array.from(elements, async (element) => {
+          elements.map(async ({ element }) => {
             try {
               if (element.srcObject) await timeboxed([element.play()], 2000);
-              elements.delete(element);
+              next = next.filter((entry) => entry.element !== element);
             } catch (err) {
               this.logger.warn(`Can't resume audio for element`, element, err);
             }
           }),
         );
-        return elements;
+        return next;
       },
     );
   };

package/src/helpers/DynascaleManager.ts

@@ -328,6 +328,12 @@ export class DynascaleManager {
     let gainNode: GainNode | undefined = undefined;
     let audioWatchdog: MediaPlaybackWatchdog | undefined = undefined;
 
+    const clearBlockedAudio = () => {
+      if (!this.blockedAudioTracker.isBlocked(audioElement)) return;
+      this.blockedAudioTracker.markBlocked(audioElement, false);
+    };
+    audioElement.addEventListener('playing', clearBlockedAudio);
+
     const isAudioTrack = trackType === 'audioTrack';
     const trackKey = isAudioTrack ? 'audioStream' : 'screenShareAudioStream';
     const updateMediaStreamSubscription = participant$
@@ -341,7 +347,7 @@ export class DynascaleManager {
           audioWatchdog?.dispose();
           audioWatchdog = undefined;
           if (!source) {
-            this.blockedAudioTracker.markBlocked(audioElement, false);
+            clearBlockedAudio();
             return;
           }
 
@@ -369,7 +375,11 @@ export class DynascaleManager {
               this.tracer.trace('audioPlaybackError', e.message);
               if (e.name === 'NotAllowedError') {
                 this.tracer.trace('audioPlaybackBlocked', null);
-                this.blockedAudioTracker.markBlocked(audioElement, true);
+                this.blockedAudioTracker.markBlocked(
+                  audioElement,
+                  true,
+                  sessionId,
+                );
               }
               this.logger.warn(`Failed to play audio stream`, e);
             });
@@ -405,7 +415,8 @@ export class DynascaleManager {
     audioElement.autoplay = true;
 
     return () => {
-      this.blockedAudioTracker.markBlocked(audioElement, false);
+      audioElement.removeEventListener('playing', clearBlockedAudio);
+      clearBlockedAudio();
       sinkIdSubscription?.unsubscribe();
       volumeSubscription.unsubscribe();
       updateMediaStreamSubscription.unsubscribe();

package/src/helpers/MediaPlaybackWatchdog.ts

@@ -23,8 +23,10 @@ export class MediaPlaybackWatchdog {
   private tracer: Tracer;
   private controller = new AbortController();
   private pendingTimer: ReturnType<typeof setTimeout> | undefined;
+  private settleTimer: ReturnType<typeof setTimeout> | undefined;
   private attempt = 0;
   private disposed = false;
+  private stopped = false;
 
   constructor(opts: MediaPlaybackWatchdogOptions) {
     this.element = opts.element;
@@ -46,33 +48,47 @@ export class MediaPlaybackWatchdog {
     if (this.disposed) return;
     this.disposed = true;
     this.controller.abort();
+    this.clearTimers();
+  };
+
+  private clearTimers = () => {
     if (this.pendingTimer) clearTimeout(this.pendingTimer);
     this.pendingTimer = undefined;
+    if (this.settleTimer) clearTimeout(this.settleTimer);
+    this.settleTimer = undefined;
   };
 
   private onPlaying = () => {
-    if (this.attempt > 0) {
-      this.tracer.trace('mediaPlayback.recover.success', {
-        kind: this.kind,
-        attempts: this.attempt,
-      });
-    }
-    this.attempt = 0;
     if (this.pendingTimer) clearTimeout(this.pendingTimer);
     this.pendingTimer = undefined;
+    if (this.attempt === 0 || this.settleTimer) return;
+    this.settleTimer ??= setTimeout(this.onSettled, 1000);
+  };
+
+  private onSettled = () => {
+    this.settleTimer = undefined;
+    if (this.element.paused) return;
+    this.tracer.trace('mediaPlayback.recover.success', {
+      kind: this.kind,
+      attempts: this.attempt,
+    });
+    this.attempt = 0;
+    this.stopped = false;
   };
 
   private onPauseOrSuspend = (event: Event) => {
-    if (this.disposed) return;
+    if (this.disposed || this.stopped) return;
     this.tracer.trace('mediaPlayback.paused', {
       kind: this.kind,
       reason: event.type,
     });
+    if (this.settleTimer) clearTimeout(this.settleTimer);
+    this.settleTimer = undefined;
     this.scheduleRecovery();
   };
 
   private scheduleRecovery = () => {
-    if (this.disposed || this.pendingTimer) return;
+    if (this.disposed || this.stopped || this.pendingTimer) return;
     const skipReason = this.computeSkipReason();
     if (skipReason) {
       this.tracer.trace('mediaPlayback.recover.skipped', {
@@ -81,6 +97,16 @@ export class MediaPlaybackWatchdog {
       });
       return;
     }
+
+    if (this.attempt >= 10) {
+      this.stopped = true;
+      this.clearTimers();
+      this.tracer.trace('mediaPlayback.recover.giveUp', {
+        kind: this.kind,
+        attempts: this.attempt,
+      });
+      return;
+    }
     const delay = this.attempt === 0 ? 0 : retryInterval(this.attempt);
     this.pendingTimer = setTimeout(this.attemptPlay, delay);
   };
@@ -109,13 +135,6 @@ export class MediaPlaybackWatchdog {
     } catch (err) {
       if (this.disposed) return;
       this.logger.warn(`Failed to recover ${this.kind} playback`, err);
-      if (this.attempt >= 10) {
-        this.tracer.trace('mediaPlayback.recover.giveUp', {
-          kind: this.kind,
-          attempts: this.attempt,
-        });
-        return;
-      }
       this.scheduleRecovery();
     }
   };

package/src/helpers/__tests__/BlockedAudioTracker.test.ts

@@ -2,7 +2,7 @@
  * @vitest-environment happy-dom
  */
 
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { BlockedAudioTracker } from '../BlockedAudioTracker';
 import { getCurrentValue } from '../../store/rxUtils';
 import type { Tracer } from '../../stats';
@@ -30,6 +30,10 @@ describe('BlockedAudioTracker', () => {
     tracker = new BlockedAudioTracker(tracer);
   });
 
+  afterEach(() => {
+    tracker.reset();
+  });
+
   it('emits autoplayBlocked$ true after markBlocked(el, true)', () => {
     expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
     tracker.markBlocked(createAudioElement(), true);
@@ -58,14 +62,58 @@ describe('BlockedAudioTracker', () => {
     expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
   });
 
+  it('emits blocked participant session ids', () => {
+    const el1 = createAudioElement();
+    const el2 = createAudioElement();
+
+    tracker.markBlocked(el1, true, 'session-id-1');
+    tracker.markBlocked(el2, true, 'session-id-2');
+
+    expect(getCurrentValue(tracker.blockedSessionIds$)).toEqual([
+      'session-id-1',
+      'session-id-2',
+    ]);
+
+    tracker.markBlocked(el1, false);
+
+    expect(getCurrentValue(tracker.blockedSessionIds$)).toEqual([
+      'session-id-2',
+    ]);
+  });
+
+  it(`doesn't emit when blocked participant session ids don't change`, () => {
+    const el1 = createAudioElement();
+    const el2 = createAudioElement();
+    tracker.markBlocked(el1, true, 'session-id-1');
+
+    const subscriber = vi.fn();
+    const subscription = tracker.blockedSessionIds$.subscribe(subscriber);
+
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el1, true, 'session-id-1');
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el2, true, 'session-id-1');
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el1, false);
+    expect(subscriber).toHaveBeenCalledTimes(1);
+
+    tracker.markBlocked(el2, false);
+    expect(subscriber).toHaveBeenCalledTimes(2);
+
+    subscription.unsubscribe();
+  });
+
   it('resumeAudio plays each element with srcObject and clears successful ones', async () => {
     const el1 = createAudioElement();
     const el2 = createAudioElement();
     const play1 = vi.spyOn(el1, 'play').mockResolvedValue();
     const play2 = vi.spyOn(el2, 'play').mockResolvedValue();
 
-    tracker.markBlocked(el1, true);
-    tracker.markBlocked(el2, true);
+    tracker.markBlocked(el1, true, 'session-id-1');
+    tracker.markBlocked(el2, true, 'session-id-2');
 
     await tracker.resumeAudio();
 
@@ -82,8 +130,8 @@ describe('BlockedAudioTracker', () => {
       new DOMException('', 'NotAllowedError'),
     );
 
-    tracker.markBlocked(ok, true);
-    tracker.markBlocked(stillBlocked, true);
+    tracker.markBlocked(ok, true, 'session-id-1');
+    tracker.markBlocked(stillBlocked, true, 'session-id-2');
 
     await tracker.resumeAudio();
 
@@ -100,7 +148,7 @@ describe('BlockedAudioTracker', () => {
     const detached = createAudioElement(false);
     const play = vi.spyOn(detached, 'play').mockResolvedValue();
 
-    tracker.markBlocked(detached, true);
+    tracker.markBlocked(detached, true, 'session-id');
     await tracker.resumeAudio();
 
     expect(play).not.toHaveBeenCalled();

package/src/helpers/__tests__/DynascaleManager.test.ts

@@ -143,6 +143,51 @@ describe('DynascaleManager', () => {
       cleanup?.();
     });
 
+    it('audio: tracks the blocked session and clears it once the element plays', async () => {
+      vi.useFakeTimers();
+      let cleanup: ReturnType<typeof dynascaleManager.bindAudioElement>;
+      try {
+        const audioElement = document.createElement('audio');
+        Object.defineProperty(audioElement, 'srcObject', { writable: true });
+        vi.spyOn(audioElement, 'play').mockRejectedValue(
+          new DOMException('', 'NotAllowedError'),
+        );
+
+        // @ts-expect-error incomplete data
+        call.state.updateOrAddParticipant('session-id', {
+          userId: 'user-id',
+          sessionId: 'session-id',
+          publishedTracks: [],
+        });
+
+        cleanup = dynascaleManager.bindAudioElement(
+          audioElement,
+          'session-id',
+          'audioTrack',
+        );
+
+        call.state.updateParticipant('session-id', {
+          audioStream: new MediaStream(),
+        });
+        await vi.advanceTimersByTimeAsync(0); // flush the rejected play()
+
+        expect(call.blockedAudioTracker.isBlocked(audioElement)).toBe(true);
+        expect(
+          getCurrentValue(call.blockedAudioTracker.blockedSessionIds$),
+        ).toEqual(['session-id']);
+
+        audioElement.dispatchEvent(new Event('playing'));
+
+        expect(call.blockedAudioTracker.isBlocked(audioElement)).toBe(false);
+        expect(
+          getCurrentValue(call.blockedAudioTracker.blockedSessionIds$),
+        ).toEqual([]);
+      } finally {
+        cleanup?.();
+        vi.useRealTimers();
+      }
+    });
+
     it('audio: Safari should use AudioContext for audio playback', () => {
       globalThis._isSafari = true;
       dynascaleManager.setUseWebAudio(true); // enabled by default on Safari