@stream-io/video-client 1.61.1 → 1.62.0

package/src/helpers/AudioBindingsWatchdog.ts

@@ -10,12 +10,43 @@ const toBindingKey = (
   trackType: AudioTrackType = 'audioTrack',
 ) => `${sessionId}/${trackType}`;
 
+type AudioElementState = {
+  paused: boolean;
+  muted: boolean;
+  volume: number;
+  readyState: number;
+  sinkId?: string;
+  track: { enabled: boolean; muted: boolean; readyState: string } | null;
+};
+
+const toElementState = (element: HTMLAudioElement): AudioElementState => {
+  const stream = element.srcObject as MediaStream | null;
+  const [track] = stream?.getAudioTracks?.() ?? [];
+  return {
+    paused: element.paused,
+    muted: element.muted,
+    volume: Math.round(element.volume * 100) / 100,
+    readyState: element.readyState,
+    sinkId: element.sinkId,
+    track: track
+      ? {
+          enabled: track.enabled,
+          muted: track.muted,
+          readyState: track.readyState,
+        }
+      : null,
+  };
+};
+
 /**
  * Tracks audio element bindings and periodically warns about
  * remote participants whose audio streams have no bound element.
+ * On the same tick, it traces the playback state of the bound elements,
+ * limited to the bindings whose state changed since the previous trace.
  */
 export class AudioBindingsWatchdog {
   private bindings = new Map<string, HTMLAudioElement>();
+  private tracedStates = new Map<string, string>();
   private enabled = true;
   private watchdogInterval?: NodeJS.Timeout;
   private readonly unsubscribeCallingState: () => void;
@@ -30,7 +61,6 @@ export class AudioBindingsWatchdog {
     this.unsubscribeCallingState = createSubscription(
       state.callingState$,
       (callingState) => {
-        if (!this.enabled) return;
         if (callingState !== CallingState.JOINED) {
           this.stop();
         } else {
@@ -68,14 +98,14 @@ export class AudioBindingsWatchdog {
   };
 
   /**
-   * Enables or disables the watchdog.
-   * When disabled, the periodic check stops but bindings are still tracked.
+   * Enables or disables the dangling binding warnings.
+   * Bindings are still tracked and their playback state is still traced.
    */
   setEnabled = (enabled: boolean) => {
     this.enabled = enabled;
     if (enabled) {
       this.start();
-    } else {
+    } else if (this.state.callingState !== CallingState.JOINED) {
       this.stop();
     }
   };
@@ -86,45 +116,71 @@ export class AudioBindingsWatchdog {
   dispose = () => {
     this.stop();
     this.bindings.clear();
+    this.tracedStates.clear();
     this.unsubscribeCallingState();
   };
 
   private start = () => {
     clearInterval(this.watchdogInterval);
     this.watchdogInterval = setInterval(() => {
-      const danglingUserIds: string[] = [];
-      for (const p of this.state.participants) {
-        if (p.isLocalParticipant) continue;
-        const {
-          audioStream,
-          screenShareAudioStream,
-          sessionId,
-          userId,
-          publishedTracks,
-        } = p;
-        if (
-          audioStream &&
-          publishedTracks.includes(TrackType.AUDIO) &&
-          !this.bindings.has(toBindingKey(sessionId))
-        ) {
-          danglingUserIds.push(userId);
-        }
-        if (
-          screenShareAudioStream &&
-          publishedTracks.includes(TrackType.SCREEN_SHARE_AUDIO) &&
-          !this.bindings.has(toBindingKey(sessionId, 'screenShareAudioTrack'))
-        ) {
-          danglingUserIds.push(userId);
-        }
+      if (this.enabled) this.warnAboutDanglingBindings();
+      this.traceBindingStates();
+    }, 3000);
+  };
+
+  private warnAboutDanglingBindings = () => {
+    const danglingUserIds: string[] = [];
+    for (const p of this.state.participants) {
+      if (p.isLocalParticipant) continue;
+      const {
+        audioStream,
+        screenShareAudioStream,
+        sessionId,
+        userId,
+        publishedTracks,
+      } = p;
+      if (
+        audioStream &&
+        publishedTracks.includes(TrackType.AUDIO) &&
+        !this.bindings.has(toBindingKey(sessionId))
+      ) {
+        danglingUserIds.push(userId);
       }
-      if (danglingUserIds.length > 0) {
-        const key = 'audioBinding.danglingWarning';
-        this.tracer.traceOnce(key, key, danglingUserIds);
-        this.logger.warn(
-          `Dangling audio bindings detected. Did you forget to bind the audio element? user_ids: ${danglingUserIds}.`,
-        );
+      if (
+        screenShareAudioStream &&
+        publishedTracks.includes(TrackType.SCREEN_SHARE_AUDIO) &&
+        !this.bindings.has(toBindingKey(sessionId, 'screenShareAudioTrack'))
+      ) {
+        danglingUserIds.push(userId);
       }
-    }, 3000);
+    }
+    if (danglingUserIds.length > 0) {
+      const key = 'audioBinding.danglingWarning';
+      this.tracer.traceOnce(key, key, danglingUserIds);
+      this.logger.warn(
+        `Dangling audio bindings detected. Did you forget to bind the audio element? user_ids: ${danglingUserIds}.`,
+      );
+    }
+  };
+
+  private traceBindingStates = () => {
+    const changes: Record<string, AudioElementState | null> = {};
+    let hasChanges = false;
+    for (const [key, element] of this.bindings) {
+      const state = toElementState(element);
+      const serialized = JSON.stringify(state);
+      if (this.tracedStates.get(key) === serialized) continue;
+      this.tracedStates.set(key, serialized);
+      changes[key] = state;
+      hasChanges = true;
+    }
+    for (const key of this.tracedStates.keys()) {
+      if (this.bindings.has(key)) continue;
+      this.tracedStates.delete(key);
+      changes[key] = null;
+      hasChanges = true;
+    }
+    if (hasChanges) this.tracer.trace('audioBinding.state', changes);
   };
 
   private stop = () => {

package/src/helpers/DynascaleManager.ts

@@ -14,7 +14,11 @@ import {
   takeWhile,
 } from 'rxjs';
 import type { BlockedAudioTracker } from './BlockedAudioTracker';
-import { MediaPlaybackWatchdog } from './MediaPlaybackWatchdog';
+import {
+  MediaPlaybackWatchdog,
+  type MediaPlaybackWatchdogOptions,
+} from './MediaPlaybackWatchdog';
+import { watchMediaInterruptions } from './mediaInterruptions';
 import type { TrackSubscriptionManager } from './TrackSubscriptionManager';
 import { isFirefox, isSafari } from './browsers';
 import { hasScreenShare, hasVideo } from './participantUtils';
@@ -37,6 +41,9 @@ export class DynascaleManager {
   private readonly tracer: Tracer;
   private useWebAudio = false;
   private audioContext: AudioContext | undefined;
+  private playbackWatchdogs = new Set<MediaPlaybackWatchdog>();
+  private stopWatchingInterruptions: (() => void) | undefined;
+  private stopTracingAudioSession: (() => void) | undefined;
 
   private trackSubscriptionManager: TrackSubscriptionManager;
   private blockedAudioTracker: BlockedAudioTracker;
@@ -62,6 +69,10 @@ export class DynascaleManager {
    * Closes the audio context if it was created.
    */
   dispose = async () => {
+    this.stopWatchingInterruptions?.();
+    this.stopWatchingInterruptions = undefined;
+    this.stopTracingAudioSession?.();
+    this.stopTracingAudioSession = undefined;
     const context = this.audioContext;
     if (context && context.state !== 'closed') {
       document.removeEventListener('click', this.resumeAudioContext);
@@ -244,10 +255,12 @@ export class DynascaleManager {
     // https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide
     videoElement.muted = true;
 
-    const playbackWatchdog = new MediaPlaybackWatchdog({
+    const playbackWatchdog = this.createPlaybackWatchdog({
       element: videoElement,
       kind: 'video',
       tracer: this.tracer,
+      sessionId,
+      trackType,
     });
 
     const trackKey = isVideoTrack ? 'videoStream' : 'screenShareStream';
@@ -275,7 +288,7 @@ export class DynascaleManager {
       publishedTracksSubscription?.unsubscribe();
       streamSubscription.unsubscribe();
       resizeObserver?.disconnect();
-      playbackWatchdog.dispose();
+      this.disposePlaybackWatchdog(playbackWatchdog);
     };
   };
 
@@ -297,6 +310,7 @@ export class DynascaleManager {
   ) => {
     const participant = this.callState.findParticipantBySessionId(sessionId);
     if (!participant || participant.isLocalParticipant) return;
+    this.traceAudioSessionState();
 
     const participant$ = this.callState.participants$.pipe(
       map((ps) => ps.find((p) => p.sessionId === sessionId)),
@@ -309,7 +323,7 @@ export class DynascaleManager {
       deviceId: string,
       audioContext: AudioContext | undefined,
     ) => {
-      if (!deviceId) return;
+      if (!deviceId || !this.speaker.state.isDeviceSelectionSupported) return;
       if ('setSinkId' in audioElement) {
         audioElement.setSinkId(deviceId).catch((e) => {
           this.logger.warn(`Can't to set AudioElement sinkId`, e);
@@ -327,8 +341,10 @@ export class DynascaleManager {
     let sourceNode: MediaStreamAudioSourceNode | undefined = undefined;
     let gainNode: GainNode | undefined = undefined;
     let audioWatchdog: MediaPlaybackWatchdog | undefined = undefined;
+    let blockedByGiveUp = false;
 
     const clearBlockedAudio = () => {
+      blockedByGiveUp = false;
       if (!this.blockedAudioTracker.isBlocked(audioElement)) return;
       this.blockedAudioTracker.markBlocked(audioElement, false);
     };
@@ -336,15 +352,18 @@ export class DynascaleManager {
 
     const isAudioTrack = trackType === 'audioTrack';
     const trackKey = isAudioTrack ? 'audioStream' : 'screenShareAudioStream';
+    let sourceChangeTimer: ReturnType<typeof setTimeout> | undefined;
     const updateMediaStreamSubscription = participant$
       .pipe(distinctUntilKeyChanged(trackKey))
       .subscribe((p) => {
         const source = isAudioTrack ? p.audioStream : p.screenShareAudioStream;
         if (audioElement.srcObject === source) return;
 
-        setTimeout(() => {
+        clearTimeout(sourceChangeTimer);
+        sourceChangeTimer = setTimeout(() => {
+          sourceChangeTimer = undefined;
           audioElement.srcObject = source ?? null;
-          audioWatchdog?.dispose();
+          this.disposePlaybackWatchdog(audioWatchdog);
           audioWatchdog = undefined;
           if (!source) {
             clearBlockedAudio();
@@ -375,6 +394,7 @@ export class DynascaleManager {
               this.tracer.trace('audioPlaybackError', e.message);
               if (e.name === 'NotAllowedError') {
                 this.tracer.trace('audioPlaybackBlocked', null);
+                blockedByGiveUp = false;
                 this.blockedAudioTracker.markBlocked(
                   audioElement,
                   true,
@@ -383,11 +403,25 @@ export class DynascaleManager {
               }
               this.logger.warn(`Failed to play audio stream`, e);
             });
-            audioWatchdog = new MediaPlaybackWatchdog({
+            audioWatchdog = this.createPlaybackWatchdog({
               element: audioElement,
               kind: 'audio',
               tracer: this.tracer,
-              isBlocked: () => this.blockedAudioTracker.isBlocked(audioElement),
+              sessionId,
+              trackType,
+              // only an autoplay-policy block needs a user gesture; a give-up
+              // block is for the UI and must not stop a re-armed cycle
+              isBlocked: () =>
+                !blockedByGiveUp &&
+                this.blockedAudioTracker.isBlocked(audioElement),
+              onGiveUp: () => {
+                blockedByGiveUp = true;
+                this.blockedAudioTracker.markBlocked(
+                  audioElement,
+                  true,
+                  sessionId,
+                );
+              },
             });
           }
 
@@ -396,7 +430,7 @@ export class DynascaleManager {
         });
       });
 
-    const sinkIdSubscription = !('setSinkId' in audioElement)
+    const sinkIdSubscription = !this.speaker.state.isDeviceSelectionSupported
       ? null
       : this.speaker.state.selectedDevice$.subscribe((deviceId) => {
           const audioContext = this.getOrCreateAudioContext();
@@ -420,14 +454,56 @@ export class DynascaleManager {
       sinkIdSubscription?.unsubscribe();
       volumeSubscription.unsubscribe();
       updateMediaStreamSubscription.unsubscribe();
+      clearTimeout(sourceChangeTimer);
       audioElement.srcObject = null;
       sourceNode?.disconnect();
       gainNode?.disconnect();
-      audioWatchdog?.dispose();
+      this.disposePlaybackWatchdog(audioWatchdog);
       audioWatchdog = undefined;
     };
   };
 
+  /**
+   * Creates a playback watchdog that is re-armed whenever a media
+   * interruption (backgrounding, phone call, Siri) ends.
+   */
+  private createPlaybackWatchdog = (opts: MediaPlaybackWatchdogOptions) => {
+    const watchdog = new MediaPlaybackWatchdog(opts);
+    this.playbackWatchdogs.add(watchdog);
+    this.stopWatchingInterruptions ??= watchMediaInterruptions(
+      this.rearmPlayback,
+    );
+    return watchdog;
+  };
+
+  private disposePlaybackWatchdog = (watchdog?: MediaPlaybackWatchdog) => {
+    if (!watchdog) return;
+    watchdog.dispose();
+    this.playbackWatchdogs.delete(watchdog);
+  };
+
+  private rearmPlayback = () => {
+    this.tracer.trace('mediaPlayback.rearm', this.playbackWatchdogs.size);
+    this.playbackWatchdogs.forEach((watchdog) => watchdog.rearm());
+  };
+
+  /**
+   * Traces the WebKit audio session state (initial value and changes) in
+   * both playback modes, as it explains most iOS playback interruptions.
+   */
+  private traceAudioSessionState = () => {
+    const audioSession = navigator.audioSession;
+    if (!audioSession || this.stopTracingAudioSession) return;
+    const trace = () => {
+      this.tracer.trace('audioSession.state', audioSession.state);
+    };
+    trace();
+    audioSession.addEventListener('statechange', trace);
+    this.stopTracingAudioSession = () => {
+      audioSession.removeEventListener('statechange', trace);
+    };
+  };
+
   private getOrCreateAudioContext = (): AudioContext | undefined => {
     if (!this.useWebAudio) return;
     if (this.audioContext) return this.audioContext;
@@ -450,7 +526,6 @@ export class DynascaleManager {
 
       let isSessionInterrupted = false;
       audioSession.addEventListener('statechange', () => {
-        this.tracer.trace('audioSession.state', audioSession.state);
         if (audioSession.state === 'interrupted') {
           isSessionInterrupted = true;
         } else if (isSessionInterrupted) {

package/src/helpers/MediaPlaybackWatchdog.ts

@@ -8,7 +8,10 @@ export type MediaPlaybackWatchdogOptions = {
   element: HTMLMediaElement;
   kind: MediaKind;
   tracer: Tracer;
+  sessionId?: string;
+  trackType?: string;
   isBlocked?: () => boolean;
+  onGiveUp?: () => void;
 };
 
 /**
@@ -18,7 +21,13 @@ export type MediaPlaybackWatchdogOptions = {
 export class MediaPlaybackWatchdog {
   private logger = videoLoggerSystem.getLogger('MediaPlaybackWatchdog');
   private readonly kind: MediaKind;
+  private readonly traceInfo: {
+    kind: MediaKind;
+    sessionId?: string;
+    trackType?: string;
+  };
   private readonly isBlocked: () => boolean;
+  private readonly onGiveUp: () => void;
   private element: HTMLMediaElement;
   private tracer: Tracer;
   private controller = new AbortController();
@@ -31,8 +40,14 @@ export class MediaPlaybackWatchdog {
   constructor(opts: MediaPlaybackWatchdogOptions) {
     this.element = opts.element;
     this.kind = opts.kind;
+    this.traceInfo = {
+      kind: opts.kind,
+      sessionId: opts.sessionId,
+      trackType: opts.trackType,
+    };
     this.tracer = opts.tracer;
     this.isBlocked = opts.isBlocked ?? (() => false);
+    this.onGiveUp = opts.onGiveUp ?? (() => {});
     this.attach();
   }
 
@@ -51,6 +66,22 @@ export class MediaPlaybackWatchdog {
     this.clearTimers();
   };
 
+  /**
+   * Starts a fresh recovery cycle, e.g. after a media interruption ended.
+   * Unlike a regular pause-triggered recovery, it also tries elements that
+   * were skipped as blocked or not ready, since those conditions may have
+   * cleared together with the interruption.
+   */
+  rearm = () => {
+    if (this.disposed) return;
+    this.clearTimers();
+    this.attempt = 0;
+    this.stopped = false;
+    const { paused, srcObject, ended } = this.element;
+    if (!paused || !srcObject || ended) return;
+    this.pendingTimer = setTimeout(this.attemptPlay, 0);
+  };
+
   private clearTimers = () => {
     if (this.pendingTimer) clearTimeout(this.pendingTimer);
     this.pendingTimer = undefined;
@@ -69,17 +100,34 @@ export class MediaPlaybackWatchdog {
     this.settleTimer = undefined;
     if (this.element.paused) return;
     this.tracer.trace('mediaPlayback.recover.success', {
-      kind: this.kind,
+      ...this.traceInfo,
       attempts: this.attempt,
     });
     this.attempt = 0;
     this.stopped = false;
   };
 
+  /**
+   * Media elements backed by a MediaStream fire `suspend` routinely (after
+   * the initial load, on source swaps, on layer switches), so a `suspend`
+   * that needs no recovery is ignored without a trace. An explicit `pause`
+   * is rare and always traced, along with the reason recovery was skipped.
+   */
   private onPauseOrSuspend = (event: Event) => {
-    if (this.disposed || this.stopped) return;
+    if (this.disposed || this.stopped || this.pendingTimer) return;
+    const skipReason = this.computeSkipReason();
+    if (skipReason) {
+      if (event.type === 'pause') {
+        this.tracer.trace('mediaPlayback.paused', {
+          ...this.traceInfo,
+          reason: event.type,
+          skipped: skipReason,
+        });
+      }
+      return;
+    }
     this.tracer.trace('mediaPlayback.paused', {
-      kind: this.kind,
+      ...this.traceInfo,
       reason: event.type,
     });
     if (this.settleTimer) clearTimeout(this.settleTimer);
@@ -92,7 +140,7 @@ export class MediaPlaybackWatchdog {
     const skipReason = this.computeSkipReason();
     if (skipReason) {
       this.tracer.trace('mediaPlayback.recover.skipped', {
-        kind: this.kind,
+        ...this.traceInfo,
         reason: skipReason,
       });
       return;
@@ -102,9 +150,10 @@ export class MediaPlaybackWatchdog {
       this.stopped = true;
       this.clearTimers();
       this.tracer.trace('mediaPlayback.recover.giveUp', {
-        kind: this.kind,
+        ...this.traceInfo,
         attempts: this.attempt,
       });
+      this.onGiveUp();
       return;
     }
     const delay = this.attempt === 0 ? 0 : retryInterval(this.attempt);
@@ -127,7 +176,7 @@ export class MediaPlaybackWatchdog {
     if (this.disposed) return;
     this.attempt += 1;
     this.tracer.trace('mediaPlayback.recover.attempt', {
-      kind: this.kind,
+      ...this.traceInfo,
       attempt: this.attempt,
     });
     try {