@stream-io/video-client 1.44.4 → 1.44.5

package/dist/src/helpers/AudioBindingsWatchdog.d.ts

@@ -0,0 +1,37 @@
+import type { AudioTrackType } from '../types';
+import { CallState } from '../store';
+import { Tracer } from '../stats';
+/**
+ * Tracks audio element bindings and periodically warns about
+ * remote participants whose audio streams have no bound element.
+ */
+export declare class AudioBindingsWatchdog {
+    private state;
+    private tracer;
+    private bindings;
+    private enabled;
+    private watchdogInterval?;
+    private readonly unsubscribeCallingState;
+    private logger;
+    constructor(state: CallState, tracer: Tracer);
+    /**
+     * Registers an audio element binding for the given session and track type.
+     * Warns if a different element is already bound to the same key.
+     */
+    register: (audioElement: HTMLAudioElement, sessionId: string, trackType: AudioTrackType) => void;
+    /**
+     * Removes the audio element binding for the given session and track type.
+     */
+    unregister: (sessionId: string, trackType: AudioTrackType) => void;
+    /**
+     * Enables or disables the watchdog.
+     * When disabled, the periodic check stops but bindings are still tracked.
+     */
+    setEnabled: (enabled: boolean) => void;
+    /**
+     * Stops the watchdog and unsubscribes from callingState changes.
+     */
+    dispose: () => void;
+    private start;
+    private stop;
+}

package/dist/src/helpers/DynascaleManager.d.ts

@@ -1,8 +1,9 @@
 import { AudioTrackType, DebounceType, VideoTrackType } from '../types';
 import { VideoDimension } from '../gen/video/sfu/models/models';
 import { ViewportTracker } from './ViewportTracker';
+import { AudioBindingsWatchdog } from './AudioBindingsWatchdog';
 import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signal';
-import type { CallState } from '../store';
+import { CallState } from '../store';
 import type { StreamSfuClient } from '../StreamSfuClient';
 import { SpeakerManager } from '../devices';
 import { Tracer } from '../stats';
@@ -40,6 +41,7 @@ export declare class DynascaleManager {
     private audioContext;
     private sfuClient;
     private pendingSubscriptionsUpdate;
+    readonly audioBindingsWatchdog: AudioBindingsWatchdog | undefined;
     private videoTrackSubscriptionOverridesSubject;
     videoTrackSubscriptionOverrides$: import("rxjs").Observable<VideoTrackSubscriptionOverrides>;
     incomingVideoSettings$: import("rxjs").Observable<{

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-client",
-  "version": "1.44.4",
+  "version": "1.44.5",
   "main": "dist/index.cjs.js",
   "module": "dist/index.es.js",
   "browser": "dist/index.browser.es.js",

package/src/devices/devices.ts

@@ -11,7 +11,7 @@ import {
 } from 'rxjs';
 import { BrowserPermission } from './BrowserPermission';
 import { lazy } from '../helpers/lazy';
-import { isFirefox, isSafari } from '../helpers/browsers';
+import { isFirefox } from '../helpers/browsers';
 import { dumpStream, Tracer } from '../stats';
 import { getCurrentValue } from '../store/rxUtils';
 import { videoLoggerSystem } from '../logger';
@@ -61,8 +61,6 @@ const getDevices = (
  */
 export const checkIfAudioOutputChangeSupported = () => {
   if (typeof document === 'undefined') return false;
-  // Safari uses WebAudio API for playing audio, so we check the AudioContext prototype
-  if (isSafari()) return 'setSinkId' in AudioContext.prototype;
   const element = document.createElement('audio');
   return 'setSinkId' in element;
 };

package/src/helpers/AudioBindingsWatchdog.ts

@@ -0,0 +1,118 @@
+import type { AudioTrackType } from '../types';
+import { CallingState, CallState } from '../store';
+import { createSubscription } from '../store/rxUtils';
+import { videoLoggerSystem } from '../logger';
+import { Tracer } from '../stats';
+
+const toBindingKey = (
+  sessionId: string,
+  trackType: AudioTrackType = 'audioTrack',
+) => `${sessionId}/${trackType}`;
+
+/**
+ * Tracks audio element bindings and periodically warns about
+ * remote participants whose audio streams have no bound element.
+ */
+export class AudioBindingsWatchdog {
+  private bindings = new Map<string, HTMLAudioElement>();
+  private enabled = true;
+  private watchdogInterval?: NodeJS.Timeout;
+  private readonly unsubscribeCallingState: () => void;
+  private logger = videoLoggerSystem.getLogger('AudioBindingsWatchdog');
+
+  constructor(
+    private state: CallState,
+    private tracer: Tracer,
+  ) {
+    this.unsubscribeCallingState = createSubscription(
+      state.callingState$,
+      (callingState) => {
+        if (!this.enabled) return;
+        if (callingState !== CallingState.JOINED) {
+          this.stop();
+        } else {
+          this.start();
+        }
+      },
+    );
+  }
+
+  /**
+   * Registers an audio element binding for the given session and track type.
+   * Warns if a different element is already bound to the same key.
+   */
+  register = (
+    audioElement: HTMLAudioElement,
+    sessionId: string,
+    trackType: AudioTrackType,
+  ) => {
+    const key = toBindingKey(sessionId, trackType);
+    const existing = this.bindings.get(key);
+    if (existing && existing !== audioElement) {
+      this.logger.warn(
+        `Audio element already bound to ${sessionId} and ${trackType}`,
+      );
+      this.tracer.trace('audioBinding.alreadyBoundWarning', trackType);
+    }
+    this.bindings.set(key, audioElement);
+  };
+
+  /**
+   * Removes the audio element binding for the given session and track type.
+   */
+  unregister = (sessionId: string, trackType: AudioTrackType) => {
+    this.bindings.delete(toBindingKey(sessionId, trackType));
+  };
+
+  /**
+   * Enables or disables the watchdog.
+   * When disabled, the periodic check stops but bindings are still tracked.
+   */
+  setEnabled = (enabled: boolean) => {
+    this.enabled = enabled;
+    if (enabled) {
+      this.start();
+    } else {
+      this.stop();
+    }
+  };
+
+  /**
+   * Stops the watchdog and unsubscribes from callingState changes.
+   */
+  dispose = () => {
+    this.stop();
+    this.unsubscribeCallingState();
+  };
+
+  private start = () => {
+    clearInterval(this.watchdogInterval);
+    this.watchdogInterval = setInterval(() => {
+      const danglingUserIds: string[] = [];
+      for (const p of this.state.participants) {
+        if (p.isLocalParticipant) continue;
+        const { audioStream, screenShareAudioStream, sessionId, userId } = p;
+        if (audioStream && !this.bindings.has(toBindingKey(sessionId))) {
+          danglingUserIds.push(userId);
+        }
+        if (
+          screenShareAudioStream &&
+          !this.bindings.has(toBindingKey(sessionId, 'screenShareAudioTrack'))
+        ) {
+          danglingUserIds.push(userId);
+        }
+      }
+      if (danglingUserIds.length > 0) {
+        const key = 'audioBinding.danglingWarning';
+        this.tracer.traceOnce(key, key, danglingUserIds);
+        this.logger.warn(
+          `Dangling audio bindings detected. Did you forget to bind the audio element? user_ids: ${danglingUserIds}.`,
+        );
+      }
+    }, 3000);
+  };
+
+  private stop = () => {
+    clearInterval(this.watchdogInterval);
+  };
+}

package/src/helpers/DynascaleManager.ts

@@ -15,14 +15,16 @@ import {
   takeWhile,
 } from 'rxjs';
 import { ViewportTracker } from './ViewportTracker';
+import { AudioBindingsWatchdog } from './AudioBindingsWatchdog';
 import { isFirefox, isSafari } from './browsers';
+import { isReactNative } from './platforms';
 import {
   hasScreenShare,
   hasScreenShareAudio,
   hasVideo,
 } from './participantUtils';
 import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signal';
-import type { CallState } from '../store';
+import { CallState } from '../store';
 import type { StreamSfuClient } from '../StreamSfuClient';
 import { SpeakerManager } from '../devices';
 import { getCurrentValue, setCurrentValue } from '../store/rxUtils';
@@ -71,10 +73,11 @@ export class DynascaleManager {
   private callState: CallState;
   private speaker: SpeakerManager;
   private tracer: Tracer;
-  private useWebAudio = isSafari();
+  private useWebAudio = false;
   private audioContext: AudioContext | undefined;
   private sfuClient: StreamSfuClient | undefined;
   private pendingSubscriptionsUpdate: NodeJS.Timeout | null = null;
+  readonly audioBindingsWatchdog: AudioBindingsWatchdog | undefined;
 
   private videoTrackSubscriptionOverridesSubject =
     new BehaviorSubject<VideoTrackSubscriptionOverrides>({});
@@ -120,6 +123,9 @@ export class DynascaleManager {
     this.callState = callState;
     this.speaker = speaker;
     this.tracer = tracer;
+    if (!isReactNative()) {
+      this.audioBindingsWatchdog = new AudioBindingsWatchdog(callState, tracer);
+    }
   }
 
   /**
@@ -129,7 +135,8 @@ export class DynascaleManager {
     if (this.pendingSubscriptionsUpdate) {
       clearTimeout(this.pendingSubscriptionsUpdate);
     }
-    const context = this.getOrCreateAudioContext();
+    this.audioBindingsWatchdog?.dispose();
+    const context = this.audioContext;
     if (context && context.state !== 'closed') {
       document.removeEventListener('click', this.resumeAudioContext);
       await context.close();
@@ -447,6 +454,7 @@ export class DynascaleManager {
         });
     resizeObserver?.observe(videoElement);
 
+    const isVideoTrack = trackType === 'videoTrack';
     // element renders and gets bound - track subscription gets
     // triggered first other ones get skipped on initial subscriptions
     const publishedTracksSubscription = boundParticipant.isLocalParticipant
@@ -454,9 +462,7 @@ export class DynascaleManager {
       : participant$
           .pipe(
             distinctUntilKeyChanged('publishedTracks'),
-            map((p) =>
-              trackType === 'videoTrack' ? hasVideo(p) : hasScreenShare(p),
-            ),
+            map((p) => (isVideoTrack ? hasVideo(p) : hasScreenShare(p))),
             distinctUntilChanged(),
           )
           .subscribe((isPublishing) => {
@@ -480,15 +486,11 @@ export class DynascaleManager {
     // https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide
     videoElement.muted = true;
 
+    const trackKey = isVideoTrack ? 'videoStream' : 'screenShareStream';
     const streamSubscription = participant$
-      .pipe(
-        distinctUntilKeyChanged(
-          trackType === 'videoTrack' ? 'videoStream' : 'screenShareStream',
-        ),
-      )
+      .pipe(distinctUntilKeyChanged(trackKey))
       .subscribe((p) => {
-        const source =
-          trackType === 'videoTrack' ? p.videoStream : p.screenShareStream;
+        const source = isVideoTrack ? p.videoStream : p.screenShareStream;
         if (videoElement.srcObject === source) return;
         videoElement.srcObject = source ?? null;
         if (isSafari() || isFirefox()) {
@@ -532,6 +534,8 @@ export class DynascaleManager {
     const participant = this.callState.findParticipantBySessionId(sessionId);
     if (!participant || participant.isLocalParticipant) return;
 
+    this.audioBindingsWatchdog?.register(audioElement, sessionId, trackType);
+
     const participant$ = this.callState.participants$.pipe(
       map((ps) => ps.find((p) => p.sessionId === sessionId)),
       takeWhile((p) => !!p),
@@ -561,19 +565,12 @@ export class DynascaleManager {
     let sourceNode: MediaStreamAudioSourceNode | undefined = undefined;
     let gainNode: GainNode | undefined = undefined;
 
+    const isAudioTrack = trackType === 'audioTrack';
+    const trackKey = isAudioTrack ? 'audioStream' : 'screenShareAudioStream';
     const updateMediaStreamSubscription = participant$
-      .pipe(
-        distinctUntilKeyChanged(
-          trackType === 'screenShareAudioTrack'
-            ? 'screenShareAudioStream'
-            : 'audioStream',
-        ),
-      )
+      .pipe(distinctUntilKeyChanged(trackKey))
       .subscribe((p) => {
-        const source =
-          trackType === 'screenShareAudioTrack'
-            ? p.screenShareAudioStream
-            : p.audioStream;
+        const source = isAudioTrack ? p.audioStream : p.screenShareAudioStream;
         if (audioElement.srcObject === source) return;
 
         setTimeout(() => {
@@ -630,6 +627,7 @@ export class DynascaleManager {
     audioElement.autoplay = true;
 
     return () => {
+      this.audioBindingsWatchdog?.unregister(sessionId, trackType);
       sinkIdSubscription?.unsubscribe();
       volumeSubscription.unsubscribe();
       updateMediaStreamSubscription.unsubscribe();

package/src/helpers/__tests__/AudioBindingsWatchdog.test.ts

@@ -0,0 +1,325 @@
+/**
+ * @vitest-environment happy-dom
+ */
+
+import '../../rtc/__tests__/mocks/webrtc.mocks';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { AudioBindingsWatchdog } from '../AudioBindingsWatchdog';
+import { Call } from '../../Call';
+import { StreamClient } from '../../coordinator/connection/client';
+import { CallingState, StreamVideoWriteableStateStore } from '../../store';
+import { noopComparator } from '../../sorting';
+import { fromPartial } from '@total-typescript/shoehorn';
+
+describe('AudioBindingsWatchdog', () => {
+  let watchdog: AudioBindingsWatchdog;
+  let call: Call;
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+    call = new Call({
+      id: 'id',
+      type: 'default',
+      streamClient: new StreamClient('api-key', {
+        devicePersistence: { enabled: false },
+      }),
+      clientStore: new StreamVideoWriteableStateStore(),
+    });
+    call.setSortParticipantsBy(noopComparator());
+    watchdog = new AudioBindingsWatchdog(call.state, call.tracer);
+  });
+
+  afterEach(() => {
+    watchdog.dispose();
+    call.leave();
+    vi.useRealTimers();
+  });
+
+  const addRemoteParticipant = (
+    sessionId: string,
+    userId: string,
+    streams?: {
+      audioStream?: MediaStream;
+      screenShareAudioStream?: MediaStream;
+    },
+  ) => {
+    call.state.updateOrAddParticipant(
+      sessionId,
+      fromPartial({
+        userId,
+        sessionId,
+        publishedTracks: [],
+        ...streams,
+      }),
+    );
+  };
+
+  it('should warn about dangling audio streams when active', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+    });
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Dangling audio bindings detected'),
+    );
+    expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('user-1'));
+  });
+
+  it('should not warn when all audio elements are bound', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+    });
+    watchdog.register(
+      document.createElement('audio'),
+      'session-1',
+      'audioTrack',
+    );
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('should skip local participant', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    // @ts-expect-error incomplete data
+    call.state.updateOrAddParticipant('local-session', {
+      userId: 'local-user',
+      sessionId: 'local-session',
+      isLocalParticipant: true,
+      publishedTracks: [],
+      audioStream: new MediaStream(),
+    });
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('should start on JOINED and stop on non-JOINED state', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+    });
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+    expect(warnSpy).toHaveBeenCalled();
+
+    warnSpy.mockClear();
+
+    call.state.setCallingState(CallingState.IDLE);
+    vi.advanceTimersByTime(6000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('should be disableable via setEnabled', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+    });
+
+    watchdog.setEnabled(false);
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(6000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('should re-enable after disabling', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+    });
+
+    watchdog.setEnabled(false);
+    watchdog.setEnabled(true);
+
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Dangling audio bindings detected'),
+    );
+  });
+
+  it('should warn when binding a different element to the same key', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    const audioElement1 = document.createElement('audio');
+    const audioElement2 = document.createElement('audio');
+
+    watchdog.register(audioElement1, 'session-1', 'audioTrack');
+    watchdog.register(audioElement2, 'session-1', 'audioTrack');
+
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Audio element already bound'),
+    );
+  });
+
+  it('should not warn when re-binding the same element', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    const audioElement = document.createElement('audio');
+
+    watchdog.register(audioElement, 'session-1', 'audioTrack');
+    watchdog.register(audioElement, 'session-1', 'audioTrack');
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('unregisterBinding should remove the binding', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+    });
+    watchdog.register(
+      document.createElement('audio'),
+      'session-1',
+      'audioTrack',
+    );
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+    expect(warnSpy).not.toHaveBeenCalled();
+
+    watchdog.unregister('session-1', 'audioTrack');
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Dangling audio bindings detected'),
+    );
+  });
+
+  it('should warn about dangling screenShareAudioStream', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      screenShareAudioStream: new MediaStream(),
+    });
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Dangling audio bindings detected'),
+    );
+    expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('user-1'));
+  });
+
+  it('should not warn when screenShareAudio element is bound', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      screenShareAudioStream: new MediaStream(),
+    });
+    watchdog.register(
+      document.createElement('audio'),
+      'session-1',
+      'screenShareAudioTrack',
+    );
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('should warn only about the unbound track type', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+      screenShareAudioStream: new MediaStream(),
+    });
+
+    // bind only the regular audio track
+    watchdog.register(
+      document.createElement('audio'),
+      'session-1',
+      'audioTrack',
+    );
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    // should still warn because screenShareAudio is unbound
+    expect(warnSpy).toHaveBeenCalledWith(
+      expect.stringContaining('Dangling audio bindings detected'),
+    );
+  });
+
+  it('should not warn when both audio and screenShareAudio are bound', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+      screenShareAudioStream: new MediaStream(),
+    });
+
+    watchdog.register(
+      document.createElement('audio'),
+      'session-1',
+      'audioTrack',
+    );
+    watchdog.register(
+      document.createElement('audio'),
+      'session-1',
+      'screenShareAudioTrack',
+    );
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
+  it('dispose should stop the watchdog', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    addRemoteParticipant('session-1', 'user-1', {
+      audioStream: new MediaStream(),
+    });
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+    expect(warnSpy).toHaveBeenCalled();
+
+    warnSpy.mockClear();
+
+    watchdog.dispose();
+    vi.advanceTimersByTime(6000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+});