@stream-io/video-client 1.49.0 → 1.50.0

package/src/devices/__tests__/DeviceManager.test.ts

@@ -395,6 +395,289 @@ describe('Device Manager', () => {
     vi.useRealTimers();
   });
 
+  describe('interruptedTracks (hardware mute/unmute events)', () => {
+    const localSessionId = 'local-session-id';
+
+    beforeEach(() => {
+      manager['call'].state.setParticipants([
+        {
+          sessionId: localSessionId,
+          userId: 'local-user',
+          isLocalParticipant: true,
+          publishedTracks: [],
+        } as any,
+      ]);
+    });
+
+    const fireOn = async (track: MockTrack, event: 'mute' | 'unmute') => {
+      const handler = track.eventHandlers[event] as Function;
+      await handler();
+    };
+
+    const currentTrack = () =>
+      manager.state.mediaStream?.getTracks()[0] as MockTrack;
+
+    const isInterrupted = () =>
+      !!manager['call'].state.localParticipant?.interruptedTracks?.includes(
+        TrackType.VIDEO,
+      );
+
+    it('adds the track type on a mute event without touching status', async () => {
+      await manager.enable();
+      expect(manager.state.status).toBe('enabled');
+      expect(isInterrupted()).toBe(false);
+
+      await fireOn(currentTrack(), 'mute');
+
+      expect(isInterrupted()).toBe(true);
+      expect(manager.state.status).toBe('enabled');
+      expect(manager.state.optimisticStatus).toBe('enabled');
+    });
+
+    it('removes the track type on the matching unmute event', async () => {
+      await manager.enable();
+      const track = currentTrack();
+      await fireOn(track, 'mute');
+      expect(isInterrupted()).toBe(true);
+
+      await fireOn(track, 'unmute');
+
+      expect(isInterrupted()).toBe(false);
+      expect(manager.state.status).toBe('enabled');
+    });
+
+    it('notifies the SFU for video track mute/unmute events', async () => {
+      await manager.enable();
+      const track = currentTrack();
+
+      await fireOn(track, 'mute');
+      expect(manager['call'].notifyTrackMuteState).toHaveBeenCalledWith(
+        true,
+        TrackType.VIDEO,
+      );
+
+      await fireOn(track, 'unmute');
+      expect(manager['call'].notifyTrackMuteState).toHaveBeenCalledWith(
+        false,
+        TrackType.VIDEO,
+      );
+    });
+
+    it('emits localParticipant$ transitions to subscribers', async () => {
+      const observed: boolean[] = [];
+      const subscription = manager['call'].state.localParticipant$.subscribe(
+        (p) => observed.push(!!p?.interruptedTracks?.includes(TrackType.VIDEO)),
+      );
+
+      await manager.enable();
+      const track = currentTrack();
+      await fireOn(track, 'mute');
+      await fireOn(track, 'unmute');
+
+      expect(observed).toContain(true);
+      expect(observed[observed.length - 1]).toBe(false);
+      subscription.unsubscribe();
+    });
+
+    it('reacquires a fresh stream when the device is replaced mid-interruption', async () => {
+      vi.useFakeTimers();
+      emitDeviceIds(mockVideoDevices);
+
+      await manager.enable();
+      const device = mockVideoDevices[0];
+      await manager.select(device.deviceId);
+      await fireOn(currentTrack(), 'mute');
+      expect(isInterrupted()).toBe(true);
+      expect(manager.state.status).toBe('enabled');
+
+      manager.getStream.mockClear();
+
+      emitDeviceIds([
+        { ...device, groupId: device.groupId + 'new' },
+        ...mockVideoDevices.slice(1),
+      ]);
+
+      await vi.runAllTimersAsync();
+
+      // Status stays 'enabled' so the replacement flows through
+      // applySettingsToStream, which forces a fresh getStream call.
+      expect(manager.getStream).toHaveBeenCalled();
+      expect(manager.state.status).toBe('enabled');
+      vi.useRealTimers();
+    });
+
+    it('leaves manager.enabled === true so capability cleanup can still disable it', async () => {
+      await manager.enable();
+      await fireOn(currentTrack(), 'mute');
+
+      // `enabled` continues to reflect requested-publishing intent. Code
+      // that revokes SEND_AUDIO / SEND_VIDEO at the Call layer iterates
+      // managers whose `enabled` is true; if system-muted hid that bit,
+      // the cleanup would skip a still-published track.
+      expect(manager.enabled).toBe(true);
+    });
+
+    it('clears interruptedTracks when the user toggles the device off and back on', async () => {
+      await manager.enable();
+      await fireOn(currentTrack(), 'mute');
+      expect(isInterrupted()).toBe(true);
+
+      await manager.disable();
+      // Stream is cleared on disable; the prior hardware-mute signal
+      // belonged to the now-gone track.
+      expect(isInterrupted()).toBe(false);
+
+      await manager.enable();
+      // Re-acquired stream is fresh; the stale flag must not carry over.
+      expect(isInterrupted()).toBe(false);
+    });
+
+    it('clears interruptedTracks when select() swaps to a different device', async () => {
+      await manager.enable();
+      await fireOn(currentTrack(), 'mute');
+      expect(isInterrupted()).toBe(true);
+
+      await manager.select(mockVideoDevices[1].deviceId);
+
+      expect(isInterrupted()).toBe(false);
+    });
+
+    it('removes mute/unmute listeners from the prior track when select() swaps the stream', async () => {
+      await manager.enable();
+      const oldTrack = currentTrack();
+      expect(oldTrack.eventHandlers['mute']).toBeDefined();
+      expect(oldTrack.eventHandlers['unmute']).toBeDefined();
+
+      await manager.select(mockVideoDevices[1].deviceId);
+
+      // Listeners on the prior track are torn down so a delayed
+      // mute/unmute event cannot clobber the fresh stream's state.
+      expect(oldTrack.eventHandlers['mute']).toBeUndefined();
+      expect(oldTrack.eventHandlers['unmute']).toBeUndefined();
+      expect(oldTrack.eventHandlers['ended']).toBeUndefined();
+    });
+
+    it('removes mute/unmute listeners when the stream is cleared on disable', async () => {
+      await manager.enable();
+      const oldTrack = currentTrack();
+      expect(oldTrack.eventHandlers['mute']).toBeDefined();
+
+      await manager.disable();
+
+      expect(oldTrack.eventHandlers['mute']).toBeUndefined();
+      expect(oldTrack.eventHandlers['unmute']).toBeUndefined();
+      expect(oldTrack.eventHandlers['ended']).toBeUndefined();
+    });
+
+    describe('WebKit refreshTrack on unmute (encoder stall workaround)', () => {
+      const SAFARI_UA =
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15';
+      const IOS_WKWEBVIEW_UA =
+        'Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148';
+      const CHROME_UA =
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+      const originalUserAgentDescriptor = Object.getOwnPropertyDescriptor(
+        window.navigator,
+        'userAgent',
+      );
+
+      const setUserAgent = (ua: string) => {
+        Object.defineProperty(window.navigator, 'userAgent', {
+          configurable: true,
+          get: () => ua,
+        });
+      };
+
+      afterEach(() => {
+        if (originalUserAgentDescriptor) {
+          Object.defineProperty(
+            window.navigator,
+            'userAgent',
+            originalUserAgentDescriptor,
+          );
+        }
+      });
+
+      it('calls refreshPublishedTrack on Safari unmute', async () => {
+        setUserAgent(SAFARI_UA);
+        await manager.enable();
+        const track = currentTrack();
+        await fireOn(track, 'mute');
+
+        await fireOn(track, 'unmute');
+
+        expect(manager['call'].refreshPublishedTrack).toHaveBeenCalledWith(
+          TrackType.VIDEO,
+        );
+      });
+
+      it('calls refreshPublishedTrack on a bare iOS WKWebView (no Safari token)', async () => {
+        setUserAgent(IOS_WKWEBVIEW_UA);
+        await manager.enable();
+        const track = currentTrack();
+        await fireOn(track, 'mute');
+
+        await fireOn(track, 'unmute');
+
+        expect(manager['call'].refreshPublishedTrack).toHaveBeenCalledWith(
+          TrackType.VIDEO,
+        );
+      });
+
+      it('does not call refreshPublishedTrack on Chrome unmute', async () => {
+        setUserAgent(CHROME_UA);
+        await manager.enable();
+        const track = currentTrack();
+        await fireOn(track, 'mute');
+
+        await fireOn(track, 'unmute');
+
+        expect(manager['call'].refreshPublishedTrack).not.toHaveBeenCalled();
+      });
+
+      it('does not call refreshPublishedTrack on the mute leg', async () => {
+        setUserAgent(SAFARI_UA);
+        await manager.enable();
+        const track = currentTrack();
+
+        await fireOn(track, 'mute');
+
+        expect(manager['call'].refreshPublishedTrack).not.toHaveBeenCalled();
+      });
+
+      it('skips refreshPublishedTrack while the page is hidden', async () => {
+        setUserAgent(SAFARI_UA);
+        const visibilityDescriptor = Object.getOwnPropertyDescriptor(
+          document,
+          'visibilityState',
+        );
+        Object.defineProperty(document, 'visibilityState', {
+          configurable: true,
+          get: () => 'hidden',
+        });
+
+        try {
+          await manager.enable();
+          const track = currentTrack();
+          await fireOn(track, 'mute');
+
+          await fireOn(track, 'unmute');
+
+          expect(manager['call'].refreshPublishedTrack).not.toHaveBeenCalled();
+        } finally {
+          if (visibilityDescriptor) {
+            Object.defineProperty(
+              document,
+              'visibilityState',
+              visibilityDescriptor,
+            );
+          }
+        }
+      });
+    });
+  });
+
   describe('persistPreference', () => {
     it('stores selected device and muted state', () => {
       const persistenceEnabledManager = new TestInputMediaDeviceManager(

package/src/devices/__tests__/mocks.ts

@@ -104,6 +104,8 @@ export const mockCall = (): Partial<Call> => {
     }),
     notifyNoiseCancellationStarting: vi.fn().mockResolvedValue(undefined),
     notifyNoiseCancellationStopped: vi.fn().mockResolvedValue(undefined),
+    notifyTrackMuteState: vi.fn().mockResolvedValue(undefined),
+    refreshPublishedTrack: vi.fn().mockResolvedValue(undefined),
     tracer: new Tracer('tests'),
   };
 };

package/src/gen/video/sfu/event/events.ts

@@ -10,6 +10,7 @@ import {
   ClientDetails,
   Codec,
   ConnectionQuality,
+  DegradationPreference,
   Error as Error$,
   GoAwayReason,
   ICETrickle as ICETrickle$,
@@ -836,6 +837,10 @@ export interface VideoSender {
    * @generated from protobuf field: int32 publish_option_id = 5;
    */
   publishOptionId: number;
+  /**
+   * @generated from protobuf field: stream.video.sfu.models.DegradationPreference degradation_preference = 6;
+   */
+  degradationPreference: DegradationPreference;
 }
 /**
  * sent to users when they need to change the quality of their video
@@ -1796,6 +1801,16 @@ class VideoSender$Type extends MessageType<VideoSender> {
         kind: 'scalar',
         T: 5 /*ScalarType.INT32*/,
       },
+      {
+        no: 6,
+        name: 'degradation_preference',
+        kind: 'enum',
+        T: () => [
+          'stream.video.sfu.models.DegradationPreference',
+          DegradationPreference,
+          'DEGRADATION_PREFERENCE_',
+        ],
+      },
     ]);
   }
 }

package/src/gen/video/sfu/models/models.ts

@@ -307,6 +307,12 @@ export interface PublishOption {
    * @generated from protobuf field: repeated stream.video.sfu.models.AudioBitrate audio_bitrate_profiles = 10;
    */
   audioBitrateProfiles: AudioBitrate[];
+  /**
+   * The degradation preference for video encoding.
+   *
+   * @generated from protobuf field: stream.video.sfu.models.DegradationPreference degradation_preference = 11;
+   */
+  degradationPreference: DegradationPreference;
 }
 /**
  * @generated from protobuf message stream.video.sfu.models.Codec
@@ -1172,6 +1178,34 @@ export enum ClientCapability {
    */
   SUBSCRIBER_VIDEO_PAUSE = 1,
 }
+/**
+ * DegradationPreference represents the RTCDegradationPreference from WebRTC.
+ * See https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpSender/setParameters#degradationpreference
+ *
+ * @generated from protobuf enum stream.video.sfu.models.DegradationPreference
+ */
+export enum DegradationPreference {
+  /**
+   * @generated from protobuf enum value: DEGRADATION_PREFERENCE_UNSPECIFIED = 0;
+   */
+  UNSPECIFIED = 0,
+  /**
+   * @generated from protobuf enum value: DEGRADATION_PREFERENCE_BALANCED = 1;
+   */
+  BALANCED = 1,
+  /**
+   * @generated from protobuf enum value: DEGRADATION_PREFERENCE_MAINTAIN_FRAMERATE = 2;
+   */
+  MAINTAIN_FRAMERATE = 2,
+  /**
+   * @generated from protobuf enum value: DEGRADATION_PREFERENCE_MAINTAIN_RESOLUTION = 3;
+   */
+  MAINTAIN_RESOLUTION = 3,
+  /**
+   * @generated from protobuf enum value: DEGRADATION_PREFERENCE_MAINTAIN_FRAMERATE_AND_RESOLUTION = 4;
+   */
+  MAINTAIN_FRAMERATE_AND_RESOLUTION = 4,
+}
 // @generated message type with reflection information, may provide speed optimized methods
 class CallState$Type extends MessageType<CallState> {
   constructor() {
@@ -1441,6 +1475,16 @@ class PublishOption$Type extends MessageType<PublishOption> {
         repeat: 2 /*RepeatType.UNPACKED*/,
         T: () => AudioBitrate,
       },
+      {
+        no: 11,
+        name: 'degradation_preference',
+        kind: 'enum',
+        T: () => [
+          'stream.video.sfu.models.DegradationPreference',
+          DegradationPreference,
+          'DEGRADATION_PREFERENCE_',
+        ],
+      },
     ]);
   }
 }

package/src/helpers/AudioBindingsWatchdog.ts

@@ -21,10 +21,12 @@ export class AudioBindingsWatchdog {
   private readonly unsubscribeCallingState: () => void;
   private logger = videoLoggerSystem.getLogger('AudioBindingsWatchdog');
 
-  constructor(
-    private state: CallState,
-    private tracer: Tracer,
-  ) {
+  private readonly state: CallState;
+  private readonly tracer: Tracer;
+
+  constructor(state: CallState, tracer: Tracer) {
+    this.tracer = tracer;
+    this.state = state;
     this.unsubscribeCallingState = createSubscription(
       state.callingState$,
       (callingState) => {
@@ -43,19 +45,19 @@ export class AudioBindingsWatchdog {
    * Warns if a different element is already bound to the same key.
    */
   register = (
-    audioElement: HTMLAudioElement,
+    element: HTMLAudioElement,
     sessionId: string,
     trackType: AudioTrackType,
   ) => {
     const key = toBindingKey(sessionId, trackType);
     const existing = this.bindings.get(key);
-    if (existing && existing !== audioElement) {
+    if (existing && existing !== element) {
       this.logger.warn(
         `Audio element already bound to ${sessionId} and ${trackType}`,
       );
       this.tracer.trace('audioBinding.alreadyBoundWarning', trackType);
     }
-    this.bindings.set(key, audioElement);
+    this.bindings.set(key, element);
   };
 
   /**
@@ -83,6 +85,7 @@ export class AudioBindingsWatchdog {
    */
   dispose = () => {
     this.stop();
+    this.bindings.clear();
     this.unsubscribeCallingState();
   };
 

package/src/helpers/BlockedAudioTracker.ts

@@ -0,0 +1,74 @@
+import { BehaviorSubject, distinctUntilChanged, map } from 'rxjs';
+import { videoLoggerSystem } from '../logger';
+import { Tracer } from '../stats';
+import { setCurrentValue, setCurrentValueAsync } from '../store/rxUtils';
+import { timeboxed } from '../coordinator/connection/utils';
+
+/**
+ * Tracks audio elements that the browser's autoplay policy has blocked.
+ */
+export class BlockedAudioTracker {
+  private logger = videoLoggerSystem.getLogger('BlockedAudioTracker');
+  private tracer: Tracer;
+
+  private blockedElementsSubject = new BehaviorSubject(
+    new Set<HTMLAudioElement>(),
+  );
+
+  /**
+   * Whether the browser's autoplay policy is blocking audio playback.
+   * Will be `true` when at least one audio element is currently blocked.
+   * Use {@link resumeAudio} within a user gesture to unblock.
+   */
+  autoplayBlocked$ = this.blockedElementsSubject.pipe(
+    map((elements) => elements.size > 0),
+    distinctUntilChanged(),
+  );
+
+  constructor(tracer: Tracer) {
+    this.tracer = tracer;
+  }
+
+  /**
+   * Registers an audio element as blocked by the browser's autoplay policy.
+   */
+  markBlocked = (audioElement: HTMLAudioElement, blocked: boolean) => {
+    setCurrentValue(this.blockedElementsSubject, (elements) => {
+      if (blocked) elements.add(audioElement);
+      else elements.delete(audioElement);
+      return elements;
+    });
+  };
+
+  /**
+   * Returns whether the given audio element is currently flagged as blocked
+   * by the browser's autoplay policy.
+   */
+  isBlocked = (audioElement: HTMLAudioElement): boolean => {
+    return this.blockedElementsSubject.getValue().has(audioElement);
+  };
+
+  /**
+   * Plays all audio elements blocked by the browser's autoplay policy.
+   * Must be called from within a user gesture (e.g., click handler).
+   */
+  resumeAudio = async () => {
+    this.tracer.trace('resumeAudio', null);
+    await setCurrentValueAsync(
+      this.blockedElementsSubject,
+      async (elements) => {
+        await Promise.all(
+          Array.from(elements, async (element) => {
+            try {
+              if (element.srcObject) await timeboxed([element.play()], 2000);
+              elements.delete(element);
+            } catch (err) {
+              this.logger.warn(`Can't resume audio for element`, element, err);
+            }
+          }),
+        );
+        return elements;
+      },
+    );
+  };
+}