@stream-io/video-client 1.55.2 → 1.57.0

package/src/helpers/__tests__/MediaPlaybackWatchdog.test.ts

@@ -28,20 +28,33 @@ const createMediaElement = (
   state: FakeMediaState = {},
 ) => {
   const el = document.createElement(kind) as HTMLMediaElement;
-  Object.defineProperty(el, 'srcObject', { writable: true });
+  Object.defineProperty(el, 'srcObject', {
+    value: 'srcObject' in state ? state.srcObject : new MediaStream(),
+    writable: true,
+  });
   Object.defineProperty(el, 'paused', { writable: true, configurable: true });
   Object.defineProperty(el, 'readyState', {
+    value: state.readyState ?? 4,
+    writable: true,
+    configurable: true,
+  });
+  Object.defineProperty(el, 'ended', {
+    value: state.ended ?? false,
     writable: true,
     configurable: true,
   });
-  Object.defineProperty(el, 'ended', { writable: true, configurable: true });
-  el.srcObject = 'srcObject' in state ? state.srcObject : new MediaStream();
-  el.paused = state.paused ?? true;
-  el.readyState = state.readyState ?? 4;
-  el.ended = state.ended ?? false;
+  setPaused(el, state.paused ?? true);
   return el;
 };
 
+const setPaused = (el: HTMLMediaElement, paused: boolean) => {
+  Object.defineProperty(el, 'paused', {
+    value: paused,
+    writable: true,
+    configurable: true,
+  });
+};
+
 type SetupOpts = {
   kind?: 'audio' | 'video';
   state?: FakeMediaState;
@@ -111,15 +124,131 @@ describe('MediaPlaybackWatchdog', () => {
     await vi.advanceTimersByTimeAsync(6000);
     expect(play).toHaveBeenCalledTimes(2);
 
-    // Simulate the element actually starting to play.
+    // Simulate the element actually starting to play, and staying up long
+    // enough for the recovery to count as settled.
+    setPaused(el, false);
     el.dispatchEvent(new Event('playing'));
+    await vi.advanceTimersByTimeAsync(1000);
 
-    // A subsequent pause should still trigger a fresh recovery attempt.
+    // A subsequent pause should trigger a fresh, immediate recovery attempt.
+    setPaused(el, true);
     el.dispatchEvent(new Event('pause'));
     await vi.advanceTimersByTimeAsync(0);
     expect(play).toHaveBeenCalledTimes(3);
   });
 
+  it('backs off instead of retrying immediately when playback has not settled', async () => {
+    el.dispatchEvent(new Event('pause'));
+    await vi.advanceTimersByTimeAsync(0);
+    expect(play).toHaveBeenCalledTimes(1);
+
+    // resumed, but re-paused before the settle delay elapsed
+    setPaused(el, false);
+    el.dispatchEvent(new Event('playing'));
+    expect(tracer.trace).not.toHaveBeenCalledWith(
+      'mediaPlayback.recover.success',
+      {
+        kind: 'audio',
+        attempts: 1,
+      },
+    );
+    await vi.advanceTimersByTimeAsync(500);
+    setPaused(el, true);
+    el.dispatchEvent(new Event('pause'));
+
+    // the next attempt is subject to backoff, not scheduled on the next tick
+    await vi.advanceTimersByTimeAsync(0);
+    expect(play).toHaveBeenCalledTimes(1);
+
+    await vi.advanceTimersByTimeAsync(6000);
+    expect(play).toHaveBeenCalledTimes(2);
+  });
+
+  it('stops fighting an element that re-pauses after every successful play', async () => {
+    setup({ kind: 'video' });
+    // Safari re-pauses the element right after play() resolves. This used to
+    // reset the attempt counter, so the loop spun with a zero delay forever.
+    play.mockImplementation(async () => {
+      setPaused(el, false);
+      el.dispatchEvent(new Event('playing'));
+      setPaused(el, true);
+      el.dispatchEvent(new Event('pause'));
+    });
+
+    el.dispatchEvent(new Event('pause'));
+    for (let i = 0; i < 20; i++) {
+      await vi.advanceTimersByTimeAsync(6000);
+    }
+
+    expect(play).toHaveBeenCalledTimes(10);
+    expect(tracer.trace).toHaveBeenCalledWith('mediaPlayback.recover.giveUp', {
+      kind: 'video',
+      attempts: 10,
+    });
+
+    // External playback can still emit `playing` after the watchdog gave up
+    // (for example from a user gesture or another playback helper). If it
+    // re-pauses before settling, it must not revive the watchdog or trace a
+    // false recovery success.
+    setPaused(el, false);
+    el.dispatchEvent(new Event('playing'));
+    setPaused(el, true);
+    el.dispatchEvent(new Event('pause'));
+    await vi.advanceTimersByTimeAsync(1000);
+
+    expect(tracer.trace).not.toHaveBeenCalledWith(
+      'mediaPlayback.recover.success',
+      {
+        kind: 'video',
+        attempts: 10,
+      },
+    );
+
+    // and it stays stopped, without tracing the pause storm
+    const traceMock = tracer.trace as unknown as MockInstance;
+    const tracesBefore = traceMock.mock.calls.length;
+    for (let i = 0; i < 100; i++) {
+      el.dispatchEvent(new Event('pause'));
+    }
+    await vi.advanceTimersByTimeAsync(6000);
+
+    expect(play).toHaveBeenCalledTimes(10);
+    expect(traceMock.mock.calls.length).toBe(tracesBefore);
+  });
+
+  it('revives once playback genuinely holds for the settle window', async () => {
+    setup({ kind: 'video' });
+    play.mockImplementation(async () => {
+      setPaused(el, false);
+      el.dispatchEvent(new Event('playing'));
+      setPaused(el, true);
+      el.dispatchEvent(new Event('pause'));
+    });
+
+    el.dispatchEvent(new Event('pause'));
+    for (let i = 0; i < 20; i++) {
+      await vi.advanceTimersByTimeAsync(6000);
+    }
+    expect(play).toHaveBeenCalledTimes(10); // gave up
+
+    play.mockResolvedValue(undefined);
+    setPaused(el, false);
+    el.dispatchEvent(new Event('playing'));
+    await vi.advanceTimersByTimeAsync(1500);
+
+    expect(tracer.trace).toHaveBeenCalledWith('mediaPlayback.recover.success', {
+      kind: 'video',
+      attempts: 10,
+    });
+
+    // recovery is live again: the attempt counter was reset, so a fresh pause
+    // gets an immediate attempt rather than being ignored
+    setPaused(el, true);
+    el.dispatchEvent(new Event('pause'));
+    await vi.advanceTimersByTimeAsync(0);
+    expect(play).toHaveBeenCalledTimes(11);
+  });
+
   it('does not attempt recovery when srcObject is null', async () => {
     setup({ state: { srcObject: null } });
 

package/src/helpers/__tests__/ViewportTracker.test.ts

@@ -38,8 +38,7 @@ describe('ViewportTracker', () => {
 
   it('updates participant viewportVisibilityState as visibility changes', () => {
     let visibilityHandler:
-      | ((entry: IntersectionObserverEntry) => void)
-      | undefined;
+      ((entry: IntersectionObserverEntry) => void) | undefined;
     vi.spyOn(viewportTracker, 'observe').mockImplementation((_el, handler) => {
       visibilityHandler = handler;
       return vi.fn();

package/src/reporting/ClientEventReporter.ts

@@ -33,16 +33,12 @@ export type ClientEventStage =
   | 'FirstAudioFrame';
 
 export type MediaPermissionState =
-  | 'INITIATED'
-  | 'FAILED'
-  | 'GRANTED'
-  | 'NOT_INITIATED';
+  'INITIATED' | 'FAILED' | 'GRANTED' | 'NOT_INITIATED';
+
+export type ReportedIceState = 'CONNECTED' | 'FAILED' | 'NOT_CONNECTED';
 
 export type JoinReason =
-  | 'first-attempt'
-  | 'network-available'
-  | 'migration'
-  | 'full-rejoin';
+  'first-attempt' | 'network-available' | 'migration' | 'full-rejoin';
 
 export type ClientEventStandardCode =
   | 'CLIENT_ABORTED'
@@ -86,6 +82,7 @@ type PeerConnectionPairState = StagePairState & {
   sfuId: string;
   userSessionId: string;
   wasPreviouslyConnected: boolean;
+  lastIceState?: RTCIceConnectionState;
 };
 
 const pcKey = (cid: string, role: ClientEventPeerConnection): string =>
@@ -356,20 +353,8 @@ export class ClientEventReporter {
       this.failCoordinator(cid);
       this.failWs(cid);
 
-      this.emitPeerConnectionFailure(
-        cid,
-        'publish',
-        code,
-        reason,
-        'NOT_CONNECTED',
-      );
-      this.emitPeerConnectionFailure(
-        cid,
-        'subscribe',
-        code,
-        reason,
-        'NOT_CONNECTED',
-      );
+      this.emitPeerConnectionFailure(cid, 'publish', code, reason);
+      this.emitPeerConnectionFailure(cid, 'subscribe', code, reason);
     } catch (err) {
       this.logger.warn('Failed to report abort', err);
     }
@@ -384,7 +369,6 @@ export class ClientEventReporter {
         role,
         'CLIENT_ABORTED',
         'superseded by a new join attempt',
-        'NOT_CONNECTED',
       );
     }
   };
@@ -573,34 +557,35 @@ export class ClientEventReporter {
     const role: ClientEventPeerConnection =
       event.peerType === PeerType.SUBSCRIBER ? 'subscribe' : 'publish';
 
-    if (event.stateType === 'ice' && event.state === 'failed') {
-      this.emitPeerConnectionFailure(
-        cid,
-        role,
-        'ICE_CONNECTIVITY_FAILED',
-        'ICE connectivity checks failed',
-        'FAILED',
-      );
+    const pair = this.peerConnectionPairs.get(pcKey(cid, role));
+    if (pair) pair.lastIceState = event.iceConnectionState;
+
+    if (event.stateType === 'ice') {
+      if (event.state === 'failed') {
+        this.emitPeerConnectionFailure(
+          cid,
+          role,
+          'ICE_CONNECTIVITY_FAILED',
+          'ICE connectivity checks failed',
+        );
+      }
       return;
     }
 
-    if (event.stateType === 'peerConnection' && event.state === 'failed') {
+    if (event.state === 'failed') {
       this.emitPeerConnectionFailure(
         cid,
         role,
         'DTLS_CONNECTIVITY_FAILED',
         'DTLS connectivity checks failed',
-        'CONNECTED',
       );
       return;
     }
 
-    if (event.stateType !== 'peerConnection') return;
-
     switch (event.state) {
       case 'connecting':
-        if (this.peerConnectionPairs.has(pcKey(cid, role))) return;
-        this.openPeerConnectionPair(cid, role);
+        if (pair) return;
+        this.openPeerConnectionPair(cid, role, event.iceConnectionState);
         break;
       case 'connected':
         this.emitPeerConnectionSuccess(cid, role);
@@ -614,6 +599,7 @@ export class ClientEventReporter {
   private openPeerConnectionPair = (
     cid: string,
     role: ClientEventPeerConnection,
+    iceConnectionState: RTCIceConnectionState,
   ) => {
     const key = pcKey(cid, role);
     const pair: PeerConnectionPairState = {
@@ -624,6 +610,7 @@ export class ClientEventReporter {
       sfuId: this.getSfuId(cid),
       userSessionId: this.getUserSessionId(cid),
       wasPreviouslyConnected: this.pcEverConnected.get(key) === true,
+      lastIceState: iceConnectionState,
     };
     this.peerConnectionPairs.set(key, pair);
 
@@ -670,7 +657,6 @@ export class ClientEventReporter {
     role: ClientEventPeerConnection,
     code: ClientEventStandardCode,
     reason: string,
-    iceState: 'CONNECTED' | 'FAILED' | 'NOT_CONNECTED',
   ) => {
     const key = pcKey(cid, role);
     const pair = this.peerConnectionPairs.get(key);
@@ -689,7 +675,7 @@ export class ClientEventReporter {
       outcome: 'failure',
       retry_count_attempt: 0,
       elapsed_time: Date.now() - pair.startedAt,
-      ice_state: iceState,
+      ice_state: toReportedIceState(pair.lastIceState),
       retry_failure_reason: reason,
       retry_failure_code: code,
     });
@@ -782,6 +768,20 @@ export class ClientEventReporter {
   };
 }
 
+const toReportedIceState = (
+  state: RTCIceConnectionState | undefined,
+): ReportedIceState => {
+  switch (state) {
+    case 'connected':
+    case 'completed':
+      return 'CONNECTED';
+    case 'failed':
+      return 'FAILED';
+    default:
+      return 'NOT_CONNECTED';
+  }
+};
+
 const readPermissionStatus = (
   permission: BrowserPermission,
 ): MediaPermissionState => {

package/src/reporting/__tests__/ClientEventReporter.test.ts

@@ -7,6 +7,7 @@ import { ErrorFromResponse } from '../../coordinator/connection/types';
 import { SfuTimeoutError } from '../../errors';
 import type { AxiosResponse } from 'axios';
 import { PeerType, TrackType } from '../../gen/video/sfu/models/models';
+import type { PeerConnectionStateChangeEvent } from '../../rtc';
 
 vi.mock('../../devices', () => ({
   getAudioBrowserPermission: () => ({ asStateObservable: () => of('granted') }),
@@ -15,6 +16,27 @@ vi.mock('../../devices', () => ({
 
 const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
 
+const peerConnectionEvent = (
+  peerType: PeerType,
+  state: RTCPeerConnectionState,
+  iceConnectionState: RTCIceConnectionState,
+): PeerConnectionStateChangeEvent => ({
+  peerType,
+  stateType: 'peerConnection',
+  state,
+  iceConnectionState,
+});
+
+const iceEvent = (
+  peerType: PeerType,
+  state: RTCIceConnectionState,
+): PeerConnectionStateChangeEvent => ({
+  peerType,
+  stateType: 'ice',
+  state,
+  iceConnectionState: state,
+});
+
 describe('ClientEventReporter', () => {
   const cid = 'default:call-1';
   let doAxiosRequest: ReturnType<typeof vi.fn>;
@@ -24,6 +46,21 @@ describe('ClientEventReporter', () => {
   const postedEvents = (): Array<Record<string, any>> =>
     doAxiosRequest.mock.calls.map((call) => call[2].events[0]);
 
+  const reportPeerConnectionState = (
+    peerType: PeerType,
+    state: RTCPeerConnectionState,
+    iceConnectionState: RTCIceConnectionState,
+  ) => {
+    reporter.onPeerConnectionStateChange(
+      cid,
+      peerConnectionEvent(peerType, state, iceConnectionState),
+    );
+  };
+
+  const reportIceState = (peerType: PeerType, state: RTCIceConnectionState) => {
+    reporter.onPeerConnectionStateChange(cid, iceEvent(peerType, state));
+  };
+
   beforeEach(() => {
     doAxiosRequest = vi.fn().mockResolvedValue({});
     const streamClient = fromPartial<StreamClient>({
@@ -465,16 +502,16 @@ describe('ClientEventReporter', () => {
 
   it('tracks a publisher peer connection connect', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connecting',
-    });
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connected',
-    });
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connected',
+      'connected',
+    );
     await flush();
 
     const events = postedEvents().filter(
@@ -497,16 +534,8 @@ describe('ClientEventReporter', () => {
 
   it('reports an ICE failure on the peer connection', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.SUBSCRIBER,
-      stateType: 'peerConnection',
-      state: 'connecting',
-    });
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.SUBSCRIBER,
-      stateType: 'ice',
-      state: 'failed',
-    });
+    reportPeerConnectionState(PeerType.SUBSCRIBER, 'connecting', 'checking');
+    reportIceState(PeerType.SUBSCRIBER, 'failed');
     await flush();
 
     const completed = postedEvents().filter(
@@ -522,25 +551,111 @@ describe('ClientEventReporter', () => {
     });
   });
 
-  it('opens a fresh peer connection pair after a new correlation', async () => {
+  it('reports a DTLS failure with the real ICE state (connected)', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connecting',
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    reportIceState(PeerType.PUBLISHER_UNSPECIFIED, 'connected');
+    // Aggregate connection fails while ICE stays connected -> genuine DTLS.
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'failed',
+      'connected',
+    );
+    await flush();
+
+    const completed = postedEvents().filter(
+      (e) =>
+        e.stage === 'PeerConnectionConnect' && e.event_type === 'completed',
+    );
+    expect(completed).toHaveLength(1);
+    expect(completed[0]).toMatchObject({
+      outcome: 'failure',
+      peer_connection: 'publish',
+      retry_failure_code: 'DTLS_CONNECTIVITY_FAILED',
+      ice_state: 'CONNECTED',
     });
+  });
 
-    reporter.startCorrelation(cid, 'full-rejoin');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connecting',
+  it('reports a DTLS failure with the real ICE state (never connected)', async () => {
+    reporter.startCorrelation(cid, 'first-attempt');
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    // ICE is still `checking` when the aggregate peer-connection state fails:
+    // the reported ice_state must be NOT_CONNECTED, not an assumed CONNECTED.
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'failed',
+      'checking',
+    );
+    await flush();
+
+    const completed = postedEvents().filter(
+      (e) =>
+        e.stage === 'PeerConnectionConnect' && e.event_type === 'completed',
+    );
+    expect(completed).toHaveLength(1);
+    expect(completed[0]).toMatchObject({
+      outcome: 'failure',
+      peer_connection: 'publish',
+      retry_failure_code: 'DTLS_CONNECTIVITY_FAILED',
+      ice_state: 'NOT_CONNECTED',
     });
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connected',
+  });
+
+  it('reports the ICE state captured when the pair opened, with no ICE event', async () => {
+    reporter.startCorrelation(cid, 'first-attempt');
+    // The pair opens with ICE already connected (carried on the connecting
+    // event); no separate ICE event is ever delivered.
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'connected',
+    );
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'failed',
+      'connected',
+    );
+    await flush();
+
+    const completed = postedEvents().filter(
+      (e) =>
+        e.stage === 'PeerConnectionConnect' && e.event_type === 'completed',
+    );
+    expect(completed).toHaveLength(1);
+    expect(completed[0]).toMatchObject({
+      outcome: 'failure',
+      retry_failure_code: 'DTLS_CONNECTIVITY_FAILED',
+      ice_state: 'CONNECTED',
     });
+  });
+
+  it('opens a fresh peer connection pair after a new correlation', async () => {
+    reporter.startCorrelation(cid, 'first-attempt');
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+
+    reporter.startCorrelation(cid, 'full-rejoin');
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connected',
+      'connected',
+    );
     await flush();
 
     const pc = postedEvents().filter(
@@ -566,16 +681,16 @@ describe('ClientEventReporter', () => {
   it('marks a peer connection as previously connected on reconnect', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
     const connect = () => {
-      reporter.onPeerConnectionStateChange(cid, {
-        peerType: PeerType.PUBLISHER_UNSPECIFIED,
-        stateType: 'peerConnection',
-        state: 'connecting',
-      });
-      reporter.onPeerConnectionStateChange(cid, {
-        peerType: PeerType.PUBLISHER_UNSPECIFIED,
-        stateType: 'peerConnection',
-        state: 'connected',
-      });
+      reportPeerConnectionState(
+        PeerType.PUBLISHER_UNSPECIFIED,
+        'connecting',
+        'checking',
+      );
+      reportPeerConnectionState(
+        PeerType.PUBLISHER_UNSPECIFIED,
+        'connected',
+        'connected',
+      );
     };
     connect();
     connect();

package/src/rtc/BasePeerConnection.ts

@@ -362,6 +362,7 @@ export abstract class BasePeerConnection {
     try {
       this.onPeerConnectionStateChange?.({
         peerType: this.peerType,
+        iceConnectionState: this.pc.iceConnectionState,
         ...event,
       });
     } catch (err) {