@stream-io/video-client 1.58.0 → 2.0.0-beta.0

package/src/reporting/ClientEventReporter.ts

@@ -19,6 +19,11 @@ import type {
   BrowserPermissionState,
 } from '../devices/BrowserPermission';
 import { getCurrentValue } from '../store/rxUtils';
+import type {
+  ClientEvent,
+  ReportClientEventRequest,
+  ReportClientEventResponse,
+} from '../gen/coordinator';
 
 export type ClientEventPeerConnection = 'publish' | 'subscribe';
 
@@ -55,7 +60,6 @@ export type CallReportContext = {
   callId: string;
   getSfuId: () => string;
   getCallSessionId: () => string;
-  getUserSessionId: () => string;
 };
 
 export type ClientEventReporterOptions = {
@@ -80,7 +84,6 @@ type StagePairState = {
 
 type PeerConnectionPairState = StagePairState & {
   sfuId: string;
-  userSessionId: string;
   wasPreviouslyConnected: boolean;
   lastIceState?: RTCIceConnectionState;
 };
@@ -192,9 +195,7 @@ export class ClientEventReporter {
     this.coordinatorWsPair = undefined;
   };
 
-  private buildCoordinatorWsCommon = (
-    pair: StagePairState,
-  ): Record<string, unknown> => ({
+  private buildCoordinatorWsCommon = (pair: StagePairState): ClientEvent => ({
     user_id: pair.userIdSnapshot ?? this.streamClient.userID,
     stage: 'CoordinatorWS',
     stage_id: pair.sid,
@@ -383,7 +384,6 @@ export class ClientEventReporter {
       user_id: this.streamClient.userID || this.coordinatorConnectUserId,
       type: ctx?.callType,
       id: ctx?.callId,
-      call_cid: cid,
       stage: 'JoinInitiated',
       join_attempt_id: joinAttemptId,
       ...(coordinatorConnectId && {
@@ -608,7 +608,6 @@ export class ClientEventReporter {
       startedAt: Date.now(),
       joinAttemptIdSnapshot: this.joinAttemptIds.get(cid),
       sfuId: this.getSfuId(cid),
-      userSessionId: this.getUserSessionId(cid),
       wasPreviouslyConnected: this.pcEverConnected.get(key) === true,
       lastIceState: iceConnectionState,
     };
@@ -620,9 +619,6 @@ export class ClientEventReporter {
       peer_connection: role,
       was_previously_connected: pair.wasPreviouslyConnected,
       ...(pair.sfuId && { sfu_id: pair.sfuId }),
-      ...(pair.userSessionId && {
-        user_session_id: pair.userSessionId,
-      }),
       event_type: 'initiated',
     });
   };
@@ -641,9 +637,6 @@ export class ClientEventReporter {
       peer_connection: role,
       was_previously_connected: pair.wasPreviouslyConnected,
       ...(pair.sfuId && { sfu_id: pair.sfuId }),
-      ...(pair.userSessionId && {
-        user_session_id: pair.userSessionId,
-      }),
       event_type: 'completed',
       outcome: 'success',
       retry_count_attempt: 0,
@@ -667,9 +660,6 @@ export class ClientEventReporter {
       ...this.sessionIdField(cid),
       peer_connection: role,
       was_previously_connected: pair.wasPreviouslyConnected,
-      ...(pair.userSessionId && {
-        user_session_id: pair.userSessionId,
-      }),
       ...(pair.sfuId && { sfu_id: pair.sfuId }),
       event_type: 'completed',
       outcome: 'failure',
@@ -685,10 +675,7 @@ export class ClientEventReporter {
   private getSfuId = (cid: string): string =>
     this.callContexts.get(cid)?.getSfuId() ?? '';
 
-  private getUserSessionId = (cid: string): string =>
-    this.callContexts.get(cid)?.getUserSessionId() ?? '';
-
-  private sessionIdField = (cid: string): Record<string, unknown> => {
+  private sessionIdField = (cid: string): { call_session_id?: string } => {
     const callSessionId = this.callContexts.get(cid)?.getCallSessionId() ?? '';
     return callSessionId ? { call_session_id: callSessionId } : {};
   };
@@ -697,14 +684,13 @@ export class ClientEventReporter {
     cid: string,
     stage: ClientEventStage,
     pair: StagePairState,
-  ): Record<string, unknown> => {
+  ): ClientEvent => {
     const ctx = this.callContexts.get(cid);
     const coordinatorConnectId = this.coordinatorConnectId;
     return {
       user_id: this.streamClient.userID || this.coordinatorConnectUserId,
       type: ctx?.callType ?? '',
       id: ctx?.callId ?? '',
-      call_cid: cid,
       stage,
       stage_id: pair.sid,
       ...(pair.joinAttemptIdSnapshot && {
@@ -719,22 +705,23 @@ export class ClientEventReporter {
     };
   };
 
-  private send = (body: Record<string, unknown>) => {
+  private send = (body: ClientEvent) => {
     if (!this.enabled) return;
     void this.sendWithRetry(body);
   };
 
-  private sendForCall = (cid: string, body: Record<string, unknown>) => {
+  private sendForCall = (cid: string, body: ClientEvent) => {
     if (!this.callContexts.has(cid)) return;
     this.send(body);
   };
 
-  private sendWithRetry = async (
-    body: Record<string, unknown>,
-  ): Promise<boolean> => {
+  private sendWithRetry = async (body: ClientEvent): Promise<boolean> => {
     for (let attempt = 0; attempt < 5; attempt++) {
       try {
-        await this.streamClient.doAxiosRequest(
+        await this.streamClient.doAxiosRequest<
+          ReportClientEventResponse,
+          ReportClientEventRequest
+        >(
           'post',
           '/call_client_event',
           { events: [body] },

package/src/reporting/__tests__/ClientEventReporter.test.ts

@@ -77,7 +77,6 @@ describe('ClientEventReporter', () => {
       callId: 'call-1',
       getCallSessionId: () => 'session-1',
       getSfuId: () => 'sfu-1',
-      getUserSessionId: () => 'user-session-1',
     };
     reporter.registerCall(cid, ctx);
   });
@@ -91,7 +90,7 @@ describe('ClientEventReporter', () => {
     expect(events).toHaveLength(2);
     expect(events[0]).toMatchObject({
       event_type: 'initiated',
-      call_cid: cid,
+      id: 'call-1',
       user_id: 'user-1',
       coordinator_connect_id: connectId,
       join_reason: 'first-attempt',
@@ -523,7 +522,6 @@ describe('ClientEventReporter', () => {
       peer_connection: 'publish',
       was_previously_connected: false,
       sfu_id: 'sfu-1',
-      user_session_id: 'user-session-1',
     });
     expect(events[1]).toMatchObject({
       event_type: 'completed',
@@ -712,7 +710,6 @@ describe('ClientEventReporter', () => {
       callId: 'call-2',
       getCallSessionId: () => 'session-2',
       getSfuId: () => 'sfu-2',
-      getUserSessionId: () => 'user-session-2',
     });
 
     reporter.startCorrelation(cid, 'first-attempt');
@@ -722,10 +719,10 @@ describe('ClientEventReporter', () => {
     await flush();
 
     const ws1 = postedEvents().filter(
-      (e) => e.stage === 'WSJoin' && e.call_cid === cid,
+      (e) => e.stage === 'WSJoin' && e.id === 'call-1',
     );
     const ws2 = postedEvents().filter(
-      (e) => e.stage === 'WSJoin' && e.call_cid === cid2,
+      (e) => e.stage === 'WSJoin' && e.id === 'call-2',
     );
     expect(ws1).toHaveLength(2);
     expect(ws2).toHaveLength(2);
@@ -806,7 +803,6 @@ describe('ClientEventReporter (disabled)', () => {
       callId: 'call-1',
       getCallSessionId: () => 'session-1',
       getSfuId: () => 'sfu-1',
-      getUserSessionId: () => 'user-session-1',
     });
   });
 

package/src/rtc/Subscriber.ts

@@ -94,6 +94,13 @@ export class Subscriber extends BasePeerConnection {
     }
   };
 
+  /**
+   * Whether the given stream is one the SFU echoed back to us
+   * (`allowOwnTracksLoopback`) rather than a local capture stream.
+   */
+  isSelfSubscribedStream = (stream: MediaStream | undefined): boolean =>
+    !!stream && !!this.trackedStreams?.has(stream);
+
   private handleOnTrack = (e: RTCTrackEvent) => {
     const { streams, track } = e;
     const [primaryStream] = streams;
@@ -212,6 +219,7 @@ export class Subscriber extends BasePeerConnection {
       (p) => p.trackLookupPrefix === trackId,
     );
     if (!target) return;
+    if (target.isLocalParticipant) return;
     this.state.updateParticipant(target.sessionId, (p) => {
       const current = p.interruptedTracks ?? [];
       const has = current.includes(trackType);

package/src/rtc/__tests__/Subscriber.test.ts

@@ -447,6 +447,65 @@ describe('Subscriber', () => {
   });
 
   describe('OnTrack', () => {
+    const audioStreamNamed = (trackLookupPrefix: string) => {
+      const stream = new MediaStream();
+      // @ts-expect-error - mock
+      stream.id = `${trackLookupPrefix}:TRACK_TYPE_AUDIO`;
+      return stream;
+    };
+
+    const audioTrack = () => {
+      const track = new MediaStreamTrack();
+      // @ts-expect-error - mock
+      track.kind = 'audio';
+      return track;
+    };
+
+    it('replaces the local capture stream with the echoed one, leaving the capture tracks live', () => {
+      const captureTrack = audioTrack();
+      const captureStream = new MediaStream();
+      captureStream.getTracks = () => [captureTrack];
+      const echoedStream = audioStreamNamed('self-lookup');
+      // @ts-expect-error - incomplete mock
+      state.updateOrAddParticipant('self-session', {
+        sessionId: 'self-session',
+        trackLookupPrefix: 'self-lookup',
+        isLocalParticipant: true,
+        audioStream: captureStream,
+      });
+
+      const onTrack = subscriber['handleOnTrack'];
+      // @ts-expect-error - incomplete mock
+      onTrack({ streams: [echoedStream], track: audioTrack() });
+
+      const participant = state.participants.find(
+        (p) => p.sessionId === 'self-session',
+      );
+      expect(participant?.audioStream).toBe(echoedStream);
+      // the device manager owns the capture stream and is still publishing it
+      expect(captureTrack.stop).not.toHaveBeenCalled();
+      expect(captureStream.removeTrack).not.toHaveBeenCalled();
+    });
+
+    it('reports streams it received as self-subscribed, and capture streams as not', () => {
+      const echoedStream = audioStreamNamed('self-lookup');
+      const captureStream = new MediaStream();
+      // @ts-expect-error - incomplete mock
+      state.updateOrAddParticipant('self-session', {
+        sessionId: 'self-session',
+        trackLookupPrefix: 'self-lookup',
+        isLocalParticipant: true,
+      });
+
+      const onTrack = subscriber['handleOnTrack'];
+      // @ts-expect-error - incomplete mock
+      onTrack({ streams: [echoedStream], track: audioTrack() });
+
+      expect(subscriber.isSelfSubscribedStream(echoedStream)).toBe(true);
+      expect(subscriber.isSelfSubscribedStream(captureStream)).toBe(false);
+      expect(subscriber.isSelfSubscribedStream(undefined)).toBe(false);
+    });
+
     it('should add unknown tracks to the to the call state', () => {
       const mediaStream = new MediaStream();
       const mediaStreamTrack = new MediaStreamTrack();
@@ -683,6 +742,48 @@ describe('Subscriber', () => {
       expect(interruptedFor('session-id')).toEqual([]);
     });
 
+    it('leaves the local participant alone on a self-sub track', () => {
+      const mediaStream = new MediaStream();
+      const track = new MediaStreamTrack();
+      // @ts-expect-error - mock
+      mediaStream.id = 'self-lookup:TRACK_TYPE_AUDIO';
+      // @ts-expect-error - mock
+      track.kind = 'audio';
+      Object.defineProperty(track, 'muted', {
+        configurable: true,
+        get: () => true,
+      });
+      // @ts-expect-error - incomplete mock
+      state.updateOrAddParticipant('self-session', {
+        sessionId: 'self-session',
+        trackLookupPrefix: 'self-lookup',
+        isLocalParticipant: true,
+        interruptedTracks: [],
+      });
+
+      const onTrack = subscriber['handleOnTrack'];
+      // @ts-expect-error - incomplete mock
+      onTrack({ streams: [mediaStream], track });
+
+      expect(interruptedFor('self-session')).toEqual([]);
+
+      const handlers: Record<string, () => void> = {};
+      for (const [event, handler] of (
+        track.addEventListener as ReturnType<typeof vi.fn>
+      ).mock.calls) {
+        handlers[event] = handler as () => void;
+      }
+
+      handlers['mute']();
+      expect(interruptedFor('self-session')).toEqual([]);
+
+      state.updateParticipant('self-session', {
+        interruptedTracks: [TrackType.AUDIO],
+      });
+      handlers['unmute']();
+      expect(interruptedFor('self-session')).toEqual([TrackType.AUDIO]);
+    });
+
     it('ignores non-audio remote tracks to avoid Dynascale false positives', () => {
       // Remote video track.muted is dominated by viewport-driven
       // SFU unsubscriptions, so we deliberately only track audio

package/src/sorting/__tests__/presets.test.ts

@@ -1,8 +1,54 @@
 import { describe, expect, it } from 'vitest';
 import * as TestData from './participant-data';
-import { VisibilityState } from '../../types';
-import { paginatedLayoutSortPreset } from '../presets';
-import { TrackType } from '../../gen/video/sfu/models/models';
+import { StreamVideoParticipant, VisibilityState } from '../../types';
+import { Comparator } from '../comparator';
+import {
+  defaultSortPreset,
+  livestreamOrAudioRoomSortPreset,
+  paginatedLayoutSortPreset,
+  speakerLayoutSortPreset,
+} from '../presets';
+import {
+  ParticipantSource,
+  TrackType,
+} from '../../gen/video/sfu/models/models';
+
+const participant = (
+  name: string,
+  publishedTracks: TrackType[],
+  visibility: VisibilityState = VisibilityState.VISIBLE,
+  overrides: Partial<StreamVideoParticipant> = {},
+): StreamVideoParticipant =>
+  ({
+    name,
+    userId: name,
+    sessionId: name,
+    roles: ['user'],
+    publishedTracks,
+    trackLookupPrefix: name,
+    isSpeaking: false,
+    isDominantSpeaker: false,
+    audioLevel: 0,
+    image: '',
+    source: ParticipantSource.WEBRTC_UNSPECIFIED,
+    viewportVisibilityState: {
+      videoTrack: visibility,
+      screenShareTrack: visibility,
+    },
+    ...overrides,
+  }) as StreamVideoParticipant;
+
+const names = (ps: StreamVideoParticipant[]) => ps.map((p) => p.name);
+
+// every preset that prioritizes publishing participants over passive ones
+const presets: [string, Comparator<StreamVideoParticipant>][] = [
+  ['defaultSortPreset', defaultSortPreset],
+  ['speakerLayoutSortPreset', speakerLayoutSortPreset],
+  ['livestreamOrAudioRoomSortPreset', livestreamOrAudioRoomSortPreset],
+];
+
+const { UNKNOWN, VISIBLE, INVISIBLE } = VisibilityState;
+const { AUDIO, VIDEO, SCREEN_SHARE } = TrackType;
 
 describe('presets', () => {
   it('paginatedLayoutSortPreset', () => {
@@ -60,4 +106,143 @@ describe('presets', () => {
       ]
     `);
   });
+
+  // In SpeakerLayout the viewport is the participants bar, so the spotlight
+  // tile is never observed and stays UNKNOWN. Regression test for a passive
+  // participant holding the spotlight over a participant publishing video.
+  it.each(presets)(
+    '%s keeps passive participants out of the spotlight',
+    (_, preset) => {
+      const ps = [
+        participant('passive-viewer', [], UNKNOWN),
+        participant('host', [VIDEO]),
+        participant('local-viewer', [], VISIBLE, { isLocalParticipant: true }),
+      ];
+
+      expect(names(ps.sort(preset))).toEqual([
+        'host',
+        'passive-viewer',
+        'local-viewer',
+      ]);
+    },
+  );
+
+  it.each(presets)(
+    '%s promotes a rejoined publisher appended last',
+    (_, preset) => {
+      const ps = [
+        participant('v1', [], UNKNOWN),
+        participant('v2', []),
+        participant('v3', []),
+      ].sort(preset);
+      // a rejoining participant is appended to the end of the list
+      ps.push(participant('host', [AUDIO, VIDEO], UNKNOWN));
+
+      expect(names(ps.sort(preset))).toEqual(['host', 'v1', 'v2', 'v3']);
+    },
+  );
+
+  it.each(presets)(
+    '%s ranks passive participants below publishing ones regardless of visibility',
+    (_, preset) => {
+      const passive = [
+        participant('p1', [], UNKNOWN),
+        participant('p2', [], VISIBLE),
+        participant('p3', [], INVISIBLE),
+      ];
+      const publishing = [
+        participant('a1', [VIDEO], UNKNOWN),
+        participant('a2', [AUDIO], VISIBLE),
+        participant('a3', [AUDIO, VIDEO], INVISIBLE),
+      ];
+
+      for (const a of passive) {
+        for (const b of publishing) {
+          expect(preset(a, b)).toBe(1);
+          expect(preset(b, a)).toBe(-1);
+        }
+      }
+    },
+  );
+
+  // `publishing` ignores the track type, so a participant keeps its rank while
+  // it still publishes something. In speakerLayoutSortPreset the gated
+  // comparators cannot override this, because the gate only opens on INVISIBLE.
+  it.each<[string, TrackType[]]>([
+    ['camera and microphone on', [AUDIO, VIDEO]],
+    ['the microphone muted', [VIDEO]],
+    ['the camera turned off', [AUDIO]],
+  ])(
+    'speakerLayoutSortPreset keeps the spotlight with %s',
+    (_, spotlightTracks) => {
+      const ps = [
+        participant('spotlight', spotlightTracks, UNKNOWN),
+        participant('bar', [AUDIO, VIDEO]),
+      ];
+
+      expect(names(ps.sort(speakerLayoutSortPreset))).toEqual([
+        'spotlight',
+        'bar',
+      ]);
+    },
+  );
+
+  it('speakerLayoutSortPreset drops a spotlight that stops publishing', () => {
+    const ps = [
+      participant('spotlight', [], UNKNOWN),
+      participant('bar', [AUDIO, VIDEO]),
+    ];
+
+    expect(names(ps.sort(speakerLayoutSortPreset))).toEqual([
+      'bar',
+      'spotlight',
+    ]);
+  });
+
+  // livestreamOrAudioRoomSortPreset gates on INVISIBLE *or* UNKNOWN, so an
+  // unobserved spotlight tile stays comparable by audio and video state. This
+  // predates the `publishing` comparator, which only breaks ties between a
+  // publishing and a non-publishing participant.
+  it('livestreamOrAudioRoomSortPreset still re-ranks publishers by track state', () => {
+    const ps = [
+      participant('spotlight', [VIDEO], UNKNOWN),
+      participant('bar', [AUDIO, VIDEO]),
+    ];
+
+    expect(names(ps.sort(livestreamOrAudioRoomSortPreset))).toEqual([
+      'bar',
+      'spotlight',
+    ]);
+  });
+
+  it.each<[string, TrackType[], Partial<StreamVideoParticipant>]>([
+    ['a pinned participant', [], { pin: { isLocalPin: true, pinnedAt: 1 } }],
+    ['a presenter', [SCREEN_SHARE], {}],
+    ['the dominant speaker', [AUDIO], { isDominantSpeaker: true }],
+  ])(
+    'speakerLayoutSortPreset ranks %s above a publishing participant',
+    (_, tracks, overrides) => {
+      const ps = [
+        participant('publisher', [AUDIO, VIDEO], UNKNOWN),
+        participant('contender', tracks, VISIBLE, overrides),
+      ];
+
+      expect(names(ps.sort(speakerLayoutSortPreset))).toEqual([
+        'contender',
+        'publisher',
+      ]);
+    },
+  );
+
+  it('livestreamOrAudioRoomSortPreset still uses role as a tie breaker between publishers', () => {
+    const ps = [
+      participant('viewer', [AUDIO, VIDEO]),
+      participant('host', [AUDIO, VIDEO], UNKNOWN, { roles: ['host'] }),
+    ];
+
+    expect(names(ps.sort(livestreamOrAudioRoomSortPreset))).toEqual([
+      'host',
+      'viewer',
+    ]);
+  });
 });

package/src/sorting/__tests__/sorting.test.ts

@@ -1,11 +1,16 @@
 import { describe, expect, it } from 'vitest';
-import { ParticipantSource } from '../../gen/video/sfu/models/models';
+import {
+  ParticipantSource,
+  TrackType,
+} from '../../gen/video/sfu/models/models';
+import { StreamVideoParticipant } from '../../types';
 import {
   combineComparators,
   Comparator,
   conditional,
   dominantSpeaker,
   pinned,
+  publishing,
   publishingAudio,
   publishingVideo,
   screenSharing,
@@ -37,6 +42,22 @@ describe('Sorting', () => {
     expect(sorted.map((p) => p.name)).toEqual(['F', 'D', 'B', 'A', 'E', 'C']);
   });
 
+  it('publishing', () => {
+    const { AUDIO, VIDEO, SCREEN_SHARE } = TrackType;
+    const p = (publishedTracks: TrackType[]) =>
+      ({ publishedTracks }) as StreamVideoParticipant;
+
+    // publishing anything ranks above publishing nothing
+    expect(publishing(p([AUDIO]), p([]))).toEqual(-1);
+    expect(publishing(p([SCREEN_SHARE]), p([]))).toEqual(-1);
+    expect(publishing(p([]), p([VIDEO]))).toEqual(1);
+
+    // the track type is irrelevant, so device toggles do not change the rank
+    expect(publishing(p([AUDIO, VIDEO]), p([AUDIO]))).toEqual(0);
+    expect(publishing(p([VIDEO]), p([AUDIO]))).toEqual(0);
+    expect(publishing(p([]), p([]))).toEqual(0);
+  });
+
   it('conditional comparator', () => {
     const byValue: Comparator<number> = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
     const input = [2, 3, 1];

package/src/sorting/participants.ts

@@ -73,6 +73,26 @@ export const publishingAudio: Comparator<StreamVideoParticipant> = (a, b) => {
   return 0;
 };
 
+/**
+ * A comparator which prioritizes participants who are publishing any track
+ * over participants who are not publishing at all.
+ *
+ * Unlike `publishingVideo` and `publishingAudio`, this comparator does not
+ * distinguish between track types. Muting the microphone or turning off the
+ * camera does not change a participant's rank as long as they still publish
+ * something, which keeps the ordering stable across ordinary device toggles.
+ *
+ * @param a the first participant.
+ * @param b the second participant.
+ */
+export const publishing: Comparator<StreamVideoParticipant> = (a, b) => {
+  const hasA = a.publishedTracks.length > 0;
+  const hasB = b.publishedTracks.length > 0;
+  if (hasA && !hasB) return -1;
+  if (!hasA && hasB) return 1;
+  return 0;
+};
+
 /**
  * A comparator which prioritizes participants who are pinned.
  *

package/src/sorting/presets.ts

@@ -4,6 +4,7 @@ import { ParticipantSource } from '../gen/video/sfu/models/models';
 import {
   dominantSpeaker,
   pinned,
+  publishing,
   publishingAudio,
   publishingVideo,
   reactionType,
@@ -51,6 +52,7 @@ const withVideoIngressSource = withParticipantSource(
 export const defaultSortPreset = combineComparators(
   screenSharing,
   pinned,
+  publishing,
   ifInvisibleBy(
     combineComparators(
       dominantSpeaker,
@@ -69,6 +71,7 @@ export const speakerLayoutSortPreset = combineComparators(
   screenSharing,
   pinned,
   dominantSpeaker,
+  publishing,
   ifInvisibleBy(
     combineComparators(
       speaking,
@@ -102,6 +105,7 @@ export const paginatedLayoutSortPreset = combineComparators(
  * The sorting preset for livestreams and audio rooms.
  */
 export const livestreamOrAudioRoomSortPreset = combineComparators(
+  publishing,
   ifInvisibleOrUnknownBy(
     combineComparators(
       dominantSpeaker,

package/src/store/__tests__/CallState.test.ts

@@ -266,8 +266,8 @@ describe('CallState', () => {
           "E",
           "F",
           "A",
-          "C",
           "D",
+          "C",
         ]
       `);
 
@@ -288,8 +288,8 @@ describe('CallState', () => {
           "E",
           "F",
           "A",
-          "C",
           "D",
+          "C",
         ]
       `);
     });