@stream-io/video-client 1.49.0 → 1.50.0

package/src/rtc/Subscriber.ts

@@ -1,9 +1,10 @@
 import { BasePeerConnection } from './BasePeerConnection';
 import { BasePeerConnectionOpts } from './types';
 import { NegotiationError } from './NegotiationError';
-import { PeerType } from '../gen/video/sfu/models/models';
+import { PeerType, TrackType } from '../gen/video/sfu/models/models';
 import { SubscriberOffer } from '../gen/video/sfu/event/events';
 import { toTrackType, trackTypeToParticipantStreamKey } from './helpers/tracks';
+import { pushToIfMissing, removeFromIfPresent } from '../helpers/array';
 import { enableStereo, removeCodecsExcept } from './helpers/sdp';
 
 /**
@@ -67,7 +68,8 @@ export class Subscriber extends BasePeerConnection {
   };
 
   private handleOnTrack = (e: RTCTrackEvent) => {
-    const [primaryStream] = e.streams;
+    const { streams, track } = e;
+    const [primaryStream] = streams;
     // example: `e3f6aaf8-b03d-4911-be36-83f47d37a76a:TRACK_TYPE_VIDEO`
     const [trackId, rawTrackType] = primaryStream.id.split(':');
     const participantToUpdate = this.state.participants.find(
@@ -75,30 +77,35 @@ export class Subscriber extends BasePeerConnection {
     );
     this.logger.debug(
       `[onTrack]: Got remote ${rawTrackType} track for userId: ${participantToUpdate?.userId}`,
-      e.track.id,
-      e.track,
+      track.id,
+      track,
     );
 
+    const trackType = toTrackType(rawTrackType);
+    if (!trackType) {
+      return this.logger.error(`Unknown track type: ${rawTrackType}`);
+    }
+
     const trackDebugInfo = `${participantToUpdate?.userId} ${rawTrackType}:${trackId}`;
-    e.track.addEventListener('mute', () => {
+    track.addEventListener('mute', () => {
       this.logger.info(`[onTrack]: Track muted: ${trackDebugInfo}`);
+      this.setRemoteTrackInterrupted(trackId, trackType, true);
     });
-
-    e.track.addEventListener('unmute', () => {
+    track.addEventListener('unmute', () => {
       this.logger.info(`[onTrack]: Track unmuted: ${trackDebugInfo}`);
+      this.setRemoteTrackInterrupted(trackId, trackType, false);
     });
-
-    e.track.addEventListener('ended', () => {
+    track.addEventListener('ended', () => {
       this.logger.info(`[onTrack]: Track ended: ${trackDebugInfo}`);
+      this.setRemoteTrackInterrupted(trackId, trackType, false);
       this.state.removeOrphanedTrack(primaryStream.id);
     });
 
-    const trackType = toTrackType(rawTrackType);
-    if (!trackType) {
-      return this.logger.error(`Unknown track type: ${rawTrackType}`);
+    if (track.muted) {
+      this.setRemoteTrackInterrupted(trackId, trackType, true);
     }
 
-    this.trackIdToTrackType.set(e.track.id, trackType);
+    this.trackIdToTrackType.set(track.id, trackType);
 
     if (!participantToUpdate) {
       this.logger.warn(
@@ -133,7 +140,7 @@ export class Subscriber extends BasePeerConnection {
     // now, dispose the previous stream if it exists
     if (previousStream) {
       this.logger.info(
-        `[onTrack]: Cleaning up previous remote ${e.track.kind} tracks for userId: ${participantToUpdate.userId}`,
+        `[onTrack]: Cleaning up previous remote ${track.kind} tracks for userId: ${participantToUpdate.userId}`,
       );
       previousStream.getTracks().forEach((t) => {
         t.stop();
@@ -142,6 +149,27 @@ export class Subscriber extends BasePeerConnection {
     }
   };
 
+  private setRemoteTrackInterrupted = (
+    trackId: string,
+    trackType: TrackType,
+    interrupted: boolean,
+  ) => {
+    if (trackType !== TrackType.AUDIO) return;
+    const target = this.state.participants.find(
+      (p) => p.trackLookupPrefix === trackId,
+    );
+    if (!target) return;
+    this.state.updateParticipant(target.sessionId, (p) => {
+      const current = p.interruptedTracks ?? [];
+      const has = current.includes(trackType);
+      if (interrupted === has) return {};
+      const next = interrupted
+        ? pushToIfMissing([...current], trackType)
+        : removeFromIfPresent([...current], trackType);
+      return { interruptedTracks: next };
+    });
+  };
+
   private negotiate = async (subscriberOffer: SubscriberOffer) => {
     await this.pc.setRemoteDescription({
       type: 'offer',

package/src/rtc/__tests__/Publisher.test.ts

@@ -2,6 +2,7 @@ import './mocks/webrtc.mocks';
 
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { anyString } from 'vitest-mock-extended';
+import { fromPartial } from '@total-typescript/shoehorn';
 import { NegotiationError } from '../NegotiationError';
 import { Publisher } from '../Publisher';
 import { ReconnectReason } from '../types';
@@ -9,6 +10,7 @@ import { CallState } from '../../store';
 import { StreamSfuClient } from '../../StreamSfuClient';
 import { DispatchableMessage, Dispatcher } from '../Dispatcher';
 import {
+  DegradationPreference,
   ErrorCode,
   PeerType,
   PublishOption,
@@ -82,6 +84,7 @@ describe('Publisher', () => {
           fps: 30,
           maxTemporalLayers: 3,
           maxSpatialLayers: 3,
+          degradationPreference: DegradationPreference.UNSPECIFIED,
         },
       ],
     );
@@ -168,16 +171,19 @@ describe('Publisher', () => {
             changePublishQuality: {
               audioSenders: [],
               videoSenders: [
-                {
+                fromPartial({
                   publishOptionId: 1,
                   trackType: TrackType.VIDEO,
                   layers: [],
-                },
-                {
+                  degradationPreference: DegradationPreference.BALANCED,
+                }),
+                fromPartial({
                   publishOptionId: 2,
                   trackType: TrackType.SCREEN_SHARE,
                   layers: [],
-                },
+                  degradationPreference:
+                    DegradationPreference.MAINTAIN_RESOLUTION,
+                }),
               ],
             },
           },
@@ -657,6 +663,7 @@ describe('Publisher', () => {
       await publisher['changePublishQuality']({
         publishOptionId: 1,
         trackType: TrackType.VIDEO,
+        degradationPreference: DegradationPreference.UNSPECIFIED,
         layers: [
           {
             name: 'q',
@@ -733,6 +740,7 @@ describe('Publisher', () => {
       await publisher['changePublishQuality']({
         publishOptionId: 1,
         trackType: TrackType.VIDEO,
+        degradationPreference: DegradationPreference.UNSPECIFIED,
         layers: [
           {
             name: 'q',
@@ -796,6 +804,7 @@ describe('Publisher', () => {
       await publisher['changePublishQuality']({
         publishOptionId: 1,
         trackType: TrackType.VIDEO,
+        degradationPreference: DegradationPreference.UNSPECIFIED,
         layers: [
           {
             name: 'q',
@@ -855,6 +864,7 @@ describe('Publisher', () => {
       await publisher['changePublishQuality']({
         publishOptionId: 1,
         trackType: TrackType.VIDEO,
+        degradationPreference: DegradationPreference.UNSPECIFIED,
         layers: [
           {
             name: 'q',
@@ -879,6 +889,93 @@ describe('Publisher', () => {
         },
       ]);
     });
+
+    it('applies degradationPreference from the SFU event', async () => {
+      const transceiver = new RTCRtpTransceiver();
+      const setParametersSpy = vi
+        .spyOn(transceiver.sender, 'setParameters')
+        .mockResolvedValue();
+      vi.spyOn(transceiver.sender, 'getParameters').mockReturnValue({
+        // @ts-expect-error incomplete data
+        codecs: [{ mimeType: 'video/VP8' }],
+        encodings: [{ rid: 'q', active: true }],
+        degradationPreference: 'maintain-framerate',
+      });
+
+      publisher['transceiverCache'].add({
+        // @ts-expect-error incomplete data
+        publishOption: { trackType: TrackType.VIDEO, id: 1 },
+        transceiver,
+        options: {},
+      });
+
+      await publisher['changePublishQuality']({
+        publishOptionId: 1,
+        trackType: TrackType.VIDEO,
+        degradationPreference: DegradationPreference.BALANCED,
+        layers: [
+          {
+            name: 'q',
+            active: true,
+            maxBitrate: 100,
+            scaleResolutionDownBy: 1,
+            maxFramerate: 30,
+            scalabilityMode: '',
+          },
+        ],
+      });
+
+      expect(setParametersSpy).toHaveBeenCalled();
+      expect(setParametersSpy.mock.calls[0][0].degradationPreference).toBe(
+        'balanced',
+      );
+    });
+
+    it('does not call setParameters when nothing changes and degradationPreference is UNSPECIFIED', async () => {
+      const transceiver = new RTCRtpTransceiver();
+      const setParametersSpy = vi
+        .spyOn(transceiver.sender, 'setParameters')
+        .mockResolvedValue();
+      vi.spyOn(transceiver.sender, 'getParameters').mockReturnValue({
+        // @ts-expect-error incomplete data
+        codecs: [{ mimeType: 'video/VP8' }],
+        encodings: [
+          {
+            rid: 'q',
+            active: true,
+            maxBitrate: 100,
+            scaleResolutionDownBy: 1,
+            maxFramerate: 30,
+          },
+        ],
+        degradationPreference: 'maintain-framerate',
+      });
+
+      publisher['transceiverCache'].add({
+        // @ts-expect-error incomplete data
+        publishOption: { trackType: TrackType.VIDEO, id: 1 },
+        transceiver,
+        options: {},
+      });
+
+      await publisher['changePublishQuality']({
+        publishOptionId: 1,
+        trackType: TrackType.VIDEO,
+        degradationPreference: DegradationPreference.UNSPECIFIED,
+        layers: [
+          {
+            name: 'q',
+            active: true,
+            maxBitrate: 100,
+            scaleResolutionDownBy: 1,
+            maxFramerate: 30,
+            scalabilityMode: '',
+          },
+        ],
+      });
+
+      expect(setParametersSpy).not.toHaveBeenCalled();
+    });
   });
 
   describe('changePublishOptions', () => {
@@ -893,12 +990,27 @@ describe('Publisher', () => {
       vi.spyOn(publisher, 'negotiate').mockResolvedValue();
 
       publisher['publishOptions'] = [
-        // @ts-expect-error incomplete data
-        { trackType: TrackType.VIDEO, id: 0, codec: { name: 'vp8' } },
-        // @ts-expect-error incomplete data
-        { trackType: TrackType.VIDEO, id: 1, codec: { name: 'av1' } },
-        // @ts-expect-error incomplete data
-        { trackType: TrackType.VIDEO, id: 2, codec: { name: 'vp9' } },
+        {
+          trackType: TrackType.VIDEO,
+          id: 0,
+          // @ts-expect-error incomplete data
+          codec: { name: 'vp8' },
+          degradationPreference: DegradationPreference.UNSPECIFIED,
+        },
+        {
+          trackType: TrackType.VIDEO,
+          id: 1,
+          // @ts-expect-error incomplete data
+          codec: { name: 'av1' },
+          degradationPreference: DegradationPreference.UNSPECIFIED,
+        },
+        {
+          trackType: TrackType.VIDEO,
+          id: 2,
+          // @ts-expect-error incomplete data
+          codec: { name: 'vp9' },
+          degradationPreference: DegradationPreference.UNSPECIFIED,
+        },
       ];
 
       publisher['transceiverCache'].add({

package/src/rtc/__tests__/Subscriber.test.ts

@@ -28,10 +28,11 @@ vi.mock('../../StreamSfuClient', () => {
 describe('Subscriber', () => {
   let sfuClient: StreamSfuClient;
   let subscriber: Subscriber;
-  const state = new CallState();
+  let state: CallState;
   let dispatcher: Dispatcher;
 
   beforeEach(() => {
+    state = new CallState();
     dispatcher = new Dispatcher();
     sfuClient = new StreamSfuClient({
       dispatcher,
@@ -272,6 +273,150 @@ describe('Subscriber', () => {
     });
   });
 
+  describe('interruptedTracks', () => {
+    const setup = ({ muted = false }: { muted?: boolean } = {}) => {
+      const mediaStream = new MediaStream();
+      const track = new MediaStreamTrack();
+      // @ts-expect-error - mock
+      mediaStream.id = 'lookup:TRACK_TYPE_AUDIO';
+      // @ts-expect-error - mock
+      track.kind = 'audio';
+      Object.defineProperty(track, 'muted', {
+        configurable: true,
+        get: () => muted,
+      });
+      // @ts-expect-error - incomplete mock
+      state.updateOrAddParticipant('session-id', {
+        sessionId: 'session-id',
+        trackLookupPrefix: 'lookup',
+      });
+
+      const onTrack = subscriber['handleOnTrack'];
+      // @ts-expect-error - incomplete mock
+      onTrack({ streams: [mediaStream], track });
+
+      const calls = (track.addEventListener as ReturnType<typeof vi.fn>).mock
+        .calls;
+      const handlers: Record<string, () => void> = {};
+      for (const [event, handler] of calls) {
+        handlers[event] = handler as () => void;
+      }
+      return { track, handlers };
+    };
+
+    const interruptedFor = (sessionId: string) =>
+      state.participants.find((p) => p.sessionId === sessionId)
+        ?.interruptedTracks ?? [];
+
+    it('adds the track type when the mute handler fires', () => {
+      const { handlers } = setup();
+      expect(interruptedFor('session-id')).toEqual([]);
+
+      handlers['mute']();
+
+      expect(interruptedFor('session-id')).toEqual([TrackType.AUDIO]);
+    });
+
+    it('removes the track type when the unmute handler fires', () => {
+      const { handlers } = setup();
+      handlers['mute']();
+      expect(interruptedFor('session-id')).toEqual([TrackType.AUDIO]);
+
+      handlers['unmute']();
+
+      expect(interruptedFor('session-id')).toEqual([]);
+    });
+
+    it('seeds the track type when the track arrives already muted', () => {
+      setup({ muted: true });
+
+      expect(interruptedFor('session-id')).toEqual([TrackType.AUDIO]);
+    });
+
+    it('clears the track type when the track ends', () => {
+      const { handlers } = setup();
+      handlers['mute']();
+      expect(interruptedFor('session-id')).toEqual([TrackType.AUDIO]);
+
+      handlers['ended']();
+
+      expect(interruptedFor('session-id')).toEqual([]);
+    });
+
+    it('ignores non-audio remote tracks to avoid Dynascale false positives', () => {
+      // Remote video track.muted is dominated by viewport-driven
+      // SFU unsubscriptions, so we deliberately only track audio
+      // interruption on remote participants.
+      const mediaStream = new MediaStream();
+      const track = new MediaStreamTrack();
+      // @ts-expect-error - mock
+      mediaStream.id = 'video-lookup:TRACK_TYPE_VIDEO';
+      // @ts-expect-error - mock
+      track.kind = 'video';
+      Object.defineProperty(track, 'muted', {
+        configurable: true,
+        get: () => true,
+      });
+      // @ts-expect-error - incomplete mock
+      state.updateOrAddParticipant('video-session', {
+        sessionId: 'video-session',
+        trackLookupPrefix: 'video-lookup',
+      });
+
+      const onTrack = subscriber['handleOnTrack'];
+      // @ts-expect-error - incomplete mock
+      onTrack({ streams: [mediaStream], track });
+
+      // Seeded muted track is ignored.
+      expect(interruptedFor('video-session')).toEqual([]);
+
+      // Subsequent mute / unmute events are ignored too.
+      const calls = (track.addEventListener as ReturnType<typeof vi.fn>).mock
+        .calls;
+      const handlers: Record<string, () => void> = {};
+      for (const [event, handler] of calls) {
+        handlers[event] = handler as () => void;
+      }
+      handlers['mute']();
+      handlers['unmute']();
+      expect(interruptedFor('video-session')).toEqual([]);
+    });
+
+    it('does not mutate state for orphaned tracks until associated', () => {
+      const mediaStream = new MediaStream();
+      const track = new MediaStreamTrack();
+      // @ts-expect-error - mock
+      mediaStream.id = 'orphan:TRACK_TYPE_AUDIO';
+      // @ts-expect-error - mock
+      track.kind = 'audio';
+
+      const onTrack = subscriber['handleOnTrack'];
+      // @ts-expect-error - incomplete mock
+      onTrack({ streams: [mediaStream], track });
+
+      const calls = (track.addEventListener as ReturnType<typeof vi.fn>).mock
+        .calls;
+      const handlers: Record<string, () => void> = {};
+      for (const [event, handler] of calls) {
+        handlers[event] = handler as () => void;
+      }
+
+      // Orphan: handler fires before the participant exists.
+      handlers['mute']();
+      expect(state.participants).toEqual([]);
+
+      // Once the participant is registered, the next event lands.
+      // @ts-expect-error - incomplete mock
+      state.updateOrAddParticipant('orphan-session', {
+        sessionId: 'orphan-session',
+        trackLookupPrefix: 'orphan',
+      });
+      handlers['mute']();
+
+      expect(interruptedFor('orphan-session')).toEqual([TrackType.AUDIO]);
+    });
+  });
+
   describe('Negotiation', () => {
     it('negotiates with the SFU', async () => {
       sfuClient.sendAnswer = vi.fn();

package/src/rtc/__tests__/mocks/webrtc.mocks.ts

@@ -16,8 +16,8 @@ const RTCPeerConnectionMock = vi.fn((): Partial<RTCPeerConnection> => {
     close: vi.fn(),
     connectionState: 'connected',
     signalingState: 'stable',
-    getReceivers: vi.fn(),
-    getSenders: vi.fn(),
+    getReceivers: vi.fn().mockReturnValue([]),
+    getSenders: vi.fn().mockReturnValue([]),
     removeTrack: vi.fn(),
   };
 });
@@ -109,6 +109,16 @@ const AudioContextMock = vi.fn((): Partial<AudioContext> => {
         gain: { value: 1 },
       } as unknown as GainNode;
     }),
+    // Silent keep-alive node used by DynascaleManager's probe AudioContext.
+    createConstantSource: vi.fn(() => {
+      return {
+        offset: { value: 0 },
+        connect: vi.fn((v) => v),
+        disconnect: vi.fn(),
+        start: vi.fn(),
+        stop: vi.fn(),
+      } as unknown as ConstantSourceNode;
+    }),
     close: vi.fn(async function () {
       this.state = 'closed';
     }),
@@ -119,6 +129,7 @@ const AudioContextMock = vi.fn((): Partial<AudioContext> => {
       this.sinkId = sinkId;
     }),
     addEventListener: vi.fn(),
+    removeEventListener: vi.fn(),
   };
 });
 vi.stubGlobal('AudioContext', AudioContextMock);

package/src/rtc/helpers/__tests__/degradationPreference.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { DegradationPreference } from '../../../gen/video/sfu/models/models';
+import { toRTCDegradationPreference } from '../degradationPreference';
+
+describe('toRTCDegradationPreference', () => {
+  it.each([
+    [DegradationPreference.BALANCED, 'balanced'],
+    [DegradationPreference.MAINTAIN_FRAMERATE, 'maintain-framerate'],
+    [DegradationPreference.MAINTAIN_RESOLUTION, 'maintain-resolution'],
+    [
+      DegradationPreference.MAINTAIN_FRAMERATE_AND_RESOLUTION,
+      'maintain-framerate-and-resolution',
+    ],
+  ])('maps %s to "%s"', (preference, expected) => {
+    expect(toRTCDegradationPreference(preference)).toBe(expected);
+  });
+
+  it('returns undefined for UNSPECIFIED', () => {
+    expect(
+      toRTCDegradationPreference(DegradationPreference.UNSPECIFIED),
+    ).toBeUndefined();
+  });
+});

package/src/rtc/helpers/degradationPreference.ts

@@ -0,0 +1,22 @@
+import { DegradationPreference } from '../../gen/video/sfu/models/models';
+import { ensureExhausted } from '../../helpers/ensureExhausted';
+
+export const toRTCDegradationPreference = (
+  preference: DegradationPreference,
+): RTCDegradationPreference | undefined => {
+  switch (preference) {
+    case DegradationPreference.BALANCED:
+      return 'balanced';
+    case DegradationPreference.MAINTAIN_FRAMERATE:
+      return 'maintain-framerate';
+    case DegradationPreference.MAINTAIN_RESOLUTION:
+      return 'maintain-resolution';
+    case DegradationPreference.MAINTAIN_FRAMERATE_AND_RESOLUTION:
+      // @ts-expect-error not in the typedefs yet
+      return 'maintain-framerate-and-resolution';
+    case DegradationPreference.UNSPECIFIED:
+      return undefined;
+    default:
+      ensureExhausted(preference, 'Unknown degradation preference');
+  }
+};

package/src/stats/rtc/types.ts

@@ -16,6 +16,7 @@ export type RTCStatsDataType =
   | RTCSessionDescriptionInit
   | (RTCIceCandidateInit | RTCIceCandidate) // addIceCandidate
   | object
+  | number
   | null
   | undefined;