@stream-io/video-client 1.7.4 → 1.8.0

package/src/helpers/DynascaleManager.ts

@@ -1,4 +1,3 @@
-import { Call } from '../Call';
 import {
   AudioTrackType,
   DebounceType,
@@ -6,8 +5,9 @@ import {
   VideoTrackType,
   VisibilityState,
 } from '../types';
-import { VideoDimension } from '../gen/video/sfu/models/models';
+import { TrackType, VideoDimension } from '../gen/video/sfu/models/models';
 import {
+  BehaviorSubject,
   combineLatest,
   distinctUntilChanged,
   distinctUntilKeyChanged,
@@ -18,7 +18,16 @@ import {
 import { ViewportTracker } from './ViewportTracker';
 import { getLogger } from '../logger';
 import { isFirefox, isSafari } from './browsers';
-import { hasScreenShare, hasVideo } from './participantUtils';
+import {
+  hasScreenShare,
+  hasScreenShareAudio,
+  hasVideo,
+} from './participantUtils';
+import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signal';
+import type { CallState } from '../store';
+import type { StreamSfuClient } from '../StreamSfuClient';
+import { SpeakerManager } from '../devices';
+import { getCurrentValue, setCurrentValue } from '../store/rxUtils';
 
 const DEFAULT_VIEWPORT_VISIBILITY_STATE: Record<
   VideoTrackType,
@@ -28,6 +37,20 @@ const DEFAULT_VIEWPORT_VISIBILITY_STATE: Record<
   screenShareTrack: VisibilityState.UNKNOWN,
 } as const;
 
+type VideoTrackSubscriptionOverride =
+  | {
+      enabled: true;
+      dimension: VideoDimension;
+    }
+  | { enabled: false };
+
+const globalOverrideKey = Symbol('globalOverrideKey');
+
+interface VideoTrackSubscriptionOverrides {
+  [sessionId: string]: VideoTrackSubscriptionOverride | undefined;
+  [globalOverrideKey]?: VideoTrackSubscriptionOverride;
+}
+
 /**
  * A manager class that handles dynascale related tasks like:
  *
@@ -45,17 +68,151 @@ export class DynascaleManager {
   readonly viewportTracker = new ViewportTracker();
 
   private logger = getLogger(['DynascaleManager']);
-  private call: Call;
+  private callState: CallState;
+  private speaker: SpeakerManager;
+  private sfuClient: StreamSfuClient | undefined;
+  private pendingSubscriptionsUpdate: NodeJS.Timeout | null = null;
+
+  private videoTrackSubscriptionOverridesSubject =
+    new BehaviorSubject<VideoTrackSubscriptionOverrides>({});
+
+  videoTrackSubscriptionOverrides$ =
+    this.videoTrackSubscriptionOverridesSubject.asObservable();
+
+  incomingVideoSettings$ = this.videoTrackSubscriptionOverrides$.pipe(
+    map((overrides) => {
+      const { [globalOverrideKey]: globalSettings, ...participants } =
+        overrides;
+      return {
+        enabled: globalSettings?.enabled !== false,
+        preferredResolution: globalSettings?.enabled
+          ? globalSettings.dimension
+          : undefined,
+        participants: Object.fromEntries(
+          Object.entries(participants).map(
+            ([sessionId, participantOverride]) => [
+              sessionId,
+              {
+                enabled: participantOverride?.enabled !== false,
+                preferredResolution: participantOverride?.enabled
+                  ? participantOverride.dimension
+                  : undefined,
+              },
+            ],
+          ),
+        ),
+        isParticipantVideoEnabled: (sessionId: string) =>
+          overrides[sessionId]?.enabled ??
+          overrides[globalOverrideKey]?.enabled ??
+          true,
+      };
+    }),
+    shareReplay(1),
+  );
 
   /**
    * Creates a new DynascaleManager instance.
    *
    * @param call the call to manage.
    */
-  constructor(call: Call) {
-    this.call = call;
+  constructor(callState: CallState, speaker: SpeakerManager) {
+    this.callState = callState;
+    this.speaker = speaker;
+  }
+
+  setSfuClient(sfuClient: StreamSfuClient | undefined) {
+    this.sfuClient = sfuClient;
+  }
+
+  get trackSubscriptions() {
+    const subscriptions: TrackSubscriptionDetails[] = [];
+    for (const p of this.callState.remoteParticipants) {
+      // NOTE: audio tracks don't have to be requested explicitly
+      // as the SFU will implicitly subscribe us to all of them,
+      // once they become available.
+      if (p.videoDimension && hasVideo(p)) {
+        const override =
+          this.videoTrackSubscriptionOverrides[p.sessionId] ??
+          this.videoTrackSubscriptionOverrides[globalOverrideKey];
+
+        if (override?.enabled !== false) {
+          subscriptions.push({
+            userId: p.userId,
+            sessionId: p.sessionId,
+            trackType: TrackType.VIDEO,
+            dimension: override?.dimension ?? p.videoDimension,
+          });
+        }
+      }
+      if (p.screenShareDimension && hasScreenShare(p)) {
+        subscriptions.push({
+          userId: p.userId,
+          sessionId: p.sessionId,
+          trackType: TrackType.SCREEN_SHARE,
+          dimension: p.screenShareDimension,
+        });
+      }
+      if (hasScreenShareAudio(p)) {
+        subscriptions.push({
+          userId: p.userId,
+          sessionId: p.sessionId,
+          trackType: TrackType.SCREEN_SHARE_AUDIO,
+        });
+      }
+    }
+    return subscriptions;
   }
 
+  get videoTrackSubscriptionOverrides() {
+    return getCurrentValue(this.videoTrackSubscriptionOverrides$);
+  }
+
+  setVideoTrackSubscriptionOverrides = (
+    override: VideoTrackSubscriptionOverride | undefined,
+    sessionIds?: string[],
+  ) => {
+    if (!sessionIds) {
+      return setCurrentValue(
+        this.videoTrackSubscriptionOverridesSubject,
+        override ? { [globalOverrideKey]: override } : {},
+      );
+    }
+
+    return setCurrentValue(
+      this.videoTrackSubscriptionOverridesSubject,
+      (overrides) => ({
+        ...overrides,
+        ...Object.fromEntries(sessionIds.map((id) => [id, override])),
+      }),
+    );
+  };
+
+  applyTrackSubscriptions = (
+    debounceType: DebounceType = DebounceType.SLOW,
+  ) => {
+    if (this.pendingSubscriptionsUpdate) {
+      clearTimeout(this.pendingSubscriptionsUpdate);
+    }
+
+    const updateSubscriptions = () => {
+      this.pendingSubscriptionsUpdate = null;
+      this.sfuClient
+        ?.updateSubscriptions(this.trackSubscriptions)
+        .catch((err: unknown) => {
+          this.logger('debug', `Failed to update track subscriptions`, err);
+        });
+    };
+
+    if (debounceType) {
+      this.pendingSubscriptionsUpdate = setTimeout(
+        updateSubscriptions,
+        debounceType,
+      );
+    } else {
+      updateSubscriptions();
+    }
+  };
+
   /**
    * Will begin tracking the given element for visibility changes within the
    * configured viewport element (`call.setViewport`).
@@ -71,7 +228,7 @@ export class DynascaleManager {
     trackType: VideoTrackType,
   ) => {
     const cleanup = this.viewportTracker.observe(element, (entry) => {
-      this.call.state.updateParticipant(sessionId, (participant) => {
+      this.callState.updateParticipant(sessionId, (participant) => {
         const previousVisibilityState =
           participant.viewportVisibilityState ??
           DEFAULT_VIEWPORT_VISIBILITY_STATE;
@@ -97,7 +254,7 @@ export class DynascaleManager {
       // reset visibility state to UNKNOWN upon cleanup
       // so that the layouts that are not actively observed
       // can still function normally (runtime layout switching)
-      this.call.state.updateParticipant(sessionId, (participant) => {
+      this.callState.updateParticipant(sessionId, (participant) => {
         const previousVisibilityState =
           participant.viewportVisibilityState ??
           DEFAULT_VIEWPORT_VISIBILITY_STATE;
@@ -142,7 +299,7 @@ export class DynascaleManager {
     trackType: VideoTrackType,
   ) => {
     const boundParticipant =
-      this.call.state.findParticipantBySessionId(sessionId);
+      this.callState.findParticipantBySessionId(sessionId);
     if (!boundParticipant) return;
 
     const requestTrackWithDimensions = (
@@ -157,14 +314,13 @@ export class DynascaleManager {
         this.logger('debug', `Ignoring 0x0 dimension`, boundParticipant);
         dimension = undefined;
       }
-      this.call.updateSubscriptionsPartial(
-        trackType,
-        { [sessionId]: { dimension } },
-        debounceType,
-      );
+      this.callState.updateParticipantTracks(trackType, {
+        [sessionId]: { dimension },
+      });
+      this.applyTrackSubscriptions(debounceType);
     };
 
-    const participant$ = this.call.state.participants$.pipe(
+    const participant$ = this.callState.participants$.pipe(
       map(
         (participants) =>
           participants.find(
@@ -324,10 +480,10 @@ export class DynascaleManager {
     sessionId: string,
     trackType: AudioTrackType,
   ) => {
-    const participant = this.call.state.findParticipantBySessionId(sessionId);
+    const participant = this.callState.findParticipantBySessionId(sessionId);
     if (!participant || participant.isLocalParticipant) return;
 
-    const participant$ = this.call.state.participants$.pipe(
+    const participant$ = this.callState.participants$.pipe(
       map(
         (participants) =>
           participants.find(
@@ -364,7 +520,7 @@ export class DynascaleManager {
             // audio output device shall be set after the audio element is played
             // otherwise, the browser will not pick it up, and will always
             // play audio through the system's default device
-            const { selectedDevice } = this.call.speaker.state;
+            const { selectedDevice } = this.speaker.state;
             if (selectedDevice && 'setSinkId' in audioElement) {
               audioElement.setSinkId(selectedDevice);
             }
@@ -374,14 +530,14 @@ export class DynascaleManager {
 
     const sinkIdSubscription = !('setSinkId' in audioElement)
       ? null
-      : this.call.speaker.state.selectedDevice$.subscribe((deviceId) => {
+      : this.speaker.state.selectedDevice$.subscribe((deviceId) => {
           if (deviceId) {
             audioElement.setSinkId(deviceId);
           }
         });
 
     const volumeSubscription = combineLatest([
-      this.call.speaker.state.volume$,
+      this.speaker.state.volume$,
       participant$.pipe(distinctUntilKeyChanged('audioVolume')),
     ]).subscribe(([volume, p]) => {
       audioElement.volume = p.audioVolume ?? volume;

package/src/helpers/__tests__/DynascaleManager.test.ts

@@ -9,7 +9,7 @@ import { DynascaleManager } from '../DynascaleManager';
 import { Call } from '../../Call';
 import { StreamClient } from '../../coordinator/connection/client';
 import { StreamVideoWriteableStateStore } from '../../store';
-import { DebounceType, VisibilityState } from '../../types';
+import { VisibilityState } from '../../types';
 import { noopComparator } from '../../sorting';
 import { TrackType } from '../../gen/video/sfu/models/models';
 
@@ -25,7 +25,7 @@ describe('DynascaleManager', () => {
       clientStore: new StreamVideoWriteableStateStore(),
     });
     call.setSortParticipantsBy(noopComparator());
-    dynascaleManager = new DynascaleManager(call);
+    dynascaleManager = new DynascaleManager(call.state, call.speaker);
   });
 
   afterEach(() => {
@@ -194,7 +194,10 @@ describe('DynascaleManager', () => {
     });
 
     it('video: should update subscription when track becomes available', () => {
-      const updateSubscription = vi.spyOn(call, 'updateSubscriptionsPartial');
+      const updateSubscription = vi.spyOn(
+        call.state,
+        'updateParticipantTracks',
+      );
 
       // @ts-ignore
       call.state.updateOrAddParticipant('session-id', {
@@ -213,52 +216,45 @@ describe('DynascaleManager', () => {
       expect(videoElement.muted).toBe(true);
       expect(videoElement.playsInline).toBe(true);
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
 
       call.state.updateParticipant('session-id', {
         publishedTracks: [TrackType.VIDEO],
       });
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        {
-          'session-id': {
-            dimension: {
-              width: videoElement.clientWidth,
-              height: videoElement.clientHeight,
-            },
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': {
+          dimension: {
+            width: videoElement.clientWidth,
+            height: videoElement.clientHeight,
           },
         },
-        DebounceType.FAST,
-      );
+      });
 
       call.state.updateParticipant('session-id', {
         publishedTracks: [TrackType.VIDEO],
       });
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
 
       cleanup?.();
 
-      expect(updateSubscription).toHaveBeenLastCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenLastCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
     });
 
     it('video: should play video when track becomes available', () => {
       vi.useFakeTimers();
 
-      const updateSubscription = vi.spyOn(call, 'updateSubscriptionsPartial');
+      const updateSubscription = vi.spyOn(
+        call.state,
+        'updateParticipantTracks',
+      );
       const play = vi.spyOn(videoElement, 'play').mockResolvedValue();
 
       // @ts-ignore
@@ -282,28 +278,22 @@ describe('DynascaleManager', () => {
 
       vi.runAllTimers();
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        {
-          'session-id': {
-            dimension: {
-              width: videoElement.clientWidth,
-              height: videoElement.clientHeight,
-            },
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': {
+          dimension: {
+            width: videoElement.clientWidth,
+            height: videoElement.clientHeight,
           },
         },
-        DebounceType.FAST,
-      );
+      });
       expect(play).toHaveBeenCalled();
       expect(videoElement.srcObject).toBe(mediaStream);
 
       cleanup?.();
 
-      expect(updateSubscription).toHaveBeenLastCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenLastCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
     });
 
     it('video: should update subscription when element becomes visible', () => {
@@ -318,7 +308,10 @@ describe('DynascaleManager', () => {
         },
       });
 
-      const updateSubscription = vi.spyOn(call, 'updateSubscriptionsPartial');
+      const updateSubscription = vi.spyOn(
+        call.state,
+        'updateParticipantTracks',
+      );
 
       const cleanup = dynascaleManager.bindVideoElement(
         videoElement,
@@ -326,11 +319,9 @@ describe('DynascaleManager', () => {
         'videoTrack',
       );
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
 
       call.state.updateParticipant('session-id', {
         viewportVisibilityState: {
@@ -339,18 +330,14 @@ describe('DynascaleManager', () => {
         },
       });
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        {
-          'session-id': {
-            dimension: {
-              width: videoElement.clientWidth,
-              height: videoElement.clientHeight,
-            },
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': {
+          dimension: {
+            width: videoElement.clientWidth,
+            height: videoElement.clientHeight,
           },
         },
-        DebounceType.MEDIUM,
-      );
+      });
 
       call.state.updateParticipant('session-id', {
         viewportVisibilityState: {
@@ -359,11 +346,9 @@ describe('DynascaleManager', () => {
         },
       });
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.MEDIUM,
-      );
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
 
       call.state.updateParticipant('session-id', {
         viewportVisibilityState: {
@@ -372,26 +357,20 @@ describe('DynascaleManager', () => {
         },
       });
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        {
-          'session-id': {
-            dimension: {
-              width: videoElement.clientWidth,
-              height: videoElement.clientHeight,
-            },
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': {
+          dimension: {
+            width: videoElement.clientWidth,
+            height: videoElement.clientHeight,
           },
         },
-        DebounceType.MEDIUM,
-      );
+      });
 
       cleanup?.();
 
-      expect(updateSubscription).toHaveBeenLastCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenLastCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
     });
 
     it('video: should update subscription when element resizes', () => {
@@ -406,7 +385,7 @@ describe('DynascaleManager', () => {
         },
       });
 
-      let updateSubscription = vi.spyOn(call, 'updateSubscriptionsPartial');
+      let updateSubscription = vi.spyOn(call.state, 'updateParticipantTracks');
 
       let resizeObserverCallback: ResizeObserverCallback;
       window.ResizeObserver = class ResizeObserver {
@@ -428,18 +407,14 @@ describe('DynascaleManager', () => {
         'videoTrack',
       );
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        {
-          'session-id': {
-            dimension: {
-              width: videoElement.clientWidth,
-              height: videoElement.clientHeight,
-            },
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': {
+          dimension: {
+            width: videoElement.clientWidth,
+            height: videoElement.clientHeight,
           },
         },
-        DebounceType.FAST,
-      );
+      });
 
       // @ts-ignore simulate resize
       videoElement.clientHeight = 101;
@@ -449,19 +424,15 @@ describe('DynascaleManager', () => {
       // @ts-ignore simulate resize
       resizeObserverCallback();
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: { width: 101, height: 101 } } },
-        DebounceType.SLOW,
-      );
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': { dimension: { width: 101, height: 101 } },
+      });
 
       cleanup?.();
 
-      expect(updateSubscription).toHaveBeenLastCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenLastCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
     });
 
     it('video: should unsubscribe when element dimensions are zero', () => {
@@ -476,7 +447,7 @@ describe('DynascaleManager', () => {
         },
       });
 
-      let updateSubscription = vi.spyOn(call, 'updateSubscriptionsPartial');
+      let updateSubscription = vi.spyOn(call.state, 'updateParticipantTracks');
 
       // @ts-ignore simulate resize
       videoElement.clientHeight = 0;
@@ -489,19 +460,15 @@ describe('DynascaleManager', () => {
         'videoTrack',
       );
 
-      expect(updateSubscription).toHaveBeenCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
 
       cleanup?.();
 
-      expect(updateSubscription).toHaveBeenLastCalledWith(
-        'videoTrack',
-        { 'session-id': { dimension: undefined } },
-        DebounceType.FAST,
-      );
+      expect(updateSubscription).toHaveBeenLastCalledWith('videoTrack', {
+        'session-id': { dimension: undefined },
+      });
     });
   });
 });

package/src/store/CallState.ts

@@ -9,9 +9,11 @@ import type { Patch } from './rxUtils';
 import * as RxUtils from './rxUtils';
 import { CallingState } from './CallingState';
 import {
-  StreamVideoParticipant,
-  StreamVideoParticipantPatch,
-  StreamVideoParticipantPatches,
+  type StreamVideoParticipant,
+  type StreamVideoParticipantPatch,
+  type StreamVideoParticipantPatches,
+  type SubscriptionChanges,
+  VideoTrackType,
   VisibilityState,
 } from '../types';
 import { CallStatsReport } from '../stats';
@@ -903,6 +905,44 @@ export class CallState {
     );
   };
 
+  /**
+   * Update track subscription configuration for one or more participants.
+   * You have to create a subscription for each participant for all the different kinds of tracks you want to receive.
+   * You can only subscribe for tracks after the participant started publishing the given kind of track.
+   *
+   * @param trackType the kind of subscription to update.
+   * @param changes the list of subscription changes to do.
+   * @param type the debounce type to use for the update.
+   */
+  updateParticipantTracks = (
+    trackType: VideoTrackType,
+    changes: SubscriptionChanges,
+  ) => {
+    return this.updateParticipants(
+      Object.entries(changes).reduce<StreamVideoParticipantPatches>(
+        (acc, [sessionId, change]) => {
+          if (change.dimension) {
+            change.dimension.height = Math.ceil(change.dimension.height);
+            change.dimension.width = Math.ceil(change.dimension.width);
+          }
+          const prop: keyof StreamVideoParticipant | undefined =
+            trackType === 'videoTrack'
+              ? 'videoDimension'
+              : trackType === 'screenShareTrack'
+                ? 'screenShareDimension'
+                : undefined;
+          if (prop) {
+            acc[sessionId] = {
+              [prop]: change.dimension,
+            };
+          }
+          return acc;
+        },
+        {},
+      ),
+    );
+  };
+
   /**
    * Updates the call state with the data received from the server.
    *