@stream-io/video-client 1.49.0 → 1.50.0

package/src/helpers/TrackSubscriptionManager.ts

@@ -0,0 +1,243 @@
+import { BehaviorSubject, map, shareReplay } from 'rxjs';
+import { DebounceType } from '../types';
+import type { TrackSubscriptionDetails } from '../gen/video/sfu/signal_rpc/signal';
+import { TrackType, VideoDimension } from '../gen/video/sfu/models/models';
+import { CallState } from '../store';
+import { getCurrentValue, setCurrentValue } from '../store/rxUtils';
+import type { StreamSfuClient } from '../StreamSfuClient';
+import { videoLoggerSystem } from '../logger';
+import { Tracer } from '../stats';
+import {
+  hasScreenShare,
+  hasScreenShareAudio,
+  hasVideo,
+} from './participantUtils';
+
+/**
+ * Per-participant (or global) video-subscription override.
+ *
+ * - `{ enabled: true, dimension }`: request video at a specific
+ *   resolution. If set globally, applies to every remote participant
+ *   that doesn't have a per-session override.
+ * - `{ enabled: false }`: unsubscribe from video entirely. The SFU
+ *   sends nothing; bandwidth is saved.
+ */
+export type VideoTrackSubscriptionOverride =
+  | {
+      enabled: true;
+      dimension: VideoDimension;
+    }
+  | { enabled: false };
+
+/** Symbol key for the "applies to all participants" override slot. */
+const globalOverrideKey = Symbol('globalOverrideKey');
+
+/**
+ * Map of per-session overrides plus one optional global override under
+ * the `globalOverrideKey` symbol slot.
+ */
+export interface VideoTrackSubscriptionOverrides {
+  [sessionId: string]: VideoTrackSubscriptionOverride | undefined;
+  [globalOverrideKey]?: VideoTrackSubscriptionOverride;
+}
+
+/**
+ * Owns the SFU-side video-subscription machinery for a `Call`:
+ *
+ * - Holds the per-session / global override state in a
+ *   `BehaviorSubject<VideoTrackSubscriptionOverrides>`.
+ * - Derives the SFU subscription list from `CallState` participants +
+ *   current overrides via the `subscriptions` getter.
+ * - Debounces and pushes the list to the SFU through
+ *   `sfuClient.updateSubscriptions`.
+ * - Exposes `incomingVideoSettings$`: a consumer-friendly projection of
+ *   the override state for React hooks.
+ *
+ * Owned by `DynascaleManager` as `readonly trackSubscriptionManager`.
+ * `DynascaleManager.bindVideoElement` triggers `apply()` on every
+ * dimension / visibility change.
+ */
+export class TrackSubscriptionManager {
+  private logger = videoLoggerSystem.getLogger('TrackSubscriptionManager');
+  private callState: CallState;
+  private tracer: Tracer;
+
+  private sfuClient: StreamSfuClient | undefined;
+  private pendingUpdate: NodeJS.Timeout | null = null;
+
+  private overridesSubject =
+    new BehaviorSubject<VideoTrackSubscriptionOverrides>({});
+
+  overrides$ = this.overridesSubject.asObservable();
+
+  /**
+   * Consumer-friendly projection of the override state. Used by the
+   * `useIncomingVideoSettings()` React hook.
+   */
+  incomingVideoSettings$ = this.overrides$.pipe(
+    map((overrides) => {
+      const { [globalOverrideKey]: globalSettings, ...participants } =
+        overrides;
+      return {
+        enabled: globalSettings?.enabled !== false,
+        preferredResolution: globalSettings?.enabled
+          ? globalSettings.dimension
+          : undefined,
+        participants: Object.fromEntries(
+          Object.entries(participants).map(
+            ([sessionId, participantOverride]) => [
+              sessionId,
+              {
+                enabled: participantOverride?.enabled !== false,
+                preferredResolution: participantOverride?.enabled
+                  ? participantOverride.dimension
+                  : undefined,
+              },
+            ],
+          ),
+        ),
+        isParticipantVideoEnabled: (sessionId: string) =>
+          overrides[sessionId]?.enabled ??
+          overrides[globalOverrideKey]?.enabled ??
+          true,
+      };
+    }),
+    shareReplay(1),
+  );
+
+  /**
+   * Constructs new TrackSubscriptionManager instance.
+   *
+   * @param callState the call state.
+   * @param tracer the tracer to use.
+   */
+  constructor(callState: CallState, tracer: Tracer) {
+    this.tracer = tracer;
+    this.callState = callState;
+  }
+
+  /**
+   * Sets the SFU client used by `apply()` to push subscription updates.
+   * Called by the owner on call join; cleared on leave.
+   */
+  setSfuClient = (sfuClient: StreamSfuClient | undefined) => {
+    this.sfuClient = sfuClient;
+  };
+
+  /**
+   * Cancels any pending debounced subscription push. Idempotent.
+   */
+  dispose = () => {
+    if (this.pendingUpdate) {
+      clearTimeout(this.pendingUpdate);
+      this.pendingUpdate = null;
+    }
+  };
+
+  /**
+   * The current SFU subscription list, computed from `CallState`
+   * participants and the override state. Used by:
+   *
+   * - `apply()` to push to the SFU each time the set changes.
+   * - `Call.getReconnectDetails` to include the subscription list in
+   *   the reconnect payload.
+   */
+  get subscriptions(): TrackSubscriptionDetails[] {
+    const subscriptions: TrackSubscriptionDetails[] = [];
+    // Use getParticipantsSnapshot() to bypass the observable pipeline
+    // and avoid stale data caused by shareReplay with no active subscribers
+    const participants = this.callState.getParticipantsSnapshot();
+    const overrides = this.overridesSubject.getValue();
+    for (const p of participants) {
+      if (p.isLocalParticipant) continue;
+      // NOTE: audio tracks don't have to be requested explicitly
+      // as the SFU will implicitly subscribe us to all of them,
+      // once they become available.
+      if (p.videoDimension && hasVideo(p)) {
+        const override = overrides[p.sessionId] ?? overrides[globalOverrideKey];
+
+        if (override?.enabled !== false) {
+          subscriptions.push({
+            userId: p.userId,
+            sessionId: p.sessionId,
+            trackType: TrackType.VIDEO,
+            dimension: override?.dimension ?? p.videoDimension,
+          });
+        }
+      }
+      if (p.screenShareDimension && hasScreenShare(p)) {
+        subscriptions.push({
+          userId: p.userId,
+          sessionId: p.sessionId,
+          trackType: TrackType.SCREEN_SHARE,
+          dimension: p.screenShareDimension,
+        });
+      }
+      if (hasScreenShareAudio(p)) {
+        subscriptions.push({
+          userId: p.userId,
+          sessionId: p.sessionId,
+          trackType: TrackType.SCREEN_SHARE_AUDIO,
+        });
+      }
+    }
+    return subscriptions;
+  }
+
+  get overrides() {
+    return getCurrentValue(this.overrides$);
+  }
+
+  /**
+   * Sets video-subscription overrides. Called by
+   * `Call.setIncomingVideoEnabled` and
+   * `Call.setPreferredIncomingVideoResolution`.
+   *
+   * - `sessionIds` omitted → applies `override` globally (or clears the
+   *   global override if `override` is `undefined`).
+   * - `sessionIds` provided → applies `override` to each listed session.
+   */
+  setOverrides = (
+    override: VideoTrackSubscriptionOverride | undefined,
+    sessionIds?: string[],
+  ) => {
+    this.tracer.trace('setOverrides', [override, sessionIds]);
+    if (!sessionIds) {
+      return setCurrentValue(
+        this.overridesSubject,
+        override ? { [globalOverrideKey]: override } : {},
+      );
+    }
+
+    return setCurrentValue(this.overridesSubject, (overrides) => ({
+      ...overrides,
+      ...Object.fromEntries(sessionIds.map((id) => [id, override])),
+    }));
+  };
+
+  /**
+   * Pushes `subscriptions` to the SFU. Debounced by `debounceType`
+   * (SLOW by default). Multiple rapid calls coalesce into one RPC.
+   * Passing `0` fires synchronously.
+   */
+  apply = (debounceType: DebounceType = DebounceType.SLOW) => {
+    if (this.pendingUpdate) {
+      clearTimeout(this.pendingUpdate);
+    }
+
+    const updateSubscriptions = () => {
+      this.pendingUpdate = null;
+      this.sfuClient
+        ?.updateSubscriptions(this.subscriptions)
+        .catch((err: unknown) => {
+          this.logger.debug(`Failed to update track subscriptions`, err);
+        });
+    };
+
+    if (debounceType) {
+      this.pendingUpdate = setTimeout(updateSubscriptions, debounceType);
+    } else {
+      updateSubscriptions();
+    }
+  };
+}

package/src/helpers/ViewportTracker.ts

@@ -1,5 +1,16 @@
+import type { CallState } from '../store';
+import { VideoTrackType, VisibilityState } from '../types';
+
 const DEFAULT_THRESHOLD = 0.35;
 
+const DEFAULT_VIEWPORT_VISIBILITY_STATE: Record<
+  VideoTrackType,
+  VisibilityState
+> = {
+  videoTrack: VisibilityState.UNKNOWN,
+  screenShareTrack: VisibilityState.UNKNOWN,
+} as const;
+
 export type EntryHandler = (entry: IntersectionObserverEntry) => void;
 
 export type Unobserve = () => void;
@@ -10,33 +21,28 @@ export type Observe = (
 ) => Unobserve;
 
 export class ViewportTracker {
-  /**
-   * @private
-   */
+  private callState: CallState;
+
   private elementHandlerMap: Map<
     HTMLElement,
     (entry: IntersectionObserverEntry) => void
   > = new Map();
-  /**
-   * @private
-   */
+
   private observer: IntersectionObserver | null = null;
+
   // in React children render before viewport is set, add
   // them to the queue and observe them once the observer is ready
-  /**
-   * @private
-   */
   private queueSet: Set<readonly [HTMLElement, EntryHandler]> = new Set();
 
+  constructor(callState: CallState) {
+    this.callState = callState;
+  }
+
   /**
    * Method to set scrollable viewport as root for the IntersectionObserver, returns
    * cleanup function to be invoked upon disposing of the DOM element to prevent memory leaks
-   *
-   * @param viewportElement
-   * @param options
-   * @returns Unobserve
    */
-  public setViewport = (
+  setViewport = (
     viewportElement: HTMLElement,
     options?: Pick<IntersectionObserverInit, 'threshold' | 'rootMargin'>,
   ) => {
@@ -81,12 +87,8 @@ export class ViewportTracker {
    * Method to set element to observe and handler to be triggered whenever IntersectionObserver
    * detects a possible change in element's visibility within specified viewport, returns
    * cleanup function to be invoked upon disposing of the DOM element to prevent memory leaks
-   *
-   * @param element
-   * @param handler
-   * @returns Unobserve
    */
-  public observe: Observe = (element, handler) => {
+  observe: Observe = (element, handler) => {
     const queueItem = [element, handler] as const;
 
     const cleanup = () => {
@@ -109,4 +111,57 @@ export class ViewportTracker {
 
     return cleanup;
   };
+
+  /**
+   * Tracks the given element for visibility changes and mirrors the result
+   * into `participant.viewportVisibilityState[trackType]` in `CallState`.
+   * Returns a function that unobserves the element and resets the visibility
+   * state back to `UNKNOWN`.
+   */
+  trackElementVisibility = <T extends HTMLElement>(
+    element: T,
+    sessionId: string,
+    trackType: VideoTrackType,
+  ) => {
+    const cleanup = this.observe(element, (entry) => {
+      this.callState.updateParticipant(sessionId, (participant) => {
+        const previousVisibilityState =
+          participant.viewportVisibilityState ??
+          DEFAULT_VIEWPORT_VISIBILITY_STATE;
+
+        // observer triggers when the element is "moved" to be a fullscreen element
+        // keep it VISIBLE if that happens to prevent fullscreen with placeholder
+        const isVisible =
+          entry.isIntersecting || document.fullscreenElement === element
+            ? VisibilityState.VISIBLE
+            : VisibilityState.INVISIBLE;
+        return {
+          ...participant,
+          viewportVisibilityState: {
+            ...previousVisibilityState,
+            [trackType]: isVisible,
+          },
+        };
+      });
+    });
+
+    return () => {
+      cleanup();
+      // reset visibility state to UNKNOWN upon cleanup
+      // so that the layouts that are not actively observed
+      // can still function normally (runtime layout switching)
+      this.callState.updateParticipant(sessionId, (participant) => {
+        const previousVisibilityState =
+          participant.viewportVisibilityState ??
+          DEFAULT_VIEWPORT_VISIBILITY_STATE;
+        return {
+          ...participant,
+          viewportVisibilityState: {
+            ...previousVisibilityState,
+            [trackType]: VisibilityState.UNKNOWN,
+          },
+        };
+      });
+    };
+  };
 }

package/src/helpers/__tests__/BlockedAudioTracker.test.ts

@@ -0,0 +1,114 @@
+/**
+ * @vitest-environment happy-dom
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { BlockedAudioTracker } from '../BlockedAudioTracker';
+import { getCurrentValue } from '../../store/rxUtils';
+import type { Tracer } from '../../stats';
+
+const createTracer = () =>
+  ({
+    trace: vi.fn(),
+  }) as unknown as Tracer;
+
+const createAudioElement = (withSrcObject = true) => {
+  const el = document.createElement('audio');
+  Object.defineProperty(el, 'srcObject', { writable: true });
+  if (withSrcObject) {
+    (el as HTMLAudioElement).srcObject = new MediaStream();
+  }
+  return el;
+};
+
+describe('BlockedAudioTracker', () => {
+  let tracer: Tracer;
+  let tracker: BlockedAudioTracker;
+
+  beforeEach(() => {
+    tracer = createTracer();
+    tracker = new BlockedAudioTracker(tracer);
+  });
+
+  it('emits autoplayBlocked$ true after markBlocked(el, true)', () => {
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
+    tracker.markBlocked(createAudioElement(), true);
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(true);
+  });
+
+  it('emits autoplayBlocked$ false after the last element is unmarked', () => {
+    const el1 = createAudioElement();
+    const el2 = createAudioElement();
+    tracker.markBlocked(el1, true);
+    tracker.markBlocked(el2, true);
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(true);
+
+    tracker.markBlocked(el1, false);
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(true);
+
+    tracker.markBlocked(el2, false);
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
+  });
+
+  it('is idempotent: marking the same element twice keeps a single entry', () => {
+    const el = createAudioElement();
+    tracker.markBlocked(el, true);
+    tracker.markBlocked(el, true);
+    tracker.markBlocked(el, false);
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
+  });
+
+  it('resumeAudio plays each element with srcObject and clears successful ones', async () => {
+    const el1 = createAudioElement();
+    const el2 = createAudioElement();
+    const play1 = vi.spyOn(el1, 'play').mockResolvedValue();
+    const play2 = vi.spyOn(el2, 'play').mockResolvedValue();
+
+    tracker.markBlocked(el1, true);
+    tracker.markBlocked(el2, true);
+
+    await tracker.resumeAudio();
+
+    expect(play1).toHaveBeenCalled();
+    expect(play2).toHaveBeenCalled();
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
+  });
+
+  it('resumeAudio keeps elements whose play() still rejects', async () => {
+    const ok = createAudioElement();
+    const stillBlocked = createAudioElement();
+    vi.spyOn(ok, 'play').mockResolvedValue();
+    vi.spyOn(stillBlocked, 'play').mockRejectedValue(
+      new DOMException('', 'NotAllowedError'),
+    );
+
+    tracker.markBlocked(ok, true);
+    tracker.markBlocked(stillBlocked, true);
+
+    await tracker.resumeAudio();
+
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(true);
+
+    // Resuming again with the now-cooperative element should clear the flag.
+    vi.spyOn(stillBlocked, 'play').mockResolvedValue();
+    await tracker.resumeAudio();
+
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
+  });
+
+  it('resumeAudio drops elements without srcObject without calling play()', async () => {
+    const detached = createAudioElement(false);
+    const play = vi.spyOn(detached, 'play').mockResolvedValue();
+
+    tracker.markBlocked(detached, true);
+    await tracker.resumeAudio();
+
+    expect(play).not.toHaveBeenCalled();
+    expect(getCurrentValue(tracker.autoplayBlocked$)).toBe(false);
+  });
+
+  it('traces resumeAudio', async () => {
+    await tracker.resumeAudio();
+    expect(tracer.trace).toHaveBeenCalledWith('resumeAudio', null);
+  });
+});