livekit-client 2.22.2 → 2.22.4

package/src/room/track/RemoteVideoTrack.test.ts

@@ -98,6 +98,216 @@ describe('RemoteVideoTrack', () => {
   });
 });
 
+describe('Document Picture-in-Picture detection', () => {
+  /**
+   * happy-dom's MediaStream implements everything except getTracks(), which Track.attach()
+   * reads. Subclassing keeps it assignable to the element's srcObject, which type checks the
+   * value against happy-dom's own MediaStream.
+   */
+  class TestMediaStream extends MediaStream {
+    getTracks() {
+      return [...this.getAudioTracks(), ...this.getVideoTracks()];
+    }
+  }
+
+  /** Stands in for window.documentPictureInPicture, which happy-dom does not implement. */
+  class FakeDocumentPictureInPicture extends EventTarget {
+    window?: Window;
+  }
+
+  let documentPiP: FakeDocumentPictureInPicture;
+
+  /** Gives an element the layout box that isElementInViewport reads. */
+  function placeElement(
+    el: HTMLElement,
+    { top, left, width, height }: { top: number; left: number; width: number; height: number },
+  ) {
+    Object.defineProperties(el, {
+      offsetTop: { value: top, configurable: true },
+      offsetLeft: { value: left, configurable: true },
+      offsetWidth: { value: width, configurable: true },
+      offsetHeight: { value: height, configurable: true },
+      offsetParent: { value: null, configurable: true },
+    });
+  }
+
+  function createVideo(
+    ownerDocument: Document,
+    box = { top: 40, left: 40, width: 320, height: 180 },
+  ) {
+    const video = ownerDocument.createElement('video');
+    ownerDocument.body.appendChild(video);
+    placeElement(video, box);
+    return video;
+  }
+
+  /**
+   * Creates a frame in the given document and returns its document. happy-dom implements
+   * neither frameElement nor a correct top on a frame's window, so both are wired up here.
+   */
+  function createFrame(parentDocument: Document, topWindow: Window) {
+    const frame = parentDocument.createElement('iframe');
+    parentDocument.body.appendChild(frame);
+    const frameDocument = frame.contentDocument!;
+    Object.defineProperties(frameDocument.defaultView!, {
+      frameElement: { value: frame, configurable: true },
+      top: { value: topWindow, configurable: true },
+    });
+    return frameDocument;
+  }
+
+  /**
+   * Makes a frame document look like one reached through a cross-origin frame: the containing
+   * frame belongs to another origin, so frameElement reads null while top stays readable.
+   */
+  function hideContainingFrame(frameDocument: Document) {
+    Object.defineProperty(frameDocument.defaultView!, 'frameElement', {
+      value: null,
+      configurable: true,
+    });
+    return frameDocument;
+  }
+
+  /** Opens a Document PiP window and fires the 'enter' event the SDK listens for. */
+  function openPiPWindow({ width, height } = { width: 1280, height: 820 }) {
+    const pipDocument = document.implementation.createHTMLDocument('pip');
+    documentPiP.window = Object.assign(new EventTarget(), {
+      document: pipDocument,
+      innerWidth: width,
+      innerHeight: height,
+      pageXOffset: 0,
+      pageYOffset: 0,
+    }) as unknown as Window;
+    documentPiP.dispatchEvent(new Event('enter'));
+    return pipDocument;
+  }
+
+  /** Attaches a fresh track to the element and records every VisibilityChanged emission. */
+  function attachTrack(video: HTMLElement) {
+    const track = new RemoteVideoTrack(new MockMediaStreamTrack(), 'sid', undefined!, {});
+    const events: boolean[] = [];
+    track.on(TrackEvent.VisibilityChanged, (visible) => events.push(visible));
+    track.attach(video as HTMLMediaElement);
+    return events;
+  }
+
+  /** Flushes the microtask + animation frame that onEnterPiP defers its re-check behind. */
+  async function flushDeferredPiPCheck() {
+    await vi.advanceTimersByTimeAsync(50);
+  }
+
+  function setOpenerHidden() {
+    Object.defineProperty(document, 'visibilityState', { value: 'hidden', configurable: true });
+  }
+
+  beforeEach(() => {
+    vi.stubGlobal('MediaStream', TestMediaStream);
+    documentPiP = new FakeDocumentPictureInPicture();
+    (window as any).documentPictureInPicture = documentPiP;
+    // The opener tab is backgrounded in every case, so pauseVideoInBackground (on by default)
+    // reports the element hidden unless it is genuinely in PiP.
+    setOpenerHidden();
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    delete (window as any).documentPictureInPicture;
+    Object.defineProperty(document, 'visibilityState', { value: 'visible', configurable: true });
+    document.body.innerHTML = '';
+  });
+
+  it('does not report an opener-document element as being in PiP', () => {
+    openPiPWindow();
+    const video = createVideo(document);
+
+    // The top-window check must not open a hole here: the opener's top is the opener itself.
+    expect(window.top).toBe(window);
+    expect(window.top).not.toBe(documentPiP.window);
+    // The element sits at (40, 40) of the OPENER page, which falls inside the PiP window's
+    // 1280x820 box. It is not in PiP, so the background pause applies.
+    expect(attachTrack(video)).toEqual([false]);
+  });
+
+  it('does not report an element in an opener frame as being in PiP', () => {
+    openPiPWindow();
+    const openerFrame = createFrame(document, window);
+    const video = createVideo(openerFrame);
+
+    // An opener frame tops out at the opener, so neither the top check nor the walk matches.
+    expect(openerFrame.defaultView!.top).toBe(window);
+    expect(attachTrack(video)).toEqual([false]);
+  });
+
+  it('reports an element moved into the PiP document after the enter event', async () => {
+    const video = createVideo(document);
+    const events = attachTrack(video);
+    expect(events).toEqual([false]);
+
+    const pipDocument = openPiPWindow();
+    // The SDK defers its re-check behind a microtask and an animation frame precisely so the
+    // embedder can move its subtree across after the enter event fires.
+    pipDocument.body.appendChild(video);
+    expect(video.ownerDocument).toBe(pipDocument);
+    await flushDeferredPiPCheck();
+
+    expect(events).toEqual([false, true]);
+  });
+
+  it('leaves an unrelated opener element hidden when another element enters PiP', async () => {
+    const openerVideo = createVideo(document);
+    const movedVideo = createVideo(document);
+    const openerEvents = attachTrack(openerVideo);
+    const movedEvents = attachTrack(movedVideo);
+
+    const pipDocument = openPiPWindow();
+    pipDocument.body.appendChild(movedVideo);
+    await flushDeferredPiPCheck();
+
+    expect(movedEvents).toEqual([false, true]);
+    expect(openerEvents).toEqual([false]);
+  });
+
+  it('reports an element inside a frame nested in the PiP document', () => {
+    const pipDocument = openPiPWindow();
+    const video = createVideo(createFrame(pipDocument, documentPiP.window!), {
+      top: 10,
+      left: 10,
+      width: 320,
+      height: 180,
+    });
+
+    expect(attachTrack(video)).toEqual([true]);
+  });
+
+  it('reports an element behind a cross-origin frame nested in the PiP window', () => {
+    const pipDocument = openPiPWindow();
+    const pipWindow = documentPiP.window!;
+    // PiP document (origin A) -> frame (origin B) -> frame (origin A). The opener can reach
+    // the innermost document, but its containing frame belongs to B, so frameElement is null
+    // there and the same-origin walk cannot get back to the PiP document.
+    const crossOriginFrame = createFrame(pipDocument, pipWindow);
+    const innerFrame = hideContainingFrame(createFrame(crossOriginFrame, pipWindow));
+    const video = createVideo(innerFrame, { top: 10, left: 10, width: 320, height: 180 });
+
+    expect(innerFrame.defaultView!.frameElement).toBeNull();
+    expect(innerFrame.defaultView!.top).toBe(pipWindow);
+    expect(attachTrack(video)).toEqual([true]);
+  });
+
+  it('does not report an element scrolled out of the PiP window viewport', () => {
+    const pipDocument = openPiPWindow();
+    const video = createVideo(pipDocument, { top: 2000, left: 40, width: 320, height: 180 });
+
+    expect(attachTrack(video)).toEqual([false]);
+  });
+
+  it('does not report an element as being in PiP when no PiP window is open', () => {
+    const video = createVideo(document);
+
+    expect(attachTrack(video)).toEqual([false]);
+  });
+});
+
 class MockElementInfo implements ElementInfo {
   element: object = {};
 

package/src/room/track/RemoteVideoTrack.ts

@@ -435,9 +435,35 @@ class HTMLElementInfo implements ElementInfo {
 function isElementInPiP(el: HTMLElement) {
   // Simple video PiP
   if (document.pictureInPictureElement === el) return true;
-  // Document PiP
-  if (window.documentPictureInPicture?.window)
-    return isElementInViewport(el, window.documentPictureInPicture?.window);
+  // Document PiP: the element has to actually live in the PiP window, otherwise an
+  // opener-document element whose coordinates happen to fall inside the PiP window's
+  // bounds is reported as being in PiP.
+  const pipWindow = window.documentPictureInPicture?.window;
+  if (pipWindow) {
+    return isDocumentInPiPWindow(el.ownerDocument, pipWindow) && isElementInViewport(el, pipWindow);
+  }
+  return false;
+}
+
+// Whether the document is the Document PiP window's own document, or the document of a frame
+// nested inside it. The spec allows frames in a PiP window, so an element in one is in PiP:
+// https://wicg.github.io/document-picture-in-picture/#iframes
+function isDocumentInPiPWindow(doc: Document, pipWindow: Window) {
+  // A cross-origin frame anywhere between the element and the PiP window hides frameElement
+  // from the walk below, but top stays readable across origins and identifies the top-level
+  // window the element belongs to. For an opener element that is the opener, not the PiP window.
+  if (doc.defaultView?.top === pipWindow) return true;
+  let current: Document | null | undefined = doc;
+  while (current) {
+    if (current === pipWindow.document) return true;
+    try {
+      // Reading frameElement across a cross-origin boundary throws, which means we cannot
+      // establish the ancestry and must not claim the element is in PiP.
+      current = current.defaultView?.frameElement?.ownerDocument;
+    } catch {
+      return false;
+    }
+  }
   return false;
 }
 

package/src/room/track/options.ts

@@ -79,7 +79,7 @@ export interface TrackPublishDefaults {
 
   /**
    * scalability mode for svc codecs, defaults to 'L3T3_KEY'.
-   * for svc codecs, simulcast is disabled.
+   * for svc codecs, simulcast is disabled if more than one spatial layer is used ('L2Tx' or 'L3Tx').
    */
   scalabilityMode?: ScalabilityMode;
 

package/src/room/track/utils.test.ts

@@ -1,7 +1,12 @@
-import { describe, expect, it } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { audioDefaults, videoDefaults } from '../defaults';
 import { type AudioCaptureOptions, VideoPresets } from './options';
-import { constraintsForOptions, diffAttributes, mergeDefaultOptions } from './utils';
+import {
+  constraintsForOptions,
+  diffAttributes,
+  mergeDefaultOptions,
+  waitForFirstVideoFrame,
+} from './utils';
 
 describe('mergeDefaultOptions', () => {
   it('does not enable undefined options', () => {
@@ -135,3 +140,197 @@ describe('diffAttributes', () => {
     expect(diff.a).toBe('value');
   });
 });
+
+class FakeVideoTrack extends EventTarget {
+  kind = 'video';
+
+  id = 'fake-video-track';
+
+  label = 'fake';
+
+  enabled = true;
+
+  muted = false;
+
+  readyState: MediaStreamTrackState = 'live';
+
+  settings: MediaTrackSettings;
+
+  constructor(settings: MediaTrackSettings = { width: 1280, height: 720 }) {
+    super();
+    this.settings = settings;
+  }
+
+  getSettings() {
+    return this.settings;
+  }
+
+  getConstraints() {
+    return {};
+  }
+}
+
+const asMediaStreamTrack = (track: FakeVideoTrack) => track as unknown as MediaStreamTrack;
+
+describe('waitForFirstVideoFrame', () => {
+  let frameCallbacks: Map<number, VideoFrameRequestCallback>;
+  let cancelledHandles: number[];
+  let createdVideoElements: HTMLVideoElement[];
+
+  /** happy-dom has no requestVideoFrameCallback, install a controllable one */
+  function enableFrameCallbacks() {
+    let nextHandle = 1;
+    HTMLVideoElement.prototype.requestVideoFrameCallback = function requestVideoFrameCallback(
+      callback: VideoFrameRequestCallback,
+    ) {
+      const handle = nextHandle;
+      nextHandle += 1;
+      frameCallbacks.set(handle, callback);
+      return handle;
+    };
+    HTMLVideoElement.prototype.cancelVideoFrameCallback = function cancelVideoFrameCallback(
+      handle: number,
+    ) {
+      cancelledHandles.push(handle);
+      frameCallbacks.delete(handle);
+    };
+  }
+
+  function deliverFrame() {
+    Array.from(frameCallbacks.values()).forEach((callback) =>
+      callback(0, {} as VideoFrameCallbackMetadata),
+    );
+  }
+
+  beforeEach(() => {
+    frameCallbacks = new Map();
+    cancelledHandles = [];
+    createdVideoElements = [];
+    const createElement = document.createElement.bind(document);
+    vi.spyOn(document, 'createElement').mockImplementation(((
+      tagName: string,
+      options?: ElementCreationOptions,
+    ) => {
+      const element = createElement(tagName, options);
+      if (tagName === 'video') {
+        createdVideoElements.push(element as HTMLVideoElement);
+      }
+      return element;
+    }) as typeof document.createElement);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    // @ts-expect-error removing the stand-in we installed above
+    delete HTMLVideoElement.prototype.requestVideoFrameCallback;
+    // @ts-expect-error removing the stand-in we installed above
+    delete HTMLVideoElement.prototype.cancelVideoFrameCallback;
+  });
+
+  it('resolves once a frame has been delivered', async () => {
+    enableFrameCallbacks();
+    const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
+    expect(createdVideoElements).toHaveLength(1);
+
+    deliverFrame();
+
+    await expect(pending).resolves.toBe(true);
+  });
+
+  it('gives up once the timeout elapses', async () => {
+    enableFrameCallbacks();
+
+    await expect(
+      waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 20),
+    ).resolves.toBe(false);
+  });
+
+  it.each([
+    ['ended', { readyState: 'ended' as MediaStreamTrackState }],
+    ['muted', { muted: true }],
+    ['disabled', { enabled: false }],
+  ])('does not wait on a track that is %s', async (_label, overrides) => {
+    enableFrameCallbacks();
+    const track = Object.assign(new FakeVideoTrack(), overrides);
+
+    await expect(waitForFirstVideoFrame(asMediaStreamTrack(track), 10_000)).resolves.toBe(false);
+    expect(createdVideoElements).toHaveLength(0);
+  });
+
+  it('falls back to loadeddata when requestVideoFrameCallback is unavailable', async () => {
+    const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
+
+    createdVideoElements[0].dispatchEvent(new Event('loadeddata'));
+
+    await expect(pending).resolves.toBe(true);
+  });
+
+  it('keeps waiting when play() is rejected', async () => {
+    enableFrameCallbacks();
+    vi.spyOn(HTMLMediaElement.prototype, 'play').mockRejectedValue(new Error('NotAllowedError'));
+    const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
+
+    deliverFrame();
+
+    await expect(pending).resolves.toBe(true);
+  });
+
+  it('reuses an element that already renders the track', async () => {
+    enableFrameCallbacks();
+    const track = new FakeVideoTrack();
+    const attached = document.createElement('video');
+    attached.srcObject = new MediaStream([asMediaStreamTrack(track)]);
+    createdVideoElements.length = 0;
+
+    const pending = waitForFirstVideoFrame(asMediaStreamTrack(track), 1000, [attached]);
+    expect(createdVideoElements).toHaveLength(0);
+    deliverFrame();
+
+    await expect(pending).resolves.toBe(true);
+    // the caller owns that element, it must be left rendering
+    expect(attached.srcObject).not.toBeNull();
+  });
+
+  it('reuses an element that has not decoded a frame yet without requestVideoFrameCallback', async () => {
+    const track = new FakeVideoTrack();
+    const attached = document.createElement('video');
+    attached.srcObject = new MediaStream([asMediaStreamTrack(track)]);
+    Object.defineProperty(attached, 'readyState', { value: 1, configurable: true });
+    createdVideoElements.length = 0;
+
+    const pending = waitForFirstVideoFrame(asMediaStreamTrack(track), 1000, [attached]);
+    expect(createdVideoElements).toHaveLength(0);
+    // `loadeddata` has not fired on that element yet, so listening for it still works
+    attached.dispatchEvent(new Event('loadeddata'));
+
+    await expect(pending).resolves.toBe(true);
+  });
+
+  it('declines an already loaded element without requestVideoFrameCallback', async () => {
+    const track = new FakeVideoTrack();
+    const attached = document.createElement('video');
+    attached.srcObject = new MediaStream([asMediaStreamTrack(track)]);
+    Object.defineProperty(attached, 'readyState', { value: 2, configurable: true });
+    createdVideoElements.length = 0;
+
+    const pending = waitForFirstVideoFrame(asMediaStreamTrack(track), 1000, [attached]);
+    // `loadeddata` already fired on that element and is never replayed, so we need our own
+    expect(createdVideoElements).toHaveLength(1);
+    createdVideoElements[0].dispatchEvent(new Event('loadeddata'));
+
+    await expect(pending).resolves.toBe(true);
+  });
+
+  it('tears down the element it created', async () => {
+    enableFrameCallbacks();
+    const pause = vi.spyOn(HTMLMediaElement.prototype, 'pause');
+    const pending = waitForFirstVideoFrame(asMediaStreamTrack(new FakeVideoTrack()), 1000);
+
+    deliverFrame();
+    await pending;
+
+    expect(createdVideoElements[0].srcObject).toBeNull();
+    expect(pause).toHaveBeenCalled();
+    expect(cancelledHandles).toHaveLength(1);
+  });
+});

package/src/room/track/utils.ts

@@ -1,5 +1,6 @@
 import { TrackInfo, TrackPublishedResponse, TrackSource, VideoQuality } from '@livekit/protocol';
 import type { AudioProcessorOptions, TrackProcessor, VideoProcessorOptions } from '../..';
+import log from '../../logger';
 import { cloneDeep } from '../../utils/cloneDeep';
 import { isSafari, sleep } from '../utils';
 import { Track } from './Track';
@@ -351,3 +352,90 @@ export function layerDimensionsFor(
 ): Track.Dimensions | undefined {
   return trackInfo.layers?.find((l) => l.quality === quality);
 }
+
+/**
+ * Waits until `track` has actually produced a video frame.
+ *
+ * iOS keeps reporting the camera's sensor frame from `getSettings()` until the camera has
+ * delivered its first picture, so a portrait capture initially reads as landscape. The arrival
+ * of that first frame is the only reliable signal that the reported settings describe reality.
+ *
+ * Best effort, never throws. Resolves `true` when a frame was observed, `false` when no frame
+ * can be observed (the track is already ended, muted or disabled) or `timeoutMs` elapsed first.
+ *
+ * @internal
+ */
+export async function waitForFirstVideoFrame(
+  track: MediaStreamTrack,
+  timeoutMs: number,
+  candidateElements: HTMLMediaElement[] = [],
+): Promise<boolean> {
+  if (typeof document === 'undefined') {
+    return false;
+  }
+  if (track.readyState === 'ended' || track.muted || !track.enabled) {
+    // no frame is ever going to arrive, don't hold the caller up for the full timeout
+    return false;
+  }
+
+  // requestVideoFrameCallback (Safari 15.4 and up) is a platform capability, so the answer is the
+  // same for every element
+  const supportsFrameCallback =
+    typeof HTMLVideoElement.prototype.requestVideoFrameCallback === 'function';
+
+  // prefer an element that is already rendering this track (a local preview, typically) over
+  // spinning up a second decode just to observe a frame
+  const existingElement = candidateElements.find(
+    (el): el is HTMLVideoElement =>
+      el instanceof HTMLVideoElement &&
+      el.srcObject instanceof MediaStream &&
+      el.srcObject.getVideoTracks().includes(track) &&
+      // without requestVideoFrameCallback we depend on `loadeddata`, which is one-shot: an element
+      // that already decoded a frame (readyState >= HAVE_CURRENT_DATA) never fires it again.
+      // Leave those alone and set up our own element, which starts out empty.
+      (supportsFrameCallback || el.readyState < 2),
+  );
+
+  const element = existingElement ?? document.createElement('video');
+  if (!existingElement) {
+    element.playsInline = true;
+    element.muted = true;
+    element.srcObject = new MediaStream([track]);
+    // the element is deliberately never added to the DOM, it only exists to pull frames.
+    // WebKit delivers frames to a paused element too, so a rejected play() (Safari's low power
+    // mode, for instance) is not a reason to stop waiting
+    element.play().catch((error) => {
+      log.debug('could not play element while waiting for the first video frame', { error });
+    });
+  }
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let frameHandle: number | undefined;
+  let onLoadedData: (() => void) | undefined;
+
+  try {
+    return await new Promise<boolean>((resolve) => {
+      timer = setTimeout(() => resolve(false), timeoutMs);
+      if (supportsFrameCallback) {
+        frameHandle = element.requestVideoFrameCallback(() => resolve(true));
+      } else {
+        // Safari below 15.4 has no requestVideoFrameCallback. `loadeddata` fires once the first
+        // frame has been decoded, which is the same signal a few milliseconds earlier.
+        onLoadedData = () => resolve(true);
+        element.addEventListener('loadeddata', onLoadedData);
+      }
+    });
+  } finally {
+    clearTimeout(timer);
+    if (frameHandle !== undefined) {
+      element.cancelVideoFrameCallback?.(frameHandle);
+    }
+    if (onLoadedData) {
+      element.removeEventListener('loadeddata', onLoadedData);
+    }
+    if (!existingElement) {
+      element.pause();
+      element.srcObject = null;
+    }
+  }
+}