livekit-client 2.22.3 → 2.23.0

package/src/room/track/LocalTrack.ts

@@ -5,12 +5,20 @@ import { debounce } from '../debounce';
 import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
 import { TrackEvent } from '../events';
 import type { LoggerOptions } from '../types';
-import { compareVersions, isMobile, sleep, unwrapConstraint } from '../utils';
+import {
+  compareVersions,
+  isAppleMobile,
+  isMobile,
+  markDeviceAcquisitionFailure,
+  sleep,
+  unwrapConstraint,
+} from '../utils';
 import { Track, attachToElement, detachTrack } from './Track';
 import type { VideoCodec } from './options';
 import type { TrackProcessor } from './processor/types';
 import { LocalTrackRecorder, isRecordingSupported } from './record';
 import type { ReplaceTrackOptions } from './types';
+import { waitForFirstVideoFrame } from './utils';
 
 const DEFAULT_DIMENSIONS_TIMEOUT = 1000;
 const PRE_CONNECT_BUFFER_TIMEOUT = 10_000;
@@ -131,6 +139,10 @@ export default abstract class LocalTrack<
     return this.providedByUser;
   }
 
+  get isStopped() {
+    return this.manuallyStopped;
+  }
+
   get mediaStreamTrack() {
     return this.processor?.processedTrack ?? this._mediaStreamTrack;
   }
@@ -194,6 +206,9 @@ export default abstract class LocalTrack<
         track: newTrack,
         kind: this.kind,
         element: this.processorElement,
+        // audio processors need the context they were initialised with in order to rebuild
+        // their graph, omitting it here would leave them without one
+        audioContext: this.audioContext,
         localTrack: this,
       });
       processedTrack = this.processor.processedTrack;
@@ -217,6 +232,17 @@ export default abstract class LocalTrack<
         attachToElement(processedTrack ?? newTrack, el);
       });
     }
+    await this.onMediaStreamTrackChanged();
+  }
+
+  /**
+   * Hook invoked after the underlying MediaStreamTrack has been swapped and the processor has
+   * been restarted around it, so `processor.processedTrack` already holds the new track.
+   * Unlike `onSenderTrackSwapped` this also fires on the restart and unmute paths.
+   */
+  protected async onMediaStreamTrackChanged(): Promise<void> {
+    // base implementation is a no-op; LocalAudioTrack overrides this to move the krisp
+    // feature listeners onto the new processed track.
   }
 
   async waitForDimensions(timeout = DEFAULT_DIMENSIONS_TIMEOUT): Promise<Track.Dimensions> {
@@ -224,18 +250,28 @@ export default abstract class LocalTrack<
       throw new Error('cannot get dimensions for audio tracks');
     }
 
-    if (getBrowser()?.os === 'iOS') {
-      // browsers report wrong initial resolution on iOS.
-      // when slightly delaying the call to .getSettings(), the correct resolution is being reported
-      await sleep(10);
+    const started = Date.now();
+
+    if (isAppleMobile()) {
+      // iOS and iPadOS keep reporting the camera's sensor frame from getSettings() until the
+      // camera has delivered its first picture, so a portrait capture initially reads as
+      // landscape. Wait for that first frame instead of guessing at a delay.
+      // Gated to the Apple capture pipeline rather than run everywhere: elsewhere getSettings() is
+      // already right and the first frame can be hundreds of milliseconds out (measured ~500ms on
+      // macOS Safari), which would be pure added publish latency.
+      // https://github.com/livekit/client-sdk-js/issues/2099
+      // https://github.com/livekit/client-sdk-js/issues/2107
+      await waitForFirstVideoFrame(this._mediaStreamTrack, timeout, this.attachedElements);
     }
 
-    const started = Date.now();
-    while (Date.now() - started < timeout) {
+    while (true) {
       const dims = this.dimensions;
       if (dims) {
         return dims;
       }
+      if (Date.now() - started >= timeout) {
+        break;
+      }
       await sleep(50);
     }
     throw new TrackInvalidError('unable to get track dimensions after timeout');
@@ -244,7 +280,8 @@ export default abstract class LocalTrack<
   async setDeviceId(deviceId: ConstrainDOMString): Promise<boolean> {
     if (
       this._constraints.deviceId === deviceId &&
-      this._mediaStreamTrack.getSettings().deviceId === unwrapConstraint(deviceId)
+      this._mediaStreamTrack.getSettings().deviceId === unwrapConstraint(deviceId) &&
+      this._mediaStreamTrack.readyState === 'live'
     ) {
       return true;
     }
@@ -375,19 +412,37 @@ export default abstract class LocalTrack<
       this._mediaStreamTrack.stop();
 
       // create new track and attach
-      const mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
+      let mediaStream: MediaStream;
+      try {
+        mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
+      } catch (e) {
+        markDeviceAcquisitionFailure(e);
+        throw e;
+      }
       const newTrack = mediaStream.getTracks()[0];
-      if (this.kind === Track.Kind.Video) {
-        // we already captured the audio track with the constraints, so we only need to apply the video constraints
-        await newTrack.applyConstraints(otherConstraints);
+      try {
+        if (this.kind === Track.Kind.Video) {
+          // we already captured the audio track with the constraints, so we only need to apply the video constraints
+          await newTrack.applyConstraints(otherConstraints);
+        }
+        newTrack.addEventListener('ended', this.handleEnded);
+        this.log.debug('re-acquired MediaStreamTrack', this.logContext);
+
+        await this.setMediaStreamTrack(newTrack, false, isUnmuting);
+      } catch (e) {
+        // setMediaStreamTrack can throw after acquisition but before it adopts the new track,
+        // which would leave the capture device live with nothing referencing it
+        if (this._mediaStreamTrack !== newTrack) {
+          newTrack.stop();
+        }
+        throw e;
       }
-      newTrack.addEventListener('ended', this.handleEnded);
-      this.log.debug('re-acquired MediaStreamTrack', this.logContext);
-
-      await this.setMediaStreamTrack(newTrack, false, isUnmuting);
       this._constraints = constraints;
       this.pendingDeviceChange = false;
       this.emit(TrackEvent.Restarted, this);
+      return this;
+    } finally {
+      // in the finally because a throw can also leave the new track adopted, and so still live
       if (this.manuallyStopped) {
         this.log.warn(
           'track was stopped during a restart, stopping restarted track',
@@ -395,8 +450,6 @@ export default abstract class LocalTrack<
         );
         this.stop();
       }
-      return this;
-    } finally {
       unlock();
     }
   }
@@ -464,7 +517,11 @@ export default abstract class LocalTrack<
     this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
     this._mediaStreamTrack.removeEventListener('mute', this.handleTrackMuteEvent);
     this._mediaStreamTrack.removeEventListener('unmute', this.handleTrackUnmuteEvent);
-    this.processor?.destroy();
+    // `stop` is synchronous, so we can't await the teardown of the processor here.
+    // make sure a failing teardown doesn't surface as an unhandled rejection.
+    this.processor?.destroy().catch((error) => {
+      this.log.error('failed to destroy processor', { ...this.logContext, error });
+    });
     this.processor = undefined;
   }
 
@@ -540,6 +597,11 @@ export default abstract class LocalTrack<
    * Sets a processor on this track.
    * See https://github.com/livekit/track-processors-js for example usage
    *
+   * The processor lives on the track, so stopping the track destroys it. Unpublishing and
+   * disconnecting both stop the track by default, which means a processor has to be set again on
+   * the track published after a reconnect. See {@link TrackProcessor} for the full lifecycle,
+   * including what `stopLocalTrackOnUnpublish: false` changes.
+   *
    * @param processor
    * @param showProcessedStreamLocally
    * @returns
@@ -606,6 +668,10 @@ export default abstract class LocalTrack<
     await this.onSenderTrackSwapped();
   }
 
+  /**
+   * Returns the processor currently set on this track, or `undefined` if there is none. A track
+   * that was stopped by an unpublish or a disconnect has none. See {@link TrackProcessor}.
+   */
   getProcessor() {
     return this.processor;
   }
@@ -614,6 +680,8 @@ export default abstract class LocalTrack<
    * Stops the track processor
    * See https://github.com/livekit/track-processors-js for example usage
    *
+   * Only needed to remove a processor from a track that stays published. Unpublishing or
+   * disconnecting already destroys it. See {@link TrackProcessor}.
    */
   async stopProcessor(keepElement = true) {
     const unlock = await this.trackChangeLock.lock();

package/src/room/track/LocalVideoTrack.ts

@@ -401,6 +401,13 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     await sender.setParameters(params);
   }
 
+  /**
+   * Sets a processor on this track, and wires its processed track into the simulcast codecs.
+   *
+   * The processor is destroyed when the track is stopped, which unpublishing and disconnecting
+   * both do by default, so it has to be set again after reconnecting. See {@link TrackProcessor}
+   * for the full lifecycle.
+   */
   async setProcessor(
     processor: TrackProcessor<Track.Kind.Video>,
     showProcessedStreamLocally = true,
@@ -482,12 +489,19 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     // browser will reenable disabled codec/layers after new codec has been published,
     // so refresh subscribedCodecs after publish a new codec
     setTimeout(() => {
-      if (this.subscribedCodecs) {
-        this.setPublishingCodecs(this.subscribedCodecs);
-      }
+      this.refreshSubscribedCodecs();
     }, refreshSubscribedCodecAfterNewCodec);
   }
 
+  /** @interal */
+  async refreshSubscribedCodecs() {
+    if (this.subscribedCodecs) {
+      return this.setPublishingCodecs(this.subscribedCodecs).catch((e) =>
+        this.log.warn(`failed to refresh codec for track`, { ...this.logContext, error: e }),
+      );
+    }
+  }
+
   /**
    * @internal
    * Sets codecs that should be publishing, returns new codecs that have not yet

package/src/room/track/RemoteAudioTrack.ts

@@ -20,6 +20,13 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
 
   private webAudioPluginNodes: AudioNode[];
 
+  /**
+   * the nodes of the graph currently wired up, in order, so teardown can undo exactly the edges
+   * `connectWebAudio` created. Reading `webAudioPluginNodes` instead would miss the old plugins
+   * once `setWebAudioPlugins` has replaced them.
+   */
+  private connectedNodes: AudioNode[] = [];
+
   private sinkId?: string;
 
   constructor(
@@ -42,10 +49,11 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
    * sets the volume for all attached audio elements
    */
   setVolume(volume: number) {
-    for (const el of this.attachedElements) {
-      if (this.audioContext) {
-        this.gainNode?.gain.setTargetAtTime(volume, 0, 0.1);
-      } else {
+    if (this.audioContext) {
+      // playback goes through the web audio graph, the attached elements themselves stay silent
+      this.gainNode?.gain.setTargetAtTime(volume, 0, 0.1);
+    } else {
+      for (const el of this.attachedElements) {
         el.volume = volume;
       }
     }
@@ -60,7 +68,7 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
    * gets the volume of attached audio elements (loudest)
    */
   getVolume(): number {
-    if (this.elementVolume) {
+    if (this.elementVolume !== undefined) {
       return this.elementVolume;
     }
     if (isReactNative()) {
@@ -96,7 +104,9 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
   attach(): HTMLMediaElement;
   attach(element: HTMLMediaElement): HTMLMediaElement;
   attach(element?: HTMLMediaElement): HTMLMediaElement {
-    const needsNewWebAudioConnection = this.attachedElements.length === 0;
+    // the graph is fed by the track rather than by a specific element, so one connection covers
+    // all attached elements
+    const needsNewWebAudioConnection = !this.sourceNode;
     if (!element) {
       element = super.attach();
     } else {
@@ -108,14 +118,18 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
         this.log.error('Failed to set sink id on remote audio track', e, this.logContext);
       });
     }
-    if (this.audioContext && needsNewWebAudioConnection) {
-      this.log.debug('using audio context mapping', this.logContext);
-      this.connectWebAudio(this.audioContext, element);
+    if (this.audioContext) {
+      // audio is routed through the web audio graph, so every attached element has to stay
+      // silent, no matter whether it is the one the graph was built from
       element.volume = 0;
       element.muted = true;
+      if (needsNewWebAudioConnection) {
+        this.log.debug('using audio context mapping', this.logContext);
+        this.connectWebAudio(this.audioContext, element);
+      }
     }
 
-    if (this.elementVolume) {
+    if (this.elementVolume !== undefined) {
       // make sure volume setting is being applied to the newly attached element
       this.setVolume(this.elementVolume);
     }
@@ -158,11 +172,26 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
    * @experimental
    */
   setAudioContext(audioContext: AudioContext | undefined) {
+    if (this.audioContext === audioContext) {
+      return;
+    }
     this.audioContext = audioContext;
-    if (audioContext && this.attachedElements.length > 0) {
-      this.connectWebAudio(audioContext, this.attachedElements[0]);
-    } else if (!audioContext) {
+    if (audioContext) {
+      // playback moves into the web audio graph, mute the elements to avoid double playback
+      for (const element of this.attachedElements) {
+        element.volume = 0;
+        element.muted = true;
+      }
+      if (this.attachedElements.length > 0) {
+        this.connectWebAudio(audioContext, this.attachedElements[0]);
+      }
+    } else {
       this.disconnectWebAudio();
+      // playback moves back into the elements, which `attach` had muted in favour of the graph
+      for (const element of this.attachedElements) {
+        element.muted = false;
+        element.volume = this.elementVolume ?? 1;
+      }
     }
   }
 
@@ -183,15 +212,18 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
     // @ts-ignore attached elements always have a srcObject set
     this.sourceNode = context.createMediaStreamSource(element.srcObject);
     let lastNode: AudioNode = this.sourceNode;
+    this.connectedNodes = [this.sourceNode];
     this.webAudioPluginNodes.forEach((node) => {
       lastNode.connect(node);
       lastNode = node;
+      this.connectedNodes.push(node);
     });
     this.gainNode = context.createGain();
     lastNode.connect(this.gainNode);
+    this.connectedNodes.push(this.gainNode);
     this.gainNode.connect(context.destination);
 
-    if (this.elementVolume) {
+    if (this.elementVolume !== undefined) {
       this.gainNode.gain.setTargetAtTime(this.elementVolume, 0, 0.1);
     }
 
@@ -214,6 +246,19 @@ export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
   }
 
   private disconnectWebAudio() {
+    // the plugin nodes belong to whoever passed them in and may feed graphs of their own, so only
+    // the edges this track created come down. A bare `node.disconnect()` would drop the
+    // application's own routing with them
+    for (let i = 0; i < this.connectedNodes.length - 1; i += 1) {
+      try {
+        this.connectedNodes[i].disconnect(this.connectedNodes[i + 1]);
+      } catch {
+        // a plugin node the application already disconnected itself, the rest of the chain still
+        // has to come apart
+      }
+    }
+    this.connectedNodes = [];
+    // the source and gain nodes are ours alone, so every remaining edge of theirs is ours to drop
     this.gainNode?.disconnect();
     this.sourceNode?.disconnect();
     this.gainNode = undefined;

package/src/room/track/RemoteVideoTrack.test.ts

@@ -98,6 +98,216 @@ describe('RemoteVideoTrack', () => {
   });
 });
 
+describe('Document Picture-in-Picture detection', () => {
+  /**
+   * happy-dom's MediaStream implements everything except getTracks(), which Track.attach()
+   * reads. Subclassing keeps it assignable to the element's srcObject, which type checks the
+   * value against happy-dom's own MediaStream.
+   */
+  class TestMediaStream extends MediaStream {
+    getTracks() {
+      return [...this.getAudioTracks(), ...this.getVideoTracks()];
+    }
+  }
+
+  /** Stands in for window.documentPictureInPicture, which happy-dom does not implement. */
+  class FakeDocumentPictureInPicture extends EventTarget {
+    window?: Window;
+  }
+
+  let documentPiP: FakeDocumentPictureInPicture;
+
+  /** Gives an element the layout box that isElementInViewport reads. */
+  function placeElement(
+    el: HTMLElement,
+    { top, left, width, height }: { top: number; left: number; width: number; height: number },
+  ) {
+    Object.defineProperties(el, {
+      offsetTop: { value: top, configurable: true },
+      offsetLeft: { value: left, configurable: true },
+      offsetWidth: { value: width, configurable: true },
+      offsetHeight: { value: height, configurable: true },
+      offsetParent: { value: null, configurable: true },
+    });
+  }
+
+  function createVideo(
+    ownerDocument: Document,
+    box = { top: 40, left: 40, width: 320, height: 180 },
+  ) {
+    const video = ownerDocument.createElement('video');
+    ownerDocument.body.appendChild(video);
+    placeElement(video, box);
+    return video;
+  }
+
+  /**
+   * Creates a frame in the given document and returns its document. happy-dom implements
+   * neither frameElement nor a correct top on a frame's window, so both are wired up here.
+   */
+  function createFrame(parentDocument: Document, topWindow: Window) {
+    const frame = parentDocument.createElement('iframe');
+    parentDocument.body.appendChild(frame);
+    const frameDocument = frame.contentDocument!;
+    Object.defineProperties(frameDocument.defaultView!, {
+      frameElement: { value: frame, configurable: true },
+      top: { value: topWindow, configurable: true },
+    });
+    return frameDocument;
+  }
+
+  /**
+   * Makes a frame document look like one reached through a cross-origin frame: the containing
+   * frame belongs to another origin, so frameElement reads null while top stays readable.
+   */
+  function hideContainingFrame(frameDocument: Document) {
+    Object.defineProperty(frameDocument.defaultView!, 'frameElement', {
+      value: null,
+      configurable: true,
+    });
+    return frameDocument;
+  }
+
+  /** Opens a Document PiP window and fires the 'enter' event the SDK listens for. */
+  function openPiPWindow({ width, height } = { width: 1280, height: 820 }) {
+    const pipDocument = document.implementation.createHTMLDocument('pip');
+    documentPiP.window = Object.assign(new EventTarget(), {
+      document: pipDocument,
+      innerWidth: width,
+      innerHeight: height,
+      pageXOffset: 0,
+      pageYOffset: 0,
+    }) as unknown as Window;
+    documentPiP.dispatchEvent(new Event('enter'));
+    return pipDocument;
+  }
+
+  /** Attaches a fresh track to the element and records every VisibilityChanged emission. */
+  function attachTrack(video: HTMLElement) {
+    const track = new RemoteVideoTrack(new MockMediaStreamTrack(), 'sid', undefined!, {});
+    const events: boolean[] = [];
+    track.on(TrackEvent.VisibilityChanged, (visible) => events.push(visible));
+    track.attach(video as HTMLMediaElement);
+    return events;
+  }
+
+  /** Flushes the microtask + animation frame that onEnterPiP defers its re-check behind. */
+  async function flushDeferredPiPCheck() {
+    await vi.advanceTimersByTimeAsync(50);
+  }
+
+  function setOpenerHidden() {
+    Object.defineProperty(document, 'visibilityState', { value: 'hidden', configurable: true });
+  }
+
+  beforeEach(() => {
+    vi.stubGlobal('MediaStream', TestMediaStream);
+    documentPiP = new FakeDocumentPictureInPicture();
+    (window as any).documentPictureInPicture = documentPiP;
+    // The opener tab is backgrounded in every case, so pauseVideoInBackground (on by default)
+    // reports the element hidden unless it is genuinely in PiP.
+    setOpenerHidden();
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    delete (window as any).documentPictureInPicture;
+    Object.defineProperty(document, 'visibilityState', { value: 'visible', configurable: true });
+    document.body.innerHTML = '';
+  });
+
+  it('does not report an opener-document element as being in PiP', () => {
+    openPiPWindow();
+    const video = createVideo(document);
+
+    // The top-window check must not open a hole here: the opener's top is the opener itself.
+    expect(window.top).toBe(window);
+    expect(window.top).not.toBe(documentPiP.window);
+    // The element sits at (40, 40) of the OPENER page, which falls inside the PiP window's
+    // 1280x820 box. It is not in PiP, so the background pause applies.
+    expect(attachTrack(video)).toEqual([false]);
+  });
+
+  it('does not report an element in an opener frame as being in PiP', () => {
+    openPiPWindow();
+    const openerFrame = createFrame(document, window);
+    const video = createVideo(openerFrame);
+
+    // An opener frame tops out at the opener, so neither the top check nor the walk matches.
+    expect(openerFrame.defaultView!.top).toBe(window);
+    expect(attachTrack(video)).toEqual([false]);
+  });
+
+  it('reports an element moved into the PiP document after the enter event', async () => {
+    const video = createVideo(document);
+    const events = attachTrack(video);
+    expect(events).toEqual([false]);
+
+    const pipDocument = openPiPWindow();
+    // The SDK defers its re-check behind a microtask and an animation frame precisely so the
+    // embedder can move its subtree across after the enter event fires.
+    pipDocument.body.appendChild(video);
+    expect(video.ownerDocument).toBe(pipDocument);
+    await flushDeferredPiPCheck();
+
+    expect(events).toEqual([false, true]);
+  });
+
+  it('leaves an unrelated opener element hidden when another element enters PiP', async () => {
+    const openerVideo = createVideo(document);
+    const movedVideo = createVideo(document);
+    const openerEvents = attachTrack(openerVideo);
+    const movedEvents = attachTrack(movedVideo);
+
+    const pipDocument = openPiPWindow();
+    pipDocument.body.appendChild(movedVideo);
+    await flushDeferredPiPCheck();
+
+    expect(movedEvents).toEqual([false, true]);
+    expect(openerEvents).toEqual([false]);
+  });
+
+  it('reports an element inside a frame nested in the PiP document', () => {
+    const pipDocument = openPiPWindow();
+    const video = createVideo(createFrame(pipDocument, documentPiP.window!), {
+      top: 10,
+      left: 10,
+      width: 320,
+      height: 180,
+    });
+
+    expect(attachTrack(video)).toEqual([true]);
+  });
+
+  it('reports an element behind a cross-origin frame nested in the PiP window', () => {
+    const pipDocument = openPiPWindow();
+    const pipWindow = documentPiP.window!;
+    // PiP document (origin A) -> frame (origin B) -> frame (origin A). The opener can reach
+    // the innermost document, but its containing frame belongs to B, so frameElement is null
+    // there and the same-origin walk cannot get back to the PiP document.
+    const crossOriginFrame = createFrame(pipDocument, pipWindow);
+    const innerFrame = hideContainingFrame(createFrame(crossOriginFrame, pipWindow));
+    const video = createVideo(innerFrame, { top: 10, left: 10, width: 320, height: 180 });
+
+    expect(innerFrame.defaultView!.frameElement).toBeNull();
+    expect(innerFrame.defaultView!.top).toBe(pipWindow);
+    expect(attachTrack(video)).toEqual([true]);
+  });
+
+  it('does not report an element scrolled out of the PiP window viewport', () => {
+    const pipDocument = openPiPWindow();
+    const video = createVideo(pipDocument, { top: 2000, left: 40, width: 320, height: 180 });
+
+    expect(attachTrack(video)).toEqual([false]);
+  });
+
+  it('does not report an element as being in PiP when no PiP window is open', () => {
+    const video = createVideo(document);
+
+    expect(attachTrack(video)).toEqual([false]);
+  });
+});
+
 class MockElementInfo implements ElementInfo {
   element: object = {};
 

package/src/room/track/RemoteVideoTrack.ts

@@ -435,9 +435,35 @@ class HTMLElementInfo implements ElementInfo {
 function isElementInPiP(el: HTMLElement) {
   // Simple video PiP
   if (document.pictureInPictureElement === el) return true;
-  // Document PiP
-  if (window.documentPictureInPicture?.window)
-    return isElementInViewport(el, window.documentPictureInPicture?.window);
+  // Document PiP: the element has to actually live in the PiP window, otherwise an
+  // opener-document element whose coordinates happen to fall inside the PiP window's
+  // bounds is reported as being in PiP.
+  const pipWindow = window.documentPictureInPicture?.window;
+  if (pipWindow) {
+    return isDocumentInPiPWindow(el.ownerDocument, pipWindow) && isElementInViewport(el, pipWindow);
+  }
+  return false;
+}
+
+// Whether the document is the Document PiP window's own document, or the document of a frame
+// nested inside it. The spec allows frames in a PiP window, so an element in one is in PiP:
+// https://wicg.github.io/document-picture-in-picture/#iframes
+function isDocumentInPiPWindow(doc: Document, pipWindow: Window) {
+  // A cross-origin frame anywhere between the element and the PiP window hides frameElement
+  // from the walk below, but top stays readable across origins and identifies the top-level
+  // window the element belongs to. For an opener element that is the opener, not the PiP window.
+  if (doc.defaultView?.top === pipWindow) return true;
+  let current: Document | null | undefined = doc;
+  while (current) {
+    if (current === pipWindow.document) return true;
+    try {
+      // Reading frameElement across a cross-origin boundary throws, which means we cannot
+      // establish the ancestry and must not claim the element is in PiP.
+      current = current.defaultView?.frameElement?.ownerDocument;
+    } catch {
+      return false;
+    }
+  }
   return false;
 }
 

package/src/room/track/processor/types.ts

@@ -22,6 +22,17 @@ export interface AudioProcessorOptions extends ProcessorOptions<Track.Kind.Audio
  */
 export interface VideoProcessorOptions extends ProcessorOptions<Track.Kind.Video> {}
 
+/**
+ * A processor is bound to the track it was set on, not to the room, and it does not survive the
+ * track being stopped: unpublishing a track or disconnecting from a room destroys its processor.
+ * Both paths stop the track by default, so after reconnecting or republishing, call
+ * `setProcessor` again on the new track to put the processor back.
+ *
+ * The exception is `stopLocalTrackOnUnpublish: false`, which keeps the track alive across an
+ * unpublish or disconnect. A video processor then survives with it. An audio processor only
+ * survives if the audio context came from the application, because a context the SDK created is
+ * closed on disconnect and the processor goes with it.
+ */
 export interface TrackProcessor<
   T extends Track.Kind,
   U extends ProcessorOptions<T> = ProcessorOptions<T>,