livekit-client 2.22.3 → 2.22.4

package/src/room/track/utils.ts

@@ -1,5 +1,6 @@
 import { TrackInfo, TrackPublishedResponse, TrackSource, VideoQuality } from '@livekit/protocol';
 import type { AudioProcessorOptions, TrackProcessor, VideoProcessorOptions } from '../..';
+import log from '../../logger';
 import { cloneDeep } from '../../utils/cloneDeep';
 import { isSafari, sleep } from '../utils';
 import { Track } from './Track';
@@ -351,3 +352,90 @@ export function layerDimensionsFor(
 ): Track.Dimensions | undefined {
   return trackInfo.layers?.find((l) => l.quality === quality);
 }
+
+/**
+ * Waits until `track` has actually produced a video frame.
+ *
+ * iOS keeps reporting the camera's sensor frame from `getSettings()` until the camera has
+ * delivered its first picture, so a portrait capture initially reads as landscape. The arrival
+ * of that first frame is the only reliable signal that the reported settings describe reality.
+ *
+ * Best effort, never throws. Resolves `true` when a frame was observed, `false` when no frame
+ * can be observed (the track is already ended, muted or disabled) or `timeoutMs` elapsed first.
+ *
+ * @internal
+ */
+export async function waitForFirstVideoFrame(
+  track: MediaStreamTrack,
+  timeoutMs: number,
+  candidateElements: HTMLMediaElement[] = [],
+): Promise<boolean> {
+  if (typeof document === 'undefined') {
+    return false;
+  }
+  if (track.readyState === 'ended' || track.muted || !track.enabled) {
+    // no frame is ever going to arrive, don't hold the caller up for the full timeout
+    return false;
+  }
+
+  // requestVideoFrameCallback (Safari 15.4 and up) is a platform capability, so the answer is the
+  // same for every element
+  const supportsFrameCallback =
+    typeof HTMLVideoElement.prototype.requestVideoFrameCallback === 'function';
+
+  // prefer an element that is already rendering this track (a local preview, typically) over
+  // spinning up a second decode just to observe a frame
+  const existingElement = candidateElements.find(
+    (el): el is HTMLVideoElement =>
+      el instanceof HTMLVideoElement &&
+      el.srcObject instanceof MediaStream &&
+      el.srcObject.getVideoTracks().includes(track) &&
+      // without requestVideoFrameCallback we depend on `loadeddata`, which is one-shot: an element
+      // that already decoded a frame (readyState >= HAVE_CURRENT_DATA) never fires it again.
+      // Leave those alone and set up our own element, which starts out empty.
+      (supportsFrameCallback || el.readyState < 2),
+  );
+
+  const element = existingElement ?? document.createElement('video');
+  if (!existingElement) {
+    element.playsInline = true;
+    element.muted = true;
+    element.srcObject = new MediaStream([track]);
+    // the element is deliberately never added to the DOM, it only exists to pull frames.
+    // WebKit delivers frames to a paused element too, so a rejected play() (Safari's low power
+    // mode, for instance) is not a reason to stop waiting
+    element.play().catch((error) => {
+      log.debug('could not play element while waiting for the first video frame', { error });
+    });
+  }
+
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let frameHandle: number | undefined;
+  let onLoadedData: (() => void) | undefined;
+
+  try {
+    return await new Promise<boolean>((resolve) => {
+      timer = setTimeout(() => resolve(false), timeoutMs);
+      if (supportsFrameCallback) {
+        frameHandle = element.requestVideoFrameCallback(() => resolve(true));
+      } else {
+        // Safari below 15.4 has no requestVideoFrameCallback. `loadeddata` fires once the first
+        // frame has been decoded, which is the same signal a few milliseconds earlier.
+        onLoadedData = () => resolve(true);
+        element.addEventListener('loadeddata', onLoadedData);
+      }
+    });
+  } finally {
+    clearTimeout(timer);
+    if (frameHandle !== undefined) {
+      element.cancelVideoFrameCallback?.(frameHandle);
+    }
+    if (onLoadedData) {
+      element.removeEventListener('loadeddata', onLoadedData);
+    }
+    if (!existingElement) {
+      element.pause();
+      element.srcObject = null;
+    }
+  }
+}

package/src/room/utils.test.ts

@@ -1,12 +1,22 @@
 import { ClientInfo_Capability } from '@livekit/protocol';
 import { afterEach, describe, expect, it, vi } from 'vitest';
+import {
+  stubMissingCompressionStreams,
+  stubMissingDeflateRawSupport,
+} from '../test/compressionStreamStubs';
+import { getBrowser } from '../utils/browserParser';
 import {
   ddExtensionURI,
   extractMaxAgeFromRequestHeaders,
   getClientInfo,
+  isAppleMobile,
+  isCompressionStreamSupported,
+  isDeflateRawCompressionSupported,
+  isIPadOS,
   isSVCSimulcast,
   isSVCSimulcastSupportedByServer,
   negotiateDependencyDescriptor,
+  resetDeflateRawCompressionSupportCache,
   splitUtf8,
   supportsAdaptiveStream,
   toWebsocketUrl,
@@ -367,3 +377,117 @@ describe('usesLegacySVCEncodings', () => {
     expect(usesLegacySVCEncodings()).toBe(true);
   });
 });
+
+describe('isIPadOS / isAppleMobile', () => {
+  const stub = (userAgent: string, maxTouchPoints: number) =>
+    vi.stubGlobal('navigator', { userAgent, maxTouchPoints, product: 'Gecko' });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  // measured on an iPad 6 running iPadOS 17: the default (desktop) user agent, maxTouchPoints 5
+  const IPAD_DESKTOP_UA =
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.14 Safari/605.1.15';
+  // measured on macOS Safari 26.4: the same shape of user agent, maxTouchPoints 0
+  const MAC_SAFARI_UA =
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.4 Safari/605.1.15';
+  const IPHONE_UA =
+    'Mozilla/5.0 (iPhone; CPU iPhone OS 18_7 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1';
+  const IPAD_MOBILE_UA =
+    'Mozilla/5.0 (iPad; CPU OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1';
+  const MAC_CHROME_UA =
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36';
+  const ANDROID_CHROME_UA =
+    'Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.36';
+
+  it('detects an iPad sending the desktop user agent', () => {
+    stub(IPAD_DESKTOP_UA, 5);
+    expect(isIPadOS()).toBe(true);
+    expect(isAppleMobile()).toBe(true);
+  });
+
+  it('is the case getBrowser() alone gets wrong', () => {
+    // the gap this exists to close: iPadOS parses as macOS, so `os === 'iOS'` misses it
+    stub(IPAD_DESKTOP_UA, 5);
+    expect(getBrowser()?.os).toBe('macOS');
+  });
+
+  it('leaves macOS Safari alone, where the camera reports correctly and frames are slow', () => {
+    stub(MAC_SAFARI_UA, 0);
+    expect(isIPadOS()).toBe(false);
+    expect(isAppleMobile()).toBe(false);
+  });
+
+  it('covers iPhones through the existing iOS check', () => {
+    stub(IPHONE_UA, 5);
+    expect(isIPadOS()).toBe(false);
+    expect(isAppleMobile()).toBe(true);
+  });
+
+  it('covers an iPad set to request the mobile site', () => {
+    stub(IPAD_MOBILE_UA, 5);
+    expect(isIPadOS()).toBe(false);
+    expect(isAppleMobile()).toBe(true);
+  });
+
+  it('does not fire for a Mac with a touchscreen attached, which is not Safari', () => {
+    stub(MAC_CHROME_UA, 5);
+    expect(isIPadOS()).toBe(false);
+    expect(isAppleMobile()).toBe(false);
+  });
+
+  it('does not fire on android', () => {
+    stub(ANDROID_CHROME_UA, 5);
+    expect(isIPadOS()).toBe(false);
+    expect(isAppleMobile()).toBe(false);
+  });
+
+  it('does not fire on a desktop safari reporting no touch support', () => {
+    stub(MAC_SAFARI_UA, 1);
+    expect(isIPadOS()).toBe(false);
+  });
+});
+
+describe('isDeflateRawCompressionSupported', () => {
+  afterEach(() => {
+    resetDeflateRawCompressionSupportCache();
+  });
+
+  it('is true in a runtime that supports the deflate-raw format', () => {
+    expect(isDeflateRawCompressionSupported()).toBe(true);
+  });
+
+  it('is false without the compression streams api at all', () => {
+    const restoreCompressionStreams = stubMissingCompressionStreams();
+    try {
+      expect(isDeflateRawCompressionSupported()).toBe(false);
+      expect(isCompressionStreamSupported()).toBe(false);
+    } finally {
+      restoreCompressionStreams();
+    }
+  });
+
+  it('is false when the api exists but deflate-raw is not a supported format', () => {
+    const restoreCompressionStreams = stubMissingDeflateRawSupport();
+    try {
+      expect(isDeflateRawCompressionSupported()).toBe(false);
+      // Chromium 80-102 still compresses gzip, which is all the wrapped join request needs.
+      expect(isCompressionStreamSupported()).toBe(true);
+    } finally {
+      restoreCompressionStreams();
+    }
+  });
+
+  it('probes the runtime only once', () => {
+    expect(isDeflateRawCompressionSupported()).toBe(true);
+
+    const originalCompressionStream = globalThis.CompressionStream;
+    try {
+      (globalThis as any).CompressionStream = undefined;
+      expect(isDeflateRawCompressionSupported()).toBe(true);
+    } finally {
+      (globalThis as any).CompressionStream = originalCompressionStream;
+    }
+  });
+});

package/src/room/utils.ts

@@ -300,6 +300,33 @@ export function isSafariBased(): boolean {
   return b?.name === 'Safari' || b?.os === 'iOS';
 }
 
+/**
+ * iPadOS has sent the desktop Macintosh user agent by default since iPadOS 13, so `getBrowser()`
+ * reports `os: 'macOS'` there and every `os === 'iOS'` check misses it. macOS Safari implements no
+ * touch events at all (`TouchEvent` undefined, `maxTouchPoints` 0) regardless of any touchscreen
+ * attached, so touch capability separates the two. Measured on an iPad 6 / iPadOS 17
+ * (`maxTouchPoints` 5) against macOS Safari 26.4 (`maxTouchPoints` 0).
+ *
+ * Deliberately a separate helper rather than a fix in `browserParser`: iPadOS spoofs the Mac
+ * version too, so reporting `os: 'iOS'` there would leave `osVersion` at `10.15.7` and silently
+ * break {@link isSafari17Based} and {@link isSafariSvcApi}, which compare it.
+ */
+export function isIPadOS(): boolean {
+  if (!isWeb()) {
+    return false;
+  }
+  const b = getBrowser();
+  return b?.name === 'Safari' && b?.os === 'macOS' && navigator.maxTouchPoints > 1;
+}
+
+/**
+ * iPhone, iPad and iOS-hosted browsers — everything running the Apple camera capture pipeline,
+ * including iPads that present themselves as a Mac.
+ */
+export function isAppleMobile(): boolean {
+  return getBrowser()?.os === 'iOS' || isIPadOS();
+}
+
 export function isSafari17Based(): boolean {
   const b = getBrowser();
   return (
@@ -961,6 +988,49 @@ export function isCompressionStreamSupported() {
   return typeof CompressionStream !== 'undefined';
 }
 
+let deflateRawCompressionSupported: boolean | null = null;
+
+/**
+ * Whether this runtime can compress AND decompress the `deflate-raw` format, which is a stricter
+ * requirement than {@link isCompressionStreamSupported}: Chromium 80-102 included
+ * `CompressionStream` supporting only `gzip` / `deflate`, and `deflate-raw` was added in Chromium
+ * 103 (Firefox 113, Safari 16.4).
+ */
+export function isDeflateRawCompressionSupported(): boolean {
+  if (typeof deflateRawCompressionSupported !== 'boolean') {
+    deflateRawCompressionSupported = probeDeflateRawCompressionSupport();
+  }
+  return deflateRawCompressionSupported;
+}
+
+function probeDeflateRawCompressionSupport(): boolean {
+  if (typeof CompressionStream === 'undefined' || typeof DecompressionStream === 'undefined') {
+    return false;
+  }
+  try {
+    // There is no api to query format support, and constructing with an unsupported format throws
+    // synchronously, so constructing one of each is the probe. Both are checked because the
+    // deflate-raw capability advertised to peers is as much about decompressing what they send us
+    // as it is about compressing what we send them.
+    new CompressionStream('deflate-raw');
+    new DecompressionStream('deflate-raw');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Clears the memoized {@link isDeflateRawCompressionSupported} probe result. A runtime doesn't
+ * gain or lose compression formats while it is running, so this only exists for tests that swap
+ * the global `CompressionStream` / `DecompressionStream`.
+ *
+ * @internal
+ */
+export function resetDeflateRawCompressionSupportCache() {
+  deflateRawCompressionSupported = null;
+}
+
 export function isPublisherOfferWithJoinSupported() {
   // we have connectivity issue about publisher offer with join on firefox #1919
   return isCompressionStreamSupported() && !isFireFox();

package/src/test/compressionStreamStubs.ts

@@ -0,0 +1,53 @@
+import { resetDeflateRawCompressionSupportCache } from '../room/utils';
+
+/**
+ * Swaps the global `CompressionStream` / `DecompressionStream` away entirely, emulating a runtime
+ * that predates the compression streams api. Returns a function that restores them.
+ */
+export function stubMissingCompressionStreams(): () => void {
+  return stubCompressionStreams(undefined, undefined);
+}
+
+/**
+ * Swaps in global `CompressionStream` / `DecompressionStream` constructors that reject the
+ * `deflate-raw` format the way Chromium 80-102 does (the api exists, but only `gzip` / `deflate`
+ * are supported). Returns a function that restores the real ones.
+ */
+export function stubMissingDeflateRawSupport(): () => void {
+  return stubCompressionStreams(
+    legacyFormatsOnly('CompressionStream', CompressionStream),
+    legacyFormatsOnly('DecompressionStream', DecompressionStream),
+  );
+}
+
+type CompressionStreamCtor<T> = new (format: CompressionFormat) => T;
+
+function legacyFormatsOnly<T>(
+  name: string,
+  Real: CompressionStreamCtor<T>,
+): CompressionStreamCtor<T> {
+  return function LegacyCompressionStream(format: CompressionFormat) {
+    if (format === 'deflate-raw') {
+      throw new TypeError(
+        `Failed to construct '${name}': Unsupported compression format: 'deflate-raw'`,
+      );
+    }
+    return new Real(format);
+  } as unknown as CompressionStreamCtor<T>;
+}
+
+function stubCompressionStreams(compression: unknown, decompression: unknown): () => void {
+  const originalCompressionStream = globalThis.CompressionStream;
+  const originalDecompressionStream = globalThis.DecompressionStream;
+  (globalThis as any).CompressionStream = compression;
+  (globalThis as any).DecompressionStream = decompression;
+  // The support probe memoizes, which is only safe because a real runtime never changes formats
+  // mid-session; swapping the globals has to invalidate it.
+  resetDeflateRawCompressionSupportCache();
+
+  return () => {
+    (globalThis as any).CompressionStream = originalCompressionStream;
+    (globalThis as any).DecompressionStream = originalDecompressionStream;
+    resetDeflateRawCompressionSupportCache();
+  };
+}