@voltkit-rtc/client 0.7.3 → 0.9.0

package/src/adaptive.ts

@@ -0,0 +1,295 @@
+/**
+ * Adaptive stream: receive each remote video at the size it is shown, and not
+ * at all while nobody can see it.
+ *
+ * The app attaches the element that shows a video (`attachVideo`). From that
+ * element's size on screen (CSS pixels × devicePixelRatio, at most 2) the
+ * smallest simulcast layer that fills it sharply is chosen; while no attached
+ * element is visible (scrolled away, `display: none`, the tab in the
+ * background) the video is paused at the media server, so it costs no
+ * bandwidth and no decoding. This is what LiveKit calls adaptiveStream and
+ * Tencent TRTC "small stream + receiveWhenViewVisible": the single biggest
+ * saving on a viewer with several tiles, where every tile otherwise decodes
+ * the publisher's full-size layer.
+ *
+ * Pure decisions are exported for tests; the browser parts (observers,
+ * timers) are injected.
+ */
+
+import type { VideoLayer } from './quality.js';
+
+/** Where a video is drawn, in CSS pixels, and how it is fitted into that box. */
+export interface ViewBox {
+  width: number;
+  height: number;
+  fit: 'cover' | 'contain';
+}
+
+/** Never ask for more than twice the CSS size: a 3× phone screen gains nothing visible from it. */
+export const MAX_PIXEL_RATIO = 2;
+/** A layer shown at most this much bigger than it is still looks sharp: avoids a whole layer for a few pixels. */
+export const MAX_UPSCALE = 1.15;
+
+/**
+ * The smallest layer that fills `box` sharply, or null when the layers are
+ * not known (an older publisher): then the server chooses, as before.
+ *
+ * With `cover` the picture is scaled until it fills the box (a part is
+ * cropped), with `contain` until it fits. Either way the scale is measured
+ * against each layer's own size, so a portrait layer list works too.
+ */
+export function layerForBox(layers: readonly VideoLayer[], box: ViewBox, pixelRatio = 1): number | null {
+  const usable = layers.filter((l) => l.width > 0 && l.height > 0).sort((a, b) => a.width * a.height - b.width * b.height);
+  if (!usable.length || box.width <= 0 || box.height <= 0) return null;
+  const ratio = Math.min(MAX_PIXEL_RATIO, Math.max(1, pixelRatio || 1));
+  const w = box.width * ratio;
+  const h = box.height * ratio;
+  for (const layer of usable) {
+    const scale = box.fit === 'cover' ? Math.max(w / layer.width, h / layer.height) : Math.min(w / layer.width, h / layer.height);
+    // The layer is at most slightly upscaled on screen.
+    if (scale <= MAX_UPSCALE) return layer.spatialLayer;
+  }
+  return usable[usable.length - 1]!.spatialLayer;
+}
+
+/** What one video should get now: paused, or a layer (null = let the server choose). */
+export interface AdaptiveWant {
+  visible: boolean;
+  spatialLayer: number | null;
+}
+
+/**
+ * The largest view among the visible ones decides; any visible view keeps the video flowing. A view not measured
+ * yet (just attached, before layout) counts as shown at the server's choice, so attaching never pauses a video
+ * for the moment before the browser has sized the element.
+ */
+export function wantFor(views: readonly { box: ViewBox; visible: boolean; measured?: boolean }[], layers: readonly VideoLayer[], pixelRatio: number, pageVisible: boolean): AdaptiveWant {
+  if (pageVisible && views.some((v) => v.measured === false)) return { visible: true, spatialLayer: null };
+  const shown = pageVisible ? views.filter((v) => v.visible && v.box.width > 0 && v.box.height > 0) : [];
+  if (!shown.length) return { visible: false, spatialLayer: null };
+  let best: number | null = null;
+  for (const view of shown) {
+    const layer = layerForBox(layers, view.box, pixelRatio);
+    if (layer === null) return { visible: true, spatialLayer: null };
+    best = best === null ? layer : Math.max(best, layer);
+  }
+  return { visible: true, spatialLayer: best };
+}
+
+/** Pause a hidden video only after this long, so scrolling past or a quick tab switch does not stop it. */
+export const PAUSE_AFTER_MS = 1500;
+/** Go down a layer only after this long (going up is immediate): a tile shrinking for a moment keeps its picture. */
+export const LOWER_AFTER_MS = 1200;
+
+export interface AdaptiveHost {
+  /** The layers this producer's publisher declared (smallest first or any order); [] when unknown. */
+  layers(producerId: string): readonly VideoLayer[];
+  /** Apply a decision for one video. */
+  apply(producerId: string, want: AdaptiveWant): void;
+  /** For tests: the environment. */
+  pixelRatio?(): number;
+  pageVisible?(): boolean;
+  now?(): number;
+  setTimer?(fn: () => void, ms: number): unknown;
+  clearTimer?(handle: unknown): void;
+  /** For tests: stand-ins for ResizeObserver / IntersectionObserver / document visibility. */
+  observe?(element: Element, onChange: (box: ViewBox | null, visible: boolean | null) => void): () => void;
+  onPageVisibility?(listener: () => void): () => void;
+}
+
+interface View {
+  box: ViewBox;
+  visible: boolean;
+  measured: boolean;
+  stop: () => void;
+}
+
+interface Entry {
+  views: Map<Element, View>;
+  applied: AdaptiveWant | null;
+  timer: unknown;
+}
+
+const FLOWING: AdaptiveWant = { visible: true, spatialLayer: null };
+const sameWant = (a: AdaptiveWant | null, b: AdaptiveWant) => a !== null && a.visible === b.visible && a.spatialLayer === b.spatialLayer;
+
+/**
+ * The videos being shown, by producer id. Decisions are applied through the
+ * host; going up (or becoming visible) is applied at once, going down (or
+ * hiding) after a short delay.
+ */
+export class AdaptiveVideos {
+  private readonly entries = new Map<string, Entry>();
+  private stopPage: (() => void) | null = null;
+
+  constructor(private readonly host: AdaptiveHost) {}
+
+  /** Show `producerId` in `element`; returns the function that stops. */
+  attach(producerId: string, element: Element): () => void {
+    let entry = this.entries.get(producerId);
+    if (!entry) {
+      entry = { views: new Map(), applied: null, timer: null };
+      this.entries.set(producerId, entry);
+    }
+    if (entry.views.has(element)) return () => this.detach(producerId, element);
+    const view: View = { box: { width: 0, height: 0, fit: 'contain' }, visible: true, measured: false, stop: () => undefined };
+    entry.views.set(element, view);
+    view.stop = this.observe(element, (box, visible) => {
+      if (box) {
+        view.box = box;
+        view.measured = true;
+      }
+      if (visible !== null) view.visible = visible;
+      this.update(producerId);
+    });
+    if (!this.stopPage) this.stopPage = this.watchPage(() => { for (const id of this.entries.keys()) this.update(id); });
+    this.update(producerId);
+    return () => this.detach(producerId, element);
+  }
+
+  detach(producerId: string, element: Element): void {
+    const entry = this.entries.get(producerId);
+    const view = entry?.views.get(element);
+    if (!entry || !view) return;
+    view.stop();
+    entry.views.delete(element);
+    if (entry.views.size) {
+      this.update(producerId);
+      return;
+    }
+    // Nothing shows it any more: back to how it was before (flowing, the server's choice).
+    this.clear(entry);
+    this.entries.delete(producerId);
+    if (entry.applied && !sameWant(entry.applied, { visible: true, spatialLayer: null })) this.host.apply(producerId, { visible: true, spatialLayer: null });
+    if (!this.entries.size) {
+      this.stopPage?.();
+      this.stopPage = null;
+    }
+  }
+
+  /** Whether the app shows this video through attachVideo (its decisions then apply to new consumers too). */
+  has(producerId: string): boolean {
+    return this.entries.has(producerId);
+  }
+
+  /** The decision in force, for a consumer made (or made again) after it: null when the app does not attach it. */
+  current(producerId: string): AdaptiveWant | null {
+    const entry = this.entries.get(producerId);
+    return entry ? this.decide(producerId, entry) : null;
+  }
+
+  /**
+   * A new consumer was made with `current()` applied (its layers, and paused when hidden): record that as applied,
+   * so nothing is sent twice, and drop a pending change made for the old one.
+   */
+  consumed(producerId: string): void {
+    const entry = this.entries.get(producerId);
+    if (!entry) return;
+    this.clear(entry);
+    entry.applied = this.decide(producerId, entry);
+  }
+
+  /** Stop everything (the session closed). */
+  dispose(): void {
+    for (const entry of this.entries.values()) {
+      this.clear(entry);
+      for (const view of entry.views.values()) view.stop();
+    }
+    this.entries.clear();
+    this.stopPage?.();
+    this.stopPage = null;
+  }
+
+  private decide(producerId: string, entry: Entry): AdaptiveWant {
+    return wantFor([...entry.views.values()], this.host.layers(producerId), this.host.pixelRatio?.() ?? 1, this.host.pageVisible?.() ?? true);
+  }
+
+  private update(producerId: string): void {
+    const entry = this.entries.get(producerId);
+    if (!entry) return;
+    const want = this.decide(producerId, entry);
+    // Nothing applied yet is the server's choice, flowing: asking for exactly that sends nothing.
+    if (sameWant(entry.applied ?? FLOWING, want)) {
+      this.clear(entry);
+      entry.applied ??= want;
+      return;
+    }
+    // Nothing applied yet means the video flows at the server's choice.
+    const prev = entry.applied ?? { visible: true, spatialLayer: null };
+    // Hiding, or a smaller layer than one applied, waits; anything else (showing, a bigger layer, a first layer) is at once.
+    const lower = (prev.visible && !want.visible) ||
+      (want.visible && prev.visible && want.spatialLayer !== null && prev.spatialLayer !== null && want.spatialLayer < prev.spatialLayer);
+    // Already waiting to lower: a repeated report must not restart the wait; it applies what is wanted when it runs out.
+    if (lower && entry.timer !== null) return;
+    this.clear(entry);
+    if (!lower) {
+      entry.applied = want;
+      this.host.apply(producerId, want);
+      return;
+    }
+    const delay = want.visible ? LOWER_AFTER_MS : PAUSE_AFTER_MS;
+    entry.timer = (this.host.setTimer ?? ((fn, ms) => setTimeout(fn, ms)))(() => {
+      entry.timer = null;
+      const now = this.decide(producerId, entry);
+      if (sameWant(entry.applied, now)) return;
+      entry.applied = now;
+      this.host.apply(producerId, now);
+    }, delay);
+  }
+
+  private clear(entry: Entry): void {
+    if (entry.timer !== null) (this.host.clearTimer ?? ((h) => clearTimeout(h as ReturnType<typeof setTimeout>)))(entry.timer);
+    entry.timer = null;
+  }
+
+  private observe(element: Element, onChange: (box: ViewBox | null, visible: boolean | null) => void): () => void {
+    if (this.host.observe) return this.host.observe(element, onChange);
+    return observeElement(element, onChange);
+  }
+
+  private watchPage(listener: () => void): () => void {
+    if (this.host.onPageVisibility) return this.host.onPageVisibility(listener);
+    if (typeof document === 'undefined') return () => undefined;
+    document.addEventListener('visibilitychange', listener);
+    return () => document.removeEventListener('visibilitychange', listener);
+  }
+}
+
+/** How an element fits its picture: a video's object-fit, else contain. */
+function fitOf(element: Element): 'cover' | 'contain' {
+  try {
+    const fit = typeof getComputedStyle === 'function' ? getComputedStyle(element).objectFit : '';
+    return fit === 'cover' || fit === 'fill' ? 'cover' : 'contain';
+  } catch {
+    return 'contain';
+  }
+}
+
+/** The browser's observers: size (ResizeObserver) and visibility on screen (IntersectionObserver). */
+function observeElement(element: Element, onChange: (box: ViewBox | null, visible: boolean | null) => void): () => void {
+  const stops: (() => void)[] = [];
+  const measure = () => {
+    const rect = element.getBoundingClientRect();
+    onChange({ width: rect.width, height: rect.height, fit: fitOf(element) }, null);
+  };
+  if (typeof ResizeObserver !== 'undefined') {
+    const resize = new ResizeObserver(() => measure());
+    resize.observe(element);
+    stops.push(() => resize.disconnect());
+  }
+  if (typeof IntersectionObserver !== 'undefined') {
+    const seen = new IntersectionObserver((entries) => {
+      const last = entries[entries.length - 1];
+      if (last) onChange(null, last.isIntersecting);
+    });
+    seen.observe(element);
+    stops.push(() => seen.disconnect());
+  }
+  measure();
+  return () => stops.forEach((stop) => stop());
+}
+
+/** The page is visible (not a background tab). */
+export function pageIsVisible(): boolean {
+  return typeof document === 'undefined' || document.visibilityState !== 'hidden';
+}

package/src/devices.ts

@@ -0,0 +1,70 @@
+/**
+ * Cameras, microphones and speakers (docs/implementation-progress/RTC_PERF_PROGRESS.md I1).
+ *
+ * What the big SDKs do, read from their code:
+ * - Amazon Chime (`DefaultDeviceController`): lists devices by kind, watches `devicechange` and tells observers
+ *   which lists changed; a microphone that ends is replaced so audio keeps flowing, a camera that ends is stopped and
+ *   reported (`audioInputStreamEnded` / `videoInputStreamEnded`) -- never swapped for another camera on its own.
+ * - LiveKit (`Room.handleDeviceChange`, `switchActiveDevice`): switches the published track to a chosen device; when
+ *   the active audio device disappears it moves to the first available one.
+ * - Tencent TRTC: `DEVICE_CHANGED {type, action: add | remove | active}`.
+ *
+ * This module holds the pure parts: comparing device lists and building the constraints for a chosen device.
+ */
+
+export type DeviceKind = 'audioinput' | 'videoinput' | 'audiooutput';
+
+/** A camera, microphone or speaker, as the browser lists it. Labels are empty until a permission was granted. */
+export interface MediaDevice {
+  deviceId: string;
+  groupId: string;
+  kind: DeviceKind;
+  label: string;
+}
+
+export interface DevicesChangedEvent {
+  /** Every device now. */
+  devices: MediaDevice[];
+  added: MediaDevice[];
+  removed: MediaDevice[];
+}
+
+/** The browser's device list, optionally of one kind. Empty outside a browser. */
+export async function enumerate(kind?: DeviceKind): Promise<MediaDevice[]> {
+  const md = typeof navigator !== 'undefined' ? navigator.mediaDevices : undefined;
+  if (!md?.enumerateDevices) return [];
+  const all = await md.enumerateDevices();
+  return all
+    .filter((d) => d.kind === 'audioinput' || d.kind === 'videoinput' || d.kind === 'audiooutput')
+    .filter((d) => !kind || d.kind === kind)
+    .map((d) => ({ deviceId: d.deviceId, groupId: d.groupId, kind: d.kind as DeviceKind, label: d.label }));
+}
+
+const key = (d: MediaDevice) => `${d.kind}:${d.deviceId}`;
+
+/** What appeared and what went between two lists. */
+export function diffDevices(before: readonly MediaDevice[], after: readonly MediaDevice[]): { added: MediaDevice[]; removed: MediaDevice[] } {
+  const was = new Set(before.map(key));
+  const now = new Set(after.map(key));
+  return {
+    added: after.filter((d) => !was.has(key(d))),
+    removed: before.filter((d) => !now.has(key(d))),
+  };
+}
+
+/**
+ * The constraints for capturing `deviceId` (or the system default when null), keeping everything else the app asked
+ * for: size, frame rate, echo cancellation. `base` is what publish was given for that kind.
+ */
+export function constraintsFor(base: boolean | MediaTrackConstraints | undefined, deviceId: string | null): MediaTrackConstraints {
+  const rest: MediaTrackConstraints = typeof base === 'object' && base !== null ? { ...base } : {};
+  delete rest.deviceId;
+  delete rest.groupId;
+  return deviceId === null ? rest : { ...rest, deviceId: { exact: deviceId } };
+}
+
+/** The device a live track captures, when it is a device capture at all (a canvas or a processed track has none). */
+export function deviceIdOf(track: MediaStreamTrack | null | undefined): string | null {
+  const id = track?.getSettings?.().deviceId;
+  return typeof id === 'string' && id.length > 0 ? id : null;
+}

package/src/diagnostics.ts

@@ -0,0 +1,94 @@
+/**
+ * Diagnostics for support (docs/implementation-progress/RTC_PERF_PROGRESS.md S4).
+ *
+ * Agora uploads its SDK log on request (`enableLogUpload`), Twilio sends call telemetry to Insights, Stream Video
+ * traces session events to its servers. Shipping telemetry needs a backend; what helps today is a snapshot an app can
+ * attach to a support ticket: the SDK's recent events (a ring buffer) and its state now.
+ */
+
+/** This package's version; kept equal to package.json by a test. */
+export const SDK_VERSION = '0.9.0';
+
+/** How many recent events the log keeps. */
+export const DIAGNOSTICS_LOG_SIZE = 200;
+
+export interface DiagnosticsEntry {
+  /** Unix ms. */
+  at: number;
+  event: string;
+  /** A short summary of what the event carried (no media objects). */
+  detail?: string;
+}
+
+/** The events worth keeping for support, and how to summarise each. */
+const SUMMARY: Record<string, (args: unknown[]) => string | undefined> = {
+  connectionStateChanged: ([e]) => { const s = e as { previous?: string; state?: string }; return `${s.previous} → ${s.state}`; },
+  disconnected: ([reason]) => String(reason),
+  closed: ([reason]) => String(reason),
+  mustReconnect: ([e]) => (e as { reason?: string }).reason,
+  mediaReconnecting: ([e]) => { const m = e as { transportId?: string; reconnecting?: boolean }; return `${m.transportId} ${m.reconnecting ? 'recovering' : 'recovered'}`; },
+  error: ([e]) => {
+    const err = e as { code?: string; message?: string };
+    const message = err.message ?? '';
+    // RpcError's message already starts with its code.
+    return (err.code && !message.startsWith(`${err.code}:`) ? `${err.code}: ${message}` : message).slice(0, 300);
+  },
+  streamFallback: ([e]) => { const f = e as { producerId?: string; fallback?: boolean; reason?: string }; return `${f.producerId} ${f.fallback ? 'held' : 'back'} (${f.reason})`; },
+  mediaWarning: ([e]) => { const w = e as { code?: string; active?: boolean }; return `${w.code} ${w.active ? 'started' : 'cleared'}`; },
+  cpuConstrained: ([e]) => String((e as { constrained?: boolean }).constrained),
+  videoLimited: ([e]) => (e as { reason?: string }).reason,
+  localTrackEnded: ([e]) => { const t = e as { kind?: string; recovered?: boolean }; return `${t.kind} ${t.recovered ? 'recovered' : 'off'}`; },
+  devicesChanged: ([e]) => { const d = e as { added?: unknown[]; removed?: unknown[] }; return `+${d.added?.length ?? 0} -${d.removed?.length ?? 0}`; },
+  kicked: ([e]) => (e as { reason?: string }).reason,
+  roomClosed: ([e]) => (e as { reason?: string }).reason,
+  usageLimit: ([e]) => (e as { reason?: string }).reason,
+  autoplayFailed: () => undefined,
+  trackEnded: ([e]) => (e as { producerId?: string }).producerId,
+  track: ([e]) => { const t = e as { identity?: string; source?: string }; return `${t.identity} ${t.source}`; },
+};
+
+/** A ring buffer of the SDK's recent events. */
+export class DiagnosticsLog {
+  private readonly entries: DiagnosticsEntry[] = [];
+
+  constructor(private readonly now: () => number = Date.now) {}
+
+  /** Keep an emitted event, when it is one worth keeping. */
+  record(event: string, args: unknown[]): void {
+    const summarize = SUMMARY[event];
+    if (!summarize) return;
+    let detail: string | undefined;
+    try {
+      detail = summarize(args);
+    } catch {
+      detail = undefined;
+    }
+    this.entries.push(detail === undefined ? { at: this.now(), event } : { at: this.now(), event, detail });
+    if (this.entries.length > DIAGNOSTICS_LOG_SIZE) this.entries.shift();
+  }
+
+  list(): DiagnosticsEntry[] {
+    return this.entries.map((e) => ({ ...e }));
+  }
+}
+
+export interface DiagnosticsTransport {
+  state: string;
+  path: { protocol: 'udp' | 'tcp' | null; relayed: boolean } | null;
+}
+
+export interface Diagnostics {
+  sdk: string;
+  at: number;
+  userAgent: string;
+  connectionState: string;
+  room: string | null;
+  identity: string | null;
+  role: string | null;
+  transports: { send: DiagnosticsTransport | null; recv: DiagnosticsTransport | null };
+  network: { rttMs: number | null; availableKbps: number | null } | null;
+  published: { source: string; kind: string; paused: boolean }[];
+  received: { identity: string; source: string; kind: string; paused: boolean }[];
+  /** Recent events, oldest first. */
+  log: DiagnosticsEntry[];
+}

package/src/effects-core.ts

@@ -0,0 +1,177 @@
+/**
+ * Video effects, the parts that need no browser (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-5): settings,
+ * fitting a background image, the size segmentation runs at, and pacing it on a slow device. Kept apart from the
+ * WebGL and MediaPipe code so they are tested in Node.
+ */
+
+/** A background picture: a URL, or anything the browser can draw. */
+export type BackgroundImageSource = string | Blob | ImageBitmap | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas;
+
+/** What happens behind the person: nothing, a blur, or a picture. */
+export type BackgroundSetting = 'none' | 'blur' | { image: BackgroundImageSource };
+
+/** Beauty, each 0-1 (Agora's beauty effect has the same four). */
+export interface BeautyOptions {
+  /** Skin smoothing. Default 0.5. */
+  smooth?: number;
+  /** Brightening. Default 0.3. */
+  brighten?: number;
+  /** A rosier skin. Default 0.1. */
+  redness?: number;
+  /** Sharper eyes, hair and edges (not skin). Default 0.3. */
+  sharpen?: number;
+}
+
+export interface BeautySettings {
+  smooth: number;
+  brighten: number;
+  redness: number;
+  sharpen: number;
+}
+
+export const DEFAULT_BEAUTY: Readonly<BeautySettings> = Object.freeze({ smooth: 0.5, brighten: 0.3, redness: 0.1, sharpen: 0.3 });
+
+/** The blur behind the person, 1 (light) to 10 (strong). */
+export const DEFAULT_BLUR = 5;
+
+const unit = (value: unknown, fallback: number): number =>
+  typeof value === 'number' && Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : fallback;
+
+/** Beauty settings from what the app passed: `true` is the defaults, `false`/`null` is off. */
+export function beautySettings(options: BeautyOptions | boolean | null | undefined): BeautySettings | null {
+  if (!options) return null;
+  const o: BeautyOptions = options === true ? {} : options;
+  return {
+    smooth: unit(o.smooth, DEFAULT_BEAUTY.smooth),
+    brighten: unit(o.brighten, DEFAULT_BEAUTY.brighten),
+    redness: unit(o.redness, DEFAULT_BEAUTY.redness),
+    sharpen: unit(o.sharpen, DEFAULT_BEAUTY.sharpen),
+  };
+}
+
+/** A blur level from what the app passed: 1-10, rounded; anything else is the default. */
+export function blurLevel(value: unknown): number {
+  return typeof value === 'number' && Number.isFinite(value) ? Math.min(10, Math.max(1, Math.round(value))) : DEFAULT_BLUR;
+}
+
+/**
+ * The blur's Gaussian sigma, in texels of the quarter-size picture it is computed on: a share of the picture's height,
+ * so a level looks the same at 480p and 1080p.
+ */
+export function blurSigma(level: number, blurHeight: number): number {
+  return Math.max(1, (blurLevel(level) / 10) * (blurHeight / 14));
+}
+
+/** What the background setting asks of the pipeline. */
+export function backgroundMode(setting: BackgroundSetting | undefined | null): 'none' | 'blur' | 'image' {
+  if (setting === 'blur') return 'blur';
+  if (setting && typeof setting === 'object' && 'image' in setting && setting.image) return 'image';
+  return 'none';
+}
+
+/**
+ * Fit a background picture to the frame the way CSS `object-fit: cover` does: it fills the frame, cropped evenly on
+ * the long side, never stretched. Returns the texture transform `imageUv = frameUv * scale + offset`.
+ */
+export function coverFit(imageWidth: number, imageHeight: number, frameWidth: number, frameHeight: number): { sx: number; sy: number; ox: number; oy: number } {
+  if (!(imageWidth > 0 && imageHeight > 0 && frameWidth > 0 && frameHeight > 0)) return { sx: 1, sy: 1, ox: 0, oy: 0 };
+  const image = imageWidth / imageHeight;
+  const frame = frameWidth / frameHeight;
+  if (image > frame) {
+    // Wider than the frame: its sides are cropped.
+    const sx = frame / image;
+    return { sx, sy: 1, ox: (1 - sx) / 2, oy: 0 };
+  }
+  const sy = image / frame;
+  return { sx: 1, sy, ox: 0, oy: (1 - sy) / 2 };
+}
+
+/** The size the person is found at: the frame scaled so its longer side is `longSide` (the model's input). */
+export function segmentationSize(width: number, height: number, longSide = 256): { width: number; height: number } {
+  if (!(width > 0 && height > 0)) return { width: longSide, height: longSide };
+  const scale = longSide / Math.max(width, height);
+  return { width: Math.max(1, Math.round(width * scale)), height: Math.max(1, Math.round(height * scale)) };
+}
+
+/** How fast the mask follows a new one: lower is steadier between frames, higher follows movement sooner. */
+export const MASK_FOLLOW = 0.65;
+
+/**
+ * Smooth the person mask between frames (Stream's temporal smoothing), against flicker at the edges, and write it as
+ * bytes for the GPU. `steady` keeps the running mask; a new size starts it over.
+ */
+export function smoothMask(confidence: Float32Array, steady: Float32Array | null, out: Uint8Array, follow = MASK_FOLLOW): Float32Array {
+  const fresh = steady === null || steady.length !== confidence.length;
+  const running = fresh ? new Float32Array(confidence) : steady;
+  for (let i = 0; i < confidence.length; i += 1) {
+    const value = Math.min(1, Math.max(0, confidence[i]));
+    const next = fresh ? value : running[i] + (value - running[i]) * follow;
+    running[i] = next;
+    out[i] = Math.round(next * 255);
+  }
+  return running;
+}
+
+/** The most frames a mask is reused for before the device is called overloaded. */
+export const MAX_SEGMENT_EVERY = 6;
+
+/**
+ * Amazon Chime's adaptive background filter: finding the person is the expensive part, so when it takes too much of
+ * each frame's time it runs only every Nth frame and the last mask is reused -- the picture still runs at the camera's
+ * rate. N comes back down once it is cheap again. `overloaded` is the point where even that is not enough.
+ */
+export class SegmentationPacer {
+  private average: number | null = null;
+  private changedAt = Number.NEGATIVE_INFINITY;
+  private every = 1;
+
+  constructor(private readonly maxEvery = MAX_SEGMENT_EVERY) {}
+
+  /** Run segmentation on every this-many frames. */
+  get interval(): number {
+    return this.every;
+  }
+
+  /** Average milliseconds one segmentation takes. */
+  get segmentMs(): number {
+    return this.average ?? 0;
+  }
+
+  shouldSegment(frameIndex: number): boolean {
+    return frameIndex % this.every === 0;
+  }
+
+  /** The share of the frame time segmentation takes now, given how often it runs. */
+  load(frameIntervalMs: number): number {
+    if (this.average === null || !(frameIntervalMs > 0)) return 0;
+    return this.average / (this.every * frameIntervalMs);
+  }
+
+  /** Whether segmenting even this rarely still takes too long. */
+  overloaded(frameIntervalMs: number): boolean {
+    return this.every >= this.maxEvery && this.load(frameIntervalMs) > 0.3;
+  }
+
+  /** One segmentation took `ms`, at a frame interval of `frameIntervalMs`. */
+  report(ms: number, frameIntervalMs: number, now: number): void {
+    if (!Number.isFinite(ms) || ms < 0) return;
+    this.average = this.average === null ? ms : this.average * 0.8 + ms * 0.2;
+    const load = this.load(frameIntervalMs);
+    if (load > 0.3 && this.every < this.maxEvery && now - this.changedAt >= 1000) {
+      this.every += 1;
+      this.changedAt = now;
+    } else if (load < 0.12 && this.every > 1 && now - this.changedAt >= 3000) {
+      this.every -= 1;
+      this.changedAt = now;
+    }
+  }
+}
+
+/** Timestamps MediaPipe's video mode accepts: in milliseconds and always increasing. */
+export class MonotonicClock {
+  private last = 0;
+  next(now: number): number {
+    this.last = Math.max(this.last + 1, Math.round(now));
+    return this.last;
+  }
+}