@voltkit-rtc/client 0.7.3 → 0.8.0

package/src/adaptive.ts

@@ -0,0 +1,295 @@
+/**
+ * Adaptive stream: receive each remote video at the size it is shown, and not
+ * at all while nobody can see it.
+ *
+ * The app attaches the element that shows a video (`attachVideo`). From that
+ * element's size on screen (CSS pixels × devicePixelRatio, at most 2) the
+ * smallest simulcast layer that fills it sharply is chosen; while no attached
+ * element is visible (scrolled away, `display: none`, the tab in the
+ * background) the video is paused at the media server, so it costs no
+ * bandwidth and no decoding. This is what LiveKit calls adaptiveStream and
+ * Tencent TRTC "small stream + receiveWhenViewVisible": the single biggest
+ * saving on a viewer with several tiles, where every tile otherwise decodes
+ * the publisher's full-size layer.
+ *
+ * Pure decisions are exported for tests; the browser parts (observers,
+ * timers) are injected.
+ */
+
+import type { VideoLayer } from './quality.js';
+
+/** Where a video is drawn, in CSS pixels, and how it is fitted into that box. */
+export interface ViewBox {
+  width: number;
+  height: number;
+  fit: 'cover' | 'contain';
+}
+
+/** Never ask for more than twice the CSS size: a 3× phone screen gains nothing visible from it. */
+export const MAX_PIXEL_RATIO = 2;
+/** A layer shown at most this much bigger than it is still looks sharp: avoids a whole layer for a few pixels. */
+export const MAX_UPSCALE = 1.15;
+
+/**
+ * The smallest layer that fills `box` sharply, or null when the layers are
+ * not known (an older publisher): then the server chooses, as before.
+ *
+ * With `cover` the picture is scaled until it fills the box (a part is
+ * cropped), with `contain` until it fits. Either way the scale is measured
+ * against each layer's own size, so a portrait layer list works too.
+ */
+export function layerForBox(layers: readonly VideoLayer[], box: ViewBox, pixelRatio = 1): number | null {
+  const usable = layers.filter((l) => l.width > 0 && l.height > 0).sort((a, b) => a.width * a.height - b.width * b.height);
+  if (!usable.length || box.width <= 0 || box.height <= 0) return null;
+  const ratio = Math.min(MAX_PIXEL_RATIO, Math.max(1, pixelRatio || 1));
+  const w = box.width * ratio;
+  const h = box.height * ratio;
+  for (const layer of usable) {
+    const scale = box.fit === 'cover' ? Math.max(w / layer.width, h / layer.height) : Math.min(w / layer.width, h / layer.height);
+    // The layer is at most slightly upscaled on screen.
+    if (scale <= MAX_UPSCALE) return layer.spatialLayer;
+  }
+  return usable[usable.length - 1]!.spatialLayer;
+}
+
+/** What one video should get now: paused, or a layer (null = let the server choose). */
+export interface AdaptiveWant {
+  visible: boolean;
+  spatialLayer: number | null;
+}
+
+/**
+ * The largest view among the visible ones decides; any visible view keeps the video flowing. A view not measured
+ * yet (just attached, before layout) counts as shown at the server's choice, so attaching never pauses a video
+ * for the moment before the browser has sized the element.
+ */
+export function wantFor(views: readonly { box: ViewBox; visible: boolean; measured?: boolean }[], layers: readonly VideoLayer[], pixelRatio: number, pageVisible: boolean): AdaptiveWant {
+  if (pageVisible && views.some((v) => v.measured === false)) return { visible: true, spatialLayer: null };
+  const shown = pageVisible ? views.filter((v) => v.visible && v.box.width > 0 && v.box.height > 0) : [];
+  if (!shown.length) return { visible: false, spatialLayer: null };
+  let best: number | null = null;
+  for (const view of shown) {
+    const layer = layerForBox(layers, view.box, pixelRatio);
+    if (layer === null) return { visible: true, spatialLayer: null };
+    best = best === null ? layer : Math.max(best, layer);
+  }
+  return { visible: true, spatialLayer: best };
+}
+
+/** Pause a hidden video only after this long, so scrolling past or a quick tab switch does not stop it. */
+export const PAUSE_AFTER_MS = 1500;
+/** Go down a layer only after this long (going up is immediate): a tile shrinking for a moment keeps its picture. */
+export const LOWER_AFTER_MS = 1200;
+
+export interface AdaptiveHost {
+  /** The layers this producer's publisher declared (smallest first or any order); [] when unknown. */
+  layers(producerId: string): readonly VideoLayer[];
+  /** Apply a decision for one video. */
+  apply(producerId: string, want: AdaptiveWant): void;
+  /** For tests: the environment. */
+  pixelRatio?(): number;
+  pageVisible?(): boolean;
+  now?(): number;
+  setTimer?(fn: () => void, ms: number): unknown;
+  clearTimer?(handle: unknown): void;
+  /** For tests: stand-ins for ResizeObserver / IntersectionObserver / document visibility. */
+  observe?(element: Element, onChange: (box: ViewBox | null, visible: boolean | null) => void): () => void;
+  onPageVisibility?(listener: () => void): () => void;
+}
+
+interface View {
+  box: ViewBox;
+  visible: boolean;
+  measured: boolean;
+  stop: () => void;
+}
+
+interface Entry {
+  views: Map<Element, View>;
+  applied: AdaptiveWant | null;
+  timer: unknown;
+}
+
+const FLOWING: AdaptiveWant = { visible: true, spatialLayer: null };
+const sameWant = (a: AdaptiveWant | null, b: AdaptiveWant) => a !== null && a.visible === b.visible && a.spatialLayer === b.spatialLayer;
+
+/**
+ * The videos being shown, by producer id. Decisions are applied through the
+ * host; going up (or becoming visible) is applied at once, going down (or
+ * hiding) after a short delay.
+ */
+export class AdaptiveVideos {
+  private readonly entries = new Map<string, Entry>();
+  private stopPage: (() => void) | null = null;
+
+  constructor(private readonly host: AdaptiveHost) {}
+
+  /** Show `producerId` in `element`; returns the function that stops. */
+  attach(producerId: string, element: Element): () => void {
+    let entry = this.entries.get(producerId);
+    if (!entry) {
+      entry = { views: new Map(), applied: null, timer: null };
+      this.entries.set(producerId, entry);
+    }
+    if (entry.views.has(element)) return () => this.detach(producerId, element);
+    const view: View = { box: { width: 0, height: 0, fit: 'contain' }, visible: true, measured: false, stop: () => undefined };
+    entry.views.set(element, view);
+    view.stop = this.observe(element, (box, visible) => {
+      if (box) {
+        view.box = box;
+        view.measured = true;
+      }
+      if (visible !== null) view.visible = visible;
+      this.update(producerId);
+    });
+    if (!this.stopPage) this.stopPage = this.watchPage(() => { for (const id of this.entries.keys()) this.update(id); });
+    this.update(producerId);
+    return () => this.detach(producerId, element);
+  }
+
+  detach(producerId: string, element: Element): void {
+    const entry = this.entries.get(producerId);
+    const view = entry?.views.get(element);
+    if (!entry || !view) return;
+    view.stop();
+    entry.views.delete(element);
+    if (entry.views.size) {
+      this.update(producerId);
+      return;
+    }
+    // Nothing shows it any more: back to how it was before (flowing, the server's choice).
+    this.clear(entry);
+    this.entries.delete(producerId);
+    if (entry.applied && !sameWant(entry.applied, { visible: true, spatialLayer: null })) this.host.apply(producerId, { visible: true, spatialLayer: null });
+    if (!this.entries.size) {
+      this.stopPage?.();
+      this.stopPage = null;
+    }
+  }
+
+  /** Whether the app shows this video through attachVideo (its decisions then apply to new consumers too). */
+  has(producerId: string): boolean {
+    return this.entries.has(producerId);
+  }
+
+  /** The decision in force, for a consumer made (or made again) after it: null when the app does not attach it. */
+  current(producerId: string): AdaptiveWant | null {
+    const entry = this.entries.get(producerId);
+    return entry ? this.decide(producerId, entry) : null;
+  }
+
+  /**
+   * A new consumer was made with `current()` applied (its layers, and paused when hidden): record that as applied,
+   * so nothing is sent twice, and drop a pending change made for the old one.
+   */
+  consumed(producerId: string): void {
+    const entry = this.entries.get(producerId);
+    if (!entry) return;
+    this.clear(entry);
+    entry.applied = this.decide(producerId, entry);
+  }
+
+  /** Stop everything (the session closed). */
+  dispose(): void {
+    for (const entry of this.entries.values()) {
+      this.clear(entry);
+      for (const view of entry.views.values()) view.stop();
+    }
+    this.entries.clear();
+    this.stopPage?.();
+    this.stopPage = null;
+  }
+
+  private decide(producerId: string, entry: Entry): AdaptiveWant {
+    return wantFor([...entry.views.values()], this.host.layers(producerId), this.host.pixelRatio?.() ?? 1, this.host.pageVisible?.() ?? true);
+  }
+
+  private update(producerId: string): void {
+    const entry = this.entries.get(producerId);
+    if (!entry) return;
+    const want = this.decide(producerId, entry);
+    // Nothing applied yet is the server's choice, flowing: asking for exactly that sends nothing.
+    if (sameWant(entry.applied ?? FLOWING, want)) {
+      this.clear(entry);
+      entry.applied ??= want;
+      return;
+    }
+    // Nothing applied yet means the video flows at the server's choice.
+    const prev = entry.applied ?? { visible: true, spatialLayer: null };
+    // Hiding, or a smaller layer than one applied, waits; anything else (showing, a bigger layer, a first layer) is at once.
+    const lower = (prev.visible && !want.visible) ||
+      (want.visible && prev.visible && want.spatialLayer !== null && prev.spatialLayer !== null && want.spatialLayer < prev.spatialLayer);
+    // Already waiting to lower: a repeated report must not restart the wait; it applies what is wanted when it runs out.
+    if (lower && entry.timer !== null) return;
+    this.clear(entry);
+    if (!lower) {
+      entry.applied = want;
+      this.host.apply(producerId, want);
+      return;
+    }
+    const delay = want.visible ? LOWER_AFTER_MS : PAUSE_AFTER_MS;
+    entry.timer = (this.host.setTimer ?? ((fn, ms) => setTimeout(fn, ms)))(() => {
+      entry.timer = null;
+      const now = this.decide(producerId, entry);
+      if (sameWant(entry.applied, now)) return;
+      entry.applied = now;
+      this.host.apply(producerId, now);
+    }, delay);
+  }
+
+  private clear(entry: Entry): void {
+    if (entry.timer !== null) (this.host.clearTimer ?? ((h) => clearTimeout(h as ReturnType<typeof setTimeout>)))(entry.timer);
+    entry.timer = null;
+  }
+
+  private observe(element: Element, onChange: (box: ViewBox | null, visible: boolean | null) => void): () => void {
+    if (this.host.observe) return this.host.observe(element, onChange);
+    return observeElement(element, onChange);
+  }
+
+  private watchPage(listener: () => void): () => void {
+    if (this.host.onPageVisibility) return this.host.onPageVisibility(listener);
+    if (typeof document === 'undefined') return () => undefined;
+    document.addEventListener('visibilitychange', listener);
+    return () => document.removeEventListener('visibilitychange', listener);
+  }
+}
+
+/** How an element fits its picture: a video's object-fit, else contain. */
+function fitOf(element: Element): 'cover' | 'contain' {
+  try {
+    const fit = typeof getComputedStyle === 'function' ? getComputedStyle(element).objectFit : '';
+    return fit === 'cover' || fit === 'fill' ? 'cover' : 'contain';
+  } catch {
+    return 'contain';
+  }
+}
+
+/** The browser's observers: size (ResizeObserver) and visibility on screen (IntersectionObserver). */
+function observeElement(element: Element, onChange: (box: ViewBox | null, visible: boolean | null) => void): () => void {
+  const stops: (() => void)[] = [];
+  const measure = () => {
+    const rect = element.getBoundingClientRect();
+    onChange({ width: rect.width, height: rect.height, fit: fitOf(element) }, null);
+  };
+  if (typeof ResizeObserver !== 'undefined') {
+    const resize = new ResizeObserver(() => measure());
+    resize.observe(element);
+    stops.push(() => resize.disconnect());
+  }
+  if (typeof IntersectionObserver !== 'undefined') {
+    const seen = new IntersectionObserver((entries) => {
+      const last = entries[entries.length - 1];
+      if (last) onChange(null, last.isIntersecting);
+    });
+    seen.observe(element);
+    stops.push(() => seen.disconnect());
+  }
+  measure();
+  return () => stops.forEach((stop) => stop());
+}
+
+/** The page is visible (not a background tab). */
+export function pageIsVisible(): boolean {
+  return typeof document === 'undefined' || document.visibilityState !== 'hidden';
+}

package/src/devices.ts

@@ -0,0 +1,70 @@
+/**
+ * Cameras, microphones and speakers (docs/implementation-progress/RTC_PERF_PROGRESS.md I1).
+ *
+ * What the big SDKs do, read from their code:
+ * - Amazon Chime (`DefaultDeviceController`): lists devices by kind, watches `devicechange` and tells observers
+ *   which lists changed; a microphone that ends is replaced so audio keeps flowing, a camera that ends is stopped and
+ *   reported (`audioInputStreamEnded` / `videoInputStreamEnded`) -- never swapped for another camera on its own.
+ * - LiveKit (`Room.handleDeviceChange`, `switchActiveDevice`): switches the published track to a chosen device; when
+ *   the active audio device disappears it moves to the first available one.
+ * - Tencent TRTC: `DEVICE_CHANGED {type, action: add | remove | active}`.
+ *
+ * This module holds the pure parts: comparing device lists and building the constraints for a chosen device.
+ */
+
+export type DeviceKind = 'audioinput' | 'videoinput' | 'audiooutput';
+
+/** A camera, microphone or speaker, as the browser lists it. Labels are empty until a permission was granted. */
+export interface MediaDevice {
+  deviceId: string;
+  groupId: string;
+  kind: DeviceKind;
+  label: string;
+}
+
+export interface DevicesChangedEvent {
+  /** Every device now. */
+  devices: MediaDevice[];
+  added: MediaDevice[];
+  removed: MediaDevice[];
+}
+
+/** The browser's device list, optionally of one kind. Empty outside a browser. */
+export async function enumerate(kind?: DeviceKind): Promise<MediaDevice[]> {
+  const md = typeof navigator !== 'undefined' ? navigator.mediaDevices : undefined;
+  if (!md?.enumerateDevices) return [];
+  const all = await md.enumerateDevices();
+  return all
+    .filter((d) => d.kind === 'audioinput' || d.kind === 'videoinput' || d.kind === 'audiooutput')
+    .filter((d) => !kind || d.kind === kind)
+    .map((d) => ({ deviceId: d.deviceId, groupId: d.groupId, kind: d.kind as DeviceKind, label: d.label }));
+}
+
+const key = (d: MediaDevice) => `${d.kind}:${d.deviceId}`;
+
+/** What appeared and what went between two lists. */
+export function diffDevices(before: readonly MediaDevice[], after: readonly MediaDevice[]): { added: MediaDevice[]; removed: MediaDevice[] } {
+  const was = new Set(before.map(key));
+  const now = new Set(after.map(key));
+  return {
+    added: after.filter((d) => !was.has(key(d))),
+    removed: before.filter((d) => !now.has(key(d))),
+  };
+}
+
+/**
+ * The constraints for capturing `deviceId` (or the system default when null), keeping everything else the app asked
+ * for: size, frame rate, echo cancellation. `base` is what publish was given for that kind.
+ */
+export function constraintsFor(base: boolean | MediaTrackConstraints | undefined, deviceId: string | null): MediaTrackConstraints {
+  const rest: MediaTrackConstraints = typeof base === 'object' && base !== null ? { ...base } : {};
+  delete rest.deviceId;
+  delete rest.groupId;
+  return deviceId === null ? rest : { ...rest, deviceId: { exact: deviceId } };
+}
+
+/** The device a live track captures, when it is a device capture at all (a canvas or a processed track has none). */
+export function deviceIdOf(track: MediaStreamTrack | null | undefined): string | null {
+  const id = track?.getSettings?.().deviceId;
+  return typeof id === 'string' && id.length > 0 ? id : null;
+}

package/src/diagnostics.ts

@@ -0,0 +1,94 @@
+/**
+ * Diagnostics for support (docs/implementation-progress/RTC_PERF_PROGRESS.md S4).
+ *
+ * Agora uploads its SDK log on request (`enableLogUpload`), Twilio sends call telemetry to Insights, Stream Video
+ * traces session events to its servers. Shipping telemetry needs a backend; what helps today is a snapshot an app can
+ * attach to a support ticket: the SDK's recent events (a ring buffer) and its state now.
+ */
+
+/** This package's version; kept equal to package.json by a test. */
+export const SDK_VERSION = '0.8.0';
+
+/** How many recent events the log keeps. */
+export const DIAGNOSTICS_LOG_SIZE = 200;
+
+export interface DiagnosticsEntry {
+  /** Unix ms. */
+  at: number;
+  event: string;
+  /** A short summary of what the event carried (no media objects). */
+  detail?: string;
+}
+
+/** The events worth keeping for support, and how to summarise each. */
+const SUMMARY: Record<string, (args: unknown[]) => string | undefined> = {
+  connectionStateChanged: ([e]) => { const s = e as { previous?: string; state?: string }; return `${s.previous} → ${s.state}`; },
+  disconnected: ([reason]) => String(reason),
+  closed: ([reason]) => String(reason),
+  mustReconnect: ([e]) => (e as { reason?: string }).reason,
+  mediaReconnecting: ([e]) => { const m = e as { transportId?: string; reconnecting?: boolean }; return `${m.transportId} ${m.reconnecting ? 'recovering' : 'recovered'}`; },
+  error: ([e]) => {
+    const err = e as { code?: string; message?: string };
+    const message = err.message ?? '';
+    // RpcError's message already starts with its code.
+    return (err.code && !message.startsWith(`${err.code}:`) ? `${err.code}: ${message}` : message).slice(0, 300);
+  },
+  streamFallback: ([e]) => { const f = e as { producerId?: string; fallback?: boolean; reason?: string }; return `${f.producerId} ${f.fallback ? 'held' : 'back'} (${f.reason})`; },
+  mediaWarning: ([e]) => { const w = e as { code?: string; active?: boolean }; return `${w.code} ${w.active ? 'started' : 'cleared'}`; },
+  cpuConstrained: ([e]) => String((e as { constrained?: boolean }).constrained),
+  videoLimited: ([e]) => (e as { reason?: string }).reason,
+  localTrackEnded: ([e]) => { const t = e as { kind?: string; recovered?: boolean }; return `${t.kind} ${t.recovered ? 'recovered' : 'off'}`; },
+  devicesChanged: ([e]) => { const d = e as { added?: unknown[]; removed?: unknown[] }; return `+${d.added?.length ?? 0} -${d.removed?.length ?? 0}`; },
+  kicked: ([e]) => (e as { reason?: string }).reason,
+  roomClosed: ([e]) => (e as { reason?: string }).reason,
+  usageLimit: ([e]) => (e as { reason?: string }).reason,
+  autoplayFailed: () => undefined,
+  trackEnded: ([e]) => (e as { producerId?: string }).producerId,
+  track: ([e]) => { const t = e as { identity?: string; source?: string }; return `${t.identity} ${t.source}`; },
+};
+
+/** A ring buffer of the SDK's recent events. */
+export class DiagnosticsLog {
+  private readonly entries: DiagnosticsEntry[] = [];
+
+  constructor(private readonly now: () => number = Date.now) {}
+
+  /** Keep an emitted event, when it is one worth keeping. */
+  record(event: string, args: unknown[]): void {
+    const summarize = SUMMARY[event];
+    if (!summarize) return;
+    let detail: string | undefined;
+    try {
+      detail = summarize(args);
+    } catch {
+      detail = undefined;
+    }
+    this.entries.push(detail === undefined ? { at: this.now(), event } : { at: this.now(), event, detail });
+    if (this.entries.length > DIAGNOSTICS_LOG_SIZE) this.entries.shift();
+  }
+
+  list(): DiagnosticsEntry[] {
+    return this.entries.map((e) => ({ ...e }));
+  }
+}
+
+export interface DiagnosticsTransport {
+  state: string;
+  path: { protocol: 'udp' | 'tcp' | null; relayed: boolean } | null;
+}
+
+export interface Diagnostics {
+  sdk: string;
+  at: number;
+  userAgent: string;
+  connectionState: string;
+  room: string | null;
+  identity: string | null;
+  role: string | null;
+  transports: { send: DiagnosticsTransport | null; recv: DiagnosticsTransport | null };
+  network: { rttMs: number | null; availableKbps: number | null } | null;
+  published: { source: string; kind: string; paused: boolean }[];
+  received: { identity: string; source: string; kind: string; paused: boolean }[];
+  /** Recent events, oldest first. */
+  log: DiagnosticsEntry[];
+}

package/src/fallback.ts

@@ -0,0 +1,169 @@
+/**
+ * Audio-only fallback (docs/implementation-progress/RTC_PERF_PROGRESS.md H1).
+ *
+ * When this viewer's connection is so poor that even the sound breaks up, remote camera videos are paused at the
+ * server so what bandwidth there is goes to the audio, and they come back once the connection has recovered.
+ *
+ * - Agora's `setStreamFallbackOption(uid, AUDIO_ONLY)` drops to the low stream and then to audio only, and reports
+ *   `stream-fallback` (fallback / recover).
+ * - Amazon Chime watches the audio it receives (`UnusableAudioWarningConnectionHealthPolicy`): about half of the
+ *   expected packets missing over 15 one-second samples suggests stopping video, with a 60 s cool-down.
+ *
+ * This follows Chime's signal -- the audio actually received -- measured as RTP loss (sequence gaps) rather than
+ * packet counts, because Opus DTX sends almost nothing during silence and a count would read silence as loss.
+ * Extreme only: {@link FALLBACK_LOSS} of the audio lost over {@link FALLBACK_SAMPLES} samples, with at least
+ * {@link MIN_PACKETS} packets to judge by. Back after a hold ({@link FIRST_HOLD_MS}, doubled for each relapse within
+ * {@link RELAPSE_MS}, up to {@link MAX_HOLD_MS}) once the loss is under {@link RECOVER_LOSS} -- or nobody spoke
+ * enough to tell, in which case video comes back and a still-bad connection falls back again, holding longer.
+ *
+ * The SFU cutting a video for bandwidth by itself is reported separately, by the server (`reason: 'bandwidth'`).
+ * Pure: the decisions take counters and a clock.
+ */
+import type { StatsEntry } from './stats.js';
+
+/** How often the received audio is sampled while the fallback could be needed. */
+export const PROTECT_SAMPLE_MS = 3000;
+/** This share of the audio lost (RTP sequence gaps) over the window: fall back. */
+export const FALLBACK_LOSS = 0.25;
+/** Below this share lost over the recovery window: video may come back. */
+export const RECOVER_LOSS = 0.05;
+/** Samples judged for falling back (15 s at 3 s) -- at least 3 of them. */
+export const FALLBACK_SAMPLES = 5;
+/** Samples judged for coming back (12 s). */
+export const RECOVER_SAMPLES = 4;
+/** Fewer audio packets than this over the fallback window are too few to judge. */
+export const MIN_PACKETS = 150;
+/** Fewer than this over the recovery window: nobody spoke enough to tell. */
+export const MIN_RECOVER_PACKETS = 30;
+/** The first fallback lasts at least this long. */
+export const FIRST_HOLD_MS = 20_000;
+/** A relapse doubles the hold, up to this. */
+export const MAX_HOLD_MS = 160_000;
+/** Falling back again within this long of coming back is a relapse. */
+export const RELAPSE_MS = 120_000;
+
+export interface AudioCounters {
+  received: number;
+  lost: number;
+}
+
+const num = (value: unknown): number => (typeof value === 'number' && Number.isFinite(value) ? value : 0);
+
+/** The received-audio counters of a receive transport's report, summed over its audio streams. */
+export function readInboundAudio(report: Iterable<StatsEntry>): AudioCounters {
+  let received = 0;
+  let lost = 0;
+  for (const e of report) {
+    if (typeof e !== 'object' || e === null || e.type !== 'inbound-rtp') continue;
+    if (e.kind !== 'audio' && e.mediaType !== 'audio') continue;
+    received += num(e.packetsReceived);
+    lost += Math.max(0, num(e.packetsLost));
+  }
+  return { received, lost };
+}
+
+/**
+ * The sent-audio counters of a send transport's report: packets sent, and how many of them the server reported
+ * lost (its RTCP receiver reports). For the publisher side (Agora's setLocalPublishFallbackOption): `received` is what
+ * got through.
+ */
+export function readOutboundAudio(report: Iterable<StatsEntry>): AudioCounters {
+  let sent = 0;
+  let lost = 0;
+  for (const e of report) {
+    if (typeof e !== 'object' || e === null) continue;
+    const audio = e.kind === 'audio' || e.mediaType === 'audio';
+    if (!audio) continue;
+    if (e.type === 'outbound-rtp') sent += num(e.packetsSent);
+    else if (e.type === 'remote-inbound-rtp') lost += Math.max(0, num(e.packetsLost));
+  }
+  return { received: Math.max(0, sent - lost), lost };
+}
+
+/** Decides when to fall back to audio only and when video may come back. */
+export class AudioProtection {
+  private samples: AudioCounters[] = [];
+  private previous: AudioCounters | null = null;
+  private activeSince: number | null = null;
+  private hold = FIRST_HOLD_MS;
+  private recoveredAt: number | null = null;
+
+  constructor(
+    private readonly onChange: (active: boolean) => void,
+    private readonly now: () => number = Date.now,
+  ) {}
+
+  /** Whether remote camera videos are held back now. */
+  get active(): boolean {
+    return this.activeSince !== null;
+  }
+
+  /** How long the current (or next first) fallback holds, in ms. */
+  get holdMs(): number {
+    return this.hold;
+  }
+
+  /**
+   * One reading of the cumulative counters. They only go up on one transport; lower ones are a new transport, and
+   * judging starts over from them.
+   */
+  feed(counters: AudioCounters): void {
+    const before = this.previous;
+    this.previous = counters;
+    if (before === null || counters.received < before.received || counters.lost < before.lost) {
+      this.samples = [];
+      return;
+    }
+    this.samples.push({ received: counters.received - before.received, lost: counters.lost - before.lost });
+    if (this.samples.length > Math.max(FALLBACK_SAMPLES, RECOVER_SAMPLES)) this.samples.shift();
+
+    if (this.activeSince === null) {
+      const window = this.samples.slice(-FALLBACK_SAMPLES);
+      if (window.length < 3) return;
+      const { total, lost } = sum(window);
+      if (total >= MIN_PACKETS && lost / total >= FALLBACK_LOSS) this.activate();
+      return;
+    }
+    if (this.now() - this.activeSince < this.hold || this.samples.length < RECOVER_SAMPLES) return;
+    const { total, lost } = sum(this.samples.slice(-RECOVER_SAMPLES));
+    if (total < MIN_RECOVER_PACKETS || lost / total < RECOVER_LOSS) this.recover();
+  }
+
+  /** Come back now (the app turned the fallback off). */
+  release(): void {
+    if (this.activeSince !== null) this.recover();
+  }
+
+  /** The transports went away (reconnect, close): forget everything, without telling. */
+  reset(): void {
+    this.activeSince = null;
+    this.samples = [];
+    this.previous = null;
+  }
+
+  private activate(): void {
+    const now = this.now();
+    const relapse = this.recoveredAt !== null && now - this.recoveredAt < RELAPSE_MS;
+    this.hold = relapse ? Math.min(this.hold * 2, MAX_HOLD_MS) : FIRST_HOLD_MS;
+    this.activeSince = now;
+    this.samples = [];
+    this.onChange(true);
+  }
+
+  private recover(): void {
+    this.activeSince = null;
+    this.recoveredAt = this.now();
+    this.samples = [];
+    this.onChange(false);
+  }
+}
+
+function sum(samples: readonly AudioCounters[]): { total: number; lost: number } {
+  let total = 0;
+  let lost = 0;
+  for (const s of samples) {
+    total += s.received + s.lost;
+    lost += s.lost;
+  }
+  return { total, lost };
+}