@voltkit-rtc/client 0.7.3 → 0.8.0

package/src/index.ts

@@ -30,7 +30,15 @@ import {
   type JoinMedia,
   type MediaPermissions,
 } from './media-permissions.js';
-import { NetworkQualityWatch, type NetworkQuality } from './network.js';
+import { NetworkQualityWatch, networkQualityOf, readTransportStats, type NetworkQuality } from './network.js';
+import { MediaPathWatch, SocketLiveness, type RestartOutcome } from './link.js';
+import { AdaptiveVideos, pageIsVisible, type AdaptiveWant } from './adaptive.js';
+import { AudioProtection, PROTECT_SAMPLE_MS, readInboundAudio, readOutboundAudio } from './fallback.js';
+import { ActiveSpeakers, AUDIO_LEVEL_INTERVAL_MS, SPEAKING_LEVEL } from './speakers.js';
+import { connectionPath, runPreflight, syntheticMedia, type PreflightOptions, type PreflightReport } from './preflight.js';
+import { DiagnosticsLog, SDK_VERSION, type Diagnostics, type DiagnosticsTransport } from './diagnostics.js';
+import { HEALTH_SAMPLE_MS, HealthDecisions, INTERRUPTED_AFTER_MS, readMediaSource, type MediaWarning } from './health.js';
+import { constraintsFor, deviceIdOf, diffDevices, enumerate, type DeviceKind, type DevicesChangedEvent, type MediaDevice } from './devices.js';
 import { socketUrl, TOKEN_EXPIRING_LEAD_MS, tokenExpiresAt } from './token.js';
 import {
   chooseCodec,
@@ -89,6 +97,12 @@ export { TOKEN_EXPIRING_LEAD_MS, tokenExpiresAt } from './token.js';
 export { allowedQuality, layerBitrate, videoEncodings, VIDEO_QUALITIES } from './quality.js';
 export type { LayerCount, VideoEncoding, VideoPolicy, VideoQuality, VideoLayer, ReceiveQuality, ScreenMode } from './quality.js';
 export type { CodecPreference } from './device.js';
+export type { DeviceKind, DevicesChangedEvent, MediaDevice } from './devices.js';
+export { SPEAKING_LEVEL } from './speakers.js';
+export type { MediaWarning, MediaWarningCode } from './health.js';
+export type { PreflightOptions, PreflightReport, PreflightStep, PreflightStepName } from './preflight.js';
+export { SDK_VERSION } from './diagnostics.js';
+export type { Diagnostics, DiagnosticsEntry, DiagnosticsTransport } from './diagnostics.js';
 export type { VideoLayerStats, VideoLimitedEvent, VideoLimitReason, VideoReceiveStats, VideoSendStats } from './stats.js';
 export type {
   ChatClientEvents,
@@ -261,6 +275,37 @@ export interface ClientOptions {
    * reserved for future native hosts, which have no user agent to detect.
    */
   handlerName?: DeviceHandlerName;
+  /**
+   * Audio-only fallback (0.8.0, on unless `false`): when this viewer's connection is so poor that the received sound
+   * breaks up, remote camera videos are paused so the audio keeps going, and resumed once it recovers. See
+   * `setAudioOnlyFallback` and the `streamFallback` event.
+   */
+  audioOnlyFallback?: boolean;
+}
+
+/** One participant's sound now (Agora's volume-indicator, Tencent TRTC's AUDIO_VOLUME). */
+export interface AudioLevel {
+  identity: string;
+  producerId: string;
+  source: TrackSource;
+  /** WebRTC's audioLevel: linear, 0 = silence, 1 = 0 dBov. */
+  level: number;
+  /** Above `SPEAKING_LEVEL` (-35 dBov, LiveKit's threshold). */
+  speaking: boolean;
+  /** This client's own microphone. */
+  local: boolean;
+}
+
+/** Why a remote video is held back: the SFU cut it for this viewer's bandwidth, or the SDK paused it to save the audio. */
+export type StreamFallbackReason = 'bandwidth' | 'audio-protection';
+
+export interface StreamFallbackEvent {
+  producerId: string;
+  identity: string;
+  source: TrackSource;
+  /** true: the video is held back (show the person's name or avatar); false: it is coming back. */
+  fallback: boolean;
+  reason: StreamFallbackReason;
 }
 
 export interface SlotClaim {
@@ -295,7 +340,26 @@ export interface RemoteProducer {
 
 const defaultSource = (kind: 'audio' | 'video'): TrackSource => (kind === 'audio' ? 'mic' : 'camera');
 
+/** A receiver's most recent audio level (0..1), from its synchronization sources: cheap, made for this. */
+function receiverLevel(receiver: RTCRtpReceiver | undefined): number {
+  const sources = receiver?.getSynchronizationSources?.() ?? [];
+  let level = 0;
+  for (const s of sources) if (typeof s.audioLevel === 'number') level = Math.max(level, s.audioLevel);
+  return level;
+}
+
+/** A microphone producer's level now (0..1), from its media-source statistics. */
+async function senderLevel(producer: Producer): Promise<number> {
+  const stats = await producer.getStats();
+  for (const s of stats.values() as Iterable<Record<string, unknown>>) {
+    if (s.type === 'media-source' && typeof s.audioLevel === 'number') return s.audioLevel;
+  }
+  return 0;
+}
+
 export interface ClientEvents extends ChatEvents {
+  /** A transport is recovering its media path; the room stays joined. */
+  mediaReconnecting: (e: { transportId: string; reconnecting: boolean }) => void;
   connected: () => void;
   disconnected: (reason: string) => void;
   /** idle → connecting → connected ⇄ reconnecting → closed. Read it any time as `connectionState`. */
@@ -390,6 +454,66 @@ export interface ClientEvents extends ChatEvents {
    * every 2 s, only while publishing a camera and someone listens.
    */
   videoLimited: (e: VideoLimitedEvent) => void;
+  /**
+   * This device is too busy to encode its camera at full quality (WebRTC's
+   * `qualityLimitationReason: "cpu"`, held for about 4 s), or no longer is.
+   * The moment to lighten its load, as LiveKit's and Agora's guides advise:
+   * `prioritizePerformance()`, video effects off, remote videos smaller (or
+   * attach them with `attachVideo`). Checked every 2 s while a camera is
+   * published and someone listens.
+   */
+  cpuConstrained: (e: { constrained: boolean }) => void;
+  /**
+   * A remote video is held back because this viewer's connection cannot carry it (`fallback: true`), or is coming
+   * back (`false`) -- Agora's `stream-fallback`. `reason: 'bandwidth'`: the media server cut it, the audio still
+   * flows. `'audio-protection'`: the received sound was breaking up, so the SDK paused remote cameras to save it
+   * (screen shares stay). Show the person's name or avatar meanwhile.
+   */
+  streamFallback: (e: StreamFallbackEvent) => void;
+  /**
+   * The publisher-side audio-only fallback (`setPublishFallback(true)`, off by default) paused this client's camera
+   * because its upload could not carry the sound (`fallback: true`), or brought it back (`false`). Agora's
+   * onLocalPublishFallbackToAudioOnly. Viewers see the camera paused meanwhile.
+   */
+  publishFallback: (e: { fallback: boolean }) => void;
+  /**
+   * A camera, microphone or speaker was plugged in or removed (Tencent TRTC's DEVICE_CHANGED, Chime's
+   * audioInputsChanged / videoInputsChanged). Watched only while someone listens.
+   */
+  devicesChanged: (e: DevicesChangedEvent) => void;
+  /**
+   * The camera or microphone this client publishes stopped from outside: unplugged, a Bluetooth headset switched off,
+   * the permission revoked. A microphone is captured again from the system default and keeps publishing
+   * (`recovered: true`), or is switched off if there is none; a camera is switched off (`recovered: false`) and never
+   * swapped for another one on its own -- offer `listDevices('videoinput')` and `switchDevice`, then
+   * `setCamEnabled(true)`.
+   */
+  localTrackEnded: (e: { kind: 'audio' | 'video'; deviceId: string | null; recovered: boolean }) => void;
+  /**
+   * Every participant's sound about four times a second while listened to (Agora's volume-indicator, TRTC's
+   * AUDIO_VOLUME): each remote microphone, music and shared audio this client receives, and its own microphone while
+   * on. For a "speaking" ring use `speaking`.
+   */
+  audioLevels: (levels: AudioLevel[]) => void;
+  /**
+   * Who is speaking, most active first, when that changes (Amazon Chime's active speaker ranking): for a speaker
+   * view or a highlighted tile. Microphones only; music does not count. This client is in it while its own
+   * microphone is on.
+   */
+  activeSpeakers: (e: { speakers: string[] }) => void;
+  /**
+   * A problem with this client's own camera or microphone starts (`active: true`) or clears (`false`), like Agora's
+   * exception events: `mic-silent` (on, but exact silence: a mute switch or a broken device), `camera-fps-low` (far
+   * fewer frames than asked: usually a dark room), `camera-interrupted` / `mic-interrupted` (the system took the
+   * device: a call, another app). Checked every 2 s while listened to.
+   */
+  mediaWarning: (e: MediaWarning) => void;
+  /**
+   * The browser refused to play an element handed to `play()` until the viewer interacts with the page (Agora's
+   * autoplay-failed, Tencent TRTC's AUTOPLAY_FAILED). Show "Tap to unmute" and call `resume()` from that tap; the SDK
+   * also retries every refused element on the next tap or key press by itself.
+   */
+  autoplayFailed: (e: { element: HTMLMediaElement; resume: () => Promise<void> }) => void;
 }
 
 type Handler = (...args: never[]) => void;
@@ -398,12 +522,14 @@ type Handler = (...args: never[]) => void;
 // event without listing it here (or listing one that does not exist) fails to
 // compile; the contract check compares this list with docs/sdk-contract.json.
 const EVENT_NAMES: { readonly [K in keyof ClientEvents]-?: true } = {
+  mediaReconnecting: true,
   connected: true, disconnected: true, connectionStateChanged: true, joined: true,
   track: true, trackEnded: true, trackMuted: true,
   participantJoined: true, participantLeft: true, participantMuted: true, participantUpdated: true,
   roleChanged: true, handRaised: true, forceMuted: true, claimChanged: true, roomMetadataChanged: true,
   mustReconnect: true, roomClosed: true, kicked: true, streamInterrupted: true, closed: true,
-  usageLimit: true, walletLow: true, tokenExpiring: true, networkQuality: true, videoLimited: true,
+  usageLimit: true, walletLow: true, tokenExpiring: true, networkQuality: true, videoLimited: true, cpuConstrained: true,
+  streamFallback: true, publishFallback: true, devicesChanged: true, localTrackEnded: true, audioLevels: true, activeSpeakers: true, mediaWarning: true, autoplayFailed: true,
   screenShareStarted: true, screenShareEnded: true, error: true, localMediaChanged: true, localStreamChanged: true,
   message: true, messageDeleted: true, chatMuted: true, typing: true,
 };
@@ -426,8 +552,6 @@ const CLOSE_REPLACED = 4001;
 /** The server ended the session on purpose and said why first: do not reconnect. */
 const CLOSE_ENDED = 4003;
 
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
 /** What setPreferredLayers sends; no temporal layer means the full frame rate. */
 interface PreferredLayers {
   spatialLayer: number;
@@ -478,6 +602,72 @@ function declareAudioQuality(rtpParameters: ms.RtpParameters, appData: Record<st
 }
 
 export class StreamClient {
+  private backoffWake: (() => void) | null = null;
+  private onlineTarget: Window | null = null;
+  private readonly online = () => {
+    if (this.closedByUs) return;
+    this.backoffWake?.();
+    this.socketLiveness.checkNow();
+  };
+  private readonly socketLiveness = new SocketLiveness({
+    active: () => !this.closedByUs && this.state === 'connected',
+    ping: () => { void this.rpc('ping').catch(() => undefined); },
+    dead: () => this.recoverConnection('socket-timeout'),
+  });
+  private readonly mediaPaths = new MediaPathWatch({
+    active: () => !this.closedByUs && this.state === 'connected',
+    restartIce: id => this.restartMediaIce(id),
+    reconnect: reason => this.recoverConnection(reason),
+    onPathChange: (transportId, reconnecting) => {
+      if (!this.closedByUs) this.emit('mediaReconnecting', { transportId, reconnecting });
+    },
+  });
+
+  private recoverConnection(reason: string): void {
+    if (this.closedByUs || !this.joinedOnce || this.state !== 'connected') return;
+    this.setState('reconnecting');
+    if (this.closedByUs) return; // a state listener may close synchronously
+    this.mediaPaths.reset();
+    if (this.closedByUs) return;
+    this.detachSocket(reason);
+    this.networkWatch.set(false);
+    this.emit('disconnected', reason);
+    void this.scheduleReconnect();
+  }
+
+  private async restartMediaIce(id: string): Promise<RestartOutcome> {
+    const transport = [this.sendTransport, this.recvTransport].find(t => t?.id === id);
+    const socket = this.ws;
+    if (!transport || transport.closed || this.closedByUs || this.state !== 'connected') return 'skipped';
+    const { iceParameters } = await this.rpc('restartIce', { transportId: id }) as { iceParameters: ms.IceParameters };
+    if (this.ws !== socket || transport.closed || this.closedByUs ||
+        (transport !== this.sendTransport && transport !== this.recvTransport)) return 'skipped';
+    await transport.restartIce({ iceParameters });
+    if (this.ws === socket && !this.closedByUs && !transport.closed && transport.connectionState === 'connected') {
+      this.mediaPaths.update(id, 'connected');
+    }
+    return 'restarted';
+  }
+
+  private watchTransport(transport: Transport): void {
+    transport.on('connectionstatechange', state => {
+      if (transport !== this.sendTransport && transport !== this.recvTransport) return;
+      if (!this.closedByUs && this.state === 'connected') this.mediaPaths.update(transport.id, state);
+    });
+    transport.observer?.on('close', () => this.mediaPaths.forget(transport.id));
+  }
+
+  private waitBackoff(ms: number): Promise<void> {
+    return new Promise(resolve => {
+      const finish = () => {
+        clearTimeout(timer);
+        if (this.backoffWake === finish) this.backoffWake = null;
+        resolve();
+      };
+      const timer = setTimeout(finish, ms);
+      this.backoffWake = finish;
+    });
+  }
   private ws: WebSocket | null = null;
   private device: Device | null = null;
   private sendTransport: Transport | null = null;
@@ -550,6 +740,67 @@ export class StreamClient {
    * the full picture.
    */
   private readonly preferredLayers = new Map<string, PreferredLayers>();
+  /**
+   * Adaptive stream (attachVideo): each attached remote video at the size it is
+   * shown, and paused at the server while no attached element is visible.
+   */
+  private readonly adaptive = new AdaptiveVideos({
+    layers: (id) => this.remoteVideoLayers.get(id) ?? [],
+    apply: (id, want) => { void this.applyAdaptive(id, this.withFallback(id, want)); },
+    pixelRatio: () => (typeof window !== 'undefined' ? window.devicePixelRatio : 1),
+    pageVisible: pageIsVisible,
+  });
+  /** Audio-only fallback: whether it may happen, its decisions, and the sampling that feeds them. */
+  private audioOnlyEnabled = true;
+  private readonly protection = new AudioProtection((active) => this.onAudioProtection(active));
+  private protectionTimer: ReturnType<typeof setInterval> | null = null;
+  private protectionSampling = false;
+  /** Publisher-side fallback: off unless asked for; its decisions, timer, and whether it holds the camera now. */
+  private publishFallbackEnabled = false;
+  private readonly publishProtection = new AudioProtection((active) => { void this.onPublishProtection(active); });
+  private publishProtectionTimer: ReturnType<typeof setInterval> | null = null;
+  private publishProtectionSampling = false;
+  private cameraHeldByFallback = false;
+  /** Remote videos the media server reports it cut for this viewer's bandwidth. */
+  private readonly bandwidthCut = new Set<string>();
+  /** Remote cameras the audio-only fallback holds back now (reported with fallback: true). */
+  private readonly protectionHeld = new Set<string>();
+  /** audioLevels / activeSpeakers: read only while someone listens. */
+  private levelTimer: ReturnType<typeof setInterval> | null = null;
+  private levelSampling = false;
+  private readonly speakers = new ActiveSpeakers();
+  /** mediaWarning: decisions, the sampling timer, and the system interruptions being timed. */
+  private readonly health = new HealthDecisions((warning) => this.emit('mediaWarning', warning));
+  private healthTimer: ReturnType<typeof setInterval> | null = null;
+  private healthSampling = false;
+  private readonly interruptions = new Map<'camera' | 'mic', ReturnType<typeof setTimeout>>();
+  /** getDiagnostics(): the SDK's recent events, for a support ticket. */
+  private readonly diagnosticsLog = new DiagnosticsLog();
+  /** play(): elements the browser refused to play until the viewer interacts, retried on the next gesture. */
+  private readonly autoplayBlocked = new Set<HTMLMediaElement>();
+  private gestureWatchOn = false;
+  private readonly onGesture = (): void => { void this.retryAutoplay(); };
+  /** setPlayoutDelay: the receive buffer asked for every remote track, in ms (null: the browser's own). */
+  private playoutDelayMs: number | null = null;
+  /** devicesChanged: the device list last seen, kept while someone listens. */
+  private knownDevices: MediaDevice[] = [];
+  private deviceWatchOn = false;
+  private readonly onDeviceChange = (): void => { void this.refreshDevices(); };
+  /** Videos adaptive stream has paused at the server, by producer id. */
+  private readonly adaptivePaused = new Set<string>();
+  /** The server has no pauseConsumer (an older controller): a hidden video drops to its smallest layer instead. */
+  private pauseUnsupported = false;
+  /** The last cpuConstrained said. */
+  private cpuConstrainedNow = false;
+  /** prioritizePerformance(): the camera's largest simulcast layer is not encoded while true. */
+  private performanceFirst = false;
+  /**
+   * Dynacast (the server's `producerLayers`): the highest layer any viewer needs, by producer id (-1 = nobody
+   * watches, the smallest layer stays on). No entry: every layer, as before the server said anything.
+   */
+  private readonly layersNeeded = new Map<string, number>();
+  /** The highest camera layer this client set active, and on which producer. */
+  private cameraMaxLayer: { producerId: string; layer: number } | null = null;
   /** endRoom() calls waiting for their reply; a roomClosed that arrives first answers them. */
   private readonly endRoomWaiters = new Set<() => void>();
   private closedByUs = false;
@@ -578,7 +829,14 @@ export class StreamClient {
   private readonly recvStatsBaseline: StatsBaseline = new Map();
   private readonly limitWatch = new VideoLimitWatch(
     (baseline, isCurrent) => this.sampleCamera(baseline, isCurrent),
-    (event) => this.emit('videoLimited', event),
+    (event) => {
+      this.emit('videoLimited', event);
+      const constrained = event.reason === 'cpu';
+      if (constrained !== this.cpuConstrainedNow) {
+        this.cpuConstrainedNow = constrained;
+        this.emit('cpuConstrained', { constrained });
+      }
+    },
     undefined,
     () => { this.frameRateLowSamples = 0; this.frameRateWarmupSamples = 2; },
   );
@@ -617,9 +875,53 @@ export class StreamClient {
     (quality) => this.emit('networkQuality', quality),
   );
 
+  /**
+   * A pre-call check (Twilio's runPreflight, LiveKit's ConnectionCheck, Agora's last-mile probe): joins the room the
+   * token names, publishes `options.media` (cloned, your tracks keep running) or a tone and a moving picture it makes,
+   * measures for `durationMs` (5 s) and leaves. Mint a host token for a test room nobody else is in: the session is
+   * real, and billed like any other for its few seconds. Resolves with each step, how media travelled (UDP is best;
+   * TCP or a relay means a firewall in the way), the round trip, loss and upload, and any camera or microphone warning
+   * seen. Never rejects: a failure is a step with `ok: false` and its error.
+   */
+  static async preflight(wsUrl: string, token: string, options: PreflightOptions = {}): Promise<PreflightReport> {
+    const client = new StreamClient(wsUrl, token, { audioOnlyFallback: false });
+    const synthetic = options.media ? null : syntheticMedia();
+    const stream = options.media ? new MediaStream(options.media.getTracks().map((t) => t.clone())) : synthetic!.stream;
+    try {
+      return await runPreflight({
+        connect: () => client.connect(),
+        publish: (media) => client.publish(media),
+        sendState: () => client.sendTransport?.connectionState ?? null,
+        sendStats: async () => (client.sendTransport ? ((await client.sendTransport.getStats()).values() as Iterable<Record<string, unknown>>) : null),
+        onWarning: (listener) => client.on('mediaWarning', (w) => listener(w.code, w.active)),
+        close: () => client.close(),
+      }, stream, options);
+    } finally {
+      synthetic?.stop();
+    }
+  }
+
+  /**
+   * Warm the connection before `connect()` (LiveKit's prepareConnection): a HEAD request to the server, so the
+   * name lookup and the TLS handshake are done by the time the socket opens. Call it as the user heads for the
+   * room (opening the room screen, say). Never rejects; it only saves time.
+   */
+  static async prepareConnection(wsUrl: string): Promise<void> {
+    try {
+      const url = new URL(wsUrl);
+      url.protocol = url.protocol === 'ws:' ? 'http:' : 'https:';
+      url.pathname = '/';
+      url.search = '';
+      await fetch(url.toString(), { method: 'HEAD', mode: 'no-cors', cache: 'no-store' });
+    } catch {
+      // Only a warm-up: connect() does the real work either way.
+    }
+  }
+
   constructor(private readonly wsUrl: string, token: string, private readonly options: ClientOptions = {}) {
     this.token = token;
     this.tokenExpiry = tokenExpiresAt(token);
+    this.audioOnlyEnabled = options.audioOnlyFallback !== false;
   }
 
   on<K extends keyof ClientEvents>(event: K, fn: ClientEvents[K]): () => void {
@@ -631,12 +933,18 @@ export class StreamClient {
     set.add(fn as Handler);
     // videoLimited and networkQuality cost a getStats every few seconds, so
     // they run only while heard.
-    if (event === 'videoLimited') this.syncLimitWatch();
+    if (event === 'videoLimited' || event === 'cpuConstrained') this.syncLimitWatch();
     if (event === 'networkQuality') this.syncNetworkWatch();
+    if (event === 'devicesChanged') this.syncDeviceWatch();
+    if (event === 'audioLevels' || event === 'activeSpeakers') this.syncLevelWatch();
+    if (event === 'mediaWarning') this.syncHealthWatch();
     return () => {
       set!.delete(fn as Handler);
-      if (event === 'videoLimited') this.syncLimitWatch();
+      if (event === 'videoLimited' || event === 'cpuConstrained') this.syncLimitWatch();
       if (event === 'networkQuality') this.syncNetworkWatch();
+      if (event === 'devicesChanged') this.syncDeviceWatch();
+      if (event === 'audioLevels' || event === 'activeSpeakers') this.syncLevelWatch();
+      if (event === 'mediaWarning') this.syncHealthWatch();
     };
   }
 
@@ -650,6 +958,10 @@ export class StreamClient {
     const previous = this.state;
     this.state = next;
     if (next === 'closed') this.stopTimers();
+    else {
+      this.syncLevelWatch();
+      this.syncHealthWatch();
+    }
     this.emit('connectionStateChanged', { state: next, previous });
   }
 
@@ -674,6 +986,19 @@ export class StreamClient {
   }
 
   private stopTimers(): void {
+    if (this.publishProtectionTimer) clearInterval(this.publishProtectionTimer);
+    this.publishProtectionTimer = null;
+    this.autoplayBlocked.clear();
+    this.watchGesture(false);
+    this.stopProtection();
+    this.stopLevelWatch();
+    this.stopHealthWatch(false);
+    this.stopDeviceWatch();
+    this.socketLiveness.stop();
+    this.mediaPaths.reset();
+    this.backoffWake?.();
+    this.onlineTarget?.removeEventListener('online', this.online);
+    this.onlineTarget = null;
     if (this.tokenTimer) clearTimeout(this.tokenTimer);
     this.tokenTimer = null;
     this.networkWatch.set(false);
@@ -684,6 +1009,7 @@ export class StreamClient {
   }
 
   private emit<K extends keyof ClientEvents>(event: K, ...args: Parameters<ClientEvents[K]>): void {
+    this.diagnosticsLog.record(event, args);
     for (const fn of this.listeners.get(event) ?? []) {
       try {
         (fn as (...a: unknown[]) => void)(...args);
@@ -737,6 +1063,11 @@ export class StreamClient {
     this.joinedOnce = true;
     this.skippedVideos.clear();
     this.setState('connected');
+    this.socketLiveness.start();
+    if (!this.onlineTarget && typeof window !== 'undefined' && window.addEventListener) {
+      this.onlineTarget = window;
+      window.addEventListener('online', this.online);
+    }
     this.scheduleTokenExpiring();
 
     // A role or mute can change while the socket is away. The fresh roster
@@ -767,10 +1098,11 @@ export class StreamClient {
 
     // Subscribe to everything already publishing before we announce readiness.
     // A close() part-way stops it: what is left would only fail, after closed.
-    for (const p of result.producers) {
-      if (closed()) throw closedWhileConnecting();
-      await this.consume(p).catch((err) => { if (!closed()) this.emit('error', err as Error); });
-    }
+    // All at once, as newProducer events already are (and the Flutter SDK does): each subscription reserves its slot
+    // before its first await, in this order, and the receive transport is made once. A room with six publishers
+    // starts six videos together instead of one after another, two round trips each.
+    if (closed()) throw closedWhileConnecting();
+    await Promise.all(result.producers.map((p) => this.consume(p).catch((err) => { if (!closed()) this.emit('error', err as Error); })));
 
     // Restore our own broadcast. This is what makes an origin failover
     // survivable for the host: it reconnects onto the new origin and the
@@ -780,6 +1112,9 @@ export class StreamClient {
     }
     if (closed()) throw closedWhileConnecting();
     this.syncNetworkWatch();
+    this.syncProtection();
+    this.syncLevelWatch();
+    this.syncPublishProtection();
     return result;
   }
 
@@ -823,12 +1158,17 @@ export class StreamClient {
       ws.addEventListener('open', onOpen, { once: true });
       ws.addEventListener('error', onError, { once: true });
       ws.addEventListener('message', (ev) => {
-        if (this.ws === ws) this.onMessage(ev.data as string);
+        if (this.ws === ws) {
+          this.socketLiveness.received();
+          this.onMessage(ev.data as string);
+        }
       });
       ws.addEventListener('close', (ev) => {
         ws.removeEventListener('error', onError);
         // Only the socket we currently consider ours may drive reconnection.
         if (this.ws !== ws) return;
+        this.socketLiveness.stop();
+        this.mediaPaths.reset();
         this.ws = null;
         this.failAllPending(new RpcError('not-connected', `socket closed (${ev.code})`));
         this.emit('disconnected', ev.reason || String(ev.code));
@@ -861,6 +1201,7 @@ export class StreamClient {
   }
 
   private detachSocket(reason = 'superseded', why = 'the connection was replaced'): void {
+    this.socketLiveness.stop();
     clearTimeout(this.openDeadline);
     const ws = this.ws;
     this.ws = null;
@@ -910,7 +1251,7 @@ export class StreamClient {
           return;
         }
         const backoff = Math.min(500 * 2 ** (this.reconnectAttempt - 1), 15_000);
-        await sleep(backoff + Math.random() * 400);
+        await this.waitBackoff(backoff + Math.random() * 400);
         if (this.closedByUs) return;
         try {
           this.teardownMedia();
@@ -942,6 +1283,9 @@ export class StreamClient {
 
   close(): void {
     void this.stopRecording().catch(() => undefined);
+    // Adaptive stream stops watching elements: nothing is received any more.
+    this.adaptive.dispose();
+    this.adaptivePaused.clear();
     // Only an app closing a live session is 'left'; kicked, room-closed and the
     // others have already reported themselves and set closedByUs.
     const wasOpen = !this.closedByUs;
@@ -975,7 +1319,15 @@ export class StreamClient {
   }
 
   private teardownMedia(): void {
+    this.mediaPaths.reset();
+    // The videos go with their consumers; a reconnect decides afresh. What was reported held back is withdrawn, so an
+    // app's "poor connection" overlay does not outlive it.
+    this.clearFallbacks();
     this.subscriptions.clear();
+    // Producers made after this are new: the server says again what their viewers need.
+    this.layersNeeded.clear();
+    // New consumers start paused at the server; adaptive stream decides again for each.
+    this.adaptivePaused.clear();
     for (const c of this.consumers.values()) c.close();
     this.consumers.clear();
     this.remoteInfo.clear();
@@ -1198,6 +1550,27 @@ export class StreamClient {
       case 'newProducer':
         await this.consume(data as RemoteProducer).catch((err) => this.emit('error', err as Error));
         break;
+      case 'streamFallback': {
+        // The media server cut (or resumed) one of this viewer's videos for its bandwidth.
+        const e = data as { producerId?: string; fallback?: boolean };
+        if (typeof e.producerId !== 'string' || !this.consumers.has(e.producerId)) break;
+        const cut = e.fallback === true;
+        if (cut === this.bandwidthCut.has(e.producerId)) break;
+        if (cut) this.bandwidthCut.add(e.producerId);
+        else this.bandwidthCut.delete(e.producerId);
+        this.emitFallback(e.producerId, cut, 'bandwidth');
+        break;
+      }
+      case 'producerLayers': {
+        // Dynacast: what this client's viewers need from one of its producers. Layers nobody receives are not
+        // encoded (the smallest always is, so a new viewer starts at once); null means every layer.
+        const e = data as { producerId?: string; spatialLayer?: number | null };
+        if (typeof e.producerId !== 'string') break;
+        if (typeof e.spatialLayer === 'number' && Number.isInteger(e.spatialLayer)) this.layersNeeded.set(e.producerId, e.spatialLayer);
+        else this.layersNeeded.delete(e.producerId);
+        if (this.cameraProducer()?.id === e.producerId) void this.cameraTask(() => this.syncCameraLayers());
+        break;
+      }
       case 'producerClosed': {
         const { producerId, identity } = data as { producerId: string; identity: string };
         this.subscriptions.delete(producerId);
@@ -1205,6 +1578,9 @@ export class StreamClient {
         this.receiveQualities.delete(producerId);
         this.remoteVideoLayers.delete(producerId);
         this.skippedVideos.delete(producerId);
+        this.adaptivePaused.delete(producerId);
+        this.bandwidthCut.delete(producerId);
+        this.protectionHeld.delete(producerId);
         const consumer = this.consumers.get(producerId);
         consumer?.close();
         this.consumers.delete(producerId);
@@ -1258,6 +1634,8 @@ export class StreamClient {
         const e = data as { identity: string; reason?: string };
         this.updateRoster((list) => list.filter((p) => p.identity !== e.identity));
         this.emit('participantLeft', e);
+        const speakers = this.speakers.remove(e.identity);
+        if (speakers) this.emit('activeSpeakers', { speakers });
         break;
       }
       case 'roleChanged': {
@@ -1400,17 +1778,19 @@ export class StreamClient {
   private ensureSendTransport(): Promise<Transport> {
     if (this.sendTransport) return Promise.resolve(this.sendTransport);
     if (this.sendTransportPending) return this.sendTransportPending;
-    this.sendTransportPending = this.createSendTransport().finally(() => {
-      this.sendTransportPending = null;
+    const pending = this.createSendTransport().finally(() => {
+      if (this.sendTransportPending === pending) this.sendTransportPending = null;
     });
-    return this.sendTransportPending;
+    return this.sendTransportPending = pending;
   }
 
   private async createSendTransport(): Promise<Transport> {
     if (!this.device) throw new RpcError('not-connected', 'not connected');
-
+    const device = this.device;
+    const socket = this.ws;
     const params = (await this.rpc('createTransport', { direction: 'send' })) as TransportOptions;
-    const transport = this.device.createSendTransport({
+    if (this.closedByUs || device !== this.device || socket !== this.ws) throw closedWhileConnecting();
+    const transport = device.createSendTransport({
       ...params,
       iceServers: this.joinResult?.iceServers,
     });
@@ -1430,23 +1810,26 @@ export class StreamClient {
     });
 
     this.sendTransport = transport;
+    this.watchTransport(transport);
     return transport;
   }
 
   private ensureRecvTransport(): Promise<Transport> {
     if (this.recvTransport) return Promise.resolve(this.recvTransport);
     if (this.recvTransportPending) return this.recvTransportPending;
-    this.recvTransportPending = this.createRecvTransport().finally(() => {
-      this.recvTransportPending = null;
+    const pending = this.createRecvTransport().finally(() => {
+      if (this.recvTransportPending === pending) this.recvTransportPending = null;
     });
-    return this.recvTransportPending;
+    return this.recvTransportPending = pending;
   }
 
   private async createRecvTransport(): Promise<Transport> {
     if (!this.device) throw new RpcError('not-connected', 'not connected');
-
+    const device = this.device;
+    const socket = this.ws;
     const params = (await this.rpc('createTransport', { direction: 'recv' })) as TransportOptions;
-    const transport = this.device.createRecvTransport({
+    if (this.closedByUs || device !== this.device || socket !== this.ws) throw closedWhileConnecting();
+    const transport = device.createRecvTransport({
       ...params,
       iceServers: this.joinResult?.iceServers,
     });
@@ -1457,6 +1840,7 @@ export class StreamClient {
     });
 
     this.recvTransport = transport;
+    this.watchTransport(transport);
     return transport;
   }
 
@@ -1621,6 +2005,7 @@ export class StreamClient {
       appData: { source: 'mic', audioQuality: quality },
     });
     await this.keepProducer(producer, generation, 'microphone');
+    this.watchTrackEnded(producer);
     this.micQuality = quality;
     // A microphone handed over switched off, or muted before a reconnect.
     await this.pauseOnServer(producer);
@@ -1768,6 +2153,7 @@ export class StreamClient {
         if (this.cameraEncodings && sameLayout(this.cameraEncodings, next) &&
             previousSize?.width === width && previousSize?.height === height) {
           await current.replaceTrack({ track });
+          this.watchTrackEnded(current);
           await this.applyLayerBitrates(current, next);
           return;
         }
@@ -1867,7 +2253,9 @@ export class StreamClient {
     this.receiveQualities.delete(producerId);
     // Kept first: a reconnect that interrupts the request still applies it.
     this.preferredLayers.set(producerId, layers);
-    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
+    // Shown through attachVideo: the lower of this and the size it is shown at.
+    const adaptive = this.adaptive.current(producerId);
+    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...(adaptive ? this.effectiveLayers(producerId, adaptive) : layers) });
   }
 
   private async collectVideoStats(baseline: StatsBaseline): Promise<VideoSendStats | null> {
@@ -1898,7 +2286,8 @@ export class StreamClient {
     }
     this.receiveQualities.set(producerId, quality);
     this.preferredLayers.set(producerId, layers);
-    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
+    const adaptive = this.adaptive.current(producerId);
+    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...(adaptive ? this.effectiveLayers(producerId, adaptive) : layers) });
   }
 
   private cameraProducer(): Producer | undefined {
@@ -1996,15 +2385,63 @@ export class StreamClient {
     // unpublish() may have run while this was being made (a publish, a
     // quality change or a track swap under way).
     await this.keepProducer(producer, generation, 'camera');
+    this.watchTrackEnded(producer);
+    this.syncPublishProtection();
     this.cameraEncodings = encodings;
     this.frameRateLowSamples = 0;
     this.frameRateWarmupSamples = 2;
     await this.pauseOnServer(producer);
     await this.applyDegradationPreference(producer);
     this.syncLimitWatch();
+    // A new camera producer starts with every layer: put the performance and viewer limits back on it.
+    await this.syncCameraLayers();
     return producer;
   }
 
+  /**
+   * Lighten this device's work while it is overloaded (LiveKit's
+   * `prioritizePerformance`): the camera's largest simulcast layer stops being
+   * encoded -- the most expensive part of publishing -- and viewers get the
+   * next one. `false` brings it back. Call it on `cpuConstrained`, together
+   * with your app's own savings (effects off, smaller remote videos). Kept for
+   * later publishes and reconnects until changed. A camera sent as one layer
+   * is left as it is.
+   */
+  async prioritizePerformance(on = true): Promise<void> {
+    this.performanceFirst = on;
+    await this.cameraTask(() => this.syncCameraLayers());
+  }
+
+  /** Whether `prioritizePerformance()` is in force. */
+  get performancePrioritized(): boolean {
+    return this.performanceFirst;
+  }
+
+  /**
+   * The camera's active simulcast layers: every layer, except the top one while
+   * performance comes first, and none above the highest any viewer needs
+   * (dynacast; the smallest always stays on, so a new viewer starts at once).
+   */
+  private async syncCameraLayers(): Promise<void> {
+    const producer = this.cameraProducer();
+    const count = this.cameraEncodings?.length ?? 0;
+    if (!producer || producer.closed || count < 2) return;
+    const top = count - 1;
+    let max = top;
+    if (this.performanceFirst) max = Math.min(max, top - 1);
+    const needed = this.layersNeeded.get(producer.id);
+    if (needed !== undefined) max = Math.min(max, Math.max(0, needed));
+    // A producer just made sends every layer: nothing to ask of it until a limit applies.
+    const applied = this.cameraMaxLayer?.producerId === producer.id ? this.cameraMaxLayer.layer : top;
+    if (applied === max) return;
+    try {
+      await producer.setMaxSpatialLayer(max);
+      this.cameraMaxLayer = { producerId: producer.id, layer: max };
+    } catch (err) {
+      if (!producer.closed) this.emit('error', err as Error);
+    }
+  }
+
   /**
    * Keep a producer just made -- unless unpublish() ran while it was being
    * made. Kept, it would publish again what was just unpublished, so it is
@@ -2082,7 +2519,7 @@ export class StreamClient {
 
   /** videoLimited sampling runs while a camera is published and someone listens. */
   private syncLimitWatch(): void {
-    this.limitWatch.set(!this.closedByUs && this.cameraProducer() !== undefined && (this.listenerCount('videoLimited') > 0 || this.cameraFrameRate === 60));
+    this.limitWatch.set(!this.closedByUs && this.cameraProducer() !== undefined && (this.listenerCount('videoLimited') > 0 || this.listenerCount('cpuConstrained') > 0 || this.cameraFrameRate === 60));
   }
 
   /**
@@ -2515,6 +2952,7 @@ export class StreamClient {
     const consumer = await transport.consume(params);
     current(consumer);
     this.consumers.set(remote.producerId, consumer);
+    if (this.playoutDelayMs !== null) this.applyPlayoutDelay(consumer.rtpReceiver);
     this.remoteVideoLayers.set(remote.producerId, remote.videoLayers ?? remote.appData?.videoLayers ?? (params as { videoLayers?: VideoLayer[] }).videoLayers ?? []);
 
     // Layers asked for before a reconnect: the new consumer gets them before
@@ -2526,6 +2964,12 @@ export class StreamClient {
       try { layers = { spatialLayer: receiveSpatialLayer(this.remoteVideoLayers.get(remote.producerId) ?? [], receiveQuality) }; }
       catch { layers = undefined; this.emit('error', new RpcError('quality-unavailable', 'this publisher no longer describes its video layers; using Auto')); }
     }
+    // Adaptive stream: a video the app shows through attachVideo starts at the
+    // size it is shown, and one nobody can see stays paused.
+    const adaptive = consumer.kind === 'video' ? this.adaptive.current(remote.producerId) : null;
+    if (adaptive) layers = this.effectiveLayers(remote.producerId, adaptive);
+    // A camera arriving while the audio-only fallback holds cameras back starts held back too.
+    const held = consumer.kind === 'video' && this.protection.active && (remote.source ?? 'camera') === 'camera';
     if (layers) {
       await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers }).catch((err) => this.emit('error', err as Error));
     }
@@ -2533,7 +2977,12 @@ export class StreamClient {
 
     // The server created it paused. Resume only now that our end exists,
     // otherwise the first frames arrive before there is anything to decode them.
-    await this.rpc('resumeConsumer', { consumerId: consumer.id });
+    this.adaptivePaused.delete(remote.producerId);
+    if ((adaptive && !adaptive.visible) || (held && !this.pauseUnsupported)) this.adaptivePaused.add(remote.producerId);
+    else await this.rpc('resumeConsumer', { consumerId: consumer.id });
+    if (held && this.pauseUnsupported) {
+      await this.rpc('setPreferredLayers', { consumerId: consumer.id, spatialLayer: 0, temporalLayer: 0 }).catch(() => undefined);
+    }
     current(consumer);
 
     const kind = consumer.kind as 'audio' | 'video';
@@ -2547,6 +2996,620 @@ export class StreamClient {
       producerId: remote.producerId,
       source,
     });
+    if (kind === 'video') this.adaptive.consumed(remote.producerId);
+    if (held) {
+      this.protectionHeld.add(remote.producerId);
+      this.emitFallback(remote.producerId, true, 'audio-protection');
+    }
+    this.syncProtection();
+  }
+
+  /**
+   * Adaptive stream (like LiveKit's adaptiveStream and Tencent TRTC's small
+   * stream with receiveWhenViewVisible): tell the SDK which element shows a
+   * remote video. It then receives the smallest layer that fills that element
+   * sharply (its size on screen × devicePixelRatio, at most 2) and pauses the
+   * video at the server while the element is not visible (scrolled away,
+   * `display: none`, or the tab in the background), so a grid of small tiles
+   * stops decoding full-size video. Attach the same video to several elements
+   * and the largest visible one decides. Returns the function that detaches
+   * it; detach when the element goes away. A ceiling set with
+   * `setReceiveQuality` or `setPreferredLayers` still applies. Works with any
+   * server; one without `pauseConsumer` gets the smallest layer instead of a
+   * pause.
+   */
+  attachVideo(producerId: string, element: Element): () => void {
+    return this.adaptive.attach(producerId, element);
+  }
+
+  /** The layers to ask for: the app's ceiling (setReceiveQuality / setPreferredLayers) and adaptive stream's, the lower. */
+  private effectiveLayers(producerId: string, want: AdaptiveWant | null): PreferredLayers {
+    let manual: PreferredLayers | undefined = this.preferredLayers.get(producerId);
+    const quality = this.receiveQualities.get(producerId);
+    if (quality && quality !== 'auto') {
+      try { manual = { spatialLayer: receiveSpatialLayer(this.remoteVideoLayers.get(producerId) ?? [], quality) }; } catch { manual = undefined; }
+    }
+    const adaptive = want?.visible ? want.spatialLayer : null;
+    if (adaptive === null || adaptive === undefined) return manual ?? { spatialLayer: -1, temporalLayer: -1 };
+    if (!manual || manual.spatialLayer < 0) return { spatialLayer: adaptive };
+    return { ...manual, spatialLayer: Math.min(manual.spatialLayer, adaptive) };
+  }
+
+  // ------------------------------------------------------- diagnostics
+
+  /**
+   * A snapshot for a support ticket (what Agora's log upload and Twilio's Insights are used for): the SDK version and
+   * browser, the session (room, identity, role, state), how each media transport is connected (UDP, TCP or relayed),
+   * the round trip and upload now, what is published and received, and the last 200 notable events (state changes,
+   * errors, recoveries, fallbacks, warnings) with their times. Plain JSON: `JSON.stringify` it and attach it. Holds
+   * no tokens and no media.
+   */
+  async getDiagnostics(): Promise<Diagnostics> {
+    const transport = async (t: Transport | null): Promise<DiagnosticsTransport | null> => {
+      if (!t || t.closed) return null;
+      let path: DiagnosticsTransport['path'] = null;
+      try {
+        path = connectionPath((await t.getStats()).values() as Iterable<Record<string, unknown>>);
+      } catch {
+        // Best effort.
+      }
+      return { state: t.connectionState, path };
+    };
+    let network: Diagnostics['network'] = null;
+    try {
+      const reports = await this.transportStats();
+      const current = new Map([...reports].map(([key, report]) => [key, readTransportStats(report)] as const));
+      if (current.size > 0) {
+        const q = networkQualityOf(current, new Map(), this.producers.size > 0);
+        network = { rttMs: q.rttMs, availableKbps: q.availableKbps };
+      }
+    } catch {
+      // Best effort.
+    }
+    const join = this.joinResult;
+    return {
+      sdk: SDK_VERSION,
+      at: Date.now(),
+      userAgent: typeof navigator !== 'undefined' ? navigator.userAgent : '',
+      connectionState: this.state,
+      room: join?.roomId ?? null,
+      identity: join?.identity ?? null,
+      role: join?.role ?? null,
+      transports: { send: await transport(this.sendTransport), recv: await transport(this.recvTransport) },
+      network,
+      published: [...this.producers.values()].filter((p) => !p.closed)
+        .map((p) => ({ source: String(p.appData.source ?? p.kind), kind: p.kind, paused: p.paused })),
+      received: [...this.consumers.entries()].filter(([, c]) => !c.closed).map(([producerId, c]) => ({
+        identity: this.remoteInfo.get(producerId)?.identity ?? '',
+        source: this.remoteSources.get(producerId) ?? (c.kind === 'audio' ? 'mic' : 'camera'),
+        kind: c.kind,
+        paused: c.paused || this.adaptivePaused.has(producerId),
+      })),
+      log: this.diagnosticsLog.list(),
+    };
+  }
+
+  // ------------------------------------------------------- autoplay
+
+  /**
+   * Play a `<video>` or `<audio>` showing a remote track, the way autoplay rules allow (Agora's and Tencent TRTC's
+   * autoplay handling). Resolves true when it plays. When the browser refuses until the viewer interacts with the
+   * page, resolves false, emits `autoplayFailed` with a `resume()` for a "Tap to unmute" button, and plays it -- with
+   * every other refused element -- on the next tap, click or key press by itself.
+   */
+  async play(element: HTMLMediaElement): Promise<boolean> {
+    try {
+      await element.play();
+      this.autoplayBlocked.delete(element);
+      return true;
+    } catch (err) {
+      if ((err as { name?: string })?.name !== 'NotAllowedError') throw err;
+      this.autoplayBlocked.add(element);
+      this.watchGesture(true);
+      this.emit('autoplayFailed', { element, resume: async () => { await this.play(element); } });
+      return false;
+    }
+  }
+
+  private watchGesture(on: boolean): void {
+    if (typeof document === 'undefined' || on === this.gestureWatchOn) return;
+    this.gestureWatchOn = on;
+    for (const type of ['pointerdown', 'keydown', 'touchend'] as const) {
+      if (on) document.addEventListener(type, this.onGesture, { capture: true });
+      else document.removeEventListener(type, this.onGesture, { capture: true });
+    }
+  }
+
+  private async retryAutoplay(): Promise<void> {
+    for (const element of [...this.autoplayBlocked]) {
+      try {
+        await element.play();
+        this.autoplayBlocked.delete(element);
+      } catch {
+        // Still refused (or the element went away); the next gesture tries again.
+        if (!element.isConnected) this.autoplayBlocked.delete(element);
+      }
+    }
+    if (this.autoplayBlocked.size === 0) this.watchGesture(false);
+  }
+
+  // ------------------------------------------------------- playout delay
+
+  /**
+   * Trade a little delay for smoother playback (Tencent TRTC's playoutDelay): every remote track is buffered about
+   * `ms` before it plays, which rides out the jitter of a mobile network without freezes. For an audience watching a
+   * stream, 300-500 ms is a good value; people talking with each other want `null` (the browser's own, as small as
+   * it can keep). Applies to what is received now and later. Browsers without the setting (before Chrome 114) keep
+   * their own; 0-4000 ms.
+   */
+  setPlayoutDelay(ms: number | null): void {
+    this.playoutDelayMs = ms === null ? null : Math.min(4000, Math.max(0, Math.round(ms)));
+    for (const consumer of this.consumers.values()) this.applyPlayoutDelay(consumer.rtpReceiver);
+  }
+
+  private applyPlayoutDelay(receiver: RTCRtpReceiver | undefined): void {
+    if (!receiver || !('jitterBufferTarget' in receiver)) return;
+    try {
+      (receiver as RTCRtpReceiver & { jitterBufferTarget: number | null }).jitterBufferTarget = this.playoutDelayMs;
+    } catch {
+      // Out of the browser's range: it keeps its own.
+    }
+  }
+
+  // ------------------------------------------------------- audio levels
+
+  /** Levels are read only while someone listens and the session is connected. */
+  private syncLevelWatch(): void {
+    const wanted = !this.closedByUs && this.state === 'connected' &&
+      (this.listenerCount('audioLevels') > 0 || this.listenerCount('activeSpeakers') > 0);
+    if (wanted && !this.levelTimer) {
+      this.levelTimer = setInterval(() => void this.sampleAudioLevels(), AUDIO_LEVEL_INTERVAL_MS);
+      (this.levelTimer as { unref?: () => void }).unref?.();
+    } else if (!wanted) {
+      this.stopLevelWatch();
+    }
+  }
+
+  private stopLevelWatch(): void {
+    if (this.levelTimer) clearInterval(this.levelTimer);
+    this.levelTimer = null;
+    this.speakers.reset();
+  }
+
+  /**
+   * One reading: each remote audio from its receiver's synchronization sources (the most recent packet's level, no
+   * statistics report needed), the own microphone from its sender's media-source statistics.
+   */
+  private async sampleAudioLevels(): Promise<void> {
+    if (this.levelSampling) return;
+    this.levelSampling = true;
+    try {
+      const levels: AudioLevel[] = [];
+      for (const [producerId, consumer] of this.consumers) {
+        if (consumer.kind !== 'audio' || consumer.closed) continue;
+        const info = this.remoteInfo.get(producerId);
+        if (!info) continue;
+        const level = info.paused ? 0 : receiverLevel(consumer.rtpReceiver);
+        levels.push({ identity: info.identity, producerId, source: this.remoteSources.get(producerId) ?? 'mic', level, speaking: level > SPEAKING_LEVEL, local: false });
+      }
+      const mic = this.micProducer();
+      const self = this.joinResult?.identity;
+      if (mic && !mic.paused && self) {
+        const level = await senderLevel(mic);
+        levels.push({ identity: self, producerId: mic.id, source: 'mic', level, speaking: level > SPEAKING_LEVEL, local: true });
+      }
+      if (!this.levelTimer) return;
+      if (this.listenerCount('audioLevels') > 0) this.emit('audioLevels', levels);
+      const loudest = new Map<string, number>();
+      for (const l of levels) if (l.source === 'mic') loudest.set(l.identity, Math.max(loudest.get(l.identity) ?? 0, l.level));
+      const speakers = this.speakers.update(loudest);
+      if (speakers) this.emit('activeSpeakers', { speakers });
+    } catch {
+      // Best effort; the next reading tries again.
+    } finally {
+      this.levelSampling = false;
+    }
+  }
+
+  // ------------------------------------------------------- media health
+
+  /** The published devices are sampled only while someone listens and the session is connected. */
+  private syncHealthWatch(): void {
+    const wanted = !this.closedByUs && this.state === 'connected' && this.listenerCount('mediaWarning') > 0;
+    if (wanted && !this.healthTimer) {
+      this.healthTimer = setInterval(() => void this.sampleHealth(), HEALTH_SAMPLE_MS);
+      (this.healthTimer as { unref?: () => void }).unref?.();
+    } else if (!wanted) {
+      this.stopHealthWatch(this.listenerCount('mediaWarning') > 0 && !this.closedByUs);
+    }
+  }
+
+  /** Stop sampling; `report` says each active warning cleared (nothing is said after close). */
+  private stopHealthWatch(report: boolean): void {
+    if (this.healthTimer) clearInterval(this.healthTimer);
+    this.healthTimer = null;
+    for (const timer of this.interruptions.values()) clearTimeout(timer);
+    this.interruptions.clear();
+    if (report) this.health.clearAll();
+  }
+
+  private async sampleHealth(): Promise<void> {
+    if (this.healthSampling) return;
+    this.healthSampling = true;
+    try {
+      const mic = this.micProducer();
+      const camera = this.cameraProducer();
+      const micLevel = mic && !mic.paused ? readMediaSource((await mic.getStats()).values() as Iterable<Record<string, unknown>>).level : null;
+      const cameraFps = camera && !camera.paused ? readMediaSource((await camera.getStats()).values() as Iterable<Record<string, unknown>>).fps : null;
+      if (!this.healthTimer) return;
+      this.health.feed({ micLevel, cameraFps, askedFps: this.videoFrameRate });
+      // An interrupted device that is no longer published never says it came back: that is over too.
+      if (!mic) this.health.interrupted('mic-interrupted', false);
+      if (!camera) this.health.interrupted('camera-interrupted', false);
+    } catch {
+      // Best effort; the next sample tries again.
+    } finally {
+      this.healthSampling = false;
+    }
+  }
+
+  // ---------------------------------------------------------------- devices
+
+  /**
+   * The cameras, microphones and speakers the browser offers, optionally of one kind. Labels are empty until the
+   * page has been granted camera or microphone access.
+   */
+  listDevices(kind?: DeviceKind): Promise<MediaDevice[]> {
+    return enumerate(kind);
+  }
+
+  /**
+   * Publish from another camera or microphone (LiveKit's switchActiveDevice, Agora's setDevice). The device is
+   * captured with the constraints publish was given (size, frame rate, echo cancellation) and its track replaces the
+   * published one: a camera of the same size swaps with no pause for viewers, a microphone always does. The track it
+   * replaces is stopped, releasing that device; the client's own stream follows (`localStreamChanged`), and a
+   * reconnect captures the chosen device again. A camera or microphone that is switched off stays off.
+   *
+   * For a camera published as a canvas or another processed track, capture the new camera yourself and pass your
+   * processed track to `replaceVideoTrack`; this rejects rather than replace your picture with the raw camera.
+   * Rejects when that kind is not being published.
+   */
+  async switchDevice(kind: 'audioinput' | 'videoinput', deviceId: string): Promise<void> {
+    const md = typeof navigator !== 'undefined' ? navigator.mediaDevices : undefined;
+    if (!md?.getUserMedia) throw new Error('switchDevice: this browser cannot capture media');
+    if (kind === 'videoinput') {
+      const current = this.cameraProducer()?.track ?? this.localStream?.getVideoTracks()[0] ?? null;
+      if (!current) throw new Error('switchDevice: no camera is being published');
+      if (!deviceIdOf(current)) {
+        throw new Error('switchDevice: the published video is not a camera capture (a canvas or processed track); capture the camera yourself and call replaceVideoTrack');
+      }
+      const video = constraintsFor(this.publishConstraints?.video, deviceId);
+      const track = (await md.getUserMedia({ video })).getVideoTracks()[0];
+      if (!track) throw new Error('switchDevice: the camera gave no video');
+      try {
+        await this.replaceVideoTrack(track);
+      } catch (err) {
+        track.stop();
+        throw err;
+      }
+      if (current !== track) current.stop();
+      this.publishConstraints = { ...(this.publishConstraints ?? {}), video };
+      return;
+    }
+    await this.micTask(async () => {
+      const producer = this.micProducer();
+      if (!producer) throw new Error('switchDevice: no microphone is being published');
+      const audio = constraintsFor(this.publishConstraints?.audio, deviceId);
+      const track = (await md.getUserMedia({ audio })).getAudioTracks()[0];
+      if (!track) throw new Error('switchDevice: the microphone gave no audio');
+      if (producer.closed || this.micProducer() !== producer) {
+        track.stop();
+        throw new Error('switchDevice: the microphone was unpublished meanwhile');
+      }
+      await this.adoptAudioTrack(producer, track);
+      this.publishConstraints = { ...(this.publishConstraints ?? {}), audio };
+    });
+  }
+
+  /** Put `track` on the microphone producer and in the client's own stream; the replaced track is stopped. */
+  private async adoptAudioTrack(producer: Producer, track: MediaStreamTrack): Promise<void> {
+    const previous = producer.track;
+    await producer.replaceTrack({ track });
+    const video = this.localStream ? this.localStream.getTracks().filter((t) => t.kind !== 'audio') : [];
+    this.localStream = new MediaStream([track, ...video]);
+    this.emit('localStreamChanged', this.localStream);
+    if (previous && previous !== track) previous.stop();
+    this.watchTrackEnded(producer);
+  }
+
+  /** Notice when the producer's present track stops from outside (unplugged, permission revoked). */
+  private watchTrackEnded(producer: Producer): void {
+    const track = producer.track;
+    if (!track || typeof track.addEventListener !== 'function') return;
+    track.addEventListener('ended', () => { void this.onTrackEnded(producer, track); }, { once: true });
+    // The system taking the device (a call, another app) mutes the track; held a moment, it is reported.
+    const which = producer.appData.source === 'mic' ? 'mic' : producer.appData.source === 'camera' ? 'camera' : null;
+    if (!which) return;
+    const code = which === 'mic' ? 'mic-interrupted' : 'camera-interrupted';
+    const current = () => !this.closedByUs && !producer.closed && producer.track === track;
+    track.addEventListener('mute', () => {
+      if (!current() || this.interruptions.has(which)) return;
+      this.interruptions.set(which, setTimeout(() => {
+        this.interruptions.delete(which);
+        if (current() && track.muted) this.health.interrupted(code, true);
+      }, INTERRUPTED_AFTER_MS));
+    });
+    track.addEventListener('unmute', () => {
+      const timer = this.interruptions.get(which);
+      if (timer) clearTimeout(timer);
+      this.interruptions.delete(which);
+      if (current()) this.health.interrupted(code, false);
+    });
+  }
+
+  /**
+   * A published camera or microphone stopped from outside. The microphone comes back from the system default (Chime
+   * keeps audio flowing, LiveKit moves to the first available device); a camera is switched off and reported, never
+   * swapped for another camera on its own (Chime stops it the same way).
+   */
+  private async onTrackEnded(producer: Producer, track: MediaStreamTrack): Promise<void> {
+    const live = () => !this.closedByUs && !producer.closed && producer.track === track && this.producers.get(producer.id) === producer;
+    if (!live()) return;
+    const deviceId = deviceIdOf(track);
+    if (producer.appData.source === 'mic') {
+      await this.micTask(async () => {
+        if (!live()) return;
+        try {
+          const md = navigator.mediaDevices;
+          const audio = constraintsFor(this.publishConstraints?.audio, null);
+          const next = (await md.getUserMedia({ audio })).getAudioTracks()[0];
+          if (!next) throw new Error('no microphone');
+          if (!live()) { next.stop(); return; }
+          await this.adoptAudioTrack(producer, next);
+          this.publishConstraints = { ...(this.publishConstraints ?? {}), audio };
+          this.emit('localTrackEnded', { kind: 'audio', deviceId, recovered: true });
+        } catch {
+          if (!live()) return;
+          await this.setMicEnabled(false).catch(() => undefined);
+          this.emit('localTrackEnded', { kind: 'audio', deviceId, recovered: false });
+        }
+      });
+      return;
+    }
+    if (producer.appData.source === 'camera') {
+      await this.setCamEnabled(false).catch(() => undefined);
+      if (!this.closedByUs) this.emit('localTrackEnded', { kind: 'video', deviceId, recovered: false });
+    }
+  }
+
+  /** devicesChanged is watched only while someone listens. */
+  private syncDeviceWatch(): void {
+    const md = typeof navigator !== 'undefined' ? navigator.mediaDevices : undefined;
+    const wanted = !this.closedByUs && this.listenerCount('devicesChanged') > 0 && typeof md?.addEventListener === 'function';
+    if (wanted && !this.deviceWatchOn) {
+      this.deviceWatchOn = true;
+      md!.addEventListener('devicechange', this.onDeviceChange);
+      void enumerate().then((list) => { if (this.deviceWatchOn) this.knownDevices = list; }).catch(() => undefined);
+    } else if (!wanted) {
+      this.stopDeviceWatch();
+    }
+  }
+
+  private stopDeviceWatch(): void {
+    if (!this.deviceWatchOn) return;
+    this.deviceWatchOn = false;
+    navigator.mediaDevices?.removeEventListener?.('devicechange', this.onDeviceChange);
+  }
+
+  private async refreshDevices(): Promise<void> {
+    let now: MediaDevice[];
+    try {
+      now = await enumerate();
+    } catch {
+      return;
+    }
+    if (!this.deviceWatchOn) return;
+    const { added, removed } = diffDevices(this.knownDevices, now);
+    this.knownDevices = now;
+    if (added.length > 0 || removed.length > 0) this.emit('devicesChanged', { devices: now, added, removed });
+  }
+
+  /** Adaptive stream's decision for a video, with the audio-only fallback holding remote cameras back. */
+  private withFallback(producerId: string, want: AdaptiveWant): AdaptiveWant {
+    if (this.protection.active && this.remoteSources.get(producerId) === 'camera') return { visible: false, spatialLayer: null };
+    return want;
+  }
+
+  /**
+   * Audio-only fallback (like Agora's AUDIO_ONLY stream fallback): on unless `audioOnlyFallback: false` was given.
+   * While this viewer's received sound breaks up badly -- a quarter of it lost over about 15 s -- remote camera
+   * videos are paused at the server (screen shares stay) so what bandwidth there is goes to the audio; they come
+   * back once it has recovered for a while, holding longer if it falls back again soon. Each held video is reported
+   * by `streamFallback`. `false` brings any held video back now and stops watching.
+   */
+  setAudioOnlyFallback(enabled: boolean): void {
+    this.audioOnlyEnabled = enabled;
+    if (!enabled) this.protection.release();
+    this.syncProtection();
+  }
+
+  /**
+   * The publisher side of the audio-only fallback (Agora's setLocalPublishFallbackOption(AUDIO_ONLY)), off by default:
+   * while this client's own sound breaks up on the way to the server -- a quarter of it lost over about 15 s -- its
+   * camera is paused (viewers see it paused) so the upload carries the voice, and resumed once it recovers, holding
+   * longer after a relapse. Reported by `publishFallback`. A camera you or a moderator switch off meanwhile stays off.
+   * `false` brings a held camera back now.
+   */
+  setPublishFallback(enabled: boolean): void {
+    this.publishFallbackEnabled = enabled;
+    if (!enabled) this.publishProtection.release();
+    this.syncPublishProtection();
+  }
+
+  private syncPublishProtection(): void {
+    const wanted = this.publishFallbackEnabled && !this.closedByUs && this.state === 'connected' &&
+      this.micProducer() !== undefined && (this.cameraProducer() !== undefined || this.publishProtection.active);
+    if (wanted && !this.publishProtectionTimer) {
+      this.publishProtectionTimer = setInterval(() => void this.samplePublishProtection(), PROTECT_SAMPLE_MS);
+      (this.publishProtectionTimer as { unref?: () => void }).unref?.();
+    } else if (!wanted && this.publishProtectionTimer) {
+      clearInterval(this.publishProtectionTimer);
+      this.publishProtectionTimer = null;
+    }
+  }
+
+  private async samplePublishProtection(): Promise<void> {
+    const transport = this.sendTransport;
+    if (this.publishProtectionSampling || !transport || transport.closed) return;
+    this.publishProtectionSampling = true;
+    try {
+      const report = await transport.getStats();
+      if (transport !== this.sendTransport || !this.publishProtectionTimer) return;
+      this.publishProtection.feed(readOutboundAudio(report.values() as Iterable<Record<string, unknown>>));
+    } catch {
+      // Best effort.
+    } finally {
+      this.publishProtectionSampling = false;
+    }
+  }
+
+  /** The publisher-side fallback decided: hold the camera, or bring it back if it is still ours to bring back. */
+  private async onPublishProtection(active: boolean): Promise<void> {
+    const camera = this.cameraProducer();
+    if (active) {
+      if (!camera || camera.paused) { this.syncPublishProtection(); return; }
+      this.cameraHeldByFallback = true;
+      camera.pause();
+      this.emit('publishFallback', { fallback: true });
+      await this.rpc('pauseProducer', { producerId: camera.id, paused: true }).catch(() => undefined);
+      this.emitLocalMedia();
+    } else if (this.cameraHeldByFallback) {
+      this.cameraHeldByFallback = false;
+      if (camera && camera.paused && !this.forcedOff.video) {
+        camera.resume();
+        await this.rpc('pauseProducer', { producerId: camera.id, paused: false }).catch(() => undefined);
+        this.emitLocalMedia();
+      }
+      this.emit('publishFallback', { fallback: false });
+    }
+    this.syncPublishProtection();
+  }
+
+  /** Whether the audio-only fallback is holding remote cameras back now. */
+  get audioOnlyFallbackActive(): boolean {
+    return this.protection.active;
+  }
+
+  /** The audio-only fallback decided: hold remote cameras back, or let them come back. */
+  private onAudioProtection(active: boolean): void {
+    for (const [producerId, consumer] of this.consumers) {
+      if (consumer.kind !== 'video' || this.remoteSources.get(producerId) !== 'camera') continue;
+      const want = this.withFallback(producerId, this.adaptive.current(producerId) ?? { visible: true, spatialLayer: null });
+      void this.applyAdaptive(producerId, want);
+      if (active && !this.protectionHeld.has(producerId)) {
+        this.protectionHeld.add(producerId);
+        this.emitFallback(producerId, true, 'audio-protection');
+      } else if (!active && this.protectionHeld.delete(producerId)) {
+        this.emitFallback(producerId, false, 'audio-protection');
+      }
+    }
+    this.syncProtection();
+  }
+
+  private emitFallback(producerId: string, fallback: boolean, reason: StreamFallbackEvent['reason']): void {
+    const info = this.remoteInfo.get(producerId);
+    this.emit('streamFallback', {
+      producerId,
+      identity: info?.identity ?? '',
+      source: this.remoteSources.get(producerId) ?? 'camera',
+      fallback,
+      reason,
+    });
+  }
+
+  /** Withdraw every held-back report (teardown; nothing is said after close) and forget the decisions. */
+  private clearFallbacks(): void {
+    if (!this.closedByUs) {
+      for (const producerId of this.bandwidthCut) this.emitFallback(producerId, false, 'bandwidth');
+      for (const producerId of this.protectionHeld) this.emitFallback(producerId, false, 'audio-protection');
+    }
+    this.bandwidthCut.clear();
+    this.protectionHeld.clear();
+    this.protection.reset();
+    this.stopProtection();
+  }
+
+  /**
+   * The received audio is sampled only while the fallback could matter: enabled, connected, with remote audio to
+   * judge by and a remote camera to hold back (or one held back already).
+   */
+  private syncProtection(): void {
+    let audio = false;
+    let camera = false;
+    for (const [producerId, consumer] of this.consumers) {
+      if (consumer.closed) continue;
+      if (consumer.kind === 'audio') audio = true;
+      else if (this.remoteSources.get(producerId) === 'camera') camera = true;
+    }
+    const wanted = this.audioOnlyEnabled && !this.closedByUs && this.state === 'connected' && audio && (camera || this.protection.active);
+    if (wanted && !this.protectionTimer) {
+      this.protectionTimer = setInterval(() => void this.sampleProtection(), PROTECT_SAMPLE_MS);
+      // Background sampling never keeps a server-side or test process alive (Node); browsers have no unref.
+      (this.protectionTimer as { unref?: () => void }).unref?.();
+    } else if (!wanted) {
+      this.stopProtection();
+    }
+  }
+
+  private stopProtection(): void {
+    if (this.protectionTimer) clearInterval(this.protectionTimer);
+    this.protectionTimer = null;
+  }
+
+  private async sampleProtection(): Promise<void> {
+    const transport = this.recvTransport;
+    if (this.protectionSampling || !transport || transport.closed) return;
+    this.protectionSampling = true;
+    try {
+      const report = await transport.getStats();
+      if (transport !== this.recvTransport || !this.protectionTimer) return;
+      this.protection.feed(readInboundAudio(report.values() as Iterable<Record<string, unknown>>));
+    } catch {
+      // Statistics are best effort; the next sample tries again.
+    } finally {
+      this.protectionSampling = false;
+    }
+  }
+
+  /** Apply adaptive stream's decision for one video: pause it, or resume it at its layer. */
+  private async applyAdaptive(producerId: string, want: AdaptiveWant): Promise<void> {
+    const consumer = this.consumers.get(producerId);
+    // Not received (yet): a consumer made later starts from the decision in force.
+    if (!consumer || consumer.kind !== 'video' || consumer.closed || this.closedByUs) return;
+    try {
+      if (!want.visible) {
+        if (this.pauseUnsupported) {
+          await this.rpc('setPreferredLayers', { consumerId: consumer.id, spatialLayer: 0, temporalLayer: 0 });
+          return;
+        }
+        if (this.adaptivePaused.has(producerId)) return;
+        this.adaptivePaused.add(producerId);
+        await this.rpc('pauseConsumer', { consumerId: consumer.id });
+        return;
+      }
+      if (this.adaptivePaused.delete(producerId)) await this.rpc('resumeConsumer', { consumerId: consumer.id });
+      await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...this.effectiveLayers(producerId, want) });
+    } catch (err) {
+      if (this.consumers.get(producerId) !== consumer || consumer.closed || this.closedByUs) return;
+      if (!want.visible && err instanceof RpcError && err.code === 'unknown-method') {
+        // An older server: drop to the smallest layer instead of pausing.
+        this.adaptivePaused.delete(producerId);
+        this.pauseUnsupported = true;
+        await this.applyAdaptive(producerId, want);
+        return;
+      }
+      this.emit('error', err as Error);
+    }
   }
 
   // ------------------------------------------------------------- moderation
@@ -2599,6 +3662,12 @@ export class StreamClient {
   }
 
   private async setKindEnabled(kind: 'audio' | 'video', enabled: boolean): Promise<void> {
+    // Your own choice about the camera ends the publisher-side fallback's hold on it: it never undoes your choice.
+    if (kind === 'video' && this.cameraHeldByFallback) {
+      this.cameraHeldByFallback = false;
+      this.publishProtection.reset();
+      this.emit('publishFallback', { fallback: false });
+    }
     const generation = this.publishGeneration;
     // Unpublished or closed while waiting: nothing is left to switch, and the
     // call resolves, as it does with nothing published (V072-03).
@@ -2643,6 +3712,7 @@ export class StreamClient {
   /** Stop publishing entirely and release the local devices. */
   async unpublish(): Promise<void> {
     void this.stopRecording().catch(error => this.emit('error', error as Error));
+    this.layersNeeded.clear();
     // All of it takes effect now; only telling the server is waited for. So
     // publish() and unpublish() act in the order they are called: a publish
     // called while this still waits is not undone by it, and one already