@voltkit-rtc/client 0.3.0 → 0.5.0

package/src/index.ts

@@ -13,12 +13,50 @@ import { Device, type types as ms } from 'mediasoup-client';
 type Consumer = ms.Consumer;
 type Producer = ms.Producer;
 type RtpCapabilities = ms.RtpCapabilities;
+type RtpCodecCapability = ms.RtpCodecCapability;
 type Transport = ms.Transport;
 type TransportOptions = ms.TransportOptions;
 
 import type { ChatEvents, ChatMessage, ChatState, SendMessageOptions } from './chat.js';
+import {
+  chooseCodec,
+  currentEnv,
+  defaultLayerCount,
+  findCodec,
+  h264HardwareEncode,
+  isAppleMobile,
+  isPhoneOrTablet,
+  normalizeCodec,
+  type CodecPreference,
+} from './device.js';
 import { MusicSource, type MusicInput, type MusicOptions, type TrackSource } from './music.js';
-import { allowedQuality, DEFAULT_VIDEO_POLICY, videoEncodings, type VideoPolicy, type VideoQuality } from './quality.js';
+import {
+  allowedQuality,
+  DEFAULT_VIDEO_POLICY,
+  sameLayout,
+  describeVideoLayers,
+  receiveSpatialLayer,
+  screenEncodings,
+  type ScreenMode,
+  type ReceiveQuality,
+  type VideoLayer,
+  topLayerSize,
+  videoEncodings,
+  type LayerCount,
+  type VideoEncoding,
+  type VideoPolicy,
+  type VideoQuality,
+} from './quality.js';
+import {
+  parseReceiveStats,
+  parseSendStats,
+  pruneBaseline,
+  VideoLimitWatch,
+  type StatsBaseline,
+  type VideoLimitedEvent,
+  type VideoReceiveStats,
+  type VideoSendStats,
+} from './stats.js';
 
 export { BroadcastPublisher, WhipError, whipEnd, whipOffer } from './broadcast.js';
 export { AudioMixer, MusicSource } from './music.js';
@@ -26,7 +64,9 @@ export type { MusicInput, MusicOptions, TrackSource } from './music.js';
 export type { BroadcastEvents, BroadcastStartOptions, BroadcastState } from './broadcast.js';
 export { ChatClient, ChatRpcError } from './chat.js';
 export { allowedQuality, layerBitrate, videoEncodings, VIDEO_QUALITIES } from './quality.js';
-export type { VideoEncoding, VideoPolicy, VideoQuality } from './quality.js';
+export type { LayerCount, VideoEncoding, VideoPolicy, VideoQuality, VideoLayer, ReceiveQuality, ScreenMode } from './quality.js';
+export type { CodecPreference } from './device.js';
+export type { VideoLayerStats, VideoLimitedEvent, VideoLimitReason, VideoReceiveStats, VideoSendStats } from './stats.js';
 export type {
   ChatClientEvents,
   ChatEvents,
@@ -64,18 +104,55 @@ export interface JoinResult {
   blockedSlots?: string[];
   /**
    * The highest video quality this project may publish, and the server's cap
-   * on a publisher's incoming bitrate. Absent from older servers: 720p.
+   * on a publisher's incoming bitrate. Absent from older servers: 1080p.
    */
   video?: VideoPolicy;
 }
 
 export interface PublishOptions {
+  /** 30 by default. Only the top layer uses 60; capture at 60 too. */
+  frameRate?: 30 | 60;
   /**
    * The video quality to send: 1080p, 720p or 480p. Defaults to the highest
    * the project allows (`videoPolicy.maxQuality`), and is lowered to it if
    * higher, and to the camera's own size if that is smaller.
    */
   quality?: VideoQuality;
+  /**
+   * The most simulcast layers to send: 1, 2 or 3. Defaults to 2 on phones and
+   * tablets and 3 elsewhere. Each layer is a separate encode. When a phone is
+   * too busy, the browser shrinks the picture and then sends fewer layers,
+   * dropping the top one: from three, viewers were left with a quarter of the
+   * picture; from two, the full picture survives every step down to 480×270.
+   * Never more than the picture allows: 3 from 960×540 up, 2 from 480×270,
+   * else 1. Two layers are half and full size. One is full size only, and
+   * every viewer then gets the same stream, whatever their connection.
+   */
+  layers?: LayerCount;
+  /**
+   * The video codec. `'auto'` (the default) sends H.264 on phones and tablets
+   * that encode it in hardware and VP8 everywhere else. `'h264'` and `'vp8'`
+   * force one; a forced H.264 the browser cannot send falls back to VP8 and
+   * fires `error`. A viewer whose browser cannot decode the codec cannot
+   * watch, so force VP8 if your audience uses such browsers.
+   */
+  codec?: CodecPreference;
+  /**
+   * What the encoder gives up first when the device cannot keep up:
+   * `'maintain-resolution'` lowers the frame rate, `'maintain-framerate'` the
+   * picture size, `'balanced'` some of each. Unset leaves the browser's
+   * default. Chrome also takes it from the track's `contentHint`: `'detail'`
+   * keeps resolution, `'motion'` keeps frame rate. Browsers that do not
+   * support it ignore it.
+   */
+  degradationPreference?: 'maintain-resolution' | 'maintain-framerate' | 'balanced';
+}
+
+export interface ScreenPublishOptions {
+  mode?: ScreenMode;
+  audio?: boolean;
+  /** An existing screen stream; otherwise capture is requested from this user gesture. */
+  stream?: MediaStream;
 }
 
 export interface SlotClaim {
@@ -91,7 +168,9 @@ export interface ParticipantInfo {
   identity: string;
   role: Role;
   handRaised: boolean;
+  handMetadata?: Record<string, unknown>;
   muted: boolean;
+  videoBlocked?: boolean;
   metadata?: unknown;
 }
 
@@ -102,6 +181,8 @@ export interface RemoteProducer {
   paused: boolean;
   /** What the track carries. A host playing music publishes two audio tracks: `mic` and `music`. */
   source?: TrackSource;
+  videoLayers?: VideoLayer[];
+  appData?: { videoLayers?: VideoLayer[] };
 }
 
 const defaultSource = (kind: 'audio' | 'video'): TrackSource => (kind === 'audio' ? 'mic' : 'camera');
@@ -123,8 +204,9 @@ export interface ClientEvents extends ChatEvents {
   trackMuted: (info: { identity: string; kind: 'audio' | 'video'; source: TrackSource; producerId: string; paused: boolean }) => void;
   participantJoined: (p: ParticipantInfo) => void;
   participantLeft: (p: { identity: string; reason?: string }) => void;
+  participantMuted: (e: { identity: string; kind?: 'audio' | 'video' | ''; paused: boolean }) => void;
   roleChanged: (e: { identity: string; role: Role }) => void;
-  handRaised: (e: { identity: string; handRaised: boolean }) => void;
+  handRaised: (e: { identity: string; handRaised: boolean; metadata?: Record<string, unknown>; reason?: string }) => void;
   /**
    * A moderator switched your microphone (`audio`), camera (`video`) or both
    * (no kind) off, or released it (`paused: false`). While paused you cannot
@@ -134,6 +216,8 @@ export interface ClientEvents extends ChatEvents {
   mustReconnect: (e: { reason: string; token?: string; retry: boolean }) => void;
   roomClosed: (e: { reason: string }) => void;
   kicked: (e: { by: string; reason?: string }) => void;
+  screenShareStarted: (e: { stream: MediaStream; mode: ScreenMode }) => void;
+  screenShareEnded: (e: { reason: 'stopped' | 'track-ended' }) => void;
   error: (err: Error) => void;
   localMediaChanged: (state: { micEnabled: boolean; camEnabled: boolean; musicEnabled: boolean }) => void;
   /** The local stream was replaced, typically after reacquiring devices on a reconnect. */
@@ -148,6 +232,15 @@ export interface ClientEvents extends ChatEvents {
    * account is topped up first -- tell the user, and stop asking them to wait.
    */
   usageLimit: (e: UsageLimitNotice) => void;
+  /**
+   * Your camera video is being held back by this device (`cpu`: too busy to
+   * encode it), by the upload (`bandwidth`), or by something else (`other`);
+   * or that has cleared (`none`). Fires when the reason changes and has held
+   * for about 4 s, so brief spikes stay quiet; never `none` first. Tell the
+   * publisher: "your device is busy" or "your connection is slow". Checked
+   * every 2 s, only while publishing a camera and someone listens.
+   */
+  videoLimited: (e: VideoLimitedEvent) => void;
 }
 
 type Handler = (...args: never[]) => void;
@@ -195,6 +288,23 @@ const CLOSE_ENDED = 4003;
 
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
+/** What setPreferredLayers sends; no temporal layer means the full frame rate. */
+interface PreferredLayers {
+  spatialLayer: number;
+  temporalLayer?: number;
+}
+
+const isLayer = (value: unknown): value is number => Number.isInteger(value) && (value as number) >= 0;
+
+/**
+ * Where a new camera stream's bandwidth estimate starts, in kbps. Chrome
+ * otherwise starts low and takes many seconds to climb to the top layer.
+ */
+const VIDEO_START_BITRATE_KBPS = 2000;
+
+/** Why a publish gave up: unpublish() ran while it was under way, and unpublish wins. */
+const unpublishedWhile = (what: string) => `the ${what} was unpublished while it was being published`;
+
 export class StreamClient {
   private ws: WebSocket | null = null;
   private device: Device | null = null;
@@ -224,11 +334,50 @@ export class StreamClient {
   /** In-app music being published, kept across reconnects so it can be published again. */
   private music: MusicSource | null = null;
   private musicProducerId: string | null = null;
+  private screenStream: MediaStream | null = null;
+  private screenMode: ScreenMode = 'detail';
+  private screenAudioEnabled = true;
+  private screenGeneration = 0;
+  private screenPending = false;
+  private screenStarted = false;
+  private screenEndedListener: (() => void) | null = null;
+  private readonly screenProducerIds = new Set<string>();
+  private readonly remoteVideoLayers = new Map<string, VideoLayer[]>();
+  private readonly receiveQualities = new Map<string, ReceiveQuality>();
+  private cameraFrameRate: 30 | 60 = 30;
+  private frameRateFallback = false;
+  private frameRateLowSamples = 0;
+  private frameRateWarmupSamples = 2;
   /** What each remote producer carries, so a pause can say "music" rather than just "audio". */
   private readonly remoteSources = new Map<string, TrackSource>();
+  /**
+   * The layers asked for each remote video, by producerId. A reconnect
+   * consumes everything again, and the new consumer would otherwise start at
+   * the full picture.
+   */
+  private readonly preferredLayers = new Map<string, PreferredLayers>();
   private closedByUs = false;
   private reconnecting = false;
   private reconnectAttempt = 0;
+  /** Set once a connect() has joined and set up. Only such a session is reconnected. */
+  private joinedOnce = false;
+  /** The camera producer's layers, to tell whether a new track fits them. */
+  private cameraEncodings: VideoEncoding[] | null = null;
+  // Changes to the camera producer run one at a time. Two overlapping swaps
+  // (an effect toggled on and off quickly) would otherwise both close the
+  // same producer and both produce a new one.
+  private cameraQueue: Promise<unknown> = Promise.resolve();
+  // Bumped by unpublish(). A producer still being made when it runs (camera,
+  // microphone or music) belongs to the publish it ended: it is closed, not
+  // kept.
+  private publishGeneration = 0;
+  private codecFallbackLogged = false;
+  private readonly sendStatsBaseline: StatsBaseline = new Map();
+  private readonly recvStatsBaseline: StatsBaseline = new Map();
+  private readonly limitWatch = new VideoLimitWatch(
+    (baseline) => this.sampleCamera(baseline),
+    (event) => this.emit('videoLimited', event),
+  );
 
   constructor(private readonly wsUrl: string, token: string) {
     this.token = token;
@@ -241,7 +390,16 @@ export class StreamClient {
       this.listeners.set(event, set);
     }
     set.add(fn as Handler);
-    return () => set!.delete(fn as Handler);
+    // videoLimited costs a getStats every 2 s, so it runs only while heard.
+    if (event === 'videoLimited') this.syncLimitWatch();
+    return () => {
+      set!.delete(fn as Handler);
+      if (event === 'videoLimited') this.syncLimitWatch();
+    };
+  }
+
+  private listenerCount(event: keyof ClientEvents): number {
+    return this.listeners.get(event)?.size ?? 0;
   }
 
   private emit<K extends keyof ClientEvents>(event: K, ...args: Parameters<ClientEvents[K]>): void {
@@ -257,15 +415,51 @@ export class StreamClient {
   // ------------------------------------------------------------- connection
 
   async connect(): Promise<JoinResult> {
-    await this.openSocket();
-    const result = (await this.rpc('join', { token: this.token })) as JoinResult;
-    this.joinResult = result;
-
-    this.device = new Device();
-    await this.device.load({ routerRtpCapabilities: result.routerRtpCapabilities });
+    let result: JoinResult;
+    try {
+      await this.openSocket();
+      result = (await this.rpc('join', { token: this.token })) as JoinResult;
+      this.joinResult = result;
 
+      this.device = this.createDevice();
+      await this.device.load({ routerRtpCapabilities: result.routerRtpCapabilities });
+    } catch (err) {
+      // A session that never got going: the app has the error. Left open,
+      // the socket would be closed by the server's join deadline instead, and
+      // that close would start reconnecting a session that never began.
+      if (!this.joinedOnce) this.detachSocket();
+      throw err;
+    }
+    // Joined and set up: the next drop gets every reconnect attempt again.
+    // Not when the socket opens -- a join refused with a code worth retrying
+    // (no-capacity) never used them up, and every client of a full cluster
+    // retried about twice a second, forever.
+    this.reconnectAttempt = 0;
+    this.joinedOnce = true;
+
+    // A role or mute can change while the socket is away. The fresh roster
+    // decides what may return; reconnecting never turns a blocked device on.
+    if (result.role !== 'host' && result.role !== 'cohost' && (this.wasPublishing || this.screenPending)) {
+      await this.unpublish();
+    }
+    const own = result.participants.find((p) => p.identity === result.identity);
+    if (own?.muted) this.applyLocalModeration('audio');
+    if (own?.videoBlocked) this.applyLocalModeration('video');
     this.emit('joined', result);
 
+    // A video that ended while this client was away sends it no
+    // producerClosed: forget the layers asked for it.
+    const present = new Set(result.producers.map((p) => p.producerId));
+    for (const producerId of this.preferredLayers.keys()) {
+      if (!present.has(producerId)) {
+        this.preferredLayers.delete(producerId);
+        this.receiveQualities.delete(producerId);
+        this.remoteVideoLayers.delete(producerId);
+      }
+    }
+
+    for (const producerId of this.remoteVideoLayers.keys()) if (!present.has(producerId)) this.remoteVideoLayers.delete(producerId);
+
     // Subscribe to everything already publishing before we announce readiness.
     for (const p of result.producers) {
       await this.consume(p).catch((err) => this.emit('error', err as Error));
@@ -280,6 +474,11 @@ export class StreamClient {
     return result;
   }
 
+  /** A new mediasoup Device for each join. Its own method so tests, which have no WebRTC, can stand one in. */
+  private createDevice(): Device {
+    return new Device();
+  }
+
   private openSocket(): Promise<void> {
     // Abandon any previous socket first. Its close handler checks identity
     // against `this.ws`, so once it is detached it can no longer trigger a
@@ -293,7 +492,6 @@ export class StreamClient {
       this.ws = ws;
 
       const onOpen = () => {
-        this.reconnectAttempt = 0;
         this.emit('connected');
         resolve();
       };
@@ -318,6 +516,9 @@ export class StreamClient {
           // displace it, and it would displace us right back, forever.
           this.emit('error', new Error('this identity connected from somewhere else'));
           this.closedByUs = true;
+          // The producers outlive this close until the app calls close(); the
+          // videoLimited sampling must not, or it reports after `closed`.
+          this.limitWatch.reset();
           this.emit('closed', 'replaced');
           return;
         }
@@ -325,9 +526,13 @@ export class StreamClient {
           // Normally roomClosed has already ended the session; this covers a
           // close that arrived without it.
           this.closedByUs = true;
+          this.limitWatch.reset();
           this.emit('closed', 'room-closed');
           return;
         }
+        // Before a first successful join the app is still waiting on
+        // connect(), which reports the failure; there is nothing to restore.
+        if (!this.joinedOnce) return;
         void this.scheduleReconnect();
       });
     });
@@ -370,8 +575,12 @@ export class StreamClient {
         if (this.closedByUs) return;
         try {
           this.teardownMedia();
-          await this.connect();
-          return; // connect() resets the attempt counter once the socket opens
+          await this.connect(); // which resets the attempt counter once joined
+          // The socket can drop again while the session is being restored,
+          // and its close handler then finds this loop still running and
+          // leaves it to us. Returning would leave the client disconnected
+          // for good, with no `closed`.
+          if (this.ws) return;
         } catch (err) {
           this.emit('error', err as Error);
           // Stop on failures a retry cannot resolve. Hammering a room that has
@@ -394,8 +603,17 @@ export class StreamClient {
     // others have already reported themselves and set closedByUs.
     const wasOpen = !this.closedByUs;
     this.closedByUs = true;
+    this.publishGeneration += 1;
     this.wasPublishing = false;
     this.teardownMedia();
+    this.preferredLayers.clear();
+    this.receiveQualities.clear();
+    this.remoteVideoLayers.clear();
+    this.releaseScreen();
+    this.music?.close();
+    this.music = null;
+    // Nothing is emitted after closing, so not even a clearing videoLimited.
+    this.limitWatch.reset();
     // Producers no longer stop their tracks, so releasing the camera and
     // microphone is ours to do -- otherwise the device light stays on.
     this.localStream?.getTracks().forEach((t) => t.stop());
@@ -412,6 +630,11 @@ export class StreamClient {
     for (const p of this.producers.values()) p.close();
     this.producers.clear();
     this.musicProducerId = null;
+    this.screenProducerIds.clear();
+    this.cameraEncodings = null;
+    // Paused, not reset: a reconnect's republish starts it again, and a
+    // limitation already reported is not reported twice.
+    this.syncLimitWatch();
     this.sendTransport?.close();
     this.recvTransport?.close();
     this.sendTransport = null;
@@ -484,6 +707,9 @@ export class StreamClient {
         break;
       case 'producerClosed': {
         const { producerId, identity } = data as { producerId: string; identity: string };
+        this.preferredLayers.delete(producerId);
+        this.receiveQualities.delete(producerId);
+        this.remoteVideoLayers.delete(producerId);
         const consumer = this.consumers.get(producerId);
         if (!consumer) break; // never consumed it, or already cleaned up
         consumer.close();
@@ -506,16 +732,28 @@ export class StreamClient {
       case 'participantJoined':
         this.emit('participantJoined', data as ParticipantInfo);
         break;
+      case 'participantMuted': {
+        const wire = data as { identity: string; kind?: 'audio' | 'video' | ''; muted?: boolean; paused?: boolean };
+        const e = { identity: wire.identity, kind: wire.kind, paused: wire.paused ?? wire.muted ?? false };
+        if (this.joinResult) this.joinResult.participants = this.joinResult.participants.map((p) =>
+          p.identity !== e.identity ? p : { ...p,
+            ...(!e.kind || e.kind === 'audio' ? { muted: e.paused } : {}),
+            ...(!e.kind || e.kind === 'video' ? { videoBlocked: e.paused } : {}),
+          });
+        this.emit('participantMuted', e);
+        break;
+      }
       case 'participantLeft':
         this.emit('participantLeft', data as { identity: string; reason?: string });
         break;
       case 'roleChanged': {
         const e = data as { identity: string; role: Role };
+        if (e.identity === this.joinResult?.identity) this.joinResult.role = e.role;
         this.emit('roleChanged', e);
         // Demoted: the server has already closed our producers. Release the
         // camera and microphone too, instead of capturing for nobody.
         const canPublish = e.role === 'host' || e.role === 'cohost';
-        if (e.identity === this.joinResult?.identity && !canPublish && this.producers.size > 0) {
+        if (e.identity === this.joinResult?.identity && !canPublish && (this.wasPublishing || this.screenPending)) {
           void this.unpublish().catch(() => undefined);
         }
         break;
@@ -530,9 +768,7 @@ export class StreamClient {
         // changes nothing here -- switching back on is the user's choice.
         const e = data as { by: string; kind?: 'audio' | 'video' | ''; paused: boolean };
         if (e.paused) {
-          for (const p of this.producers.values()) {
-            if (!e.kind || p.kind === e.kind) p.pause();
-          }
+          this.applyLocalModeration(e.kind);
           this.emitLocalMedia();
         }
         this.emit('forceMuted', e);
@@ -620,7 +856,7 @@ export class StreamClient {
         .catch(fail);
     });
     transport.on('produce', ({ kind, rtpParameters, appData }, ok, fail) => {
-      this.rpc('produce', { transportId: transport.id, kind, rtpParameters, appData, source: appData.source })
+      this.rpc('produce', { transportId: transport.id, kind, rtpParameters, appData, source: appData.source, videoLayers: appData.videoLayers })
         .then((res) => ok({ id: (res as { producerId: string }).producerId }))
         .catch(fail);
     });
@@ -664,13 +900,24 @@ export class StreamClient {
    *
    * `options.quality` picks 1080p, 720p or 480p (see `videoPolicy` for what
    * the project allows). Capture at least that size for the full picture.
+   * `options.layers`, `codec` and `degradationPreference` tune how the video
+   * is encoded; the defaults suit both phones and computers. The options are
+   * kept for a reconnect's republish.
    */
   async publish(stream: MediaStream, constraints?: MediaStreamConstraints, options?: PublishOptions): Promise<void> {
+    const generation = this.publishGeneration;
     const transport = await this.ensureSendTransport();
+    // unpublish() ran first: this publish is over before it began, and must
+    // leave nothing behind that a reconnect would publish again.
+    if (generation !== this.publishGeneration) throw new Error(unpublishedWhile('stream'));
     this.localStream = stream;
     this.wasPublishing = true;
     if (constraints) this.publishConstraints = constraints;
-    if (options) this.publishOptions = { ...options };
+    if (options) {
+      this.publishOptions = { ...options };
+      this.frameRateFallback = false;
+      this.frameRateLowSamples = 0;
+    }
 
     const audio = stream.getAudioTracks()[0];
     if (audio) {
@@ -682,14 +929,20 @@ export class StreamClient {
         codecOptions: { opusStereo: true, opusDtx: true, opusFec: true },
         appData: { source: 'mic' },
       });
-      this.producers.set(producer.id, producer);
+      await this.keepProducer(producer, generation, 'microphone');
+      // A microphone handed over switched off, or muted before a reconnect.
+      await this.pauseOnServer(producer);
     }
 
     const video = stream.getVideoTracks()[0];
-    if (video) await this.produceCamera(transport, video);
+    if (video) {
+      // The track is read when the camera's turn comes: one handed to
+      // replaceVideoTrack while this publish was under way is the one to send.
+      await this.cameraTask(() => this.produceCamera(transport, this.localStream?.getVideoTracks()[0] ?? video, generation));
+    }
   }
 
-  /** What this project may publish. Known once joined; 720p before that. */
+  /** What this project may publish. Known once joined; 1080p before that. */
   get videoPolicy(): VideoPolicy {
     return this.joinResult?.video ?? DEFAULT_VIDEO_POLICY;
   }
@@ -707,39 +960,393 @@ export class StreamClient {
    */
   async setVideoQuality(quality: VideoQuality): Promise<VideoQuality> {
     this.publishOptions = { ...this.publishOptions, quality };
-    const current = [...this.producers.values()].find(
-      (p) => p.kind === 'video' && (p.appData as { source?: string }).source === 'camera',
-    );
-    const track = current?.track;
-    if (!current || !track || track.readyState !== 'live') return this.videoQuality;
+    await this.cameraTask(async () => {
+      const current = this.cameraProducer();
+      const track = current?.track;
+      if (!current || !track || track.readyState !== 'live') return;
+      await this.reproduceCamera(current, track);
+    });
+    return this.videoQuality;
+  }
+
+  /** Change the camera frame rate without stopping screen, microphone or music. */
+  async setVideoFrameRate(frameRate: 30 | 60): Promise<30 | 60> {
+    if (frameRate !== 30 && frameRate !== 60) throw new RangeError('setVideoFrameRate: expected 30 or 60');
+    this.publishOptions = { ...this.publishOptions, frameRate };
+    this.frameRateFallback = false;
+    this.frameRateLowSamples = 0;
+    await this.cameraTask(async () => {
+      const current = this.cameraProducer();
+      const track = current?.track;
+      if (!current || !track || track.readyState !== 'live') return;
+      try { await track.applyConstraints?.({ frameRate: { ideal: frameRate, max: frameRate } }); }
+      catch { /* Existing capture settings decide whether 60 fps is possible. */ }
+      await this.reproduceCamera(current, track);
+    });
+    return this.cameraProducer() ? this.cameraFrameRate : this.videoPolicy.maxFrameRate === 30 ? 30 : frameRate;
+  }
+
+  /**
+   * Send a different video track from now on: a canvas with effects drawn on
+   * it, the raw camera again, another camera. When the new track needs the
+   * same dimensions and layers as the current one, viewers see no pause, and the layers
+   * take the new picture's bitrates. Otherwise the camera producer is
+   * replaced, as `setVideoQuality` does. Either way a paused camera stays
+   * paused.
+   *
+   * Your own stream is left as it is: the client keeps a stream of its own
+   * (`local`, announced by `localStreamChanged`) with the new track and your
+   * microphone, and a reconnect republishes that. The track you replaced is
+   * not stopped; it is yours again, switched back on if pausing the camera
+   * had switched it off. Rejects when no camera is being published.
+   */
+  async replaceVideoTrack(track: MediaStreamTrack): Promise<void> {
+    if (!track || track.kind !== 'video') throw new TypeError('replaceVideoTrack needs a video track');
+    if (track.readyState === 'ended') throw new Error('replaceVideoTrack: that track has ended');
+    await this.cameraTask(async () => {
+      const current = this.cameraProducer();
+      // Between a dropped connection and the republish there is no producer,
+      // but the camera is still being published: the republish sends this.
+      const reconnecting = !current && this.wasPublishing && (this.localStream?.getVideoTracks().length ?? 0) > 0;
+      if (!current && !reconnecting) throw new Error('replaceVideoTrack: no camera is being published');
+      if (!current) {
+        // The republish reads the camera's state off the track it sends, and
+        // the track being replaced is switched off exactly when the camera is
+        // paused. Hand that state on, or a camera that was off comes back on;
+        // and the old track back switched on, as below.
+        const replaced = this.localStream?.getVideoTracks()[0];
+        if (replaced && replaced !== track) {
+          track.enabled = replaced.enabled;
+          replaced.enabled = true;
+        }
+      }
+      this.adoptVideoTrack(track);
+      if (!current || current.track === track) return;
+      const previous = current.track;
+      const paused = current.paused;
+      try {
+        const { width = 0, height = 0 } = track.getSettings();
+        const next = videoEncodings(width, height, this.videoQuality, this.layerCount(), this.frameRateFor(track));
+        const previousSize = current.track?.getSettings();
+        if (this.cameraEncodings && sameLayout(this.cameraEncodings, next) &&
+            previousSize?.width === width && previousSize?.height === height) {
+          await current.replaceTrack({ track });
+          await this.applyLayerBitrates(current, next);
+          return;
+        }
+        await this.reproduceCamera(current, track);
+      } finally {
+        // Pausing switched the old track off (mediasoup-client pauses that
+        // way), and nothing switches it back on once it is replaced. It is the
+        // app's again, often the camera its canvas draws from: handed back
+        // off, that canvas stays black after the camera is resumed.
+        if (paused && previous && previous !== this.cameraProducer()?.track) previous.enabled = true;
+      }
+    });
+  }
+
+  /**
+   * Give a camera producer whose track was swapped in place the bitrates of
+   * its new picture. The layout matched, but a bigger picture at the same
+   * scales (a 1280×720 canvas for a 960×540 camera) would otherwise stay
+   * capped at the smaller one's. Best effort: a browser that refuses keeps
+   * the old caps.
+   */
+  private async applyLayerBitrates(producer: Producer, encodings: VideoEncoding[]): Promise<void> {
+    const before = this.cameraEncodings;
+    if (before && before.every((encoding, i) => encoding.maxBitrate === encodings[i]?.maxBitrate)) {
+      this.cameraEncodings = encodings;
+      return;
+    }
+    const sender = producer.rtpSender;
+    if (!sender) return;
+    try {
+      const parameters = sender.getParameters();
+      // One entry per layer, smallest first, as they were produced. A browser
+      // that reports them some other way is left with its own.
+      if (parameters.encodings.length !== encodings.length) return;
+      parameters.encodings.forEach((encoding, i) => {
+        encoding.maxBitrate = encodings[i].maxBitrate;
+      });
+      await sender.setParameters(parameters);
+      this.cameraEncodings = encodings;
+    } catch {
+      // Unsupported here; the layers keep the caps they were produced with.
+    }
+  }
+
+  /**
+   * What your camera video is sending now, layer by layer, or null when no
+   * camera is being published. The top-level fields describe the biggest
+   * layer going out. `qualityLimitation` says why it is less than asked:
+   * `cpu` is this device, `bandwidth` the upload, `none` neither (then check
+   * the camera and the light). Bitrates are averaged since the previous call,
+   * so call it every second or two; the first call reports 0.
+   */
+  async getVideoStats(): Promise<VideoSendStats | null> {
+    return this.collectVideoStats(this.sendStatsBaseline);
+  }
+
+  /**
+   * What this client receives of a remote video (by its `producerId`, from
+   * the `track` event), or null when it is not receiving that video. Bitrates
+   * are averaged since the previous call for it; the first call reports 0.
+   */
+  async getReceiveStats(producerId: string): Promise<VideoReceiveStats | null> {
+    const consumer = this.consumers.get(producerId);
+    if (!consumer || consumer.kind !== 'video') return null;
+    let report: RTCStatsReport;
+    try {
+      report = await consumer.getStats();
+    } catch (err) {
+      if (consumer.closed) return null;
+      throw err;
+    }
+    const live = new Set([...this.consumers.values()].map((c) => c.id));
+    pruneBaseline(this.recvStatsBaseline, (key) => live.has(key));
+    return parseReceiveStats(report.values(), { producerId, baseline: this.recvStatsBaseline, key: consumer.id });
+  }
+
+  /**
+   * Receive a smaller copy of someone's video (by its `producerId`, from the
+   * `track` event), for a tile that does not need the full picture. Spatial
+   * layer 0 is the smallest copy the publisher sends: a computer sends three
+   * (at 720p 320×180, 640×360 and 1280×720), a phone two (640×360 and
+   * 1280×720), and a layer above the biggest means the biggest. Leave
+   * `temporalLayer` out to keep the full frame rate. It is the most this
+   * client is sent: a connection that cannot carry it still gets less. The
+   * choice is kept across reconnects and forgotten when the track ends.
+   * Rejects for a producer whose video this client is not receiving.
+   */
+  async setPreferredLayers(producerId: string, spatialLayer: number, temporalLayer?: number): Promise<void> {
+    if (!isLayer(spatialLayer) || (temporalLayer !== undefined && !isLayer(temporalLayer))) {
+      throw new RangeError('setPreferredLayers: layers are whole numbers from 0');
+    }
+    const consumer = this.consumers.get(producerId);
+    if (!consumer || consumer.kind !== 'video') {
+      throw new Error(`setPreferredLayers: this client is not receiving a video with producerId ${producerId}`);
+    }
+    const layers: PreferredLayers = temporalLayer === undefined ? { spatialLayer } : { spatialLayer, temporalLayer };
+    this.receiveQualities.delete(producerId);
+    // Kept first: a reconnect that interrupts the request still applies it.
+    this.preferredLayers.set(producerId, layers);
+    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
+  }
+
+  private async collectVideoStats(baseline: StatsBaseline): Promise<VideoSendStats | null> {
+    const producer = this.cameraProducer();
+    if (!producer) return null;
+    let report: RTCStatsReport;
+    try {
+      report = await producer.getStats();
+    } catch (err) {
+      if (producer.closed) return null;
+      throw err;
+    }
+    const prefix = `${producer.id}:`;
+    pruneBaseline(baseline, (key) => key.startsWith(prefix));
+    return parseSendStats(report.values(), { quality: this.videoQuality, baseline, keyPrefix: prefix });
+  }
+
+  /** Friendly receive ceiling using this producer's real layer sizes; auto restores adaptation. */
+  async setReceiveQuality(producerId: string, quality: ReceiveQuality): Promise<void> {
+    if (!['auto', '1080p', '720p', '480p'].includes(quality)) throw new RangeError('setReceiveQuality: unsupported quality');
+    const consumer = this.consumers.get(producerId);
+    if (!consumer || consumer.kind !== 'video') throw new Error('setReceiveQuality: this client is not receiving that video');
+    let layers: PreferredLayers;
+    if (quality === 'auto') layers = { spatialLayer: -1, temporalLayer: -1 };
+    else {
+      try { layers = { spatialLayer: receiveSpatialLayer(this.remoteVideoLayers.get(producerId) ?? [], quality) }; }
+      catch { throw new RpcError('quality-unavailable', 'this publisher does not describe its video layers; use Auto'); }
+    }
+    this.receiveQualities.set(producerId, quality);
+    this.preferredLayers.set(producerId, layers);
+    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
+  }
+
+  private cameraProducer(): Producer | undefined {
+    for (const p of this.producers.values()) {
+      if (p.kind === 'video' && (p.appData as { source?: string }).source === 'camera') return p;
+    }
+    return undefined;
+  }
+
+  private cameraTask<T>(task: () => Promise<T>): Promise<T> {
+    const run = this.cameraQueue.then(task, task);
+    this.cameraQueue = run.catch(() => undefined);
+    return run;
+  }
+
+  /** Swap the video in the client's own stream, never the app's. */
+  private adoptVideoTrack(track: MediaStreamTrack): void {
+    const previous = this.localStream;
+    if (previous && previous.getVideoTracks()[0] === track) return;
+    const others = previous ? previous.getTracks().filter((t) => t.kind !== 'video') : [];
+    this.localStream = new MediaStream([...others, track]);
+    this.emit('localStreamChanged', this.localStream);
+  }
+
+  /** Close the camera producer and produce `track` in its place, paused on both ends if it was. */
+  private async reproduceCamera(current: Producer, track: MediaStreamTrack): Promise<Producer> {
+    const generation = this.publishGeneration;
     const paused = current.paused;
     current.close();
     this.producers.delete(current.id);
+    this.cameraEncodings = null;
+    this.syncLimitWatch();
     await this.rpc('closeProducer', { producerId: current.id }).catch(() => undefined);
-    const producer = await this.produceCamera(await this.ensureSendTransport(), track);
-    if (paused) {
-      producer.pause();
-      await this.rpc('pauseProducer', { producerId: producer.id, paused: true });
-    }
-    return this.videoQuality;
+    return this.produceCamera(await this.ensureSendTransport(), track, generation, paused);
+  }
+
+  /** `options.layers` when valid, else 2 on phones and tablets and 3 elsewhere. */
+  private layerCount(): LayerCount {
+    const asked = this.publishOptions.layers;
+    return asked === 1 || asked === 2 || asked === 3 ? asked : defaultLayerCount();
   }
 
-  private async produceCamera(transport: Transport, track: MediaStreamTrack): Promise<Producer> {
+  /**
+   * Produce the camera from `track`, paused on both ends when `paused`. By
+   * default the track says: one handed over switched off is a camera that is
+   * off, and so is one a pause switched off before a reconnect.
+   */
+  private async produceCamera(
+    transport: Transport,
+    track: MediaStreamTrack,
+    generation: number,
+    paused = !track.enabled,
+  ): Promise<Producer> {
+    if (generation !== this.publishGeneration) throw new Error(unpublishedWhile('camera'));
+    // mediasoup-client starts a producer paused exactly when its track is
+    // off. Setting the track to the camera's state first makes the producer
+    // start in it, and a camera that is off never sends a frame of the new
+    // track; the server is told below.
+    track.enabled = !paused;
     // Simulcast from day one: it is what lets one node serve far more viewers.
-    // The layers follow the picture's real size and the project's quality, so
-    // the browser sends every layer asked for (see videoEncodings).
+    // The layers follow the picture's real size, the project's quality and
+    // the device, so the browser sends every layer asked for (see
+    // videoEncodings).
     const { width = 0, height = 0 } = track.getSettings();
-    const producer = await transport.produce({
-      track,
-      stopTracks: false,
-      encodings: videoEncodings(width, height, this.videoQuality),
-      codecOptions: { videoGoogleStartBitrate: 1000 },
-      appData: { source: 'camera' },
-    });
-    this.producers.set(producer.id, producer);
+    const quality = this.videoQuality;
+    this.cameraFrameRate = this.frameRateFor(track);
+    const encodings = videoEncodings(width, height, quality, this.layerCount(), this.cameraFrameRate);
+    let codecs: RtpCodecCapability[] | undefined;
+    try {
+      codecs = this.device?.sendRtpCapabilities.codecs;
+    } catch {
+      codecs = undefined;
+    }
+    const chosen = await this.cameraCodec(codecs, encodings, width, height, quality);
+    const produce = (codec: RtpCodecCapability | undefined) =>
+      transport.produce({
+        track,
+        stopTracks: false,
+        encodings,
+        codecOptions: { videoGoogleStartBitrate: VIDEO_START_BITRATE_KBPS },
+        ...(codec ? { codec } : {}),
+        appData: { source: 'camera', videoLayers: describeVideoLayers(width, height, encodings, quality) },
+      });
+    let producer: Producer;
+    try {
+      producer = await produce(chosen.capability);
+    } catch (err) {
+      // Never fail a publish over the codec: H.264 refused is VP8 instead.
+      // A closed transport (a reconnect under way) fails either way.
+      if (chosen.codec !== 'h264' || transport.closed) throw err;
+      this.reportCodecFallback(`H.264 was refused (${(err as Error).message}): publishing VP8 instead`);
+      producer = await produce(findCodec(codecs, 'vp8'));
+    }
+    // unpublish() may have run while this was being made (a publish, a
+    // quality change or a track swap under way).
+    await this.keepProducer(producer, generation, 'camera');
+    this.cameraEncodings = encodings;
+    this.frameRateLowSamples = 0;
+    this.frameRateWarmupSamples = 2;
+    await this.pauseOnServer(producer);
+    await this.applyDegradationPreference(producer);
+    this.syncLimitWatch();
     return producer;
   }
 
+  /**
+   * Keep a producer just made -- unless unpublish() ran while it was being
+   * made. Kept, it would publish again what was just unpublished, so it is
+   * closed instead, on both ends.
+   */
+  private async keepProducer(producer: Producer, generation: number, what: string): Promise<void> {
+    if (generation !== this.publishGeneration) {
+      producer.close();
+      await this.rpc('closeProducer', { producerId: producer.id }).catch(() => undefined);
+      throw new Error(unpublishedWhile(what));
+    }
+    this.producers.set(producer.id, producer);
+  }
+
+  /**
+   * mediasoup-client makes a producer paused when its track is off, and the
+   * server makes every producer live. Unless the server is told, viewers get
+   * black video or silence from a camera or microphone they are told is on.
+   */
+  private async pauseOnServer(producer: Producer): Promise<void> {
+    if (producer.paused) await this.rpc('pauseProducer', { producerId: producer.id, paused: true });
+  }
+
+  /**
+   * H.264 or VP8 for the camera (see chooseCodec), with the device's own
+   * capability entry for it: produce() must only ever get one of those.
+   */
+  private async cameraCodec(
+    codecs: RtpCodecCapability[] | undefined,
+    encodings: VideoEncoding[],
+    width: number,
+    height: number,
+    quality: VideoQuality,
+  ): Promise<{ codec: 'vp8' | 'h264'; capability?: RtpCodecCapability }> {
+    const requested = normalizeCodec(this.publishOptions.codec);
+    const env = currentEnv();
+    const phoneOrTablet = isPhoneOrTablet(env);
+    const h264 = findCodec(codecs, 'h264');
+    let h264Hardware: boolean | null = null;
+    if (requested === 'auto' && phoneOrTablet && h264) {
+      const top = topLayerSize(width, height, quality);
+      h264Hardware = await h264HardwareEncode({ width: top.width, height: top.height, bitrate: encodings[encodings.length - 1].maxBitrate });
+    }
+    const choice = chooseCodec({ requested, phoneOrTablet, appleMobile: isAppleMobile(env), h264Available: Boolean(h264), h264Hardware });
+    if (choice.warning) this.reportCodecFallback(choice.warning);
+    else if (requested === 'auto' && phoneOrTablet && !h264) this.reportCodecFallback('this browser cannot send H.264 to this server: publishing VP8');
+    if (choice.codec === 'h264' && h264) return { codec: 'h264', capability: h264 };
+    return { codec: 'vp8', capability: findCodec(codecs, 'vp8') };
+  }
+
+  /** A forced H.264 that could not be used is the app's to hear about; 'auto' just notes it, once. */
+  private reportCodecFallback(message: string): void {
+    if (normalizeCodec(this.publishOptions.codec) === 'h264') {
+      this.emit('error', new Error(message));
+    } else if (!this.codecFallbackLogged) {
+      this.codecFallbackLogged = true;
+      console.warn(`[voltkit-rtc] ${message}`);
+    }
+  }
+
+  /** Best effort: a browser that ignores or refuses it keeps its own default. */
+  private async applyDegradationPreference(producer: Producer): Promise<void> {
+    const preference = this.publishOptions.degradationPreference;
+    const sender = producer.rtpSender;
+    if (!sender) return;
+    if (preference !== 'maintain-resolution' && preference !== 'maintain-framerate' && preference !== 'balanced') return;
+    try {
+      const parameters = sender.getParameters();
+      parameters.degradationPreference = preference;
+      await sender.setParameters(parameters);
+    } catch {
+      // Unsupported here; the encoder keeps the browser's own trade-off.
+    }
+  }
+
+  /** videoLimited sampling runs while a camera is published and someone listens. */
+  private syncLimitWatch(): void {
+    this.limitWatch.set(!this.closedByUs && this.cameraProducer() !== undefined && (this.listenerCount('videoLimited') > 0 || this.cameraFrameRate === 60));
+  }
+
   /**
    * Re-establish this client's broadcast after a reconnect.
    *
@@ -748,9 +1355,18 @@ export class StreamClient {
    * when the tracks have genuinely ended.
    */
   private async republish(): Promise<void> {
+    // unpublish() can run while this is under way, and wins: nothing it
+    // stopped is published again.
+    const generation = this.publishGeneration;
     // Music is not a device and cannot be "gone": publish it again whatever
     // happens to the camera and microphone below.
-    if (this.music) await this.produceMusic(this.music).catch((err) => this.emit('error', err as Error));
+    const screen = this.screenStream;
+    if (screen) await this.restoreScreen().catch(async (err) => {
+      if (this.screenStream === screen) await this.stopScreen('stopped');
+      this.emit('error', err as Error);
+    });
+    if (this.music) await this.produceMusic(this.music, generation).catch((err) => this.emit('error', err as Error));
+    if (generation !== this.publishGeneration) return;
 
     let stream = this.localStream;
     if (!stream && !this.publishConstraints) return; // music was all that was being published
@@ -761,7 +1377,18 @@ export class StreamClient {
         this.emit('error', new Error('cannot restore the broadcast: the devices are gone'));
         return;
       }
+      const ended = stream;
       stream = await navigator.mediaDevices.getUserMedia(this.publishConstraints);
+      if (generation !== this.publishGeneration) {
+        for (const track of stream.getTracks()) track.stop();
+        return;
+      }
+      // New devices come back switched on. A camera or microphone the user
+      // had switched off stays off: a dropped connection is no reason to
+      // switch it back on.
+      for (const track of stream.getTracks()) {
+        if (ended?.getTracks().some((t) => t.kind === track.kind && !t.enabled)) track.enabled = false;
+      }
       this.localStream = stream;
       this.emit('localStreamChanged', stream);
     }
@@ -770,6 +1397,150 @@ export class StreamClient {
     await this.publish(stream);
   }
 
+  // ------------------------------------------------------------- screen sharing
+
+  /** Publish a screen beside the camera. Call from a click so capture permission can open. */
+  async publishScreen(options: ScreenPublishOptions = {}): Promise<MediaStream> {
+    if (this.joinResult?.role !== 'host' && this.joinResult?.role !== 'cohost') throw new RpcError('forbidden', 'only hosts and cohosts can share a screen');
+    if (this.screenStream || this.screenPending) throw new RpcError('screen-share-active', 'a screen share is already active');
+    if (options.mode !== undefined && options.mode !== 'detail' && options.mode !== 'motion') throw new TypeError('publishScreen: mode must be detail or motion');
+    const mode = options.mode ?? 'detail';
+    if (!options.stream && (typeof navigator === 'undefined' || !navigator.mediaDevices?.getDisplayMedia)) {
+      throw new RpcError('screen-share-unsupported', 'screen capture is not supported by this browser; use a supported desktop browser');
+    }
+    const generation = this.screenGeneration;
+    this.screenPending = true;
+    let stream: MediaStream | undefined;
+    try {
+      stream = options.stream ?? await navigator.mediaDevices.getDisplayMedia({
+        video: { width: { ideal: 1920, max: 1920 }, height: { ideal: 1080, max: 1080 }, frameRate: { ideal: mode === 'motion' ? 30 : 15, max: mode === 'motion' ? 30 : 15 } },
+        audio: options.audio ?? false,
+      });
+      const video = stream.getVideoTracks()[0];
+      if (!video || video.readyState !== 'live') throw new RpcError('screen-share-no-video', 'screen capture did not provide a live video track');
+      if (generation !== this.screenGeneration || this.closedByUs) throw new Error(unpublishedWhile('screen'));
+      this.screenStream = stream;
+      this.screenMode = mode;
+      this.screenAudioEnabled = options.audio !== false;
+      video.contentHint = mode;
+      const onEnded = () => { void this.stopScreen('track-ended').catch((err) => this.emit('error', err as Error)); };
+      this.screenEndedListener = onEnded;
+      video.addEventListener('ended', onEnded, { once: true });
+      await this.restoreScreen();
+      if (generation !== this.screenGeneration || video.readyState !== 'live') throw new Error(unpublishedWhile('screen'));
+      this.screenStarted = true;
+      this.wasPublishing = true;
+      this.emit('screenShareStarted', { stream, mode });
+      return stream;
+    } catch (error) {
+      if (generation === this.screenGeneration) await this.stopScreen('stopped');
+      stream?.getTracks().forEach((track) => track.stop());
+      throw error;
+    } finally {
+      if (generation === this.screenGeneration) this.screenPending = false;
+    }
+  }
+
+  /** Stop only the screen and its tab/system audio; keep camera, microphone and music. */
+  async unpublishScreen(): Promise<void> {
+    await this.stopScreen('stopped');
+  }
+
+  private releaseScreen(): void {
+    this.screenGeneration += 1;
+    this.screenPending = false;
+    const stream = this.screenStream;
+    if (this.screenEndedListener) stream?.getVideoTracks()[0]?.removeEventListener('ended', this.screenEndedListener);
+    this.screenEndedListener = null;
+    this.screenStream = null;
+    this.screenStarted = false;
+    stream?.getTracks().forEach((track) => track.stop());
+  }
+
+  private async stopScreen(reason: 'stopped' | 'track-ended'): Promise<void> {
+    const started = this.screenStarted;
+    const ids = [...this.screenProducerIds];
+    this.screenProducerIds.clear();
+    this.releaseScreen();
+    for (const id of ids) { this.producers.get(id)?.close(); this.producers.delete(id); }
+    if (!this.localStream && !this.music) this.wasPublishing = false;
+    if (started && !this.closedByUs) this.emit('screenShareEnded', { reason });
+    await Promise.all(ids.map((producerId) => this.rpc('closeProducer', { producerId }).catch(() => undefined)));
+  }
+
+  private async restoreScreen(): Promise<void> {
+    const stream = this.screenStream;
+    const video = stream?.getVideoTracks()[0];
+    if (!stream || !video) return;
+    if (video.readyState !== 'live') { await this.stopScreen('track-ended'); return; }
+    const generation = this.screenGeneration;
+    const publishing = this.publishGeneration;
+    const transport = await this.ensureSendTransport();
+    if (generation !== this.screenGeneration) return;
+    const { width = 0, height = 0 } = video.getSettings();
+    const encodings = screenEncodings(width, height, this.screenMode);
+    // A project's explicit safety restriction also applies to screen video.
+    const cap = topLayerSize(width, height, this.videoPolicy.maxQuality);
+    const screenScale = encodings.at(-1)!.scaleResolutionDownBy;
+    if (cap.scale > screenScale) encodings.forEach((e) => { e.scaleResolutionDownBy *= cap.scale / screenScale; });
+    const keep = async (producer: Producer) => {
+      if (generation !== this.screenGeneration || publishing !== this.publishGeneration || video.readyState !== 'live') {
+        producer.close();
+        await this.rpc('closeProducer', { producerId: producer.id }).catch(() => undefined);
+        throw new Error(unpublishedWhile('screen'));
+      }
+      this.producers.set(producer.id, producer);
+      this.screenProducerIds.add(producer.id);
+      await this.pauseOnServer(producer);
+    };
+    await keep(await transport.produce({
+      track: video, stopTracks: false, encodings,
+      codecOptions: { videoGoogleStartBitrate: VIDEO_START_BITRATE_KBPS },
+      appData: { source: 'screen', videoLayers: describeVideoLayers(width, height, encodings) },
+    }));
+    const audio = stream.getAudioTracks().find((track) => track.readyState === 'live');
+    if (audio && this.screenAudioEnabled && generation === this.screenGeneration) await keep(await transport.produce({
+      track: audio, stopTracks: false,
+      codecOptions: { opusStereo: true, opusDtx: false, opusFec: true, opusMaxAverageBitrate: 128000 },
+      appData: { source: 'screen-audio' },
+    }));
+  }
+
+  private frameRateFor(track: MediaStreamTrack): 30 | 60 {
+    if (this.publishOptions.frameRate !== 60 || this.frameRateFallback) return 30;
+    const captured = track.getSettings().frameRate;
+    if (this.videoPolicy.maxFrameRate === 30 || (typeof captured === 'number' && captured < 50)) {
+      this.frameRateFallback = true;
+      this.emit('error', new RpcError('frame-rate-fallback', '60 fps is unavailable from this camera or project; publishing at 30 fps'));
+      return 30;
+    }
+    return 60;
+  }
+
+  private async sampleCamera(baseline: StatsBaseline): Promise<VideoSendStats | null> {
+    const stats = await this.collectVideoStats(baseline);
+    if (this.cameraFrameRate !== 60 || this.frameRateFallback || !stats) return stats;
+    if (this.cameraProducer()?.paused) return stats;
+    // The lower simulcast copies intentionally run at 30 fps. During the
+    // initial bandwidth ramp they may be the only active copies, which does
+    // not establish that the camera cannot sustain 60. Match native warm-up:
+    // two 2-second samples, then require three consecutive limited samples.
+    if (this.frameRateWarmupSamples-- > 0) return stats;
+    const limited = stats.layers.some((layer) => layer.active && (layer.qualityLimitation === 'cpu' || layer.qualityLimitation === 'bandwidth'));
+    this.frameRateLowSamples = limited && stats.fps > 0 && stats.fps < 50 ? this.frameRateLowSamples + 1 : 0;
+    if (this.frameRateLowSamples >= 3) {
+      this.frameRateFallback = true;
+      void this.cameraTask(async () => {
+        const producer = this.cameraProducer();
+        if (!producer?.track || producer.closed) return;
+        await producer.track.applyConstraints?.({ frameRate: { ideal: 30, max: 30 } }).catch(() => undefined);
+        await this.reproduceCamera(producer, producer.track);
+        this.emit('error', new RpcError('frame-rate-fallback', 'this device or upload cannot sustain 60 fps; publishing at 30 fps'));
+      }).catch((error) => this.emit('error', error as Error));
+    }
+    return stats;
+  }
+
   // ------------------------------------------------------------------ music
 
   /**
@@ -783,10 +1554,15 @@ export class StreamClient {
    * the user has not interacted with.
    */
   async publishMusic(input: MusicInput, options?: MusicOptions): Promise<void> {
+    const generation = this.publishGeneration;
     if (this.music) await this.stopMusic();
     const music = new MusicSource(input, options);
     try {
-      await this.produceMusic(music);
+      await this.produceMusic(music, generation);
+      // Checked again here: an unpublish() after the producer was kept has
+      // already closed it, and the music must not be remembered for a
+      // reconnect to publish again.
+      if (generation !== this.publishGeneration) throw new Error(unpublishedWhile('music'));
     } catch (err) {
       music.close();
       throw err;
@@ -796,8 +1572,9 @@ export class StreamClient {
     this.emitLocalMedia();
   }
 
-  private async produceMusic(music: MusicSource): Promise<void> {
+  private async produceMusic(music: MusicSource, generation: number): Promise<void> {
     const transport = await this.ensureSendTransport();
+    if (generation !== this.publishGeneration) throw new Error(unpublishedWhile('music'));
     const producer = await transport.produce({
       track: music.track,
       stopTracks: false,
@@ -806,8 +1583,10 @@ export class StreamClient {
       codecOptions: { opusStereo: true, opusDtx: false, opusFec: true, opusMaxAverageBitrate: 128_000 },
       appData: { source: 'music' },
     });
-    this.producers.set(producer.id, producer);
+    await this.keepProducer(producer, generation, 'music');
     this.musicProducerId = producer.id;
+    // Music switched off before a reconnect comes back paused here.
+    await this.pauseOnServer(producer);
   }
 
   /** True while music is published, whether or not it is switched on. */
@@ -873,6 +1652,20 @@ export class StreamClient {
 
     const consumer = await transport.consume(params);
     this.consumers.set(remote.producerId, consumer);
+    this.remoteVideoLayers.set(remote.producerId, remote.videoLayers ?? remote.appData?.videoLayers ?? (params as { videoLayers?: VideoLayer[] }).videoLayers ?? []);
+
+    // Layers asked for before a reconnect: the new consumer gets them before
+    // its first frame instead of starting at the full picture. Not getting
+    // them costs data, not the track, so a refusal is only reported.
+    let layers = this.preferredLayers.get(remote.producerId);
+    const receiveQuality = this.receiveQualities.get(remote.producerId);
+    if (receiveQuality && receiveQuality !== 'auto') {
+      try { layers = { spatialLayer: receiveSpatialLayer(this.remoteVideoLayers.get(remote.producerId) ?? [], receiveQuality) }; }
+      catch { layers = undefined; this.emit('error', new RpcError('quality-unavailable', 'this publisher no longer describes its video layers; using Auto')); }
+    }
+    if (layers) {
+      await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers }).catch((err) => this.emit('error', err as Error));
+    }
 
     // The server created it paused. Resume only now that our end exists,
     // otherwise the first frames arrive before there is anything to decode them.
@@ -909,7 +1702,7 @@ export class StreamClient {
 
   private trackEnabled(kind: 'audio' | 'video'): boolean {
     for (const p of this.producers.values()) {
-      if (p.kind === kind && p.id !== this.musicProducerId) return !p.paused;
+      if (p.kind === kind && p.appData.source === defaultSource(kind)) return !p.paused;
     }
     return false;
   }
@@ -927,33 +1720,61 @@ export class StreamClient {
     await this.setKindEnabled('video', enabled);
   }
 
+  private applyLocalModeration(kind?: 'audio' | 'video' | ''): void {
+    for (const producer of this.producers.values()) {
+      if (!kind || producer.kind === kind) producer.pause();
+    }
+    // During reconnect there are no producers, but the retained tracks still
+    // capture. Keep their muted state for republish as well.
+    for (const stream of [this.localStream, this.screenStream]) {
+      for (const track of stream?.getTracks() ?? []) if (!kind || track.kind === kind) track.enabled = false;
+    }
+    if (!kind || kind === 'audio') this.music?.setEnabled(false);
+  }
+
   private async setKindEnabled(kind: 'audio' | 'video', enabled: boolean): Promise<void> {
     for (const producer of this.producers.values()) {
       // The music has its own switch. Muting the microphone must not stop the song.
-      if (producer.kind !== kind || producer.id === this.musicProducerId) continue;
-      if (enabled) producer.resume();
-      else producer.pause();
+      if (producer.kind !== kind || producer.appData.source !== defaultSource(kind)) continue;
+      if (!enabled) producer.pause();
       await this.rpc('pauseProducer', { producerId: producer.id, paused: !enabled });
+      if (enabled) producer.resume();
     }
     this.emitLocalMedia();
   }
 
   /** Stop publishing entirely and release the local devices. */
   async unpublish(): Promise<void> {
+    // All of it takes effect now; only telling the server is waited for. So
+    // publish() and unpublish() act in the order they are called: a publish
+    // called while this still waits is not undone by it, and one already
+    // under way stops at its next step (keepProducer).
+    this.publishGeneration += 1;
+    const screenWasStarted = this.screenStarted;
+    this.releaseScreen();
+    if (screenWasStarted) this.emit('screenShareEnded', { reason: 'stopped' });
     const ids = [...this.producers.keys()];
     for (const producer of this.producers.values()) producer.close();
     this.producers.clear();
-    for (const id of ids) {
-      await this.rpc('closeProducer', { producerId: id }).catch(() => undefined);
-    }
+    this.cameraEncodings = null;
+    // Start over for the next publish, and clear a limitation still showing:
+    // nothing is being held back once nothing is being sent.
+    const cleared = this.limitWatch.reset();
+    if (cleared) this.emit('videoLimited', cleared);
     this.localStream?.getTracks().forEach((t) => t.stop());
     this.localStream = null;
+    // Forgotten with the stream: kept, a reconnect would ask for the camera
+    // and microphone again and publish them, after the app unpublished them.
+    this.publishConstraints = null;
     this.music?.close();
     this.music = null;
     this.musicProducerId = null;
     this.wasPublishing = false;
     this.emit('localStreamChanged', null);
     this.emit('localMediaChanged', { micEnabled: false, camEnabled: false, musicEnabled: false });
+    for (const id of ids) {
+      await this.rpc('closeProducer', { producerId: id }).catch(() => undefined);
+    }
   }
 
   get local(): MediaStream | null {
@@ -962,8 +1783,8 @@ export class StreamClient {
 
   // ------------------------------------------------------------- moderation
 
-  raiseHand(): Promise<unknown> { return this.rpc('raiseHand'); }
-  lowerHand(): Promise<unknown> { return this.rpc('lowerHand'); }
+  raiseHand(metadata?: Record<string, unknown>): Promise<unknown> { return this.rpc('raiseHand', metadata ? { metadata } : {}); }
+  lowerHand(identity?: string): Promise<unknown> { return this.rpc('lowerHand', identity ? { identity } : {}); }
   muteParticipant(identity: string): Promise<unknown> { return this.rpc('requestMute', { identity }); }
   /**
    * Moderating roles: switch someone's microphone (`audio`), camera (`video`)