@voltkit-rtc/client 0.3.0 → 0.5.0

package/dist/esm/index.js

@@ -6,8 +6,10 @@
  * setup, consumer resume ordering and reconnection, each slightly wrong.
  */
 import { Device } from 'mediasoup-client';
+import { chooseCodec, currentEnv, defaultLayerCount, findCodec, h264HardwareEncode, isAppleMobile, isPhoneOrTablet, normalizeCodec, } from './device.js';
 import { MusicSource } from './music.js';
-import { allowedQuality, DEFAULT_VIDEO_POLICY, videoEncodings } from './quality.js';
+import { allowedQuality, DEFAULT_VIDEO_POLICY, sameLayout, describeVideoLayers, receiveSpatialLayer, screenEncodings, topLayerSize, videoEncodings, } from './quality.js';
+import { parseReceiveStats, parseSendStats, pruneBaseline, VideoLimitWatch, } from './stats.js';
 export { BroadcastPublisher, WhipError, whipEnd, whipOffer } from './broadcast.js';
 export { AudioMixer, MusicSource } from './music.js';
 export { ChatClient, ChatRpcError } from './chat.js';
@@ -48,6 +50,14 @@ const CLOSE_REPLACED = 4001;
 /** The server ended the session on purpose and said why first: do not reconnect. */
 const CLOSE_ENDED = 4003;
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const isLayer = (value) => Number.isInteger(value) && value >= 0;
+/**
+ * Where a new camera stream's bandwidth estimate starts, in kbps. Chrome
+ * otherwise starts low and takes many seconds to climb to the top layer.
+ */
+const VIDEO_START_BITRATE_KBPS = 2000;
+/** Why a publish gave up: unpublish() ran while it was under way, and unpublish wins. */
+const unpublishedWhile = (what) => `the ${what} was unpublished while it was being published`;
 export class StreamClient {
     wsUrl;
     ws = null;
@@ -78,11 +88,47 @@ export class StreamClient {
     /** In-app music being published, kept across reconnects so it can be published again. */
     music = null;
     musicProducerId = null;
+    screenStream = null;
+    screenMode = 'detail';
+    screenAudioEnabled = true;
+    screenGeneration = 0;
+    screenPending = false;
+    screenStarted = false;
+    screenEndedListener = null;
+    screenProducerIds = new Set();
+    remoteVideoLayers = new Map();
+    receiveQualities = new Map();
+    cameraFrameRate = 30;
+    frameRateFallback = false;
+    frameRateLowSamples = 0;
+    frameRateWarmupSamples = 2;
     /** What each remote producer carries, so a pause can say "music" rather than just "audio". */
     remoteSources = new Map();
+    /**
+     * The layers asked for each remote video, by producerId. A reconnect
+     * consumes everything again, and the new consumer would otherwise start at
+     * the full picture.
+     */
+    preferredLayers = new Map();
     closedByUs = false;
     reconnecting = false;
     reconnectAttempt = 0;
+    /** Set once a connect() has joined and set up. Only such a session is reconnected. */
+    joinedOnce = false;
+    /** The camera producer's layers, to tell whether a new track fits them. */
+    cameraEncodings = null;
+    // Changes to the camera producer run one at a time. Two overlapping swaps
+    // (an effect toggled on and off quickly) would otherwise both close the
+    // same producer and both produce a new one.
+    cameraQueue = Promise.resolve();
+    // Bumped by unpublish(). A producer still being made when it runs (camera,
+    // microphone or music) belongs to the publish it ended: it is closed, not
+    // kept.
+    publishGeneration = 0;
+    codecFallbackLogged = false;
+    sendStatsBaseline = new Map();
+    recvStatsBaseline = new Map();
+    limitWatch = new VideoLimitWatch((baseline) => this.sampleCamera(baseline), (event) => this.emit('videoLimited', event));
     constructor(wsUrl, token) {
         this.wsUrl = wsUrl;
         this.token = token;
@@ -94,7 +140,17 @@ export class StreamClient {
             this.listeners.set(event, set);
         }
         set.add(fn);
-        return () => set.delete(fn);
+        // videoLimited costs a getStats every 2 s, so it runs only while heard.
+        if (event === 'videoLimited')
+            this.syncLimitWatch();
+        return () => {
+            set.delete(fn);
+            if (event === 'videoLimited')
+                this.syncLimitWatch();
+        };
+    }
+    listenerCount(event) {
+        return this.listeners.get(event)?.size ?? 0;
     }
     emit(event, ...args) {
         for (const fn of this.listeners.get(event) ?? []) {
@@ -108,12 +164,52 @@ export class StreamClient {
     }
     // ------------------------------------------------------------- connection
     async connect() {
-        await this.openSocket();
-        const result = (await this.rpc('join', { token: this.token }));
-        this.joinResult = result;
-        this.device = new Device();
-        await this.device.load({ routerRtpCapabilities: result.routerRtpCapabilities });
+        let result;
+        try {
+            await this.openSocket();
+            result = (await this.rpc('join', { token: this.token }));
+            this.joinResult = result;
+            this.device = this.createDevice();
+            await this.device.load({ routerRtpCapabilities: result.routerRtpCapabilities });
+        }
+        catch (err) {
+            // A session that never got going: the app has the error. Left open,
+            // the socket would be closed by the server's join deadline instead, and
+            // that close would start reconnecting a session that never began.
+            if (!this.joinedOnce)
+                this.detachSocket();
+            throw err;
+        }
+        // Joined and set up: the next drop gets every reconnect attempt again.
+        // Not when the socket opens -- a join refused with a code worth retrying
+        // (no-capacity) never used them up, and every client of a full cluster
+        // retried about twice a second, forever.
+        this.reconnectAttempt = 0;
+        this.joinedOnce = true;
+        // A role or mute can change while the socket is away. The fresh roster
+        // decides what may return; reconnecting never turns a blocked device on.
+        if (result.role !== 'host' && result.role !== 'cohost' && (this.wasPublishing || this.screenPending)) {
+            await this.unpublish();
+        }
+        const own = result.participants.find((p) => p.identity === result.identity);
+        if (own?.muted)
+            this.applyLocalModeration('audio');
+        if (own?.videoBlocked)
+            this.applyLocalModeration('video');
         this.emit('joined', result);
+        // A video that ended while this client was away sends it no
+        // producerClosed: forget the layers asked for it.
+        const present = new Set(result.producers.map((p) => p.producerId));
+        for (const producerId of this.preferredLayers.keys()) {
+            if (!present.has(producerId)) {
+                this.preferredLayers.delete(producerId);
+                this.receiveQualities.delete(producerId);
+                this.remoteVideoLayers.delete(producerId);
+            }
+        }
+        for (const producerId of this.remoteVideoLayers.keys())
+            if (!present.has(producerId))
+                this.remoteVideoLayers.delete(producerId);
         // Subscribe to everything already publishing before we announce readiness.
         for (const p of result.producers) {
             await this.consume(p).catch((err) => this.emit('error', err));
@@ -126,6 +222,10 @@ export class StreamClient {
         }
         return result;
     }
+    /** A new mediasoup Device for each join. Its own method so tests, which have no WebRTC, can stand one in. */
+    createDevice() {
+        return new Device();
+    }
     openSocket() {
         // Abandon any previous socket first. Its close handler checks identity
         // against `this.ws`, so once it is detached it can no longer trigger a
@@ -137,7 +237,6 @@ export class StreamClient {
             const ws = new WebSocket(url.toString());
             this.ws = ws;
             const onOpen = () => {
-                this.reconnectAttempt = 0;
                 this.emit('connected');
                 resolve();
             };
@@ -163,6 +262,9 @@ export class StreamClient {
                     // displace it, and it would displace us right back, forever.
                     this.emit('error', new Error('this identity connected from somewhere else'));
                     this.closedByUs = true;
+                    // The producers outlive this close until the app calls close(); the
+                    // videoLimited sampling must not, or it reports after `closed`.
+                    this.limitWatch.reset();
                     this.emit('closed', 'replaced');
                     return;
                 }
@@ -170,9 +272,14 @@ export class StreamClient {
                     // Normally roomClosed has already ended the session; this covers a
                     // close that arrived without it.
                     this.closedByUs = true;
+                    this.limitWatch.reset();
                     this.emit('closed', 'room-closed');
                     return;
                 }
+                // Before a first successful join the app is still waiting on
+                // connect(), which reports the failure; there is nothing to restore.
+                if (!this.joinedOnce)
+                    return;
                 void this.scheduleReconnect();
             });
         });
@@ -216,8 +323,13 @@ export class StreamClient {
                     return;
                 try {
                     this.teardownMedia();
-                    await this.connect();
-                    return; // connect() resets the attempt counter once the socket opens
+                    await this.connect(); // which resets the attempt counter once joined
+                    // The socket can drop again while the session is being restored,
+                    // and its close handler then finds this loop still running and
+                    // leaves it to us. Returning would leave the client disconnected
+                    // for good, with no `closed`.
+                    if (this.ws)
+                        return;
                 }
                 catch (err) {
                     this.emit('error', err);
@@ -241,8 +353,17 @@ export class StreamClient {
         // others have already reported themselves and set closedByUs.
         const wasOpen = !this.closedByUs;
         this.closedByUs = true;
+        this.publishGeneration += 1;
         this.wasPublishing = false;
         this.teardownMedia();
+        this.preferredLayers.clear();
+        this.receiveQualities.clear();
+        this.remoteVideoLayers.clear();
+        this.releaseScreen();
+        this.music?.close();
+        this.music = null;
+        // Nothing is emitted after closing, so not even a clearing videoLimited.
+        this.limitWatch.reset();
         // Producers no longer stop their tracks, so releasing the camera and
         // microphone is ours to do -- otherwise the device light stays on.
         this.localStream?.getTracks().forEach((t) => t.stop());
@@ -261,6 +382,11 @@ export class StreamClient {
             p.close();
         this.producers.clear();
         this.musicProducerId = null;
+        this.screenProducerIds.clear();
+        this.cameraEncodings = null;
+        // Paused, not reset: a reconnect's republish starts it again, and a
+        // limitation already reported is not reported twice.
+        this.syncLimitWatch();
         this.sendTransport?.close();
         this.recvTransport?.close();
         this.sendTransport = null;
@@ -329,6 +455,9 @@ export class StreamClient {
                 break;
             case 'producerClosed': {
                 const { producerId, identity } = data;
+                this.preferredLayers.delete(producerId);
+                this.receiveQualities.delete(producerId);
+                this.remoteVideoLayers.delete(producerId);
                 const consumer = this.consumers.get(producerId);
                 if (!consumer)
                     break; // never consumed it, or already cleaned up
@@ -352,16 +481,29 @@ export class StreamClient {
             case 'participantJoined':
                 this.emit('participantJoined', data);
                 break;
+            case 'participantMuted': {
+                const wire = data;
+                const e = { identity: wire.identity, kind: wire.kind, paused: wire.paused ?? wire.muted ?? false };
+                if (this.joinResult)
+                    this.joinResult.participants = this.joinResult.participants.map((p) => p.identity !== e.identity ? p : { ...p,
+                        ...(!e.kind || e.kind === 'audio' ? { muted: e.paused } : {}),
+                        ...(!e.kind || e.kind === 'video' ? { videoBlocked: e.paused } : {}),
+                    });
+                this.emit('participantMuted', e);
+                break;
+            }
             case 'participantLeft':
                 this.emit('participantLeft', data);
                 break;
             case 'roleChanged': {
                 const e = data;
+                if (e.identity === this.joinResult?.identity)
+                    this.joinResult.role = e.role;
                 this.emit('roleChanged', e);
                 // Demoted: the server has already closed our producers. Release the
                 // camera and microphone too, instead of capturing for nobody.
                 const canPublish = e.role === 'host' || e.role === 'cohost';
-                if (e.identity === this.joinResult?.identity && !canPublish && this.producers.size > 0) {
+                if (e.identity === this.joinResult?.identity && !canPublish && (this.wasPublishing || this.screenPending)) {
                     void this.unpublish().catch(() => undefined);
                 }
                 break;
@@ -376,10 +518,7 @@ export class StreamClient {
                 // changes nothing here -- switching back on is the user's choice.
                 const e = data;
                 if (e.paused) {
-                    for (const p of this.producers.values()) {
-                        if (!e.kind || p.kind === e.kind)
-                            p.pause();
-                    }
+                    this.applyLocalModeration(e.kind);
                     this.emitLocalMedia();
                 }
                 this.emit('forceMuted', e);
@@ -467,7 +606,7 @@ export class StreamClient {
                 .catch(fail);
         });
         transport.on('produce', ({ kind, rtpParameters, appData }, ok, fail) => {
-            this.rpc('produce', { transportId: transport.id, kind, rtpParameters, appData, source: appData.source })
+            this.rpc('produce', { transportId: transport.id, kind, rtpParameters, appData, source: appData.source, videoLayers: appData.videoLayers })
                 .then((res) => ok({ id: res.producerId }))
                 .catch(fail);
         });
@@ -508,15 +647,26 @@ export class StreamClient {
      *
      * `options.quality` picks 1080p, 720p or 480p (see `videoPolicy` for what
      * the project allows). Capture at least that size for the full picture.
+     * `options.layers`, `codec` and `degradationPreference` tune how the video
+     * is encoded; the defaults suit both phones and computers. The options are
+     * kept for a reconnect's republish.
      */
     async publish(stream, constraints, options) {
+        const generation = this.publishGeneration;
         const transport = await this.ensureSendTransport();
+        // unpublish() ran first: this publish is over before it began, and must
+        // leave nothing behind that a reconnect would publish again.
+        if (generation !== this.publishGeneration)
+            throw new Error(unpublishedWhile('stream'));
         this.localStream = stream;
         this.wasPublishing = true;
         if (constraints)
             this.publishConstraints = constraints;
-        if (options)
+        if (options) {
             this.publishOptions = { ...options };
+            this.frameRateFallback = false;
+            this.frameRateLowSamples = 0;
+        }
         const audio = stream.getAudioTracks()[0];
         if (audio) {
             const producer = await transport.produce({
@@ -527,13 +677,18 @@ export class StreamClient {
                 codecOptions: { opusStereo: true, opusDtx: true, opusFec: true },
                 appData: { source: 'mic' },
             });
-            this.producers.set(producer.id, producer);
+            await this.keepProducer(producer, generation, 'microphone');
+            // A microphone handed over switched off, or muted before a reconnect.
+            await this.pauseOnServer(producer);
         }
         const video = stream.getVideoTracks()[0];
-        if (video)
-            await this.produceCamera(transport, video);
+        if (video) {
+            // The track is read when the camera's turn comes: one handed to
+            // replaceVideoTrack while this publish was under way is the one to send.
+            await this.cameraTask(() => this.produceCamera(transport, this.localStream?.getVideoTracks()[0] ?? video, generation));
+        }
     }
-    /** What this project may publish. Known once joined; 720p before that. */
+    /** What this project may publish. Known once joined; 1080p before that. */
     get videoPolicy() {
         return this.joinResult?.video ?? DEFAULT_VIDEO_POLICY;
     }
@@ -549,36 +704,401 @@ export class StreamClient {
      */
     async setVideoQuality(quality) {
         this.publishOptions = { ...this.publishOptions, quality };
-        const current = [...this.producers.values()].find((p) => p.kind === 'video' && p.appData.source === 'camera');
-        const track = current?.track;
-        if (!current || !track || track.readyState !== 'live')
-            return this.videoQuality;
+        await this.cameraTask(async () => {
+            const current = this.cameraProducer();
+            const track = current?.track;
+            if (!current || !track || track.readyState !== 'live')
+                return;
+            await this.reproduceCamera(current, track);
+        });
+        return this.videoQuality;
+    }
+    /** Change the camera frame rate without stopping screen, microphone or music. */
+    async setVideoFrameRate(frameRate) {
+        if (frameRate !== 30 && frameRate !== 60)
+            throw new RangeError('setVideoFrameRate: expected 30 or 60');
+        this.publishOptions = { ...this.publishOptions, frameRate };
+        this.frameRateFallback = false;
+        this.frameRateLowSamples = 0;
+        await this.cameraTask(async () => {
+            const current = this.cameraProducer();
+            const track = current?.track;
+            if (!current || !track || track.readyState !== 'live')
+                return;
+            try {
+                await track.applyConstraints?.({ frameRate: { ideal: frameRate, max: frameRate } });
+            }
+            catch { /* Existing capture settings decide whether 60 fps is possible. */ }
+            await this.reproduceCamera(current, track);
+        });
+        return this.cameraProducer() ? this.cameraFrameRate : this.videoPolicy.maxFrameRate === 30 ? 30 : frameRate;
+    }
+    /**
+     * Send a different video track from now on: a canvas with effects drawn on
+     * it, the raw camera again, another camera. When the new track needs the
+     * same dimensions and layers as the current one, viewers see no pause, and the layers
+     * take the new picture's bitrates. Otherwise the camera producer is
+     * replaced, as `setVideoQuality` does. Either way a paused camera stays
+     * paused.
+     *
+     * Your own stream is left as it is: the client keeps a stream of its own
+     * (`local`, announced by `localStreamChanged`) with the new track and your
+     * microphone, and a reconnect republishes that. The track you replaced is
+     * not stopped; it is yours again, switched back on if pausing the camera
+     * had switched it off. Rejects when no camera is being published.
+     */
+    async replaceVideoTrack(track) {
+        if (!track || track.kind !== 'video')
+            throw new TypeError('replaceVideoTrack needs a video track');
+        if (track.readyState === 'ended')
+            throw new Error('replaceVideoTrack: that track has ended');
+        await this.cameraTask(async () => {
+            const current = this.cameraProducer();
+            // Between a dropped connection and the republish there is no producer,
+            // but the camera is still being published: the republish sends this.
+            const reconnecting = !current && this.wasPublishing && (this.localStream?.getVideoTracks().length ?? 0) > 0;
+            if (!current && !reconnecting)
+                throw new Error('replaceVideoTrack: no camera is being published');
+            if (!current) {
+                // The republish reads the camera's state off the track it sends, and
+                // the track being replaced is switched off exactly when the camera is
+                // paused. Hand that state on, or a camera that was off comes back on;
+                // and the old track back switched on, as below.
+                const replaced = this.localStream?.getVideoTracks()[0];
+                if (replaced && replaced !== track) {
+                    track.enabled = replaced.enabled;
+                    replaced.enabled = true;
+                }
+            }
+            this.adoptVideoTrack(track);
+            if (!current || current.track === track)
+                return;
+            const previous = current.track;
+            const paused = current.paused;
+            try {
+                const { width = 0, height = 0 } = track.getSettings();
+                const next = videoEncodings(width, height, this.videoQuality, this.layerCount(), this.frameRateFor(track));
+                const previousSize = current.track?.getSettings();
+                if (this.cameraEncodings && sameLayout(this.cameraEncodings, next) &&
+                    previousSize?.width === width && previousSize?.height === height) {
+                    await current.replaceTrack({ track });
+                    await this.applyLayerBitrates(current, next);
+                    return;
+                }
+                await this.reproduceCamera(current, track);
+            }
+            finally {
+                // Pausing switched the old track off (mediasoup-client pauses that
+                // way), and nothing switches it back on once it is replaced. It is the
+                // app's again, often the camera its canvas draws from: handed back
+                // off, that canvas stays black after the camera is resumed.
+                if (paused && previous && previous !== this.cameraProducer()?.track)
+                    previous.enabled = true;
+            }
+        });
+    }
+    /**
+     * Give a camera producer whose track was swapped in place the bitrates of
+     * its new picture. The layout matched, but a bigger picture at the same
+     * scales (a 1280×720 canvas for a 960×540 camera) would otherwise stay
+     * capped at the smaller one's. Best effort: a browser that refuses keeps
+     * the old caps.
+     */
+    async applyLayerBitrates(producer, encodings) {
+        const before = this.cameraEncodings;
+        if (before && before.every((encoding, i) => encoding.maxBitrate === encodings[i]?.maxBitrate)) {
+            this.cameraEncodings = encodings;
+            return;
+        }
+        const sender = producer.rtpSender;
+        if (!sender)
+            return;
+        try {
+            const parameters = sender.getParameters();
+            // One entry per layer, smallest first, as they were produced. A browser
+            // that reports them some other way is left with its own.
+            if (parameters.encodings.length !== encodings.length)
+                return;
+            parameters.encodings.forEach((encoding, i) => {
+                encoding.maxBitrate = encodings[i].maxBitrate;
+            });
+            await sender.setParameters(parameters);
+            this.cameraEncodings = encodings;
+        }
+        catch {
+            // Unsupported here; the layers keep the caps they were produced with.
+        }
+    }
+    /**
+     * What your camera video is sending now, layer by layer, or null when no
+     * camera is being published. The top-level fields describe the biggest
+     * layer going out. `qualityLimitation` says why it is less than asked:
+     * `cpu` is this device, `bandwidth` the upload, `none` neither (then check
+     * the camera and the light). Bitrates are averaged since the previous call,
+     * so call it every second or two; the first call reports 0.
+     */
+    async getVideoStats() {
+        return this.collectVideoStats(this.sendStatsBaseline);
+    }
+    /**
+     * What this client receives of a remote video (by its `producerId`, from
+     * the `track` event), or null when it is not receiving that video. Bitrates
+     * are averaged since the previous call for it; the first call reports 0.
+     */
+    async getReceiveStats(producerId) {
+        const consumer = this.consumers.get(producerId);
+        if (!consumer || consumer.kind !== 'video')
+            return null;
+        let report;
+        try {
+            report = await consumer.getStats();
+        }
+        catch (err) {
+            if (consumer.closed)
+                return null;
+            throw err;
+        }
+        const live = new Set([...this.consumers.values()].map((c) => c.id));
+        pruneBaseline(this.recvStatsBaseline, (key) => live.has(key));
+        return parseReceiveStats(report.values(), { producerId, baseline: this.recvStatsBaseline, key: consumer.id });
+    }
+    /**
+     * Receive a smaller copy of someone's video (by its `producerId`, from the
+     * `track` event), for a tile that does not need the full picture. Spatial
+     * layer 0 is the smallest copy the publisher sends: a computer sends three
+     * (at 720p 320×180, 640×360 and 1280×720), a phone two (640×360 and
+     * 1280×720), and a layer above the biggest means the biggest. Leave
+     * `temporalLayer` out to keep the full frame rate. It is the most this
+     * client is sent: a connection that cannot carry it still gets less. The
+     * choice is kept across reconnects and forgotten when the track ends.
+     * Rejects for a producer whose video this client is not receiving.
+     */
+    async setPreferredLayers(producerId, spatialLayer, temporalLayer) {
+        if (!isLayer(spatialLayer) || (temporalLayer !== undefined && !isLayer(temporalLayer))) {
+            throw new RangeError('setPreferredLayers: layers are whole numbers from 0');
+        }
+        const consumer = this.consumers.get(producerId);
+        if (!consumer || consumer.kind !== 'video') {
+            throw new Error(`setPreferredLayers: this client is not receiving a video with producerId ${producerId}`);
+        }
+        const layers = temporalLayer === undefined ? { spatialLayer } : { spatialLayer, temporalLayer };
+        this.receiveQualities.delete(producerId);
+        // Kept first: a reconnect that interrupts the request still applies it.
+        this.preferredLayers.set(producerId, layers);
+        await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
+    }
+    async collectVideoStats(baseline) {
+        const producer = this.cameraProducer();
+        if (!producer)
+            return null;
+        let report;
+        try {
+            report = await producer.getStats();
+        }
+        catch (err) {
+            if (producer.closed)
+                return null;
+            throw err;
+        }
+        const prefix = `${producer.id}:`;
+        pruneBaseline(baseline, (key) => key.startsWith(prefix));
+        return parseSendStats(report.values(), { quality: this.videoQuality, baseline, keyPrefix: prefix });
+    }
+    /** Friendly receive ceiling using this producer's real layer sizes; auto restores adaptation. */
+    async setReceiveQuality(producerId, quality) {
+        if (!['auto', '1080p', '720p', '480p'].includes(quality))
+            throw new RangeError('setReceiveQuality: unsupported quality');
+        const consumer = this.consumers.get(producerId);
+        if (!consumer || consumer.kind !== 'video')
+            throw new Error('setReceiveQuality: this client is not receiving that video');
+        let layers;
+        if (quality === 'auto')
+            layers = { spatialLayer: -1, temporalLayer: -1 };
+        else {
+            try {
+                layers = { spatialLayer: receiveSpatialLayer(this.remoteVideoLayers.get(producerId) ?? [], quality) };
+            }
+            catch {
+                throw new RpcError('quality-unavailable', 'this publisher does not describe its video layers; use Auto');
+            }
+        }
+        this.receiveQualities.set(producerId, quality);
+        this.preferredLayers.set(producerId, layers);
+        await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers });
+    }
+    cameraProducer() {
+        for (const p of this.producers.values()) {
+            if (p.kind === 'video' && p.appData.source === 'camera')
+                return p;
+        }
+        return undefined;
+    }
+    cameraTask(task) {
+        const run = this.cameraQueue.then(task, task);
+        this.cameraQueue = run.catch(() => undefined);
+        return run;
+    }
+    /** Swap the video in the client's own stream, never the app's. */
+    adoptVideoTrack(track) {
+        const previous = this.localStream;
+        if (previous && previous.getVideoTracks()[0] === track)
+            return;
+        const others = previous ? previous.getTracks().filter((t) => t.kind !== 'video') : [];
+        this.localStream = new MediaStream([...others, track]);
+        this.emit('localStreamChanged', this.localStream);
+    }
+    /** Close the camera producer and produce `track` in its place, paused on both ends if it was. */
+    async reproduceCamera(current, track) {
+        const generation = this.publishGeneration;
         const paused = current.paused;
         current.close();
         this.producers.delete(current.id);
+        this.cameraEncodings = null;
+        this.syncLimitWatch();
         await this.rpc('closeProducer', { producerId: current.id }).catch(() => undefined);
-        const producer = await this.produceCamera(await this.ensureSendTransport(), track);
-        if (paused) {
-            producer.pause();
-            await this.rpc('pauseProducer', { producerId: producer.id, paused: true });
-        }
-        return this.videoQuality;
+        return this.produceCamera(await this.ensureSendTransport(), track, generation, paused);
+    }
+    /** `options.layers` when valid, else 2 on phones and tablets and 3 elsewhere. */
+    layerCount() {
+        const asked = this.publishOptions.layers;
+        return asked === 1 || asked === 2 || asked === 3 ? asked : defaultLayerCount();
     }
-    async produceCamera(transport, track) {
+    /**
+     * Produce the camera from `track`, paused on both ends when `paused`. By
+     * default the track says: one handed over switched off is a camera that is
+     * off, and so is one a pause switched off before a reconnect.
+     */
+    async produceCamera(transport, track, generation, paused = !track.enabled) {
+        if (generation !== this.publishGeneration)
+            throw new Error(unpublishedWhile('camera'));
+        // mediasoup-client starts a producer paused exactly when its track is
+        // off. Setting the track to the camera's state first makes the producer
+        // start in it, and a camera that is off never sends a frame of the new
+        // track; the server is told below.
+        track.enabled = !paused;
         // Simulcast from day one: it is what lets one node serve far more viewers.
-        // The layers follow the picture's real size and the project's quality, so
-        // the browser sends every layer asked for (see videoEncodings).
+        // The layers follow the picture's real size, the project's quality and
+        // the device, so the browser sends every layer asked for (see
+        // videoEncodings).
         const { width = 0, height = 0 } = track.getSettings();
-        const producer = await transport.produce({
+        const quality = this.videoQuality;
+        this.cameraFrameRate = this.frameRateFor(track);
+        const encodings = videoEncodings(width, height, quality, this.layerCount(), this.cameraFrameRate);
+        let codecs;
+        try {
+            codecs = this.device?.sendRtpCapabilities.codecs;
+        }
+        catch {
+            codecs = undefined;
+        }
+        const chosen = await this.cameraCodec(codecs, encodings, width, height, quality);
+        const produce = (codec) => transport.produce({
             track,
             stopTracks: false,
-            encodings: videoEncodings(width, height, this.videoQuality),
-            codecOptions: { videoGoogleStartBitrate: 1000 },
-            appData: { source: 'camera' },
+            encodings,
+            codecOptions: { videoGoogleStartBitrate: VIDEO_START_BITRATE_KBPS },
+            ...(codec ? { codec } : {}),
+            appData: { source: 'camera', videoLayers: describeVideoLayers(width, height, encodings, quality) },
         });
-        this.producers.set(producer.id, producer);
+        let producer;
+        try {
+            producer = await produce(chosen.capability);
+        }
+        catch (err) {
+            // Never fail a publish over the codec: H.264 refused is VP8 instead.
+            // A closed transport (a reconnect under way) fails either way.
+            if (chosen.codec !== 'h264' || transport.closed)
+                throw err;
+            this.reportCodecFallback(`H.264 was refused (${err.message}): publishing VP8 instead`);
+            producer = await produce(findCodec(codecs, 'vp8'));
+        }
+        // unpublish() may have run while this was being made (a publish, a
+        // quality change or a track swap under way).
+        await this.keepProducer(producer, generation, 'camera');
+        this.cameraEncodings = encodings;
+        this.frameRateLowSamples = 0;
+        this.frameRateWarmupSamples = 2;
+        await this.pauseOnServer(producer);
+        await this.applyDegradationPreference(producer);
+        this.syncLimitWatch();
         return producer;
     }
+    /**
+     * Keep a producer just made -- unless unpublish() ran while it was being
+     * made. Kept, it would publish again what was just unpublished, so it is
+     * closed instead, on both ends.
+     */
+    async keepProducer(producer, generation, what) {
+        if (generation !== this.publishGeneration) {
+            producer.close();
+            await this.rpc('closeProducer', { producerId: producer.id }).catch(() => undefined);
+            throw new Error(unpublishedWhile(what));
+        }
+        this.producers.set(producer.id, producer);
+    }
+    /**
+     * mediasoup-client makes a producer paused when its track is off, and the
+     * server makes every producer live. Unless the server is told, viewers get
+     * black video or silence from a camera or microphone they are told is on.
+     */
+    async pauseOnServer(producer) {
+        if (producer.paused)
+            await this.rpc('pauseProducer', { producerId: producer.id, paused: true });
+    }
+    /**
+     * H.264 or VP8 for the camera (see chooseCodec), with the device's own
+     * capability entry for it: produce() must only ever get one of those.
+     */
+    async cameraCodec(codecs, encodings, width, height, quality) {
+        const requested = normalizeCodec(this.publishOptions.codec);
+        const env = currentEnv();
+        const phoneOrTablet = isPhoneOrTablet(env);
+        const h264 = findCodec(codecs, 'h264');
+        let h264Hardware = null;
+        if (requested === 'auto' && phoneOrTablet && h264) {
+            const top = topLayerSize(width, height, quality);
+            h264Hardware = await h264HardwareEncode({ width: top.width, height: top.height, bitrate: encodings[encodings.length - 1].maxBitrate });
+        }
+        const choice = chooseCodec({ requested, phoneOrTablet, appleMobile: isAppleMobile(env), h264Available: Boolean(h264), h264Hardware });
+        if (choice.warning)
+            this.reportCodecFallback(choice.warning);
+        else if (requested === 'auto' && phoneOrTablet && !h264)
+            this.reportCodecFallback('this browser cannot send H.264 to this server: publishing VP8');
+        if (choice.codec === 'h264' && h264)
+            return { codec: 'h264', capability: h264 };
+        return { codec: 'vp8', capability: findCodec(codecs, 'vp8') };
+    }
+    /** A forced H.264 that could not be used is the app's to hear about; 'auto' just notes it, once. */
+    reportCodecFallback(message) {
+        if (normalizeCodec(this.publishOptions.codec) === 'h264') {
+            this.emit('error', new Error(message));
+        }
+        else if (!this.codecFallbackLogged) {
+            this.codecFallbackLogged = true;
+            console.warn(`[voltkit-rtc] ${message}`);
+        }
+    }
+    /** Best effort: a browser that ignores or refuses it keeps its own default. */
+    async applyDegradationPreference(producer) {
+        const preference = this.publishOptions.degradationPreference;
+        const sender = producer.rtpSender;
+        if (!sender)
+            return;
+        if (preference !== 'maintain-resolution' && preference !== 'maintain-framerate' && preference !== 'balanced')
+            return;
+        try {
+            const parameters = sender.getParameters();
+            parameters.degradationPreference = preference;
+            await sender.setParameters(parameters);
+        }
+        catch {
+            // Unsupported here; the encoder keeps the browser's own trade-off.
+        }
+    }
+    /** videoLimited sampling runs while a camera is published and someone listens. */
+    syncLimitWatch() {
+        this.limitWatch.set(!this.closedByUs && this.cameraProducer() !== undefined && (this.listenerCount('videoLimited') > 0 || this.cameraFrameRate === 60));
+    }
     /**
      * Re-establish this client's broadcast after a reconnect.
      *
@@ -587,10 +1107,22 @@ export class StreamClient {
      * when the tracks have genuinely ended.
      */
     async republish() {
+        // unpublish() can run while this is under way, and wins: nothing it
+        // stopped is published again.
+        const generation = this.publishGeneration;
         // Music is not a device and cannot be "gone": publish it again whatever
         // happens to the camera and microphone below.
+        const screen = this.screenStream;
+        if (screen)
+            await this.restoreScreen().catch(async (err) => {
+                if (this.screenStream === screen)
+                    await this.stopScreen('stopped');
+                this.emit('error', err);
+            });
         if (this.music)
-            await this.produceMusic(this.music).catch((err) => this.emit('error', err));
+            await this.produceMusic(this.music, generation).catch((err) => this.emit('error', err));
+        if (generation !== this.publishGeneration)
+            return;
         let stream = this.localStream;
         if (!stream && !this.publishConstraints)
             return; // music was all that was being published
@@ -600,7 +1132,20 @@ export class StreamClient {
                 this.emit('error', new Error('cannot restore the broadcast: the devices are gone'));
                 return;
             }
+            const ended = stream;
             stream = await navigator.mediaDevices.getUserMedia(this.publishConstraints);
+            if (generation !== this.publishGeneration) {
+                for (const track of stream.getTracks())
+                    track.stop();
+                return;
+            }
+            // New devices come back switched on. A camera or microphone the user
+            // had switched off stays off: a dropped connection is no reason to
+            // switch it back on.
+            for (const track of stream.getTracks()) {
+                if (ended?.getTracks().some((t) => t.kind === track.kind && !t.enabled))
+                    track.enabled = false;
+            }
             this.localStream = stream;
             this.emit('localStreamChanged', stream);
         }
@@ -608,6 +1153,170 @@ export class StreamClient {
             return;
         await this.publish(stream);
     }
+    // ------------------------------------------------------------- screen sharing
+    /** Publish a screen beside the camera. Call from a click so capture permission can open. */
+    async publishScreen(options = {}) {
+        if (this.joinResult?.role !== 'host' && this.joinResult?.role !== 'cohost')
+            throw new RpcError('forbidden', 'only hosts and cohosts can share a screen');
+        if (this.screenStream || this.screenPending)
+            throw new RpcError('screen-share-active', 'a screen share is already active');
+        if (options.mode !== undefined && options.mode !== 'detail' && options.mode !== 'motion')
+            throw new TypeError('publishScreen: mode must be detail or motion');
+        const mode = options.mode ?? 'detail';
+        if (!options.stream && (typeof navigator === 'undefined' || !navigator.mediaDevices?.getDisplayMedia)) {
+            throw new RpcError('screen-share-unsupported', 'screen capture is not supported by this browser; use a supported desktop browser');
+        }
+        const generation = this.screenGeneration;
+        this.screenPending = true;
+        let stream;
+        try {
+            stream = options.stream ?? await navigator.mediaDevices.getDisplayMedia({
+                video: { width: { ideal: 1920, max: 1920 }, height: { ideal: 1080, max: 1080 }, frameRate: { ideal: mode === 'motion' ? 30 : 15, max: mode === 'motion' ? 30 : 15 } },
+                audio: options.audio ?? false,
+            });
+            const video = stream.getVideoTracks()[0];
+            if (!video || video.readyState !== 'live')
+                throw new RpcError('screen-share-no-video', 'screen capture did not provide a live video track');
+            if (generation !== this.screenGeneration || this.closedByUs)
+                throw new Error(unpublishedWhile('screen'));
+            this.screenStream = stream;
+            this.screenMode = mode;
+            this.screenAudioEnabled = options.audio !== false;
+            video.contentHint = mode;
+            const onEnded = () => { void this.stopScreen('track-ended').catch((err) => this.emit('error', err)); };
+            this.screenEndedListener = onEnded;
+            video.addEventListener('ended', onEnded, { once: true });
+            await this.restoreScreen();
+            if (generation !== this.screenGeneration || video.readyState !== 'live')
+                throw new Error(unpublishedWhile('screen'));
+            this.screenStarted = true;
+            this.wasPublishing = true;
+            this.emit('screenShareStarted', { stream, mode });
+            return stream;
+        }
+        catch (error) {
+            if (generation === this.screenGeneration)
+                await this.stopScreen('stopped');
+            stream?.getTracks().forEach((track) => track.stop());
+            throw error;
+        }
+        finally {
+            if (generation === this.screenGeneration)
+                this.screenPending = false;
+        }
+    }
+    /** Stop only the screen and its tab/system audio; keep camera, microphone and music. */
+    async unpublishScreen() {
+        await this.stopScreen('stopped');
+    }
+    releaseScreen() {
+        this.screenGeneration += 1;
+        this.screenPending = false;
+        const stream = this.screenStream;
+        if (this.screenEndedListener)
+            stream?.getVideoTracks()[0]?.removeEventListener('ended', this.screenEndedListener);
+        this.screenEndedListener = null;
+        this.screenStream = null;
+        this.screenStarted = false;
+        stream?.getTracks().forEach((track) => track.stop());
+    }
+    async stopScreen(reason) {
+        const started = this.screenStarted;
+        const ids = [...this.screenProducerIds];
+        this.screenProducerIds.clear();
+        this.releaseScreen();
+        for (const id of ids) {
+            this.producers.get(id)?.close();
+            this.producers.delete(id);
+        }
+        if (!this.localStream && !this.music)
+            this.wasPublishing = false;
+        if (started && !this.closedByUs)
+            this.emit('screenShareEnded', { reason });
+        await Promise.all(ids.map((producerId) => this.rpc('closeProducer', { producerId }).catch(() => undefined)));
+    }
+    async restoreScreen() {
+        const stream = this.screenStream;
+        const video = stream?.getVideoTracks()[0];
+        if (!stream || !video)
+            return;
+        if (video.readyState !== 'live') {
+            await this.stopScreen('track-ended');
+            return;
+        }
+        const generation = this.screenGeneration;
+        const publishing = this.publishGeneration;
+        const transport = await this.ensureSendTransport();
+        if (generation !== this.screenGeneration)
+            return;
+        const { width = 0, height = 0 } = video.getSettings();
+        const encodings = screenEncodings(width, height, this.screenMode);
+        // A project's explicit safety restriction also applies to screen video.
+        const cap = topLayerSize(width, height, this.videoPolicy.maxQuality);
+        const screenScale = encodings.at(-1).scaleResolutionDownBy;
+        if (cap.scale > screenScale)
+            encodings.forEach((e) => { e.scaleResolutionDownBy *= cap.scale / screenScale; });
+        const keep = async (producer) => {
+            if (generation !== this.screenGeneration || publishing !== this.publishGeneration || video.readyState !== 'live') {
+                producer.close();
+                await this.rpc('closeProducer', { producerId: producer.id }).catch(() => undefined);
+                throw new Error(unpublishedWhile('screen'));
+            }
+            this.producers.set(producer.id, producer);
+            this.screenProducerIds.add(producer.id);
+            await this.pauseOnServer(producer);
+        };
+        await keep(await transport.produce({
+            track: video, stopTracks: false, encodings,
+            codecOptions: { videoGoogleStartBitrate: VIDEO_START_BITRATE_KBPS },
+            appData: { source: 'screen', videoLayers: describeVideoLayers(width, height, encodings) },
+        }));
+        const audio = stream.getAudioTracks().find((track) => track.readyState === 'live');
+        if (audio && this.screenAudioEnabled && generation === this.screenGeneration)
+            await keep(await transport.produce({
+                track: audio, stopTracks: false,
+                codecOptions: { opusStereo: true, opusDtx: false, opusFec: true, opusMaxAverageBitrate: 128000 },
+                appData: { source: 'screen-audio' },
+            }));
+    }
+    frameRateFor(track) {
+        if (this.publishOptions.frameRate !== 60 || this.frameRateFallback)
+            return 30;
+        const captured = track.getSettings().frameRate;
+        if (this.videoPolicy.maxFrameRate === 30 || (typeof captured === 'number' && captured < 50)) {
+            this.frameRateFallback = true;
+            this.emit('error', new RpcError('frame-rate-fallback', '60 fps is unavailable from this camera or project; publishing at 30 fps'));
+            return 30;
+        }
+        return 60;
+    }
+    async sampleCamera(baseline) {
+        const stats = await this.collectVideoStats(baseline);
+        if (this.cameraFrameRate !== 60 || this.frameRateFallback || !stats)
+            return stats;
+        if (this.cameraProducer()?.paused)
+            return stats;
+        // The lower simulcast copies intentionally run at 30 fps. During the
+        // initial bandwidth ramp they may be the only active copies, which does
+        // not establish that the camera cannot sustain 60. Match native warm-up:
+        // two 2-second samples, then require three consecutive limited samples.
+        if (this.frameRateWarmupSamples-- > 0)
+            return stats;
+        const limited = stats.layers.some((layer) => layer.active && (layer.qualityLimitation === 'cpu' || layer.qualityLimitation === 'bandwidth'));
+        this.frameRateLowSamples = limited && stats.fps > 0 && stats.fps < 50 ? this.frameRateLowSamples + 1 : 0;
+        if (this.frameRateLowSamples >= 3) {
+            this.frameRateFallback = true;
+            void this.cameraTask(async () => {
+                const producer = this.cameraProducer();
+                if (!producer?.track || producer.closed)
+                    return;
+                await producer.track.applyConstraints?.({ frameRate: { ideal: 30, max: 30 } }).catch(() => undefined);
+                await this.reproduceCamera(producer, producer.track);
+                this.emit('error', new RpcError('frame-rate-fallback', 'this device or upload cannot sustain 60 fps; publishing at 30 fps'));
+            }).catch((error) => this.emit('error', error));
+        }
+        return stats;
+    }
     // ------------------------------------------------------------------ music
     /**
      * Publish music playing inside the page, as its own track.
@@ -620,11 +1329,17 @@ export class StreamClient {
      * the user has not interacted with.
      */
     async publishMusic(input, options) {
+        const generation = this.publishGeneration;
         if (this.music)
             await this.stopMusic();
         const music = new MusicSource(input, options);
         try {
-            await this.produceMusic(music);
+            await this.produceMusic(music, generation);
+            // Checked again here: an unpublish() after the producer was kept has
+            // already closed it, and the music must not be remembered for a
+            // reconnect to publish again.
+            if (generation !== this.publishGeneration)
+                throw new Error(unpublishedWhile('music'));
         }
         catch (err) {
             music.close();
@@ -634,8 +1349,10 @@ export class StreamClient {
         this.wasPublishing = true;
         this.emitLocalMedia();
     }
-    async produceMusic(music) {
+    async produceMusic(music, generation) {
         const transport = await this.ensureSendTransport();
+        if (generation !== this.publishGeneration)
+            throw new Error(unpublishedWhile('music'));
         const producer = await transport.produce({
             track: music.track,
             stopTracks: false,
@@ -644,8 +1361,10 @@ export class StreamClient {
             codecOptions: { opusStereo: true, opusDtx: false, opusFec: true, opusMaxAverageBitrate: 128_000 },
             appData: { source: 'music' },
         });
-        this.producers.set(producer.id, producer);
+        await this.keepProducer(producer, generation, 'music');
         this.musicProducerId = producer.id;
+        // Music switched off before a reconnect comes back paused here.
+        await this.pauseOnServer(producer);
     }
     /** True while music is published, whether or not it is switched on. */
     get musicPublishing() {
@@ -708,6 +1427,24 @@ export class StreamClient {
         }));
         const consumer = await transport.consume(params);
         this.consumers.set(remote.producerId, consumer);
+        this.remoteVideoLayers.set(remote.producerId, remote.videoLayers ?? remote.appData?.videoLayers ?? params.videoLayers ?? []);
+        // Layers asked for before a reconnect: the new consumer gets them before
+        // its first frame instead of starting at the full picture. Not getting
+        // them costs data, not the track, so a refusal is only reported.
+        let layers = this.preferredLayers.get(remote.producerId);
+        const receiveQuality = this.receiveQualities.get(remote.producerId);
+        if (receiveQuality && receiveQuality !== 'auto') {
+            try {
+                layers = { spatialLayer: receiveSpatialLayer(this.remoteVideoLayers.get(remote.producerId) ?? [], receiveQuality) };
+            }
+            catch {
+                layers = undefined;
+                this.emit('error', new RpcError('quality-unavailable', 'this publisher no longer describes its video layers; using Auto'));
+            }
+        }
+        if (layers) {
+            await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers }).catch((err) => this.emit('error', err));
+        }
         // The server created it paused. Resume only now that our end exists,
         // otherwise the first frames arrive before there is anything to decode them.
         await this.rpc('resumeConsumer', { consumerId: consumer.id });
@@ -736,7 +1473,7 @@ export class StreamClient {
     }
     trackEnabled(kind) {
         for (const p of this.producers.values()) {
-            if (p.kind === kind && p.id !== this.musicProducerId)
+            if (p.kind === kind && p.appData.source === defaultSource(kind))
                 return !p.paused;
         }
         return false;
@@ -752,43 +1489,76 @@ export class StreamClient {
     async setCamEnabled(enabled) {
         await this.setKindEnabled('video', enabled);
     }
+    applyLocalModeration(kind) {
+        for (const producer of this.producers.values()) {
+            if (!kind || producer.kind === kind)
+                producer.pause();
+        }
+        // During reconnect there are no producers, but the retained tracks still
+        // capture. Keep their muted state for republish as well.
+        for (const stream of [this.localStream, this.screenStream]) {
+            for (const track of stream?.getTracks() ?? [])
+                if (!kind || track.kind === kind)
+                    track.enabled = false;
+        }
+        if (!kind || kind === 'audio')
+            this.music?.setEnabled(false);
+    }
     async setKindEnabled(kind, enabled) {
         for (const producer of this.producers.values()) {
             // The music has its own switch. Muting the microphone must not stop the song.
-            if (producer.kind !== kind || producer.id === this.musicProducerId)
+            if (producer.kind !== kind || producer.appData.source !== defaultSource(kind))
                 continue;
-            if (enabled)
-                producer.resume();
-            else
+            if (!enabled)
                 producer.pause();
             await this.rpc('pauseProducer', { producerId: producer.id, paused: !enabled });
+            if (enabled)
+                producer.resume();
         }
         this.emitLocalMedia();
     }
     /** Stop publishing entirely and release the local devices. */
     async unpublish() {
+        // All of it takes effect now; only telling the server is waited for. So
+        // publish() and unpublish() act in the order they are called: a publish
+        // called while this still waits is not undone by it, and one already
+        // under way stops at its next step (keepProducer).
+        this.publishGeneration += 1;
+        const screenWasStarted = this.screenStarted;
+        this.releaseScreen();
+        if (screenWasStarted)
+            this.emit('screenShareEnded', { reason: 'stopped' });
         const ids = [...this.producers.keys()];
         for (const producer of this.producers.values())
             producer.close();
         this.producers.clear();
-        for (const id of ids) {
-            await this.rpc('closeProducer', { producerId: id }).catch(() => undefined);
-        }
+        this.cameraEncodings = null;
+        // Start over for the next publish, and clear a limitation still showing:
+        // nothing is being held back once nothing is being sent.
+        const cleared = this.limitWatch.reset();
+        if (cleared)
+            this.emit('videoLimited', cleared);
         this.localStream?.getTracks().forEach((t) => t.stop());
         this.localStream = null;
+        // Forgotten with the stream: kept, a reconnect would ask for the camera
+        // and microphone again and publish them, after the app unpublished them.
+        this.publishConstraints = null;
         this.music?.close();
         this.music = null;
         this.musicProducerId = null;
         this.wasPublishing = false;
         this.emit('localStreamChanged', null);
         this.emit('localMediaChanged', { micEnabled: false, camEnabled: false, musicEnabled: false });
+        for (const id of ids) {
+            await this.rpc('closeProducer', { producerId: id }).catch(() => undefined);
+        }
     }
     get local() {
         return this.localStream;
     }
     // ------------------------------------------------------------- moderation
-    raiseHand() { return this.rpc('raiseHand'); }
-    lowerHand() { return this.rpc('lowerHand'); }
+    raiseHand(metadata) { return this.rpc('raiseHand', metadata ? { metadata } : {}); }
+    lowerHand(identity) { return this.rpc('lowerHand', identity ? { identity } : {}); }
     muteParticipant(identity) { return this.rpc('requestMute', { identity }); }
     /**
      * Moderating roles: switch someone's microphone (`audio`), camera (`video`)