@voltkit-rtc/client 0.9.2 → 0.9.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voltkit-rtc/client",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "description": "Browser SDK for VoltKit RTC: realtime audio/video rooms, chat and live broadcast publishing.",
   "keywords": [
     "voltkit",

package/src/diagnostics.ts

@@ -7,7 +7,7 @@
  */
 
 /** This package's version; kept equal to package.json by a test. */
-export const SDK_VERSION = '0.9.2';
+export const SDK_VERSION = '0.9.3';
 
 /** How many recent events the log keeps. */
 export const DIAGNOSTICS_LOG_SIZE = 200;

package/src/index.ts

@@ -564,6 +564,12 @@ interface PreferredLayers {
 
 const isLayer = (value: unknown): value is number => Number.isInteger(value) && (value as number) >= 0;
 
+/** A viewer's choice for one video: a quality (setReceiveQuality, which also sets the layers) or a ceiling (setPreferredLayers). */
+interface ReceiveChoice {
+  quality: ReceiveQuality | undefined;
+  layers: PreferredLayers | undefined;
+}
+
 /**
  * Where a new camera stream's bandwidth estimate starts, in kbps. Chrome
  * otherwise starts low and takes many seconds to climb to the top layer.
@@ -732,6 +738,14 @@ export class StreamClient {
   private readonly screenProducerIds = new Set<string>();
   private readonly remoteVideoLayers = new Map<string, VideoLayer[]>();
   private readonly receiveQualities = new Map<string, ReceiveQuality>();
+  /**
+   * BR-09 (the Flutter SDK's V092-02): per video, the choice the server last accepted, and a revision every change
+   * bumps. A choice is stored before its request (what is asked for is read from it), so a request that fails goes
+   * back to the last accepted choice -- not merely the one before it, which may itself be waiting or refused -- and
+   * only the latest change may go back, so a late failure never undoes a newer choice. Replies come in request order.
+   */
+  private readonly receiveAccepted = new Map<string, ReceiveChoice>();
+  private readonly receiveRevision = new Map<string, number>();
   private cameraFrameRate: 30 | 60 = 30;
   private frameRateFallback = false;
   private frameRateLowSamples = 0;
@@ -1104,10 +1118,9 @@ export class StreamClient {
     // A video that ended while this client was away sends it no
     // producerClosed: forget the layers asked for it.
     const present = new Set(result.producers.map((p) => p.producerId));
-    for (const producerId of this.preferredLayers.keys()) {
+    for (const producerId of new Set([...this.preferredLayers.keys(), ...this.receiveAccepted.keys()])) {
       if (!present.has(producerId)) {
-        this.preferredLayers.delete(producerId);
-        this.receiveQualities.delete(producerId);
+        this.forgetReceiveChoice(producerId);
         this.remoteVideoLayers.delete(producerId);
       }
     }
@@ -1322,6 +1335,8 @@ export class StreamClient {
     this.teardownMedia();
     this.preferredLayers.clear();
     this.receiveQualities.clear();
+    this.receiveAccepted.clear();
+    this.receiveRevision.clear();
     this.remoteVideoLayers.clear();
     this.releaseScreen();
     this.music?.close();
@@ -1603,8 +1618,7 @@ export class StreamClient {
       case 'producerClosed': {
         const { producerId, identity } = data as { producerId: string; identity: string };
         this.subscriptions.delete(producerId);
-        this.preferredLayers.delete(producerId);
-        this.receiveQualities.delete(producerId);
+        this.forgetReceiveChoice(producerId);
         this.remoteVideoLayers.delete(producerId);
         this.skippedVideos.delete(producerId);
         this.adaptivePaused.delete(producerId);
@@ -2188,7 +2202,10 @@ export class StreamClient {
       const shown = current?.track ?? this.localStream?.getVideoTracks()[0] ?? null;
       const previousSource = shown ? this.cameraSource(shown) : null;
       const before = this.camOut;
+      const beforeRaw = this.camRaw;
+      const wasPaused = current?.paused ?? false;
       const sent = await this.processCamera(track);
+      let swapped = false;
       try {
         if (!current) {
           // The republish reads the camera's state off the track it sends, and
@@ -2202,7 +2219,10 @@ export class StreamClient {
           }
         }
         this.adoptVideoTrack(sent);
-        if (!current || current.track === sent) return;
+        if (!current || current.track === sent) {
+          swapped = true;
+          return;
+        }
         const previous = current.track;
         const paused = current.paused;
         try {
@@ -2213,11 +2233,13 @@ export class StreamClient {
           if (this.cameraEncodings && sameLayout(this.cameraEncodings, next) &&
               previousSize?.width === width && previousSize?.height === height) {
             await current.replaceTrack({ track: sent });
+            swapped = true;
             this.watchTrackEnded(current);
             await this.applyLayerBitrates(current, next);
             return;
           }
           await this.reproduceCamera(current, sent);
+          swapped = true;
         } finally {
           // Pausing switched the old track off (mediasoup-client pauses that
           // way), and nothing switches it back on once it is replaced. It is the
@@ -2225,13 +2247,55 @@ export class StreamClient {
           // off, that canvas stays black after the camera is resumed.
           if (paused && previous && previous !== this.cameraProducer()?.track) previous.enabled = true;
         }
+      } catch (err) {
+        if (!swapped) {
+          await this.undoCameraSwap({ producer: current ?? null, given: track, made: sent, shown, source: previousSource, raw: beforeRaw, out: before, paused: wasPaused });
+        }
+        throw err;
       } finally {
-        // The processor's previous picture is no longer sent or shown.
-        if (before && before !== this.camOut) before.stop();
+        // The processor's previous picture is no longer sent or shown -- once the new one is.
+        if (swapped && before && before !== this.camOut) before.stop();
       }
     });
   }
 
+  /**
+   * A camera swap that failed (BR-12; the Flutter SDK's V090-02). What is sent is still the old picture -- or nothing,
+   * when the producer was replaced and the new one failed. The client's stream goes back to the old picture, so the
+   * app's preview and a reconnect never use a track that did not go on air. With a processor, the old camera is put
+   * back through it if it dropped the old picture when given the new track. The picture made for the new track is
+   * released; the new track itself is the caller's.
+   */
+  private async undoCameraSwap(undo: {
+    producer: Producer | null; given: MediaStreamTrack; made: MediaStreamTrack; shown: MediaStreamTrack | null;
+    source: MediaStreamTrack | null; raw: MediaStreamTrack | null; out: MediaStreamTrack | null; paused: boolean;
+  }): Promise<void> {
+    const { producer, given, made, shown, source, raw, out, paused } = undo;
+    if (made !== given) made.stop();
+    const sending = () => producer !== null && !producer.closed && this.cameraProducer() === producer;
+    let picture = shown;
+    if (shown && shown.readyState === 'ended' && source && source !== shown && source.readyState !== 'ended') {
+      // The processor dropped the picture on air: make it again from the old camera, and send that.
+      picture = await this.processCamera(source);
+      if (sending() && producer && producer.track !== picture) {
+        try {
+          await producer.replaceTrack({ track: picture });
+          this.watchTrackEnded(producer);
+        } catch (err) {
+          this.emit('error', new Error(`the camera could not be put back after a failed switch: ${(err as Error)?.message ?? err}`));
+        }
+      }
+    } else {
+      // The picture on air is as it was, and so is what stands behind it.
+      this.camRaw = raw;
+      this.camOut = out;
+    }
+    if (!picture || picture.readyState === 'ended') return;
+    // With nothing sending it, a republish reads the camera's state off the track: a paused camera stays paused.
+    if (!sending()) picture.enabled = !paused;
+    this.adoptVideoTrack(picture);
+  }
+
   /**
    * Give a camera producer whose track was swapped in place the bitrates of
    * its new picture. The layout matched, but a bigger picture at the same
@@ -2302,8 +2366,10 @@ export class StreamClient {
    * 1280×720), and a layer above the biggest means the biggest. Leave
    * `temporalLayer` out to keep the full frame rate. It is the most this
    * client is sent: a connection that cannot carry it still gets less. The
-   * choice is kept across reconnects and forgotten when the track ends.
-   * Rejects for a producer whose video this client is not receiving.
+   * choice is kept across reconnects and forgotten when the track ends; a
+   * call that fails (refused, or the connection lost before the answer)
+   * leaves the previous choice in place. Rejects for a producer whose video
+   * this client is not receiving.
    */
   async setPreferredLayers(producerId: string, spatialLayer: number, temporalLayer?: number): Promise<void> {
     if (!isLayer(spatialLayer) || (temporalLayer !== undefined && !isLayer(temporalLayer))) {
@@ -2314,12 +2380,18 @@ export class StreamClient {
       throw new Error(`setPreferredLayers: this client is not receiving a video with producerId ${producerId}`);
     }
     const layers: PreferredLayers = temporalLayer === undefined ? { spatialLayer } : { spatialLayer, temporalLayer };
-    this.receiveQualities.delete(producerId);
-    // Kept first: a reconnect that interrupts the request still applies it.
-    this.preferredLayers.set(producerId, layers);
+    // Stored before the request, which reads it; a request that fails (refused, or the connection lost before the
+    // answer) leaves the last accepted choice in place (BR-09).
+    const change = this.changeReceive(producerId, { quality: undefined, layers });
     // Shown through attachVideo: the lower of this and the size it is shown at.
     const adaptive = this.adaptive.current(producerId);
-    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...(adaptive ? this.effectiveLayers(producerId, adaptive) : layers) });
+    try {
+      await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...(adaptive ? this.effectiveLayers(producerId, adaptive) : layers) });
+    } catch (err) {
+      change.failed();
+      throw err;
+    }
+    change.accepted();
   }
 
   private async collectVideoStats(baseline: StatsBaseline): Promise<VideoSendStats | null> {
@@ -2343,7 +2415,7 @@ export class StreamClient {
    * big it is on screen: someone who picks 1080p expects 1080p, not the smaller copy that fits the tile. A connection
    * that cannot carry it still gets less until it recovers, and a video shown through attachVideo is still paused
    * while nobody can see it. 'auto' (the default) goes back to the size it is shown at, or the server's choice.
-   * Kept across reconnects.
+   * Kept across reconnects; a call that fails leaves the previous choice in place.
    */
   async setReceiveQuality(producerId: string, quality: ReceiveQuality): Promise<void> {
     if (!['auto', '1080p', '720p', '480p'].includes(quality)) throw new RangeError('setReceiveQuality: unsupported quality');
@@ -2355,10 +2427,48 @@ export class StreamClient {
       try { layers = { spatialLayer: receiveSpatialLayer(this.remoteVideoLayers.get(producerId) ?? [], quality) }; }
       catch { throw new RpcError('quality-unavailable', 'this publisher does not describe its video layers; use Auto'); }
     }
-    this.receiveQualities.set(producerId, quality);
-    this.preferredLayers.set(producerId, layers);
+    const change = this.changeReceive(producerId, { quality, layers });
     const adaptive = this.adaptive.current(producerId);
-    await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...(adaptive ? this.effectiveLayers(producerId, adaptive) : layers) });
+    try {
+      await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...(adaptive ? this.effectiveLayers(producerId, adaptive) : layers) });
+    } catch (err) {
+      change.failed();
+      throw err;
+    }
+    change.accepted();
+  }
+
+  /** Store `choice` for one video now, before its request; the caller reports how the request ended (BR-09). */
+  private changeReceive(producerId: string, choice: ReceiveChoice): { accepted(): void; failed(): void } {
+    if (!this.receiveAccepted.has(producerId)) {
+      this.receiveAccepted.set(producerId, { quality: this.receiveQualities.get(producerId), layers: this.preferredLayers.get(producerId) });
+    }
+    const revision = (this.receiveRevision.get(producerId) ?? 0) + 1;
+    this.receiveRevision.set(producerId, revision);
+    this.setReceiveChoice(producerId, choice);
+    return {
+      // Forgotten meanwhile (the video ended, the session closed): a late answer changes nothing.
+      accepted: () => { if (this.receiveRevision.has(producerId)) this.receiveAccepted.set(producerId, choice); },
+      failed: () => {
+        const back = this.receiveAccepted.get(producerId);
+        if (this.receiveRevision.get(producerId) === revision && back) this.setReceiveChoice(producerId, back);
+      },
+    };
+  }
+
+  private setReceiveChoice(producerId: string, choice: ReceiveChoice): void {
+    if (choice.quality === undefined) this.receiveQualities.delete(producerId);
+    else this.receiveQualities.set(producerId, choice.quality);
+    if (choice.layers === undefined) this.preferredLayers.delete(producerId);
+    else this.preferredLayers.set(producerId, choice.layers);
+  }
+
+  /** The video ended: its choices go with it. */
+  private forgetReceiveChoice(producerId: string): void {
+    this.preferredLayers.delete(producerId);
+    this.receiveQualities.delete(producerId);
+    this.receiveAccepted.delete(producerId);
+    this.receiveRevision.delete(producerId);
   }
 
   private cameraProducer(): Producer | undefined {
@@ -3117,6 +3227,9 @@ export class StreamClient {
    * setPreferredLayers and adaptive stream's size: the lower of the two.
    */
   private effectiveLayers(producerId: string, want: AdaptiveWant | null): PreferredLayers {
+    // Hidden: paused at the server, or held at the smallest layer where it cannot pause. A chosen quality or a ceiling
+    // applies once the video is shown again, never while nobody can see it (BR-08; the Flutter SDK's V092-01).
+    if (want && !want.visible) return { spatialLayer: 0, temporalLayer: 0 };
     let manual: PreferredLayers | undefined = this.preferredLayers.get(producerId);
     const quality = this.receiveQualities.get(producerId);
     if (quality && quality !== 'auto') {
@@ -3488,7 +3601,9 @@ export class StreamClient {
    * captured with the constraints publish was given (size, frame rate, echo cancellation) and its track replaces the
    * published one: a camera of the same size swaps with no pause for viewers, a microphone always does. The track it
    * replaces is stopped, releasing that device; the client's own stream follows (`localStreamChanged`), and a
-   * reconnect captures the chosen device again. A camera or microphone that is switched off stays off.
+   * reconnect captures the chosen device again. A camera or microphone that is switched off stays off. A swap that
+   * fails releases the new device and keeps the old one on air, put back through a noise suppressor or video effects
+   * if one is set.
    *
    * For a camera published as a canvas or another processed track, capture the new camera yourself and pass your
    * processed track to `replaceVideoTrack`; this rejects rather than replace your picture with the raw camera.
@@ -3528,16 +3643,35 @@ export class StreamClient {
         track.stop();
         throw new Error('switchDevice: the microphone was unpublished meanwhile');
       }
-      await this.adoptAudioTrack(producer, track);
+      try {
+        // Releases the new microphone itself if the swap fails.
+        await this.adoptAudioTrack(producer, track);
+      } catch (err) {
+        if (producer.closed || this.micProducer() !== producer) throw new Error('switchDevice: the microphone was unpublished meanwhile');
+        throw err;
+      }
       this.publishConstraints = { ...(this.publishConstraints ?? {}), audio };
     });
   }
 
-  /** Put `track` on the microphone producer and in the client's own stream; the replaced track is stopped. */
+  /**
+   * Put `track` (one the SDK captured) on the microphone producer and in the client's own stream; the replaced track is
+   * stopped. A swap that fails releases `track` and the sound made from it, and with a processor puts the microphone
+   * still on air back through it: given the new track, the processor dropped that sound (BR-10, BR-11).
+   */
   private async adoptAudioTrack(producer: Producer, track: MediaStreamTrack): Promise<void> {
     const previous = this.micRaw ?? producer.track;
-    const sent = await this.processMic(track);
-    await producer.replaceTrack({ track: sent });
+    const previousRaw = this.micRaw;
+    let sent = track;
+    try {
+      sent = await this.processMic(track);
+      await producer.replaceTrack({ track: sent });
+    } catch (err) {
+      if (sent !== track) sent.stop();
+      track.stop();
+      await this.restoreMic(producer, previousRaw);
+      throw err;
+    }
     this.micRaw = sent !== track ? track : null;
     const video = this.localStream ? this.localStream.getTracks().filter((t) => t.kind !== 'audio') : [];
     this.localStream = new MediaStream([track, ...video]);
@@ -3546,6 +3680,23 @@ export class StreamClient {
     this.watchTrackEnded(producer);
   }
 
+  /** After a failed microphone swap with a processor: send the microphone on air through it again (it dropped that sound). */
+  private async restoreMic(producer: Producer, raw: MediaStreamTrack | null): Promise<void> {
+    const current = () => !producer.closed && this.micProducer() === producer;
+    if (!this.audioProcessor || !raw || raw.readyState === 'ended' || !current()) return;
+    try {
+      const again = await this.processMic(raw);
+      if (!current()) {
+        if (again !== raw) again.stop();
+        return;
+      }
+      await producer.replaceTrack({ track: again });
+      this.micRaw = again !== raw ? raw : null;
+    } catch (err) {
+      this.emit('error', new Error(`the microphone could not be put back after a failed switch: ${(err as Error)?.message ?? err}`));
+    }
+  }
+
   /** Notice when the producer's present track stops from outside (unplugged, permission revoked). */
   private watchTrackEnded(producer: Producer): void {
     // The device's track, not the processed one a processor sends: that is the one a device ends.