@voltkit-rtc/client 0.8.0 → 0.9.1

package/src/index.ts

@@ -17,6 +17,8 @@ type RtpCapabilities = ms.RtpCapabilities;
 type RtpCodecCapability = ms.RtpCodecCapability;
 type Transport = ms.Transport;
 type TransportOptions = ms.TransportOptions;
+/** The transports a join made when asked (`join {transports}`, P3-6): exactly what createTransport returns. */
+type PreparedTransports = { send?: TransportOptions; recv?: TransportOptions };
 
 import type { ChatEvents, ChatMessage, ChatState, SendMessageOptions } from './chat.js';
 import { isTerminalError, RpcError } from './errors.js';
@@ -36,6 +38,7 @@ import { AdaptiveVideos, pageIsVisible, type AdaptiveWant } from './adaptive.js'
 import { AudioProtection, PROTECT_SAMPLE_MS, readInboundAudio, readOutboundAudio } from './fallback.js';
 import { ActiveSpeakers, AUDIO_LEVEL_INTERVAL_MS, SPEAKING_LEVEL } from './speakers.js';
 import { connectionPath, runPreflight, syntheticMedia, type PreflightOptions, type PreflightReport } from './preflight.js';
+import type { AudioTrackProcessor, VideoTrackProcessor } from './processors.js';
 import { DiagnosticsLog, SDK_VERSION, type Diagnostics, type DiagnosticsTransport } from './diagnostics.js';
 import { HEALTH_SAMPLE_MS, HealthDecisions, INTERRUPTED_AFTER_MS, readMediaSource, type MediaWarning } from './health.js';
 import { constraintsFor, deviceIdOf, diffDevices, enumerate, type DeviceKind, type DevicesChangedEvent, type MediaDevice } from './devices.js';
@@ -102,6 +105,7 @@ export { SPEAKING_LEVEL } from './speakers.js';
 export type { MediaWarning, MediaWarningCode } from './health.js';
 export type { PreflightOptions, PreflightReport, PreflightStep, PreflightStepName } from './preflight.js';
 export { SDK_VERSION } from './diagnostics.js';
+export type { AudioTrackProcessor, VideoTrackProcessor } from './processors.js';
 export type { Diagnostics, DiagnosticsEntry, DiagnosticsTransport } from './diagnostics.js';
 export type { VideoLayerStats, VideoLimitedEvent, VideoLimitReason, VideoReceiveStats, VideoSendStats } from './stats.js';
 export type {
@@ -678,6 +682,8 @@ export class StreamClient {
   // server only remembers the last one, rejecting consumes on the first.
   private sendTransportPending: Promise<Transport> | null = null;
   private recvTransportPending: Promise<Transport> | null = null;
+  /** Transports the join made (P3-6), each used once by the first transport of its direction. */
+  private preparedTransports: PreparedTransports | null = null;
   private readonly pending = new Map<number, Pending>();
   private readonly listeners = new Map<string, Set<Handler>>();
   private readonly consumers = new Map<string, Consumer>();
@@ -774,6 +780,13 @@ export class StreamClient {
   private healthTimer: ReturnType<typeof setInterval> | null = null;
   private healthSampling = false;
   private readonly interruptions = new Map<'camera' | 'mic', ReturnType<typeof setTimeout>>();
+  /** setAudioProcessor: what the microphone passes through, and the device's own track while it does. */
+  private audioProcessor: AudioTrackProcessor | null = null;
+  private micRaw: MediaStreamTrack | null = null;
+  private videoProcessor: VideoTrackProcessor | null = null;
+  /** With a video processor: the camera (or the app's track) it draws from, and the picture it makes -- sent and shown. */
+  private camRaw: MediaStreamTrack | null = null;
+  private camOut: MediaStreamTrack | null = null;
   /** getDiagnostics(): the SDK's recent events, for a support ticket. */
   private readonly diagnosticsLog = new DiagnosticsLog();
   /** play(): elements the browser refused to play until the viewer interacts, retried on the next gesture. */
@@ -1033,8 +1046,13 @@ export class StreamClient {
       const opening = this.openSocket();
       socket = this.ws;
       await opening;
-      result = (await this.rpc('join', { token: this.token })) as JoinResult;
+      // The transports come with the join where the server can make them (P3-6): the first publish and the first
+      // video skip a createTransport round trip. Older servers ignore the request and the client asks as before.
+      const joined = (await this.rpc('join', { token: this.token, transports: ['send', 'recv'] })) as JoinResult & { transports?: PreparedTransports };
+      const { transports, ...joinResult } = joined;
+      result = joinResult;
       this.joinResult = result;
+      this.preparedTransports = transports ?? null;
 
       this.device = this.createDevice();
       await this.device.load({ routerRtpCapabilities: result.routerRtpCapabilities });
@@ -1283,6 +1301,14 @@ export class StreamClient {
 
   close(): void {
     void this.stopRecording().catch(() => undefined);
+    // The processor's audio graph goes with the session.
+    const processor = this.audioProcessor;
+    this.audioProcessor = null;
+    this.micRaw = null;
+    void Promise.resolve(processor?.destroy()).catch(() => undefined);
+    const videoProcessor = this.videoProcessor;
+    this.videoProcessor = null;
+    void Promise.resolve(videoProcessor?.destroy()).catch(() => undefined);
     // Adaptive stream stops watching elements: nothing is received any more.
     this.adaptive.dispose();
     this.adaptivePaused.clear();
@@ -1305,7 +1331,7 @@ export class StreamClient {
     this.skippedVideos.clear();
     // Producers no longer stop their tracks, so releasing the camera and
     // microphone is ours to do -- otherwise the device light stays on.
-    this.localStream?.getTracks().forEach((t) => t.stop());
+    this.stopLocalTracks();
     this.localStream = null;
     const ws = this.ws;
     this.ws = null;
@@ -1319,6 +1345,9 @@ export class StreamClient {
   }
 
   private teardownMedia(): void {
+    this.micRaw = null;
+    // Made for the session that is going; a rejoin brings new ones.
+    this.preparedTransports = null;
     this.mediaPaths.reset();
     // The videos go with their consumers; a reconnect decides afresh. What was reported held back is withdrawn, so an
     // app's "poor connection" overlay does not outlive it.
@@ -1788,18 +1817,14 @@ export class StreamClient {
     if (!this.device) throw new RpcError('not-connected', 'not connected');
     const device = this.device;
     const socket = this.ws;
-    const params = (await this.rpc('createTransport', { direction: 'send' })) as TransportOptions;
+    const params = this.takePreparedTransport('send') ?? (await this.rpc('createTransport', { direction: 'send' })) as TransportOptions;
     if (this.closedByUs || device !== this.device || socket !== this.ws) throw closedWhileConnecting();
     const transport = device.createSendTransport({
       ...params,
       iceServers: this.joinResult?.iceServers,
     });
 
-    transport.on('connect', ({ dtlsParameters }, ok, fail) => {
-      this.rpc('connectTransport', { transportId: transport.id, dtlsParameters })
-        .then(() => ok())
-        .catch(fail);
-    });
+    this.connectEarly(transport);
     transport.on('produce', ({ kind, rtpParameters, appData }, ok, fail) => {
       this.rpc('produce', {
         transportId: transport.id, kind, rtpParameters: declareAudioQuality(rtpParameters, appData),
@@ -1814,6 +1839,33 @@ export class StreamClient {
     return transport;
   }
 
+  /**
+   * A faster first picture (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-6). mediasoup-client applies the
+   * remote description -- which starts ICE -- only once this transport's DTLS parameters are with the server: a round
+   * trip before any media path is even tried, for the first publish and the first video alike. mediasoup does not need
+   * them first: it starts DTLS as soon as ICE connects and checks the fingerprint whenever it arrives (its `auto`
+   * role; the browser is always the DTLS client), and the server handles a socket's requests in order. So they are
+   * sent and the transport goes on at once: ICE starts a round trip sooner, and the first produce or resume follows
+   * straight after. A refusal means the transport can never carry media: the session rejoins.
+   */
+  private connectEarly(transport: Transport): void {
+    transport.on('connect', ({ dtlsParameters }, ok) => {
+      ok();
+      this.rpc('connectTransport', { transportId: transport.id, dtlsParameters }).catch((err) => {
+        if (transport.closed || this.closedByUs || (transport !== this.sendTransport && transport !== this.recvTransport)) return;
+        this.emit('error', err as Error);
+        this.recoverConnection(`connectTransport failed: ${(err as Error)?.message ?? err}`);
+      });
+    });
+  }
+
+  /** The transport the join made for this direction, once: a later one is created as before. */
+  private takePreparedTransport(direction: 'send' | 'recv'): TransportOptions | null {
+    const prepared = this.preparedTransports?.[direction] ?? null;
+    if (this.preparedTransports) this.preparedTransports[direction] = undefined;
+    return prepared;
+  }
+
   private ensureRecvTransport(): Promise<Transport> {
     if (this.recvTransport) return Promise.resolve(this.recvTransport);
     if (this.recvTransportPending) return this.recvTransportPending;
@@ -1827,17 +1879,13 @@ export class StreamClient {
     if (!this.device) throw new RpcError('not-connected', 'not connected');
     const device = this.device;
     const socket = this.ws;
-    const params = (await this.rpc('createTransport', { direction: 'recv' })) as TransportOptions;
+    const params = this.takePreparedTransport('recv') ?? (await this.rpc('createTransport', { direction: 'recv' })) as TransportOptions;
     if (this.closedByUs || device !== this.device || socket !== this.ws) throw closedWhileConnecting();
     const transport = device.createRecvTransport({
       ...params,
       iceServers: this.joinResult?.iceServers,
     });
-    transport.on('connect', ({ dtlsParameters }, ok, fail) => {
-      this.rpc('connectTransport', { transportId: transport.id, dtlsParameters })
-        .then(() => ok())
-        .catch(fail);
-    });
+    this.connectEarly(transport);
 
     this.recvTransport = transport;
     this.watchTransport(transport);
@@ -1951,7 +1999,8 @@ export class StreamClient {
     this.audioCapReported = false;
     await this.micTask(async () => {
       const current = this.micProducer();
-      const track = current?.track;
+      // The device's track: the processed one is the old processor graph's, let go when it is processed again.
+      const track = current ? this.deviceTrackOf(current) : null;
       if (!current || !track || track.readyState !== 'live' || this.micQuality === quality) return;
       await this.reproduceMic(current, track);
     });
@@ -1991,8 +2040,10 @@ export class StreamClient {
     if (generation !== this.publishGeneration) throw unpublishedWhile('microphone');
     const quality = this.audioQualityFor();
     const settings = AUDIO_SETTINGS[quality];
+    // Through the audio processor (noise suppression) when there is one; the device's track stays the one watched.
+    const sent = await this.processMic(track);
     const producer = await transport.produce({
-      track,
+      track: sent,
       // Keep the track alive when the producer closes: a reconnect closes
       // every producer, and a stopped track cannot be republished.
       stopTracks: false,
@@ -2005,6 +2056,7 @@ export class StreamClient {
       appData: { source: 'mic', audioQuality: quality },
     });
     await this.keepProducer(producer, generation, 'microphone');
+    this.micRaw = sent !== track ? track : null;
     this.watchTrackEnded(producer);
     this.micQuality = quality;
     // A microphone handed over switched off, or muted before a reconnect.
@@ -2080,7 +2132,8 @@ export class StreamClient {
       const current = this.cameraProducer();
       const track = current?.track;
       if (!current || !track || track.readyState !== 'live') return;
-      try { await track.applyConstraints?.({ ...track.getConstraints?.(), frameRate: { ideal: frameRate, max: frameRate } }); }
+      const camera = this.cameraSource(track);
+      try { await camera.applyConstraints?.({ ...camera.getConstraints?.(), frameRate: { ideal: frameRate, max: frameRate } }); }
       catch { /* Existing capture settings decide whether 60 fps is possible. */ }
       await this.reproduceCamera(current, track);
     });
@@ -2130,40 +2183,51 @@ export class StreamClient {
       // but the camera is still being published: the republish sends this.
       const reconnecting = !current && this.wasPublishing && (this.localStream?.getVideoTracks().length ?? 0) > 0;
       if (!current && !reconnecting) throw new Error('replaceVideoTrack: no camera is being published');
-      if (!current) {
-        // The republish reads the camera's state off the track it sends, and
-        // the track being replaced is switched off exactly when the camera is
-        // paused. Hand that state on, or a camera that was off comes back on;
-        // and the old track back switched on, as below.
-        const replaced = this.localStream?.getVideoTracks()[0];
-        if (replaced && replaced !== track) {
-          track.enabled = replaced.enabled;
-          replaced.enabled = true;
-        }
-      }
-      this.adoptVideoTrack(track);
-      if (!current || current.track === track) return;
-      const previous = current.track;
-      const paused = current.paused;
+      // The picture behind what is sent now (a video processor's input), for its size; the new one goes through the
+      // processor too, when there is one.
+      const shown = current?.track ?? this.localStream?.getVideoTracks()[0] ?? null;
+      const previousSource = shown ? this.cameraSource(shown) : null;
+      const before = this.camOut;
+      const sent = await this.processCamera(track);
       try {
-        const { width = 0, height = 0 } = track.getSettings();
-        const frameRate = this.frameRateFor(track);
-        const next = videoEncodings(width, height, this.cameraQuality(frameRate), this.layerCount(), frameRate);
-        const previousSize = current.track?.getSettings();
-        if (this.cameraEncodings && sameLayout(this.cameraEncodings, next) &&
-            previousSize?.width === width && previousSize?.height === height) {
-          await current.replaceTrack({ track });
-          this.watchTrackEnded(current);
-          await this.applyLayerBitrates(current, next);
-          return;
+        if (!current) {
+          // The republish reads the camera's state off the track it sends, and
+          // the track being replaced is switched off exactly when the camera is
+          // paused. Hand that state on, or a camera that was off comes back on;
+          // and the old track back switched on, as below.
+          const replaced = this.localStream?.getVideoTracks()[0];
+          if (replaced && replaced !== sent) {
+            sent.enabled = replaced.enabled;
+            replaced.enabled = true;
+          }
+        }
+        this.adoptVideoTrack(sent);
+        if (!current || current.track === sent) return;
+        const previous = current.track;
+        const paused = current.paused;
+        try {
+          const { width = 0, height = 0 } = track.getSettings();
+          const frameRate = this.frameRateFor(track);
+          const next = videoEncodings(width, height, this.cameraQuality(frameRate), this.layerCount(), frameRate);
+          const previousSize = previousSource?.getSettings();
+          if (this.cameraEncodings && sameLayout(this.cameraEncodings, next) &&
+              previousSize?.width === width && previousSize?.height === height) {
+            await current.replaceTrack({ track: sent });
+            this.watchTrackEnded(current);
+            await this.applyLayerBitrates(current, next);
+            return;
+          }
+          await this.reproduceCamera(current, sent);
+        } finally {
+          // Pausing switched the old track off (mediasoup-client pauses that
+          // way), and nothing switches it back on once it is replaced. It is the
+          // app's again, often the camera its canvas draws from: handed back
+          // off, that canvas stays black after the camera is resumed.
+          if (paused && previous && previous !== this.cameraProducer()?.track) previous.enabled = true;
         }
-        await this.reproduceCamera(current, track);
       } finally {
-        // Pausing switched the old track off (mediasoup-client pauses that
-        // way), and nothing switches it back on once it is replaced. It is the
-        // app's again, often the camera its canvas draws from: handed back
-        // off, that canvas stays black after the camera is resumed.
-        if (paused && previous && previous !== this.cameraProducer()?.track) previous.enabled = true;
+        // The processor's previous picture is no longer sent or shown.
+        if (before && before !== this.camOut) before.stop();
       }
     });
   }
@@ -2342,18 +2406,28 @@ export class StreamClient {
     paused = !track.enabled,
   ): Promise<Producer> {
     if (generation !== this.publishGeneration) throw unpublishedWhile('camera');
+    // With a video processor its picture is sent (and shown); the camera behind it has the size and frame rate. The
+    // picture already made from that camera is kept: a reconnect or a new quality does not start the processor again.
+    const source = this.cameraSource(track);
+    const before = this.camOut;
+    const beforeSource = this.camRaw;
+    const sent = await this.processCamera(source);
+    if (generation !== this.publishGeneration) {
+      if (sent !== source && sent !== before) sent.stop();
+      throw unpublishedWhile('camera');
+    }
     // mediasoup-client starts a producer paused exactly when its track is
     // off. Setting the track to the camera's state first makes the producer
     // start in it, and a camera that is off never sends a frame of the new
     // track; the server is told below.
-    track.enabled = !paused;
+    sent.enabled = !paused;
     // Simulcast from day one: it is what lets one node serve far more viewers.
     // The layers follow the picture's real size, the project's quality and
     // the device, so the browser sends every layer asked for (see
     // videoEncodings).
-    const { width = 0, height = 0 } = track.getSettings();
+    const { width = 0, height = 0 } = source.getSettings();
     // The frame rate first: the size the project allows depends on it.
-    this.cameraFrameRate = this.frameRateFor(track);
+    this.cameraFrameRate = this.frameRateFor(source);
     const quality = this.cameraQuality(this.cameraFrameRate);
     const encodings = videoEncodings(width, height, quality, this.layerCount(), this.cameraFrameRate);
     let codecs: RtpCodecCapability[] | undefined;
@@ -2365,7 +2439,7 @@ export class StreamClient {
     const chosen = await this.cameraCodec(codecs, encodings, width, height, quality);
     const produce = (codec: RtpCodecCapability | undefined) =>
       transport.produce({
-        track,
+        track: sent,
         stopTracks: false,
         encodings,
         codecOptions: { videoGoogleStartBitrate: VIDEO_START_BITRATE_KBPS },
@@ -2385,6 +2459,10 @@ export class StreamClient {
     // unpublish() may have run while this was being made (a publish, a
     // quality change or a track swap under way).
     await this.keepProducer(producer, generation, 'camera');
+    if (sent !== track) this.adoptVideoTrack(sent);
+    if (before && before !== sent) before.stop();
+    // A second publish brought another camera: the one the processor drew from before is no longer published.
+    if (beforeSource && beforeSource !== source && beforeSource !== this.camRaw) beforeSource.stop();
     this.watchTrackEnded(producer);
     this.syncPublishProtection();
     this.cameraEncodings = encodings;
@@ -2732,7 +2810,8 @@ export class StreamClient {
         const producer = this.cameraProducer();
         if (!isCurrent() || producer !== sampledProducer || this.frameRateFallback || !producer?.track || producer.closed) return;
         this.frameRateFallback = true;
-        await producer.track.applyConstraints?.({ ...producer.track.getConstraints?.(), frameRate: { ideal: 30, max: 30 } }).catch(() => undefined);
+        const camera = this.cameraSource(producer.track);
+        await camera.applyConstraints?.({ ...camera.getConstraints?.(), frameRate: { ideal: 30, max: 30 } }).catch(() => undefined);
         await this.reproduceCamera(producer, producer.track);
         this.emit('error', new RpcError('frame-rate-fallback', 'this device or upload cannot sustain 60 fps; publishing at 30 fps'));
       }).catch((error) => this.emit('error', error as Error));
@@ -2970,8 +3049,10 @@ export class StreamClient {
     if (adaptive) layers = this.effectiveLayers(remote.producerId, adaptive);
     // A camera arriving while the audio-only fallback holds cameras back starts held back too.
     const held = consumer.kind === 'video' && this.protection.active && (remote.source ?? 'camera') === 'camera';
+    // Not waited for: the server applies them before the resume below (a socket's requests in order), so the first
+    // frame is already the right size without a round trip in front of it (P3-6).
     if (layers) {
-      await this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers }).catch((err) => this.emit('error', err as Error));
+      void this.rpc('setPreferredLayers', { consumerId: consumer.id, ...layers }).catch((err) => this.emit('error', err as Error));
     }
     current(consumer);
 
@@ -3035,6 +3116,131 @@ export class StreamClient {
     return { ...manual, spatialLayer: Math.min(manual.spatialLayer, adaptive) };
   }
 
+  // ------------------------------------------------------- audio processor
+
+  /**
+   * Send the microphone through `processor` first (LiveKit's setProcessor, Agora's extensions): above all noise
+   * suppression -- `createNoiseSuppressor()` from `@voltkit-rtc/client/noise-suppression`. Applies to the microphone
+   * published now (swapped in place, no gap for listeners) and to every one after it: another device, a reconnect, a
+   * new quality. `null` sends the microphone as it is again and lets the processor go. A processor that fails never
+   * keeps the microphone off: it is sent as it is, and `error` says why.
+   */
+  async setAudioProcessor(processor: AudioTrackProcessor | null): Promise<void> {
+    await this.micTask(async () => {
+      const previous = this.audioProcessor;
+      this.audioProcessor = processor;
+      const producer = this.micProducer();
+      const device = producer ? this.deviceTrackOf(producer) : null;
+      if (producer && device && !producer.closed) {
+        const sent = await this.processMic(device);
+        await producer.replaceTrack({ track: sent });
+        this.micRaw = sent !== device ? device : null;
+      }
+      if (previous && previous !== processor) await Promise.resolve(previous.destroy()).catch(() => undefined);
+    });
+  }
+
+  /** The processor in use now, or null. */
+  get audioProcessorInUse(): AudioTrackProcessor | null {
+    return this.audioProcessor;
+  }
+
+  /** The track to send for the microphone's device track `track`: processed when there is a processor that works. */
+  private async processMic(track: MediaStreamTrack): Promise<MediaStreamTrack> {
+    const processor = this.audioProcessor;
+    if (!processor) return track;
+    try {
+      return await processor.process(track);
+    } catch (err) {
+      this.emit('error', new Error(`audio processor ${processor.name} failed, sending the microphone as it is: ${(err as Error)?.message ?? err}`));
+      return track;
+    }
+  }
+
+  /** The device's own track behind a producer: for the microphone, the one before any processor. */
+  private deviceTrackOf(producer: Producer): MediaStreamTrack | null {
+    const source = producer.appData.source;
+    if (source === 'mic') return this.micRaw ?? producer.track;
+    return source === 'camera' && producer.track ? this.cameraSource(producer.track) : producer.track;
+  }
+
+  /**
+   * Send the camera through `processor` first (LiveKit's setProcessor, Agora's and TRTC's video extensions): a blurred
+   * or replaced background, beauty -- `createVideoEffects()` from `@voltkit-rtc/client/video-effects`. The processed
+   * picture is what viewers get and what `local` shows. Applies to the camera published now (swapped in place: the
+   * same size, so no pause for viewers) and to every one after it: `switchDevice`, `replaceVideoTrack`, a reconnect, a
+   * new quality. `null` sends the camera as it is again and lets the processor go. A processor that fails never keeps
+   * the camera off: it is sent as it is, and `error` says why.
+   */
+  async setVideoProcessor(processor: VideoTrackProcessor | null): Promise<void> {
+    await this.cameraTask(async () => {
+      const previous = this.videoProcessor;
+      this.videoProcessor = processor;
+      const producer = this.cameraProducer();
+      const shown = producer?.track ?? this.localStream?.getVideoTracks()[0] ?? null;
+      if (shown && shown.readyState === 'live') {
+        const source = this.cameraSource(shown);
+        const before = this.camOut;
+        const paused = producer ? producer.paused : !shown.enabled;
+        // Made afresh, by the new processor.
+        this.camRaw = null;
+        this.camOut = null;
+        const sent = await this.processCamera(source);
+        sent.enabled = !paused;
+        if (producer && !producer.closed && producer.track !== sent) await producer.replaceTrack({ track: sent });
+        this.adoptVideoTrack(sent);
+        if (before && before !== sent) before.stop();
+      }
+      if (previous && previous !== processor) await Promise.resolve(previous.destroy()).catch(() => undefined);
+    });
+  }
+
+  /** The video processor in use now, or null. */
+  get videoProcessorInUse(): VideoTrackProcessor | null {
+    return this.videoProcessor;
+  }
+
+  /** The camera picture behind `track`: the one a video processor draws from, when `track` is its picture. */
+  private cameraSource(track: MediaStreamTrack): MediaStreamTrack {
+    return track === this.camOut && this.camRaw ? this.camRaw : track;
+  }
+
+  /**
+   * The picture to send for the camera picture `source`: a video processor's, when there is one that works. The one
+   * already made from `source` is kept (a reconnect, a new quality); otherwise it is made afresh.
+   */
+  private async processCamera(source: MediaStreamTrack): Promise<MediaStreamTrack> {
+    const processor = this.videoProcessor;
+    if (!processor) {
+      this.camRaw = null;
+      this.camOut = null;
+      return source;
+    }
+    if (this.camRaw === source && this.camOut?.readyState === 'live') return this.camOut;
+    try {
+      const out = await processor.process(source);
+      // The processor draws from the camera whether or not the camera is paused: its picture carries the pause.
+      source.enabled = true;
+      this.camRaw = source;
+      this.camOut = out;
+      return out;
+    } catch (err) {
+      this.emit('error', new Error(`video processor ${processor.name} failed, sending the camera as it is: ${(err as Error)?.message ?? err}`));
+      this.camRaw = null;
+      this.camOut = null;
+      return source;
+    }
+  }
+
+  /** Stop the client's own tracks, and the camera behind a video processor's picture (not in the stream itself). */
+  private stopLocalTracks(): void {
+    this.localStream?.getTracks().forEach((t) => t.stop());
+    this.camRaw?.stop();
+    this.camOut?.stop();
+    this.camRaw = null;
+    this.camOut = null;
+  }
+
   // ------------------------------------------------------- diagnostics
 
   /**
@@ -3239,8 +3445,10 @@ export class StreamClient {
     try {
       const mic = this.micProducer();
       const camera = this.cameraProducer();
-      const micLevel = mic && !mic.paused ? readMediaSource((await mic.getStats()).values() as Iterable<Record<string, unknown>>).level : null;
-      const cameraFps = camera && !camera.paused ? readMediaSource((await camera.getStats()).values() as Iterable<Record<string, unknown>>).fps : null;
+      // A processed microphone (noise suppression) is near-silent between words by design: no mic-silent then.
+      const micLevel = mic && !mic.paused && !this.audioProcessor ? readMediaSource((await mic.getStats()).values() as Iterable<Record<string, unknown>>).level : null;
+      // A processed picture runs at the processor's pace, not the camera's: no camera-fps-low then.
+      const cameraFps = camera && !camera.paused && !this.videoProcessor ? readMediaSource((await camera.getStats()).values() as Iterable<Record<string, unknown>>).fps : null;
       if (!this.healthTimer) return;
       this.health.feed({ micLevel, cameraFps, askedFps: this.videoFrameRate });
       // An interrupted device that is no longer published never says it came back: that is over too.
@@ -3278,7 +3486,9 @@ export class StreamClient {
     const md = typeof navigator !== 'undefined' ? navigator.mediaDevices : undefined;
     if (!md?.getUserMedia) throw new Error('switchDevice: this browser cannot capture media');
     if (kind === 'videoinput') {
-      const current = this.cameraProducer()?.track ?? this.localStream?.getVideoTracks()[0] ?? null;
+      const shown = this.cameraProducer()?.track ?? this.localStream?.getVideoTracks()[0] ?? null;
+      // With a video processor, the camera it draws from.
+      const current = shown ? this.cameraSource(shown) : null;
       if (!current) throw new Error('switchDevice: no camera is being published');
       if (!deviceIdOf(current)) {
         throw new Error('switchDevice: the published video is not a camera capture (a canvas or processed track); capture the camera yourself and call replaceVideoTrack');
@@ -3313,8 +3523,10 @@ export class StreamClient {
 
   /** Put `track` on the microphone producer and in the client's own stream; the replaced track is stopped. */
   private async adoptAudioTrack(producer: Producer, track: MediaStreamTrack): Promise<void> {
-    const previous = producer.track;
-    await producer.replaceTrack({ track });
+    const previous = this.micRaw ?? producer.track;
+    const sent = await this.processMic(track);
+    await producer.replaceTrack({ track: sent });
+    this.micRaw = sent !== track ? track : null;
     const video = this.localStream ? this.localStream.getTracks().filter((t) => t.kind !== 'audio') : [];
     this.localStream = new MediaStream([track, ...video]);
     this.emit('localStreamChanged', this.localStream);
@@ -3324,14 +3536,15 @@ export class StreamClient {
 
   /** Notice when the producer's present track stops from outside (unplugged, permission revoked). */
   private watchTrackEnded(producer: Producer): void {
-    const track = producer.track;
+    // The device's track, not the processed one a processor sends: that is the one a device ends.
+    const track = this.deviceTrackOf(producer);
     if (!track || typeof track.addEventListener !== 'function') return;
     track.addEventListener('ended', () => { void this.onTrackEnded(producer, track); }, { once: true });
     // The system taking the device (a call, another app) mutes the track; held a moment, it is reported.
     const which = producer.appData.source === 'mic' ? 'mic' : producer.appData.source === 'camera' ? 'camera' : null;
     if (!which) return;
     const code = which === 'mic' ? 'mic-interrupted' : 'camera-interrupted';
-    const current = () => !this.closedByUs && !producer.closed && producer.track === track;
+    const current = () => !this.closedByUs && !producer.closed && this.deviceTrackOf(producer) === track;
     track.addEventListener('mute', () => {
       if (!current() || this.interruptions.has(which)) return;
       this.interruptions.set(which, setTimeout(() => {
@@ -3353,7 +3566,7 @@ export class StreamClient {
    * swapped for another camera on its own (Chime stops it the same way).
    */
   private async onTrackEnded(producer: Producer, track: MediaStreamTrack): Promise<void> {
-    const live = () => !this.closedByUs && !producer.closed && producer.track === track && this.producers.get(producer.id) === producer;
+    const live = () => !this.closedByUs && !producer.closed && this.deviceTrackOf(producer) === track && this.producers.get(producer.id) === producer;
     if (!live()) return;
     const deviceId = deviceIdOf(track);
     if (producer.appData.source === 'mic') {
@@ -3729,7 +3942,7 @@ export class StreamClient {
     // nothing is being held back once nothing is being sent.
     const cleared = this.limitWatch.reset();
     if (cleared) this.emit('videoLimited', cleared);
-    this.localStream?.getTracks().forEach((t) => t.stop());
+    this.stopLocalTracks();
     this.localStream = null;
     // Forgotten with the stream: kept, a reconnect would ask for the camera
     // and microphone again and publish them, after the app unpublished them.