@rapidmx/meet-plugin 0.5.0 → 0.8.0

package/apps/shared/media/useLocalMedia.ts

@@ -20,10 +20,18 @@
  * - **Devices**: `selectDevice()` swaps in another camera or microphone; `devicechange` refreshes the lists.
  * - **Level**: `audioLevel` (0-5) follows the microphone while it is unmuted, for the "your audio is being sent"
  * indicator.
+ * - **Filters**: while any video filter is on (`setFilters()`), `videoTrack`/`videoStream` are the *filtered*
+ * picture (`filters/VideoFilterProcessor.ts`) rather than the camera's own track - so everything downstream, the
+ * lobby preview and every path the call sends over, gets it without knowing. The camera's track stays owned here; the
+ * processor only reads it. If the filtered picture can't be made there is no video at all rather than an unfiltered
+ * one, since the filter may be hiding the participant's room.
+ * - **Remembered settings**: the devices picked, whether the microphone and camera were on, and the filters are
+ * saved on this device (`mediaPreferences.ts`) and applied the next time - a saved device is asked for as a
+ * preference (`ideal`), so one that has since been unplugged falls back to the default instead of failing.
  *
  * Nothing here runs during SSR: browser APIs are only touched from effects and event handlers.
  */
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
 import {
     type DeviceLists,
     type MediaAccessError,
@@ -33,6 +41,10 @@ import {
     requestUserMedia,
 } from "./deviceMedia.js";
 import { startLevelMeter } from "./levelMeter.js";
+import { loadPicture, prepareBackgroundImage } from "./filters/backgroundImage.js";
+import { NO_FILTERS, type VideoFilters, filtersActive } from "./filters/filterTypes.js";
+import { type BackgroundPicture, type FilterStatus, VideoFilterProcessor } from "./filters/VideoFilterProcessor.js";
+import { type MediaPreferences, loadPreferences, savePreferences } from "./mediaPreferences.js";
 
 export type MediaKind = "audio" | "video";
 
@@ -47,6 +59,9 @@ const VIDEO_CONSTRAINTS: MediaTrackConstraints = { facingMode: "user", width: {
 /** The mean deviation (the level meter's 0-100 scale) that fills the whole indicator - ordinary speech sits well
  * below full scale. */
 const LEVEL_FULL_SCALE = 20;
+const FILTERS_UNSUPPORTED =
+    "Video effects aren't supported in this browser, so your camera is off. Turn the effects off to show your camera.";
+
 export const AUDIO_LEVEL_STEPS = 5;
 
 export interface LocalMedia {
@@ -70,6 +85,16 @@ export interface LocalMedia {
     selectedDeviceIds: Partial<Record<MediaKind, string>>;
     /** 0-`AUDIO_LEVEL_STEPS`: how loud the microphone is right now - always 0 while muted. */
     audioLevel: number;
+    /** The video filters in effect - `NO_FILTERS` when the camera is sent as it is. */
+    filters: VideoFilters;
+    /** Whether the filters' models are still loading, and why a filter isn't fully working. */
+    filterStatus: FilterStatus;
+    /** Whether a custom background image has been chosen (and so "custom image" can be switched to at once). */
+    hasBackgroundImage: boolean;
+    /** Changes some of the filters, keeping the rest. */
+    setFilters(patch: Partial<VideoFilters>): void;
+    /** Uses `file` as the custom background and switches to it. Resolves to why the file can't be used, or `null`. */
+    chooseBackgroundImage(file: File): Promise<string | null>;
     /** Asks for the camera and microphone (or just `kind`), see this module's doc comment. */
     requestAccess(kind?: MediaKind): Promise<void>;
     toggleMic(): Promise<void>;
@@ -90,7 +115,17 @@ function statusFor(error: MediaAccessError): TrackStatus {
     }
 }
 
-export function useLocalMedia(): LocalMedia {
+export interface LocalMediaOptions {
+    /** Where the filters' models are hosted, when not on the default CDN - see `filters/mlModels.ts`. */
+    effectsAssetsUrl?: string;
+}
+
+/** `base`, asking for `preferredId` (the device used last time) when there is one - as a preference, not a demand. */
+function preferring(base: MediaTrackConstraints, preferredId?: string): MediaTrackConstraints {
+    return preferredId ? { ...base, deviceId: { ideal: preferredId } } : base;
+}
+
+export function useLocalMedia({ effectsAssetsUrl }: LocalMediaOptions = {}): LocalMedia {
     const [supported, setSupported] = useState(true);
     const [requesting, setRequesting] = useState(false);
     const [error, setError] = useState<MediaAccessError | null>(null);
@@ -99,10 +134,66 @@ export function useLocalMedia(): LocalMedia {
     const [micEnabled, setMicEnabled] = useState(true);
     const [devices, setDevices] = useState<DeviceLists>({ cameras: [], microphones: [] });
     const [audioLevel, setAudioLevel] = useState(0);
+    const [filters, setFiltersState] = useState<VideoFilters>(NO_FILTERS);
+    const [background, setBackground] = useState<BackgroundPicture | null>(null);
+    const [filterStatus, setFilterStatus] = useState<FilterStatus>({ loading: false, error: null });
+    const [processed, setProcessed] = useState<MediaStreamTrack | null>(null);
 
     const tracksRef = useRef<Record<MediaKind, MediaStreamTrack | null>>({ audio: null, video: null });
     const micEnabledRef = useRef(true);
     const mountedRef = useRef(true);
+    const prefsRef = useRef<MediaPreferences | null>(null);
+    const filtersRef = useRef<VideoFilters>(NO_FILTERS);
+    const backgroundRef = useRef<BackgroundPicture | null>(null);
+    const processorRef = useRef<VideoFilterProcessor | null>(null);
+
+    const applyFilters = useCallback((next: VideoFilters) => {
+        filtersRef.current = next;
+        setFiltersState(next);
+    }, []);
+
+    const applyBackground = useCallback((picture: BackgroundPicture) => {
+        backgroundRef.current = picture;
+        setBackground(picture);
+    }, []);
+
+    /** What was remembered from last time - read once, the first time anything needs it, and applied to the state it
+     * covers. Never during render: storage doesn't exist on the server, and reading it would make the first client
+     * render differ from the server's. */
+    const ensurePrefs = useCallback((): MediaPreferences => {
+        if (prefsRef.current) {
+            return prefsRef.current;
+        }
+        const prefs = loadPreferences();
+        prefsRef.current = prefs;
+        if (prefs.micEnabled === false) {
+            micEnabledRef.current = false;
+            setMicEnabled(false);
+        }
+        if (prefs.filters) {
+            applyFilters(prefs.filters);
+        }
+        if (prefs.backgroundImage) {
+            loadPicture(prefs.backgroundImage).then(
+                (picture) => {
+                    if (mountedRef.current) {
+                        applyBackground(picture);
+                    }
+                },
+                () => undefined,
+            );
+        }
+        return prefs;
+    }, [applyBackground, applyFilters]);
+
+    /** Saves a change to the remembered settings. */
+    const remember = useCallback(
+        (patch: Partial<MediaPreferences>) => {
+            prefsRef.current = { ...ensurePrefs(), ...patch };
+            savePreferences(patch);
+        },
+        [ensurePrefs],
+    );
 
     const refreshDevices = useCallback(async () => {
         const list = await listDevices();
@@ -138,7 +229,10 @@ export function useLocalMedia(): LocalMedia {
         tracksRef.current = { ...tracksRef.current, [kind]: track };
         setTracks(tracksRef.current);
         setStatus((prev) => ({ ...prev, [kind]: "live" }));
-    }, []);
+        if (kind === "video") {
+            remember({ cameraEnabled: true });
+        }
+    }, [remember]);
 
     const adoptStream = useCallback(
         (stream: MediaStream) => {
@@ -149,11 +243,14 @@ export function useLocalMedia(): LocalMedia {
         [adoptTrack],
     );
 
-    /** Asks for one kind on its own - a specific device when `deviceId` is given. */
+    /** Asks for one kind on its own - a specific device when `deviceId` is given, else the one used last time. */
     const acquire = useCallback(
         async (kind: MediaKind, deviceId?: string): Promise<boolean> => {
             const base = kind === "audio" ? AUDIO_CONSTRAINTS : VIDEO_CONSTRAINTS;
-            const constraint: MediaTrackConstraints = deviceId ? { ...base, deviceId: { exact: deviceId } } : base;
+            const prefs = ensurePrefs();
+            const constraint: MediaTrackConstraints = deviceId
+                ? { ...base, deviceId: { exact: deviceId } }
+                : preferring(base, kind === "audio" ? prefs.microphoneId : prefs.cameraId);
             const result = await requestUserMedia({ [kind]: constraint });
             if (!result.ok) {
                 if (mountedRef.current) {
@@ -168,12 +265,24 @@ export function useLocalMedia(): LocalMedia {
             }
             return true;
         },
-        [adoptStream],
+        [adoptStream, ensurePrefs],
     );
 
     /** Asks for the camera and the microphone in a single prompt. */
     const acquireBoth = useCallback(async () => {
-        const both = await requestUserMedia({ audio: AUDIO_CONSTRAINTS, video: VIDEO_CONSTRAINTS });
+        const prefs = ensurePrefs();
+        if (prefs.cameraEnabled === false) {
+            // The camera was off when the participant last left: join with it off, without switching it on first.
+            if (mountedRef.current) {
+                setStatus((prev) => ({ ...prev, video: "off" }));
+            }
+            await acquire("audio");
+            return;
+        }
+        const both = await requestUserMedia({
+            audio: preferring(AUDIO_CONSTRAINTS, prefs.microphoneId),
+            video: preferring(VIDEO_CONSTRAINTS, prefs.cameraId),
+        });
         if (both.ok) {
             adoptStream(both.value);
         } else if (both.error.kind === "permission-denied") {
@@ -187,7 +296,7 @@ export function useLocalMedia(): LocalMedia {
             await acquire("audio");
             await acquire("video");
         }
-    }, [acquire, adoptStream]);
+    }, [acquire, adoptStream, ensurePrefs]);
 
     const requestAccess = useCallback(
         async (kind?: MediaKind) => {
@@ -211,6 +320,7 @@ export function useLocalMedia(): LocalMedia {
         if (!track) {
             micEnabledRef.current = true;
             setMicEnabled(true);
+            remember({ micEnabled: true });
             await acquire("audio");
             await refreshDevices();
             return;
@@ -219,7 +329,8 @@ export function useLocalMedia(): LocalMedia {
         micEnabledRef.current = next;
         track.enabled = next;
         setMicEnabled(next);
-    }, [acquire, refreshDevices]);
+        remember({ micEnabled: next });
+    }, [acquire, refreshDevices, remember]);
 
     const toggleCamera = useCallback(async () => {
         const track = tracksRef.current.video;
@@ -233,7 +344,8 @@ export function useLocalMedia(): LocalMedia {
         tracksRef.current = { ...tracksRef.current, video: null };
         setTracks(tracksRef.current);
         setStatus((prev) => ({ ...prev, video: "off" }));
-    }, [acquire, refreshDevices]);
+        remember({ cameraEnabled: false });
+    }, [acquire, refreshDevices, remember]);
 
     const selectDevice = useCallback(
         async (kind: MediaKind, deviceId: string) => {
@@ -247,10 +359,37 @@ export function useLocalMedia(): LocalMedia {
                     setTracks(tracksRef.current);
                 }
             }
-            await acquire(kind, deviceId);
+            if (await acquire(kind, deviceId)) {
+                remember(kind === "audio" ? { microphoneId: deviceId } : { cameraId: deviceId });
+            }
             await refreshDevices();
         },
-        [acquire, refreshDevices],
+        [acquire, refreshDevices, remember],
+    );
+
+    const setFilters = useCallback(
+        (patch: Partial<VideoFilters>) => {
+            const next = { ...filtersRef.current, ...patch };
+            applyFilters(next);
+            remember({ filters: next });
+        },
+        [applyFilters, remember],
+    );
+
+    const chooseBackgroundImage = useCallback(
+        async (file: File): Promise<string | null> => {
+            const prepared = await prepareBackgroundImage(file);
+            if (!prepared.ok) {
+                return prepared.message;
+            }
+            if (mountedRef.current) {
+                applyBackground(prepared.picture);
+                remember({ backgroundImage: prepared.dataUrl });
+                setFilters({ background: "image" });
+            }
+            return null;
+        },
+        [applyBackground, remember, setFilters],
     );
 
     const release = useCallback(() => {
@@ -304,7 +443,47 @@ export function useLocalMedia(): LocalMedia {
         };
     }, [tracks.audio, micEnabled]);
 
-    const videoStream = useMemo(() => (tracks.video ? new MediaStream([tracks.video]) : null), [tracks.video]);
+    // Puts the camera through the filters for as long as any is on - see this module's doc comment. A layout effect,
+    // so the filtered track is in place before the first frame is painted rather than after an unfiltered one.
+    const filtersOn = filtersActive(filters);
+    const source = tracks.video;
+    useLayoutEffect(() => {
+        if (!source || !filtersOn) {
+            return;
+        }
+        let processor: VideoFilterProcessor;
+        try {
+            processor = new VideoFilterProcessor({
+                source,
+                filters: filtersRef.current,
+                backgroundImage: backgroundRef.current,
+                assetsUrl: effectsAssetsUrl,
+                onStatus: setFilterStatus,
+            });
+        } catch {
+            setFilterStatus({ loading: false, error: FILTERS_UNSUPPORTED });
+            // Turning the filters off (or the camera) clears the message again.
+            return () => setFilterStatus({ loading: false, error: null });
+        }
+        processorRef.current = processor;
+        setProcessed(processor.track);
+        return () => {
+            processor.stop();
+            processorRef.current = null;
+            setProcessed(null);
+            setFilterStatus({ loading: false, error: null });
+        };
+    }, [source, filtersOn, effectsAssetsUrl]);
+
+    // Passes a change of filters (or of the background picture) on to the running processor.
+    useEffect(() => {
+        processorRef.current?.update(filters, background);
+    }, [filters, background]);
+
+    // While filters are on, the camera's own track is never what's sent: the filtered one is, or nothing.
+    const videoTrack = filtersOn ? (source ? processed : null) : source;
+
+    const videoStream = useMemo(() => (videoTrack ? new MediaStream([videoTrack]) : null), [videoTrack]);
 
     const selectedDeviceIds = useMemo(
         () => ({
@@ -319,7 +498,7 @@ export function useLocalMedia(): LocalMedia {
         requesting,
         error,
         audioTrack: tracks.audio,
-        videoTrack: tracks.video,
+        videoTrack,
         videoStream,
         micEnabled,
         micOn: !!tracks.audio && micEnabled,
@@ -328,6 +507,11 @@ export function useLocalMedia(): LocalMedia {
         devices,
         selectedDeviceIds,
         audioLevel,
+        filters,
+        filterStatus,
+        hasBackgroundImage: !!background,
+        setFilters,
+        chooseBackgroundImage,
         requestAccess,
         toggleMic,
         toggleCamera,

package/apps/shared/relay/AudioPlayer.ts

@@ -0,0 +1,111 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+import { TimestampUnwrapper, type Frame } from "./frames.js";
+import type { AudioContextLike, AudioDataLike, DecoderLike, RelayEnv } from "./relayEnv.js";
+import { safeClose } from "./senderCommon.js";
+
+export const PLAYBACK_SAMPLE_RATE = 48_000;
+/** How far ahead of the clock the first packet after a start or an underrun is scheduled. It is the jitter buffer:
+ * packets arriving up to this late (relative to their neighbours) still play in order without a gap. */
+export const JITTER_BUFFER_SECONDS = 0.08;
+/** If scheduled audio already runs this far past the clock the receiver has fallen behind (a burst, or the sender's
+ * clock running fast), so further packets are dropped until playback catches up - better a brief skip than latency
+ * that keeps growing. */
+export const MAX_AHEAD_SECONDS = 0.4;
+/** Skip a packet instead of queueing it when the decoder is this far behind. */
+export const AUDIO_MAX_DECODE_QUEUE = 16;
+
+/**
+ * Decodes one sender's Opus packets and schedules them for playback into a `MediaStreamAudioDestinationNode`.
+ *
+ * Each decoded block is copied into an `AudioBuffer` and started on its own `AudioBufferSourceNode` at
+ * `max(currentTime + jitter, nextTime)`, where `nextTime` is when the previous block ends - so consecutive blocks
+ * join seamlessly, and after a gap in the stream playback restarts with the jitter buffer's worth of headroom.
+ * Opus packets are independently decodable, so a lost packet costs only its own 20 ms and needs no recovery step;
+ * the decoder is created lazily and discarded on any error, and replaced by the next packet.
+ */
+export class AudioPlayer {
+    private decoder: DecoderLike | undefined;
+    private nextTime = 0;
+    private readonly timestamps = new TimestampUnwrapper();
+
+    /** `destination` is the node whose stream the peer's `MediaStream` carries. */
+    constructor(
+        private readonly env: RelayEnv,
+        private readonly ctx: AudioContextLike,
+        private readonly destination: unknown,
+    ) {}
+
+    /** Feeds one reassembled audio frame (one Opus packet). */
+    push(frame: Frame): void {
+        const decoder = this.decoder ?? this.createDecoder();
+        if (!decoder || decoder.decodeQueueSize > AUDIO_MAX_DECODE_QUEUE) {
+            return;
+        }
+        try {
+            decoder.decode(
+                this.env.createEncodedAudioChunk({ type: "key", timestamp: this.timestamps.next(frame.timestampMs) * 1000, data: frame.data }),
+            );
+        } catch {
+            this.discardDecoder();
+        }
+    }
+
+    close(): void {
+        this.discardDecoder();
+    }
+
+    private discardDecoder(): void {
+        const decoder = this.decoder;
+        this.decoder = undefined;
+        safeClose(decoder);
+    }
+
+    private createDecoder(): DecoderLike | undefined {
+        let decoder: DecoderLike | undefined;
+        try {
+            decoder = this.env.createAudioDecoder({
+                output: (data) => this.play(data),
+                error: () => {
+                    // Only the current decoder's failure matters; a stale one was already replaced.
+                    if (this.decoder === decoder) {
+                        this.discardDecoder();
+                    }
+                },
+            });
+            decoder.configure({ codec: "opus", sampleRate: PLAYBACK_SAMPLE_RATE, numberOfChannels: 1 });
+        } catch {
+            safeClose(decoder);
+            return undefined;
+        }
+        this.decoder = decoder;
+        return decoder;
+    }
+
+    /** Schedules one decoded block. Always closes `data`. */
+    private play(data: AudioDataLike): void {
+        try {
+            const now = this.ctx.currentTime;
+            if (this.nextTime - now > MAX_AHEAD_SECONDS) {
+                return;
+            }
+            const buffer = this.ctx.createBuffer(1, data.numberOfFrames, data.sampleRate);
+            const pcm = new Float32Array(data.numberOfFrames);
+            data.copyTo(pcm, { planeIndex: 0, format: "f32-planar" });
+            buffer.copyToChannel(pcm, 0);
+            const source = this.ctx.createBufferSource();
+            source.buffer = buffer;
+            source.connect(this.destination);
+            source.onended = () => source.disconnect();
+            const start = Math.max(now + JITTER_BUFFER_SECONDS, this.nextTime);
+            source.start(start);
+            this.nextTime = start + buffer.duration;
+        } catch {
+            // A block the browser will not take (an odd sample rate, a closed context) is dropped like a lost packet.
+        } finally {
+            data.close();
+        }
+    }
+}

package/apps/shared/relay/AudioSender.ts

@@ -0,0 +1,194 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+import { resumeAudioContext } from "./audioResume.js";
+import { KIND_AUDIO } from "./frames.js";
+import type {
+    AudioContextLike,
+    AudioDataLike,
+    AudioNodeLike,
+    EncodedChunkLike,
+    EncoderLike,
+    GainLike,
+    ScriptProcessorLike,
+} from "./relayEnv.js";
+import { safeClose, type SenderDeps } from "./senderCommon.js";
+
+export const AUDIO_SAMPLE_RATE = 48_000;
+/** Frames per `ScriptProcessorNode` callback: about 43 ms, a compromise between latency and how often the main
+ * thread is interrupted. */
+export const AUDIO_PROCESS_FRAMES = 2048;
+export const AUDIO_BITRATE = 24_000;
+/** The Opus frame duration in microseconds (20 ms). */
+export const AUDIO_FRAME_DURATION_US = 20_000;
+/** Skip a block instead of queueing it when the encoder is this far behind. */
+export const AUDIO_MAX_ENCODE_QUEUE = 8;
+
+/**
+ * Captures one microphone track and publishes it as Opus.
+ *
+ * There is exactly one capture path, chosen for being testable and available everywhere the relay is: an
+ * `AudioContext` at 48 kHz, a `MediaStreamAudioSourceNode` on the track, and a `ScriptProcessorNode` that receives
+ * the raw PCM. (`ScriptProcessorNode` is deprecated in favour of `AudioWorklet`, but it needs no separate module
+ * file to load, which a plugin bundle has no good place for, and every browser still supports it.) The processor's
+ * output is routed through a zero-gain node to the destination, because browsers only run a processor that is
+ * connected to the destination, and the gain stops the microphone playing back locally.
+ *
+ * It encodes only while sending is active, a track is set and the socket can take the frames, and rebuilds the graph
+ * when the track is replaced. The encoder is created lazily and discarded on any error, so a coder that died is
+ * replaced at the next block rather than being repaired inside its own error callback.
+ */
+export class AudioSender {
+    private active = false;
+    private track: MediaStreamTrack | null = null;
+    private attachedTrack: MediaStreamTrack | null = null;
+
+    private ctx: AudioContextLike | undefined;
+    private source: AudioNodeLike | undefined;
+    private processor: ScriptProcessorLike | undefined;
+    private gain: GainLike | undefined;
+    private stopResume: (() => void) | undefined;
+    private encoder: EncoderLike<AudioDataLike> | undefined;
+    private baseMs = 0;
+    private samples = 0;
+
+    constructor(private readonly deps: SenderDeps) {}
+
+    setActive(active: boolean): void {
+        this.active = active;
+        this.sync();
+    }
+
+    setTrack(track: MediaStreamTrack | null): void {
+        this.track = track;
+        this.sync();
+    }
+
+    /** Brings the capture graph in line with (`active`, `track`): torn down when either is missing, rebuilt when
+     * the track changed, untouched otherwise. */
+    private sync(): void {
+        const wanted = this.active ? this.track : null;
+        if (wanted === this.attachedTrack) {
+            return;
+        }
+        this.detach();
+        if (wanted) {
+            this.attach(wanted);
+        }
+    }
+
+    private attach(track: MediaStreamTrack): void {
+        const env = this.deps.env;
+        this.attachedTrack = track;
+        this.baseMs = env.now();
+        this.samples = 0;
+        try {
+            const ctx = env.createAudioContext(AUDIO_SAMPLE_RATE);
+            this.ctx = ctx;
+            const source = ctx.createMediaStreamSource(env.createMediaStream([track]));
+            this.source = source;
+            const processor = ctx.createScriptProcessor(AUDIO_PROCESS_FRAMES, 1, 1);
+            this.processor = processor;
+            const gain = ctx.createGain();
+            gain.gain.value = 0;
+            this.gain = gain;
+            processor.onaudioprocess = (event) => this.process(event.inputBuffer.getChannelData(0));
+            source.connect(processor);
+            processor.connect(gain);
+            gain.connect(ctx.destination);
+            this.stopResume = resumeAudioContext(ctx, env.document);
+        } catch {
+            // No usable AudioContext (or a browser that refuses one): audio simply isn't relayed.
+            this.detach();
+        }
+    }
+
+    private detach(): void {
+        this.attachedTrack = null;
+        this.stopResume?.();
+        this.stopResume = undefined;
+        if (this.processor) {
+            this.processor.onaudioprocess = null;
+        }
+        this.source?.disconnect();
+        this.processor?.disconnect();
+        this.gain?.disconnect();
+        this.source = this.processor = this.gain = undefined;
+        const ctx = this.ctx;
+        this.ctx = undefined;
+        // `close()` rejects for a context that is already closed; nothing to do about it.
+        ctx?.close().catch(() => undefined);
+        this.discardEncoder();
+    }
+
+    private discardEncoder(): void {
+        const encoder = this.encoder;
+        this.encoder = undefined;
+        safeClose(encoder);
+    }
+
+    private process(input: Float32Array): void {
+        const samples = input.length;
+        const timestampUs = Math.round((this.samples * 1_000_000) / AUDIO_SAMPLE_RATE);
+        this.samples += samples;
+        if (!this.deps.canSend()) {
+            return;
+        }
+        const encoder = this.encoder ?? this.createEncoder();
+        if (!encoder || encoder.encodeQueueSize > AUDIO_MAX_ENCODE_QUEUE) {
+            return;
+        }
+        let audio: AudioDataLike | undefined;
+        try {
+            // The browser reuses `input` for the next block, so the encoder gets its own copy.
+            audio = this.deps.env.createAudioData({
+                format: "f32-planar",
+                sampleRate: AUDIO_SAMPLE_RATE,
+                numberOfFrames: samples,
+                numberOfChannels: 1,
+                timestamp: timestampUs,
+                data: new Float32Array(input),
+            });
+            encoder.encode(audio);
+        } catch {
+            this.discardEncoder();
+        } finally {
+            audio?.close();
+        }
+    }
+
+    private createEncoder(): EncoderLike<AudioDataLike> | undefined {
+        let encoder: EncoderLike<AudioDataLike> | undefined;
+        try {
+            encoder = this.deps.env.createAudioEncoder({
+                output: (chunk) => this.onChunk(chunk),
+                error: () => {
+                    // Only the current encoder's failure matters; a stale one was already replaced.
+                    if (this.encoder === encoder) {
+                        this.discardEncoder();
+                    }
+                },
+            });
+            encoder.configure({
+                codec: "opus",
+                sampleRate: AUDIO_SAMPLE_RATE,
+                numberOfChannels: 1,
+                bitrate: AUDIO_BITRATE,
+                opus: { frameDuration: AUDIO_FRAME_DURATION_US },
+            });
+        } catch {
+            safeClose(encoder);
+            return undefined;
+        }
+        this.encoder = encoder;
+        return encoder;
+    }
+
+    private onChunk(chunk: EncodedChunkLike): void {
+        const bytes = new Uint8Array(chunk.byteLength);
+        chunk.copyTo(bytes);
+        // Every Opus packet decodes on its own, so each is flagged as a key frame.
+        this.deps.publish(KIND_AUDIO, true, bytes, this.baseMs + chunk.timestamp / 1000);
+    }
+}