@rapidmx/meet-plugin 0.6.0 → 0.8.1

package/apps/shared/relay/RelayTransport.ts

@@ -0,0 +1,188 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The last-resort media tier of a call: when two participants' WebRTC connection cannot be made at all (typically a
+ * network that blocks UDP and TURN alike), their audio and video travel over a WebSocket to the server, which fans
+ * each participant's frames out to whoever asked for them. This module is the facade the mesh manager talks to; the
+ * pieces behind it are:
+ *
+ * - `RelayClient` - the socket: hello/ready/want, reconnection, dropping media it cannot send promptly.
+ * - `frames` - the fragment format inside each WebSocket message and the reassembly of it.
+ * - `RelaySender` (`AudioSender`, `VideoSender`) - capture and encode the local tracks (Opus and VP8 via WebCodecs).
+ * - `RelayReceiver` (`AudioPlayer`, `VideoPlayer`) - decode a remote participant back into a `MediaStream`.
+ *
+ * There is one transport (one socket) per meeting, however many peers use it, and the codecs are fixed - there is no
+ * negotiation, because both ends are this same code.
+ */
+import { apiOrigin } from "@rapidmx/web-client/lib/util/api.js";
+import { PLAYBACK_SAMPLE_RATE } from "./AudioPlayer.js";
+import { resumeAudioContext } from "./audioResume.js";
+import { RelayClient } from "./RelayClient.js";
+import { detectRelayEnv, type AudioContextLike, type RelayEnv } from "./relayEnv.js";
+import { RelayReceiver } from "./RelayReceiver.js";
+import { RelaySender } from "./RelaySender.js";
+
+export interface RelayTransport {
+    /** false when this browser lacks WebSocket, AudioEncoder/AudioDecoder/VideoEncoder/VideoDecoder, VideoFrame,
+     * AudioContext or canvas.captureStream - the mesh then never falls back to the relay. */
+    readonly supported: boolean;
+    /** Start receiving `peerId`'s media (opens/keeps the socket, adds to `want`). `onStream` gets a MediaStream that
+     * plays that peer (one audio + one video track). Called at most once per stream creation; calling receiveFrom
+     * again for the same peer is a no-op. */
+    receiveFrom(peerId: string, onStream: (stream: MediaStream) => void): void;
+    stopReceivingFrom(peerId: string): void;
+    /** Start/stop publishing the local tracks (encode + send). Only encodes while active; stopping releases the
+     * encoders and audio nodes. */
+    setSending(active: boolean): void;
+    /** Same semantics as `MeshConnectionManager.setLocalTrack`: null = send nothing for that kind. Safe to call
+     * before or after `setSending`. */
+    setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void;
+    /** Closes the socket, all coders, streams and timers. Idempotent. */
+    close(): void;
+}
+
+export interface RelayTransportOptions {
+    meetingUid: string;
+    /** This participant's peer id on the meeting's signaling channel - the id the server stamps on what it sends. */
+    peerId: string;
+    /** Overrides the socket URL (default: `relayUrl(meetingUid)`). */
+    url?: string;
+    /** Overrides every browser API the relay uses (default: `detectRelayEnv()`), so a test can drive it with fakes. */
+    env?: RelayEnv;
+}
+
+/** The relay WebSocket's URL for a meeting: this deployment's origin (the API origin when the UI is served from a
+ * different one, else the page's own), `http` becoming `ws` - the same derivation `pushUrl()` uses for `/push`.
+ * `undefined` when there is no origin at all (not running in a browser). */
+export function relayUrl(meetingUid: string): string | undefined {
+    const origin = apiOrigin() || (typeof window !== "undefined" ? window.location.origin : "");
+    return origin ? `${origin.replace(/^http/i, "ws")}/api/mail/video-meetings/relay/${encodeURIComponent(meetingUid)}` : undefined;
+}
+
+/** What `createRelayTransport()` returns on a browser that cannot run the relay: every call is a no-op. */
+const UNSUPPORTED_TRANSPORT: RelayTransport = {
+    supported: false,
+    receiveFrom: () => undefined,
+    stopReceivingFrom: () => undefined,
+    setSending: () => undefined,
+    setLocalTrack: () => undefined,
+    close: () => undefined,
+};
+
+class WebSocketRelayTransport implements RelayTransport {
+    readonly supported = true;
+    private readonly client: RelayClient;
+    private readonly sender: RelaySender;
+    private readonly receivers = new Map<string, RelayReceiver>();
+    private playback: AudioContextLike | undefined;
+    private stopResume: (() => void) | undefined;
+    private closed = false;
+
+    constructor(
+        private readonly env: RelayEnv,
+        url: string,
+        peerId: string,
+    ) {
+        this.client = new RelayClient({
+            url,
+            peerId,
+            createSocket: (target) => env.createSocket(target),
+            random: () => env.random(),
+            onMedia: (sender, payload) => this.receivers.get(sender)?.handleMedia(payload),
+        });
+        this.sender = new RelaySender({
+            env,
+            send: (message) => this.client.sendMedia(message),
+            canSend: () => this.client.ready,
+            maxMessageBytes: () => this.client.maxMessageBytes,
+        });
+    }
+
+    receiveFrom(peerId: string, onStream: (stream: MediaStream) => void): void {
+        if (this.closed || this.receivers.has(peerId)) {
+            return;
+        }
+        const receiver = this.createReceiver();
+        if (!receiver) {
+            return;
+        }
+        this.receivers.set(peerId, receiver);
+        this.client.start();
+        this.client.setWanted(this.receivers.keys());
+        onStream(receiver.stream);
+    }
+
+    stopReceivingFrom(peerId: string): void {
+        const receiver = this.receivers.get(peerId);
+        if (!receiver) {
+            return;
+        }
+        this.receivers.delete(peerId);
+        receiver.close();
+        this.client.setWanted(this.receivers.keys());
+    }
+
+    setSending(active: boolean): void {
+        if (this.closed) {
+            return;
+        }
+        if (active) {
+            this.client.start();
+        }
+        this.sender.setActive(active);
+    }
+
+    setLocalTrack(kind: "audio" | "video", track: MediaStreamTrack | null): void {
+        if (!this.closed) {
+            this.sender.setTrack(kind, track);
+        }
+    }
+
+    close(): void {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        this.sender.setActive(false);
+        for (const receiver of this.receivers.values()) {
+            receiver.close();
+        }
+        this.receivers.clear();
+        this.client.close();
+        this.stopResume?.();
+        // `close()` rejects for a context that is already closed; nothing to do about it.
+        this.playback?.close().catch(() => undefined);
+        this.playback = undefined;
+    }
+
+    /** A receiver on the shared playback context, or `undefined` if this browser will not build one (the peer then
+     * simply has no relayed media). */
+    private createReceiver(): RelayReceiver | undefined {
+        try {
+            if (!this.playback) {
+                this.playback = this.env.createAudioContext(PLAYBACK_SAMPLE_RATE);
+                // Nothing the user just did created this context, so it may start suspended - see `resumeAudioContext`.
+                this.stopResume = resumeAudioContext(this.playback, this.env.document);
+            }
+            return new RelayReceiver(this.env, this.playback);
+        } catch {
+            return undefined;
+        }
+    }
+}
+
+/**
+ * Creates the relay transport for one meeting. Returns an inert transport (`supported: false`) when the browser lacks
+ * something the relay needs, so the caller can simply check `supported` once and never fall back to it.
+ * No socket is opened until the first `receiveFrom()` or `setSending(true)`.
+ */
+export function createRelayTransport(options: RelayTransportOptions): RelayTransport {
+    const env = options.env ?? detectRelayEnv();
+    const url = options.url ?? relayUrl(options.meetingUid);
+    if (!env || !url) {
+        return UNSUPPORTED_TRANSPORT;
+    }
+    return new WebSocketRelayTransport(env, url, options.peerId);
+}

package/apps/shared/relay/VideoPlayer.ts

@@ -0,0 +1,110 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+import { TimestampUnwrapper, type Frame } from "./frames.js";
+import type { CanvasContextLike, CanvasLike, DecoderLike, RelayEnv, VideoFrameLike } from "./relayEnv.js";
+import { safeClose } from "./senderCommon.js";
+
+/** Skip a frame (and wait for the next key frame) when the decoder is this far behind. */
+export const VIDEO_MAX_DECODE_QUEUE = 8;
+const SEQ_MODULUS = 0x10000;
+
+/**
+ * Decodes one sender's VP8 frames and draws them onto a canvas, whose `captureStream()` track is what the rest of
+ * the app plays as that participant's video.
+ *
+ * VP8 is a chain of deltas hanging off key frames, so a decoder fed a delta frame without its predecessor produces
+ * garbage or errors. This therefore decodes only from a key frame onward and falls back to waiting for the next one
+ * whenever the chain may be broken: a gap in `seq` (a fragment or a whole frame was dropped on the way), the decoder
+ * being too far behind, or any decode error. The sender forces a key frame every couple of seconds, which bounds the
+ * wait. The decoder is created lazily and discarded on error.
+ */
+export class VideoPlayer {
+    private decoder: DecoderLike | undefined;
+    private needKey = true;
+    private expectedSeq: number | undefined;
+    private readonly timestamps = new TimestampUnwrapper();
+
+    constructor(
+        private readonly env: RelayEnv,
+        private readonly canvas: CanvasLike,
+        private readonly context: CanvasContextLike,
+    ) {}
+
+    /** Feeds one reassembled video frame. */
+    push(frame: Frame): void {
+        if (this.expectedSeq !== undefined && frame.seq !== this.expectedSeq) {
+            this.needKey = true;
+        }
+        this.expectedSeq = (frame.seq + 1) % SEQ_MODULUS;
+        if (frame.keyFrame) {
+            this.needKey = false;
+        } else if (this.needKey) {
+            return;
+        }
+        const decoder = this.decoder ?? this.createDecoder();
+        if (!decoder || decoder.decodeQueueSize > VIDEO_MAX_DECODE_QUEUE) {
+            this.needKey = true;
+            return;
+        }
+        try {
+            decoder.decode(
+                this.env.createEncodedVideoChunk({
+                    type: frame.keyFrame ? "key" : "delta",
+                    timestamp: this.timestamps.next(frame.timestampMs) * 1000,
+                    data: frame.data,
+                }),
+            );
+        } catch {
+            this.discardDecoder();
+        }
+    }
+
+    close(): void {
+        this.discardDecoder();
+    }
+
+    private discardDecoder(): void {
+        const decoder = this.decoder;
+        this.decoder = undefined;
+        this.needKey = true;
+        safeClose(decoder);
+    }
+
+    private createDecoder(): DecoderLike | undefined {
+        let decoder: DecoderLike | undefined;
+        try {
+            decoder = this.env.createVideoDecoder({
+                output: (picture) => this.draw(picture),
+                error: () => {
+                    // Only the current decoder's failure matters; a stale one was already replaced.
+                    if (this.decoder === decoder) {
+                        this.discardDecoder();
+                    }
+                },
+            });
+            decoder.configure({ codec: "vp8" });
+        } catch {
+            safeClose(decoder);
+            return undefined;
+        }
+        this.decoder = decoder;
+        return decoder;
+    }
+
+    /** Draws one decoded picture, resizing the canvas to it. Always closes `picture`. */
+    private draw(picture: VideoFrameLike): void {
+        try {
+            if (this.canvas.width !== picture.displayWidth || this.canvas.height !== picture.displayHeight) {
+                this.canvas.width = picture.displayWidth;
+                this.canvas.height = picture.displayHeight;
+            }
+            this.context.drawImage(picture, 0, 0);
+        } catch {
+            // A picture the canvas will not take is dropped; the next one draws over it anyway.
+        } finally {
+            picture.close();
+        }
+    }
+}

package/apps/shared/relay/VideoSender.ts

@@ -0,0 +1,216 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+import { KIND_VIDEO } from "./frames.js";
+import type { CanvasContextLike, CanvasLike, EncodedChunkLike, EncoderLike, VideoElementLike, VideoFrameLike } from "./relayEnv.js";
+import { safeClose, type SenderDeps } from "./senderCommon.js";
+
+export const VIDEO_MAX_WIDTH = 480;
+export const VIDEO_MAX_HEIGHT = 360;
+export const VIDEO_FPS = 15;
+export const VIDEO_BITRATE = 350_000;
+/** A key frame is forced this often, so a receiver that joins or loses a frame recovers within two seconds. */
+export const VIDEO_KEY_FRAME_INTERVAL = 30;
+/** Skip a tick instead of queueing a frame when the encoder is this far behind. */
+export const VIDEO_MAX_ENCODE_QUEUE = 3;
+/** `HTMLMediaElement.HAVE_CURRENT_DATA`: a frame is available to draw. */
+const HAVE_CURRENT_DATA = 2;
+
+/** The size a `width` x `height` picture is encoded at: shrunk to fit `VIDEO_MAX_WIDTH` x `VIDEO_MAX_HEIGHT` keeping
+ * its aspect ratio (never enlarged), and rounded to even numbers because VP8's chroma planes are half size. */
+export function fitSize(width: number, height: number): { width: number; height: number } {
+    const scale = Math.min(1, VIDEO_MAX_WIDTH / width, VIDEO_MAX_HEIGHT / height);
+    return {
+        width: Math.max(2, 2 * Math.round((width * scale) / 2)),
+        height: Math.max(2, 2 * Math.round((height * scale) / 2)),
+    };
+}
+
+/**
+ * Captures one camera or screen-share track and publishes it as VP8.
+ *
+ * One capture path: a hidden `<video>` plays the track, a `setInterval` at about 15 fps draws its current picture onto
+ * a canvas (which also does the down-scaling, so the encoder always sees the size it was configured for), and the
+ * canvas becomes a `VideoFrame` for the `VideoEncoder`. A key frame is forced at the start, after any change of
+ * source or size and every `VIDEO_KEY_FRAME_INTERVAL` frames, and whenever a frame could not be sent (a receiver
+ * that missed a fragment cannot decode the deltas that follow it, so the fastest cure is a new key frame).
+ *
+ * Replacing the track (a screen share swaps the camera track for a display track) keeps the loop running: the
+ * `<video>` is pointed at the new track and a key frame is forced. The encoder is created lazily at the first real
+ * picture, because its size is only known once the video has one, and discarded on any error rather than repaired
+ * inside its own error callback.
+ */
+export class VideoSender {
+    private active = false;
+    private track: MediaStreamTrack | null = null;
+    private attachedTrack: MediaStreamTrack | null = null;
+
+    private video: VideoElementLike | undefined;
+    private timer: unknown;
+    private encoder: EncoderLike<VideoFrameLike, { keyFrame: boolean }> | undefined;
+    private encodedWidth = 0;
+    private encodedHeight = 0;
+    private frameCount = 0;
+    private forceKey = true;
+    private baseMs = 0;
+
+    constructor(private readonly deps: SenderDeps) {}
+
+    setActive(active: boolean): void {
+        this.active = active;
+        this.sync();
+    }
+
+    setTrack(track: MediaStreamTrack | null): void {
+        this.track = track;
+        this.sync();
+    }
+
+    private sync(): void {
+        const wanted = this.active ? this.track : null;
+        if (wanted === this.attachedTrack) {
+            return;
+        }
+        if (!wanted) {
+            this.stop();
+        } else if (this.video) {
+            this.attachedTrack = wanted;
+            this.attachSource(this.video, wanted);
+            this.forceKey = true;
+        } else {
+            this.start(wanted);
+        }
+    }
+
+    private start(track: MediaStreamTrack): void {
+        const env = this.deps.env;
+        this.attachedTrack = track;
+        this.baseMs = env.now();
+        this.frameCount = 0;
+        this.forceKey = true;
+        try {
+            const video = env.createVideoElement();
+            video.muted = true;
+            video.autoplay = true;
+            video.playsInline = true;
+            const canvas = env.createCanvas();
+            const context = canvas.getContext("2d");
+            if (!context) {
+                throw new Error("No 2D canvas context.");
+            }
+            this.video = video;
+            this.attachSource(video, track);
+            this.timer = env.setInterval(() => this.tick(video, canvas, context), Math.round(1000 / VIDEO_FPS));
+        } catch {
+            // No usable video element or canvas: video simply isn't relayed.
+            this.stop();
+        }
+    }
+
+    private attachSource(video: VideoElementLike, track: MediaStreamTrack): void {
+        video.srcObject = this.deps.env.createMediaStream([track]);
+        // Autoplay policy can reject this; a muted, inline video is normally allowed, and a rejection just means no
+        // pictures (readyState never advances), which `tick()` already treats as "nothing to send".
+        video.play()?.catch(() => undefined);
+    }
+
+    private stop(): void {
+        this.attachedTrack = null;
+        if (this.timer !== undefined) {
+            this.deps.env.clearInterval(this.timer);
+            this.timer = undefined;
+        }
+        if (this.video) {
+            this.video.pause();
+            this.video.srcObject = null;
+            this.video = undefined;
+        }
+        this.discardEncoder();
+    }
+
+    private discardEncoder(): void {
+        const encoder = this.encoder;
+        this.encoder = undefined;
+        this.encodedWidth = this.encodedHeight = 0;
+        safeClose(encoder);
+    }
+
+    private tick(video: VideoElementLike, canvas: CanvasLike, context: CanvasContextLike): void {
+        if (!this.deps.canSend()) {
+            // Nothing is being sent, so whoever starts receiving next needs a key frame to begin with.
+            this.forceKey = true;
+            return;
+        }
+        if (video.readyState < HAVE_CURRENT_DATA || video.videoWidth === 0 || video.videoHeight === 0) {
+            return;
+        }
+        const { width, height } = fitSize(video.videoWidth, video.videoHeight);
+        const encoder =
+            this.encoder && width === this.encodedWidth && height === this.encodedHeight ? this.encoder : this.configureEncoder(width, height);
+        if (!encoder || encoder.encodeQueueSize > VIDEO_MAX_ENCODE_QUEUE) {
+            return;
+        }
+        if (canvas.width !== width || canvas.height !== height) {
+            canvas.width = width;
+            canvas.height = height;
+        }
+        context.drawImage(video, 0, 0, width, height);
+        const keyFrame = this.forceKey || this.frameCount % VIDEO_KEY_FRAME_INTERVAL === 0;
+        let frame: VideoFrameLike | undefined;
+        try {
+            frame = this.deps.env.createVideoFrame(canvas, Math.round((this.deps.env.now() - this.baseMs) * 1000));
+            encoder.encode(frame, { keyFrame });
+            this.forceKey = false;
+            this.frameCount += 1;
+        } catch {
+            this.discardEncoder();
+            this.forceKey = true;
+        } finally {
+            frame?.close();
+        }
+    }
+
+    /** (Re)creates the encoder for a `width` x `height` picture. VP8 cannot change size without a key frame, so this
+     * always forces one. Returns `undefined` when this browser cannot make the encoder. */
+    private configureEncoder(width: number, height: number): EncoderLike<VideoFrameLike, { keyFrame: boolean }> | undefined {
+        this.discardEncoder();
+        let encoder: EncoderLike<VideoFrameLike, { keyFrame: boolean }> | undefined;
+        try {
+            encoder = this.deps.env.createVideoEncoder({
+                output: (chunk) => this.onChunk(chunk),
+                error: () => {
+                    // Only the current encoder's failure matters; a stale one was already replaced.
+                    if (this.encoder === encoder) {
+                        this.discardEncoder();
+                        this.forceKey = true;
+                    }
+                },
+            });
+            encoder.configure({
+                codec: "vp8",
+                width,
+                height,
+                bitrate: VIDEO_BITRATE,
+                framerate: VIDEO_FPS,
+                latencyMode: "realtime",
+            });
+        } catch {
+            safeClose(encoder);
+            return undefined;
+        }
+        this.encoder = encoder;
+        this.encodedWidth = width;
+        this.encodedHeight = height;
+        this.forceKey = true;
+        return encoder;
+    }
+
+    private onChunk(chunk: EncodedChunkLike): void {
+        const bytes = new Uint8Array(chunk.byteLength);
+        chunk.copyTo(bytes);
+        if (!this.deps.publish(KIND_VIDEO, chunk.type === "key", bytes, this.baseMs + chunk.timestamp / 1000)) {
+            this.forceKey = true;
+        }
+    }
+}

package/apps/shared/relay/audioResume.ts

@@ -0,0 +1,55 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+import type { AudioContextLike, EventTargetLike } from "./relayEnv.js";
+
+/** The user gestures a browser accepts as permission to start audio. */
+const GESTURE_EVENTS = ["pointerdown", "click", "keydown"] as const;
+
+/**
+ * Gets a suspended `AudioContext` running. A context created without a user gesture (the relay creates its
+ * playback context when the mesh falls back, which nothing the user just did caused) starts suspended under the
+ * browsers' autoplay policies, and stays silent until `resume()` is called from a gesture. So this tries
+ * `resume()` immediately - which succeeds when the page already has user activation, the usual case in a call the
+ * user clicked "Join" for - and otherwise retries on the next `pointerdown`/`click`/`keydown`, removing the
+ * listeners as soon as the context is running (or after that one retry, so an unhelpful browser doesn't keep a
+ * listener on `document` forever). Returns a function that removes the listeners early. A no-op when the context
+ * is not suspended.
+ */
+export function resumeAudioContext(ctx: AudioContextLike, target: EventTargetLike): () => void {
+    if (ctx.state !== "suspended") {
+        return () => undefined;
+    }
+    let disposed = false;
+    const dispose = (): void => {
+        if (disposed) {
+            return;
+        }
+        disposed = true;
+        for (const type of GESTURE_EVENTS) {
+            target.removeEventListener(type, onGesture, true);
+        }
+    };
+    const tryResume = (): void => {
+        // `resume()` returns a promise that stays pending until the browser allows it (and can reject); neither is
+        // something the caller can act on, and the next gesture is the retry.
+        ctx.resume().then(
+            () => {
+                if (ctx.state === "running") {
+                    dispose();
+                }
+            },
+            () => undefined,
+        );
+    };
+    function onGesture(): void {
+        tryResume();
+        dispose();
+    }
+    for (const type of GESTURE_EVENTS) {
+        target.addEventListener(type, onGesture, true);
+    }
+    tryResume();
+    return dispose;
+}