@rapidmx/meet-plugin 0.5.0 → 0.8.0

package/dist/apps/shared/relay/VideoPlayer.d.ts

@@ -0,0 +1,31 @@
+import { type Frame } from "./frames.js";
+import type { CanvasContextLike, CanvasLike, RelayEnv } from "./relayEnv.js";
+/** Skip a frame (and wait for the next key frame) when the decoder is this far behind. */
+export declare const VIDEO_MAX_DECODE_QUEUE = 8;
+/**
+ * Decodes one sender's VP8 frames and draws them onto a canvas, whose `captureStream()` track is what the rest of
+ * the app plays as that participant's video.
+ *
+ * VP8 is a chain of deltas hanging off key frames, so a decoder fed a delta frame without its predecessor produces
+ * garbage or errors. This therefore decodes only from a key frame onward and falls back to waiting for the next one
+ * whenever the chain may be broken: a gap in `seq` (a fragment or a whole frame was dropped on the way), the decoder
+ * being too far behind, or any decode error. The sender forces a key frame every couple of seconds, which bounds the
+ * wait. The decoder is created lazily and discarded on error.
+ */
+export declare class VideoPlayer {
+    private readonly env;
+    private readonly canvas;
+    private readonly context;
+    private decoder;
+    private needKey;
+    private expectedSeq;
+    private readonly timestamps;
+    constructor(env: RelayEnv, canvas: CanvasLike, context: CanvasContextLike);
+    /** Feeds one reassembled video frame. */
+    push(frame: Frame): void;
+    close(): void;
+    private discardDecoder;
+    private createDecoder;
+    /** Draws one decoded picture, resizing the canvas to it. Always closes `picture`. */
+    private draw;
+}

package/dist/apps/shared/relay/VideoPlayer.js

@@ -0,0 +1,102 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+import { TimestampUnwrapper } from "./frames.js";
+import { safeClose } from "./senderCommon.js";
+/** Skip a frame (and wait for the next key frame) when the decoder is this far behind. */
+export const VIDEO_MAX_DECODE_QUEUE = 8;
+const SEQ_MODULUS = 0x10000;
+/**
+ * Decodes one sender's VP8 frames and draws them onto a canvas, whose `captureStream()` track is what the rest of
+ * the app plays as that participant's video.
+ *
+ * VP8 is a chain of deltas hanging off key frames, so a decoder fed a delta frame without its predecessor produces
+ * garbage or errors. This therefore decodes only from a key frame onward and falls back to waiting for the next one
+ * whenever the chain may be broken: a gap in `seq` (a fragment or a whole frame was dropped on the way), the decoder
+ * being too far behind, or any decode error. The sender forces a key frame every couple of seconds, which bounds the
+ * wait. The decoder is created lazily and discarded on error.
+ */
+export class VideoPlayer {
+    constructor(env, canvas, context) {
+        this.env = env;
+        this.canvas = canvas;
+        this.context = context;
+        this.needKey = true;
+        this.timestamps = new TimestampUnwrapper();
+    }
+    /** Feeds one reassembled video frame. */
+    push(frame) {
+        if (this.expectedSeq !== undefined && frame.seq !== this.expectedSeq) {
+            this.needKey = true;
+        }
+        this.expectedSeq = (frame.seq + 1) % SEQ_MODULUS;
+        if (frame.keyFrame) {
+            this.needKey = false;
+        }
+        else if (this.needKey) {
+            return;
+        }
+        const decoder = this.decoder ?? this.createDecoder();
+        if (!decoder || decoder.decodeQueueSize > VIDEO_MAX_DECODE_QUEUE) {
+            this.needKey = true;
+            return;
+        }
+        try {
+            decoder.decode(this.env.createEncodedVideoChunk({
+                type: frame.keyFrame ? "key" : "delta",
+                timestamp: this.timestamps.next(frame.timestampMs) * 1000,
+                data: frame.data,
+            }));
+        }
+        catch {
+            this.discardDecoder();
+        }
+    }
+    close() {
+        this.discardDecoder();
+    }
+    discardDecoder() {
+        const decoder = this.decoder;
+        this.decoder = undefined;
+        this.needKey = true;
+        safeClose(decoder);
+    }
+    createDecoder() {
+        let decoder;
+        try {
+            decoder = this.env.createVideoDecoder({
+                output: (picture) => this.draw(picture),
+                error: () => {
+                    // Only the current decoder's failure matters; a stale one was already replaced.
+                    if (this.decoder === decoder) {
+                        this.discardDecoder();
+                    }
+                },
+            });
+            decoder.configure({ codec: "vp8" });
+        }
+        catch {
+            safeClose(decoder);
+            return undefined;
+        }
+        this.decoder = decoder;
+        return decoder;
+    }
+    /** Draws one decoded picture, resizing the canvas to it. Always closes `picture`. */
+    draw(picture) {
+        try {
+            if (this.canvas.width !== picture.displayWidth || this.canvas.height !== picture.displayHeight) {
+                this.canvas.width = picture.displayWidth;
+                this.canvas.height = picture.displayHeight;
+            }
+            this.context.drawImage(picture, 0, 0);
+        }
+        catch {
+            // A picture the canvas will not take is dropped; the next one draws over it anyway.
+        }
+        finally {
+            picture.close();
+        }
+    }
+}

package/dist/apps/shared/relay/VideoSender.d.ts

@@ -0,0 +1,56 @@
+import { type SenderDeps } from "./senderCommon.js";
+export declare const VIDEO_MAX_WIDTH = 480;
+export declare const VIDEO_MAX_HEIGHT = 360;
+export declare const VIDEO_FPS = 15;
+export declare const VIDEO_BITRATE = 350000;
+/** A key frame is forced this often, so a receiver that joins or loses a frame recovers within two seconds. */
+export declare const VIDEO_KEY_FRAME_INTERVAL = 30;
+/** Skip a tick instead of queueing a frame when the encoder is this far behind. */
+export declare const VIDEO_MAX_ENCODE_QUEUE = 3;
+/** The size a `width` x `height` picture is encoded at: shrunk to fit `VIDEO_MAX_WIDTH` x `VIDEO_MAX_HEIGHT` keeping
+ * its aspect ratio (never enlarged), and rounded to even numbers because VP8's chroma planes are half size. */
+export declare function fitSize(width: number, height: number): {
+    width: number;
+    height: number;
+};
+/**
+ * Captures one camera or screen-share track and publishes it as VP8.
+ *
+ * One capture path: a hidden `<video>` plays the track, a `setInterval` at about 15 fps draws its current picture onto
+ * a canvas (which also does the down-scaling, so the encoder always sees the size it was configured for), and the
+ * canvas becomes a `VideoFrame` for the `VideoEncoder`. A key frame is forced at the start, after any change of
+ * source or size and every `VIDEO_KEY_FRAME_INTERVAL` frames, and whenever a frame could not be sent (a receiver
+ * that missed a fragment cannot decode the deltas that follow it, so the fastest cure is a new key frame).
+ *
+ * Replacing the track (a screen share swaps the camera track for a display track) keeps the loop running: the
+ * `<video>` is pointed at the new track and a key frame is forced. The encoder is created lazily at the first real
+ * picture, because its size is only known once the video has one, and discarded on any error rather than repaired
+ * inside its own error callback.
+ */
+export declare class VideoSender {
+    private readonly deps;
+    private active;
+    private track;
+    private attachedTrack;
+    private video;
+    private timer;
+    private encoder;
+    private encodedWidth;
+    private encodedHeight;
+    private frameCount;
+    private forceKey;
+    private baseMs;
+    constructor(deps: SenderDeps);
+    setActive(active: boolean): void;
+    setTrack(track: MediaStreamTrack | null): void;
+    private sync;
+    private start;
+    private attachSource;
+    private stop;
+    private discardEncoder;
+    private tick;
+    /** (Re)creates the encoder for a `width` x `height` picture. VP8 cannot change size without a key frame, so this
+     * always forces one. Returns `undefined` when this browser cannot make the encoder. */
+    private configureEncoder;
+    private onChunk;
+}

package/dist/apps/shared/relay/VideoSender.js

@@ -0,0 +1,204 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+import { KIND_VIDEO } from "./frames.js";
+import { safeClose } from "./senderCommon.js";
+export const VIDEO_MAX_WIDTH = 480;
+export const VIDEO_MAX_HEIGHT = 360;
+export const VIDEO_FPS = 15;
+export const VIDEO_BITRATE = 350000;
+/** A key frame is forced this often, so a receiver that joins or loses a frame recovers within two seconds. */
+export const VIDEO_KEY_FRAME_INTERVAL = 30;
+/** Skip a tick instead of queueing a frame when the encoder is this far behind. */
+export const VIDEO_MAX_ENCODE_QUEUE = 3;
+/** `HTMLMediaElement.HAVE_CURRENT_DATA`: a frame is available to draw. */
+const HAVE_CURRENT_DATA = 2;
+/** The size a `width` x `height` picture is encoded at: shrunk to fit `VIDEO_MAX_WIDTH` x `VIDEO_MAX_HEIGHT` keeping
+ * its aspect ratio (never enlarged), and rounded to even numbers because VP8's chroma planes are half size. */
+export function fitSize(width, height) {
+    const scale = Math.min(1, VIDEO_MAX_WIDTH / width, VIDEO_MAX_HEIGHT / height);
+    return {
+        width: Math.max(2, 2 * Math.round((width * scale) / 2)),
+        height: Math.max(2, 2 * Math.round((height * scale) / 2)),
+    };
+}
+/**
+ * Captures one camera or screen-share track and publishes it as VP8.
+ *
+ * One capture path: a hidden `<video>` plays the track, a `setInterval` at about 15 fps draws its current picture onto
+ * a canvas (which also does the down-scaling, so the encoder always sees the size it was configured for), and the
+ * canvas becomes a `VideoFrame` for the `VideoEncoder`. A key frame is forced at the start, after any change of
+ * source or size and every `VIDEO_KEY_FRAME_INTERVAL` frames, and whenever a frame could not be sent (a receiver
+ * that missed a fragment cannot decode the deltas that follow it, so the fastest cure is a new key frame).
+ *
+ * Replacing the track (a screen share swaps the camera track for a display track) keeps the loop running: the
+ * `<video>` is pointed at the new track and a key frame is forced. The encoder is created lazily at the first real
+ * picture, because its size is only known once the video has one, and discarded on any error rather than repaired
+ * inside its own error callback.
+ */
+export class VideoSender {
+    constructor(deps) {
+        this.deps = deps;
+        this.active = false;
+        this.track = null;
+        this.attachedTrack = null;
+        this.encodedWidth = 0;
+        this.encodedHeight = 0;
+        this.frameCount = 0;
+        this.forceKey = true;
+        this.baseMs = 0;
+    }
+    setActive(active) {
+        this.active = active;
+        this.sync();
+    }
+    setTrack(track) {
+        this.track = track;
+        this.sync();
+    }
+    sync() {
+        const wanted = this.active ? this.track : null;
+        if (wanted === this.attachedTrack) {
+            return;
+        }
+        if (!wanted) {
+            this.stop();
+        }
+        else if (this.video) {
+            this.attachedTrack = wanted;
+            this.attachSource(this.video, wanted);
+            this.forceKey = true;
+        }
+        else {
+            this.start(wanted);
+        }
+    }
+    start(track) {
+        const env = this.deps.env;
+        this.attachedTrack = track;
+        this.baseMs = env.now();
+        this.frameCount = 0;
+        this.forceKey = true;
+        try {
+            const video = env.createVideoElement();
+            video.muted = true;
+            video.autoplay = true;
+            video.playsInline = true;
+            const canvas = env.createCanvas();
+            const context = canvas.getContext("2d");
+            if (!context) {
+                throw new Error("No 2D canvas context.");
+            }
+            this.video = video;
+            this.attachSource(video, track);
+            this.timer = env.setInterval(() => this.tick(video, canvas, context), Math.round(1000 / VIDEO_FPS));
+        }
+        catch {
+            // No usable video element or canvas: video simply isn't relayed.
+            this.stop();
+        }
+    }
+    attachSource(video, track) {
+        video.srcObject = this.deps.env.createMediaStream([track]);
+        // Autoplay policy can reject this; a muted, inline video is normally allowed, and a rejection just means no
+        // pictures (readyState never advances), which `tick()` already treats as "nothing to send".
+        video.play()?.catch(() => undefined);
+    }
+    stop() {
+        this.attachedTrack = null;
+        if (this.timer !== undefined) {
+            this.deps.env.clearInterval(this.timer);
+            this.timer = undefined;
+        }
+        if (this.video) {
+            this.video.pause();
+            this.video.srcObject = null;
+            this.video = undefined;
+        }
+        this.discardEncoder();
+    }
+    discardEncoder() {
+        const encoder = this.encoder;
+        this.encoder = undefined;
+        this.encodedWidth = this.encodedHeight = 0;
+        safeClose(encoder);
+    }
+    tick(video, canvas, context) {
+        if (!this.deps.canSend()) {
+            // Nothing is being sent, so whoever starts receiving next needs a key frame to begin with.
+            this.forceKey = true;
+            return;
+        }
+        if (video.readyState < HAVE_CURRENT_DATA || video.videoWidth === 0 || video.videoHeight === 0) {
+            return;
+        }
+        const { width, height } = fitSize(video.videoWidth, video.videoHeight);
+        const encoder = this.encoder && width === this.encodedWidth && height === this.encodedHeight ? this.encoder : this.configureEncoder(width, height);
+        if (!encoder || encoder.encodeQueueSize > VIDEO_MAX_ENCODE_QUEUE) {
+            return;
+        }
+        if (canvas.width !== width || canvas.height !== height) {
+            canvas.width = width;
+            canvas.height = height;
+        }
+        context.drawImage(video, 0, 0, width, height);
+        const keyFrame = this.forceKey || this.frameCount % VIDEO_KEY_FRAME_INTERVAL === 0;
+        let frame;
+        try {
+            frame = this.deps.env.createVideoFrame(canvas, Math.round((this.deps.env.now() - this.baseMs) * 1000));
+            encoder.encode(frame, { keyFrame });
+            this.forceKey = false;
+            this.frameCount += 1;
+        }
+        catch {
+            this.discardEncoder();
+            this.forceKey = true;
+        }
+        finally {
+            frame?.close();
+        }
+    }
+    /** (Re)creates the encoder for a `width` x `height` picture. VP8 cannot change size without a key frame, so this
+     * always forces one. Returns `undefined` when this browser cannot make the encoder. */
+    configureEncoder(width, height) {
+        this.discardEncoder();
+        let encoder;
+        try {
+            encoder = this.deps.env.createVideoEncoder({
+                output: (chunk) => this.onChunk(chunk),
+                error: () => {
+                    // Only the current encoder's failure matters; a stale one was already replaced.
+                    if (this.encoder === encoder) {
+                        this.discardEncoder();
+                        this.forceKey = true;
+                    }
+                },
+            });
+            encoder.configure({
+                codec: "vp8",
+                width,
+                height,
+                bitrate: VIDEO_BITRATE,
+                framerate: VIDEO_FPS,
+                latencyMode: "realtime",
+            });
+        }
+        catch {
+            safeClose(encoder);
+            return undefined;
+        }
+        this.encoder = encoder;
+        this.encodedWidth = width;
+        this.encodedHeight = height;
+        this.forceKey = true;
+        return encoder;
+    }
+    onChunk(chunk) {
+        const bytes = new Uint8Array(chunk.byteLength);
+        chunk.copyTo(bytes);
+        if (!this.deps.publish(KIND_VIDEO, chunk.type === "key", bytes, this.baseMs + chunk.timestamp / 1000)) {
+            this.forceKey = true;
+        }
+    }
+}

package/dist/apps/shared/relay/audioResume.d.ts

@@ -0,0 +1,12 @@
+import type { AudioContextLike, EventTargetLike } from "./relayEnv.js";
+/**
+ * Gets a suspended `AudioContext` running. A context created without a user gesture (the relay creates its
+ * playback context when the mesh falls back, which nothing the user just did caused) starts suspended under the
+ * browsers' autoplay policies, and stays silent until `resume()` is called from a gesture. So this tries
+ * `resume()` immediately - which succeeds when the page already has user activation, the usual case in a call the
+ * user clicked "Join" for - and otherwise retries on the next `pointerdown`/`click`/`keydown`, removing the
+ * listeners as soon as the context is running (or after that one retry, so an unhelpful browser doesn't keep a
+ * listener on `document` forever). Returns a function that removes the listeners early. A no-op when the context
+ * is not suspended.
+ */
+export declare function resumeAudioContext(ctx: AudioContextLike, target: EventTargetLike): () => void;

package/dist/apps/shared/relay/audioResume.js

@@ -0,0 +1,45 @@
+/** The user gestures a browser accepts as permission to start audio. */
+const GESTURE_EVENTS = ["pointerdown", "click", "keydown"];
+/**
+ * Gets a suspended `AudioContext` running. A context created without a user gesture (the relay creates its
+ * playback context when the mesh falls back, which nothing the user just did caused) starts suspended under the
+ * browsers' autoplay policies, and stays silent until `resume()` is called from a gesture. So this tries
+ * `resume()` immediately - which succeeds when the page already has user activation, the usual case in a call the
+ * user clicked "Join" for - and otherwise retries on the next `pointerdown`/`click`/`keydown`, removing the
+ * listeners as soon as the context is running (or after that one retry, so an unhelpful browser doesn't keep a
+ * listener on `document` forever). Returns a function that removes the listeners early. A no-op when the context
+ * is not suspended.
+ */
+export function resumeAudioContext(ctx, target) {
+    if (ctx.state !== "suspended") {
+        return () => undefined;
+    }
+    let disposed = false;
+    const dispose = () => {
+        if (disposed) {
+            return;
+        }
+        disposed = true;
+        for (const type of GESTURE_EVENTS) {
+            target.removeEventListener(type, onGesture, true);
+        }
+    };
+    const tryResume = () => {
+        // `resume()` returns a promise that stays pending until the browser allows it (and can reject); neither is
+        // something the caller can act on, and the next gesture is the retry.
+        ctx.resume().then(() => {
+            if (ctx.state === "running") {
+                dispose();
+            }
+        }, () => undefined);
+    };
+    function onGesture() {
+        tryResume();
+        dispose();
+    }
+    for (const type of GESTURE_EVENTS) {
+        target.addEventListener(type, onGesture, true);
+    }
+    tryResume();
+    return dispose;
+}

package/dist/apps/shared/relay/frames.d.ts

@@ -0,0 +1,101 @@
+/**
+ * The media frame format carried inside a relay WebSocket message - see `RelayClient.ts` for the envelope the
+ * server wraps around it. The server treats the payload as opaque bytes, so everything in this module is a contract
+ * between two browsers running this same code, and it has to survive an unreliable transport: a WebSocket to the
+ * relay is ordered and lossless per connection, but the server drops binary messages under backpressure, so a
+ * receiver must cope with missing fragments without throwing or growing its memory without bound.
+ *
+ * ## Layout (all big-endian)
+ *
+ * A frame (one encoded audio packet or one encoded video picture) is split into fragments sized to the largest
+ * WebSocket message the server accepts (`fragmentPayloadSize()`): an older server allows only 16384 bytes (a larger
+ * message closes the socket) and a key frame easily runs to tens of kilobytes, while a current one announces a
+ * larger limit in its `ready` reply. Each fragment is
+ *
+ * `[kind u8][flags u8][seq u16][fragIndex u8][fragCount u8][timestampMs u32][fragment bytes]`
+ *
+ * `seq` is a per-kind frame counter that wraps at 65536, so a receiver can tell fragments of different frames
+ * apart and notice a gap. `timestampMs` is the sender's monotonic clock and also wraps (at about 49.7 days), which
+ * is why the receiver runs it through `TimestampUnwrapper` before handing it to a decoder.
+ */
+export declare const KIND_AUDIO = 1;
+export declare const KIND_VIDEO = 2;
+export type FrameKind = typeof KIND_AUDIO | typeof KIND_VIDEO;
+/** Bit 0 of the flags byte: this frame is independently decodable (always set for audio packets). */
+export declare const FLAG_KEY_FRAME = 1;
+export declare const HEADER_BYTES = 10;
+/** The largest client-to-server message an older server accepts, and what a client assumes until the server's
+ * `ready` says otherwise. */
+export declare const DEFAULT_MESSAGE_BYTES = 16384;
+/** The payload size used under the default limit. It is well below 16384 - 10 so an older server keeps behaving
+ * exactly as it always has, with headroom besides. */
+export declare const DEFAULT_FRAGMENT_BYTES = 12000;
+/** The ceiling on the payload of one fragment, however large a message the server accepts: bigger messages gain
+ * nothing for a 480x360 stream and make one lost message cost more. */
+export declare const MAX_FRAGMENT_BYTES = 60000;
+/** The most fragments a frame may have (`fragCount` is one byte). */
+export declare const MAX_FRAGMENTS = 255;
+/** The largest frame a receiver will buffer. A 480x360 VP8 key frame at the rates the sender uses is a small
+ * fraction of this; the cap exists so a hostile or buggy peer cannot make a receiver allocate without bound. */
+export declare const MAX_FRAME_BYTES: number;
+/** One decoded fragment header plus a view of its bytes. */
+export interface Fragment {
+    kind: FrameKind;
+    keyFrame: boolean;
+    seq: number;
+    fragIndex: number;
+    fragCount: number;
+    timestampMs: number;
+    data: Uint8Array;
+}
+/** A completely reassembled frame. */
+export interface Frame {
+    kind: FrameKind;
+    keyFrame: boolean;
+    seq: number;
+    timestampMs: number;
+    data: Uint8Array;
+}
+/** The payload bytes per fragment for a server that accepts messages of `maxMessageBytes`. At or below the
+ * default limit that is `DEFAULT_FRAGMENT_BYTES` (or less, for a smaller limit); above it, everything the limit
+ * leaves after the header, capped at `MAX_FRAGMENT_BYTES`. A message is therefore never larger than the limit. */
+export declare function fragmentPayloadSize(maxMessageBytes: number): number;
+/** Splits `data` into wire-ready fragments (header included), each at most `maxMessageBytes` long (default: what an
+ * older server accepts). An empty frame still produces one (header-only) fragment. Returns an empty array for a frame
+ * the receiver would refuse anyway - larger than `MAX_FRAME_BYTES`, or needing more than `MAX_FRAGMENTS` fragments at
+ * this message size - so it is not worth sending. `seq` and `timestampMs` are reduced modulo their field widths, so a
+ * caller can pass a free-running counter or clock. */
+export declare function fragmentFrame(kind: FrameKind, keyFrame: boolean, seq: number, timestampMs: number, data: Uint8Array, maxMessageBytes?: number): Uint8Array[];
+/** Parses one fragment, or returns `undefined` for anything that is not a well-formed one (too short, unknown
+ * kind, a zero fragment count, an index beyond the count). Never throws: the bytes come off the network. */
+export declare function decodeFragment(bytes: Uint8Array): Fragment | undefined;
+/** True when `a` is a later sequence number than `b`, treating the 16 bit counter as circular. */
+export declare function seqIsNewer(a: number, b: number): boolean;
+/**
+ * Collects the fragments of one sender's one media kind back into whole frames. It holds at most one frame in
+ * flight: the moment a fragment of a newer sequence arrives, an incomplete older frame is abandoned (its missing
+ * piece was dropped by the server, and waiting longer would only add latency). Everything is validated and nothing
+ * throws, so a garbage or hostile stream can at worst cost the frames it corrupts.
+ */
+export declare class FrameReassembler {
+    private pending;
+    private lastSeq;
+    private stale;
+    /** Feeds one fragment. Returns the frame if this fragment completed it. */
+    push(fragment: Fragment): Frame | undefined;
+    /** Drops any frame in flight, e.g. after the consumer reset its decoder. */
+    reset(): void;
+    private collect;
+    private finish;
+}
+/**
+ * Turns the wrapping 32 bit millisecond timestamps into a monotonic number a decoder can be given. The signed
+ * difference from the previous value is taken modulo 2^32, so it is correct across the wrap point and also for a
+ * frame that arrives slightly out of order (a small negative step).
+ */
+export declare class TimestampUnwrapper {
+    private last;
+    private unwrapped;
+    /** Returns the unwrapped time in milliseconds relative to the first timestamp seen. */
+    next(timestampMs: number): number;
+}