@rapidmx/meet-plugin 0.5.0 → 0.8.0

package/apps/shared/media/filters/backgroundImage.ts

@@ -0,0 +1,69 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The picture behind a participant for the "custom image" background: read from a file they pick on their own
+ * device, shrunk to what a video call can show anyway, and kept as a small JPEG data URL so it can be remembered
+ * (`mediaPreferences.ts`) without filling the browser's storage. The file never leaves the browser.
+ */
+import type { BackgroundPicture } from "./VideoFilterProcessor.js";
+
+/** A bigger file than this is refused before the browser tries to decode it. */
+export const MAX_BACKGROUND_FILE_BYTES = 20 * 1024 * 1024;
+/** The longest edge kept - a call's video is 1280 wide, so more would only cost storage. */
+export const MAX_BACKGROUND_EDGE = 1280;
+const JPEG_QUALITY = 0.85;
+
+export type PreparedBackground = { ok: true; dataUrl: string; picture: BackgroundPicture } | { ok: false; message: string };
+
+/** Decodes an image (a URL or data URL) into something that can be drawn. Rejects if it isn't a picture. */
+export function loadPicture(src: string): Promise<BackgroundPicture> {
+    return new Promise((resolve, reject) => {
+        const image = new Image();
+        image.onload = () => resolve({ source: image, width: image.naturalWidth, height: image.naturalHeight });
+        image.onerror = () => reject(new Error("The image could not be read."));
+        image.src = src;
+    });
+}
+
+function readAsDataUrl(file: File): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(String(reader.result));
+        reader.onerror = () => reject(new Error("The file could not be read."));
+        reader.readAsDataURL(file);
+    });
+}
+
+/** Reads a picked file into a background: validated, downscaled and ready both to draw and to store. */
+export async function prepareBackgroundImage(file: File): Promise<PreparedBackground> {
+    if (!file.type.startsWith("image/")) {
+        return { ok: false, message: "Choose an image file (a JPEG, PNG or similar)." };
+    }
+    if (file.size > MAX_BACKGROUND_FILE_BYTES) {
+        return { ok: false, message: "That image is too large. Choose one under 20 MB." };
+    }
+    try {
+        const original = await loadPicture(await readAsDataUrl(file));
+        const scale = Math.min(1, MAX_BACKGROUND_EDGE / Math.max(original.width, original.height));
+        const canvas = document.createElement("canvas");
+        canvas.width = Math.max(1, Math.round(original.width * scale));
+        canvas.height = Math.max(1, Math.round(original.height * scale));
+        const ctx = canvas.getContext("2d");
+        if (!ctx) {
+            return { ok: false, message: "This browser can't prepare that image." };
+        }
+        // JPEG has no transparency: a transparent PNG would otherwise turn black.
+        ctx.fillStyle = "#ffffff";
+        ctx.fillRect(0, 0, canvas.width, canvas.height);
+        ctx.drawImage(original.source, 0, 0, canvas.width, canvas.height);
+        return {
+            ok: true,
+            dataUrl: canvas.toDataURL("image/jpeg", JPEG_QUALITY),
+            picture: { source: canvas, width: canvas.width, height: canvas.height },
+        };
+    } catch {
+        return { ok: false, message: "That file couldn't be opened as an image." };
+    }
+}

package/apps/shared/media/filters/filterTypes.ts

@@ -0,0 +1,89 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The video filters a participant can put on their camera. They come in three independent layers, so they combine
+ * (blurred background *and* black and white *and* sunglasses):
+ *
+ * - **Background**: what is behind the participant - untouched, blurred, or a picture they chose. Needs person
+ * segmentation (`mlModels.ts`).
+ * - **Effect**: a look applied to the whole picture - black and white, sepia, night vision, pixelated.
+ * - **Accessory**: something drawn on the participant's face - sunglasses, cat ears, a party hat. Needs face landmarks
+ * (`mlModels.ts`).
+ *
+ * Adding a filter is: add its id here (with a label), then implement it in `VideoFilterProcessor.ts` (an effect), or
+ * in `accessories.ts` (an accessory).
+ */
+
+export type BackgroundFilter = "none" | "blur" | "image";
+export type ColorEffect = "none" | "bw" | "sepia" | "night-vision" | "pixelate";
+export type Accessory = "none" | "sunglasses" | "cat-ears" | "party-hat" | "crown" | "mustache";
+
+export interface VideoFilters {
+    background: BackgroundFilter;
+    effect: ColorEffect;
+    accessory: Accessory;
+}
+
+export const NO_FILTERS: VideoFilters = { background: "none", effect: "none", accessory: "none" };
+
+export interface FilterOption<T extends string> {
+    id: T;
+    label: string;
+    /** A glyph shown beside the label, purely decorative. */
+    emoji: string;
+}
+
+export const BACKGROUND_OPTIONS: FilterOption<BackgroundFilter>[] = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "blur", label: "Blur", emoji: "🌫️" },
+    { id: "image", label: "Custom image", emoji: "🖼️" },
+];
+
+export const EFFECT_OPTIONS: FilterOption<ColorEffect>[] = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "bw", label: "Black & white", emoji: "🎞️" },
+    { id: "sepia", label: "Sepia", emoji: "📜" },
+    { id: "night-vision", label: "Night vision", emoji: "🥽" },
+    { id: "pixelate", label: "Pixelate", emoji: "👾" },
+];
+
+export const ACCESSORY_OPTIONS: FilterOption<Accessory>[] = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "sunglasses", label: "Sunglasses", emoji: "😎" },
+    { id: "cat-ears", label: "Cat", emoji: "🐱" },
+    { id: "party-hat", label: "Party hat", emoji: "🥳" },
+    { id: "crown", label: "Crown", emoji: "👑" },
+    { id: "mustache", label: "Mustache", emoji: "🥸" },
+];
+
+/** Whether any filter is on - when none is, the camera's own track is sent untouched and nothing is processed. */
+export function filtersActive(filters: VideoFilters): boolean {
+    return filters.background !== "none" || filters.effect !== "none" || filters.accessory !== "none";
+}
+
+/** Whether the filters need the person-segmentation model. */
+export function needsSegmentation(filters: VideoFilters): boolean {
+    return filters.background !== "none";
+}
+
+/** Whether the filters need the face-landmark model. */
+export function needsFace(filters: VideoFilters): boolean {
+    return filters.accessory !== "none";
+}
+
+function pick<T extends string>(options: FilterOption<T>[], value: unknown, fallback: T): T {
+    return options.some((option) => option.id === value) ? (value as T) : fallback;
+}
+
+/** Turns whatever was read back from storage into valid filters - anything unrecognised (a filter since removed, a
+ * hand-edited value, another version's shape) falls back to "none" rather than breaking the call. */
+export function sanitizeFilters(value: unknown): VideoFilters {
+    const raw = (typeof value === "object" && value !== null ? value : {}) as Record<string, unknown>;
+    return {
+        background: pick(BACKGROUND_OPTIONS, raw.background, "none"),
+        effect: pick(EFFECT_OPTIONS, raw.effect, "none"),
+        accessory: pick(ACCESSORY_OPTIONS, raw.accessory, "none"),
+    };
+}

package/apps/shared/media/filters/mlModels.ts

@@ -0,0 +1,174 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The two machine-learning models behind the filters, run in the browser with MediaPipe Tasks:
+ *
+ * - **Person segmentation** (`createPersonSegmenter`) - which pixels are the participant: what the background
+ * blur and the custom background need.
+ * - **Face landmarks** (`createFaceTracker`) - where the participant's face is: what the accessories need.
+ *
+ * Nothing here is loaded until a participant turns a filter on that needs it, and each model is loaded once and
+ * shared. The MediaPipe library itself is `import()`ed on demand for the same reason - a participant who never uses a
+ * filter never downloads it.
+ *
+ * **Where the assets come from**: by default the WebAssembly runtime is fetched from jsDelivr and the models from
+ * Google's public MediaPipe model bucket. That is a request to a third party, so an administrator can host them
+ * instead (the `mail:videoconf:effects:assets_url` setting, delivered by the join response): the runtime's `wasm/`
+ * directory from the `@mediapipe/tasks-vision` package, plus `selfie_segmenter.tflite` and `face_landmarker.task`,
+ * all under one base URL.
+ *
+ * Everything the rest of the pipeline needs from MediaPipe is behind the small `PersonSegmenter`/`FaceTracker`
+ * interfaces, so `VideoFilterProcessor` never touches the library and its tests use fakes.
+ */
+import type { FaceLandmark } from "./accessories.js";
+
+/** Must match the installed `@mediapipe/tasks-vision` (a test checks): the `.wasm` fetched at runtime has to be the
+ * version the JavaScript was built against. */
+export const MEDIAPIPE_VERSION = "1.0.1";
+
+const CDN_WASM = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MEDIAPIPE_VERSION}/wasm`;
+const CDN_SEGMENTER_MODEL =
+    "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite";
+const CDN_FACE_MODEL =
+    "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task";
+
+export interface ModelUrls {
+    wasm: string;
+    segmenter: string;
+    face: string;
+}
+
+/** Where to fetch the runtime and models from - the CDN when `assetsUrl` is empty, else under `assetsUrl`. */
+export function modelUrls(assetsUrl?: string): ModelUrls {
+    if (!assetsUrl) {
+        return { wasm: CDN_WASM, segmenter: CDN_SEGMENTER_MODEL, face: CDN_FACE_MODEL };
+    }
+    const base = assetsUrl.replace(/\/+$/, "");
+    return { wasm: `${base}/wasm`, segmenter: `${base}/selfie_segmenter.tflite`, face: `${base}/face_landmarker.task` };
+}
+
+/** How likely each pixel of a frame is to be the participant, 0-1. */
+export interface PersonMask {
+    width: number;
+    height: number;
+    confidence: Float32Array;
+}
+
+export interface PersonSegmenter {
+    /** Segments `frame`, or `null` when the model produced nothing for it. */
+    segment(frame: HTMLVideoElement, timestampMs: number): PersonMask | null;
+}
+
+export interface FaceTracker {
+    /** The landmarks of the first face in `frame`, or `null` when there is none. */
+    detect(frame: HTMLVideoElement, timestampMs: number): FaceLandmark[] | null;
+}
+
+/** Which of a segmentation model's masks is the person: the one labelled as such (the selfie model's only label is
+ * "selfie"), else the last (models list the background first). */
+export function personMaskIndex(labels: string[], maskCount: number): number {
+    const index = labels.findIndex((label) => /person|selfie|foreground/i.test(label));
+    return index >= 0 && index < maskCount ? index : maskCount - 1;
+}
+
+/** MediaPipe rejects a frame whose timestamp isn't later than the last one it saw. */
+function monotonic(): (timestampMs: number) => number {
+    let last = -1;
+    return (timestampMs) => {
+        last = Math.max(timestampMs, last + 1);
+        return last;
+    };
+}
+
+type Vision = typeof import("@mediapipe/tasks-vision");
+
+/** Creates a task on the GPU, or on the CPU when the GPU can't run it (no WebGL, or a driver that refuses). */
+async function withFallback<T>(create: (delegate: "GPU" | "CPU") => Promise<T>): Promise<T> {
+    try {
+        return await create("GPU");
+    } catch {
+        return create("CPU");
+    }
+}
+
+async function loadSegmenter(assetsUrl?: string): Promise<PersonSegmenter> {
+    const urls = modelUrls(assetsUrl);
+    const vision: Vision = await import("@mediapipe/tasks-vision");
+    const fileset = await vision.FilesetResolver.forVisionTasks(urls.wasm);
+    const segmenter = await withFallback((delegate) =>
+        vision.ImageSegmenter.createFromOptions(fileset, {
+            baseOptions: { modelAssetPath: urls.segmenter, delegate },
+            runningMode: "VIDEO",
+            outputConfidenceMasks: true,
+            outputCategoryMask: false,
+        }),
+    );
+    const labels = segmenter.getLabels();
+    const stamp = monotonic();
+    return {
+        segment(frame, timestampMs) {
+            let mask: PersonMask | null = null;
+            segmenter.segmentForVideo(frame, stamp(timestampMs), (result) => {
+                const masks = result.confidenceMasks;
+                if (!masks?.length) {
+                    return;
+                }
+                const person = masks[personMaskIndex(labels, masks.length)];
+                // The mask is only valid inside this callback, so it is copied out.
+                mask = { width: person.width, height: person.height, confidence: new Float32Array(person.getAsFloat32Array()) };
+            });
+            return mask;
+        },
+    };
+}
+
+async function loadFaceTracker(assetsUrl?: string): Promise<FaceTracker> {
+    const urls = modelUrls(assetsUrl);
+    const vision: Vision = await import("@mediapipe/tasks-vision");
+    const fileset = await vision.FilesetResolver.forVisionTasks(urls.wasm);
+    const landmarker = await withFallback((delegate) =>
+        vision.FaceLandmarker.createFromOptions(fileset, {
+            baseOptions: { modelAssetPath: urls.face, delegate },
+            runningMode: "VIDEO",
+            numFaces: 1,
+        }),
+    );
+    const stamp = monotonic();
+    return {
+        detect(frame, timestampMs) {
+            return landmarker.detectForVideo(frame, stamp(timestampMs)).faceLandmarks[0] ?? null;
+        },
+    };
+}
+
+const segmenters = new Map<string, Promise<PersonSegmenter>>();
+const faceTrackers = new Map<string, Promise<FaceTracker>>();
+
+/** Loads `load` once per key, and forgets a failure so the next attempt (a retry) tries again. */
+function shared<T>(cache: Map<string, Promise<T>>, key: string, load: () => Promise<T>): Promise<T> {
+    let pending = cache.get(key);
+    if (!pending) {
+        pending = load();
+        cache.set(key, pending);
+        pending.catch(() => cache.delete(key));
+    }
+    return pending;
+}
+
+/** The person-segmentation model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export function createPersonSegmenter(assetsUrl?: string): Promise<PersonSegmenter> {
+    return shared(segmenters, assetsUrl ?? "", () => loadSegmenter(assetsUrl));
+}
+
+/** The face-landmark model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export function createFaceTracker(assetsUrl?: string): Promise<FaceTracker> {
+    return shared(faceTrackers, assetsUrl ?? "", () => loadFaceTracker(assetsUrl));
+}
+
+/** Forgets every loaded model - for tests. */
+export function resetModelCache(): void {
+    segmenters.clear();
+    faceTrackers.clear();
+}

package/apps/shared/media/filters/pixelEffects.ts

@@ -0,0 +1,61 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The colour effects, as plain functions over an RGBA pixel buffer (`ImageData.data`) so they need no canvas API
+ * beyond `getImageData`/`putImageData`. They are done here, one pixel at a time, rather than with the canvas
+ * `filter` property, which Safari does not support.
+ *
+ * Each function edits `data` in place and leaves the alpha channel alone.
+ */
+
+/** Rec. 601 luma - how bright a colour looks. */
+function luma(r: number, g: number, b: number): number {
+    return 0.299 * r + 0.587 * g + 0.114 * b;
+}
+
+export function applyGrayscale(data: Uint8ClampedArray): void {
+    for (let i = 0; i < data.length; i += 4) {
+        const y = luma(data[i], data[i + 1], data[i + 2]);
+        data[i] = y;
+        data[i + 1] = y;
+        data[i + 2] = y;
+    }
+}
+
+export function applySepia(data: Uint8ClampedArray): void {
+    for (let i = 0; i < data.length; i += 4) {
+        const r = data[i];
+        const g = data[i + 1];
+        const b = data[i + 2];
+        // `Uint8ClampedArray` clamps to 0-255 on assignment.
+        data[i] = 0.393 * r + 0.769 * g + 0.189 * b;
+        data[i + 1] = 0.349 * r + 0.686 * g + 0.168 * b;
+        data[i + 2] = 0.272 * r + 0.534 * g + 0.131 * b;
+    }
+}
+
+/** A brightened, green-tinted picture, like a night-vision scope. */
+export function applyNightVision(data: Uint8ClampedArray): void {
+    for (let i = 0; i < data.length; i += 4) {
+        const y = luma(data[i], data[i + 1], data[i + 2]) * 1.4;
+        data[i] = y * 0.25;
+        data[i + 1] = y;
+        data[i + 2] = y * 0.25;
+    }
+}
+
+/** Below this confidence a pixel is background; above `MASK_HIGH` it is the person; in between it fades, so the edge of
+ * the person is soft rather than jagged. */
+const MASK_LOW = 0.3;
+const MASK_HIGH = 0.7;
+
+/** Turns a person-confidence mask (0-1 per pixel) into the alpha channel of an RGBA buffer, sharpening the edge with a
+ * smoothstep. `rgba` must hold `4 * confidence.length` values. */
+export function maskToAlpha(confidence: Float32Array, rgba: Uint8ClampedArray): void {
+    for (let i = 0; i < confidence.length; i++) {
+        const t = Math.min(1, Math.max(0, (confidence[i] - MASK_LOW) / (MASK_HIGH - MASK_LOW)));
+        rgba[i * 4 + 3] = t * t * (3 - 2 * t) * 255;
+    }
+}

package/apps/shared/media/mediaPreferences.ts

@@ -0,0 +1,105 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * What a participant chose last time - which camera and microphone, whether each was on, and their video filters -
+ * kept in this browser's `localStorage` so the next meeting starts the same way.
+ *
+ * It lives on the device rather than on the server on purpose: most participants are guests with no account, and a
+ * camera's id means nothing on another machine anyway. Nothing here is sent anywhere.
+ *
+ * Storage can be missing or refuse to work (private windows, blocked site data, a full quota, server-side rendering),
+ * and none of that is worth failing a call over: every function tolerates it and behaves as if nothing was saved.
+ * Whatever is read back is validated, since it may be from another version of this plugin or edited by hand.
+ *
+ * The custom background picture is stored under its own key: it is by far the largest thing kept, and if it doesn't
+ * fit the rest of the settings should still be saved.
+ */
+import { type VideoFilters, sanitizeFilters } from "./filters/filterTypes.js";
+
+export interface MediaPreferences {
+    /** The camera and microphone the participant picked, by `deviceId`. */
+    cameraId?: string;
+    microphoneId?: string;
+    /** Whether the microphone / camera was on when they left. */
+    micEnabled?: boolean;
+    cameraEnabled?: boolean;
+    filters?: VideoFilters;
+    /** The custom background, as a (downscaled) image data URL. */
+    backgroundImage?: string;
+}
+
+export const PREFERENCES_KEY = "rapidmx.meet.preferences";
+export const BACKGROUND_KEY = "rapidmx.meet.background";
+
+/** The `Storage` to use, or `undefined` where there is none - looked up lazily and guarded, since merely reading
+ * `localStorage` can throw (a browser blocking site data) and it doesn't exist on the server. */
+export function defaultStorage(): Storage | undefined {
+    try {
+        return typeof localStorage === "undefined" ? undefined : localStorage;
+    } catch {
+        return undefined;
+    }
+}
+
+function readJson(storage: Storage, key: string): Record<string, unknown> {
+    try {
+        const parsed: unknown = JSON.parse(storage.getItem(key) ?? "{}");
+        return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : {};
+    } catch {
+        return {};
+    }
+}
+
+const text = (value: unknown): string | undefined => (typeof value === "string" && value ? value : undefined);
+const flag = (value: unknown): boolean | undefined => (typeof value === "boolean" ? value : undefined);
+
+export function loadPreferences(storage: Storage | undefined = defaultStorage()): MediaPreferences {
+    if (!storage) {
+        return {};
+    }
+    const raw = readJson(storage, PREFERENCES_KEY);
+    let backgroundImage: string | undefined;
+    try {
+        const stored = storage.getItem(BACKGROUND_KEY);
+        backgroundImage = stored?.startsWith("data:image/") ? stored : undefined;
+    } catch {
+        backgroundImage = undefined;
+    }
+    return {
+        cameraId: text(raw.cameraId),
+        microphoneId: text(raw.microphoneId),
+        micEnabled: flag(raw.micEnabled),
+        cameraEnabled: flag(raw.cameraEnabled),
+        filters: raw.filters === undefined ? undefined : sanitizeFilters(raw.filters),
+        backgroundImage,
+    };
+}
+
+/** Merges `patch` into what is saved. A key set to `undefined` is forgotten; a failure to save is ignored. */
+export function savePreferences(patch: Partial<MediaPreferences>, storage: Storage | undefined = defaultStorage()): void {
+    if (!storage) {
+        return;
+    }
+    const { backgroundImage, ...rest } = patch;
+    if (Object.keys(rest).length > 0) {
+        try {
+            const merged = { ...readJson(storage, PREFERENCES_KEY), ...rest };
+            storage.setItem(PREFERENCES_KEY, JSON.stringify(merged));
+        } catch {
+            // Nowhere to save - the settings just won't carry over.
+        }
+    }
+    if ("backgroundImage" in patch) {
+        try {
+            if (backgroundImage) {
+                storage.setItem(BACKGROUND_KEY, backgroundImage);
+            } else {
+                storage.removeItem(BACKGROUND_KEY);
+            }
+        } catch {
+            // Most likely the image is too big for what is left of the quota.
+        }
+    }
+}