@rapidmx/meet-plugin 0.5.0 → 0.8.0

package/dist/apps/shared/media/filters/filterTypes.js

@@ -0,0 +1,63 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The video filters a participant can put on their camera. They come in three independent layers, so they combine
+ * (blurred background *and* black and white *and* sunglasses):
+ *
+ * - **Background**: what is behind the participant - untouched, blurred, or a picture they chose. Needs person
+ * segmentation (`mlModels.ts`).
+ * - **Effect**: a look applied to the whole picture - black and white, sepia, night vision, pixelated.
+ * - **Accessory**: something drawn on the participant's face - sunglasses, cat ears, a party hat. Needs face landmarks
+ * (`mlModels.ts`).
+ *
+ * Adding a filter is: add its id here (with a label), then implement it in `VideoFilterProcessor.ts` (an effect), or
+ * in `accessories.ts` (an accessory).
+ */
+export const NO_FILTERS = { background: "none", effect: "none", accessory: "none" };
+export const BACKGROUND_OPTIONS = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "blur", label: "Blur", emoji: "🌫️" },
+    { id: "image", label: "Custom image", emoji: "🖼️" },
+];
+export const EFFECT_OPTIONS = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "bw", label: "Black & white", emoji: "🎞️" },
+    { id: "sepia", label: "Sepia", emoji: "📜" },
+    { id: "night-vision", label: "Night vision", emoji: "🥽" },
+    { id: "pixelate", label: "Pixelate", emoji: "👾" },
+];
+export const ACCESSORY_OPTIONS = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "sunglasses", label: "Sunglasses", emoji: "😎" },
+    { id: "cat-ears", label: "Cat", emoji: "🐱" },
+    { id: "party-hat", label: "Party hat", emoji: "🥳" },
+    { id: "crown", label: "Crown", emoji: "👑" },
+    { id: "mustache", label: "Mustache", emoji: "🥸" },
+];
+/** Whether any filter is on - when none is, the camera's own track is sent untouched and nothing is processed. */
+export function filtersActive(filters) {
+    return filters.background !== "none" || filters.effect !== "none" || filters.accessory !== "none";
+}
+/** Whether the filters need the person-segmentation model. */
+export function needsSegmentation(filters) {
+    return filters.background !== "none";
+}
+/** Whether the filters need the face-landmark model. */
+export function needsFace(filters) {
+    return filters.accessory !== "none";
+}
+function pick(options, value, fallback) {
+    return options.some((option) => option.id === value) ? value : fallback;
+}
+/** Turns whatever was read back from storage into valid filters - anything unrecognised (a filter since removed, a
+ * hand-edited value, another version's shape) falls back to "none" rather than breaking the call. */
+export function sanitizeFilters(value) {
+    const raw = (typeof value === "object" && value !== null ? value : {});
+    return {
+        background: pick(BACKGROUND_OPTIONS, raw.background, "none"),
+        effect: pick(EFFECT_OPTIONS, raw.effect, "none"),
+        accessory: pick(ACCESSORY_OPTIONS, raw.accessory, "none"),
+    };
+}

package/dist/apps/shared/media/filters/mlModels.d.ts

@@ -0,0 +1,54 @@
+/**
+ * The two machine-learning models behind the filters, run in the browser with MediaPipe Tasks:
+ *
+ * - **Person segmentation** (`createPersonSegmenter`) - which pixels are the participant: what the background
+ * blur and the custom background need.
+ * - **Face landmarks** (`createFaceTracker`) - where the participant's face is: what the accessories need.
+ *
+ * Nothing here is loaded until a participant turns a filter on that needs it, and each model is loaded once and
+ * shared. The MediaPipe library itself is `import()`ed on demand for the same reason - a participant who never uses a
+ * filter never downloads it.
+ *
+ * **Where the assets come from**: by default the WebAssembly runtime is fetched from jsDelivr and the models from
+ * Google's public MediaPipe model bucket. That is a request to a third party, so an administrator can host them
+ * instead (the `mail:videoconf:effects:assets_url` setting, delivered by the join response): the runtime's `wasm/`
+ * directory from the `@mediapipe/tasks-vision` package, plus `selfie_segmenter.tflite` and `face_landmarker.task`,
+ * all under one base URL.
+ *
+ * Everything the rest of the pipeline needs from MediaPipe is behind the small `PersonSegmenter`/`FaceTracker`
+ * interfaces, so `VideoFilterProcessor` never touches the library and its tests use fakes.
+ */
+import type { FaceLandmark } from "./accessories.js";
+/** Must match the installed `@mediapipe/tasks-vision` (a test checks): the `.wasm` fetched at runtime has to be the
+ * version the JavaScript was built against. */
+export declare const MEDIAPIPE_VERSION = "1.0.1";
+export interface ModelUrls {
+    wasm: string;
+    segmenter: string;
+    face: string;
+}
+/** Where to fetch the runtime and models from - the CDN when `assetsUrl` is empty, else under `assetsUrl`. */
+export declare function modelUrls(assetsUrl?: string): ModelUrls;
+/** How likely each pixel of a frame is to be the participant, 0-1. */
+export interface PersonMask {
+    width: number;
+    height: number;
+    confidence: Float32Array;
+}
+export interface PersonSegmenter {
+    /** Segments `frame`, or `null` when the model produced nothing for it. */
+    segment(frame: HTMLVideoElement, timestampMs: number): PersonMask | null;
+}
+export interface FaceTracker {
+    /** The landmarks of the first face in `frame`, or `null` when there is none. */
+    detect(frame: HTMLVideoElement, timestampMs: number): FaceLandmark[] | null;
+}
+/** Which of a segmentation model's masks is the person: the one labelled as such (the selfie model's only label is
+ * "selfie"), else the last (models list the background first). */
+export declare function personMaskIndex(labels: string[], maskCount: number): number;
+/** The person-segmentation model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export declare function createPersonSegmenter(assetsUrl?: string): Promise<PersonSegmenter>;
+/** The face-landmark model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export declare function createFaceTracker(assetsUrl?: string): Promise<FaceTracker>;
+/** Forgets every loaded model - for tests. */
+export declare function resetModelCache(): void;

package/dist/apps/shared/media/filters/mlModels.js

@@ -0,0 +1,106 @@
+/** Must match the installed `@mediapipe/tasks-vision` (a test checks): the `.wasm` fetched at runtime has to be the
+ * version the JavaScript was built against. */
+export const MEDIAPIPE_VERSION = "1.0.1";
+const CDN_WASM = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MEDIAPIPE_VERSION}/wasm`;
+const CDN_SEGMENTER_MODEL = "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite";
+const CDN_FACE_MODEL = "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task";
+/** Where to fetch the runtime and models from - the CDN when `assetsUrl` is empty, else under `assetsUrl`. */
+export function modelUrls(assetsUrl) {
+    if (!assetsUrl) {
+        return { wasm: CDN_WASM, segmenter: CDN_SEGMENTER_MODEL, face: CDN_FACE_MODEL };
+    }
+    const base = assetsUrl.replace(/\/+$/, "");
+    return { wasm: `${base}/wasm`, segmenter: `${base}/selfie_segmenter.tflite`, face: `${base}/face_landmarker.task` };
+}
+/** Which of a segmentation model's masks is the person: the one labelled as such (the selfie model's only label is
+ * "selfie"), else the last (models list the background first). */
+export function personMaskIndex(labels, maskCount) {
+    const index = labels.findIndex((label) => /person|selfie|foreground/i.test(label));
+    return index >= 0 && index < maskCount ? index : maskCount - 1;
+}
+/** MediaPipe rejects a frame whose timestamp isn't later than the last one it saw. */
+function monotonic() {
+    let last = -1;
+    return (timestampMs) => {
+        last = Math.max(timestampMs, last + 1);
+        return last;
+    };
+}
+/** Creates a task on the GPU, or on the CPU when the GPU can't run it (no WebGL, or a driver that refuses). */
+async function withFallback(create) {
+    try {
+        return await create("GPU");
+    }
+    catch {
+        return create("CPU");
+    }
+}
+async function loadSegmenter(assetsUrl) {
+    const urls = modelUrls(assetsUrl);
+    const vision = await import("@mediapipe/tasks-vision");
+    const fileset = await vision.FilesetResolver.forVisionTasks(urls.wasm);
+    const segmenter = await withFallback((delegate) => vision.ImageSegmenter.createFromOptions(fileset, {
+        baseOptions: { modelAssetPath: urls.segmenter, delegate },
+        runningMode: "VIDEO",
+        outputConfidenceMasks: true,
+        outputCategoryMask: false,
+    }));
+    const labels = segmenter.getLabels();
+    const stamp = monotonic();
+    return {
+        segment(frame, timestampMs) {
+            let mask = null;
+            segmenter.segmentForVideo(frame, stamp(timestampMs), (result) => {
+                const masks = result.confidenceMasks;
+                if (!masks?.length) {
+                    return;
+                }
+                const person = masks[personMaskIndex(labels, masks.length)];
+                // The mask is only valid inside this callback, so it is copied out.
+                mask = { width: person.width, height: person.height, confidence: new Float32Array(person.getAsFloat32Array()) };
+            });
+            return mask;
+        },
+    };
+}
+async function loadFaceTracker(assetsUrl) {
+    const urls = modelUrls(assetsUrl);
+    const vision = await import("@mediapipe/tasks-vision");
+    const fileset = await vision.FilesetResolver.forVisionTasks(urls.wasm);
+    const landmarker = await withFallback((delegate) => vision.FaceLandmarker.createFromOptions(fileset, {
+        baseOptions: { modelAssetPath: urls.face, delegate },
+        runningMode: "VIDEO",
+        numFaces: 1,
+    }));
+    const stamp = monotonic();
+    return {
+        detect(frame, timestampMs) {
+            return landmarker.detectForVideo(frame, stamp(timestampMs)).faceLandmarks[0] ?? null;
+        },
+    };
+}
+const segmenters = new Map();
+const faceTrackers = new Map();
+/** Loads `load` once per key, and forgets a failure so the next attempt (a retry) tries again. */
+function shared(cache, key, load) {
+    let pending = cache.get(key);
+    if (!pending) {
+        pending = load();
+        cache.set(key, pending);
+        pending.catch(() => cache.delete(key));
+    }
+    return pending;
+}
+/** The person-segmentation model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export function createPersonSegmenter(assetsUrl) {
+    return shared(segmenters, assetsUrl ?? "", () => loadSegmenter(assetsUrl));
+}
+/** The face-landmark model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export function createFaceTracker(assetsUrl) {
+    return shared(faceTrackers, assetsUrl ?? "", () => loadFaceTracker(assetsUrl));
+}
+/** Forgets every loaded model - for tests. */
+export function resetModelCache() {
+    segmenters.clear();
+    faceTrackers.clear();
+}

package/dist/apps/shared/media/filters/pixelEffects.d.ts

@@ -0,0 +1,14 @@
+/**
+ * The colour effects, as plain functions over an RGBA pixel buffer (`ImageData.data`) so they need no canvas API
+ * beyond `getImageData`/`putImageData`. They are done here, one pixel at a time, rather than with the canvas
+ * `filter` property, which Safari does not support.
+ *
+ * Each function edits `data` in place and leaves the alpha channel alone.
+ */
+export declare function applyGrayscale(data: Uint8ClampedArray): void;
+export declare function applySepia(data: Uint8ClampedArray): void;
+/** A brightened, green-tinted picture, like a night-vision scope. */
+export declare function applyNightVision(data: Uint8ClampedArray): void;
+/** Turns a person-confidence mask (0-1 per pixel) into the alpha channel of an RGBA buffer, sharpening the edge with a
+ * smoothstep. `rgba` must hold `4 * confidence.length` values. */
+export declare function maskToAlpha(confidence: Float32Array, rgba: Uint8ClampedArray): void;

package/dist/apps/shared/media/filters/pixelEffects.js

@@ -0,0 +1,55 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The colour effects, as plain functions over an RGBA pixel buffer (`ImageData.data`) so they need no canvas API
+ * beyond `getImageData`/`putImageData`. They are done here, one pixel at a time, rather than with the canvas
+ * `filter` property, which Safari does not support.
+ *
+ * Each function edits `data` in place and leaves the alpha channel alone.
+ */
+/** Rec. 601 luma - how bright a colour looks. */
+function luma(r, g, b) {
+    return 0.299 * r + 0.587 * g + 0.114 * b;
+}
+export function applyGrayscale(data) {
+    for (let i = 0; i < data.length; i += 4) {
+        const y = luma(data[i], data[i + 1], data[i + 2]);
+        data[i] = y;
+        data[i + 1] = y;
+        data[i + 2] = y;
+    }
+}
+export function applySepia(data) {
+    for (let i = 0; i < data.length; i += 4) {
+        const r = data[i];
+        const g = data[i + 1];
+        const b = data[i + 2];
+        // `Uint8ClampedArray` clamps to 0-255 on assignment.
+        data[i] = 0.393 * r + 0.769 * g + 0.189 * b;
+        data[i + 1] = 0.349 * r + 0.686 * g + 0.168 * b;
+        data[i + 2] = 0.272 * r + 0.534 * g + 0.131 * b;
+    }
+}
+/** A brightened, green-tinted picture, like a night-vision scope. */
+export function applyNightVision(data) {
+    for (let i = 0; i < data.length; i += 4) {
+        const y = luma(data[i], data[i + 1], data[i + 2]) * 1.4;
+        data[i] = y * 0.25;
+        data[i + 1] = y;
+        data[i + 2] = y * 0.25;
+    }
+}
+/** Below this confidence a pixel is background; above `MASK_HIGH` it is the person; in between it fades, so the edge of
+ * the person is soft rather than jagged. */
+const MASK_LOW = 0.3;
+const MASK_HIGH = 0.7;
+/** Turns a person-confidence mask (0-1 per pixel) into the alpha channel of an RGBA buffer, sharpening the edge with a
+ * smoothstep. `rgba` must hold `4 * confidence.length` values. */
+export function maskToAlpha(confidence, rgba) {
+    for (let i = 0; i < confidence.length; i++) {
+        const t = Math.min(1, Math.max(0, (confidence[i] - MASK_LOW) / (MASK_HIGH - MASK_LOW)));
+        rgba[i * 4 + 3] = t * t * (3 - 2 * t) * 255;
+    }
+}

package/dist/apps/shared/media/mediaPreferences.d.ts

@@ -0,0 +1,34 @@
+/**
+ * What a participant chose last time - which camera and microphone, whether each was on, and their video filters -
+ * kept in this browser's `localStorage` so the next meeting starts the same way.
+ *
+ * It lives on the device rather than on the server on purpose: most participants are guests with no account, and a
+ * camera's id means nothing on another machine anyway. Nothing here is sent anywhere.
+ *
+ * Storage can be missing or refuse to work (private windows, blocked site data, a full quota, server-side rendering),
+ * and none of that is worth failing a call over: every function tolerates it and behaves as if nothing was saved.
+ * Whatever is read back is validated, since it may be from another version of this plugin or edited by hand.
+ *
+ * The custom background picture is stored under its own key: it is by far the largest thing kept, and if it doesn't
+ * fit the rest of the settings should still be saved.
+ */
+import { type VideoFilters } from "./filters/filterTypes.js";
+export interface MediaPreferences {
+    /** The camera and microphone the participant picked, by `deviceId`. */
+    cameraId?: string;
+    microphoneId?: string;
+    /** Whether the microphone / camera was on when they left. */
+    micEnabled?: boolean;
+    cameraEnabled?: boolean;
+    filters?: VideoFilters;
+    /** The custom background, as a (downscaled) image data URL. */
+    backgroundImage?: string;
+}
+export declare const PREFERENCES_KEY = "rapidmx.meet.preferences";
+export declare const BACKGROUND_KEY = "rapidmx.meet.background";
+/** The `Storage` to use, or `undefined` where there is none - looked up lazily and guarded, since merely reading
+ * `localStorage` can throw (a browser blocking site data) and it doesn't exist on the server. */
+export declare function defaultStorage(): Storage | undefined;
+export declare function loadPreferences(storage?: Storage | undefined): MediaPreferences;
+/** Merges `patch` into what is saved. A key set to `undefined` is forgotten; a failure to save is ignored. */
+export declare function savePreferences(patch: Partial<MediaPreferences>, storage?: Storage | undefined): void;

package/dist/apps/shared/media/mediaPreferences.js

@@ -0,0 +1,93 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * What a participant chose last time - which camera and microphone, whether each was on, and their video filters -
+ * kept in this browser's `localStorage` so the next meeting starts the same way.
+ *
+ * It lives on the device rather than on the server on purpose: most participants are guests with no account, and a
+ * camera's id means nothing on another machine anyway. Nothing here is sent anywhere.
+ *
+ * Storage can be missing or refuse to work (private windows, blocked site data, a full quota, server-side rendering),
+ * and none of that is worth failing a call over: every function tolerates it and behaves as if nothing was saved.
+ * Whatever is read back is validated, since it may be from another version of this plugin or edited by hand.
+ *
+ * The custom background picture is stored under its own key: it is by far the largest thing kept, and if it doesn't
+ * fit the rest of the settings should still be saved.
+ */
+import { sanitizeFilters } from "./filters/filterTypes.js";
+export const PREFERENCES_KEY = "rapidmx.meet.preferences";
+export const BACKGROUND_KEY = "rapidmx.meet.background";
+/** The `Storage` to use, or `undefined` where there is none - looked up lazily and guarded, since merely reading
+ * `localStorage` can throw (a browser blocking site data) and it doesn't exist on the server. */
+export function defaultStorage() {
+    try {
+        return typeof localStorage === "undefined" ? undefined : localStorage;
+    }
+    catch {
+        return undefined;
+    }
+}
+function readJson(storage, key) {
+    try {
+        const parsed = JSON.parse(storage.getItem(key) ?? "{}");
+        return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed : {};
+    }
+    catch {
+        return {};
+    }
+}
+const text = (value) => (typeof value === "string" && value ? value : undefined);
+const flag = (value) => (typeof value === "boolean" ? value : undefined);
+export function loadPreferences(storage = defaultStorage()) {
+    if (!storage) {
+        return {};
+    }
+    const raw = readJson(storage, PREFERENCES_KEY);
+    let backgroundImage;
+    try {
+        const stored = storage.getItem(BACKGROUND_KEY);
+        backgroundImage = stored?.startsWith("data:image/") ? stored : undefined;
+    }
+    catch {
+        backgroundImage = undefined;
+    }
+    return {
+        cameraId: text(raw.cameraId),
+        microphoneId: text(raw.microphoneId),
+        micEnabled: flag(raw.micEnabled),
+        cameraEnabled: flag(raw.cameraEnabled),
+        filters: raw.filters === undefined ? undefined : sanitizeFilters(raw.filters),
+        backgroundImage,
+    };
+}
+/** Merges `patch` into what is saved. A key set to `undefined` is forgotten; a failure to save is ignored. */
+export function savePreferences(patch, storage = defaultStorage()) {
+    if (!storage) {
+        return;
+    }
+    const { backgroundImage, ...rest } = patch;
+    if (Object.keys(rest).length > 0) {
+        try {
+            const merged = { ...readJson(storage, PREFERENCES_KEY), ...rest };
+            storage.setItem(PREFERENCES_KEY, JSON.stringify(merged));
+        }
+        catch {
+            // Nowhere to save - the settings just won't carry over.
+        }
+    }
+    if ("backgroundImage" in patch) {
+        try {
+            if (backgroundImage) {
+                storage.setItem(BACKGROUND_KEY, backgroundImage);
+            }
+            else {
+                storage.removeItem(BACKGROUND_KEY);
+            }
+        }
+        catch {
+            // Most likely the image is too big for what is left of the quota.
+        }
+    }
+}

package/dist/apps/shared/media/useLocalMedia.d.ts

@@ -1,4 +1,6 @@
 import { type DeviceLists, type MediaAccessError } from "./deviceMedia.js";
+import { type VideoFilters } from "./filters/filterTypes.js";
+import { type FilterStatus } from "./filters/VideoFilterProcessor.js";
 export type MediaKind = "audio" | "video";
 /** `pending`: not asked yet, or being asked. `live`: a track is running. `off`: the participant turned the camera
  * off. `denied`/`unavailable`/`error`: the browser said no, there is no such device (or it was unplugged), or
@@ -26,6 +28,16 @@ export interface LocalMedia {
     selectedDeviceIds: Partial<Record<MediaKind, string>>;
     /** 0-`AUDIO_LEVEL_STEPS`: how loud the microphone is right now - always 0 while muted. */
     audioLevel: number;
+    /** The video filters in effect - `NO_FILTERS` when the camera is sent as it is. */
+    filters: VideoFilters;
+    /** Whether the filters' models are still loading, and why a filter isn't fully working. */
+    filterStatus: FilterStatus;
+    /** Whether a custom background image has been chosen (and so "custom image" can be switched to at once). */
+    hasBackgroundImage: boolean;
+    /** Changes some of the filters, keeping the rest. */
+    setFilters(patch: Partial<VideoFilters>): void;
+    /** Uses `file` as the custom background and switches to it. Resolves to why the file can't be used, or `null`. */
+    chooseBackgroundImage(file: File): Promise<string | null>;
     /** Asks for the camera and microphone (or just `kind`), see this module's doc comment. */
     requestAccess(kind?: MediaKind): Promise<void>;
     toggleMic(): Promise<void>;
@@ -34,4 +46,8 @@ export interface LocalMedia {
     /** Stops both tracks. */
     release(): void;
 }
-export declare function useLocalMedia(): LocalMedia;
+export interface LocalMediaOptions {
+    /** Where the filters' models are hosted, when not on the default CDN - see `filters/mlModels.ts`. */
+    effectsAssetsUrl?: string;
+}
+export declare function useLocalMedia({ effectsAssetsUrl }?: LocalMediaOptions): LocalMedia;