@rapidmx/meet-plugin 0.6.0 → 0.8.1

package/apps/shared/media/filters/accessories.ts

@@ -0,0 +1,262 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The face accessories - sunglasses, cat ears, a party hat, a crown, a mustache - drawn as vector paths onto the
+ * filtered frame, so the plugin ships no image assets at all.
+ *
+ * Each one is drawn in the face's own coordinate frame: the origin sits on a landmark (between the eyes, the top of
+ * the forehead, the nose), the x axis runs along the line between the eyes so the accessory tilts with the head, and
+ * one unit is the distance between the outer corners of the eyes so it grows and shrinks with the head. That keeps
+ * every drawing routine below a few plain numbers in "face widths".
+ */
+import type { Accessory } from "./filterTypes.js";
+
+/** A landmark as the face model reports it: 0-1 across and down the frame. */
+export interface FaceLandmark {
+    x: number;
+    y: number;
+}
+
+export interface Point {
+    x: number;
+    y: number;
+}
+
+/** Where the parts of a face are, in canvas pixels. */
+export interface FaceGeometry {
+    /** Halfway between the outer corners of the eyes. */
+    eyes: Point;
+    /** The top of the forehead. */
+    forehead: Point;
+    noseTip: Point;
+    upperLip: Point;
+    /** The distance between the outer corners of the eyes - one unit of the accessories' coordinate frame. */
+    unit: number;
+    /** The tilt of the line between the eyes, in radians. */
+    angle: number;
+}
+
+// Indices into the 478-point face mesh (MediaPipe's canonical face model).
+const RIGHT_EYE_OUTER = 33;
+const LEFT_EYE_OUTER = 263;
+const FOREHEAD_TOP = 10;
+const NOSE_TIP = 1;
+const UPPER_LIP = 0;
+const MESH_POINTS = 468;
+
+/** Works out where the parts of a face are, in pixels, from the face mesh's landmarks - `null` when there are not
+ * enough of them to be a face mesh at all. */
+export function faceGeometry(landmarks: FaceLandmark[], width: number, height: number): FaceGeometry | null {
+    if (landmarks.length < MESH_POINTS) {
+        return null;
+    }
+    const at = (index: number): Point => ({ x: landmarks[index].x * width, y: landmarks[index].y * height });
+    const right = at(RIGHT_EYE_OUTER);
+    const left = at(LEFT_EYE_OUTER);
+    return {
+        eyes: { x: (right.x + left.x) / 2, y: (right.y + left.y) / 2 },
+        forehead: at(FOREHEAD_TOP),
+        noseTip: at(NOSE_TIP),
+        upperLip: at(UPPER_LIP),
+        unit: Math.hypot(left.x - right.x, left.y - right.y),
+        angle: Math.atan2(left.y - right.y, left.x - right.x),
+    };
+}
+
+/** Runs `draw` with the canvas moved to `origin`, turned by the face's tilt and scaled to one face width per unit. */
+function inFaceFrame(ctx: CanvasRenderingContext2D, origin: Point, face: FaceGeometry, draw: () => void): void {
+    ctx.save();
+    ctx.translate(origin.x, origin.y);
+    ctx.rotate(face.angle);
+    ctx.scale(face.unit, face.unit);
+    draw();
+    ctx.restore();
+}
+
+/** Runs `draw` once as it is and once flipped left-right, for the two matching halves of a symmetrical accessory. */
+function bothSides(ctx: CanvasRenderingContext2D, draw: () => void): void {
+    for (const side of [1, -1]) {
+        ctx.save();
+        ctx.scale(side, 1);
+        draw();
+        ctx.restore();
+    }
+}
+
+function roundedRect(ctx: CanvasRenderingContext2D, x: number, y: number, w: number, h: number, r: number): void {
+    ctx.beginPath();
+    ctx.moveTo(x + r, y);
+    ctx.arcTo(x + w, y, x + w, y + h, r);
+    ctx.arcTo(x + w, y + h, x, y + h, r);
+    ctx.arcTo(x, y + h, x, y, r);
+    ctx.arcTo(x, y, x + w, y, r);
+    ctx.closePath();
+}
+
+function drawSunglasses(ctx: CanvasRenderingContext2D, face: FaceGeometry): void {
+    inFaceFrame(ctx, face.eyes, face, () => {
+        ctx.fillStyle = "rgba(12, 12, 18, 0.93)";
+        ctx.strokeStyle = "rgba(12, 12, 18, 0.93)";
+        ctx.lineCap = "round";
+        bothSides(ctx, () => {
+            roundedRect(ctx, 0.06, -0.17, 0.54, 0.38, 0.12);
+            ctx.fill();
+            // The arm that runs back over the ear.
+            ctx.lineWidth = 0.04;
+            ctx.beginPath();
+            ctx.moveTo(0.6, -0.06);
+            ctx.lineTo(0.8, -0.1);
+            ctx.stroke();
+            // A glint across the lens.
+            ctx.fillStyle = "rgba(255, 255, 255, 0.2)";
+            ctx.beginPath();
+            ctx.moveTo(0.14, -0.1);
+            ctx.lineTo(0.28, -0.1);
+            ctx.lineTo(0.16, 0.12);
+            ctx.lineTo(0.1, 0.12);
+            ctx.closePath();
+            ctx.fill();
+        });
+        // The bridge between the lenses.
+        ctx.lineWidth = 0.05;
+        ctx.beginPath();
+        ctx.moveTo(-0.08, -0.08);
+        ctx.lineTo(0.08, -0.08);
+        ctx.stroke();
+    });
+}
+
+function drawCat(ctx: CanvasRenderingContext2D, face: FaceGeometry): void {
+    inFaceFrame(ctx, face.forehead, face, () => {
+        bothSides(ctx, () => {
+            ctx.fillStyle = "#3b3b44";
+            ctx.beginPath();
+            ctx.moveTo(0.68, 0.14);
+            ctx.lineTo(0.12, 0);
+            ctx.lineTo(0.62, -0.55);
+            ctx.closePath();
+            ctx.fill();
+            ctx.fillStyle = "#f4a6b8";
+            ctx.beginPath();
+            ctx.moveTo(0.56, 0.06);
+            ctx.lineTo(0.22, 0);
+            ctx.lineTo(0.58, -0.33);
+            ctx.closePath();
+            ctx.fill();
+        });
+    });
+    inFaceFrame(ctx, face.noseTip, face, () => {
+        ctx.fillStyle = "#f06a8a";
+        ctx.beginPath();
+        ctx.moveTo(-0.09, -0.04);
+        ctx.lineTo(0.09, -0.04);
+        ctx.lineTo(0, 0.06);
+        ctx.closePath();
+        ctx.fill();
+        ctx.lineCap = "round";
+        bothSides(ctx, () => {
+            for (const tilt of [-1, 0, 1]) {
+                ctx.beginPath();
+                ctx.moveTo(0.3, 0.04 + tilt * 0.07);
+                ctx.lineTo(0.85, -0.02 + tilt * 0.15);
+                // A dark line under a white one, so the whiskers read against a light or a dark background.
+                ctx.strokeStyle = "rgba(0, 0, 0, 0.45)";
+                ctx.lineWidth = 0.035;
+                ctx.stroke();
+                ctx.strokeStyle = "#ffffff";
+                ctx.lineWidth = 0.018;
+                ctx.stroke();
+            }
+        });
+    });
+}
+
+function drawPartyHat(ctx: CanvasRenderingContext2D, face: FaceGeometry): void {
+    inFaceFrame(ctx, face.forehead, face, () => {
+        ctx.save();
+        ctx.beginPath();
+        ctx.moveTo(-0.42, 0.1);
+        ctx.lineTo(0.42, 0.1);
+        ctx.lineTo(0.04, -1.15);
+        ctx.closePath();
+        ctx.clip();
+        ctx.fillStyle = "#ff5fa2";
+        ctx.fillRect(-0.6, -1.3, 1.2, 1.5);
+        ctx.strokeStyle = "#ffd23f";
+        ctx.lineWidth = 0.09;
+        for (const y of [-0.12, -0.42, -0.72]) {
+            ctx.beginPath();
+            ctx.moveTo(-0.6, y + 0.2);
+            ctx.lineTo(0.6, y - 0.2);
+            ctx.stroke();
+        }
+        ctx.restore();
+        ctx.fillStyle = "#ffffff";
+        ctx.beginPath();
+        ctx.arc(0.04, -1.15, 0.09, 0, Math.PI * 2);
+        ctx.fill();
+    });
+}
+
+function drawCrown(ctx: CanvasRenderingContext2D, face: FaceGeometry): void {
+    inFaceFrame(ctx, face.forehead, face, () => {
+        ctx.beginPath();
+        ctx.moveTo(-0.5, 0.1);
+        ctx.lineTo(-0.5, -0.45);
+        ctx.lineTo(-0.25, -0.2);
+        ctx.lineTo(0, -0.55);
+        ctx.lineTo(0.25, -0.2);
+        ctx.lineTo(0.5, -0.45);
+        ctx.lineTo(0.5, 0.1);
+        ctx.closePath();
+        ctx.fillStyle = "#f5c518";
+        ctx.fill();
+        ctx.strokeStyle = "#b8860b";
+        ctx.lineWidth = 0.03;
+        ctx.lineJoin = "round";
+        ctx.stroke();
+        for (const [x, y, color] of [
+            [-0.5, -0.45, "#e63946"],
+            [0, -0.55, "#3a86ff"],
+            [0.5, -0.45, "#e63946"],
+            [0, -0.05, "#2a9d8f"],
+        ] as const) {
+            ctx.fillStyle = color;
+            ctx.beginPath();
+            ctx.arc(x, y, 0.05, 0, Math.PI * 2);
+            ctx.fill();
+        }
+    });
+}
+
+function drawMustache(ctx: CanvasRenderingContext2D, face: FaceGeometry): void {
+    const between: Point = { x: (face.noseTip.x + face.upperLip.x) / 2, y: (face.noseTip.y + face.upperLip.y) / 2 };
+    inFaceFrame(ctx, between, face, () => {
+        ctx.fillStyle = "#3a2618";
+        ctx.beginPath();
+        ctx.moveTo(0, -0.03);
+        ctx.bezierCurveTo(-0.12, -0.12, -0.32, -0.1, -0.42, 0.02);
+        ctx.bezierCurveTo(-0.46, 0.07, -0.4, 0.13, -0.34, 0.09);
+        ctx.bezierCurveTo(-0.26, 0.04, -0.12, 0.06, 0, 0.09);
+        ctx.bezierCurveTo(0.12, 0.06, 0.26, 0.04, 0.34, 0.09);
+        ctx.bezierCurveTo(0.4, 0.13, 0.46, 0.07, 0.42, 0.02);
+        ctx.bezierCurveTo(0.32, -0.1, 0.12, -0.12, 0, -0.03);
+        ctx.closePath();
+        ctx.fill();
+    });
+}
+
+const DRAWERS: Record<Exclude<Accessory, "none">, (ctx: CanvasRenderingContext2D, face: FaceGeometry) => void> = {
+    sunglasses: drawSunglasses,
+    "cat-ears": drawCat,
+    "party-hat": drawPartyHat,
+    crown: drawCrown,
+    mustache: drawMustache,
+};
+
+/** Draws `accessory` onto `ctx` for the face at `face`. */
+export function drawAccessory(ctx: CanvasRenderingContext2D, accessory: Exclude<Accessory, "none">, face: FaceGeometry): void {
+    DRAWERS[accessory](ctx, face);
+}

package/apps/shared/media/filters/backgroundImage.ts

@@ -0,0 +1,69 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The picture behind a participant for the "custom image" background: read from a file they pick on their own
+ * device, shrunk to what a video call can show anyway, and kept as a small JPEG data URL so it can be remembered
+ * (`mediaPreferences.ts`) without filling the browser's storage. The file never leaves the browser.
+ */
+import type { BackgroundPicture } from "./VideoFilterProcessor.js";
+
+/** A bigger file than this is refused before the browser tries to decode it. */
+export const MAX_BACKGROUND_FILE_BYTES = 20 * 1024 * 1024;
+/** The longest edge kept - a call's video is 1280 wide, so more would only cost storage. */
+export const MAX_BACKGROUND_EDGE = 1280;
+const JPEG_QUALITY = 0.85;
+
+export type PreparedBackground = { ok: true; dataUrl: string; picture: BackgroundPicture } | { ok: false; message: string };
+
+/** Decodes an image (a URL or data URL) into something that can be drawn. Rejects if it isn't a picture. */
+export function loadPicture(src: string): Promise<BackgroundPicture> {
+    return new Promise((resolve, reject) => {
+        const image = new Image();
+        image.onload = () => resolve({ source: image, width: image.naturalWidth, height: image.naturalHeight });
+        image.onerror = () => reject(new Error("The image could not be read."));
+        image.src = src;
+    });
+}
+
+function readAsDataUrl(file: File): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(String(reader.result));
+        reader.onerror = () => reject(new Error("The file could not be read."));
+        reader.readAsDataURL(file);
+    });
+}
+
+/** Reads a picked file into a background: validated, downscaled and ready both to draw and to store. */
+export async function prepareBackgroundImage(file: File): Promise<PreparedBackground> {
+    if (!file.type.startsWith("image/")) {
+        return { ok: false, message: "Choose an image file (a JPEG, PNG or similar)." };
+    }
+    if (file.size > MAX_BACKGROUND_FILE_BYTES) {
+        return { ok: false, message: "That image is too large. Choose one under 20 MB." };
+    }
+    try {
+        const original = await loadPicture(await readAsDataUrl(file));
+        const scale = Math.min(1, MAX_BACKGROUND_EDGE / Math.max(original.width, original.height));
+        const canvas = document.createElement("canvas");
+        canvas.width = Math.max(1, Math.round(original.width * scale));
+        canvas.height = Math.max(1, Math.round(original.height * scale));
+        const ctx = canvas.getContext("2d");
+        if (!ctx) {
+            return { ok: false, message: "This browser can't prepare that image." };
+        }
+        // JPEG has no transparency: a transparent PNG would otherwise turn black.
+        ctx.fillStyle = "#ffffff";
+        ctx.fillRect(0, 0, canvas.width, canvas.height);
+        ctx.drawImage(original.source, 0, 0, canvas.width, canvas.height);
+        return {
+            ok: true,
+            dataUrl: canvas.toDataURL("image/jpeg", JPEG_QUALITY),
+            picture: { source: canvas, width: canvas.width, height: canvas.height },
+        };
+    } catch {
+        return { ok: false, message: "That file couldn't be opened as an image." };
+    }
+}

package/apps/shared/media/filters/filterTypes.ts

@@ -0,0 +1,89 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The video filters a participant can put on their camera. They come in three independent layers, so they combine
+ * (blurred background *and* black and white *and* sunglasses):
+ *
+ * - **Background**: what is behind the participant - untouched, blurred, or a picture they chose. Needs person
+ * segmentation (`mlModels.ts`).
+ * - **Effect**: a look applied to the whole picture - black and white, sepia, night vision, pixelated.
+ * - **Accessory**: something drawn on the participant's face - sunglasses, cat ears, a party hat. Needs face landmarks
+ * (`mlModels.ts`).
+ *
+ * Adding a filter is: add its id here (with a label), then implement it in `VideoFilterProcessor.ts` (an effect), or
+ * in `accessories.ts` (an accessory).
+ */
+
+export type BackgroundFilter = "none" | "blur" | "image";
+export type ColorEffect = "none" | "bw" | "sepia" | "night-vision" | "pixelate";
+export type Accessory = "none" | "sunglasses" | "cat-ears" | "party-hat" | "crown" | "mustache";
+
+export interface VideoFilters {
+    background: BackgroundFilter;
+    effect: ColorEffect;
+    accessory: Accessory;
+}
+
+export const NO_FILTERS: VideoFilters = { background: "none", effect: "none", accessory: "none" };
+
+export interface FilterOption<T extends string> {
+    id: T;
+    label: string;
+    /** A glyph shown beside the label, purely decorative. */
+    emoji: string;
+}
+
+export const BACKGROUND_OPTIONS: FilterOption<BackgroundFilter>[] = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "blur", label: "Blur", emoji: "🌫️" },
+    { id: "image", label: "Custom image", emoji: "🖼️" },
+];
+
+export const EFFECT_OPTIONS: FilterOption<ColorEffect>[] = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "bw", label: "Black & white", emoji: "🎞️" },
+    { id: "sepia", label: "Sepia", emoji: "📜" },
+    { id: "night-vision", label: "Night vision", emoji: "🥽" },
+    { id: "pixelate", label: "Pixelate", emoji: "👾" },
+];
+
+export const ACCESSORY_OPTIONS: FilterOption<Accessory>[] = [
+    { id: "none", label: "None", emoji: "🚫" },
+    { id: "sunglasses", label: "Sunglasses", emoji: "😎" },
+    { id: "cat-ears", label: "Cat", emoji: "🐱" },
+    { id: "party-hat", label: "Party hat", emoji: "🥳" },
+    { id: "crown", label: "Crown", emoji: "👑" },
+    { id: "mustache", label: "Mustache", emoji: "🥸" },
+];
+
+/** Whether any filter is on - when none is, the camera's own track is sent untouched and nothing is processed. */
+export function filtersActive(filters: VideoFilters): boolean {
+    return filters.background !== "none" || filters.effect !== "none" || filters.accessory !== "none";
+}
+
+/** Whether the filters need the person-segmentation model. */
+export function needsSegmentation(filters: VideoFilters): boolean {
+    return filters.background !== "none";
+}
+
+/** Whether the filters need the face-landmark model. */
+export function needsFace(filters: VideoFilters): boolean {
+    return filters.accessory !== "none";
+}
+
+function pick<T extends string>(options: FilterOption<T>[], value: unknown, fallback: T): T {
+    return options.some((option) => option.id === value) ? (value as T) : fallback;
+}
+
+/** Turns whatever was read back from storage into valid filters - anything unrecognised (a filter since removed, a
+ * hand-edited value, another version's shape) falls back to "none" rather than breaking the call. */
+export function sanitizeFilters(value: unknown): VideoFilters {
+    const raw = (typeof value === "object" && value !== null ? value : {}) as Record<string, unknown>;
+    return {
+        background: pick(BACKGROUND_OPTIONS, raw.background, "none"),
+        effect: pick(EFFECT_OPTIONS, raw.effect, "none"),
+        accessory: pick(ACCESSORY_OPTIONS, raw.accessory, "none"),
+    };
+}

package/apps/shared/media/filters/mlModels.ts

@@ -0,0 +1,174 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The two machine-learning models behind the filters, run in the browser with MediaPipe Tasks:
+ *
+ * - **Person segmentation** (`createPersonSegmenter`) - which pixels are the participant: what the background
+ * blur and the custom background need.
+ * - **Face landmarks** (`createFaceTracker`) - where the participant's face is: what the accessories need.
+ *
+ * Nothing here is loaded until a participant turns a filter on that needs it, and each model is loaded once and
+ * shared. The MediaPipe library itself is `import()`ed on demand for the same reason - a participant who never uses a
+ * filter never downloads it.
+ *
+ * **Where the assets come from**: by default the WebAssembly runtime is fetched from jsDelivr and the models from
+ * Google's public MediaPipe model bucket. That is a request to a third party, so an administrator can host them
+ * instead (the `mail:videoconf:effects:assets_url` setting, delivered by the join response): the runtime's `wasm/`
+ * directory from the `@mediapipe/tasks-vision` package, plus `selfie_segmenter.tflite` and `face_landmarker.task`,
+ * all under one base URL.
+ *
+ * Everything the rest of the pipeline needs from MediaPipe is behind the small `PersonSegmenter`/`FaceTracker`
+ * interfaces, so `VideoFilterProcessor` never touches the library and its tests use fakes.
+ */
+import type { FaceLandmark } from "./accessories.js";
+
+/** Must match the installed `@mediapipe/tasks-vision` (a test checks): the `.wasm` fetched at runtime has to be the
+ * version the JavaScript was built against. */
+export const MEDIAPIPE_VERSION = "1.0.1";
+
+const CDN_WASM = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MEDIAPIPE_VERSION}/wasm`;
+const CDN_SEGMENTER_MODEL =
+    "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite";
+const CDN_FACE_MODEL =
+    "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task";
+
+export interface ModelUrls {
+    wasm: string;
+    segmenter: string;
+    face: string;
+}
+
+/** Where to fetch the runtime and models from - the CDN when `assetsUrl` is empty, else under `assetsUrl`. */
+export function modelUrls(assetsUrl?: string): ModelUrls {
+    if (!assetsUrl) {
+        return { wasm: CDN_WASM, segmenter: CDN_SEGMENTER_MODEL, face: CDN_FACE_MODEL };
+    }
+    const base = assetsUrl.replace(/\/+$/, "");
+    return { wasm: `${base}/wasm`, segmenter: `${base}/selfie_segmenter.tflite`, face: `${base}/face_landmarker.task` };
+}
+
+/** How likely each pixel of a frame is to be the participant, 0-1. */
+export interface PersonMask {
+    width: number;
+    height: number;
+    confidence: Float32Array;
+}
+
+export interface PersonSegmenter {
+    /** Segments `frame`, or `null` when the model produced nothing for it. */
+    segment(frame: HTMLVideoElement, timestampMs: number): PersonMask | null;
+}
+
+export interface FaceTracker {
+    /** The landmarks of the first face in `frame`, or `null` when there is none. */
+    detect(frame: HTMLVideoElement, timestampMs: number): FaceLandmark[] | null;
+}
+
+/** Which of a segmentation model's masks is the person: the one labelled as such (the selfie model's only label is
+ * "selfie"), else the last (models list the background first). */
+export function personMaskIndex(labels: string[], maskCount: number): number {
+    const index = labels.findIndex((label) => /person|selfie|foreground/i.test(label));
+    return index >= 0 && index < maskCount ? index : maskCount - 1;
+}
+
+/** MediaPipe rejects a frame whose timestamp isn't later than the last one it saw. */
+function monotonic(): (timestampMs: number) => number {
+    let last = -1;
+    return (timestampMs) => {
+        last = Math.max(timestampMs, last + 1);
+        return last;
+    };
+}
+
+type Vision = typeof import("@mediapipe/tasks-vision");
+
+/** Creates a task on the GPU, or on the CPU when the GPU can't run it (no WebGL, or a driver that refuses). */
+async function withFallback<T>(create: (delegate: "GPU" | "CPU") => Promise<T>): Promise<T> {
+    try {
+        return await create("GPU");
+    } catch {
+        return create("CPU");
+    }
+}
+
+async function loadSegmenter(assetsUrl?: string): Promise<PersonSegmenter> {
+    const urls = modelUrls(assetsUrl);
+    const vision: Vision = await import("@mediapipe/tasks-vision");
+    const fileset = await vision.FilesetResolver.forVisionTasks(urls.wasm);
+    const segmenter = await withFallback((delegate) =>
+        vision.ImageSegmenter.createFromOptions(fileset, {
+            baseOptions: { modelAssetPath: urls.segmenter, delegate },
+            runningMode: "VIDEO",
+            outputConfidenceMasks: true,
+            outputCategoryMask: false,
+        }),
+    );
+    const labels = segmenter.getLabels();
+    const stamp = monotonic();
+    return {
+        segment(frame, timestampMs) {
+            let mask: PersonMask | null = null;
+            segmenter.segmentForVideo(frame, stamp(timestampMs), (result) => {
+                const masks = result.confidenceMasks;
+                if (!masks?.length) {
+                    return;
+                }
+                const person = masks[personMaskIndex(labels, masks.length)];
+                // The mask is only valid inside this callback, so it is copied out.
+                mask = { width: person.width, height: person.height, confidence: new Float32Array(person.getAsFloat32Array()) };
+            });
+            return mask;
+        },
+    };
+}
+
+async function loadFaceTracker(assetsUrl?: string): Promise<FaceTracker> {
+    const urls = modelUrls(assetsUrl);
+    const vision: Vision = await import("@mediapipe/tasks-vision");
+    const fileset = await vision.FilesetResolver.forVisionTasks(urls.wasm);
+    const landmarker = await withFallback((delegate) =>
+        vision.FaceLandmarker.createFromOptions(fileset, {
+            baseOptions: { modelAssetPath: urls.face, delegate },
+            runningMode: "VIDEO",
+            numFaces: 1,
+        }),
+    );
+    const stamp = monotonic();
+    return {
+        detect(frame, timestampMs) {
+            return landmarker.detectForVideo(frame, stamp(timestampMs)).faceLandmarks[0] ?? null;
+        },
+    };
+}
+
+const segmenters = new Map<string, Promise<PersonSegmenter>>();
+const faceTrackers = new Map<string, Promise<FaceTracker>>();
+
+/** Loads `load` once per key, and forgets a failure so the next attempt (a retry) tries again. */
+function shared<T>(cache: Map<string, Promise<T>>, key: string, load: () => Promise<T>): Promise<T> {
+    let pending = cache.get(key);
+    if (!pending) {
+        pending = load();
+        cache.set(key, pending);
+        pending.catch(() => cache.delete(key));
+    }
+    return pending;
+}
+
+/** The person-segmentation model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export function createPersonSegmenter(assetsUrl?: string): Promise<PersonSegmenter> {
+    return shared(segmenters, assetsUrl ?? "", () => loadSegmenter(assetsUrl));
+}
+
+/** The face-landmark model, loaded on first use and shared from then on. Rejects if it can't be loaded. */
+export function createFaceTracker(assetsUrl?: string): Promise<FaceTracker> {
+    return shared(faceTrackers, assetsUrl ?? "", () => loadFaceTracker(assetsUrl));
+}
+
+/** Forgets every loaded model - for tests. */
+export function resetModelCache(): void {
+    segmenters.clear();
+    faceTrackers.clear();
+}

package/apps/shared/media/filters/pixelEffects.ts

@@ -0,0 +1,61 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * The colour effects, as plain functions over an RGBA pixel buffer (`ImageData.data`) so they need no canvas API
+ * beyond `getImageData`/`putImageData`. They are done here, one pixel at a time, rather than with the canvas
+ * `filter` property, which Safari does not support.
+ *
+ * Each function edits `data` in place and leaves the alpha channel alone.
+ */
+
+/** Rec. 601 luma - how bright a colour looks. */
+function luma(r: number, g: number, b: number): number {
+    return 0.299 * r + 0.587 * g + 0.114 * b;
+}
+
+export function applyGrayscale(data: Uint8ClampedArray): void {
+    for (let i = 0; i < data.length; i += 4) {
+        const y = luma(data[i], data[i + 1], data[i + 2]);
+        data[i] = y;
+        data[i + 1] = y;
+        data[i + 2] = y;
+    }
+}
+
+export function applySepia(data: Uint8ClampedArray): void {
+    for (let i = 0; i < data.length; i += 4) {
+        const r = data[i];
+        const g = data[i + 1];
+        const b = data[i + 2];
+        // `Uint8ClampedArray` clamps to 0-255 on assignment.
+        data[i] = 0.393 * r + 0.769 * g + 0.189 * b;
+        data[i + 1] = 0.349 * r + 0.686 * g + 0.168 * b;
+        data[i + 2] = 0.272 * r + 0.534 * g + 0.131 * b;
+    }
+}
+
+/** A brightened, green-tinted picture, like a night-vision scope. */
+export function applyNightVision(data: Uint8ClampedArray): void {
+    for (let i = 0; i < data.length; i += 4) {
+        const y = luma(data[i], data[i + 1], data[i + 2]) * 1.4;
+        data[i] = y * 0.25;
+        data[i + 1] = y;
+        data[i + 2] = y * 0.25;
+    }
+}
+
+/** Below this confidence a pixel is background; above `MASK_HIGH` it is the person; in between it fades, so the edge of
+ * the person is soft rather than jagged. */
+const MASK_LOW = 0.3;
+const MASK_HIGH = 0.7;
+
+/** Turns a person-confidence mask (0-1 per pixel) into the alpha channel of an RGBA buffer, sharpening the edge with a
+ * smoothstep. `rgba` must hold `4 * confidence.length` values. */
+export function maskToAlpha(confidence: Float32Array, rgba: Uint8ClampedArray): void {
+    for (let i = 0; i < confidence.length; i++) {
+        const t = Math.min(1, Math.max(0, (confidence[i] - MASK_LOW) / (MASK_HIGH - MASK_LOW)));
+        rgba[i * 4 + 3] = t * t * (3 - 2 * t) * 255;
+    }
+}