@rapidmx/meet-plugin 0.5.0 → 0.8.0

package/dist/apps/shared/media/filters/VideoFilterProcessor.js

@@ -0,0 +1,304 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (C) 2026 Jean-Philippe Steinmetz. All rights reserved.
+// SPDX-License-Identifier: MPL-2.0
+///////////////////////////////////////////////////////////////////////////////
+/**
+ * Turns the camera's `MediaStreamTrack` into a *filtered* one: frames from the camera are drawn onto a canvas, put
+ * through the chosen filters, and the canvas's `captureStream()` track is what the rest of the app sends to the call.
+ * `useLocalMedia()` owns one of these while any filter is on and swaps it in for the camera track, so the lobby
+ * preview, the peer connections and the WebSocket relay all get the filtered picture without knowing about it.
+ *
+ * Per frame: the background (untouched, or blurred / replaced behind the person-segmentation mask), then the colour
+ * effect over the whole picture, then the face accessory on top - see `filterTypes.ts`.
+ *
+ * - **Fails closed.** A participant who asked for a hidden background must never be shown with their real one - not
+ * while the segmentation model is still loading, and not if it can't be loaded. Until the mask is available the frame
+ * is the background alone (the blurred or replacement picture, no person), and `onStatus` reports the load and any
+ * failure so the UI can say why. Accessories don't matter that way: without the face model there simply is no accessory.
+ * - **Timing.** Frames are driven by a timer rather than `requestAnimationFrame`, which stops entirely in a
+ * background tab - the call would freeze for everyone else the moment the participant switched to another window.
+ * Browsers still slow a background tab's timers, so a filtered picture can drop to a low frame rate there.
+ *
+ * Everything browser-facing (canvases, the video element, the models) comes in through `deps`, so the tests drive a
+ * processor with fakes.
+ */
+import { drawAccessory, faceGeometry } from "./accessories.js";
+import { needsFace, needsSegmentation } from "./filterTypes.js";
+import { createFaceTracker, createPersonSegmenter } from "./mlModels.js";
+import { applyGrayscale, applyNightVision, applySepia, maskToAlpha } from "./pixelEffects.js";
+const DEFAULT_FRAME_RATE = 30;
+/** The background is blurred by drawing the frame at 1/this size and scaling it back up. */
+const BLUR_DIVISOR = 12;
+/** The picture is pixelated into about this many blocks across. */
+const PIXEL_BLOCKS = 80;
+/** How many frames in a row without a detected face before the accessory is taken off - a face is briefly lost when
+ * the head turns, and the accessory shouldn't flicker for that. */
+const FACE_HOLD_FRAMES = 6;
+const SEGMENTATION_ERROR = "The background effect couldn't be loaded, so your background stays hidden. Turn the effect off to show your camera.";
+const FACE_ERROR = "The face effect couldn't be loaded.";
+const RENDER_ERROR = "Video effects stopped working.";
+const PIXEL_EFFECTS = { bw: applyGrayscale, sepia: applySepia, "night-vision": applyNightVision };
+const DEFAULT_DEPS = {
+    createCanvas: () => document.createElement("canvas"),
+    createVideo: () => document.createElement("video"),
+    createSegmenter: createPersonSegmenter,
+    createFaceTracker,
+    now: () => performance.now(),
+};
+function context2d(canvas) {
+    const ctx = canvas.getContext("2d", { willReadFrequently: true });
+    if (!ctx) {
+        throw new Error("This browser can't draw video effects.");
+    }
+    return ctx;
+}
+export class VideoFilterProcessor {
+    constructor(options) {
+        this.layers = new Map();
+        this.stopped = false;
+        this.segmenter = null;
+        this.segmenterState = "idle";
+        this.faceTracker = null;
+        this.faceState = "idle";
+        this.lastFace = null;
+        this.framesWithoutFace = 0;
+        this.maskImage = null;
+        this.renderFailed = false;
+        this.status = { loading: false, error: null };
+        this.tick = () => {
+            if (this.stopped) {
+                return;
+            }
+            const started = this.deps.now();
+            if (this.video.readyState >= 2 && this.video.videoWidth > 0) {
+                try {
+                    this.render(started);
+                    if (this.renderFailed) {
+                        // Drawing works again - a single bad frame doesn't leave the warning up.
+                        this.renderFailed = false;
+                        this.publishStatus();
+                    }
+                }
+                catch {
+                    // A frame that can't be drawn is skipped; the loop carries on, and the participant is told.
+                    if (!this.renderFailed) {
+                        this.renderFailed = true;
+                        this.publishStatus();
+                    }
+                }
+            }
+            this.timer = setTimeout(this.tick, Math.max(0, this.interval - (this.deps.now() - started)));
+        };
+        this.deps = { ...DEFAULT_DEPS, ...options.deps };
+        this.filters = options.filters;
+        this.backgroundImage = options.backgroundImage ?? null;
+        this.assetsUrl = options.assetsUrl;
+        this.onStatus = options.onStatus;
+        const frameRate = options.frameRate ?? DEFAULT_FRAME_RATE;
+        this.interval = 1000 / frameRate;
+        this.canvas = this.deps.createCanvas();
+        this.ctx = context2d(this.canvas);
+        this.track = this.canvas.captureStream(frameRate).getVideoTracks()[0];
+        this.video = this.deps.createVideo();
+        this.video.muted = true;
+        this.video.playsInline = true;
+        this.video.srcObject = new MediaStream([options.source]);
+        void this.video.play().catch(() => undefined);
+        this.syncModels();
+        this.publishStatus();
+        this.tick();
+    }
+    /** Changes the filters (and the background picture) without interrupting the video. Also retries a model that failed
+     * to load, so choosing a filter again is a retry. */
+    update(filters, backgroundImage) {
+        this.filters = filters;
+        this.backgroundImage = backgroundImage;
+        this.syncModels(true);
+        this.publishStatus();
+    }
+    stop() {
+        this.stopped = true;
+        clearTimeout(this.timer);
+        this.video.pause();
+        this.video.srcObject = null;
+        this.track.stop();
+    }
+    syncModels(retry = false) {
+        if (needsSegmentation(this.filters) && (this.segmenterState === "idle" || (retry && this.segmenterState === "failed"))) {
+            this.segmenterState = "loading";
+            this.deps.createSegmenter(this.assetsUrl).then((segmenter) => {
+                this.segmenter = segmenter;
+                this.segmenterState = "ready";
+                this.publishStatus();
+            }, () => {
+                this.segmenterState = "failed";
+                this.publishStatus();
+            });
+        }
+        if (needsFace(this.filters) && (this.faceState === "idle" || (retry && this.faceState === "failed"))) {
+            this.faceState = "loading";
+            this.deps.createFaceTracker(this.assetsUrl).then((tracker) => {
+                this.faceTracker = tracker;
+                this.faceState = "ready";
+                this.publishStatus();
+            }, () => {
+                this.faceState = "failed";
+                this.publishStatus();
+            });
+        }
+    }
+    publishStatus() {
+        if (this.stopped) {
+            return;
+        }
+        const wantsSegmentation = needsSegmentation(this.filters);
+        const wantsFace = needsFace(this.filters);
+        let error = null;
+        if (wantsSegmentation && this.segmenterState === "failed") {
+            error = SEGMENTATION_ERROR;
+        }
+        else if (wantsFace && this.faceState === "failed") {
+            error = FACE_ERROR;
+        }
+        else if (this.renderFailed) {
+            error = RENDER_ERROR;
+        }
+        const loading = (wantsSegmentation && this.segmenterState === "loading") || (wantsFace && this.faceState === "loading");
+        if (loading !== this.status.loading || error !== this.status.error) {
+            this.status = { loading, error };
+            this.onStatus?.(this.status);
+        }
+    }
+    /** A scratch canvas of the given size, kept between frames. */
+    layer(name, width, height) {
+        let layer = this.layers.get(name);
+        if (!layer) {
+            const canvas = this.deps.createCanvas();
+            layer = { canvas, ctx: context2d(canvas) };
+            this.layers.set(name, layer);
+        }
+        if (layer.canvas.width !== width || layer.canvas.height !== height) {
+            layer.canvas.width = width;
+            layer.canvas.height = height;
+        }
+        return layer;
+    }
+    render(now) {
+        const { video, canvas, ctx, filters } = this;
+        const width = video.videoWidth;
+        const height = video.videoHeight;
+        if (canvas.width !== width || canvas.height !== height) {
+            canvas.width = width;
+            canvas.height = height;
+        }
+        if (filters.background === "none") {
+            ctx.drawImage(video, 0, 0, width, height);
+        }
+        else {
+            this.drawBackground(now, width, height);
+        }
+        this.applyEffect(width, height);
+        if (filters.accessory !== "none") {
+            this.paintAccessory(filters.accessory, now, width, height);
+        }
+    }
+    /** The background, then - once the segmentation model is ready - the participant on top of it. */
+    drawBackground(now, width, height) {
+        const { video, ctx } = this;
+        const picture = this.backgroundImage;
+        if (this.filters.background === "image" && picture) {
+            const scale = Math.max(width / picture.width, height / picture.height);
+            const drawnWidth = picture.width * scale;
+            const drawnHeight = picture.height * scale;
+            ctx.drawImage(picture.source, (width - drawnWidth) / 2, (height - drawnHeight) / 2, drawnWidth, drawnHeight);
+        }
+        else {
+            // Also the stand-in for a "custom image" background that has no image: blurred, never the real room.
+            const small = this.layer("blur", Math.ceil(width / BLUR_DIVISOR), Math.ceil(height / BLUR_DIVISOR));
+            small.ctx.imageSmoothingQuality = "high";
+            small.ctx.drawImage(video, 0, 0, small.canvas.width, small.canvas.height);
+            ctx.imageSmoothingQuality = "high";
+            ctx.drawImage(small.canvas, 0, 0, width, height);
+        }
+        const mask = this.segment(now);
+        if (!mask) {
+            return;
+        }
+        const maskLayer = this.layer("mask", mask.width, mask.height);
+        let image = this.maskImage;
+        if (!image || image.width !== mask.width || image.height !== mask.height) {
+            image = this.maskImage = maskLayer.ctx.createImageData(mask.width, mask.height);
+        }
+        maskToAlpha(mask.confidence, image.data);
+        maskLayer.ctx.putImageData(image, 0, 0);
+        const person = this.layer("person", width, height);
+        person.ctx.globalCompositeOperation = "source-over";
+        person.ctx.clearRect(0, 0, width, height);
+        person.ctx.drawImage(maskLayer.canvas, 0, 0, width, height);
+        person.ctx.globalCompositeOperation = "source-in";
+        person.ctx.drawImage(video, 0, 0, width, height);
+        person.ctx.globalCompositeOperation = "source-over";
+        ctx.drawImage(person.canvas, 0, 0);
+    }
+    segment(now) {
+        if (!this.segmenter) {
+            return null;
+        }
+        try {
+            return this.segmenter.segment(this.video, now);
+        }
+        catch {
+            this.segmenter = null;
+            this.segmenterState = "failed";
+            this.publishStatus();
+            return null;
+        }
+    }
+    applyEffect(width, height) {
+        const { ctx } = this;
+        const effect = this.filters.effect;
+        if (effect === "pixelate") {
+            const small = this.layer("pixel", PIXEL_BLOCKS, Math.ceil((height / width) * PIXEL_BLOCKS));
+            small.ctx.drawImage(this.canvas, 0, 0, small.canvas.width, small.canvas.height);
+            ctx.imageSmoothingEnabled = false;
+            ctx.drawImage(small.canvas, 0, 0, width, height);
+            ctx.imageSmoothingEnabled = true;
+        }
+        else if (effect !== "none") {
+            const pixels = ctx.getImageData(0, 0, width, height);
+            PIXEL_EFFECTS[effect](pixels.data);
+            ctx.putImageData(pixels, 0, 0);
+        }
+    }
+    paintAccessory(accessory, now, width, height) {
+        const face = this.trackFace(now);
+        const geometry = face ? faceGeometry(face, width, height) : null;
+        if (geometry) {
+            drawAccessory(this.ctx, accessory, geometry);
+        }
+    }
+    /** The face's landmarks for this frame - the last known ones for a few frames after it is lost. */
+    trackFace(now) {
+        if (!this.faceTracker) {
+            return null;
+        }
+        let found = null;
+        try {
+            found = this.faceTracker.detect(this.video, now);
+        }
+        catch {
+            this.faceTracker = null;
+            this.faceState = "failed";
+            this.publishStatus();
+            return null;
+        }
+        if (found) {
+            this.lastFace = found;
+            this.framesWithoutFace = 0;
+        }
+        else if (++this.framesWithoutFace > FACE_HOLD_FRAMES) {
+            this.lastFace = null;
+        }
+        return this.lastFace;
+    }
+}

package/dist/apps/shared/media/filters/accessories.d.ts

@@ -0,0 +1,37 @@
+/**
+ * The face accessories - sunglasses, cat ears, a party hat, a crown, a mustache - drawn as vector paths onto the
+ * filtered frame, so the plugin ships no image assets at all.
+ *
+ * Each one is drawn in the face's own coordinate frame: the origin sits on a landmark (between the eyes, the top of
+ * the forehead, the nose), the x axis runs along the line between the eyes so the accessory tilts with the head, and
+ * one unit is the distance between the outer corners of the eyes so it grows and shrinks with the head. That keeps
+ * every drawing routine below a few plain numbers in "face widths".
+ */
+import type { Accessory } from "./filterTypes.js";
+/** A landmark as the face model reports it: 0-1 across and down the frame. */
+export interface FaceLandmark {
+    x: number;
+    y: number;
+}
+export interface Point {
+    x: number;
+    y: number;
+}
+/** Where the parts of a face are, in canvas pixels. */
+export interface FaceGeometry {
+    /** Halfway between the outer corners of the eyes. */
+    eyes: Point;
+    /** The top of the forehead. */
+    forehead: Point;
+    noseTip: Point;
+    upperLip: Point;
+    /** The distance between the outer corners of the eyes - one unit of the accessories' coordinate frame. */
+    unit: number;
+    /** The tilt of the line between the eyes, in radians. */
+    angle: number;
+}
+/** Works out where the parts of a face are, in pixels, from the face mesh's landmarks - `null` when there are not
+ * enough of them to be a face mesh at all. */
+export declare function faceGeometry(landmarks: FaceLandmark[], width: number, height: number): FaceGeometry | null;
+/** Draws `accessory` onto `ctx` for the face at `face`. */
+export declare function drawAccessory(ctx: CanvasRenderingContext2D, accessory: Exclude<Accessory, "none">, face: FaceGeometry): void;

package/dist/apps/shared/media/filters/accessories.js

@@ -0,0 +1,211 @@
+// Indices into the 478-point face mesh (MediaPipe's canonical face model).
+const RIGHT_EYE_OUTER = 33;
+const LEFT_EYE_OUTER = 263;
+const FOREHEAD_TOP = 10;
+const NOSE_TIP = 1;
+const UPPER_LIP = 0;
+const MESH_POINTS = 468;
+/** Works out where the parts of a face are, in pixels, from the face mesh's landmarks - `null` when there are not
+ * enough of them to be a face mesh at all. */
+export function faceGeometry(landmarks, width, height) {
+    if (landmarks.length < MESH_POINTS) {
+        return null;
+    }
+    const at = (index) => ({ x: landmarks[index].x * width, y: landmarks[index].y * height });
+    const right = at(RIGHT_EYE_OUTER);
+    const left = at(LEFT_EYE_OUTER);
+    return {
+        eyes: { x: (right.x + left.x) / 2, y: (right.y + left.y) / 2 },
+        forehead: at(FOREHEAD_TOP),
+        noseTip: at(NOSE_TIP),
+        upperLip: at(UPPER_LIP),
+        unit: Math.hypot(left.x - right.x, left.y - right.y),
+        angle: Math.atan2(left.y - right.y, left.x - right.x),
+    };
+}
+/** Runs `draw` with the canvas moved to `origin`, turned by the face's tilt and scaled to one face width per unit. */
+function inFaceFrame(ctx, origin, face, draw) {
+    ctx.save();
+    ctx.translate(origin.x, origin.y);
+    ctx.rotate(face.angle);
+    ctx.scale(face.unit, face.unit);
+    draw();
+    ctx.restore();
+}
+/** Runs `draw` once as it is and once flipped left-right, for the two matching halves of a symmetrical accessory. */
+function bothSides(ctx, draw) {
+    for (const side of [1, -1]) {
+        ctx.save();
+        ctx.scale(side, 1);
+        draw();
+        ctx.restore();
+    }
+}
+function roundedRect(ctx, x, y, w, h, r) {
+    ctx.beginPath();
+    ctx.moveTo(x + r, y);
+    ctx.arcTo(x + w, y, x + w, y + h, r);
+    ctx.arcTo(x + w, y + h, x, y + h, r);
+    ctx.arcTo(x, y + h, x, y, r);
+    ctx.arcTo(x, y, x + w, y, r);
+    ctx.closePath();
+}
+function drawSunglasses(ctx, face) {
+    inFaceFrame(ctx, face.eyes, face, () => {
+        ctx.fillStyle = "rgba(12, 12, 18, 0.93)";
+        ctx.strokeStyle = "rgba(12, 12, 18, 0.93)";
+        ctx.lineCap = "round";
+        bothSides(ctx, () => {
+            roundedRect(ctx, 0.06, -0.17, 0.54, 0.38, 0.12);
+            ctx.fill();
+            // The arm that runs back over the ear.
+            ctx.lineWidth = 0.04;
+            ctx.beginPath();
+            ctx.moveTo(0.6, -0.06);
+            ctx.lineTo(0.8, -0.1);
+            ctx.stroke();
+            // A glint across the lens.
+            ctx.fillStyle = "rgba(255, 255, 255, 0.2)";
+            ctx.beginPath();
+            ctx.moveTo(0.14, -0.1);
+            ctx.lineTo(0.28, -0.1);
+            ctx.lineTo(0.16, 0.12);
+            ctx.lineTo(0.1, 0.12);
+            ctx.closePath();
+            ctx.fill();
+        });
+        // The bridge between the lenses.
+        ctx.lineWidth = 0.05;
+        ctx.beginPath();
+        ctx.moveTo(-0.08, -0.08);
+        ctx.lineTo(0.08, -0.08);
+        ctx.stroke();
+    });
+}
+function drawCat(ctx, face) {
+    inFaceFrame(ctx, face.forehead, face, () => {
+        bothSides(ctx, () => {
+            ctx.fillStyle = "#3b3b44";
+            ctx.beginPath();
+            ctx.moveTo(0.68, 0.14);
+            ctx.lineTo(0.12, 0);
+            ctx.lineTo(0.62, -0.55);
+            ctx.closePath();
+            ctx.fill();
+            ctx.fillStyle = "#f4a6b8";
+            ctx.beginPath();
+            ctx.moveTo(0.56, 0.06);
+            ctx.lineTo(0.22, 0);
+            ctx.lineTo(0.58, -0.33);
+            ctx.closePath();
+            ctx.fill();
+        });
+    });
+    inFaceFrame(ctx, face.noseTip, face, () => {
+        ctx.fillStyle = "#f06a8a";
+        ctx.beginPath();
+        ctx.moveTo(-0.09, -0.04);
+        ctx.lineTo(0.09, -0.04);
+        ctx.lineTo(0, 0.06);
+        ctx.closePath();
+        ctx.fill();
+        ctx.lineCap = "round";
+        bothSides(ctx, () => {
+            for (const tilt of [-1, 0, 1]) {
+                ctx.beginPath();
+                ctx.moveTo(0.3, 0.04 + tilt * 0.07);
+                ctx.lineTo(0.85, -0.02 + tilt * 0.15);
+                // A dark line under a white one, so the whiskers read against a light or a dark background.
+                ctx.strokeStyle = "rgba(0, 0, 0, 0.45)";
+                ctx.lineWidth = 0.035;
+                ctx.stroke();
+                ctx.strokeStyle = "#ffffff";
+                ctx.lineWidth = 0.018;
+                ctx.stroke();
+            }
+        });
+    });
+}
+function drawPartyHat(ctx, face) {
+    inFaceFrame(ctx, face.forehead, face, () => {
+        ctx.save();
+        ctx.beginPath();
+        ctx.moveTo(-0.42, 0.1);
+        ctx.lineTo(0.42, 0.1);
+        ctx.lineTo(0.04, -1.15);
+        ctx.closePath();
+        ctx.clip();
+        ctx.fillStyle = "#ff5fa2";
+        ctx.fillRect(-0.6, -1.3, 1.2, 1.5);
+        ctx.strokeStyle = "#ffd23f";
+        ctx.lineWidth = 0.09;
+        for (const y of [-0.12, -0.42, -0.72]) {
+            ctx.beginPath();
+            ctx.moveTo(-0.6, y + 0.2);
+            ctx.lineTo(0.6, y - 0.2);
+            ctx.stroke();
+        }
+        ctx.restore();
+        ctx.fillStyle = "#ffffff";
+        ctx.beginPath();
+        ctx.arc(0.04, -1.15, 0.09, 0, Math.PI * 2);
+        ctx.fill();
+    });
+}
+function drawCrown(ctx, face) {
+    inFaceFrame(ctx, face.forehead, face, () => {
+        ctx.beginPath();
+        ctx.moveTo(-0.5, 0.1);
+        ctx.lineTo(-0.5, -0.45);
+        ctx.lineTo(-0.25, -0.2);
+        ctx.lineTo(0, -0.55);
+        ctx.lineTo(0.25, -0.2);
+        ctx.lineTo(0.5, -0.45);
+        ctx.lineTo(0.5, 0.1);
+        ctx.closePath();
+        ctx.fillStyle = "#f5c518";
+        ctx.fill();
+        ctx.strokeStyle = "#b8860b";
+        ctx.lineWidth = 0.03;
+        ctx.lineJoin = "round";
+        ctx.stroke();
+        for (const [x, y, color] of [
+            [-0.5, -0.45, "#e63946"],
+            [0, -0.55, "#3a86ff"],
+            [0.5, -0.45, "#e63946"],
+            [0, -0.05, "#2a9d8f"],
+        ]) {
+            ctx.fillStyle = color;
+            ctx.beginPath();
+            ctx.arc(x, y, 0.05, 0, Math.PI * 2);
+            ctx.fill();
+        }
+    });
+}
+function drawMustache(ctx, face) {
+    const between = { x: (face.noseTip.x + face.upperLip.x) / 2, y: (face.noseTip.y + face.upperLip.y) / 2 };
+    inFaceFrame(ctx, between, face, () => {
+        ctx.fillStyle = "#3a2618";
+        ctx.beginPath();
+        ctx.moveTo(0, -0.03);
+        ctx.bezierCurveTo(-0.12, -0.12, -0.32, -0.1, -0.42, 0.02);
+        ctx.bezierCurveTo(-0.46, 0.07, -0.4, 0.13, -0.34, 0.09);
+        ctx.bezierCurveTo(-0.26, 0.04, -0.12, 0.06, 0, 0.09);
+        ctx.bezierCurveTo(0.12, 0.06, 0.26, 0.04, 0.34, 0.09);
+        ctx.bezierCurveTo(0.4, 0.13, 0.46, 0.07, 0.42, 0.02);
+        ctx.bezierCurveTo(0.32, -0.1, 0.12, -0.12, 0, -0.03);
+        ctx.closePath();
+        ctx.fill();
+    });
+}
+const DRAWERS = {
+    sunglasses: drawSunglasses,
+    "cat-ears": drawCat,
+    "party-hat": drawPartyHat,
+    crown: drawCrown,
+    mustache: drawMustache,
+};
+/** Draws `accessory` onto `ctx` for the face at `face`. */
+export function drawAccessory(ctx, accessory, face) {
+    DRAWERS[accessory](ctx, face);
+}

package/dist/apps/shared/media/filters/backgroundImage.d.ts

@@ -0,0 +1,22 @@
+/**
+ * The picture behind a participant for the "custom image" background: read from a file they pick on their own
+ * device, shrunk to what a video call can show anyway, and kept as a small JPEG data URL so it can be remembered
+ * (`mediaPreferences.ts`) without filling the browser's storage. The file never leaves the browser.
+ */
+import type { BackgroundPicture } from "./VideoFilterProcessor.js";
+/** A bigger file than this is refused before the browser tries to decode it. */
+export declare const MAX_BACKGROUND_FILE_BYTES: number;
+/** The longest edge kept - a call's video is 1280 wide, so more would only cost storage. */
+export declare const MAX_BACKGROUND_EDGE = 1280;
+export type PreparedBackground = {
+    ok: true;
+    dataUrl: string;
+    picture: BackgroundPicture;
+} | {
+    ok: false;
+    message: string;
+};
+/** Decodes an image (a URL or data URL) into something that can be drawn. Rejects if it isn't a picture. */
+export declare function loadPicture(src: string): Promise<BackgroundPicture>;
+/** Reads a picked file into a background: validated, downscaled and ready both to draw and to store. */
+export declare function prepareBackgroundImage(file: File): Promise<PreparedBackground>;

package/dist/apps/shared/media/filters/backgroundImage.js

@@ -0,0 +1,54 @@
+/** A bigger file than this is refused before the browser tries to decode it. */
+export const MAX_BACKGROUND_FILE_BYTES = 20 * 1024 * 1024;
+/** The longest edge kept - a call's video is 1280 wide, so more would only cost storage. */
+export const MAX_BACKGROUND_EDGE = 1280;
+const JPEG_QUALITY = 0.85;
+/** Decodes an image (a URL or data URL) into something that can be drawn. Rejects if it isn't a picture. */
+export function loadPicture(src) {
+    return new Promise((resolve, reject) => {
+        const image = new Image();
+        image.onload = () => resolve({ source: image, width: image.naturalWidth, height: image.naturalHeight });
+        image.onerror = () => reject(new Error("The image could not be read."));
+        image.src = src;
+    });
+}
+function readAsDataUrl(file) {
+    return new Promise((resolve, reject) => {
+        const reader = new FileReader();
+        reader.onload = () => resolve(String(reader.result));
+        reader.onerror = () => reject(new Error("The file could not be read."));
+        reader.readAsDataURL(file);
+    });
+}
+/** Reads a picked file into a background: validated, downscaled and ready both to draw and to store. */
+export async function prepareBackgroundImage(file) {
+    if (!file.type.startsWith("image/")) {
+        return { ok: false, message: "Choose an image file (a JPEG, PNG or similar)." };
+    }
+    if (file.size > MAX_BACKGROUND_FILE_BYTES) {
+        return { ok: false, message: "That image is too large. Choose one under 20 MB." };
+    }
+    try {
+        const original = await loadPicture(await readAsDataUrl(file));
+        const scale = Math.min(1, MAX_BACKGROUND_EDGE / Math.max(original.width, original.height));
+        const canvas = document.createElement("canvas");
+        canvas.width = Math.max(1, Math.round(original.width * scale));
+        canvas.height = Math.max(1, Math.round(original.height * scale));
+        const ctx = canvas.getContext("2d");
+        if (!ctx) {
+            return { ok: false, message: "This browser can't prepare that image." };
+        }
+        // JPEG has no transparency: a transparent PNG would otherwise turn black.
+        ctx.fillStyle = "#ffffff";
+        ctx.fillRect(0, 0, canvas.width, canvas.height);
+        ctx.drawImage(original.source, 0, 0, canvas.width, canvas.height);
+        return {
+            ok: true,
+            dataUrl: canvas.toDataURL("image/jpeg", JPEG_QUALITY),
+            picture: { source: canvas, width: canvas.width, height: canvas.height },
+        };
+    }
+    catch {
+        return { ok: false, message: "That file couldn't be opened as an image." };
+    }
+}

package/dist/apps/shared/media/filters/filterTypes.d.ts

@@ -0,0 +1,40 @@
+/**
+ * The video filters a participant can put on their camera. They come in three independent layers, so they combine
+ * (blurred background *and* black and white *and* sunglasses):
+ *
+ * - **Background**: what is behind the participant - untouched, blurred, or a picture they chose. Needs person
+ * segmentation (`mlModels.ts`).
+ * - **Effect**: a look applied to the whole picture - black and white, sepia, night vision, pixelated.
+ * - **Accessory**: something drawn on the participant's face - sunglasses, cat ears, a party hat. Needs face landmarks
+ * (`mlModels.ts`).
+ *
+ * Adding a filter is: add its id here (with a label), then implement it in `VideoFilterProcessor.ts` (an effect), or
+ * in `accessories.ts` (an accessory).
+ */
+export type BackgroundFilter = "none" | "blur" | "image";
+export type ColorEffect = "none" | "bw" | "sepia" | "night-vision" | "pixelate";
+export type Accessory = "none" | "sunglasses" | "cat-ears" | "party-hat" | "crown" | "mustache";
+export interface VideoFilters {
+    background: BackgroundFilter;
+    effect: ColorEffect;
+    accessory: Accessory;
+}
+export declare const NO_FILTERS: VideoFilters;
+export interface FilterOption<T extends string> {
+    id: T;
+    label: string;
+    /** A glyph shown beside the label, purely decorative. */
+    emoji: string;
+}
+export declare const BACKGROUND_OPTIONS: FilterOption<BackgroundFilter>[];
+export declare const EFFECT_OPTIONS: FilterOption<ColorEffect>[];
+export declare const ACCESSORY_OPTIONS: FilterOption<Accessory>[];
+/** Whether any filter is on - when none is, the camera's own track is sent untouched and nothing is processed. */
+export declare function filtersActive(filters: VideoFilters): boolean;
+/** Whether the filters need the person-segmentation model. */
+export declare function needsSegmentation(filters: VideoFilters): boolean;
+/** Whether the filters need the face-landmark model. */
+export declare function needsFace(filters: VideoFilters): boolean;
+/** Turns whatever was read back from storage into valid filters - anything unrecognised (a filter since removed, a
+ * hand-edited value, another version's shape) falls back to "none" rather than breaking the call. */
+export declare function sanitizeFilters(value: unknown): VideoFilters;