@voltkit-rtc/client 0.8.0 → 0.9.1

package/src/video-effects.ts

@@ -0,0 +1,552 @@
+/**
+ * Virtual background and beauty for the camera (docs/implementation-progress/RTC_PERF_PROGRESS.md P3-5): a blurred or
+ * replaced background and skin smoothing, as LiveKit's track processors, Agora's and ZEGOCLOUD's virtual background and
+ * beauty extensions, and Tencent TRTC's video effect plugin do -- in the browser, on the GPU.
+ *
+ * Imported on its own (`@voltkit-rtc/client/video-effects`), so an app that does not use it never ships it:
+ *
+ *   import { createVideoEffects } from '@voltkit-rtc/client/video-effects';
+ *   const effects = createVideoEffects({ background: 'blur', beauty: true });
+ *   await client.setVideoProcessor(effects);
+ *   await effects.update({ background: { image: '/office.jpg' } });   // live, no pause for viewers
+ *
+ * Finding the person takes Google's MediaPipe selfie segmenter: install `@mediapipe/tasks-vision@0.10.35` next to the
+ * SDK (an optional peer dependency, 0.10.35 exactly -- later versions report usage to Google). Its wasm (3 MB
+ * compressed) and the model (250 kB) load the first time a background effect is used: the wasm from jsDelivr at that
+ * version and the model from Google, unless you serve them yourself (`wasmPath`, `modelUrl`). Beauty alone needs none
+ * of it.
+ *
+ * Chrome and Edge process each camera frame as it arrives (insertable streams: smooth, and it keeps going in a
+ * background tab). Firefox and Safari draw from a hidden video on a worker's timer into a captured canvas -- what
+ * LiveKit, Twilio and Amazon Chime fall back to -- which a background tab can slow down.
+ */
+import {
+  backgroundMode,
+  beautySettings,
+  blurLevel,
+  blurSigma,
+  coverFit,
+  MonotonicClock,
+  SegmentationPacer,
+  segmentationSize,
+  smoothMask,
+  type BackgroundImageSource,
+  type BackgroundSetting,
+  type BeautyOptions,
+  type BeautySettings,
+} from './effects-core.js';
+import { EffectsRenderer } from './effects-gl.js';
+import type { VideoTrackProcessor } from './processors.js';
+
+export type { BackgroundImageSource, BackgroundSetting, BeautyOptions, BeautySettings } from './effects-core.js';
+
+/** The `@mediapipe/tasks-vision` version this SDK was tested with; the default wasm URL points at it. */
+export const MEDIAPIPE_VERSION = '0.10.35';
+const MEDIAPIPE_WASM = `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${MEDIAPIPE_VERSION}/wasm`;
+/** Google's selfie segmenter for portrait and square pictures (phones), at a fixed version. */
+export const DEFAULT_SEGMENTER_MODEL = 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/1/selfie_segmenter.tflite';
+/**
+ * Its landscape version, for wide pictures (webcams): fed a 16:9 frame, the square model leaves patches of the room
+ * beside the head, the landscape one traces it cleanly (as LiveKit, Twilio, Chime and TRTC use for webcams).
+ */
+export const DEFAULT_LANDSCAPE_MODEL = 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/1/selfie_segmenter_landscape.tflite';
+
+export interface VideoEffectsOptions {
+  /** `'none'` (default), `'blur'`, or `{ image }`: a URL, Blob, image, canvas or ImageBitmap, fitted to cover. */
+  background?: BackgroundSetting;
+  /** Blur strength, 1 (light) to 10 (strong). Default 5. */
+  blur?: number;
+  /** Beauty: `true` for the defaults, or each of `smooth`, `brighten`, `redness`, `sharpen` (0-1). Off by default. */
+  beauty?: BeautyOptions | boolean | null;
+  /** A folder serving `vision_wasm_internal.{js,wasm}` and `vision_wasm_nosimd_internal.{js,wasm}` (from
+   *  `node_modules/@mediapipe/tasks-vision/wasm`). Default: jsDelivr at MEDIAPIPE_VERSION. */
+  wasmPath?: string;
+  /** Where the segmentation model is served, for every picture. Default: DEFAULT_LANDSCAPE_MODEL for wide pictures,
+   *  DEFAULT_SEGMENTER_MODEL for portrait and square ones. */
+  modelUrl?: string;
+  /** The most frames a second to process; the camera's rate by default. */
+  maxFrameRate?: number;
+  /**
+   * Told when this device cannot keep up even finding the person only every few frames (Agora's `onoverload`): turn
+   * the background off, or lower the camera's frame rate or size. Told again with `false` when it can.
+   */
+  onOverload?: (overloaded: boolean) => void;
+}
+
+/** What `update` can change while the effects run. */
+export type VideoEffectsUpdate = Pick<VideoEffectsOptions, 'background' | 'blur' | 'beauty'>;
+
+export interface VideoEffects extends VideoTrackProcessor {
+  /**
+   * Change the effects while they run: no new camera, no pause for viewers. Resolves once the change shows; a picture
+   * that cannot be loaded, or a segmenter that cannot start, rejects and leaves the effects as they were.
+   */
+  update(changes: VideoEffectsUpdate): Promise<void>;
+  /** The effects in force now. */
+  readonly settings: { background: 'none' | 'blur' | 'image'; blur: number; beauty: BeautySettings | null };
+  /** How hard the device works: the person is found every `segmentEvery` frames, in `segmentMs` on average. */
+  readonly stats: { segmentEvery: number; segmentMs: number; overloaded: boolean };
+}
+
+/** What finds the person: MediaPipe's ImageSegmenter, or a stand-in in tests. */
+export interface PersonSegmenter {
+  /** Find the person in `input`; `use` gets the confidence (0-1) of each pixel being the person, at `input`'s size. */
+  segment(input: TexImageSource, timestampMs: number, use: (confidence: Float32Array) => void): void;
+  close(): void;
+}
+
+/** Stood in for by tests. */
+export interface VideoEffectsDeps {
+  loadSegmenter(wasmPath: string, modelUrl: string): Promise<PersonSegmenter>;
+  /** How frames are taken and given back; chosen from the browser's features when absent. */
+  pump?: 'insertable' | 'canvas';
+}
+
+/** Whether this browser can run the effects (WebGL2, and a way to take and give back frames). */
+export function isVideoEffectsSupported(): boolean {
+  if (typeof document === 'undefined' || typeof WebGL2RenderingContext === 'undefined') return false;
+  if (pumpKind() === null) return false;
+  try {
+    const gl = document.createElement('canvas').getContext('webgl2');
+    if (!gl) return false;
+    gl.getExtension('WEBGL_lose_context')?.loseContext();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+type FrameGenerator = MediaStreamTrack & { writable: WritableStream<VideoFrame> };
+type InsertableGlobals = {
+  MediaStreamTrackProcessor?: new (init: { track: MediaStreamTrack }) => { readable: ReadableStream<VideoFrame> };
+  MediaStreamTrackGenerator?: new (init: { kind: 'video' }) => FrameGenerator;
+};
+
+function pumpKind(): 'insertable' | 'canvas' | null {
+  const g = globalThis as unknown as InsertableGlobals;
+  if (typeof g.MediaStreamTrackProcessor === 'function' && typeof g.MediaStreamTrackGenerator === 'function' && typeof VideoFrame === 'function') return 'insertable';
+  if (typeof HTMLCanvasElement !== 'undefined' && typeof HTMLCanvasElement.prototype.captureStream === 'function') return 'canvas';
+  return null;
+}
+
+/** MediaPipe's selfie segmenter, on the GPU where it starts, else the CPU. */
+async function loadMediaPipe(wasmPath: string, modelUrl: string): Promise<PersonSegmenter> {
+  const vision = await import('@mediapipe/tasks-vision');
+  const fileset = await vision.FilesetResolver.forVisionTasks(wasmPath);
+  const make = (delegate: 'GPU' | 'CPU') => vision.ImageSegmenter.createFromOptions(fileset, {
+    baseOptions: { modelAssetPath: modelUrl, delegate },
+    runningMode: 'VIDEO',
+    outputConfidenceMasks: true,
+    outputCategoryMask: false,
+  });
+  let segmenter: Awaited<ReturnType<typeof make>>;
+  try {
+    segmenter = await make('GPU');
+  } catch {
+    segmenter = await make('CPU');
+  }
+  return {
+    segment(input, timestampMs, use) {
+      segmenter.segmentForVideo(input, timestampMs, (result) => {
+        // The selfie segmenter's one mask: how sure it is that each pixel is the person. Only valid in here.
+        const mask = result.confidenceMasks?.[0];
+        if (mask) use(mask.getAsFloat32Array());
+      });
+    },
+    close() {
+      segmenter.close();
+    },
+  };
+}
+
+const browserDeps: VideoEffectsDeps = { loadSegmenter: loadMediaPipe };
+
+/** The largest side a background picture is kept at, to spare the GPU's memory. */
+const PICTURE_MAX = 1920;
+
+interface Picture {
+  image: TexImageSource;
+  width: number;
+  height: number;
+  close(): void;
+}
+
+async function loadPicture(source: BackgroundImageSource): Promise<Picture> {
+  let input: ImageBitmapSource;
+  if (typeof source === 'string') {
+    const img = new Image();
+    img.crossOrigin = 'anonymous';
+    img.decoding = 'async';
+    img.src = source;
+    await img.decode();
+    input = img;
+  } else {
+    input = source as ImageBitmapSource;
+  }
+  let bitmap = await createImageBitmap(input);
+  const longest = Math.max(bitmap.width, bitmap.height);
+  if (longest > PICTURE_MAX) {
+    const scale = PICTURE_MAX / longest;
+    const smaller = await createImageBitmap(bitmap, { resizeWidth: Math.round(bitmap.width * scale), resizeHeight: Math.round(bitmap.height * scale), resizeQuality: 'high' });
+    bitmap.close();
+    bitmap = smaller;
+  }
+  const b = bitmap;
+  return { image: b, width: b.width, height: b.height, close: () => b.close() };
+}
+
+/** One camera track being processed. */
+interface Pipeline {
+  output: MediaStreamTrack;
+  stop(): void;
+}
+
+type Draw = (input: TexImageSource & CanvasImageSource, width: number, height: number, frameIntervalMs: number) => void;
+
+/** Chrome and Edge: every camera frame as it arrives, out through a generator track; no timer, no page rendering. */
+function insertablePump(source: MediaStreamTrack, draw: Draw, canvas: HTMLCanvasElement | OffscreenCanvas, frameIntervalMs: number, minGapUs: number): Pipeline {
+  const g = globalThis as unknown as Required<InsertableGlobals>;
+  const reader = new g.MediaStreamTrackProcessor({ track: source }).readable.getReader();
+  const generator = new g.MediaStreamTrackGenerator({ kind: 'video' });
+  const writer = generator.writable.getWriter();
+  let stopped = false;
+  let lastOut = Number.NEGATIVE_INFINITY;
+  let failures = 0;
+  const stop = () => {
+    if (stopped) return;
+    stopped = true;
+    void reader.cancel().catch(() => undefined);
+    void writer.close().catch(() => undefined);
+    generator.stop();
+  };
+  void (async () => {
+    try {
+      for (;;) {
+        const { value: frame, done } = await reader.read();
+        if (done || !frame) return;
+        if (stopped || generator.readyState === 'ended') {
+          frame.close();
+          return;
+        }
+        // Paused (the camera is off), or sooner than maxFrameRate allows: nothing to do for this one.
+        if (!generator.enabled || frame.timestamp - lastOut < minGapUs) {
+          frame.close();
+          continue;
+        }
+        let out: VideoFrame | null = null;
+        try {
+          draw(frame, frame.displayWidth, frame.displayHeight, frameIntervalMs);
+          out = new VideoFrame(canvas, { timestamp: frame.timestamp });
+          lastOut = frame.timestamp;
+          failures = 0;
+        } catch (err) {
+          // A frame that cannot be drawn is dropped, never sent as it is: the background may be what is hidden.
+          if ((failures += 1) === 1) console.warn('video effects: a frame could not be processed', err);
+        } finally {
+          frame.close();
+        }
+        if (out) await writer.write(out);
+      }
+    } catch {
+      // The output was stopped, or the camera went.
+    } finally {
+      stop();
+    }
+  })();
+  return { output: generator, stop };
+}
+
+/** A timer that keeps time in a background tab: a worker's (LiveKit's FrameTicker), the page's if a CSP refuses it. */
+function startTicker(ms: number, tick: () => void): () => void {
+  let interval: ReturnType<typeof setInterval> | null = null;
+  const onPage = () => {
+    if (interval === null) interval = setInterval(tick, ms);
+  };
+  let worker: Worker | null = null;
+  let url: string | null = null;
+  try {
+    url = URL.createObjectURL(new Blob(['let t=null;onmessage=(e)=>{clearInterval(t);t=e.data>0?setInterval(()=>postMessage(0),e.data):null;};'], { type: 'text/javascript' }));
+    worker = new Worker(url);
+    worker.onmessage = () => tick();
+    worker.onerror = () => {
+      worker?.terminate();
+      worker = null;
+      onPage();
+    };
+    worker.postMessage(ms);
+  } catch {
+    worker = null;
+    onPage();
+  }
+  return () => {
+    worker?.terminate();
+    worker = null;
+    if (interval !== null) clearInterval(interval);
+    interval = null;
+    if (url) URL.revokeObjectURL(url);
+  };
+}
+
+/** Firefox and Safari: a hidden video drawn on a worker's timer, the canvas captured as the output. */
+function canvasPump(source: MediaStreamTrack, draw: Draw, canvas: HTMLCanvasElement, frameIntervalMs: number): Pipeline {
+  const video = document.createElement('video');
+  video.muted = true;
+  video.playsInline = true;
+  video.autoplay = true;
+  video.srcObject = new MediaStream([source]);
+  const play = () => { void video.play().catch(() => undefined); };
+  play();
+  const output = canvas.captureStream().getVideoTracks()[0];
+  if (!output) throw new Error('video effects: this browser gives no track from a canvas');
+  let stopped = false;
+  let lastTime = -1;
+  let failures = 0;
+  const stop = () => {
+    if (stopped) return;
+    stopped = true;
+    stopTicker();
+    video.pause();
+    video.srcObject = null;
+    output.stop();
+  };
+  const stopTicker = startTicker(frameIntervalMs, () => {
+    if (stopped) return;
+    if (output.readyState === 'ended' || source.readyState === 'ended') {
+      stop();
+      return;
+    }
+    // WebKit can refuse play() until later: keep asking.
+    if (video.paused) play();
+    if (!output.enabled || video.readyState < 2 || video.videoWidth === 0) return;
+    if (video.currentTime === lastTime) return;
+    lastTime = video.currentTime;
+    try {
+      draw(video, video.videoWidth, video.videoHeight, frameIntervalMs);
+      failures = 0;
+    } catch (err) {
+      if ((failures += 1) === 1) console.warn('video effects: a frame could not be processed', err);
+    }
+  });
+  return { output, stop };
+}
+
+/** Virtual background and beauty, for `client.setVideoProcessor()`. */
+export function createVideoEffects(options: VideoEffectsOptions = {}, stand: Partial<VideoEffectsDeps> = {}): VideoEffects {
+  const deps: VideoEffectsDeps = { ...browserDeps, ...stand };
+  const wasmPath = options.wasmPath ?? MEDIAPIPE_WASM;
+  const modelFor = (width: number, height: number): string => options.modelUrl ?? (width > height ? DEFAULT_LANDSCAPE_MODEL : DEFAULT_SEGMENTER_MODEL);
+  const maxFrameRate = typeof options.maxFrameRate === 'number' && options.maxFrameRate > 0 ? options.maxFrameRate : null;
+  const state: { mode: 'none' | 'blur' | 'image'; blur: number; beauty: BeautySettings | null } = {
+    mode: 'none',
+    blur: blurLevel(options.blur),
+    beauty: beautySettings(options.beauty),
+  };
+  let picture: Picture | null = null;
+  // By model: one for wide pictures, one for tall ones, each loaded when first needed.
+  const segmenters = new Map<string, PersonSegmenter>();
+  const loading = new Map<string, Promise<PersonSegmenter>>();
+  // The camera's shape, for the model to load before it is seen; a webcam's until a camera is given.
+  let shape = { width: 1280, height: 720 };
+  let renderer: EffectsRenderer | null = null;
+  let pipeline: Pipeline | null = null;
+  let destroyed = false;
+  let overloaded = false;
+  const pacer = new SegmentationPacer();
+  const clock = new MonotonicClock();
+  // Per camera: the mask and how far along it is.
+  let maskReady = false;
+  let steady: Float32Array | null = null;
+  let maskBytes = new Uint8Array(0);
+  let frameIndex = 0;
+  let segCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
+  let segContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null = null;
+  let pictureUploaded: Picture | null = null;
+
+  const ensureSegmenter = (model: string): Promise<PersonSegmenter> => {
+    const ready = segmenters.get(model);
+    if (ready) return Promise.resolve(ready);
+    let pending = loading.get(model);
+    if (!pending) {
+      pending = deps.loadSegmenter(wasmPath, model).then(
+        (s) => {
+          loading.delete(model);
+          if (destroyed) {
+            s.close();
+            throw new Error('video effects: destroyed');
+          }
+          segmenters.set(model, s);
+          return s;
+        },
+        (err: unknown) => {
+          loading.delete(model); // tried again next time (a network blip)
+          throw err;
+        },
+      );
+      loading.set(model, pending);
+    }
+    return pending;
+  };
+
+  /** A fresh start for the mask: nothing behind the person shows until the next one is found. */
+  const restartMask = () => {
+    maskReady = false;
+    steady = null;
+    frameIndex = 0;
+  };
+
+  const segmentInto = (input: TexImageSource & CanvasImageSource, width: number, height: number, frameIntervalMs: number) => {
+    const model = modelFor(width, height);
+    let seg = segmenters.get(model);
+    if (!seg) {
+      // The camera turned (a phone rotated): the other model loads meanwhile, and the one there is goes on.
+      void ensureSegmenter(model).catch(() => undefined);
+      seg = segmenters.values().next().value;
+    }
+    if (!seg || !renderer) return;
+    const size = segmentationSize(width, height);
+    if (!segCanvas || segCanvas.width !== size.width || segCanvas.height !== size.height) {
+      segCanvas = typeof OffscreenCanvas === 'function' ? new OffscreenCanvas(size.width, size.height) : Object.assign(document.createElement('canvas'), size);
+      segContext = segCanvas.getContext('2d') as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D | null;
+      maskBytes = new Uint8Array(size.width * size.height);
+      steady = null;
+    }
+    if (!segContext) return;
+    segContext.drawImage(input, 0, 0, size.width, size.height);
+    const started = performance.now();
+    seg.segment(segCanvas, clock.next(started), (confidence) => {
+      if (confidence.length !== size.width * size.height || !renderer) return;
+      steady = smoothMask(confidence, steady, maskBytes);
+      renderer.uploadMask(maskBytes, size.width, size.height);
+      maskReady = true;
+    });
+    const now = performance.now();
+    pacer.report(now - started, frameIntervalMs, now);
+    const over = pacer.overloaded(frameIntervalMs);
+    if (over !== overloaded) {
+      overloaded = over;
+      try { options.onOverload?.(over); } catch { /* the app's */ }
+    }
+  };
+
+  const draw: Draw = (input, width, height, frameIntervalMs) => {
+    const r = renderer;
+    if (!r || r.lost || !(width > 0 && height > 0)) return;
+    r.setSize(width, height);
+    r.uploadFrame(input);
+    const mode = state.mode;
+    if (mode !== 'none') {
+      if (pacer.shouldSegment(frameIndex)) segmentInto(input, width, height, frameIntervalMs);
+      frameIndex += 1;
+    }
+    if (mode === 'image' && picture && pictureUploaded !== picture) {
+      r.setPicture(picture.image, picture.width, picture.height);
+      pictureUploaded = picture;
+    } else if (mode !== 'image' && pictureUploaded) {
+      r.setPicture(null);
+      pictureUploaded = null;
+    }
+    r.render({
+      mode,
+      blurSigma: blurSigma(state.blur, r.size.blurHeight),
+      beauty: state.beauty,
+      maskReady,
+      cover: picture ? coverFit(picture.width, picture.height, width, height) : { sx: 1, sy: 1, ox: 0, oy: 0 },
+    });
+  };
+
+  const apply = async (changes: VideoEffectsUpdate): Promise<void> => {
+    if (destroyed) throw new Error('video effects: destroyed');
+    let mode = state.mode;
+    let nextPicture: Picture | null | undefined;
+    if ('background' in changes) {
+      mode = backgroundMode(changes.background);
+      // Loaded before anything changes: one that fails leaves the effects as they were.
+      if (mode === 'image') nextPicture = await loadPicture((changes.background as { image: BackgroundImageSource }).image);
+      if (mode !== 'none') {
+        try {
+          await ensureSegmenter(modelFor(shape.width, shape.height));
+        } catch (err) {
+          nextPicture?.close();
+          throw err;
+        }
+      }
+    }
+    if (destroyed) {
+      nextPicture?.close();
+      throw new Error('video effects: destroyed');
+    }
+    if ('blur' in changes) state.blur = blurLevel(changes.blur);
+    if ('beauty' in changes) state.beauty = beautySettings(changes.beauty);
+    if (mode !== state.mode) restartMask();
+    if (nextPicture !== undefined || mode !== 'image') {
+      const old = picture;
+      picture = nextPicture ?? null;
+      if (old && old !== picture) old.close();
+    }
+    state.mode = mode;
+  };
+
+  // The background asked for at creation starts loading now, so it is likely ready by the time the camera is.
+  let queue: Promise<void> = apply({ background: options.background ?? 'none' });
+  queue.catch(() => undefined);
+
+  const pumpChoice = deps.pump ?? pumpKind();
+
+  return {
+    name: 'video-effects',
+
+    async process(track: MediaStreamTrack): Promise<MediaStreamTrack> {
+      if (destroyed) throw new Error('video effects: destroyed');
+      if (!pumpChoice) throw new Error('video effects are not supported in this browser');
+      await queue; // the background asked for at creation: a failure is the SDK's to report
+      const settings = track.getSettings();
+      if (typeof settings.width === 'number' && typeof settings.height === 'number' && settings.width > 0 && settings.height > 0) {
+        shape = { width: settings.width, height: settings.height };
+      }
+      // The model for this camera's shape, before its first frame: a background shows nothing behind the person meanwhile.
+      if (state.mode !== 'none') await ensureSegmenter(modelFor(shape.width, shape.height));
+      pipeline?.stop();
+      pipeline = null;
+      if (!renderer) {
+        const canvas = pumpChoice === 'insertable' && typeof OffscreenCanvas === 'function' ? new OffscreenCanvas(2, 2) : document.createElement('canvas');
+        renderer = new EffectsRenderer(canvas);
+      }
+      restartMask();
+      const cameraRate = typeof settings.frameRate === 'number' && settings.frameRate > 0 ? settings.frameRate : 30;
+      const rate = maxFrameRate ? Math.min(maxFrameRate, cameraRate) : cameraRate;
+      const frameIntervalMs = 1000 / rate;
+      pipeline = pumpChoice === 'insertable'
+        ? insertablePump(track, draw, renderer.canvas, frameIntervalMs, maxFrameRate ? (1e6 / maxFrameRate) * 0.8 : 0)
+        : canvasPump(track, draw, renderer.canvas as HTMLCanvasElement, frameIntervalMs);
+      return pipeline.output;
+    },
+
+    update(changes: VideoEffectsUpdate): Promise<void> {
+      // One change at a time, in the order asked.
+      const next = queue.catch(() => undefined).then(() => apply(changes));
+      queue = next.catch(() => undefined);
+      return next;
+    },
+
+    get settings() {
+      return { background: state.mode, blur: state.blur, beauty: state.beauty ? { ...state.beauty } : null };
+    },
+
+    get stats() {
+      return { segmentEvery: pacer.interval, segmentMs: Math.round(pacer.segmentMs * 10) / 10, overloaded };
+    },
+
+    destroy(): void {
+      if (destroyed) return;
+      destroyed = true;
+      pipeline?.stop();
+      pipeline = null;
+      for (const s of segmenters.values()) s.close();
+      segmenters.clear();
+      renderer?.dispose();
+      renderer = null;
+      picture?.close();
+      picture = null;
+    },
+  };
+}