@livekit/track-processors 0.1.0

package/src/ProcessorPipeline.ts

@@ -0,0 +1,69 @@
+import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';
+import { VideoTrackTransformer } from './transformers';
+
+export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
+  static get isSupported() {
+    return (
+      typeof MediaStreamTrackGenerator !== 'undefined' &&
+      typeof MediaStreamTrackProcessor !== 'undefined'
+    );
+  }
+
+  name: string;
+
+  source?: MediaStreamVideoTrack;
+
+  sourceSettings?: MediaTrackSettings;
+
+  processor?: MediaStreamTrackProcessor<VideoFrame>;
+
+  trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;
+
+  canvas?: OffscreenCanvas;
+
+  sourceDummy?: HTMLMediaElement;
+
+  processedTrack?: MediaStreamTrack;
+
+  transformers: Array<VideoTrackTransformer>;
+
+  constructor(transformers: Array<VideoTrackTransformer>, name: string) {
+    this.name = name;
+    this.transformers = transformers;
+  }
+
+  async init(opts: ProcessorOptions<Track.Kind>) {
+    this.source = opts.track as MediaStreamVideoTrack;
+    this.sourceSettings = this.source.getSettings();
+    this.sourceDummy = opts.element;
+    if (!(this.sourceDummy instanceof HTMLVideoElement)) {
+      throw TypeError('Currently only video transformers are supported');
+    }
+    // TODO explore if we can do all the processing work in a webworker
+    this.processor = new MediaStreamTrackProcessor({ track: this.source });
+    this.trackGenerator = new MediaStreamTrackGenerator({ kind: 'video' });
+
+    this.canvas = new OffscreenCanvas(
+      this.sourceSettings.width ?? 300,
+      this.sourceSettings.height ?? 300,
+    );
+
+    let readableStream = this.processor.readable;
+    for (const transformer of this.transformers) {
+      transformer.init({
+        outputCanvas: this.canvas,
+        inputElement: this.sourceDummy!,
+      });
+      readableStream = readableStream.pipeThrough(transformer!.transformer!);
+    }
+    readableStream.pipeTo(this.trackGenerator.writable);
+    this.processedTrack = this.trackGenerator as MediaStreamVideoTrack;
+  }
+
+  async destroy() {
+    for (const transformer of this.transformers) {
+      await transformer.destroy();
+    }
+    this.trackGenerator?.stop();
+  }
+}

package/src/index.ts

@@ -0,0 +1,36 @@
+import ProcessorPipeline from './ProcessorPipeline';
+import BackgroundTransformer from './transformers/BackgroundTransformer';
+import DummyTransformer from './transformers/DummyTransformer';
+
+export const BackgroundBlur = (blurRadius: number = 10) => {
+  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
+  if (!isPipelineSupported) {
+    throw new Error('pipeline is not supported in this browser');
+  }
+  const pipeline = new ProcessorPipeline(
+    [new BackgroundTransformer({ blurRadius })],
+    'background-blur',
+  );
+  return pipeline;
+};
+
+export const VirtualBackground = (imagePath: string) => {
+  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
+  if (!isPipelineSupported) {
+    throw new Error('pipeline is not supported in this browser');
+  }
+  const pipeline = new ProcessorPipeline(
+    [new BackgroundTransformer({ imagePath })],
+    'virtual-background',
+  );
+  return pipeline;
+};
+
+export const Dummy = () => {
+  const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
+  if (!isPipelineSupported) {
+    throw new Error('pipeline is not supported in this browser');
+  }
+  const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');
+  return pipeline;
+};

package/src/transformers/BackgroundTransformer.ts

@@ -0,0 +1,195 @@
+import * as vision from '@mediapipe/tasks-vision';
+import VideoTransformer from './VideoTransformer';
+import { VideoTransformerInitOptions } from './types';
+
+export type BackgroundOptions = {
+  blurRadius?: number;
+  imagePath?: string;
+};
+
+export default class BackgroundProcessor extends VideoTransformer {
+  static get isSupported() {
+    return typeof OffscreenCanvas !== 'undefined';
+  }
+
+  imageSegmenter?: vision.ImageSegmenter;
+
+  segmentationResults: vision.ImageSegmenterResult | undefined;
+
+  backgroundImage: ImageBitmap | null = null;
+
+  blurRadius?: number;
+
+  constructor(opts: BackgroundOptions) {
+    super();
+    if (opts.blurRadius) {
+      this.blurRadius = opts.blurRadius;
+    } else if (opts.imagePath) {
+      this.loadBackground(opts.imagePath);
+    }
+  }
+
+  async init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions) {
+    super.init({ outputCanvas, inputElement: inputVideo });
+
+    const fileSet = await vision.FilesetResolver.forVisionTasks(
+      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm',
+    );
+
+    this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {
+      baseOptions: {
+        modelAssetPath:
+          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',
+        delegate: 'CPU',
+      },
+      runningMode: 'VIDEO',
+      outputCategoryMask: true,
+      outputConfidenceMasks: false,
+    });
+
+    // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));
+    this.sendFramesContinuouslyForSegmentation(this.inputVideo!);
+  }
+
+  async destroy() {
+    await super.destroy();
+    await this.imageSegmenter?.close();
+    this.backgroundImage = null;
+  }
+
+  async sendFramesContinuouslyForSegmentation(videoEl: HTMLVideoElement) {
+    if (!this.isDisabled) {
+      if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
+        let startTimeMs = performance.now();
+        this.imageSegmenter?.segmentForVideo(
+          videoEl,
+          startTimeMs,
+          (result) => (this.segmentationResults = result),
+        );
+      }
+      videoEl.requestVideoFrameCallback(() => {
+        this.sendFramesContinuouslyForSegmentation(videoEl);
+      });
+    }
+  }
+
+  async loadBackground(path: string) {
+    const img = new Image();
+
+    await new Promise((resolve, reject) => {
+      img.crossOrigin = 'Anonymous';
+      img.onload = () => resolve(img);
+      img.onerror = (err) => reject(err);
+      img.src = path;
+    });
+    const imageData = await createImageBitmap(img);
+    this.backgroundImage = imageData;
+  }
+
+  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
+    if (this.isDisabled) {
+      controller.enqueue(frame);
+      return;
+    }
+    if (!this.canvas) {
+      throw TypeError('Canvas needs to be initialized first');
+    }
+    if (this.blurRadius) {
+      await this.blurBackground(frame);
+    } else {
+      await this.drawVirtualBackground(frame);
+    }
+    const newFrame = new VideoFrame(this.canvas, {
+      timestamp: frame.timestamp || Date.now(),
+    });
+    frame.close();
+    controller.enqueue(newFrame);
+  }
+
+  async drawVirtualBackground(frame: VideoFrame) {
+    if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo) return;
+    // this.ctx.save();
+    // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
+    if (this.segmentationResults?.categoryMask) {
+      this.ctx.filter = 'blur(3px)';
+      this.ctx.globalCompositeOperation = 'copy';
+      const bitmap = await maskToBitmap(
+        this.segmentationResults.categoryMask,
+        this.inputVideo.videoWidth,
+        this.inputVideo.videoHeight,
+      );
+      this.ctx.drawImage(bitmap, 0, 0);
+      this.ctx.filter = 'none';
+      this.ctx.globalCompositeOperation = 'source-in';
+      if (this.backgroundImage) {
+        this.ctx.drawImage(
+          this.backgroundImage,
+          0,
+          0,
+          this.backgroundImage.width,
+          this.backgroundImage.height,
+          0,
+          0,
+          this.canvas.width,
+          this.canvas.height,
+        );
+      } else {
+        this.ctx.fillStyle = '#00FF00';
+        this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
+      }
+
+      this.ctx.globalCompositeOperation = 'destination-over';
+    }
+    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
+    // this.ctx.restore();
+  }
+
+  async blurBackground(frame: VideoFrame) {
+    const start = performance.now();
+    if (
+      !this.ctx ||
+      !this.canvas ||
+      !this.segmentationResults?.categoryMask?.canvas ||
+      !this.inputVideo
+    ) {
+      return;
+    }
+    this.ctx.save();
+    this.ctx.globalCompositeOperation = 'copy';
+
+    const bitmap = await maskToBitmap(
+      this.segmentationResults.categoryMask,
+      this.inputVideo.videoWidth,
+      this.inputVideo.videoHeight,
+    );
+    this.ctx.filter = 'blur(3px)';
+    this.ctx.globalCompositeOperation = 'copy';
+    this.ctx.drawImage(bitmap, 0, 0);
+    this.ctx.filter = 'none';
+    this.ctx.globalCompositeOperation = 'source-out';
+    this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
+    this.ctx.globalCompositeOperation = 'destination-over';
+    this.ctx.filter = `blur(${this.blurRadius}px)`;
+    this.ctx.drawImage(frame, 0, 0);
+    this.ctx.restore();
+    console.log('draw time', performance.now() - start);
+  }
+}
+
+function maskToBitmap(
+  mask: vision.MPMask,
+  videoWidth: number,
+  videoHeight: number,
+): Promise<ImageBitmap> {
+  const dataArray: Uint8ClampedArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);
+  const result = mask.getAsUint8Array();
+  for (let i = 0; i < result.length; i += 1) {
+    dataArray[i * 4] = result[i];
+    dataArray[i * 4 + 1] = result[i];
+    dataArray[i * 4 + 2] = result[i];
+    dataArray[i * 4 + 3] = result[i];
+  }
+  const dataNew = new ImageData(dataArray, videoWidth, videoHeight);
+
+  return createImageBitmap(dataNew);
+}

package/src/transformers/DummyTransformer.ts

@@ -0,0 +1,11 @@
+import VideoTransformer from './VideoTransformer';
+
+export default class DummyTransformer extends VideoTransformer {
+  async transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>) {
+    controller.enqueue(frame);
+  }
+
+  async destroy() {
+    // nothing to do
+  }
+}

package/src/transformers/MediaPipeHolisticTrackerTransformer.ts

@@ -0,0 +1,60 @@
+import { Holistic, Options, Results } from '@mediapipe/holistic';
+import VideoTransformer from './VideoTransformer';
+import { VideoTransformerInitOptions } from './types';
+
+export type MediaPipeHolisticTrackerTransformerOptions = {
+  holisticOptions?: Options;
+  callback?: (results: Results) => void;
+};
+
+export default class MediaPipeHolisticTrackerTransformer extends VideoTransformer {
+  holistic?: Holistic;
+  holisticOptions: Options;
+  callback: (results: Results) => void;
+
+  public static get isSupported(): boolean {
+    return true;
+  }
+
+  constructor({ holisticOptions, callback }: MediaPipeHolisticTrackerTransformerOptions) {
+    super();
+    this.callback = callback || (() => null);
+    this.holisticOptions = holisticOptions || {};
+  }
+
+  init({ inputElement: inputVideo, outputCanvas }: VideoTransformerInitOptions): void {
+    super.init({ outputCanvas, inputElement: inputVideo });
+
+    this.holistic = new Holistic({
+      locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`,
+    });
+    this.holistic.setOptions(this.holisticOptions);
+    this.holistic.onResults((r) => {
+      this.callback(r);
+    });
+
+    this.sendFramesContinuouslyForTracking(this.inputVideo!);
+  }
+
+  async destroy(): Promise<void> {
+    this.callback = () => null;
+    await super.destroy();
+    await this.holistic?.close();
+  }
+
+  async transform(): Promise<void> {
+    return;
+  }
+
+  async sendFramesContinuouslyForTracking(videoEl: HTMLVideoElement): Promise<void> {
+    if (!this.isDisabled) {
+      if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
+        await this.holistic?.send({ image: videoEl });
+      }
+
+      videoEl.requestVideoFrameCallback(() => {
+        this.sendFramesContinuouslyForTracking(videoEl);
+      });
+    }
+  }
+}

package/src/transformers/VideoTransformer.ts

@@ -0,0 +1,39 @@
+import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';
+
+export default abstract class VideoTransformer implements VideoTrackTransformer {
+  transformer?: TransformStream;
+
+  canvas?: OffscreenCanvas;
+
+  ctx?: OffscreenCanvasRenderingContext2D;
+
+  inputVideo?: HTMLVideoElement;
+
+  protected isDisabled?: Boolean = false;
+
+  init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): void {
+    if (!(inputVideo instanceof HTMLVideoElement)) {
+      throw TypeError('Video transformer needs a HTMLVideoElement as input');
+    }
+    this.transformer = new TransformStream({
+      transform: (frame, controller) => this.transform(frame, controller),
+    });
+    this.canvas = outputCanvas || null;
+    if (outputCanvas) {
+      this.ctx = this.canvas?.getContext('2d', { readFrequently: true }) || undefined;
+    }
+    this.inputVideo = inputVideo;
+    this.isDisabled = false;
+  }
+
+  async destroy() {
+    this.isDisabled = true;
+    this.canvas = undefined;
+    this.ctx = undefined;
+  }
+
+  abstract transform(
+    frame: VideoFrame,
+    controller: TransformStreamDefaultController<VideoFrame>,
+  ): void;
+}

package/src/transformers/index.ts

@@ -0,0 +1,2 @@
+export * from './BackgroundTransformer';
+export * from './types';

package/src/transformers/types.ts

@@ -0,0 +1,17 @@
+export type TrackTransformerInitOptions = {
+  inputElement: HTMLMediaElement;
+};
+
+export interface VideoTransformerInitOptions extends TrackTransformerInitOptions {
+  outputCanvas: OffscreenCanvas;
+  inputElement: HTMLVideoElement;
+}
+
+export interface VideoTrackTransformer<
+  T extends TrackTransformerInitOptions = VideoTransformerInitOptions,
+> {
+  init: (options: T) => void;
+  destroy: () => void;
+  transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
+  transformer?: TransformStream;
+}

package/src/utils.ts

@@ -0,0 +1,2 @@
+export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined';
+export const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined';