@livekit/track-processors 0.1.0 → 0.1.2

package/dist/ProcessorPipeline.d.ts

@@ -1,18 +1,19 @@
 /// <reference types="dom-mediacapture-transform" />
 /// <reference types="dom-webcodecs" />
-import type { ProcessorOptions, VideoProcessor } from 'livekit-client';
-import { StreamTransformer } from './transformers';
-export default class ProcessorPipeline implements VideoProcessor<ProcessorOptions> {
+import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';
+import { VideoTrackTransformer } from './transformers';
+export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
     static get isSupported(): boolean;
+    name: string;
     source?: MediaStreamVideoTrack;
     sourceSettings?: MediaTrackSettings;
     processor?: MediaStreamTrackProcessor<VideoFrame>;
     trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;
     canvas?: OffscreenCanvas;
-    sourceDummy?: HTMLVideoElement;
+    sourceDummy?: HTMLMediaElement;
     processedTrack?: MediaStreamTrack;
-    transformers: Array<StreamTransformer>;
-    constructor(transformers: Array<StreamTransformer>);
-    init(opts: ProcessorOptions): Promise<void>;
+    transformers: Array<VideoTrackTransformer>;
+    constructor(transformers: Array<VideoTrackTransformer>, name: string);
+    init(opts: ProcessorOptions<Track.Kind>): Promise<void>;
     destroy(): Promise<void>;
 }

package/dist/ProcessorPipeline.js

@@ -3,7 +3,8 @@ export default class ProcessorPipeline {
         return (typeof MediaStreamTrackGenerator !== 'undefined' &&
             typeof MediaStreamTrackProcessor !== 'undefined');
     }
-    constructor(transformers) {
+    constructor(transformers, name) {
+        this.name = name;
         this.transformers = transformers;
     }
     async init(opts) {
@@ -11,6 +12,9 @@ export default class ProcessorPipeline {
         this.source = opts.track;
         this.sourceSettings = this.source.getSettings();
         this.sourceDummy = opts.element;
+        if (!(this.sourceDummy instanceof HTMLVideoElement)) {
+            throw TypeError('Currently only video transformers are supported');
+        }
         // TODO explore if we can do all the processing work in a webworker
         this.processor = new MediaStreamTrackProcessor({ track: this.source });
         this.trackGenerator = new MediaStreamTrackGenerator({ kind: 'video' });
@@ -19,14 +23,12 @@ export default class ProcessorPipeline {
         for (const transformer of this.transformers) {
             transformer.init({
                 outputCanvas: this.canvas,
-                inputVideo: this.sourceDummy,
+                inputElement: this.sourceDummy,
             });
             readableStream = readableStream.pipeThrough(transformer.transformer);
         }
-        console.log('before pipe to');
         readableStream.pipeTo(this.trackGenerator.writable);
         this.processedTrack = this.trackGenerator;
-        console.log('processed internal', this.source, this.processedTrack, this.processor, this.trackGenerator);
     }
     async destroy() {
         var _a;

package/dist/ProcessorPipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"ProcessorPipeline.js","sourceRoot":"","sources":["../src/ProcessorPipeline.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,MAAM,KAAK,WAAW;QACpB,OAAO,CACL,OAAO,yBAAyB,KAAK,WAAW;YAChD,OAAO,yBAAyB,KAAK,WAAW,CACjD,CAAC;IACJ,CAAC;IAkBD,YAAY,YAAsC;QAChD,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAsB;;QAC/B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAA8B,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;QAChC,mEAAmE;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,yBAAyB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,cAAc,GAAG,IAAI,yBAAyB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAEvE,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAC/B,MAAA,IAAI,CAAC,cAAc,CAAC,KAAK,mCAAI,GAAG,EAChC,MAAA,IAAI,CAAC,cAAc,CAAC,MAAM,mCAAI,GAAG,CAClC,CAAC;QAEF,IAAI,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;QAC7C,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE;YAC3C,WAAW,CAAC,IAAI,CAAC;gBACf,YAAY,EAAE,IAAI,CAAC,MAAM;gBACzB,UAAU,EAAE,IAAI,CAAC,WAAY;aAC9B,CAAC,CAAC;YACH,cAAc,GAAG,cAAc,CAAC,WAAW,CAAC,WAAY,CAAC,WAAY,CAAC,CAAC;SACxE;QACD,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAE9B,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAuC,CAAC;QACnE,OAAO,CAAC,GAAG,CACT,oBAAoB,EACpB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,cAAc,CACpB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE;YAC3C,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;SAC7B;QACD,MAAA,IAAI,CAAC,cAAc,0CAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;CACF"}
+{"version":3,"file":"ProcessorPipeline.js","sourceRoot":"","sources":["../src/ProcessorPipeline.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,MAAM,KAAK,WAAW;QACpB,OAAO,CACL,OAAO,yBAAyB,KAAK,WAAW;YAChD,OAAO,yBAAyB,KAAK,WAAW,CACjD,CAAC;IACJ,CAAC;IAoBD,YAAY,YAA0C,EAAE,IAAY;QAClE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAkC;;QAC3C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAA8B,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,YAAY,gBAAgB,CAAC,EAAE;YACnD,MAAM,SAAS,CAAC,iDAAiD,CAAC,CAAC;SACpE;QACD,mEAAmE;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,yBAAyB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,cAAc,GAAG,IAAI,yBAAyB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAEvE,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAC/B,MAAA,IAAI,CAAC,cAAc,CAAC,KAAK,mCAAI,GAAG,EAChC,MAAA,IAAI,CAAC,cAAc,CAAC,MAAM,mCAAI,GAAG,CAClC,CAAC;QAEF,IAAI,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;QAC7C,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE;YAC3C,WAAW,CAAC,IAAI,CAAC;gBACf,YAAY,EAAE,IAAI,CAAC,MAAM;gBACzB,YAAY,EAAE,IAAI,CAAC,WAAY;aAChC,CAAC,CAAC;YACH,cAAc,GAAG,cAAc,CAAC,WAAW,CAAC,WAAY,CAAC,WAAY,CAAC,CAAC;SACxE;QACD,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAuC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE;YAC3C,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;SAC7B;QACD,MAAA,IAAI,CAAC,cAAc,0CAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;CACF"}

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 import ProcessorPipeline from './ProcessorPipeline';
-export declare const BlurBackground: (blurRadius?: number) => ProcessorPipeline;
+export declare const BackgroundBlur: (blurRadius?: number) => ProcessorPipeline;
 export declare const VirtualBackground: (imagePath: string) => ProcessorPipeline;
 export declare const Dummy: () => ProcessorPipeline;

package/dist/index.js

@@ -1,12 +1,12 @@
 import ProcessorPipeline from './ProcessorPipeline';
 import BackgroundTransformer from './transformers/BackgroundTransformer';
 import DummyTransformer from './transformers/DummyTransformer';
-export const BlurBackground = (blurRadius = 10) => {
+export const BackgroundBlur = (blurRadius = 10) => {
     const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
     if (!isPipelineSupported) {
         throw new Error('pipeline is not supported in this browser');
     }
-    const pipeline = new ProcessorPipeline([new BackgroundTransformer({ blurRadius })]);
+    const pipeline = new ProcessorPipeline([new BackgroundTransformer({ blurRadius })], 'background-blur');
     return pipeline;
 };
 export const VirtualBackground = (imagePath) => {
@@ -14,7 +14,7 @@ export const VirtualBackground = (imagePath) => {
     if (!isPipelineSupported) {
         throw new Error('pipeline is not supported in this browser');
     }
-    const pipeline = new ProcessorPipeline([new BackgroundTransformer({ imagePath })]);
+    const pipeline = new ProcessorPipeline([new BackgroundTransformer({ imagePath })], 'virtual-background');
     return pipeline;
 };
 export const Dummy = () => {
@@ -22,7 +22,7 @@ export const Dummy = () => {
     if (!isPipelineSupported) {
         throw new Error('pipeline is not supported in this browser');
     }
-    const pipeline = new ProcessorPipeline([new DummyTransformer()]);
+    const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');
     return pipeline;
 };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,iBAAiB,MAAM,qBAAqB,CAAC;AACpD,OAAO,qBAAqB,MAAM,sCAAsC,CAAC;AACzE,OAAO,gBAAgB,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,aAAqB,EAAE,EAAE,EAAE;IACxD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,CAAC,IAAI,qBAAqB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;IACpF,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,SAAiB,EAAE,EAAE;IACrD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,CAAC,IAAI,qBAAqB,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,GAAG,EAAE;IACxB,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,CAAC,IAAI,gBAAgB,EAAE,CAAC,CAAC,CAAC;IACjE,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,iBAAiB,MAAM,qBAAqB,CAAC;AACpD,OAAO,qBAAqB,MAAM,sCAAsC,CAAC;AACzE,OAAO,gBAAgB,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,aAAqB,EAAE,EAAE,EAAE;IACxD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CACpC,CAAC,IAAI,qBAAqB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,EAC3C,iBAAiB,CAClB,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,SAAiB,EAAE,EAAE;IACrD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CACpC,CAAC,IAAI,qBAAqB,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,EAC1C,oBAAoB,CACrB,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,GAAG,EAAE;IACxB,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,CAAC,IAAI,gBAAgB,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}

package/dist/package.json

@@ -0,0 +1,56 @@
+{
+    "name": "@livekit/track-processors",
+    "version": "0.1.2",
+    "description": "LiveKit track processors",
+    "main": "dist/index.js",
+    "source": "src/index.ts",
+    "types": "dist/index.d.ts",
+    "repository": "git@github.com:livekit/livekit-track-processors.git",
+    "author": "Lukas Seiler",
+    "license": "Apache-2.0",
+    "scripts": {
+        "build": "yarn exec tsc",
+        "build-docs": "yarn exec typedoc",
+        "build-sample": "cd example && webpack && cp styles.css index.html dist/",
+        "lint": "eslint src",
+        "test": "jest",
+        "sample": "vite serve example --port 8080 --open"
+    },
+    "files": [
+        "dist",
+        "src"
+    ],
+    "dependencies": {
+        "@mediapipe/holistic": "0.5.1675471629",
+        "@mediapipe/tasks-vision": "0.10.1"
+    },
+    "peerDependencies": {
+        "livekit-client": "^1.10.0"
+    },
+    "devDependencies": {
+        "@trivago/prettier-plugin-sort-imports": "^4.1.1",
+        "@types/dom-mediacapture-transform": "^0.1.5",
+        "@types/jest": "^27.0.3",
+        "@types/offscreencanvas": "^2019.7.0",
+        "@typescript-eslint/eslint-plugin": "^4.31.2",
+        "@webpack-cli/serve": "^1.5.2",
+        "eslint": "8.39.0",
+        "eslint-config-airbnb-typescript": "17.0.0",
+        "eslint-config-prettier": "8.8.0",
+        "eslint-plugin-ecmascript-compat": "^3.0.0",
+        "eslint-plugin-import": "2.27.5",
+        "jest": "^27.4.3",
+        "livekit-client": "^0.16.6",
+        "prettier": "^2.8.8",
+        "ts-jest": "^27.0.7",
+        "ts-loader": "^8.1.0",
+        "ts-proto": "^1.85.0",
+        "typedoc": "^0.20.35",
+        "typedoc-plugin-no-inherit": "1.3.0",
+        "typescript": "^5.0.4",
+        "vite": "^4.3.8",
+        "webpack": "^5.53.0",
+        "webpack-cli": "^4.8.0",
+        "webpack-dev-server": "^4.2.1"
+    }
+}

package/dist/src/ProcessorPipeline.d.ts

@@ -0,0 +1,19 @@
+/// <reference types="dom-mediacapture-transform" />
+/// <reference types="dom-webcodecs" />
+import type { ProcessorOptions, Track, TrackProcessor } from 'livekit-client';
+import { VideoTrackTransformer } from './transformers';
+export default class ProcessorPipeline implements TrackProcessor<Track.Kind> {
+    static get isSupported(): boolean;
+    name: string;
+    source?: MediaStreamVideoTrack;
+    sourceSettings?: MediaTrackSettings;
+    processor?: MediaStreamTrackProcessor<VideoFrame>;
+    trackGenerator?: MediaStreamTrackGenerator<VideoFrame>;
+    canvas?: OffscreenCanvas;
+    sourceDummy?: HTMLMediaElement;
+    processedTrack?: MediaStreamTrack;
+    transformers: Array<VideoTrackTransformer>;
+    constructor(transformers: Array<VideoTrackTransformer>, name: string);
+    init(opts: ProcessorOptions<Track.Kind>): Promise<void>;
+    destroy(): Promise<void>;
+}

package/dist/src/ProcessorPipeline.js

@@ -0,0 +1,41 @@
+export default class ProcessorPipeline {
+    static get isSupported() {
+        return (typeof MediaStreamTrackGenerator !== 'undefined' &&
+            typeof MediaStreamTrackProcessor !== 'undefined');
+    }
+    constructor(transformers, name) {
+        this.name = name;
+        this.transformers = transformers;
+    }
+    async init(opts) {
+        var _a, _b;
+        this.source = opts.track;
+        this.sourceSettings = this.source.getSettings();
+        this.sourceDummy = opts.element;
+        if (!(this.sourceDummy instanceof HTMLVideoElement)) {
+            throw TypeError('Currently only video transformers are supported');
+        }
+        // TODO explore if we can do all the processing work in a webworker
+        this.processor = new MediaStreamTrackProcessor({ track: this.source });
+        this.trackGenerator = new MediaStreamTrackGenerator({ kind: 'video' });
+        this.canvas = new OffscreenCanvas((_a = this.sourceSettings.width) !== null && _a !== void 0 ? _a : 300, (_b = this.sourceSettings.height) !== null && _b !== void 0 ? _b : 300);
+        let readableStream = this.processor.readable;
+        for (const transformer of this.transformers) {
+            transformer.init({
+                outputCanvas: this.canvas,
+                inputElement: this.sourceDummy,
+            });
+            readableStream = readableStream.pipeThrough(transformer.transformer);
+        }
+        readableStream.pipeTo(this.trackGenerator.writable);
+        this.processedTrack = this.trackGenerator;
+    }
+    async destroy() {
+        var _a;
+        for (const transformer of this.transformers) {
+            await transformer.destroy();
+        }
+        (_a = this.trackGenerator) === null || _a === void 0 ? void 0 : _a.stop();
+    }
+}
+//# sourceMappingURL=ProcessorPipeline.js.map

package/dist/src/ProcessorPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ProcessorPipeline.js","sourceRoot":"","sources":["../../src/ProcessorPipeline.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,MAAM,KAAK,WAAW;QACpB,OAAO,CACL,OAAO,yBAAyB,KAAK,WAAW;YAChD,OAAO,yBAAyB,KAAK,WAAW,CACjD,CAAC;IACJ,CAAC;IAoBD,YAAY,YAA0C,EAAE,IAAY;QAClE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAkC;;QAC3C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAA8B,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;QAChC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,YAAY,gBAAgB,CAAC,EAAE;YACnD,MAAM,SAAS,CAAC,iDAAiD,CAAC,CAAC;SACpE;QACD,mEAAmE;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,yBAAyB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,cAAc,GAAG,IAAI,yBAAyB,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAEvE,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAC/B,MAAA,IAAI,CAAC,cAAc,CAAC,KAAK,mCAAI,GAAG,EAChC,MAAA,IAAI,CAAC,cAAc,CAAC,MAAM,mCAAI,GAAG,CAClC,CAAC;QAEF,IAAI,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;QAC7C,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE;YAC3C,WAAW,CAAC,IAAI,CAAC;gBACf,YAAY,EAAE,IAAI,CAAC,MAAM;gBACzB,YAAY,EAAE,IAAI,CAAC,WAAY;aAChC,CAAC,CAAC;YACH,cAAc,GAAG,cAAc,CAAC,WAAW,CAAC,WAAY,CAAC,WAAY,CAAC,CAAC;SACxE;QACD,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAuC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE;YAC3C,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;SAC7B;QACD,MAAA,IAAI,CAAC,cAAc,0CAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;CACF"}

package/dist/src/index.d.ts

@@ -0,0 +1,4 @@
+import ProcessorPipeline from './ProcessorPipeline';
+export declare const BackgroundBlur: (blurRadius?: number) => ProcessorPipeline;
+export declare const VirtualBackground: (imagePath: string) => ProcessorPipeline;
+export declare const Dummy: () => ProcessorPipeline;

package/dist/src/index.js

@@ -0,0 +1,28 @@
+import ProcessorPipeline from './ProcessorPipeline';
+import BackgroundTransformer from './transformers/BackgroundTransformer';
+import DummyTransformer from './transformers/DummyTransformer';
+export const BackgroundBlur = (blurRadius = 10) => {
+    const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
+    if (!isPipelineSupported) {
+        throw new Error('pipeline is not supported in this browser');
+    }
+    const pipeline = new ProcessorPipeline([new BackgroundTransformer({ blurRadius })], 'background-blur');
+    return pipeline;
+};
+export const VirtualBackground = (imagePath) => {
+    const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
+    if (!isPipelineSupported) {
+        throw new Error('pipeline is not supported in this browser');
+    }
+    const pipeline = new ProcessorPipeline([new BackgroundTransformer({ imagePath })], 'virtual-background');
+    return pipeline;
+};
+export const Dummy = () => {
+    const isPipelineSupported = ProcessorPipeline.isSupported && BackgroundTransformer.isSupported;
+    if (!isPipelineSupported) {
+        throw new Error('pipeline is not supported in this browser');
+    }
+    const pipeline = new ProcessorPipeline([new DummyTransformer()], 'dummy');
+    return pipeline;
+};
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,iBAAiB,MAAM,qBAAqB,CAAC;AACpD,OAAO,qBAAqB,MAAM,sCAAsC,CAAC;AACzE,OAAO,gBAAgB,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,aAAqB,EAAE,EAAE,EAAE;IACxD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CACpC,CAAC,IAAI,qBAAqB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,EAC3C,iBAAiB,CAClB,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,SAAiB,EAAE,EAAE;IACrD,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CACpC,CAAC,IAAI,qBAAqB,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,EAC1C,oBAAoB,CACrB,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,KAAK,GAAG,GAAG,EAAE;IACxB,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,WAAW,IAAI,qBAAqB,CAAC,WAAW,CAAC;IAC/F,IAAI,CAAC,mBAAmB,EAAE;QACxB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;KAC9D;IACD,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,CAAC,IAAI,gBAAgB,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1E,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}

package/dist/src/transformers/BackgroundTransformer.d.ts

@@ -0,0 +1,23 @@
+/// <reference types="dom-webcodecs" />
+import * as vision from '@mediapipe/tasks-vision';
+import VideoTransformer from './VideoTransformer';
+import { VideoTransformerInitOptions } from './types';
+export type BackgroundOptions = {
+    blurRadius?: number;
+    imagePath?: string;
+};
+export default class BackgroundProcessor extends VideoTransformer {
+    static get isSupported(): boolean;
+    imageSegmenter?: vision.ImageSegmenter;
+    segmentationResults: vision.ImageSegmenterResult | undefined;
+    backgroundImage: ImageBitmap | null;
+    blurRadius?: number;
+    constructor(opts: BackgroundOptions);
+    init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): Promise<void>;
+    destroy(): Promise<void>;
+    sendFramesContinuouslyForSegmentation(videoEl: HTMLVideoElement): Promise<void>;
+    loadBackground(path: string): Promise<void>;
+    transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): Promise<void>;
+    drawVirtualBackground(frame: VideoFrame): Promise<void>;
+    blurBackground(frame: VideoFrame): Promise<void>;
+}

package/dist/src/transformers/BackgroundTransformer.js

@@ -0,0 +1,142 @@
+import * as vision from '@mediapipe/tasks-vision';
+import { dependencies } from '../../package.json';
+import VideoTransformer from './VideoTransformer';
+export default class BackgroundProcessor extends VideoTransformer {
+    static get isSupported() {
+        return typeof OffscreenCanvas !== 'undefined';
+    }
+    constructor(opts) {
+        super();
+        this.backgroundImage = null;
+        if (opts.blurRadius) {
+            this.blurRadius = opts.blurRadius;
+        }
+        else if (opts.imagePath) {
+            this.loadBackground(opts.imagePath);
+        }
+    }
+    async init({ outputCanvas, inputElement: inputVideo }) {
+        super.init({ outputCanvas, inputElement: inputVideo });
+        const fileSet = await vision.FilesetResolver.forVisionTasks(`https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`);
+        this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {
+            baseOptions: {
+                modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',
+                delegate: 'CPU',
+            },
+            runningMode: 'VIDEO',
+            outputCategoryMask: true,
+            outputConfidenceMasks: false,
+        });
+        // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));
+        this.sendFramesContinuouslyForSegmentation(this.inputVideo);
+    }
+    async destroy() {
+        var _a;
+        await super.destroy();
+        await ((_a = this.imageSegmenter) === null || _a === void 0 ? void 0 : _a.close());
+        this.backgroundImage = null;
+    }
+    async sendFramesContinuouslyForSegmentation(videoEl) {
+        var _a;
+        if (!this.isDisabled) {
+            if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
+                let startTimeMs = performance.now();
+                (_a = this.imageSegmenter) === null || _a === void 0 ? void 0 : _a.segmentForVideo(videoEl, startTimeMs, (result) => (this.segmentationResults = result));
+            }
+            videoEl.requestVideoFrameCallback(() => {
+                this.sendFramesContinuouslyForSegmentation(videoEl);
+            });
+        }
+    }
+    async loadBackground(path) {
+        const img = new Image();
+        await new Promise((resolve, reject) => {
+            img.crossOrigin = 'Anonymous';
+            img.onload = () => resolve(img);
+            img.onerror = (err) => reject(err);
+            img.src = path;
+        });
+        const imageData = await createImageBitmap(img);
+        this.backgroundImage = imageData;
+    }
+    async transform(frame, controller) {
+        if (this.isDisabled) {
+            controller.enqueue(frame);
+            return;
+        }
+        if (!this.canvas) {
+            throw TypeError('Canvas needs to be initialized first');
+        }
+        if (this.blurRadius) {
+            await this.blurBackground(frame);
+        }
+        else {
+            await this.drawVirtualBackground(frame);
+        }
+        const newFrame = new VideoFrame(this.canvas, {
+            timestamp: frame.timestamp || Date.now(),
+        });
+        frame.close();
+        controller.enqueue(newFrame);
+    }
+    async drawVirtualBackground(frame) {
+        var _a;
+        if (!this.canvas || !this.ctx || !this.segmentationResults || !this.inputVideo)
+            return;
+        // this.ctx.save();
+        // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
+        if ((_a = this.segmentationResults) === null || _a === void 0 ? void 0 : _a.categoryMask) {
+            this.ctx.filter = 'blur(3px)';
+            this.ctx.globalCompositeOperation = 'copy';
+            const bitmap = await maskToBitmap(this.segmentationResults.categoryMask, this.inputVideo.videoWidth, this.inputVideo.videoHeight);
+            this.ctx.drawImage(bitmap, 0, 0);
+            this.ctx.filter = 'none';
+            this.ctx.globalCompositeOperation = 'source-in';
+            if (this.backgroundImage) {
+                this.ctx.drawImage(this.backgroundImage, 0, 0, this.backgroundImage.width, this.backgroundImage.height, 0, 0, this.canvas.width, this.canvas.height);
+            }
+            else {
+                this.ctx.fillStyle = '#00FF00';
+                this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
+            }
+            this.ctx.globalCompositeOperation = 'destination-over';
+        }
+        this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
+        // this.ctx.restore();
+    }
+    async blurBackground(frame) {
+        var _a, _b;
+        if (!this.ctx ||
+            !this.canvas ||
+            !((_b = (_a = this.segmentationResults) === null || _a === void 0 ? void 0 : _a.categoryMask) === null || _b === void 0 ? void 0 : _b.canvas) ||
+            !this.inputVideo) {
+            return;
+        }
+        this.ctx.save();
+        this.ctx.globalCompositeOperation = 'copy';
+        const bitmap = await maskToBitmap(this.segmentationResults.categoryMask, this.inputVideo.videoWidth, this.inputVideo.videoHeight);
+        this.ctx.filter = 'blur(3px)';
+        this.ctx.globalCompositeOperation = 'copy';
+        this.ctx.drawImage(bitmap, 0, 0);
+        this.ctx.filter = 'none';
+        this.ctx.globalCompositeOperation = 'source-out';
+        this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
+        this.ctx.globalCompositeOperation = 'destination-over';
+        this.ctx.filter = `blur(${this.blurRadius}px)`;
+        this.ctx.drawImage(frame, 0, 0);
+        this.ctx.restore();
+    }
+}
+function maskToBitmap(mask, videoWidth, videoHeight) {
+    const dataArray = new Uint8ClampedArray(videoWidth * videoHeight * 4);
+    const result = mask.getAsUint8Array();
+    for (let i = 0; i < result.length; i += 1) {
+        dataArray[i * 4] = result[i];
+        dataArray[i * 4 + 1] = result[i];
+        dataArray[i * 4 + 2] = result[i];
+        dataArray[i * 4 + 3] = result[i];
+    }
+    const dataNew = new ImageData(dataArray, videoWidth, videoHeight);
+    return createImageBitmap(dataNew);
+}
+//# sourceMappingURL=BackgroundTransformer.js.map

package/dist/src/transformers/BackgroundTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BackgroundTransformer.js","sourceRoot":"","sources":["../../../src/transformers/BackgroundTransformer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAQlD,MAAM,CAAC,OAAO,OAAO,mBAAoB,SAAQ,gBAAgB;IAC/D,MAAM,KAAK,WAAW;QACpB,OAAO,OAAO,eAAe,KAAK,WAAW,CAAC;IAChD,CAAC;IAUD,YAAY,IAAuB;QACjC,KAAK,EAAE,CAAC;QALV,oBAAe,GAAuB,IAAI,CAAC;QAMzC,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;SACnC;aAAM,IAAI,IAAI,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAA+B;QAChF,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;QAEvD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,cAAc,CACzD,wDAAwD,YAAY,CAAC,yBAAyB,CAAC,OAAO,CACvG,CAAC;QAEF,IAAI,CAAC,cAAc,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,iBAAiB,CAAC,OAAO,EAAE;YAC3E,WAAW,EAAE;gBACX,cAAc,EACZ,yHAAyH;gBAC3H,QAAQ,EAAE,KAAK;aAChB;YACD,WAAW,EAAE,OAAO;YACpB,kBAAkB,EAAE,IAAI;YACxB,qBAAqB,EAAE,KAAK;SAC7B,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,CAAC,qCAAqC,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,CAAA,MAAA,IAAI,CAAC,cAAc,0CAAE,KAAK,EAAE,CAAA,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,qCAAqC,CAAC,OAAyB;;QACnE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE;gBACrD,IAAI,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBACpC,MAAA,IAAI,CAAC,cAAc,0CAAE,eAAe,CAClC,OAAO,EACP,WAAW,EACX,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,CAChD,CAAC;aACH;YACD,OAAO,CAAC,yBAAyB,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,qCAAqC,CAAC,OAAO,CAAC,CAAC;YACtD,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAY;QAC/B,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;QAExB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACpC,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;YAC9B,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,GAAG,CAAC,OAAO,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACnC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiB,EAAE,UAAwD;QACzF,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC1B,OAAO;SACR;QACD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,SAAS,CAAC,sCAAsC,CAAC,CAAC;SACzD;QACD,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;SAClC;aAAM;YACL,MAAM,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;SACzC;QACD,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE;YAC3C,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;SACzC,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,KAAiB;;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QACvF,mBAAmB;QACnB,mEAAmE;QACnE,IAAI,MAAA,IAAI,CAAC,mBAAmB,0CAAE,YAAY,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,MAAM,CAAC;YAC3C,MAAM,MAAM,GAAG,MAAM,YAAY,CAC/B,IAAI,CAAC,mBAAmB,CAAC,YAAY,EACrC,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,WAAW,CAAC;YAChD,IAAI,IAAI,CAAC,eAAe,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,SAAS,CAChB,IAAI,CAAC,eAAe,EACpB,CAAC,EACD,CAAC,EACD,IAAI,CAAC,eAAe,CAAC,KAAK,EAC1B,IAAI,CAAC,eAAe,CAAC,MAAM,EAC3B,CAAC,EACD,CAAC,EACD,IAAI,CAAC,MAAM,CAAC,KAAK,EACjB,IAAI,CAAC,MAAM,CAAC,MAAM,CACnB,CAAC;aACH;iBAAM;gBACL,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC;gBAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aAChE;YAED,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,kBAAkB,CAAC;SACxD;QACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvE,sBAAsB;IACxB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAiB;;QACpC,IACE,CAAC,IAAI,CAAC,GAAG;YACT,CAAC,IAAI,CAAC,MAAM;YACZ,CAAC,CAAA,MAAA,MAAA,IAAI,CAAC,mBAAmB,0CAAE,YAAY,0CAAE,MAAM,CAAA;YAC/C,CAAC,IAAI,CAAC,UAAU,EAChB;YACA,OAAO;SACR;QACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAChB,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,MAAM,CAAC;QAE3C,MAAM,MAAM,GAAG,MAAM,YAAY,CAC/B,IAAI,CAAC,mBAAmB,CAAC,YAAY,EACrC,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,MAAM,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACjC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;QACzB,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,YAAY,CAAC;QACjD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvE,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,kBAAkB,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,IAAI,CAAC,UAAU,KAAK,CAAC;QAC/C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACrB,CAAC;CACF;AAED,SAAS,YAAY,CACnB,IAAmB,EACnB,UAAkB,EAClB,WAAmB;IAEnB,MAAM,SAAS,GAAsB,IAAI,iBAAiB,CAAC,UAAU,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;QACzC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7B,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACjC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACjC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;KAClC;IACD,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;IAElE,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACpC,CAAC"}

package/dist/src/transformers/DummyTransformer.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="dom-webcodecs" />
+import VideoTransformer from './VideoTransformer';
+export default class DummyTransformer extends VideoTransformer {
+    transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): Promise<void>;
+    destroy(): Promise<void>;
+}

package/dist/src/transformers/DummyTransformer.js

@@ -0,0 +1,10 @@
+import VideoTransformer from './VideoTransformer';
+export default class DummyTransformer extends VideoTransformer {
+    async transform(frame, controller) {
+        controller.enqueue(frame);
+    }
+    async destroy() {
+        // nothing to do
+    }
+}
+//# sourceMappingURL=DummyTransformer.js.map

package/dist/src/transformers/DummyTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DummyTransformer.js","sourceRoot":"","sources":["../../../src/transformers/DummyTransformer.ts"],"names":[],"mappings":"AAAA,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAElD,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,gBAAgB;IAC5D,KAAK,CAAC,SAAS,CAAC,KAAiB,EAAE,UAAwD;QACzF,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,gBAAgB;IAClB,CAAC;CACF"}

package/dist/src/transformers/MediaPipeHolisticTrackerTransformer.d.ts

@@ -0,0 +1,18 @@
+import { Holistic, Options, Results } from '@mediapipe/holistic';
+import VideoTransformer from './VideoTransformer';
+import { VideoTransformerInitOptions } from './types';
+export type MediaPipeHolisticTrackerTransformerOptions = {
+    holisticOptions?: Options;
+    callback?: (results: Results) => void;
+};
+export default class MediaPipeHolisticTrackerTransformer extends VideoTransformer {
+    holistic?: Holistic;
+    holisticOptions: Options;
+    callback: (results: Results) => void;
+    static get isSupported(): boolean;
+    constructor({ holisticOptions, callback }: MediaPipeHolisticTrackerTransformerOptions);
+    init({ inputElement: inputVideo, outputCanvas }: VideoTransformerInitOptions): void;
+    destroy(): Promise<void>;
+    transform(): Promise<void>;
+    sendFramesContinuouslyForTracking(videoEl: HTMLVideoElement): Promise<void>;
+}

package/dist/src/transformers/MediaPipeHolisticTrackerTransformer.js

@@ -0,0 +1,44 @@
+import { Holistic } from '@mediapipe/holistic';
+import VideoTransformer from './VideoTransformer';
+export default class MediaPipeHolisticTrackerTransformer extends VideoTransformer {
+    static get isSupported() {
+        return true;
+    }
+    constructor({ holisticOptions, callback }) {
+        super();
+        this.callback = callback || (() => null);
+        this.holisticOptions = holisticOptions || {};
+    }
+    init({ inputElement: inputVideo, outputCanvas }) {
+        super.init({ outputCanvas, inputElement: inputVideo });
+        this.holistic = new Holistic({
+            locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`,
+        });
+        this.holistic.setOptions(this.holisticOptions);
+        this.holistic.onResults((r) => {
+            this.callback(r);
+        });
+        this.sendFramesContinuouslyForTracking(this.inputVideo);
+    }
+    async destroy() {
+        var _a;
+        this.callback = () => null;
+        await super.destroy();
+        await ((_a = this.holistic) === null || _a === void 0 ? void 0 : _a.close());
+    }
+    async transform() {
+        return;
+    }
+    async sendFramesContinuouslyForTracking(videoEl) {
+        var _a;
+        if (!this.isDisabled) {
+            if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
+                await ((_a = this.holistic) === null || _a === void 0 ? void 0 : _a.send({ image: videoEl }));
+            }
+            videoEl.requestVideoFrameCallback(() => {
+                this.sendFramesContinuouslyForTracking(videoEl);
+            });
+        }
+    }
+}
+//# sourceMappingURL=MediaPipeHolisticTrackerTransformer.js.map

package/dist/src/transformers/MediaPipeHolisticTrackerTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MediaPipeHolisticTrackerTransformer.js","sourceRoot":"","sources":["../../../src/transformers/MediaPipeHolisticTrackerTransformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAoB,MAAM,qBAAqB,CAAC;AACjE,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAQlD,MAAM,CAAC,OAAO,OAAO,mCAAoC,SAAQ,gBAAgB;IAKxE,MAAM,KAAK,WAAW;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,EAAE,eAAe,EAAE,QAAQ,EAA8C;QACnF,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAA+B;QAC1E,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;QAEvD,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,oDAAoD,IAAI,EAAE;SACjF,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE;YAC5B,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,IAAI,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;QAC3B,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,CAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,KAAK,EAAE,CAAA,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,SAAS;QACb,OAAO;IACT,CAAC;IAED,KAAK,CAAC,iCAAiC,CAAC,OAAyB;;QAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE;gBACrD,MAAM,CAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA,CAAC;aAC/C;YAED,OAAO,CAAC,yBAAyB,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;CACF"}

package/dist/src/transformers/VideoTransformer.d.ts

@@ -0,0 +1,12 @@
+/// <reference types="dom-webcodecs" />
+import { VideoTrackTransformer, VideoTransformerInitOptions } from './types';
+export default abstract class VideoTransformer implements VideoTrackTransformer {
+    transformer?: TransformStream;
+    canvas?: OffscreenCanvas;
+    ctx?: OffscreenCanvasRenderingContext2D;
+    inputVideo?: HTMLVideoElement;
+    protected isDisabled?: Boolean;
+    init({ outputCanvas, inputElement: inputVideo }: VideoTransformerInitOptions): void;
+    destroy(): Promise<void>;
+    abstract transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): void;
+}

package/dist/src/transformers/VideoTransformer.js

@@ -0,0 +1,26 @@
+export default class VideoTransformer {
+    constructor() {
+        this.isDisabled = false;
+    }
+    init({ outputCanvas, inputElement: inputVideo }) {
+        var _a;
+        if (!(inputVideo instanceof HTMLVideoElement)) {
+            throw TypeError('Video transformer needs a HTMLVideoElement as input');
+        }
+        this.transformer = new TransformStream({
+            transform: (frame, controller) => this.transform(frame, controller),
+        });
+        this.canvas = outputCanvas || null;
+        if (outputCanvas) {
+            this.ctx = ((_a = this.canvas) === null || _a === void 0 ? void 0 : _a.getContext('2d', { readFrequently: true })) || undefined;
+        }
+        this.inputVideo = inputVideo;
+        this.isDisabled = false;
+    }
+    async destroy() {
+        this.isDisabled = true;
+        this.canvas = undefined;
+        this.ctx = undefined;
+    }
+}
+//# sourceMappingURL=VideoTransformer.js.map

package/dist/src/transformers/VideoTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VideoTransformer.js","sourceRoot":"","sources":["../../../src/transformers/VideoTransformer.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAgB,gBAAgB;IAA9C;QASY,eAAU,GAAa,KAAK,CAAC;IA2BzC,CAAC;IAzBC,IAAI,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAA+B;;QAC1E,IAAI,CAAC,CAAC,UAAU,YAAY,gBAAgB,CAAC,EAAE;YAC7C,MAAM,SAAS,CAAC,qDAAqD,CAAC,CAAC;SACxE;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,eAAe,CAAC;YACrC,SAAS,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC;SACpE,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC;QACnC,IAAI,YAAY,EAAE;YAChB,IAAI,CAAC,GAAG,GAAG,CAAA,MAAA,IAAI,CAAC,MAAM,0CAAE,UAAU,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,KAAI,SAAS,CAAC;SACjF;QACD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;IACvB,CAAC;CAMF"}

package/dist/src/transformers/index.d.ts

@@ -0,0 +1,2 @@
+export * from './BackgroundTransformer';
+export * from './types';

package/dist/src/transformers/index.js

@@ -0,0 +1,3 @@
+export * from './BackgroundTransformer';
+export * from './types';
+//# sourceMappingURL=index.js.map

package/dist/src/transformers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transformers/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,SAAS,CAAC"}

package/dist/src/transformers/types.d.ts

@@ -0,0 +1,14 @@
+/// <reference types="dom-webcodecs" />
+export type TrackTransformerInitOptions = {
+    inputElement: HTMLMediaElement;
+};
+export interface VideoTransformerInitOptions extends TrackTransformerInitOptions {
+    outputCanvas: OffscreenCanvas;
+    inputElement: HTMLVideoElement;
+}
+export interface VideoTrackTransformer<T extends TrackTransformerInitOptions = VideoTransformerInitOptions> {
+    init: (options: T) => void;
+    destroy: () => void;
+    transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
+    transformer?: TransformStream;
+}