@livekit/track-processors 0.1.0

package/dist/transformers/BackgroundTransformer.js

@@ -0,0 +1,139 @@
+import * as vision from '@mediapipe/tasks-vision';
+import VideoTransformer from './VideoTransformer';
+export default class BackgroundProcessor extends VideoTransformer {
+    static get isSupported() {
+        return typeof OffscreenCanvas !== 'undefined';
+    }
+    constructor(opts) {
+        super();
+        //   backgroundImagePattern: CanvasPattern | null = null;
+        this.backgroundImage = null;
+        if (opts.blurRadius) {
+            this.blurRadius = opts.blurRadius;
+        }
+        else if (opts.imagePath) {
+            this.loadBackground(opts.imagePath);
+        }
+    }
+    async init({ outputCanvas, inputVideo }) {
+        super.init({ outputCanvas, inputVideo });
+        const fileSet = await vision.FilesetResolver.forVisionTasks('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm');
+        this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {
+            baseOptions: {
+                modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',
+                delegate: 'CPU',
+            },
+            runningMode: 'VIDEO',
+            outputCategoryMask: true,
+            outputConfidenceMasks: false,
+        });
+        // this.loadBackground(opts.backgroundUrl).catch((e) => console.error(e));
+        this.sendFramesContinuouslyForSegmentation(this.inputVideo);
+    }
+    async destroy() {
+        var _a;
+        await super.destroy();
+        await ((_a = this.imageSegmenter) === null || _a === void 0 ? void 0 : _a.close());
+        this.backgroundImage = null;
+    }
+    async sendFramesContinuouslyForSegmentation(videoEl) {
+        var _a;
+        if (!this.isDisabled) {
+            if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
+                let startTimeMs = performance.now();
+                (_a = this.imageSegmenter) === null || _a === void 0 ? void 0 : _a.segmentForVideo(videoEl, startTimeMs, (result) => (this.segmentationResults = result));
+            }
+            videoEl.requestVideoFrameCallback(() => {
+                this.sendFramesContinuouslyForSegmentation(videoEl);
+            });
+        }
+    }
+    async loadBackground(path) {
+        const img = new Image();
+        await new Promise((resolve, reject) => {
+            img.crossOrigin = 'Anonymous';
+            img.onload = () => resolve(img);
+            img.onerror = (err) => reject(err);
+            img.src = path;
+        });
+        const imageData = await createImageBitmap(img);
+        this.backgroundImage = imageData;
+    }
+    async transform(frame, controller) {
+        if (this.isDisabled) {
+            controller.enqueue(frame);
+            return;
+        }
+        if (!this.canvas) {
+            throw TypeError('Canvas needs to be initialized first');
+        }
+        if (this.blurRadius) {
+            await this.blurBackground(frame);
+        }
+        else {
+            this.drawVirtualBackground(frame);
+        }
+        const newFrame = new VideoFrame(this.canvas, {
+            timestamp: frame.timestamp || Date.now(),
+        });
+        frame.close();
+        controller.enqueue(newFrame);
+    }
+    drawVirtualBackground(frame) {
+        var _a, _b, _c;
+        if (!this.canvas || !this.ctx)
+            return;
+        // this.ctx.save();
+        // this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
+        if ((_a = this.segmentationResults) === null || _a === void 0 ? void 0 : _a.categoryMask) {
+            this.ctx.filter = 'blur(3px)';
+            this.ctx.globalCompositeOperation = 'copy';
+            const dataNew = new ImageData(Uint8ClampedArray.from(this.segmentationResults.categoryMask.getAsUint8Array()), ((_b = this.inputVideo) === null || _b === void 0 ? void 0 : _b.videoWidth) || 0, ((_c = this.inputVideo) === null || _c === void 0 ? void 0 : _c.videoHeight) || 0);
+            this.ctx.putImageData(dataNew, 0, 0);
+            this.ctx.filter = 'none';
+            this.ctx.globalCompositeOperation = 'source-in';
+            if (this.backgroundImage) {
+                this.ctx.drawImage(this.backgroundImage, 0, 0, this.backgroundImage.width, this.backgroundImage.height, 0, 0, this.canvas.width, this.canvas.height);
+            }
+            else {
+                this.ctx.fillStyle = '#00FF00';
+                this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
+            }
+            this.ctx.globalCompositeOperation = 'destination-over';
+        }
+        this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
+        // this.ctx.restore();
+    }
+    async blurBackground(frame) {
+        var _a, _b;
+        const start = performance.now();
+        if (!this.ctx ||
+            !this.canvas ||
+            !((_b = (_a = this.segmentationResults) === null || _a === void 0 ? void 0 : _a.categoryMask) === null || _b === void 0 ? void 0 : _b.canvas) ||
+            !this.inputVideo)
+            return;
+        this.ctx.save();
+        this.ctx.globalCompositeOperation = 'copy';
+        const dataArray = new Uint8ClampedArray(this.inputVideo.videoWidth * this.inputVideo.videoHeight * 4);
+        const result = this.segmentationResults.categoryMask.getAsUint8Array();
+        for (let i = 0; i < result.length; i += 1) {
+            dataArray[i * 4] = result[i];
+            dataArray[i * 4 + 1] = result[i];
+            dataArray[i * 4 + 2] = result[i];
+            dataArray[i * 4 + 3] = result[i];
+        }
+        const dataNew = new ImageData(dataArray, this.inputVideo.videoWidth, this.inputVideo.videoHeight);
+        // this.ctx.filter = 'blur(3px)';
+        this.ctx.globalCompositeOperation = 'copy';
+        this.ctx.drawImage(await createImageBitmap(dataNew), 0, 0);
+        // this.ctx.filter = 'none';
+        this.ctx.globalCompositeOperation = 'source-out';
+        this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
+        this.ctx.globalCompositeOperation = 'destination-over';
+        this.ctx.filter = `blur(${this.blurRadius}px)`;
+        this.ctx.drawImage(frame, 0, 0);
+        this.ctx.restore();
+        console.log('draw time', performance.now() - start);
+    }
+}
+//# sourceMappingURL=BackgroundTransformer.js.map

package/dist/transformers/BackgroundTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"BackgroundTransformer.js","sourceRoot":"","sources":["../../src/transformers/BackgroundTransformer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,yBAAyB,CAAC;AAClD,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAQlD,MAAM,CAAC,OAAO,OAAO,mBAAoB,SAAQ,gBAAgB;IAC/D,MAAM,KAAK,WAAW;QACpB,OAAO,OAAO,eAAe,KAAK,WAAW,CAAC;IAChD,CAAC;IAWD,YAAY,IAAuB;QACjC,KAAK,EAAE,CAAC;QANV,yDAAyD;QACzD,oBAAe,GAAuB,IAAI,CAAC;QAMzC,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;SACnC;aAAM,IAAI,IAAI,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,UAAU,EAAgC;QACnE,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;QAEzC,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,cAAc,CACzD,kEAAkE,CACnE,CAAC;QAEF,IAAI,CAAC,cAAc,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,iBAAiB,CAAC,OAAO,EAAE;YAC3E,WAAW,EAAE;gBACX,cAAc,EACZ,yHAAyH;gBAC3H,QAAQ,EAAE,KAAK;aAChB;YACD,WAAW,EAAE,OAAO;YACpB,kBAAkB,EAAE,IAAI;YACxB,qBAAqB,EAAE,KAAK;SAC7B,CAAC,CAAC;QAEH,0EAA0E;QAC1E,IAAI,CAAC,qCAAqC,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,CAAA,MAAA,IAAI,CAAC,cAAc,0CAAE,KAAK,EAAE,CAAA,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,qCAAqC,CAAC,OAAyB;;QACnE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE;gBACrD,IAAI,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;gBACpC,MAAA,IAAI,CAAC,cAAc,0CAAE,eAAe,CAClC,OAAO,EACP,WAAW,EACX,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,CAChD,CAAC;aACH;YACD,OAAO,CAAC,yBAAyB,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,qCAAqC,CAAC,OAAO,CAAC,CAAC;YACtD,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAY;QAC/B,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;QAExB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACpC,GAAG,CAAC,WAAW,GAAG,WAAW,CAAC;YAC9B,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,GAAG,CAAC,OAAO,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACnC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC;QACjB,CAAC,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiB,EAAE,UAAwD;QACzF,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC1B,OAAO;SACR;QACD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAChB,MAAM,SAAS,CAAC,sCAAsC,CAAC,CAAC;SACzD;QACD,IAAI,IAAI,CAAC,UAAU,EAAE;YACnB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;SAClC;aAAM;YACL,IAAI,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC;SACnC;QACD,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE;YAC3C,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE;SACzC,CAAC,CAAC;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,qBAAqB,CAAC,KAAiB;;QACrC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO;QACtC,mBAAmB;QACnB,mEAAmE;QACnE,IAAI,MAAA,IAAI,CAAC,mBAAmB,0CAAE,YAAY,EAAE;YAC1C,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,MAAM,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,SAAS,CAC3B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,EAC/E,CAAA,MAAA,IAAI,CAAC,UAAU,0CAAE,UAAU,KAAI,CAAC,EAChC,CAAA,MAAA,IAAI,CAAC,UAAU,0CAAE,WAAW,KAAI,CAAC,CAClC,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;YACzB,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,WAAW,CAAC;YAChD,IAAI,IAAI,CAAC,eAAe,EAAE;gBACxB,IAAI,CAAC,GAAG,CAAC,SAAS,CAChB,IAAI,CAAC,eAAe,EACpB,CAAC,EACD,CAAC,EACD,IAAI,CAAC,eAAe,CAAC,KAAK,EAC1B,IAAI,CAAC,eAAe,CAAC,MAAM,EAC3B,CAAC,EACD,CAAC,EACD,IAAI,CAAC,MAAM,CAAC,KAAK,EACjB,IAAI,CAAC,MAAM,CAAC,MAAM,CACnB,CAAC;aACH;iBAAM;gBACL,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC;gBAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aAChE;YAED,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,kBAAkB,CAAC;SACxD;QACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvE,sBAAsB;IACxB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,KAAiB;;QACpC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,IACE,CAAC,IAAI,CAAC,GAAG;YACT,CAAC,IAAI,CAAC,MAAM;YACZ,CAAC,CAAA,MAAA,MAAA,IAAI,CAAC,mBAAmB,0CAAE,YAAY,0CAAE,MAAM,CAAA;YAC/C,CAAC,IAAI,CAAC,UAAU;YAEhB,OAAO;QACT,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAChB,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,MAAM,CAAC;QAE3C,MAAM,SAAS,GAAsB,IAAI,iBAAiB,CACxD,IAAI,CAAC,UAAU,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,GAAG,CAAC,CAC7D,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC;QACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACzC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAC7B,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACjC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACjC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAClC;QACD,MAAM,OAAO,GAAG,IAAI,SAAS,CAC3B,SAAS,EACT,IAAI,CAAC,UAAU,CAAC,UAAU,EAC1B,IAAI,CAAC,UAAU,CAAC,WAAW,CAC5B,CAAC;QACF,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,MAAM,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3D,4BAA4B;QAC5B,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,YAAY,CAAC;QACjD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvE,IAAI,CAAC,GAAG,CAAC,wBAAwB,GAAG,kBAAkB,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,IAAI,CAAC,UAAU,KAAK,CAAC;QAC/C,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACtD,CAAC;CACF"}

package/dist/transformers/DummyTransformer.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="dom-webcodecs" />
+import VideoTransformer from './VideoTransformer';
+export default class DummyTransformer extends VideoTransformer {
+    transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): Promise<void>;
+    destroy(): Promise<void>;
+}

package/dist/transformers/DummyTransformer.js

@@ -0,0 +1,10 @@
+import VideoTransformer from './VideoTransformer';
+export default class DummyTransformer extends VideoTransformer {
+    async transform(frame, controller) {
+        controller.enqueue(frame);
+    }
+    async destroy() {
+        // nothing to do
+    }
+}
+//# sourceMappingURL=DummyTransformer.js.map

package/dist/transformers/DummyTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DummyTransformer.js","sourceRoot":"","sources":["../../src/transformers/DummyTransformer.ts"],"names":[],"mappings":"AAAA,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAElD,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,gBAAgB;IAC5D,KAAK,CAAC,SAAS,CAAC,KAAiB,EAAE,UAAwD;QACzF,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,gBAAgB;IAClB,CAAC;CACF"}

package/dist/transformers/MediaPipeHolisticTrackerTransformer.d.ts

@@ -0,0 +1,18 @@
+import { Holistic, Options, Results } from '@mediapipe/holistic';
+import VideoTransformer from './VideoTransformer';
+import { StreamTransformerInitOptions } from './types';
+export type MediaPipeHolisticTrackerTransformerOptions = {
+    holisticOptions?: Options;
+    callback?: (results: Results) => void;
+};
+export default class MediaPipeHolisticTrackerTransformer extends VideoTransformer {
+    holistic?: Holistic;
+    holisticOptions: Options;
+    callback: (results: Results) => void;
+    static get isSupported(): boolean;
+    constructor({ holisticOptions, callback }: MediaPipeHolisticTrackerTransformerOptions);
+    init({ inputVideo, outputCanvas }: StreamTransformerInitOptions): void;
+    destroy(): Promise<void>;
+    transform(): Promise<void>;
+    sendFramesContinuouslyForTracking(videoEl: HTMLVideoElement): Promise<void>;
+}

package/dist/transformers/MediaPipeHolisticTrackerTransformer.js

@@ -0,0 +1,44 @@
+import { Holistic } from '@mediapipe/holistic';
+import VideoTransformer from './VideoTransformer';
+export default class MediaPipeHolisticTrackerTransformer extends VideoTransformer {
+    static get isSupported() {
+        return true;
+    }
+    constructor({ holisticOptions, callback }) {
+        super();
+        this.callback = callback || (() => null);
+        this.holisticOptions = holisticOptions || {};
+    }
+    init({ inputVideo, outputCanvas }) {
+        super.init({ outputCanvas, inputVideo });
+        this.holistic = new Holistic({
+            locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`,
+        });
+        this.holistic.setOptions(this.holisticOptions);
+        this.holistic.onResults((r) => {
+            this.callback(r);
+        });
+        this.sendFramesContinuouslyForTracking(this.inputVideo);
+    }
+    async destroy() {
+        var _a;
+        this.callback = () => null;
+        await super.destroy();
+        await ((_a = this.holistic) === null || _a === void 0 ? void 0 : _a.close());
+    }
+    async transform() {
+        return;
+    }
+    async sendFramesContinuouslyForTracking(videoEl) {
+        var _a;
+        if (!this.isDisabled) {
+            if (videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
+                await ((_a = this.holistic) === null || _a === void 0 ? void 0 : _a.send({ image: videoEl }));
+            }
+            videoEl.requestVideoFrameCallback(() => {
+                this.sendFramesContinuouslyForTracking(videoEl);
+            });
+        }
+    }
+}
+//# sourceMappingURL=MediaPipeHolisticTrackerTransformer.js.map

package/dist/transformers/MediaPipeHolisticTrackerTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MediaPipeHolisticTrackerTransformer.js","sourceRoot":"","sources":["../../src/transformers/MediaPipeHolisticTrackerTransformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAoB,MAAM,qBAAqB,CAAC;AACjE,OAAO,gBAAgB,MAAM,oBAAoB,CAAC;AAQlD,MAAM,CAAC,OAAO,OAAO,mCAAoC,SAAQ,gBAAgB;IAKxE,MAAM,KAAK,WAAW;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,EAAE,eAAe,EAAE,QAAQ,EAA8C;QACnF,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,CAAC,EAAE,UAAU,EAAE,YAAY,EAAgC;QAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,CAAC;QAEzC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,oDAAoD,IAAI,EAAE;SACjF,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE;YAC5B,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,IAAI,CAAC,QAAQ,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC;QAC3B,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,CAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,KAAK,EAAE,CAAA,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,SAAS;QACb,OAAO;IACT,CAAC;IAED,KAAK,CAAC,iCAAiC,CAAC,OAAyB;;QAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;YACpB,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE;gBACrD,MAAM,CAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA,CAAC;aAC/C;YAED,OAAO,CAAC,yBAAyB,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iCAAiC,CAAC,OAAO,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;CACF"}

package/dist/transformers/Untitled-1.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/transformers/Untitled-1.js

@@ -0,0 +1,219 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+// Copyright 2023 The MediaPipe Authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//      http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import vision from 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0';
+const { ImageSegmenter, SegmentationMask, FilesetResolver } = vision;
+// Get DOM elements
+const video = document.getElementById('webcam');
+const canvasElement = document.getElementById('canvas');
+const canvasCtx = canvasElement.getContext('2d');
+const webcamPredictions = document.getElementById('webcamPredictions');
+const demosSection = document.getElementById('demos');
+let enableWebcamButton;
+let webcamRunning = false;
+const videoHeight = '360px';
+const videoWidth = '480px';
+let runningMode = 'IMAGE';
+const resultWidthHeigth = 256;
+let imageSegmenter;
+let labels;
+const legendColors = [
+    [255, 197, 0, 255],
+    [128, 62, 117, 255],
+    [255, 104, 0, 255],
+    [166, 189, 215, 255],
+    [193, 0, 32, 255],
+    [206, 162, 98, 255],
+    [129, 112, 102, 255],
+    [0, 125, 52, 255],
+    [246, 118, 142, 255],
+    [0, 83, 138, 255],
+    [255, 112, 92, 255],
+    [83, 55, 112, 255],
+    [255, 142, 0, 255],
+    [179, 40, 81, 255],
+    [244, 200, 0, 255],
+    [127, 24, 13, 255],
+    [147, 170, 0, 255],
+    [89, 51, 21, 255],
+    [241, 58, 19, 255],
+    [35, 44, 22, 255],
+    [0, 161, 194, 255], // Vivid Blue
+];
+const createImageSegmenter = () => __awaiter(void 0, void 0, void 0, function* () {
+    const audio = yield FilesetResolver.forVisionTasks('https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm');
+    imageSegmenter = yield ImageSegmenter.createFromOptions(audio, {
+        baseOptions: {
+            modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/1/deeplab_v3.tflite',
+            delegate: 'GPU',
+        },
+        runningMode: runningMode,
+        outputCategoryMask: true,
+        outputConfidenceMasks: false,
+    });
+    labels = imageSegmenter.getLabels();
+    demosSection.classList.remove('invisible');
+});
+createImageSegmenter();
+const imageContainers = document.getElementsByClassName('segmentOnClick');
+// Add click event listeners for the img elements.
+for (let i = 0; i < imageContainers.length; i++) {
+    imageContainers[i].getElementsByTagName('img')[0].addEventListener('click', handleClick);
+}
+/**
+ * Demo 1: Segmented images on click and display results.
+ */
+let canvasClick;
+function handleClick(event) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // Do not segmented if imageSegmenter hasn't loaded
+        if (imageSegmenter === undefined) {
+            return;
+        }
+        canvasClick = event.target.parentElement.getElementsByTagName('canvas')[0];
+        canvasClick.classList.remove('removed');
+        canvasClick.width = event.target.naturalWidth;
+        canvasClick.height = event.target.naturalHeight;
+        const cxt = canvasClick.getContext('2d');
+        cxt.clearRect(0, 0, canvasClick.width, canvasClick.height);
+        cxt.drawImage(event.target, 0, 0, canvasClick.width, canvasClick.height);
+        event.target.style.opacity = 0;
+        // if VIDEO mode is initialized, set runningMode to IMAGE
+        if (runningMode === 'VIDEO') {
+            runningMode = 'IMAGE';
+            yield imageSegmenter.setOptions({
+                runningMode: runningMode,
+            });
+        }
+        // imageSegmenter.segment() when resolved will call the callback function.
+        imageSegmenter.segment(event.target, callback);
+    });
+}
+function callback(result) {
+    const cxt = canvasClick.getContext('2d');
+    const { width, height } = result.categoryMask;
+    let imageData = cxt.getImageData(0, 0, width, height).data;
+    canvasClick.width = width;
+    canvasClick.height = height;
+    let category = '';
+    const mask = result.categoryMask.getAsUint8Array();
+    for (let i in mask) {
+        if (mask[i] > 0) {
+            category = labels[mask[i]];
+        }
+        const legendColor = legendColors[mask[i] % legendColors.length];
+        imageData[i * 4] = (legendColor[0] + imageData[i * 4]) / 2;
+        imageData[i * 4 + 1] = (legendColor[1] + imageData[i * 4 + 1]) / 2;
+        imageData[i * 4 + 2] = (legendColor[2] + imageData[i * 4 + 2]) / 2;
+        imageData[i * 4 + 3] = (legendColor[3] + imageData[i * 4 + 3]) / 2;
+    }
+    const uint8Array = new Uint8ClampedArray(imageData.buffer);
+    const dataNew = new ImageData(uint8Array, width, height);
+    cxt.putImageData(dataNew, 0, 0);
+    const p = event.target.parentNode.getElementsByClassName('classification')[0];
+    p.classList.remove('removed');
+    p.innerText = 'Category: ' + category;
+}
+function callbackForVideo(result) {
+    let imageData = canvasCtx.getImageData(0, 0, video.videoWidth, video.videoHeight).data;
+    const mask = result.categoryMask.getAsFloat32Array();
+    let j = 0;
+    for (let i = 0; i < mask.length; ++i) {
+        const maskVal = Math.round(mask[i] * 255.0);
+        const legendColor = legendColors[maskVal % legendColors.length];
+        imageData[j] = (legendColor[0] + imageData[j]) / 2;
+        imageData[j + 1] = (legendColor[1] + imageData[j + 1]) / 2;
+        imageData[j + 2] = (legendColor[2] + imageData[j + 2]) / 2;
+        imageData[j + 3] = (legendColor[3] + imageData[j + 3]) / 2;
+        j += 4;
+    }
+    const uint8Array = new Uint8ClampedArray(imageData.buffer);
+    const dataNew = new ImageData(uint8Array, video.videoWidth, video.videoHeight);
+    canvasCtx.putImageData(dataNew, 0, 0);
+    if (webcamRunning === true) {
+        window.requestAnimationFrame(predictWebcam);
+    }
+}
+/********************************************************************
+// Demo 2: Continuously grab image from webcam stream and segmented it.
+********************************************************************/
+// Check if webcam access is supported.
+function hasGetUserMedia() {
+    return !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia);
+}
+// Get segmentation from the webcam
+let lastWebcamTime = -1;
+function predictWebcam() {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (video.currentTime === lastWebcamTime) {
+            if (webcamRunning === true) {
+                window.requestAnimationFrame(predictWebcam);
+            }
+            return;
+        }
+        lastWebcamTime = video.currentTime;
+        canvasCtx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);
+        // Do not segmented if imageSegmenter hasn't loaded
+        if (imageSegmenter === undefined) {
+            return;
+        }
+        // if image mode is initialized, create a new segmented with video runningMode
+        if (runningMode === 'IMAGE') {
+            runningMode = 'VIDEO';
+            yield imageSegmenter.setOptions({
+                runningMode: runningMode,
+            });
+        }
+        let startTimeMs = performance.now();
+        // Start segmenting the stream.
+        imageSegmenter.segmentForVideo(video, startTimeMs, callbackForVideo);
+    });
+}
+// Enable the live webcam view and start imageSegmentation.
+function enableCam(event) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (imageSegmenter === undefined) {
+            return;
+        }
+        if (webcamRunning === true) {
+            webcamRunning = false;
+            enableWebcamButton.innerText = 'ENABLE SEGMENTATION';
+        }
+        else {
+            webcamRunning = true;
+            enableWebcamButton.innerText = 'DISABLE SEGMENTATION';
+        }
+        // getUsermedia parameters.
+        const constraints = {
+            video: true,
+        };
+        // Activate the webcam stream.
+        video.srcObject = yield navigator.mediaDevices.getUserMedia(constraints);
+        video.addEventListener('loadeddata', predictWebcam);
+    });
+}
+// If webcam supported, add event listener to button.
+if (hasGetUserMedia()) {
+    enableWebcamButton = document.getElementById('webcamButton');
+    enableWebcamButton.addEventListener('click', enableCam);
+}
+else {
+    console.warn('getUserMedia() is not supported by your browser');
+}
+//# sourceMappingURL=Untitled-1.js.map

package/dist/transformers/Untitled-1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Untitled-1.js","sourceRoot":"","sources":["../../src/transformers/Untitled-1.ts"],"names":[],"mappings":";;;;;;;;;AAAA,wCAAwC;AACxC,kEAAkE;AAClE,mEAAmE;AACnE,0CAA0C;AAC1C,kDAAkD;AAClD,sEAAsE;AACtE,oEAAoE;AACpE,2EAA2E;AAC3E,sEAAsE;AACtE,iCAAiC;AACjC,OAAO,MAAM,MAAM,6DAA6D,CAAC;AAEjF,MAAM,EAAE,cAAc,EAAE,gBAAgB,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;AAErE,mBAAmB;AACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAqB,CAAC;AACpE,MAAM,aAAa,GAAG,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAsB,CAAC;AAC7E,MAAM,SAAS,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;AACjD,MAAM,iBAAiB,GAAG,QAAQ,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;AACvE,MAAM,YAAY,GAAgB,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AACnE,IAAI,kBAAqC,CAAC;AAC1C,IAAI,aAAa,GAAY,KAAK,CAAC;AACnC,MAAM,WAAW,GAAW,OAAO,CAAC;AACpC,MAAM,UAAU,GAAW,OAAO,CAAC;AACnC,IAAI,WAAW,GAAsB,OAAO,CAAC;AAC7C,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,IAAI,cAA8B,CAAC;AACnC,IAAI,MAAqB,CAAC;AAE1B,MAAM,YAAY,GAAG;IACnB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;IAClB,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;IACnB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;IAClB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;IACpB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC;IACjB,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC;IACnB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;IACpB,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC;IACjB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;IACpB,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;IACjB,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC;IACnB,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC;IAClB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;IAClB,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC;IAClB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;IAClB,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC;IAClB,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;IAClB,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC;IACjB,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC;IAClB,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC;IACjB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,aAAa;CAClC,CAAC;AAEF,MAAM,oBAAoB,GAAG,GAAS,EAAE;IACtC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,cAAc,CAChD,kEAAkE,CACnE,CAAC;IAEF,cAAc,GAAG,MAAM,cAAc,CAAC,iBAAiB,CAAC,KAAK,EAAE;QAC7D,WAAW,EAAE;YACX,cAAc,EACZ,wGAAwG;YAC1G,QAAQ,EAAE,KAAK;SAChB;QACD,WAAW,EAAE,WAAW;QACxB,kBAAkB,EAAE,IAAI;QACxB,qBAAqB,EAAE,KAAK;KAC7B,CAAC,CAAC;IACH,MAAM,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC;IACpC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;AAC7C,CAAC,CAAA,CAAC;AACF,oBAAoB,EAAE,CAAC;AAEvB,MAAM,eAAe,GACnB,QAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;AAEpD,kDAAkD;AAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IAC/C,eAAe,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;CAC1F;AAED;;GAEG;AACH,IAAI,WAA8B,CAAC;AACnC,SAAe,WAAW,CAAC,KAAK;;QAC9B,mDAAmD;QACnD,IAAI,cAAc,KAAK,SAAS,EAAE;YAChC,OAAO;SACR;QACD,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACxC,WAAW,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC;QAC9C,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC;QAChD,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACzC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAC3D,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QACzE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;QAC/B,yDAAyD;QACzD,IAAI,WAAW,KAAK,OAAO,EAAE;YAC3B,WAAW,GAAG,OAAO,CAAC;YACtB,MAAM,cAAc,CAAC,UAAU,CAAC;gBAC9B,WAAW,EAAE,WAAW;aACzB,CAAC,CAAC;SACJ;QAED,0EAA0E;QAC1E,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;CAAA;AAED,SAAS,QAAQ,CAAC,MAA4B;IAC5C,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC;IAC9C,IAAI,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC3D,WAAW,CAAC,KAAK,GAAG,KAAK,CAAC;IAC1B,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC;IAC5B,IAAI,QAAQ,GAAW,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAa,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC;IAC7D,KAAK,IAAI,CAAC,IAAI,IAAI,EAAE;QAClB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE;YACf,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QAChE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACnE,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACnE,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;KACpE;IACD,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACzD,GAAG,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAChC,MAAM,CAAC,GAAgB,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC9B,CAAC,CAAC,SAAS,GAAG,YAAY,GAAG,QAAQ,CAAC;AACxC,CAAC;AAED,SAAS,gBAAgB,CAAC,MAA4B;IACpD,IAAI,SAAS,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC;IACvF,MAAM,IAAI,GAAa,MAAM,CAAC,YAAY,CAAC,iBAAiB,EAAE,CAAC;IAC/D,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QAChE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACnD,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3D,CAAC,IAAI,CAAC,CAAC;KACR;IACD,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/E,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,IAAI,aAAa,KAAK,IAAI,EAAE;QAC1B,MAAM,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAC;KAC7C;AACH,CAAC;AAED;;qEAEqE;AAErE,uCAAuC;AACvC,SAAS,eAAe;IACtB,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,IAAI,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;AAC3E,CAAC;AAED,mCAAmC;AACnC,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC;AACxB,SAAe,aAAa;;QAC1B,IAAI,KAAK,CAAC,WAAW,KAAK,cAAc,EAAE;YACxC,IAAI,aAAa,KAAK,IAAI,EAAE;gBAC1B,MAAM,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAC;aAC7C;YACD,OAAO;SACR;QACD,cAAc,GAAG,KAAK,CAAC,WAAW,CAAC;QACnC,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;QACtE,mDAAmD;QACnD,IAAI,cAAc,KAAK,SAAS,EAAE;YAChC,OAAO;SACR;QACD,8EAA8E;QAC9E,IAAI,WAAW,KAAK,OAAO,EAAE;YAC3B,WAAW,GAAG,OAAO,CAAC;YACtB,MAAM,cAAc,CAAC,UAAU,CAAC;gBAC9B,WAAW,EAAE,WAAW;aACzB,CAAC,CAAC;SACJ;QACD,IAAI,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,+BAA+B;QAC/B,cAAc,CAAC,eAAe,CAAC,KAAK,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC;IACvE,CAAC;CAAA;AAED,2DAA2D;AAC3D,SAAe,SAAS,CAAC,KAAK;;QAC5B,IAAI,cAAc,KAAK,SAAS,EAAE;YAChC,OAAO;SACR;QAED,IAAI,aAAa,KAAK,IAAI,EAAE;YAC1B,aAAa,GAAG,KAAK,CAAC;YACtB,kBAAkB,CAAC,SAAS,GAAG,qBAAqB,CAAC;SACtD;aAAM;YACL,aAAa,GAAG,IAAI,CAAC;YACrB,kBAAkB,CAAC,SAAS,GAAG,sBAAsB,CAAC;SACvD;QAED,2BAA2B;QAC3B,MAAM,WAAW,GAAG;YAClB,KAAK,EAAE,IAAI;SACZ,CAAC;QAEF,8BAA8B;QAC9B,KAAK,CAAC,SAAS,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QACzE,KAAK,CAAC,gBAAgB,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IACtD,CAAC;CAAA;AAED,qDAAqD;AACrD,IAAI,eAAe,EAAE,EAAE;IACrB,kBAAkB,GAAG,QAAQ,CAAC,cAAc,CAAC,cAAc,CAAsB,CAAC;IAClF,kBAAkB,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;CACzD;KAAM;IACL,OAAO,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;CACjE"}

package/dist/transformers/VideoTransformer.d.ts

@@ -0,0 +1,12 @@
+/// <reference types="dom-webcodecs" />
+import { StreamTransformer, StreamTransformerInitOptions } from './types';
+export default abstract class VideoTransformer implements StreamTransformer {
+    transformer?: TransformStream;
+    canvas?: OffscreenCanvas;
+    ctx?: OffscreenCanvasRenderingContext2D;
+    inputVideo?: HTMLVideoElement;
+    protected isDisabled?: Boolean;
+    init({ outputCanvas, inputVideo }: StreamTransformerInitOptions): void;
+    destroy(): Promise<void>;
+    abstract transform(frame: VideoFrame, controller: TransformStreamDefaultController<VideoFrame>): void;
+}

package/dist/transformers/VideoTransformer.js

@@ -0,0 +1,23 @@
+export default class VideoTransformer {
+    constructor() {
+        this.isDisabled = false;
+    }
+    init({ outputCanvas, inputVideo }) {
+        var _a;
+        this.transformer = new TransformStream({
+            transform: (frame, controller) => this.transform(frame, controller),
+        });
+        this.canvas = outputCanvas || null;
+        if (outputCanvas) {
+            this.ctx = ((_a = this.canvas) === null || _a === void 0 ? void 0 : _a.getContext('2d', { readFrequently: true })) || undefined;
+        }
+        this.inputVideo = inputVideo;
+        this.isDisabled = false;
+    }
+    async destroy() {
+        this.isDisabled = true;
+        this.canvas = undefined;
+        this.ctx = undefined;
+    }
+}
+//# sourceMappingURL=VideoTransformer.js.map

package/dist/transformers/VideoTransformer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VideoTransformer.js","sourceRoot":"","sources":["../../src/transformers/VideoTransformer.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,OAAO,OAAgB,gBAAgB;IAA9C;QASY,eAAU,GAAa,KAAK,CAAC;IAwBzC,CAAC;IAtBC,IAAI,CAAC,EAAE,YAAY,EAAE,UAAU,EAAgC;;QAC7D,IAAI,CAAC,WAAW,GAAG,IAAI,eAAe,CAAC;YACrC,SAAS,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC;SACpE,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,YAAY,IAAI,IAAI,CAAC;QACnC,IAAI,YAAY,EAAE;YAChB,IAAI,CAAC,GAAG,GAAG,CAAA,MAAA,IAAI,CAAC,MAAM,0CAAE,UAAU,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,KAAI,SAAS,CAAC;SACjF;QACD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;IACvB,CAAC;CAMF"}

package/dist/transformers/index.d.ts

@@ -0,0 +1,2 @@
+export * from './BackgroundTransformer';
+export * from './types';

package/dist/transformers/index.js

@@ -0,0 +1,3 @@
+export * from './BackgroundTransformer';
+export * from './types';
+//# sourceMappingURL=index.js.map

package/dist/transformers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/transformers/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,SAAS,CAAC"}

package/dist/transformers/types.d.ts

@@ -0,0 +1,11 @@
+/// <reference types="dom-webcodecs" />
+export type StreamTransformerInitOptions = {
+    outputCanvas: OffscreenCanvas;
+    inputVideo: HTMLVideoElement;
+};
+export interface StreamTransformer {
+    init: (options: StreamTransformerInitOptions) => void;
+    destroy: () => void;
+    transform: (frame: VideoFrame, controller: TransformStreamDefaultController) => void;
+    transformer?: TransformStream;
+}

package/dist/transformers/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/transformers/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/transformers/types.ts"],"names":[],"mappings":""}

package/dist/utils.d.ts

@@ -0,0 +1,2 @@
+export declare const supportsProcessor: boolean;
+export declare const supportsOffscreenCanvas: boolean;

package/dist/utils.js

@@ -0,0 +1,3 @@
+export const supportsProcessor = typeof MediaStreamTrackGenerator !== 'undefined';
+export const supportsOffscreenCanvas = typeof OffscreenCanvas !== 'undefined';
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAO,yBAAyB,KAAK,WAAW,CAAC;AAClF,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,eAAe,KAAK,WAAW,CAAC"}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@livekit/track-processors",
+  "version": "0.1.0",
+  "description": "LiveKit track processors",
+  "main": "dist/index.js",
+  "source": "src/index.ts",
+  "types": "dist/index.d.ts",
+  "repository": "git@github.com:livekit/livekit-track-processors.git",
+  "author": "Lukas Seiler",
+  "license": "Apache-2.0",
+  "scripts": {
+    "build": "yarn exec tsc",
+    "build-docs": "yarn exec typedoc",
+    "build-sample": "cd example && webpack && cp styles.css index.html dist/",
+    "lint": "eslint src",
+    "test": "jest",
+    "sample": "vite serve example --port 8080 --open"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@mediapipe/holistic": "^0.5.1675471629",
+    "@mediapipe/tasks-vision": "^0.10.0"
+  },
+  "peerDependencies": {
+    "livekit-client": "^1.10.0"
+  },
+  "devDependencies": {
+    "@trivago/prettier-plugin-sort-imports": "^4.1.1",
+    "@types/dom-mediacapture-transform": "^0.1.5",
+    "@types/jest": "^27.0.3",
+    "@types/offscreencanvas": "^2019.7.0",
+    "@typescript-eslint/eslint-plugin": "^4.31.2",
+    "@webpack-cli/serve": "^1.5.2",
+    "eslint": "8.39.0",
+    "eslint-config-airbnb-typescript": "17.0.0",
+    "eslint-config-prettier": "8.8.0",
+    "eslint-plugin-ecmascript-compat": "^3.0.0",
+    "eslint-plugin-import": "2.27.5",
+    "jest": "^27.4.3",
+    "livekit-client": "^0.16.6",
+    "prettier": "^2.8.8",
+    "ts-jest": "^27.0.7",
+    "ts-loader": "^8.1.0",
+    "ts-proto": "^1.85.0",
+    "typedoc": "^0.20.35",
+    "typedoc-plugin-no-inherit": "1.3.0",
+    "typescript": "^5.0.4",
+    "vite": "^4.3.8",
+    "webpack": "^5.53.0",
+    "webpack-cli": "^4.8.0",
+    "webpack-dev-server": "^4.2.1"
+  }
+}