@webex/ladon-ts 5.6.1 → 5.7.1

package/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### 5.7.1 (2025-06-17)
+
+
+### Bug Fixes
+
+* use live video element when skipping inference ([#71](https://github.com/beyonddunbar/ladon-ts/issues/71)) ([b916af0](https://github.com/beyonddunbar/ladon-ts/commit/b916af01cd66fd6e54030c95a19156d3b5656bcc))
+
+## 5.7.0 (2025-05-22)
+
+
+### Features
+
+* overhaul frame syncing ([#69](https://github.com/beyonddunbar/ladon-ts/issues/69)) ([e80969b](https://github.com/beyonddunbar/ladon-ts/commit/e80969bbe0c2a3edc9f592996c5e2179eec370dc))
+
 ### 5.6.1 (2025-05-21)
 
 

package/build/module/lib/generator-types.d.ts

@@ -26,6 +26,7 @@ export interface WorkerGeneratedResult {
 }
 export interface MaskGenerator {
     getLadonOutput(input: Uint8ClampedArray, timestamp: number): Promise<LadonOutput>;
+    getLadonOutputForTimestamp(timestamp: number): Promise<LadonOutput>;
     load(): Promise<void>;
     isLoaded(): boolean;
     postRender(input: Uint8ClampedArray, timestamp: number): Promise<void>;

package/build/module/lib/pipeline.d.ts

@@ -24,12 +24,11 @@ export declare class Pipeline implements IPipeline {
     private loaded;
     private inputCanvasCtx;
     private lastMaskResult?;
-    private frameQueue;
-    private maskQueue;
     private frameDataBuffer?;
     constructor(inputCanvas: HTMLCanvasElement, outputCanvas: HTMLCanvasElement, config: PipelineConfig, videoElement?: HTMLVideoElement | undefined);
     /**
      * Processes a single frame and returns the inference result.
+     * This method handles the frame data, performs inference if needed, and renders the output.
      *
      * @param timestamp - The timestamp of the frame.
      * @param performInference - Flag to perform mask inference.
@@ -37,10 +36,6 @@ export declare class Pipeline implements IPipeline {
      * @throws If the pipeline (generator or renderer) is not loaded.
      */
     onFrame(timestamp: number, performInference?: boolean): Promise<IFrameInferenceResult>;
-    /**
-     * Synchronizes frames and masks to ensure they are rendered together.
-     */
-    private renderSyncedFrameAndMask;
     /**
      * Gets the raw image data from the input canvas context.
      *
@@ -74,10 +69,6 @@ export declare class Pipeline implements IPipeline {
      * Loads the pipeline components such as the generator and renderer.
      */
     load(): Promise<void>;
-    /**
-     * Resets the frame and mask queues.
-     */
-    private resetQueues;
     reset(): void;
     /**
      * Destroys the pipeline and cleans up resources.

package/build/module/lib/pipeline.js

@@ -1,47 +1,38 @@
 import { WorkerGenerator as n } from "./worker-generator.js";
-import { BlurRenderer as h } from "./renderer/blur-renderer.js";
-import { PassthroughRenderer as u } from "./renderer/passthrough-renderer.js";
-import { ReplacementRenderer as o } from "./renderer/replacement-renderer.js";
-import { resizeImage as d } from "./util.js";
-class k {
-  constructor(e, t, s, r) {
-    this.inputCanvas = e, this.outputCanvas = t, this.config = s, this.videoElement = r, this.loaded = !1, this.frameQueue = [], this.maskQueue = [];
-    const a = this.inputCanvas.getContext("2d", {
+import { BlurRenderer as o } from "./renderer/blur-renderer.js";
+import { PassthroughRenderer as h } from "./renderer/passthrough-renderer.js";
+import { ReplacementRenderer as d } from "./renderer/replacement-renderer.js";
+import { resizeImage as u } from "./util.js";
+class w {
+  constructor(t, e, a, r) {
+    this.inputCanvas = t, this.outputCanvas = e, this.config = a, this.videoElement = r, this.loaded = !1;
+    const s = this.inputCanvas.getContext("2d", {
       alpha: !1,
       willReadFrequently: !0
     });
-    if (!a)
+    if (!s)
       throw new Error("[ladon-ts] Error getting input context 2d");
-    this.inputCanvasCtx = a;
+    this.inputCanvasCtx = s;
   }
   /**
    * Processes a single frame and returns the inference result.
+   * This method handles the frame data, performs inference if needed, and renders the output.
    *
    * @param timestamp - The timestamp of the frame.
    * @param performInference - Flag to perform mask inference.
    * @returns The result of the frame processing.
    * @throws If the pipeline (generator or renderer) is not loaded.
    */
-  async onFrame(e, t = !0) {
+  async onFrame(t, e = !0) {
     if (!this.generator || !this.renderer)
       throw new Error("[ladon-ts] Pipeline is not loaded.");
-    const s = this.getInputFrameData(t);
-    if (s instanceof Uint8ClampedArray ? ((!this.frameDataBuffer || this.frameDataBuffer.length !== s.length) && (this.frameDataBuffer = new Uint8ClampedArray(s.length)), this.frameDataBuffer.set(s), this.frameQueue.push({ timestamp: e, frameData: this.frameDataBuffer })) : this.frameQueue.push({ timestamp: e, frameData: s }), t || !this.lastMaskResult) {
-      const r = s, a = this.getMaskImageData(r), i = await this.generator.getLadonOutput(a, e);
-      this.maskQueue.push({ timestamp: i.timestamp, maskResult: i }), this.lastMaskResult = i, await this.renderSyncedFrameAndMask(), await this.generator.postRender(a, e);
-    } else
-      this.maskQueue.push({ timestamp: e, maskResult: this.lastMaskResult }), await this.renderSyncedFrameAndMask();
-    return this.createInferenceResult(e, t);
-  }
-  /**
-   * Synchronizes frames and masks to ensure they are rendered together.
-   */
-  async renderSyncedFrameAndMask() {
-    var e;
-    for (; this.frameQueue.length > 0 && this.maskQueue.length > 0; ) {
-      const t = this.frameQueue[0], s = this.maskQueue[0];
-      t.timestamp === s.timestamp ? (await ((e = this.renderer) == null ? void 0 : e.render(t.frameData, s.maskResult)), this.frameQueue.shift(), this.maskQueue.shift()) : t.timestamp < s.timestamp ? this.frameQueue.shift() : this.maskQueue.shift();
+    const a = this.getInputFrameData(e);
+    let r;
+    if (a instanceof Uint8ClampedArray ? ((!this.frameDataBuffer || this.frameDataBuffer.length !== a.length) && (this.frameDataBuffer = new Uint8ClampedArray(a.length)), this.frameDataBuffer.set(a), r = this.frameDataBuffer) : r = a, e || !this.lastMaskResult) {
+      const s = a, i = this.getMaskImageData(s);
+      await this.generator.postRender(i, t), this.lastMaskResult = await this.generator.getLadonOutputForTimestamp(t);
     }
+    return await this.renderer.render(r, this.lastMaskResult), this.createInferenceResult(t, e);
   }
   /**
    * Gets the raw image data from the input canvas context.
@@ -50,8 +41,8 @@ class k {
    *
    * @returns The raw image data.
    */
-  getInputFrameData(e) {
-    return e || !this.videoElement ? this.inputCanvasCtx.getImageData(
+  getInputFrameData(t) {
+    return t || !this.videoElement ? this.inputCanvasCtx.getImageData(
       0,
       0,
       this.config.input.width,
@@ -64,14 +55,14 @@ class k {
    * @param {Uint8ClampedArray} rawImageData - The raw image data.
    * @returns {Uint8ClampedArray} - The processed mask image data.
    */
-  getMaskImageData(e) {
-    const t = this.config.mask.inputSize, s = this.config.input;
-    return t.width === s.width && t.height === s.height ? e : d(
-      e,
-      s.width,
-      s.height,
-      t.width,
-      t.height
+  getMaskImageData(t) {
+    const e = this.config.mask.inputSize, a = this.config.input;
+    return e.width === a.width && e.height === a.height ? t : u(
+      t,
+      a.width,
+      a.height,
+      e.width,
+      e.height
     );
   }
   /**
@@ -81,19 +72,19 @@ class k {
    * @param {boolean} performedInference - Indicates if the inference was performed.
    * @returns {IFrameInferenceResult} - The result object containing inference details.
    */
-  createInferenceResult(e, t) {
+  createInferenceResult(t, e) {
     return this.lastMaskResult ? {
       facesAndLandmarks: this.lastMaskResult.facesAndLandmarks,
       gesture: this.lastMaskResult.gesture,
       motion: this.lastMaskResult.motion,
-      skipped: !t,
-      timestamp: e
+      skipped: !e,
+      timestamp: t
     } : {
       facesAndLandmarks: { data: new Float32Array(0), dims: [] },
       gesture: new Float32Array(0),
       motion: 0,
       skipped: !0,
-      timestamp: e
+      timestamp: t
     };
   }
   /**
@@ -110,37 +101,31 @@ class k {
   async load() {
     switch (this.config.render.type) {
       case "blur":
-        this.renderer = new h(this.outputCanvas, this.config);
+        this.renderer = new o(this.outputCanvas, this.config);
         break;
       case "passthrough":
-        this.renderer = new u(this.outputCanvas, this.config);
+        this.renderer = new h(this.outputCanvas, this.config);
         break;
       case "replacement":
-        this.renderer = new o(this.outputCanvas, this.config);
+        this.renderer = new d(this.outputCanvas, this.config);
         break;
       default:
         throw new Error("[ladon-ts] Renderer not implemented.");
     }
     await this.renderer.load(), this.generator = new n(this.config), await this.generator.load(), this.loaded = !0;
   }
-  /**
-   * Resets the frame and mask queues.
-   */
-  resetQueues() {
-    this.frameQueue = [], this.maskQueue = [];
-  }
   reset() {
-    this.resetQueues(), this.lastMaskResult = void 0;
+    this.lastMaskResult = void 0;
   }
   /**
    * Destroys the pipeline and cleans up resources.
    */
   async destroy() {
-    var e, t;
-    this.reset(), (e = this.renderer) == null || e.destroy(), await ((t = this.generator) == null ? void 0 : t.destroy()), this.frameDataBuffer = void 0, this.loaded = !1;
+    var t, e;
+    this.reset(), (t = this.renderer) == null || t.destroy(), await ((e = this.generator) == null ? void 0 : e.destroy()), this.frameDataBuffer = void 0, this.loaded = !1;
   }
 }
 export {
-  k as Pipeline
+  w as Pipeline
 };
 //# sourceMappingURL=pipeline.js.map

package/build/module/lib/pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.js","sources":["../../../src/lib/pipeline.ts"],"sourcesContent":["import { PipelineConfig } from './config-types';\nimport {\n    FacesAndLandmarks,\n    FrameData,\n    LadonOutput,\n    MaskGenerator,\n    WorkerGenerator,\n} from './generator';\nimport { BlurRenderer, LadonRenderer, PassthroughRenderer, ReplacementRenderer } from './renderer';\nimport { resizeImage } from './util';\n\nexport interface IFrameInferenceResult {\n    facesAndLandmarks: FacesAndLandmarks;\n    gesture: Float32Array;\n    motion: number;\n    skipped: boolean;\n    timestamp: number;\n}\n\nexport interface IPipeline {\n    onFrame(timestamp: number, maskNeeded?: boolean): Promise<IFrameInferenceResult>;\n    isLoaded(): boolean;\n    load(): Promise<void>;\n    destroy(): Promise<void>;\n    reset(): void;\n}\n\nexport class Pipeline implements IPipeline {\n    private generator?: MaskGenerator;\n    private renderer?: LadonRenderer;\n    private loaded = false;\n\n    private inputCanvasCtx: CanvasRenderingContext2D;\n    private lastMaskResult?: LadonOutput;\n\n    private frameQueue: Array<{ timestamp: number; frameData: FrameData }> = [];\n    private maskQueue: Array<{ timestamp: number; maskResult: LadonOutput }> = [];\n\n    private frameDataBuffer?: Uint8ClampedArray;\n\n    constructor(\n        private inputCanvas: HTMLCanvasElement,\n        private outputCanvas: HTMLCanvasElement,\n        public readonly config: PipelineConfig,\n        private readonly videoElement?: HTMLVideoElement\n    ) {\n        const inputCanvasCtx = this.inputCanvas.getContext('2d', {\n            alpha: false,\n            willReadFrequently: true,\n        });\n\n        if (!inputCanvasCtx) {\n            throw new Error('[ladon-ts] Error getting input context 2d');\n        }\n        this.inputCanvasCtx = inputCanvasCtx;\n    }\n\n    /**\n     * Processes a single frame and returns the inference result.\n     *\n     * @param timestamp - The timestamp of the frame.\n     * @param performInference - Flag to perform mask inference.\n     * @returns The result of the frame processing.\n     * @throws If the pipeline (generator or renderer) is not loaded.\n     */\n    public async onFrame(\n        timestamp: number,\n        performInference = true\n    ): Promise<IFrameInferenceResult> {\n        if (!this.generator || !this.renderer) {\n            throw new Error('[ladon-ts] Pipeline is not loaded.');\n        }\n\n        // Get the current frame (raw image data or video element)\n        const frameData = this.getInputFrameData(performInference);\n\n        // Reuse or create buffer for frame data copies, which is needed to ensure the data is not modified\n        // or garbage collected before the inference is done.\n        if (frameData instanceof Uint8ClampedArray) {\n            if (!this.frameDataBuffer || this.frameDataBuffer.length !== frameData.length) {\n                this.frameDataBuffer = new Uint8ClampedArray(frameData.length);\n            }\n            this.frameDataBuffer.set(frameData);\n            this.frameQueue.push({ timestamp, frameData: this.frameDataBuffer });\n        } else {\n            // For video elements, just pass the reference\n            this.frameQueue.push({ timestamp, frameData });\n        }\n\n        if (performInference || !this.lastMaskResult) {\n            // Perform inference and add the new mask to the mask queue\n            const rawImageData = frameData as Uint8ClampedArray;\n            const maskImageData = this.getMaskImageData(rawImageData);\n            const maskResult = await this.generator.getLadonOutput(maskImageData, timestamp);\n            // Add the mask to the queue and use the mask timestamp to sync frames and masks.\n            this.maskQueue.push({ timestamp: maskResult.timestamp, maskResult });\n            this.lastMaskResult = maskResult;\n\n            // Sync frames and masks to render together\n            await this.renderSyncedFrameAndMask();\n\n            // Post-process the mask after rendering\n            await this.generator.postRender(maskImageData, timestamp);\n        } else {\n            // Skipping inference. Update the last mask result's timestamp to match the current frame.\n            this.maskQueue.push({ timestamp, maskResult: this.lastMaskResult });\n            // Sync frames and masks to render together\n            await this.renderSyncedFrameAndMask();\n        }\n\n        // Return the inference result\n        return this.createInferenceResult(timestamp, performInference);\n    }\n\n    /**\n     * Synchronizes frames and masks to ensure they are rendered together.\n     */\n    private async renderSyncedFrameAndMask(): Promise<void> {\n        while (this.frameQueue.length > 0 && this.maskQueue.length > 0) {\n            const frame = this.frameQueue[0];\n            const maskEntry = this.maskQueue[0];\n\n            if (frame.timestamp === maskEntry.timestamp) {\n                // Render the frame and mask together\n                await this.renderer?.render(frame.frameData, maskEntry.maskResult);\n                this.frameQueue.shift();\n                this.maskQueue.shift();\n            } else if (frame.timestamp < maskEntry.timestamp) {\n                // Discard the older frame\n                this.frameQueue.shift();\n            } else {\n                // Discard the older mask\n                this.maskQueue.shift();\n            }\n        }\n    }\n\n    /**\n     * Gets the raw image data from the input canvas context.\n     *\n     * @param needFreshImageData - Flag to indicate if fresh image data is needed.\n     *\n     * @returns The raw image data.\n     */\n    private getInputFrameData(needFreshImageData: boolean): FrameData {\n        if (needFreshImageData || !this.videoElement) {\n            return this.inputCanvasCtx.getImageData(\n                0,\n                0,\n                this.config.input.width,\n                this.config.input.height\n            ).data;\n        }\n\n        return this.videoElement;\n    }\n\n    /**\n     * Resizes the image data if the model expects a fixed size input.\n     *\n     * @param {Uint8ClampedArray} rawImageData - The raw image data.\n     * @returns {Uint8ClampedArray} - The processed mask image data.\n     */\n    private getMaskImageData(rawImageData: Uint8ClampedArray): Uint8ClampedArray {\n        const modelInput = this.config.mask.inputSize;\n        const inputImage = this.config.input;\n\n        // Check if resizing is needed based on configuration.\n        if (modelInput.width === inputImage.width && modelInput.height === inputImage.height) {\n            return rawImageData;\n        }\n\n        // Resize the image data.\n        return resizeImage(\n            rawImageData,\n            inputImage.width,\n            inputImage.height,\n            modelInput.width,\n            modelInput.height\n        );\n    }\n\n    /**\n     * Creates the inference result object.\n     *\n     * @param {number} timestamp - The timestamp of the frame.\n     * @param {boolean} performedInference - Indicates if the inference was performed.\n     * @returns {IFrameInferenceResult} - The result object containing inference details.\n     */\n    private createInferenceResult(\n        timestamp: number,\n        performedInference: boolean\n    ): IFrameInferenceResult {\n        if (!this.lastMaskResult) {\n            return {\n                facesAndLandmarks: { data: new Float32Array(0), dims: [] },\n                gesture: new Float32Array(0),\n                motion: 0,\n                skipped: true,\n                timestamp,\n            };\n        }\n\n        return {\n            facesAndLandmarks: this.lastMaskResult.facesAndLandmarks,\n            gesture: this.lastMaskResult.gesture,\n            motion: this.lastMaskResult.motion,\n            skipped: !performedInference,\n            timestamp,\n        };\n    }\n\n    /**\n     * Checks if the pipeline is loaded.\n     *\n     * @returns True if the pipeline is loaded, false otherwise.\n     */\n    public isLoaded(): boolean {\n        return this.loaded;\n    }\n\n    /**\n     * Loads the pipeline components such as the generator and renderer.\n     */\n    public async load(): Promise<void> {\n        switch (this.config.render.type) {\n            case 'blur':\n                this.renderer = new BlurRenderer(this.outputCanvas, this.config);\n                break;\n            case 'passthrough':\n                this.renderer = new PassthroughRenderer(this.outputCanvas, this.config);\n                break;\n            case 'replacement':\n                this.renderer = new ReplacementRenderer(this.outputCanvas, this.config);\n                break;\n            default:\n                throw new Error('[ladon-ts] Renderer not implemented.');\n        }\n\n        await this.renderer.load();\n\n        this.generator = new WorkerGenerator(this.config);\n        await this.generator.load();\n\n        this.loaded = true;\n    }\n\n    /**\n     * Resets the frame and mask queues.\n     */\n    private resetQueues(): void {\n        this.frameQueue = [];\n        this.maskQueue = [];\n    }\n\n    public reset(): void {\n        this.resetQueues();\n        this.lastMaskResult = undefined;\n    }\n\n    /**\n     * Destroys the pipeline and cleans up resources.\n     */\n    public async destroy(): Promise<void> {\n        this.reset();\n        this.renderer?.destroy();\n        await this.generator?.destroy();\n        this.frameDataBuffer = undefined;\n        this.loaded = false;\n    }\n}\n"],"names":["Pipeline","inputCanvas","outputCanvas","config","videoElement","inputCanvasCtx","timestamp","performInference","frameData","rawImageData","maskImageData","maskResult","frame","maskEntry","_a","needFreshImageData","modelInput","inputImage","resizeImage","performedInference","BlurRenderer","PassthroughRenderer","ReplacementRenderer","WorkerGenerator","_b"],"mappings":";;;;;AA2BO,MAAMA,EAA8B;AAAA,EAavC,YACYC,GACAC,GACQC,GACCC,GACnB;AAJU,SAAA,cAAAH,GACA,KAAA,eAAAC,GACQ,KAAA,SAAAC,GACC,KAAA,eAAAC,GAdrB,KAAQ,SAAS,IAKjB,KAAQ,aAAiE,CAAC,GAC1E,KAAQ,YAAmE,CAAC;AAUxE,UAAMC,IAAiB,KAAK,YAAY,WAAW,MAAM;AAAA,MACrD,OAAO;AAAA,MACP,oBAAoB;AAAA,IAAA,CACvB;AAED,QAAI,CAACA;AACK,YAAA,IAAI,MAAM,2CAA2C;AAE/D,SAAK,iBAAiBA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAW1B,MAAa,QACTC,GACAC,IAAmB,IACW;AAC9B,QAAI,CAAC,KAAK,aAAa,CAAC,KAAK;AACnB,YAAA,IAAI,MAAM,oCAAoC;AAIlD,UAAAC,IAAY,KAAK,kBAAkBD,CAAgB;AAerD,QAXAC,aAAqB,sBACjB,CAAC,KAAK,mBAAmB,KAAK,gBAAgB,WAAWA,EAAU,YACnE,KAAK,kBAAkB,IAAI,kBAAkBA,EAAU,MAAM,IAE5D,KAAA,gBAAgB,IAAIA,CAAS,GAClC,KAAK,WAAW,KAAK,EAAE,WAAAF,GAAW,WAAW,KAAK,iBAAiB,KAGnE,KAAK,WAAW,KAAK,EAAE,WAAAA,GAAW,WAAAE,GAAW,GAG7CD,KAAoB,CAAC,KAAK,gBAAgB;AAE1C,YAAME,IAAeD,GACfE,IAAgB,KAAK,iBAAiBD,CAAY,GAClDE,IAAa,MAAM,KAAK,UAAU,eAAeD,GAAeJ,CAAS;AAE/E,WAAK,UAAU,KAAK,EAAE,WAAWK,EAAW,WAAW,YAAAA,GAAY,GACnE,KAAK,iBAAiBA,GAGtB,MAAM,KAAK,yBAAyB,GAGpC,MAAM,KAAK,UAAU,WAAWD,GAAeJ,CAAS;AAAA,IAAA;AAGxD,WAAK,UAAU,KAAK,EAAE,WAAAA,GAAW,YAAY,KAAK,gBAAgB,GAElE,MAAM,KAAK,yBAAyB;AAIjC,WAAA,KAAK,sBAAsBA,GAAWC,CAAgB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA,EAMjE,MAAc,2BAA0C;;AACpD,WAAO,KAAK,WAAW,SAAS,KAAK,KAAK,UAAU,SAAS,KAAG;AACtD,YAAAK,IAAQ,KAAK,WAAW,CAAC,GACzBC,IAAY,KAAK,UAAU,CAAC;AAE9B,MAAAD,EAAM,cAAcC,EAAU,aAE9B,QAAMC,IAAA,KAAK,aAAL,gBAAAA,EAAe,OAAOF,EAAM,WAAWC,EAAU,cACvD,KAAK,WAAW,MAAM,GACtB,KAAK,UAAU,MAAM,KACdD,EAAM,YAAYC,EAAU,YAEnC,KAAK,WAAW,MAAM,IAGtB,KAAK,UAAU,MAAM;AAAA,IACzB;AAAA,EACJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUI,kBAAkBE,GAAwC;AAC1D,WAAAA,KAAsB,CAAC,KAAK,eACrB,KAAK,eAAe;AAAA,MACvB;AAAA,MACA;AAAA,MACA,KAAK,OAAO,MAAM;AAAA,MAClB,KAAK,OAAO,MAAM;AAAA,IAAA,EACpB,OAGC,KAAK;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASR,iBAAiBN,GAAoD;AACnE,UAAAO,IAAa,KAAK,OAAO,KAAK,WAC9BC,IAAa,KAAK,OAAO;AAG/B,WAAID,EAAW,UAAUC,EAAW,SAASD,EAAW,WAAWC,EAAW,SACnER,IAIJS;AAAA,MACHT;AAAA,MACAQ,EAAW;AAAA,MACXA,EAAW;AAAA,MACXD,EAAW;AAAA,MACXA,EAAW;AAAA,IACf;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUI,sBACJV,GACAa,GACqB;AACjB,WAAC,KAAK,iBAUH;AAAA,MACH,mBAAmB,KAAK,eAAe;AAAA,MACvC,SAAS,KAAK,eAAe;AAAA,MAC7B,QAAQ,KAAK,eAAe;AAAA,MAC5B,SAAS,CAACA;AAAA,MACV,WAAAb;AAAA,IACJ,IAfW;AAAA,MACH,mBAAmB,EAAE,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,GAAG;AAAA,MACzD,SAAS,IAAI,aAAa,CAAC;AAAA,MAC3B,QAAQ;AAAA,MACR,SAAS;AAAA,MACT,WAAAA;AAAA,IACJ;AAAA,EASJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQG,WAAoB;AACvB,WAAO,KAAK;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA,EAMhB,MAAa,OAAsB;AACvB,YAAA,KAAK,OAAO,OAAO,MAAM;AAAA,MAC7B,KAAK;AACD,aAAK,WAAW,IAAIc,EAAa,KAAK,cAAc,KAAK,MAAM;AAC/D;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ;AACU,cAAA,IAAI,MAAM,sCAAsC;AAAA,IAAA;AAGxD,UAAA,KAAK,SAAS,KAAK,GAEzB,KAAK,YAAY,IAAIC,EAAgB,KAAK,MAAM,GAC1C,MAAA,KAAK,UAAU,KAAK,GAE1B,KAAK,SAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA,EAMV,cAAoB;AACxB,SAAK,aAAa,CAAC,GACnB,KAAK,YAAY,CAAC;AAAA,EAAA;AAAA,EAGf,QAAc;AACjB,SAAK,YAAY,GACjB,KAAK,iBAAiB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA,EAM1B,MAAa,UAAyB;;AAClC,SAAK,MAAM,IACXT,IAAA,KAAK,aAAL,QAAAA,EAAe,WACT,QAAAU,IAAA,KAAK,cAAL,gBAAAA,EAAgB,YACtB,KAAK,kBAAkB,QACvB,KAAK,SAAS;AAAA,EAAA;AAEtB;"}
+{"version":3,"file":"pipeline.js","sources":["../../../src/lib/pipeline.ts"],"sourcesContent":["import { PipelineConfig } from './config-types';\nimport {\n    FacesAndLandmarks,\n    FrameData,\n    LadonOutput,\n    MaskGenerator,\n    WorkerGenerator,\n} from './generator';\nimport { BlurRenderer, LadonRenderer, PassthroughRenderer, ReplacementRenderer } from './renderer';\nimport { resizeImage } from './util';\n\nexport interface IFrameInferenceResult {\n    facesAndLandmarks: FacesAndLandmarks;\n    gesture: Float32Array;\n    motion: number;\n    skipped: boolean;\n    timestamp: number;\n}\n\nexport interface IPipeline {\n    onFrame(timestamp: number, maskNeeded?: boolean): Promise<IFrameInferenceResult>;\n    isLoaded(): boolean;\n    load(): Promise<void>;\n    destroy(): Promise<void>;\n    reset(): void;\n}\n\nexport class Pipeline implements IPipeline {\n    private generator?: MaskGenerator;\n    private renderer?: LadonRenderer;\n    private loaded = false;\n\n    private inputCanvasCtx: CanvasRenderingContext2D;\n    private lastMaskResult?: LadonOutput;\n\n    private frameDataBuffer?: Uint8ClampedArray;\n\n    constructor(\n        private inputCanvas: HTMLCanvasElement,\n        private outputCanvas: HTMLCanvasElement,\n        public readonly config: PipelineConfig,\n        private readonly videoElement?: HTMLVideoElement\n    ) {\n        const inputCanvasCtx = this.inputCanvas.getContext('2d', {\n            alpha: false,\n            willReadFrequently: true,\n        });\n\n        if (!inputCanvasCtx) {\n            throw new Error('[ladon-ts] Error getting input context 2d');\n        }\n        this.inputCanvasCtx = inputCanvasCtx;\n    }\n\n    /**\n     * Processes a single frame and returns the inference result.\n     * This method handles the frame data, performs inference if needed, and renders the output.\n     *\n     * @param timestamp - The timestamp of the frame.\n     * @param performInference - Flag to perform mask inference.\n     * @returns The result of the frame processing.\n     * @throws If the pipeline (generator or renderer) is not loaded.\n     */\n    public async onFrame(\n        timestamp: number,\n        performInference = true\n    ): Promise<IFrameInferenceResult> {\n        if (!this.generator || !this.renderer) {\n            throw new Error('[ladon-ts] Pipeline is not loaded.');\n        }\n\n        // Get the current frame (raw image data or video element). Use the video element if skipping\n        // inference to avoid unnecessary processing when calling canvas getImageData.\n        const frameData = this.getInputFrameData(performInference);\n\n        // Reuse or create buffer for frame data copies, which is needed to ensure the data is not modified\n        // or garbage collected before the inference is done.\n        let frameDataCopy: FrameData;\n        if (frameData instanceof Uint8ClampedArray) {\n            if (!this.frameDataBuffer || this.frameDataBuffer.length !== frameData.length) {\n                this.frameDataBuffer = new Uint8ClampedArray(frameData.length);\n            }\n            this.frameDataBuffer.set(frameData);\n            frameDataCopy = this.frameDataBuffer;\n        } else {\n            frameDataCopy = frameData;\n        }\n\n        if (performInference || !this.lastMaskResult) {\n            // Send frame for inference and wait for the mask for this timestamp\n            const rawImageData = frameData as Uint8ClampedArray;\n            const maskImageData = this.getMaskImageData(rawImageData);\n            await this.generator.postRender(maskImageData, timestamp);\n            this.lastMaskResult = await this.generator.getLadonOutputForTimestamp(timestamp);\n        }\n\n        // Render the frame and mask together (mask may be stale if skipping inference)\n        await this.renderer.render(frameDataCopy, this.lastMaskResult);\n\n        return this.createInferenceResult(timestamp, performInference);\n    }\n\n    /**\n     * Gets the raw image data from the input canvas context.\n     *\n     * @param needFreshImageData - Flag to indicate if fresh image data is needed.\n     *\n     * @returns The raw image data.\n     */\n    private getInputFrameData(needFreshImageData: boolean): FrameData {\n        if (needFreshImageData || !this.videoElement) {\n            return this.inputCanvasCtx.getImageData(\n                0,\n                0,\n                this.config.input.width,\n                this.config.input.height\n            ).data;\n        }\n\n        return this.videoElement;\n    }\n\n    /**\n     * Resizes the image data if the model expects a fixed size input.\n     *\n     * @param {Uint8ClampedArray} rawImageData - The raw image data.\n     * @returns {Uint8ClampedArray} - The processed mask image data.\n     */\n    private getMaskImageData(rawImageData: Uint8ClampedArray): Uint8ClampedArray {\n        const modelInput = this.config.mask.inputSize;\n        const inputImage = this.config.input;\n\n        // Check if resizing is needed based on configuration.\n        if (modelInput.width === inputImage.width && modelInput.height === inputImage.height) {\n            return rawImageData;\n        }\n\n        // Resize the image data.\n        return resizeImage(\n            rawImageData,\n            inputImage.width,\n            inputImage.height,\n            modelInput.width,\n            modelInput.height\n        );\n    }\n\n    /**\n     * Creates the inference result object.\n     *\n     * @param {number} timestamp - The timestamp of the frame.\n     * @param {boolean} performedInference - Indicates if the inference was performed.\n     * @returns {IFrameInferenceResult} - The result object containing inference details.\n     */\n    private createInferenceResult(\n        timestamp: number,\n        performedInference: boolean\n    ): IFrameInferenceResult {\n        if (!this.lastMaskResult) {\n            return {\n                facesAndLandmarks: { data: new Float32Array(0), dims: [] },\n                gesture: new Float32Array(0),\n                motion: 0,\n                skipped: true,\n                timestamp,\n            };\n        }\n\n        return {\n            facesAndLandmarks: this.lastMaskResult.facesAndLandmarks,\n            gesture: this.lastMaskResult.gesture,\n            motion: this.lastMaskResult.motion,\n            skipped: !performedInference,\n            timestamp,\n        };\n    }\n\n    /**\n     * Checks if the pipeline is loaded.\n     *\n     * @returns True if the pipeline is loaded, false otherwise.\n     */\n    public isLoaded(): boolean {\n        return this.loaded;\n    }\n\n    /**\n     * Loads the pipeline components such as the generator and renderer.\n     */\n    public async load(): Promise<void> {\n        switch (this.config.render.type) {\n            case 'blur':\n                this.renderer = new BlurRenderer(this.outputCanvas, this.config);\n                break;\n            case 'passthrough':\n                this.renderer = new PassthroughRenderer(this.outputCanvas, this.config);\n                break;\n            case 'replacement':\n                this.renderer = new ReplacementRenderer(this.outputCanvas, this.config);\n                break;\n            default:\n                throw new Error('[ladon-ts] Renderer not implemented.');\n        }\n\n        await this.renderer.load();\n\n        this.generator = new WorkerGenerator(this.config);\n        await this.generator.load();\n\n        this.loaded = true;\n    }\n\n    public reset(): void {\n        this.lastMaskResult = undefined;\n    }\n\n    /**\n     * Destroys the pipeline and cleans up resources.\n     */\n    public async destroy(): Promise<void> {\n        this.reset();\n        this.renderer?.destroy();\n        await this.generator?.destroy();\n        this.frameDataBuffer = undefined;\n        this.loaded = false;\n    }\n}\n"],"names":["Pipeline","inputCanvas","outputCanvas","config","videoElement","inputCanvasCtx","timestamp","performInference","frameData","frameDataCopy","rawImageData","maskImageData","needFreshImageData","modelInput","inputImage","resizeImage","performedInference","BlurRenderer","PassthroughRenderer","ReplacementRenderer","WorkerGenerator","_a","_b"],"mappings":";;;;;AA2BO,MAAMA,EAA8B;AAAA,EAUvC,YACYC,GACAC,GACQC,GACCC,GACnB;AAJU,SAAA,cAAAH,GACA,KAAA,eAAAC,GACQ,KAAA,SAAAC,GACC,KAAA,eAAAC,GAXrB,KAAQ,SAAS;AAab,UAAMC,IAAiB,KAAK,YAAY,WAAW,MAAM;AAAA,MACrD,OAAO;AAAA,MACP,oBAAoB;AAAA,IAAA,CACvB;AAED,QAAI,CAACA;AACK,YAAA,IAAI,MAAM,2CAA2C;AAE/D,SAAK,iBAAiBA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAY1B,MAAa,QACTC,GACAC,IAAmB,IACW;AAC9B,QAAI,CAAC,KAAK,aAAa,CAAC,KAAK;AACnB,YAAA,IAAI,MAAM,oCAAoC;AAKlD,UAAAC,IAAY,KAAK,kBAAkBD,CAAgB;AAIrD,QAAAE;AAWA,QAVAD,aAAqB,sBACjB,CAAC,KAAK,mBAAmB,KAAK,gBAAgB,WAAWA,EAAU,YACnE,KAAK,kBAAkB,IAAI,kBAAkBA,EAAU,MAAM,IAE5D,KAAA,gBAAgB,IAAIA,CAAS,GAClCC,IAAgB,KAAK,mBAELA,IAAAD,GAGhBD,KAAoB,CAAC,KAAK,gBAAgB;AAE1C,YAAMG,IAAeF,GACfG,IAAgB,KAAK,iBAAiBD,CAAY;AACxD,YAAM,KAAK,UAAU,WAAWC,GAAeL,CAAS,GACxD,KAAK,iBAAiB,MAAM,KAAK,UAAU,2BAA2BA,CAAS;AAAA,IAAA;AAInF,iBAAM,KAAK,SAAS,OAAOG,GAAe,KAAK,cAAc,GAEtD,KAAK,sBAAsBH,GAAWC,CAAgB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzD,kBAAkBK,GAAwC;AAC1D,WAAAA,KAAsB,CAAC,KAAK,eACrB,KAAK,eAAe;AAAA,MACvB;AAAA,MACA;AAAA,MACA,KAAK,OAAO,MAAM;AAAA,MAClB,KAAK,OAAO,MAAM;AAAA,IAAA,EACpB,OAGC,KAAK;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASR,iBAAiBF,GAAoD;AACnE,UAAAG,IAAa,KAAK,OAAO,KAAK,WAC9BC,IAAa,KAAK,OAAO;AAG/B,WAAID,EAAW,UAAUC,EAAW,SAASD,EAAW,WAAWC,EAAW,SACnEJ,IAIJK;AAAA,MACHL;AAAA,MACAI,EAAW;AAAA,MACXA,EAAW;AAAA,MACXD,EAAW;AAAA,MACXA,EAAW;AAAA,IACf;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUI,sBACJP,GACAU,GACqB;AACjB,WAAC,KAAK,iBAUH;AAAA,MACH,mBAAmB,KAAK,eAAe;AAAA,MACvC,SAAS,KAAK,eAAe;AAAA,MAC7B,QAAQ,KAAK,eAAe;AAAA,MAC5B,SAAS,CAACA;AAAA,MACV,WAAAV;AAAA,IACJ,IAfW;AAAA,MACH,mBAAmB,EAAE,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,GAAG;AAAA,MACzD,SAAS,IAAI,aAAa,CAAC;AAAA,MAC3B,QAAQ;AAAA,MACR,SAAS;AAAA,MACT,WAAAA;AAAA,IACJ;AAAA,EASJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQG,WAAoB;AACvB,WAAO,KAAK;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA,EAMhB,MAAa,OAAsB;AACvB,YAAA,KAAK,OAAO,OAAO,MAAM;AAAA,MAC7B,KAAK;AACD,aAAK,WAAW,IAAIW,EAAa,KAAK,cAAc,KAAK,MAAM;AAC/D;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ;AACU,cAAA,IAAI,MAAM,sCAAsC;AAAA,IAAA;AAGxD,UAAA,KAAK,SAAS,KAAK,GAEzB,KAAK,YAAY,IAAIC,EAAgB,KAAK,MAAM,GAC1C,MAAA,KAAK,UAAU,KAAK,GAE1B,KAAK,SAAS;AAAA,EAAA;AAAA,EAGX,QAAc;AACjB,SAAK,iBAAiB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA,EAM1B,MAAa,UAAyB;;AAClC,SAAK,MAAM,IACXC,IAAA,KAAK,aAAL,QAAAA,EAAe,WACT,QAAAC,IAAA,KAAK,cAAL,gBAAAA,EAAgB,YACtB,KAAK,kBAAkB,QACvB,KAAK,SAAS;AAAA,EAAA;AAEtB;"}

package/build/module/lib/version.d.ts

@@ -1 +1 @@
-export declare const LIB_VERSION = "5.6.1";
+export declare const LIB_VERSION = "5.7.1";

package/build/module/lib/version.js

@@ -1,4 +1,4 @@
-const o = "5.6.1";
+const o = "5.7.1";
 export {
   o as LIB_VERSION
 };

package/build/module/lib/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sources":["../../../src/lib/version.ts"],"sourcesContent":["export const LIB_VERSION = '5.6.1';\n"],"names":["LIB_VERSION"],"mappings":"AAAO,MAAMA,IAAc;"}
+{"version":3,"file":"version.js","sources":["../../../src/lib/version.ts"],"sourcesContent":["export const LIB_VERSION = '5.7.1';\n"],"names":["LIB_VERSION"],"mappings":"AAAO,MAAMA,IAAc;"}

package/build/module/lib/worker-generator.d.ts

@@ -2,11 +2,21 @@ import { PipelineConfig } from './config-types';
 import { LadonOutput, MaskGenerator } from './generator-types';
 export declare class WorkerGenerator implements MaskGenerator {
     readonly config: PipelineConfig;
+    private pending;
     private lastResult;
     private newMask;
     constructor(config: PipelineConfig);
     isLoaded(): boolean;
     getLadonOutput(): Promise<LadonOutput>;
+    /**
+     * Returns a promise that resolves when the worker returns the mask for the given timestamp.
+     * This is useful for synchronizing the processing of frames with their corresponding masks.
+     * The promise will be resolved with the mask data when it is available.
+     *
+     * @param timestamp - The timestamp of the frame for which the mask is requested.
+     * @returns A promise that resolves with the mask data when available.
+     */
+    getLadonOutputForTimestamp(timestamp: number): Promise<LadonOutput>;
     /**
      * Loads the worker and initializes it with the provided configuration.
      * The worker is responsible for processing the input image and generating the mask.

package/build/module/lib/worker-generator.js

@@ -1,7 +1,7 @@
-import { workerManager as t } from "./worker-manager.js";
+import { workerManager as s } from "./worker-manager.js";
 class m {
   constructor(e) {
-    this.config = e, this.newMask = !0, this.lastResult = {
+    this.config = e, this.pending = /* @__PURE__ */ new Map(), this.newMask = !0, this.lastResult = {
       data: new Float32Array(
         this.config.mask.outputSize.height * this.config.mask.outputSize.width
       ),
@@ -13,7 +13,7 @@ class m {
     };
   }
   isLoaded() {
-    return !!t.getWorker();
+    return !!s.getWorker();
   }
   async getLadonOutput() {
     const e = {
@@ -26,6 +26,19 @@ class m {
     };
     return this.newMask = !1, e;
   }
+  /**
+   * Returns a promise that resolves when the worker returns the mask for the given timestamp.
+   * This is useful for synchronizing the processing of frames with their corresponding masks.
+   * The promise will be resolved with the mask data when it is available.
+   *
+   * @param timestamp - The timestamp of the frame for which the mask is requested.
+   * @returns A promise that resolves with the mask data when available.
+   */
+  async getLadonOutputForTimestamp(e) {
+    return new Promise((a) => {
+      this.pending.set(e, a);
+    });
+  }
   /**
    * Loads the worker and initializes it with the provided configuration.
    * The worker is responsible for processing the input image and generating the mask.
@@ -34,32 +47,38 @@ class m {
    * @returns A promise that resolves when the worker is loaded and initialized.
    */
   async load() {
-    const e = await t.prepareWorkerConfig(this.config);
+    const e = await s.prepareWorkerConfig(this.config);
     return new Promise((a) => {
-      t.onInitialized(() => {
+      s.onInitialized(() => {
         a();
-      }), t.onGenerated((s) => {
-        this.lastResult = {
-          data: s.mask,
-          facesAndLandmarks: s.facesAndLandmarks,
-          gesture: s.gesture,
-          motion: s.motion,
-          timestamp: s.timestamp,
+      }), s.onGenerated((t) => {
+        const r = {
+          data: t.mask,
+          facesAndLandmarks: t.facesAndLandmarks,
+          gesture: t.gesture,
+          motion: t.motion,
+          timestamp: t.timestamp,
           warm: !0
-        }, this.newMask = !0;
-      }), t.onLog((s, n) => {
-        const r = `[ladon-ts] Message received from worker: ${s}`;
-        switch (n) {
+        };
+        this.lastResult = r, this.newMask = !0;
+        const n = this.pending.get(t.timestamp);
+        n && (n(r), this.pending.delete(t.timestamp));
+      }), s.onDropped((t) => {
+        const r = this.pending.get(t);
+        r && (r(this.lastResult), this.pending.delete(t));
+      }), s.onLog((t, r) => {
+        const n = `[ladon-ts] Message received from worker: ${t}`;
+        switch (r) {
           case "warn":
-            console.warn(r);
+            console.warn(n);
             break;
           case "error":
-            console.error(r);
+            console.error(n);
             break;
           default:
-            console.log(r);
+            console.log(n);
         }
-      }), (async () => (await (t.getWorker() ?? t.initializeWorker(e)), t.sendInit({
+      }), (async () => (await (s.getWorker() ?? s.initializeWorker(e)), s.sendInit({
         baseUrl: e.baseUrl,
         input: e.input,
         mask: e.mask,
@@ -77,7 +96,7 @@ class m {
    * @param timestamp - The timestamp of the input image.
    */
   async postRender(e, a) {
-    t.sendGenerate(e, a);
+    s.sendGenerate(e, a);
   }
   /**
    * Destroys the worker and cleans up resources.
@@ -88,7 +107,7 @@ class m {
    * @returns A promise that resolves when the worker is destroyed.
    */
   destroy() {
-    return t.destroyWorker();
+    return s.destroyWorker();
   }
 }
 export {

package/build/module/lib/worker-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-generator.js","sources":["../../../src/lib/worker-generator.ts"],"sourcesContent":["import { PipelineConfig } from './config-types';\nimport { LadonOutput, MaskGenerator, WorkerGeneratedResult } from './generator-types';\nimport { workerManager } from './worker-manager';\n\nexport class WorkerGenerator implements MaskGenerator {\n    private lastResult: LadonOutput;\n    private newMask = true;\n\n    constructor(public readonly config: PipelineConfig) {\n        this.lastResult = {\n            data: new Float32Array(\n                this.config.mask.outputSize.height * this.config.mask.outputSize.width\n            ),\n            facesAndLandmarks: { data: new Float32Array(0), dims: [] },\n            gesture: new Float32Array(0),\n            motion: 0,\n            timestamp: 0,\n            warm: false,\n        };\n    }\n\n    public isLoaded(): boolean {\n        return !!workerManager.getWorker();\n    }\n\n    public async getLadonOutput(): Promise<LadonOutput> {\n        const toReturn = {\n            data: this.lastResult.data,\n            facesAndLandmarks: this.lastResult.facesAndLandmarks,\n            gesture: this.lastResult.gesture,\n            motion: this.lastResult.motion,\n            timestamp: this.lastResult.timestamp,\n            warm: this.newMask,\n        };\n\n        this.newMask = false;\n\n        return toReturn;\n    }\n\n    /**\n     * Loads the worker and initializes it with the provided configuration.\n     * The worker is responsible for processing the input image and generating the mask.\n     * Prepares the worker configuration by resolving the final URLs and paths to send to the worker.\n     *\n     * @returns A promise that resolves when the worker is loaded and initialized.\n     */\n    public async load(): Promise<void> {\n        const config = await workerManager.prepareWorkerConfig(this.config);\n\n        // Listen for worker events\n        return new Promise<void>((resolve) => {\n            workerManager.onInitialized(() => {\n                resolve();\n            });\n            workerManager.onGenerated((data: WorkerGeneratedResult) => {\n                this.lastResult = {\n                    data: data.mask,\n                    facesAndLandmarks: data.facesAndLandmarks,\n                    gesture: data.gesture,\n                    motion: data.motion,\n                    timestamp: data.timestamp,\n                    warm: true,\n                };\n                this.newMask = true;\n            });\n            workerManager.onLog((message, severity) => {\n                const msg = `[ladon-ts] Message received from worker: ${message}`;\n                switch (severity) {\n                    case 'warn':\n                        console.warn(msg);\n                        break;\n                    case 'error':\n                        console.error(msg);\n                        break;\n                    default:\n                        console.log(msg);\n                }\n            });\n\n            // Initialize the worker if not already done, then always send init\n            (async () => {\n                await (workerManager.getWorker() ?? workerManager.initializeWorker(config));\n                workerManager.sendInit({\n                    baseUrl: config.baseUrl,\n                    input: config.input,\n                    mask: config.mask,\n                    wasmUri: config.wasmUri,\n                    render: { type: 'passthrough' },\n                });\n            })();\n        });\n    }\n\n    /**\n     * Sends the input image to the worker for processing.\n     * The input image is expected to be a Uint8ClampedArray.\n     * Claims ownership of the input tensor's buffer and forwards it to the worker.\n     *\n     * @param input - The input image as a Uint8ClampedArray.\n     * @param timestamp - The timestamp of the input image.\n     */\n    public async postRender(input: Uint8ClampedArray, timestamp: number): Promise<void> {\n        workerManager.sendGenerate(input, timestamp);\n    }\n\n    /**\n     * Destroys the worker and cleans up resources.\n     * First it sends a message to the worker to destroy itself,\n     * then it waits for the worker to respond with a 'destroyed' message.\n     * After that, it terminates the worker and revokes the URL if it exists.\n     *\n     * @returns A promise that resolves when the worker is destroyed.\n     */\n    public destroy(): Promise<void> {\n        return workerManager.destroyWorker();\n    }\n}\n"],"names":["WorkerGenerator","config","workerManager","toReturn","resolve","data","message","severity","msg","input","timestamp"],"mappings":";AAIO,MAAMA,EAAyC;AAAA,EAIlD,YAA4BC,GAAwB;AAAxB,SAAA,SAAAA,GAF5B,KAAQ,UAAU,IAGd,KAAK,aAAa;AAAA,MACd,MAAM,IAAI;AAAA,QACN,KAAK,OAAO,KAAK,WAAW,SAAS,KAAK,OAAO,KAAK,WAAW;AAAA,MACrE;AAAA,MACA,mBAAmB,EAAE,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,GAAG;AAAA,MACzD,SAAS,IAAI,aAAa,CAAC;AAAA,MAC3B,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,MAAM;AAAA,IACV;AAAA,EAAA;AAAA,EAGG,WAAoB;AAChB,WAAA,CAAC,CAACC,EAAc,UAAU;AAAA,EAAA;AAAA,EAGrC,MAAa,iBAAuC;AAChD,UAAMC,IAAW;AAAA,MACb,MAAM,KAAK,WAAW;AAAA,MACtB,mBAAmB,KAAK,WAAW;AAAA,MACnC,SAAS,KAAK,WAAW;AAAA,MACzB,QAAQ,KAAK,WAAW;AAAA,MACxB,WAAW,KAAK,WAAW;AAAA,MAC3B,MAAM,KAAK;AAAA,IACf;AAEA,gBAAK,UAAU,IAERA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUX,MAAa,OAAsB;AAC/B,UAAMF,IAAS,MAAMC,EAAc,oBAAoB,KAAK,MAAM;AAG3D,WAAA,IAAI,QAAc,CAACE,MAAY;AAClC,MAAAF,EAAc,cAAc,MAAM;AACtB,QAAAE,EAAA;AAAA,MAAA,CACX,GACaF,EAAA,YAAY,CAACG,MAAgC;AACvD,aAAK,aAAa;AAAA,UACd,MAAMA,EAAK;AAAA,UACX,mBAAmBA,EAAK;AAAA,UACxB,SAASA,EAAK;AAAA,UACd,QAAQA,EAAK;AAAA,UACb,WAAWA,EAAK;AAAA,UAChB,MAAM;AAAA,QACV,GACA,KAAK,UAAU;AAAA,MAAA,CAClB,GACaH,EAAA,MAAM,CAACI,GAASC,MAAa;AACjC,cAAAC,IAAM,4CAA4CF,CAAO;AAC/D,gBAAQC,GAAU;AAAA,UACd,KAAK;AACD,oBAAQ,KAAKC,CAAG;AAChB;AAAA,UACJ,KAAK;AACD,oBAAQ,MAAMA,CAAG;AACjB;AAAA,UACJ;AACI,oBAAQ,IAAIA,CAAG;AAAA,QAAA;AAAA,MACvB,CACH,IAGA,aACG,OAAON,EAAc,UAAe,KAAAA,EAAc,iBAAiBD,CAAM,IACzEC,EAAc,SAAS;AAAA,QACnB,SAASD,EAAO;AAAA,QAChB,OAAOA,EAAO;AAAA,QACd,MAAMA,EAAO;AAAA,QACb,SAASA,EAAO;AAAA,QAChB,QAAQ,EAAE,MAAM,cAAc;AAAA,MAAA,CACjC;AAAA,IACF,CACN;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWL,MAAa,WAAWQ,GAA0BC,GAAkC;AAClE,IAAAR,EAAA,aAAaO,GAAOC,CAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWxC,UAAyB;AAC5B,WAAOR,EAAc,cAAc;AAAA,EAAA;AAE3C;"}
+{"version":3,"file":"worker-generator.js","sources":["../../../src/lib/worker-generator.ts"],"sourcesContent":["import { PipelineConfig } from './config-types';\nimport { LadonOutput, MaskGenerator, WorkerGeneratedResult } from './generator-types';\nimport { workerManager } from './worker-manager';\n\nexport class WorkerGenerator implements MaskGenerator {\n    private pending: Map<number, (result: LadonOutput) => void> = new Map();\n    private lastResult: LadonOutput;\n    private newMask = true;\n\n    constructor(public readonly config: PipelineConfig) {\n        this.lastResult = {\n            data: new Float32Array(\n                this.config.mask.outputSize.height * this.config.mask.outputSize.width\n            ),\n            facesAndLandmarks: { data: new Float32Array(0), dims: [] },\n            gesture: new Float32Array(0),\n            motion: 0,\n            timestamp: 0,\n            warm: false,\n        };\n    }\n\n    public isLoaded(): boolean {\n        return !!workerManager.getWorker();\n    }\n\n    public async getLadonOutput(): Promise<LadonOutput> {\n        const toReturn = {\n            data: this.lastResult.data,\n            facesAndLandmarks: this.lastResult.facesAndLandmarks,\n            gesture: this.lastResult.gesture,\n            motion: this.lastResult.motion,\n            timestamp: this.lastResult.timestamp,\n            warm: this.newMask,\n        };\n\n        this.newMask = false;\n\n        return toReturn;\n    }\n\n    /**\n     * Returns a promise that resolves when the worker returns the mask for the given timestamp.\n     * This is useful for synchronizing the processing of frames with their corresponding masks.\n     * The promise will be resolved with the mask data when it is available.\n     *\n     * @param timestamp - The timestamp of the frame for which the mask is requested.\n     * @returns A promise that resolves with the mask data when available.\n     */\n    public async getLadonOutputForTimestamp(timestamp: number): Promise<LadonOutput> {\n        return new Promise<LadonOutput>((resolve) => {\n            this.pending.set(timestamp, resolve);\n        });\n    }\n\n    /**\n     * Loads the worker and initializes it with the provided configuration.\n     * The worker is responsible for processing the input image and generating the mask.\n     * Prepares the worker configuration by resolving the final URLs and paths to send to the worker.\n     *\n     * @returns A promise that resolves when the worker is loaded and initialized.\n     */\n    public async load(): Promise<void> {\n        const config = await workerManager.prepareWorkerConfig(this.config);\n\n        // Listen for worker events\n        return new Promise<void>((resolve) => {\n            workerManager.onInitialized(() => {\n                resolve();\n            });\n            workerManager.onGenerated((data: WorkerGeneratedResult) => {\n                const result: LadonOutput = {\n                    data: data.mask,\n                    facesAndLandmarks: data.facesAndLandmarks,\n                    gesture: data.gesture,\n                    motion: data.motion,\n                    timestamp: data.timestamp,\n                    warm: true,\n                };\n                this.lastResult = result;\n                this.newMask = true;\n\n                // Resolve the pending promise for this timestamp, if any\n                const cb = this.pending.get(data.timestamp);\n                if (cb) {\n                    cb(result);\n                    this.pending.delete(data.timestamp);\n                }\n            });\n            workerManager.onDropped((timestamp: number) => {\n                // Resolve the pending promise for this timestamp with the last result\n                const cb = this.pending.get(timestamp);\n                if (cb) {\n                    cb(this.lastResult);\n                    this.pending.delete(timestamp);\n                }\n            });\n            workerManager.onLog((message, severity) => {\n                const msg = `[ladon-ts] Message received from worker: ${message}`;\n                switch (severity) {\n                    case 'warn':\n                        console.warn(msg);\n                        break;\n                    case 'error':\n                        console.error(msg);\n                        break;\n                    default:\n                        console.log(msg);\n                }\n            });\n\n            // Initialize the worker if not already done, then always send init\n            (async () => {\n                await (workerManager.getWorker() ?? workerManager.initializeWorker(config));\n                workerManager.sendInit({\n                    baseUrl: config.baseUrl,\n                    input: config.input,\n                    mask: config.mask,\n                    wasmUri: config.wasmUri,\n                    render: { type: 'passthrough' },\n                });\n            })();\n        });\n    }\n\n    /**\n     * Sends the input image to the worker for processing.\n     * The input image is expected to be a Uint8ClampedArray.\n     * Claims ownership of the input tensor's buffer and forwards it to the worker.\n     *\n     * @param input - The input image as a Uint8ClampedArray.\n     * @param timestamp - The timestamp of the input image.\n     */\n    public async postRender(input: Uint8ClampedArray, timestamp: number): Promise<void> {\n        workerManager.sendGenerate(input, timestamp);\n    }\n\n    /**\n     * Destroys the worker and cleans up resources.\n     * First it sends a message to the worker to destroy itself,\n     * then it waits for the worker to respond with a 'destroyed' message.\n     * After that, it terminates the worker and revokes the URL if it exists.\n     *\n     * @returns A promise that resolves when the worker is destroyed.\n     */\n    public destroy(): Promise<void> {\n        return workerManager.destroyWorker();\n    }\n}\n"],"names":["WorkerGenerator","config","workerManager","toReturn","timestamp","resolve","data","result","cb","message","severity","msg","input"],"mappings":";AAIO,MAAMA,EAAyC;AAAA,EAKlD,YAA4BC,GAAwB;AAAxB,SAAA,SAAAA,GAJpB,KAAA,8BAA0D,IAAI,GAEtE,KAAQ,UAAU,IAGd,KAAK,aAAa;AAAA,MACd,MAAM,IAAI;AAAA,QACN,KAAK,OAAO,KAAK,WAAW,SAAS,KAAK,OAAO,KAAK,WAAW;AAAA,MACrE;AAAA,MACA,mBAAmB,EAAE,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,GAAG;AAAA,MACzD,SAAS,IAAI,aAAa,CAAC;AAAA,MAC3B,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,MAAM;AAAA,IACV;AAAA,EAAA;AAAA,EAGG,WAAoB;AAChB,WAAA,CAAC,CAACC,EAAc,UAAU;AAAA,EAAA;AAAA,EAGrC,MAAa,iBAAuC;AAChD,UAAMC,IAAW;AAAA,MACb,MAAM,KAAK,WAAW;AAAA,MACtB,mBAAmB,KAAK,WAAW;AAAA,MACnC,SAAS,KAAK,WAAW;AAAA,MACzB,QAAQ,KAAK,WAAW;AAAA,MACxB,WAAW,KAAK,WAAW;AAAA,MAC3B,MAAM,KAAK;AAAA,IACf;AAEA,gBAAK,UAAU,IAERA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWX,MAAa,2BAA2BC,GAAyC;AACtE,WAAA,IAAI,QAAqB,CAACC,MAAY;AACpC,WAAA,QAAQ,IAAID,GAAWC,CAAO;AAAA,IAAA,CACtC;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUL,MAAa,OAAsB;AAC/B,UAAMJ,IAAS,MAAMC,EAAc,oBAAoB,KAAK,MAAM;AAG3D,WAAA,IAAI,QAAc,CAACG,MAAY;AAClC,MAAAH,EAAc,cAAc,MAAM;AACtB,QAAAG,EAAA;AAAA,MAAA,CACX,GACaH,EAAA,YAAY,CAACI,MAAgC;AACvD,cAAMC,IAAsB;AAAA,UACxB,MAAMD,EAAK;AAAA,UACX,mBAAmBA,EAAK;AAAA,UACxB,SAASA,EAAK;AAAA,UACd,QAAQA,EAAK;AAAA,UACb,WAAWA,EAAK;AAAA,UAChB,MAAM;AAAA,QACV;AACA,aAAK,aAAaC,GAClB,KAAK,UAAU;AAGf,cAAMC,IAAK,KAAK,QAAQ,IAAIF,EAAK,SAAS;AAC1C,QAAIE,MACAA,EAAGD,CAAM,GACJ,KAAA,QAAQ,OAAOD,EAAK,SAAS;AAAA,MACtC,CACH,GACaJ,EAAA,UAAU,CAACE,MAAsB;AAE3C,cAAMI,IAAK,KAAK,QAAQ,IAAIJ,CAAS;AACrC,QAAII,MACAA,EAAG,KAAK,UAAU,GACb,KAAA,QAAQ,OAAOJ,CAAS;AAAA,MACjC,CACH,GACaF,EAAA,MAAM,CAACO,GAASC,MAAa;AACjC,cAAAC,IAAM,4CAA4CF,CAAO;AAC/D,gBAAQC,GAAU;AAAA,UACd,KAAK;AACD,oBAAQ,KAAKC,CAAG;AAChB;AAAA,UACJ,KAAK;AACD,oBAAQ,MAAMA,CAAG;AACjB;AAAA,UACJ;AACI,oBAAQ,IAAIA,CAAG;AAAA,QAAA;AAAA,MACvB,CACH,IAGA,aACG,OAAOT,EAAc,UAAe,KAAAA,EAAc,iBAAiBD,CAAM,IACzEC,EAAc,SAAS;AAAA,QACnB,SAASD,EAAO;AAAA,QAChB,OAAOA,EAAO;AAAA,QACd,MAAMA,EAAO;AAAA,QACb,SAASA,EAAO;AAAA,QAChB,QAAQ,EAAE,MAAM,cAAc;AAAA,MAAA,CACjC;AAAA,IACF,CACN;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWL,MAAa,WAAWW,GAA0BR,GAAkC;AAClE,IAAAF,EAAA,aAAaU,GAAOR,CAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWxC,UAAyB;AAC5B,WAAOF,EAAc,cAAc;AAAA,EAAA;AAE3C;"}

package/build/module/lib/worker-manager.d.ts

@@ -12,6 +12,7 @@ export declare class WorkerManager {
     private destroyPromise;
     private onInitializedCallbacks;
     private onGeneratedCallbacks;
+    private onDroppedCallbacks;
     private onLogCallbacks;
     /**
      * Get the current worker instance.
@@ -68,6 +69,14 @@ export declare class WorkerManager {
      * @param data.motion - The detected motion.
      */
     onGenerated(callback: (data: WorkerGeneratedResult) => void): void;
+    /**
+     * Register a callback for when the worker drops a frame.
+     * This is useful for handling cases where the worker cannot keep up with the input frame rate.
+     * The callback will be called with the timestamp of the dropped frame.
+     *
+     * @param callback - The callback function to be called with the timestamp of the dropped frame.
+     */
+    onDropped(callback: (timestamp: number) => void): void;
     /**
      * Register a callback for log messages from the worker.
      *

package/build/module/lib/worker-manager.js

@@ -1,7 +1,7 @@
 import { resourceLoader as a } from "./resource-loader.js";
-class l {
+class d {
   constructor() {
-    this.destroyPromise = null, this.onInitializedCallbacks = [], this.onGeneratedCallbacks = [], this.onLogCallbacks = [];
+    this.destroyPromise = null, this.onInitializedCallbacks = [], this.onGeneratedCallbacks = [], this.onDroppedCallbacks = [], this.onLogCallbacks = [];
   }
   /**
    * Get the current worker instance.
@@ -22,11 +22,11 @@ class l {
       throw new Error("[ladon-ts] web workers are not supported in this environment");
     this.destroyPromise && (await this.destroyPromise, this.destroyPromise = null), this.worker && (this.destroyPromise = this.destroyWorker(), await this.destroyPromise, this.destroyPromise = null);
     const r = await a.resolveUri(e, e.mask.workerUri), o = await (await fetch(r, { cache: "force-cache" })).text();
-    return this.workerUrl = URL.createObjectURL(new Blob([o], { type: "text/javascript" })), this.worker = new Worker(this.workerUrl, { type: "module" }), this.worker.addEventListener("message", (i) => {
-      if (i.data.type === "log") {
-        const { message: t, severity: n } = i.data;
-        this.onLogCallbacks.forEach((k) => k(t, n));
-      } else i.data.status === "generated" ? this.onGeneratedCallbacks.forEach((t) => t(i.data)) : i.data.status === "initialized" && this.onInitializedCallbacks.forEach((t) => t());
+    return this.workerUrl = URL.createObjectURL(new Blob([o], { type: "text/javascript" })), this.worker = new Worker(this.workerUrl, { type: "module" }), this.worker.addEventListener("message", (t) => {
+      if (t.data.type === "log") {
+        const { message: s, severity: n } = t.data;
+        this.onLogCallbacks.forEach((l) => l(s, n));
+      } else t.data.status === "generated" ? this.onGeneratedCallbacks.forEach((s) => s(t.data)) : t.data.status === "dropped" ? this.onDroppedCallbacks.forEach((s) => s(t.data.timestamp)) : t.data.status === "initialized" && this.onInitializedCallbacks.forEach((s) => s());
     }), this.worker;
   }
   /**
@@ -37,9 +37,9 @@ class l {
    */
   destroyWorker() {
     return new Promise((e) => {
-      const r = (s) => {
+      const r = (i) => {
         var o;
-        s.data.status === "destroyed" && ((o = this.worker) == null || o.removeEventListener("message", r), this.terminateWorker(), e());
+        i.data.status === "destroyed" && ((o = this.worker) == null || o.removeEventListener("message", r), this.terminateWorker(), e());
       };
       if (!this.worker) {
         console.warn("[ladon-ts] No worker to destroy"), e();
@@ -73,12 +73,12 @@ class l {
    * @returns The worker configuration.
    */
   async prepareWorkerConfig(e) {
-    const r = await a.resolveUri(e, e.mask.modelUri), s = await a.resolveUri(e, e.mask.workerUri), o = await a.resolveUri(e, e.wasmUri), t = { ...{
+    const r = await a.resolveUri(e, e.mask.modelUri), i = await a.resolveUri(e, e.mask.workerUri), o = await a.resolveUri(e, e.wasmUri), s = { ...{
       ...e,
-      mask: { ...e.mask, modelUri: r, workerUri: s },
+      mask: { ...e.mask, modelUri: r, workerUri: i },
       wasmUri: o
     }, assetUrlResolver: void 0 };
-    return JSON.parse(JSON.stringify(t));
+    return JSON.parse(JSON.stringify(s));
   }
   /**
    * Register a callback for when the worker is initialized.
@@ -99,6 +99,16 @@ class l {
   onGenerated(e) {
     this.onGeneratedCallbacks.push(e);
   }
+  /**
+   * Register a callback for when the worker drops a frame.
+   * This is useful for handling cases where the worker cannot keep up with the input frame rate.
+   * The callback will be called with the timestamp of the dropped frame.
+   *
+   * @param callback - The callback function to be called with the timestamp of the dropped frame.
+   */
+  onDropped(e) {
+    this.onDroppedCallbacks.push(e);
+  }
   /**
    * Register a callback for log messages from the worker.
    *
@@ -120,8 +130,9 @@ class l {
    * @param config.assetUrlResolver - The function to resolve asset URLs.
    */
   sendInit(e) {
-    var r;
-    (r = this.worker) == null || r.postMessage({ type: "init", config: e });
+    if (!this.worker)
+      throw new Error("[ladon-ts] Worker is not initialized, cannot send init");
+    this.worker.postMessage({ type: "init", config: e });
   }
   /**
    * Send a generate message to the worker.
@@ -130,8 +141,9 @@ class l {
    * @param timestamp - The timestamp of the input image.
    */
   sendGenerate(e, r) {
-    var s;
-    (s = this.worker) == null || s.postMessage(
+    if (!this.worker)
+      throw new Error("[ladon-ts] Worker is not initialized, cannot send generate");
+    this.worker.postMessage(
       {
         type: "generate",
         image: e,
@@ -141,9 +153,9 @@ class l {
     );
   }
 }
-const w = new l();
+const w = new d();
 export {
-  l as WorkerManager,
+  d as WorkerManager,
   w as workerManager
 };
 //# sourceMappingURL=worker-manager.js.map