@webex/ladon-ts 5.9.0 → 5.9.1

package/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### 5.9.1 (2025-07-02)
+
+
+### Bug Fixes
+
+* **pipeline:** resolve mask lag regression and clarify frame data paths ([#77](https://github.com/beyonddunbar/ladon-ts/issues/77)) ([5e7a3a8](https://github.com/beyonddunbar/ladon-ts/commit/5e7a3a840f8d478ee7d62bb19275653ffe152766))
+
 ## 5.9.0 (2025-07-01)
 
 

package/build/module/lib/pipeline.d.ts

@@ -24,6 +24,8 @@ export declare class Pipeline implements IPipeline {
     private isProcessingFrame;
     /** Promise for the currently pending frame processing operation. */
     private pendingFramePromise;
+    /** Buffer for frame data copies to ensure they are not modified or garbage collected. */
+    private frameDataBuffer?;
     /**
      * Create a new pipeline instance.
      *
@@ -63,6 +65,10 @@ export declare class Pipeline implements IPipeline {
      * @returns The frame data as a Uint8ClampedArray.
      */
     private getInputFrameData;
+    /**
+     * Create a copy of frame data to prevent modifications during async operations.
+     */
+    private createFrameDataCopy;
     /**
      * Resize the image data if the model expects a different input size.
      * Handles scaling between input dimensions and model requirements.

package/build/module/lib/pipeline.js

@@ -1,9 +1,9 @@
-import { WorkerGenerator as g } from "./worker-generator.js";
-import { BlurRenderer as f } from "./renderer/blur-renderer.js";
-import { PassthroughRenderer as w } from "./renderer/passthrough-renderer.js";
+import { WorkerGenerator as f } from "./worker-generator.js";
+import { BlurRenderer as w } from "./renderer/blur-renderer.js";
+import { PassthroughRenderer as F } from "./renderer/passthrough-renderer.js";
 import { ReplacementRenderer as k } from "./renderer/replacement-renderer.js";
-import { resizeImage as F } from "./util.js";
-class M {
+import { resizeImage as y } from "./util.js";
+class A {
   /**
    * Create a new pipeline instance.
    *
@@ -13,15 +13,15 @@ class M {
    * @param inputVideoElement - Video element serving as the input source.
    * @throws Error if the input canvas 2D context cannot be obtained.
    */
-  constructor(t, e, s, n) {
-    this.inputCanvas = t, this.outputCanvas = e, this.config = s, this.inputVideoElement = n, this.loaded = !1, this.isProcessingFrame = !1, this.pendingFramePromise = null;
-    const r = this.inputCanvas.getContext("2d", {
+  constructor(e, t, r, n) {
+    this.inputCanvas = e, this.outputCanvas = t, this.config = r, this.inputVideoElement = n, this.loaded = !1, this.isProcessingFrame = !1, this.pendingFramePromise = null;
+    const s = this.inputCanvas.getContext("2d", {
       alpha: !1,
       willReadFrequently: !0
     });
-    if (!r)
+    if (!s)
       throw new Error("[ladon-ts] Error getting input context 2d");
-    this.inputCanvasContext = r;
+    this.inputCanvasContext = s;
   }
   /**
    * Process a single frame and return the inference result.
@@ -34,13 +34,13 @@ class M {
    * @returns Promise resolving to the frame processing result with timing statistics.
    * @throws Error if the pipeline (generator or renderer) is not loaded.
    */
-  async onFrame(t, e = !0, s) {
+  async onFrame(e, t = !0, r) {
     this.isProcessingFrame && this.pendingFramePromise && await this.pendingFramePromise, this.isProcessingFrame = !0;
     try {
       return this.pendingFramePromise = this.processFrameInternal(
-        t,
         e,
-        s
+        t,
+        r
       ), await this.pendingFramePromise;
     } finally {
       this.isProcessingFrame = !1, this.pendingFramePromise = null;
@@ -56,29 +56,30 @@ class M {
    * @returns Promise resolving to the frame processing result with timing statistics.
    * @throws Error if the pipeline (generator or renderer) is not loaded.
    */
-  async processFrameInternal(t, e = !0, s) {
+  async processFrameInternal(e, t = !0, r) {
     const n = performance.now();
     if (!this.generator || !this.renderer)
       throw new Error("[ladon-ts] Pipeline is not loaded.");
-    let r = 0, a = 0;
-    if (e || !this.lastMaskResult) {
-      const c = performance.now();
-      s ?? (s = this.getInputFrameData()), r = performance.now() - c;
-      const m = performance.now(), u = s, p = this.getMaskImageData(u);
-      await this.generator.process(p, t), this.lastMaskResult = await this.generator.getLadonOutputForTimestamp(t), a = performance.now() - m;
-    }
-    const h = performance.now();
-    await this.renderer.render(this.inputVideoElement, this.lastMaskResult);
-    const d = performance.now() - h, i = performance.now(), l = i - n, o = this.createInferenceResult(t, e);
-    return o.stats = {
-      totalFrameTime: l,
-      modelInferenceTime: a,
-      preprocessingTime: r,
-      postprocessingTime: d,
+    let s = 0, i = 0, a;
+    if (t || !this.lastMaskResult) {
+      const m = performance.now();
+      r ?? (r = this.getInputFrameData()), a = this.createFrameDataCopy(r), s = performance.now() - m;
+      const c = performance.now(), p = r, g = this.getMaskImageData(p);
+      await this.generator.process(g, e), this.lastMaskResult = await this.generator.getLadonOutputForTimestamp(e), i = performance.now() - c;
+    } else
+      a = this.inputVideoElement;
+    const d = performance.now();
+    await this.renderer.render(a, this.lastMaskResult);
+    const l = performance.now() - d, o = performance.now(), u = o - n, h = this.createInferenceResult(e, t);
+    return h.stats = {
+      totalFrameTime: u,
+      modelInferenceTime: i,
+      preprocessingTime: s,
+      postprocessingTime: l,
       frameStartTime: n,
-      frameEndTime: i,
-      inferenceSkipped: !e
-    }, o;
+      frameEndTime: o,
+      inferenceSkipped: !t
+    }, h;
   }
   /**
    * Get the raw image data from the input canvas context.
@@ -93,6 +94,13 @@ class M {
       this.config.input.height
     ).data;
   }
+  /**
+   * Create a copy of frame data to prevent modifications during async operations.
+   */
+  createFrameDataCopy(e) {
+    var t;
+    return e instanceof Uint8ClampedArray ? (((t = this.frameDataBuffer) == null ? void 0 : t.length) !== e.length && (this.frameDataBuffer = new Uint8ClampedArray(e.length)), this.frameDataBuffer.set(e), this.frameDataBuffer) : e;
+  }
   /**
    * Resize the image data if the model expects a different input size.
    * Handles scaling between input dimensions and model requirements.
@@ -100,14 +108,14 @@ class M {
    * @param rawImageData - The raw image data to potentially resize.
    * @returns The processed mask image data, resized if necessary.
    */
-  getMaskImageData(t) {
-    const e = this.config.mask.inputSize, s = this.config.input;
-    return e.width === s.width && e.height === s.height ? t : F(
-      t,
-      s.width,
-      s.height,
-      e.width,
-      e.height
+  getMaskImageData(e) {
+    const t = this.config.mask.inputSize, r = this.config.input;
+    return t.width === r.width && t.height === r.height ? e : y(
+      e,
+      r.width,
+      r.height,
+      t.width,
+      t.height
     );
   }
   /**
@@ -118,20 +126,20 @@ class M {
    * @param performedInference - Whether inference was actually performed for this frame.
    * @returns The inference result object containing mask data and metadata.
    */
-  createInferenceResult(t, e) {
+  createInferenceResult(e, t) {
     return this.lastMaskResult ? {
       facesAndLandmarks: this.lastMaskResult.facesAndLandmarks,
       gesture: this.lastMaskResult.gesture,
       motion: this.lastMaskResult.motion,
-      skipped: !e,
-      timestamp: t
+      skipped: !t,
+      timestamp: e
       // Stats will be added by the caller if needed
     } : {
       facesAndLandmarks: { data: new Float32Array(0), dims: [] },
       gesture: new Float32Array(0),
       motion: 0,
       skipped: !0,
-      timestamp: t
+      timestamp: e
       // Stats will be added by the caller if needed
     };
   }
@@ -153,10 +161,10 @@ class M {
   async load() {
     switch (this.config.render.type) {
       case "blur":
-        this.renderer = new f(this.outputCanvas, this.config);
+        this.renderer = new w(this.outputCanvas, this.config);
         break;
       case "passthrough":
-        this.renderer = new w(this.outputCanvas, this.config);
+        this.renderer = new F(this.outputCanvas, this.config);
         break;
       case "replacement":
         this.renderer = new k(this.outputCanvas, this.config);
@@ -164,7 +172,7 @@ class M {
       default:
         throw new Error("[ladon-ts] Renderer not implemented.");
     }
-    await this.renderer.load(), this.generator = new g(this.config), await this.generator.load(), this.loaded = !0;
+    await this.renderer.load(), this.generator = new f(this.config), await this.generator.load(), this.loaded = !0;
   }
   /**
    * Reset the pipeline state by clearing cached mask results.
@@ -180,11 +188,11 @@ class M {
    * @returns Promise that resolves when cleanup is complete.
    */
   async destroy() {
-    var t, e;
-    this.reset(), (t = this.renderer) == null || t.destroy(), await ((e = this.generator) == null ? void 0 : e.destroy()), this.loaded = !1;
+    var e, t;
+    this.reset(), (e = this.renderer) == null || e.destroy(), await ((t = this.generator) == null ? void 0 : t.destroy()), this.loaded = !1;
   }
 }
 export {
-  M as Pipeline
+  A as Pipeline
 };
 //# sourceMappingURL=pipeline.js.map

package/build/module/lib/pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.js","sources":["../../../src/lib/pipeline.ts"],"sourcesContent":["import { PipelineConfig } from './config-types';\nimport { FrameData, LadonOutput, MaskGenerator, WorkerGenerator } from './generator';\nimport { IFrameInferenceResult, IPipeline } from './pipeline-types';\nimport { BlurRenderer, LadonRenderer, PassthroughRenderer, ReplacementRenderer } from './renderer';\nimport { resizeImage } from './util';\n\n/**\n * Core pipeline implementation for processing video frames with ML inference and rendering.\n * Handles frame processing, mask generation, and output rendering with proper timing controls.\n */\nexport class Pipeline implements IPipeline {\n    /** The mask generator responsible for ML inference. */\n    private generator?: MaskGenerator;\n\n    /** The renderer responsible for compositing the final output. */\n    private renderer?: LadonRenderer;\n\n    /** Flag indicating whether the pipeline has been fully loaded and initialized. */\n    private loaded = false;\n\n    /** 2D rendering context for the input canvas used for frame processing. */\n    private inputCanvasContext: CanvasRenderingContext2D;\n\n    /** The most recent mask result from ML inference, used when skipping inference. */\n    private lastMaskResult?: LadonOutput;\n\n    /** Flag indicating whether a frame is currently being processed. */\n    private isProcessingFrame = false;\n\n    /** Promise for the currently pending frame processing operation. */\n    private pendingFramePromise: Promise<IFrameInferenceResult> | null = null;\n\n    /**\n     * Create a new pipeline instance.\n     *\n     * @param inputCanvas - Canvas element for input frame processing.\n     * @param outputCanvas - Canvas element where the final rendered output will be displayed.\n     * @param config - Pipeline configuration containing processing and rendering settings.\n     * @param inputVideoElement - Video element serving as the input source.\n     * @throws Error if the input canvas 2D context cannot be obtained.\n     */\n    constructor(\n        private inputCanvas: HTMLCanvasElement,\n        private outputCanvas: HTMLCanvasElement,\n        public readonly config: PipelineConfig,\n        private readonly inputVideoElement: HTMLVideoElement\n    ) {\n        const inputCanvasContext = this.inputCanvas.getContext('2d', {\n            alpha: false,\n            willReadFrequently: true,\n        });\n\n        if (!inputCanvasContext) {\n            throw new Error('[ladon-ts] Error getting input context 2d');\n        }\n        this.inputCanvasContext = inputCanvasContext;\n    }\n\n    /**\n     * Process a single frame and return the inference result.\n     * This method handles frame data, performs inference if needed, and renders the output.\n     * Includes frame processing serialization to prevent overlapping operations.\n     *\n     * @param timestamp - The timestamp of the frame in milliseconds.\n     * @param performInference - Whether to perform mask inference or reuse the last result.\n     * @param frameData - Optional frame data to process instead of capturing from input.\n     * @returns Promise resolving to the frame processing result with timing statistics.\n     * @throws Error if the pipeline (generator or renderer) is not loaded.\n     */\n    public async onFrame(\n        timestamp: number,\n        performInference = true,\n        frameData?: FrameData\n    ): Promise<IFrameInferenceResult> {\n        // If already processing, wait for the current frame to finish\n        if (this.isProcessingFrame && this.pendingFramePromise) {\n            await this.pendingFramePromise;\n        }\n\n        // Set processing flag\n        this.isProcessingFrame = true;\n\n        try {\n            this.pendingFramePromise = this.processFrameInternal(\n                timestamp,\n                performInference,\n                frameData\n            );\n            return await this.pendingFramePromise;\n        } finally {\n            this.isProcessingFrame = false;\n            this.pendingFramePromise = null;\n        }\n    }\n\n    /**\n     * Internal frame processing implementation.\n     * Handles the actual frame processing logic including timing measurements.\n     *\n     * @param timestamp - The timestamp of the frame in milliseconds.\n     * @param performInference - Whether to perform mask inference or reuse the last result.\n     * @param frameData - Optional frame data to process instead of capturing from input.\n     * @returns Promise resolving to the frame processing result with timing statistics.\n     * @throws Error if the pipeline (generator or renderer) is not loaded.\n     */\n    private async processFrameInternal(\n        timestamp: number,\n        performInference = true,\n        frameData?: FrameData\n    ): Promise<IFrameInferenceResult> {\n        const frameStartTime = performance.now();\n\n        if (!this.generator || !this.renderer) {\n            throw new Error('[ladon-ts] Pipeline is not loaded.');\n        }\n\n        let preprocessingTime = 0;\n        let modelInferenceTime = 0;\n\n        if (performInference || !this.lastMaskResult) {\n            // Pre-processing timing\n            const preprocessStart = performance.now();\n\n            // Get the current frame (raw image data or video element). Use the video element if skipping\n            // inference to avoid unnecessary processing when calling canvas getImageData.\n            frameData ??= this.getInputFrameData();\n\n            preprocessingTime = performance.now() - preprocessStart;\n\n            // Model inference timing\n            const inferenceStart = performance.now();\n\n            // Send frame for inference and wait for the mask for this timestamp\n            const rawImageData = frameData as Uint8ClampedArray;\n            const maskImageData = this.getMaskImageData(rawImageData);\n            await this.generator.process(maskImageData, timestamp);\n            this.lastMaskResult = await this.generator.getLadonOutputForTimestamp(timestamp);\n\n            modelInferenceTime = performance.now() - inferenceStart;\n        }\n\n        // Post-processing timing (rendering)\n        const postprocessStart = performance.now();\n\n        // Render the frame and mask together (mask may be stale if skipping inference)\n        await this.renderer.render(this.inputVideoElement, this.lastMaskResult);\n\n        const postprocessingTime = performance.now() - postprocessStart;\n\n        const frameEndTime = performance.now();\n        const totalFrameTime = frameEndTime - frameStartTime;\n\n        const result = this.createInferenceResult(timestamp, performInference);\n\n        // Add timing stats to the result\n        result.stats = {\n            totalFrameTime,\n            modelInferenceTime,\n            preprocessingTime,\n            postprocessingTime,\n            frameStartTime,\n            frameEndTime,\n            inferenceSkipped: !performInference,\n        };\n\n        return result;\n    }\n\n    /**\n     * Get the raw image data from the input canvas context.\n     *\n     * @returns The frame data as a Uint8ClampedArray.\n     */\n    private getInputFrameData(): FrameData {\n        return this.inputCanvasContext.getImageData(\n            0,\n            0,\n            this.config.input.width,\n            this.config.input.height\n        ).data;\n    }\n\n    /**\n     * Resize the image data if the model expects a different input size.\n     * Handles scaling between input dimensions and model requirements.\n     *\n     * @param rawImageData - The raw image data to potentially resize.\n     * @returns The processed mask image data, resized if necessary.\n     */\n    private getMaskImageData(rawImageData: Uint8ClampedArray): Uint8ClampedArray {\n        const modelInput = this.config.mask.inputSize;\n        const inputImage = this.config.input;\n\n        // Check if resizing is needed based on configuration.\n        if (modelInput.width === inputImage.width && modelInput.height === inputImage.height) {\n            return rawImageData;\n        }\n\n        // Resize the image data.\n        return resizeImage(\n            rawImageData,\n            inputImage.width,\n            inputImage.height,\n            modelInput.width,\n            modelInput.height\n        );\n    }\n\n    /**\n     * Create the inference result object with appropriate data.\n     * Handles both successful inference results and fallback cases.\n     *\n     * @param timestamp - The timestamp of the frame in milliseconds.\n     * @param performedInference - Whether inference was actually performed for this frame.\n     * @returns The inference result object containing mask data and metadata.\n     */\n    private createInferenceResult(\n        timestamp: number,\n        performedInference: boolean\n    ): IFrameInferenceResult {\n        if (!this.lastMaskResult) {\n            return {\n                facesAndLandmarks: { data: new Float32Array(0), dims: [] },\n                gesture: new Float32Array(0),\n                motion: 0,\n                skipped: true,\n                timestamp,\n                // Stats will be added by the caller if needed\n            };\n        }\n\n        return {\n            facesAndLandmarks: this.lastMaskResult.facesAndLandmarks,\n            gesture: this.lastMaskResult.gesture,\n            motion: this.lastMaskResult.motion,\n            skipped: !performedInference,\n            timestamp,\n            // Stats will be added by the caller if needed\n        };\n    }\n\n    /**\n     * Check if the pipeline is fully loaded and ready for processing.\n     *\n     * @returns True if the pipeline is loaded and ready, false otherwise.\n     */\n    public isLoaded(): boolean {\n        return this.loaded;\n    }\n\n    /**\n     * Load and initialize the pipeline components.\n     * Creates the appropriate renderer and generator based on configuration.\n     *\n     * @returns Promise that resolves when all components are loaded and ready.\n     * @throws Error if an unsupported renderer type is specified in the configuration.\n     */\n    public async load(): Promise<void> {\n        switch (this.config.render.type) {\n            case 'blur':\n                this.renderer = new BlurRenderer(this.outputCanvas, this.config);\n                break;\n            case 'passthrough':\n                this.renderer = new PassthroughRenderer(this.outputCanvas, this.config);\n                break;\n            case 'replacement':\n                this.renderer = new ReplacementRenderer(this.outputCanvas, this.config);\n                break;\n            default:\n                throw new Error('[ladon-ts] Renderer not implemented.');\n        }\n\n        await this.renderer.load();\n\n        this.generator = new WorkerGenerator(this.config);\n        await this.generator.load();\n\n        this.loaded = true;\n    }\n\n    /**\n     * Reset the pipeline state by clearing cached mask results.\n     * Useful when switching between different processing modes or sources.\n     */\n    public reset(): void {\n        this.lastMaskResult = undefined;\n    }\n\n    /**\n     * Destroy the pipeline and clean up all resources.\n     * Properly disposes of the renderer and generator to prevent memory leaks.\n     *\n     * @returns Promise that resolves when cleanup is complete.\n     */\n    public async destroy(): Promise<void> {\n        this.reset();\n        this.renderer?.destroy();\n        await this.generator?.destroy();\n        this.loaded = false;\n    }\n}\n"],"names":["Pipeline","inputCanvas","outputCanvas","config","inputVideoElement","inputCanvasContext","timestamp","performInference","frameData","frameStartTime","preprocessingTime","modelInferenceTime","preprocessStart","inferenceStart","rawImageData","maskImageData","postprocessStart","postprocessingTime","frameEndTime","totalFrameTime","result","modelInput","inputImage","resizeImage","performedInference","BlurRenderer","PassthroughRenderer","ReplacementRenderer","WorkerGenerator","_a","_b"],"mappings":";;;;;AAUO,MAAMA,EAA8B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA+BvC,YACYC,GACAC,GACQC,GACCC,GACnB;AAJU,SAAA,cAAAH,GACA,KAAA,eAAAC,GACQ,KAAA,SAAAC,GACC,KAAA,oBAAAC,GA3BrB,KAAQ,SAAS,IASjB,KAAQ,oBAAoB,IAG5B,KAAQ,sBAA6D;AAiBjE,UAAMC,IAAqB,KAAK,YAAY,WAAW,MAAM;AAAA,MACzD,OAAO;AAAA,MACP,oBAAoB;AAAA,IAAA,CACvB;AAED,QAAI,CAACA;AACK,YAAA,IAAI,MAAM,2CAA2C;AAE/D,SAAK,qBAAqBA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAc9B,MAAa,QACTC,GACAC,IAAmB,IACnBC,GAC8B;AAE1B,IAAA,KAAK,qBAAqB,KAAK,uBAC/B,MAAM,KAAK,qBAIf,KAAK,oBAAoB;AAErB,QAAA;AACA,kBAAK,sBAAsB,KAAK;AAAA,QAC5BF;AAAA,QACAC;AAAA,QACAC;AAAA,MACJ,GACO,MAAM,KAAK;AAAA,IAAA,UACpB;AACE,WAAK,oBAAoB,IACzB,KAAK,sBAAsB;AAAA,IAAA;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaJ,MAAc,qBACVF,GACAC,IAAmB,IACnBC,GAC8B;AACxB,UAAAC,IAAiB,YAAY,IAAI;AAEvC,QAAI,CAAC,KAAK,aAAa,CAAC,KAAK;AACnB,YAAA,IAAI,MAAM,oCAAoC;AAGxD,QAAIC,IAAoB,GACpBC,IAAqB;AAErB,QAAAJ,KAAoB,CAAC,KAAK,gBAAgB;AAEpC,YAAAK,IAAkB,YAAY,IAAI;AAIxC,MAAAJ,UAAc,KAAK,kBAAkB,IAEjBE,IAAA,YAAY,QAAQE;AAGlC,YAAAC,IAAiB,YAAY,IAAI,GAGjCC,IAAeN,GACfO,IAAgB,KAAK,iBAAiBD,CAAY;AACxD,YAAM,KAAK,UAAU,QAAQC,GAAeT,CAAS,GACrD,KAAK,iBAAiB,MAAM,KAAK,UAAU,2BAA2BA,CAAS,GAE1DK,IAAA,YAAY,QAAQE;AAAA,IAAA;AAIvC,UAAAG,IAAmB,YAAY,IAAI;AAGzC,UAAM,KAAK,SAAS,OAAO,KAAK,mBAAmB,KAAK,cAAc;AAEhE,UAAAC,IAAqB,YAAY,IAAA,IAAQD,GAEzCE,IAAe,YAAY,IAAI,GAC/BC,IAAiBD,IAAeT,GAEhCW,IAAS,KAAK,sBAAsBd,GAAWC,CAAgB;AAGrE,WAAAa,EAAO,QAAQ;AAAA,MACX,gBAAAD;AAAA,MACA,oBAAAR;AAAA,MACA,mBAAAD;AAAA,MACA,oBAAAO;AAAA,MACA,gBAAAR;AAAA,MACA,cAAAS;AAAA,MACA,kBAAkB,CAACX;AAAA,IACvB,GAEOa;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQH,oBAA+B;AACnC,WAAO,KAAK,mBAAmB;AAAA,MAC3B;AAAA,MACA;AAAA,MACA,KAAK,OAAO,MAAM;AAAA,MAClB,KAAK,OAAO,MAAM;AAAA,IAAA,EACpB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUE,iBAAiBN,GAAoD;AACnE,UAAAO,IAAa,KAAK,OAAO,KAAK,WAC9BC,IAAa,KAAK,OAAO;AAG/B,WAAID,EAAW,UAAUC,EAAW,SAASD,EAAW,WAAWC,EAAW,SACnER,IAIJS;AAAA,MACHT;AAAA,MACAQ,EAAW;AAAA,MACXA,EAAW;AAAA,MACXD,EAAW;AAAA,MACXA,EAAW;AAAA,IACf;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWI,sBACJf,GACAkB,GACqB;AACjB,WAAC,KAAK,iBAWH;AAAA,MACH,mBAAmB,KAAK,eAAe;AAAA,MACvC,SAAS,KAAK,eAAe;AAAA,MAC7B,QAAQ,KAAK,eAAe;AAAA,MAC5B,SAAS,CAACA;AAAA,MACV,WAAAlB;AAAA;AAAA,IAEJ,IAjBW;AAAA,MACH,mBAAmB,EAAE,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,GAAG;AAAA,MACzD,SAAS,IAAI,aAAa,CAAC;AAAA,MAC3B,QAAQ;AAAA,MACR,SAAS;AAAA,MACT,WAAAA;AAAA;AAAA,IAEJ;AAAA,EAUJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQG,WAAoB;AACvB,WAAO,KAAK;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUhB,MAAa,OAAsB;AACvB,YAAA,KAAK,OAAO,OAAO,MAAM;AAAA,MAC7B,KAAK;AACD,aAAK,WAAW,IAAImB,EAAa,KAAK,cAAc,KAAK,MAAM;AAC/D;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ;AACU,cAAA,IAAI,MAAM,sCAAsC;AAAA,IAAA;AAGxD,UAAA,KAAK,SAAS,KAAK,GAEzB,KAAK,YAAY,IAAIC,EAAgB,KAAK,MAAM,GAC1C,MAAA,KAAK,UAAU,KAAK,GAE1B,KAAK,SAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOX,QAAc;AACjB,SAAK,iBAAiB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAS1B,MAAa,UAAyB;;AAClC,SAAK,MAAM,IACXC,IAAA,KAAK,aAAL,QAAAA,EAAe,WACT,QAAAC,IAAA,KAAK,cAAL,gBAAAA,EAAgB,YACtB,KAAK,SAAS;AAAA,EAAA;AAEtB;"}
+{"version":3,"file":"pipeline.js","sources":["../../../src/lib/pipeline.ts"],"sourcesContent":["import { PipelineConfig } from './config-types';\nimport { FrameData, LadonOutput, MaskGenerator, WorkerGenerator } from './generator';\nimport { IFrameInferenceResult, IPipeline } from './pipeline-types';\nimport { BlurRenderer, LadonRenderer, PassthroughRenderer, ReplacementRenderer } from './renderer';\nimport { resizeImage } from './util';\n\n/**\n * Core pipeline implementation for processing video frames with ML inference and rendering.\n * Handles frame processing, mask generation, and output rendering with proper timing controls.\n */\nexport class Pipeline implements IPipeline {\n    /** The mask generator responsible for ML inference. */\n    private generator?: MaskGenerator;\n\n    /** The renderer responsible for compositing the final output. */\n    private renderer?: LadonRenderer;\n\n    /** Flag indicating whether the pipeline has been fully loaded and initialized. */\n    private loaded = false;\n\n    /** 2D rendering context for the input canvas used for frame processing. */\n    private inputCanvasContext: CanvasRenderingContext2D;\n\n    /** The most recent mask result from ML inference, used when skipping inference. */\n    private lastMaskResult?: LadonOutput;\n\n    /** Flag indicating whether a frame is currently being processed. */\n    private isProcessingFrame = false;\n\n    /** Promise for the currently pending frame processing operation. */\n    private pendingFramePromise: Promise<IFrameInferenceResult> | null = null;\n\n    /** Buffer for frame data copies to ensure they are not modified or garbage collected. */\n    private frameDataBuffer?: Uint8ClampedArray;\n\n    /**\n     * Create a new pipeline instance.\n     *\n     * @param inputCanvas - Canvas element for input frame processing.\n     * @param outputCanvas - Canvas element where the final rendered output will be displayed.\n     * @param config - Pipeline configuration containing processing and rendering settings.\n     * @param inputVideoElement - Video element serving as the input source.\n     * @throws Error if the input canvas 2D context cannot be obtained.\n     */\n    constructor(\n        private inputCanvas: HTMLCanvasElement,\n        private outputCanvas: HTMLCanvasElement,\n        public readonly config: PipelineConfig,\n        private readonly inputVideoElement: HTMLVideoElement\n    ) {\n        const inputCanvasContext = this.inputCanvas.getContext('2d', {\n            alpha: false,\n            willReadFrequently: true,\n        });\n\n        if (!inputCanvasContext) {\n            throw new Error('[ladon-ts] Error getting input context 2d');\n        }\n        this.inputCanvasContext = inputCanvasContext;\n    }\n\n    /**\n     * Process a single frame and return the inference result.\n     * This method handles frame data, performs inference if needed, and renders the output.\n     * Includes frame processing serialization to prevent overlapping operations.\n     *\n     * @param timestamp - The timestamp of the frame in milliseconds.\n     * @param performInference - Whether to perform mask inference or reuse the last result.\n     * @param frameData - Optional frame data to process instead of capturing from input.\n     * @returns Promise resolving to the frame processing result with timing statistics.\n     * @throws Error if the pipeline (generator or renderer) is not loaded.\n     */\n    public async onFrame(\n        timestamp: number,\n        performInference = true,\n        frameData?: FrameData\n    ): Promise<IFrameInferenceResult> {\n        // If already processing, wait for the current frame to finish\n        if (this.isProcessingFrame && this.pendingFramePromise) {\n            await this.pendingFramePromise;\n        }\n\n        // Set processing flag\n        this.isProcessingFrame = true;\n\n        try {\n            this.pendingFramePromise = this.processFrameInternal(\n                timestamp,\n                performInference,\n                frameData\n            );\n            return await this.pendingFramePromise;\n        } finally {\n            this.isProcessingFrame = false;\n            this.pendingFramePromise = null;\n        }\n    }\n\n    /**\n     * Internal frame processing implementation.\n     * Handles the actual frame processing logic including timing measurements.\n     *\n     * @param timestamp - The timestamp of the frame in milliseconds.\n     * @param performInference - Whether to perform mask inference or reuse the last result.\n     * @param frameData - Optional frame data to process instead of capturing from input.\n     * @returns Promise resolving to the frame processing result with timing statistics.\n     * @throws Error if the pipeline (generator or renderer) is not loaded.\n     */\n    private async processFrameInternal(\n        timestamp: number,\n        performInference = true,\n        frameData?: FrameData\n    ): Promise<IFrameInferenceResult> {\n        const frameStartTime = performance.now();\n\n        if (!this.generator || !this.renderer) {\n            throw new Error('[ladon-ts] Pipeline is not loaded.');\n        }\n\n        let preprocessingTime = 0;\n        let modelInferenceTime = 0;\n        let renderFrameData: FrameData;\n\n        if (performInference || !this.lastMaskResult) {\n            // INFERENCE PATH: Capture canvas data, generate mask, render with same data\n            const preprocessStart = performance.now();\n\n            // Get canvas data for inference\n            frameData ??= this.getInputFrameData();\n\n            // Create stable copy for rendering synchronization\n            renderFrameData = this.createFrameDataCopy(frameData);\n\n            preprocessingTime = performance.now() - preprocessStart;\n\n            // Generate mask\n            const inferenceStart = performance.now();\n            const rawImageData = frameData as Uint8ClampedArray;\n            const maskImageData = this.getMaskImageData(rawImageData);\n            await this.generator.process(maskImageData, timestamp);\n            this.lastMaskResult = await this.generator.getLadonOutputForTimestamp(timestamp);\n            modelInferenceTime = performance.now() - inferenceStart;\n        } else {\n            // SKIP INFERENCE PATH: Use video element directly for GPU optimization\n            renderFrameData = this.inputVideoElement;\n        }\n\n        // Render with appropriate frame data\n        const postprocessStart = performance.now();\n        await this.renderer.render(renderFrameData, this.lastMaskResult);\n        const postprocessingTime = performance.now() - postprocessStart;\n\n        const frameEndTime = performance.now();\n        const totalFrameTime = frameEndTime - frameStartTime;\n\n        const result = this.createInferenceResult(timestamp, performInference);\n        result.stats = {\n            totalFrameTime,\n            modelInferenceTime,\n            preprocessingTime,\n            postprocessingTime,\n            frameStartTime,\n            frameEndTime,\n            inferenceSkipped: !performInference,\n        };\n\n        return result;\n    }\n\n    /**\n     * Get the raw image data from the input canvas context.\n     *\n     * @returns The frame data as a Uint8ClampedArray.\n     */\n    private getInputFrameData(): FrameData {\n        return this.inputCanvasContext.getImageData(\n            0,\n            0,\n            this.config.input.width,\n            this.config.input.height\n        ).data;\n    }\n\n    /**\n     * Create a copy of frame data to prevent modifications during async operations.\n     */\n    private createFrameDataCopy(frameData: FrameData): FrameData {\n        if (frameData instanceof Uint8ClampedArray) {\n            if (this.frameDataBuffer?.length !== frameData.length) {\n                this.frameDataBuffer = new Uint8ClampedArray(frameData.length);\n            }\n            this.frameDataBuffer.set(frameData);\n            return this.frameDataBuffer;\n        }\n        return frameData;\n    }\n\n    /**\n     * Resize the image data if the model expects a different input size.\n     * Handles scaling between input dimensions and model requirements.\n     *\n     * @param rawImageData - The raw image data to potentially resize.\n     * @returns The processed mask image data, resized if necessary.\n     */\n    private getMaskImageData(rawImageData: Uint8ClampedArray): Uint8ClampedArray {\n        const modelInput = this.config.mask.inputSize;\n        const inputImage = this.config.input;\n\n        // Check if resizing is needed based on configuration.\n        if (modelInput.width === inputImage.width && modelInput.height === inputImage.height) {\n            return rawImageData;\n        }\n\n        // Resize the image data.\n        return resizeImage(\n            rawImageData,\n            inputImage.width,\n            inputImage.height,\n            modelInput.width,\n            modelInput.height\n        );\n    }\n\n    /**\n     * Create the inference result object with appropriate data.\n     * Handles both successful inference results and fallback cases.\n     *\n     * @param timestamp - The timestamp of the frame in milliseconds.\n     * @param performedInference - Whether inference was actually performed for this frame.\n     * @returns The inference result object containing mask data and metadata.\n     */\n    private createInferenceResult(\n        timestamp: number,\n        performedInference: boolean\n    ): IFrameInferenceResult {\n        if (!this.lastMaskResult) {\n            return {\n                facesAndLandmarks: { data: new Float32Array(0), dims: [] },\n                gesture: new Float32Array(0),\n                motion: 0,\n                skipped: true,\n                timestamp,\n                // Stats will be added by the caller if needed\n            };\n        }\n\n        return {\n            facesAndLandmarks: this.lastMaskResult.facesAndLandmarks,\n            gesture: this.lastMaskResult.gesture,\n            motion: this.lastMaskResult.motion,\n            skipped: !performedInference,\n            timestamp,\n            // Stats will be added by the caller if needed\n        };\n    }\n\n    /**\n     * Check if the pipeline is fully loaded and ready for processing.\n     *\n     * @returns True if the pipeline is loaded and ready, false otherwise.\n     */\n    public isLoaded(): boolean {\n        return this.loaded;\n    }\n\n    /**\n     * Load and initialize the pipeline components.\n     * Creates the appropriate renderer and generator based on configuration.\n     *\n     * @returns Promise that resolves when all components are loaded and ready.\n     * @throws Error if an unsupported renderer type is specified in the configuration.\n     */\n    public async load(): Promise<void> {\n        switch (this.config.render.type) {\n            case 'blur':\n                this.renderer = new BlurRenderer(this.outputCanvas, this.config);\n                break;\n            case 'passthrough':\n                this.renderer = new PassthroughRenderer(this.outputCanvas, this.config);\n                break;\n            case 'replacement':\n                this.renderer = new ReplacementRenderer(this.outputCanvas, this.config);\n                break;\n            default:\n                throw new Error('[ladon-ts] Renderer not implemented.');\n        }\n\n        await this.renderer.load();\n\n        this.generator = new WorkerGenerator(this.config);\n        await this.generator.load();\n\n        this.loaded = true;\n    }\n\n    /**\n     * Reset the pipeline state by clearing cached mask results.\n     * Useful when switching between different processing modes or sources.\n     */\n    public reset(): void {\n        this.lastMaskResult = undefined;\n    }\n\n    /**\n     * Destroy the pipeline and clean up all resources.\n     * Properly disposes of the renderer and generator to prevent memory leaks.\n     *\n     * @returns Promise that resolves when cleanup is complete.\n     */\n    public async destroy(): Promise<void> {\n        this.reset();\n        this.renderer?.destroy();\n        await this.generator?.destroy();\n        this.loaded = false;\n    }\n}\n"],"names":["Pipeline","inputCanvas","outputCanvas","config","inputVideoElement","inputCanvasContext","timestamp","performInference","frameData","frameStartTime","preprocessingTime","modelInferenceTime","renderFrameData","preprocessStart","inferenceStart","rawImageData","maskImageData","postprocessStart","postprocessingTime","frameEndTime","totalFrameTime","result","_a","modelInput","inputImage","resizeImage","performedInference","BlurRenderer","PassthroughRenderer","ReplacementRenderer","WorkerGenerator","_b"],"mappings":";;;;;AAUO,MAAMA,EAA8B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAkCvC,YACYC,GACAC,GACQC,GACCC,GACnB;AAJU,SAAA,cAAAH,GACA,KAAA,eAAAC,GACQ,KAAA,SAAAC,GACC,KAAA,oBAAAC,GA9BrB,KAAQ,SAAS,IASjB,KAAQ,oBAAoB,IAG5B,KAAQ,sBAA6D;AAoBjE,UAAMC,IAAqB,KAAK,YAAY,WAAW,MAAM;AAAA,MACzD,OAAO;AAAA,MACP,oBAAoB;AAAA,IAAA,CACvB;AAED,QAAI,CAACA;AACK,YAAA,IAAI,MAAM,2CAA2C;AAE/D,SAAK,qBAAqBA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAc9B,MAAa,QACTC,GACAC,IAAmB,IACnBC,GAC8B;AAE1B,IAAA,KAAK,qBAAqB,KAAK,uBAC/B,MAAM,KAAK,qBAIf,KAAK,oBAAoB;AAErB,QAAA;AACA,kBAAK,sBAAsB,KAAK;AAAA,QAC5BF;AAAA,QACAC;AAAA,QACAC;AAAA,MACJ,GACO,MAAM,KAAK;AAAA,IAAA,UACpB;AACE,WAAK,oBAAoB,IACzB,KAAK,sBAAsB;AAAA,IAAA;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaJ,MAAc,qBACVF,GACAC,IAAmB,IACnBC,GAC8B;AACxB,UAAAC,IAAiB,YAAY,IAAI;AAEvC,QAAI,CAAC,KAAK,aAAa,CAAC,KAAK;AACnB,YAAA,IAAI,MAAM,oCAAoC;AAGxD,QAAIC,IAAoB,GACpBC,IAAqB,GACrBC;AAEA,QAAAL,KAAoB,CAAC,KAAK,gBAAgB;AAEpC,YAAAM,IAAkB,YAAY,IAAI;AAGxC,MAAAL,UAAc,KAAK,kBAAkB,IAGnBI,IAAA,KAAK,oBAAoBJ,CAAS,GAEhCE,IAAA,YAAY,QAAQG;AAGlC,YAAAC,IAAiB,YAAY,IAAI,GACjCC,IAAeP,GACfQ,IAAgB,KAAK,iBAAiBD,CAAY;AACxD,YAAM,KAAK,UAAU,QAAQC,GAAeV,CAAS,GACrD,KAAK,iBAAiB,MAAM,KAAK,UAAU,2BAA2BA,CAAS,GAC1DK,IAAA,YAAY,QAAQG;AAAA,IAAA;AAGzC,MAAAF,IAAkB,KAAK;AAIrB,UAAAK,IAAmB,YAAY,IAAI;AACzC,UAAM,KAAK,SAAS,OAAOL,GAAiB,KAAK,cAAc;AACzD,UAAAM,IAAqB,YAAY,IAAA,IAAQD,GAEzCE,IAAe,YAAY,IAAI,GAC/BC,IAAiBD,IAAeV,GAEhCY,IAAS,KAAK,sBAAsBf,GAAWC,CAAgB;AACrE,WAAAc,EAAO,QAAQ;AAAA,MACX,gBAAAD;AAAA,MACA,oBAAAT;AAAA,MACA,mBAAAD;AAAA,MACA,oBAAAQ;AAAA,MACA,gBAAAT;AAAA,MACA,cAAAU;AAAA,MACA,kBAAkB,CAACZ;AAAA,IACvB,GAEOc;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQH,oBAA+B;AACnC,WAAO,KAAK,mBAAmB;AAAA,MAC3B;AAAA,MACA;AAAA,MACA,KAAK,OAAO,MAAM;AAAA,MAClB,KAAK,OAAO,MAAM;AAAA,IAAA,EACpB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA,EAME,oBAAoBb,GAAiC;;AACzD,WAAIA,aAAqB,uBACjBc,IAAA,KAAK,oBAAL,gBAAAA,EAAsB,YAAWd,EAAU,WAC3C,KAAK,kBAAkB,IAAI,kBAAkBA,EAAU,MAAM,IAE5D,KAAA,gBAAgB,IAAIA,CAAS,GAC3B,KAAK,mBAETA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUH,iBAAiBO,GAAoD;AACnE,UAAAQ,IAAa,KAAK,OAAO,KAAK,WAC9BC,IAAa,KAAK,OAAO;AAG/B,WAAID,EAAW,UAAUC,EAAW,SAASD,EAAW,WAAWC,EAAW,SACnET,IAIJU;AAAA,MACHV;AAAA,MACAS,EAAW;AAAA,MACXA,EAAW;AAAA,MACXD,EAAW;AAAA,MACXA,EAAW;AAAA,IACf;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWI,sBACJjB,GACAoB,GACqB;AACjB,WAAC,KAAK,iBAWH;AAAA,MACH,mBAAmB,KAAK,eAAe;AAAA,MACvC,SAAS,KAAK,eAAe;AAAA,MAC7B,QAAQ,KAAK,eAAe;AAAA,MAC5B,SAAS,CAACA;AAAA,MACV,WAAApB;AAAA;AAAA,IAEJ,IAjBW;AAAA,MACH,mBAAmB,EAAE,MAAM,IAAI,aAAa,CAAC,GAAG,MAAM,GAAG;AAAA,MACzD,SAAS,IAAI,aAAa,CAAC;AAAA,MAC3B,QAAQ;AAAA,MACR,SAAS;AAAA,MACT,WAAAA;AAAA;AAAA,IAEJ;AAAA,EAUJ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQG,WAAoB;AACvB,WAAO,KAAK;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUhB,MAAa,OAAsB;AACvB,YAAA,KAAK,OAAO,OAAO,MAAM;AAAA,MAC7B,KAAK;AACD,aAAK,WAAW,IAAIqB,EAAa,KAAK,cAAc,KAAK,MAAM;AAC/D;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ,KAAK;AACD,aAAK,WAAW,IAAIC,EAAoB,KAAK,cAAc,KAAK,MAAM;AACtE;AAAA,MACJ;AACU,cAAA,IAAI,MAAM,sCAAsC;AAAA,IAAA;AAGxD,UAAA,KAAK,SAAS,KAAK,GAEzB,KAAK,YAAY,IAAIC,EAAgB,KAAK,MAAM,GAC1C,MAAA,KAAK,UAAU,KAAK,GAE1B,KAAK,SAAS;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOX,QAAc;AACjB,SAAK,iBAAiB;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAS1B,MAAa,UAAyB;;AAClC,SAAK,MAAM,IACXR,IAAA,KAAK,aAAL,QAAAA,EAAe,WACT,QAAAS,IAAA,KAAK,cAAL,gBAAAA,EAAgB,YACtB,KAAK,SAAS;AAAA,EAAA;AAEtB;"}

package/build/module/lib/version.d.ts

@@ -1 +1 @@
-export declare const LIB_VERSION = "5.9.0";
+export declare const LIB_VERSION = "5.9.1";

package/build/module/lib/version.js

@@ -1,4 +1,4 @@
-const o = "5.9.0";
+const o = "5.9.1";
 export {
   o as LIB_VERSION
 };

package/build/module/lib/version.js.map

@@ -1 +1 @@
-{"version":3,"file":"version.js","sources":["../../../src/lib/version.ts"],"sourcesContent":["export const LIB_VERSION = '5.9.0';\n"],"names":["LIB_VERSION"],"mappings":"AAAO,MAAMA,IAAc;"}
+{"version":3,"file":"version.js","sources":["../../../src/lib/version.ts"],"sourcesContent":["export const LIB_VERSION = '5.9.1';\n"],"names":["LIB_VERSION"],"mappings":"AAAO,MAAMA,IAAc;"}

package/build/worker/segmentation.js

@@ -1,7 +1,7 @@
 /**
  * Cisco Ladon-TS 
  * Copyright (c) Cisco Systems, Inc. All rights reserved. 
- * Build timestamp: July 1, 2025 at 10:17:37 PM UTC
+ * Build timestamp: July 2, 2025 at 09:45:43 PM UTC
  */
 (function(){"use strict";var _documentCurrentScript=typeof document<"u"?document.currentScript:null;self.document={baseURI:location.origin};const originalConsoleLog=console.log,originalConsoleWarn=console.warn,originalConsoleError=console.error;console.log=(...e)=>{postMessage({type:"log",message:e.join(" "),severity:"log"}),originalConsoleLog(...e)},console.warn=(...e)=>{postMessage({type:"log",message:e.join(" "),severity:"warn"}),originalConsoleWarn(...e)},console.error=(...e)=>{postMessage({type:"log",message:e.join(" "),severity:"error"}),originalConsoleError(...e)};/*!
  * ONNX Runtime Web v1.20.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webex/ladon-ts",
-  "version": "5.9.0",
+  "version": "5.9.1",
   "description": "WebexIntelligence computer vision library",
   "type": "module",
   "main": "build/module/index.js",