@viji-dev/core 0.2.7 → 0.2.9

package/dist/index.d.ts

@@ -61,7 +61,12 @@ declare interface CoreCapabilities {
     hasGeneral: boolean;
 }
 
+declare type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation';
+
+declare type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';
+
 export declare interface FaceData {
+    id: number;
     bounds: {
         x: number;
         y: number;
@@ -69,11 +74,25 @@ export declare interface FaceData {
         height: number;
     };
     confidence: number;
-    landmarks: {
+    landmarks?: {
         x: number;
         y: number;
+        z?: number;
     }[];
-    expressions: Record<string, number>;
+    expressions: {
+        neutral: number;
+        happy: number;
+        sad: number;
+        angry: number;
+        surprised: number;
+        disgusted: number;
+        fearful: number;
+    };
+    headPose: {
+        pitch: number;
+        yaw: number;
+        roll: number;
+    };
 }
 
 declare interface FrameRateInfo {
@@ -91,15 +110,89 @@ export declare interface FrequencyBand {
 }
 
 export declare interface HandData {
-    palm: {
+    id: number;
+    handedness: 'left' | 'right';
+    confidence: number;
+    bounds: {
         x: number;
         y: number;
+        width: number;
+        height: number;
     };
-    fingers: {
+    landmarks: {
         x: number;
         y: number;
+        z: number;
     }[];
-    gestures: Record<string, boolean>;
+    palm: {
+        x: number;
+        y: number;
+        z: number;
+    };
+    fingers: {
+        thumb: {
+            tip: {
+                x: number;
+                y: number;
+                z: number;
+            };
+            extended: boolean;
+        };
+        index: {
+            tip: {
+                x: number;
+                y: number;
+                z: number;
+            };
+            extended: boolean;
+        };
+        middle: {
+            tip: {
+                x: number;
+                y: number;
+                z: number;
+            };
+            extended: boolean;
+        };
+        ring: {
+            tip: {
+                x: number;
+                y: number;
+                z: number;
+            };
+            extended: boolean;
+        };
+        pinky: {
+            tip: {
+                x: number;
+                y: number;
+                z: number;
+            };
+            extended: boolean;
+        };
+    };
+    gestures: {
+        fist: number;
+        openPalm: number;
+        peace: number;
+        thumbsUp: number;
+        pointing: number;
+    };
+}
+
+declare interface ImageConfig {
+    label: string;
+    description?: string;
+    group?: string;
+    category?: ParameterCategory;
+}
+
+declare interface ImageParameter {
+    value: ImageBitmap | OffscreenCanvas | null;
+    label: string;
+    description?: string;
+    group: string;
+    category: ParameterCategory;
 }
 
 export declare interface KeyboardAPI {
@@ -180,7 +273,7 @@ export declare interface ParameterConfig {
 }
 
 export declare interface ParameterDefinition {
-    type: 'slider' | 'color' | 'toggle' | 'select' | 'text' | 'number';
+    type: 'slider' | 'color' | 'toggle' | 'select' | 'text' | 'number' | 'image';
     defaultValue: any;
     label?: string;
     description?: string;
@@ -199,23 +292,73 @@ export declare interface ParameterGroup {
     parameters: Record<string, ParameterDefinition>;
 }
 
-declare type ParameterValue = string | number | boolean;
+declare type ParameterValue = string | number | boolean | ImageBitmap | OffscreenCanvas | null;
 
 export declare interface PerformanceStats {
     frameTime: number;
     resolution: Resolution;
     scale: number;
     frameRate: FrameRateInfo;
+    rendererType: RendererType;
     memoryUsage?: number;
     activeStreams?: number;
     parameterCount?: number;
+    cv?: {
+        activeFeatures: CVFeature[];
+        processingTime: number;
+        targetFPS: number;
+        actualFPS: number;
+        isProcessing: boolean;
+    };
 }
 
+declare interface PoseData {
+    confidence: number;
+    landmarks: {
+        x: number;
+        y: number;
+        z: number;
+        visibility: number;
+    }[];
+    face: {
+        x: number;
+        y: number;
+    }[];
+    torso: {
+        x: number;
+        y: number;
+    }[];
+    leftArm: {
+        x: number;
+        y: number;
+    }[];
+    rightArm: {
+        x: number;
+        y: number;
+    }[];
+    leftLeg: {
+        x: number;
+        y: number;
+    }[];
+    rightLeg: {
+        x: number;
+        y: number;
+    }[];
+}
+
+declare type RendererType = 'native' | 'p5' | 'shader';
+
 export declare type Resolution = {
     width: number;
     height: number;
 };
 
+declare interface SegmentationData {
+    mask: Uint8Array;
+    width: number;
+    height: number;
+}
+
 declare interface SelectConfig {
     options: string[] | number[];
     label: string;
@@ -338,7 +481,7 @@ declare interface TouchPoint {
     isEnding: boolean;
 }
 
-export declare const VERSION = "0.2.4";
+export declare const VERSION = "0.2.9";
 
 export declare interface VideoAPI {
     isConnected: boolean;
@@ -349,6 +492,17 @@ export declare interface VideoAPI {
     getFrameData: () => ImageData | null;
     faces: FaceData[];
     hands: HandData[];
+    pose: PoseData | null;
+    segmentation: SegmentationData | null;
+    cv: {
+        enableFaceDetection(enabled: boolean): Promise<void>;
+        enableFaceMesh(enabled: boolean): Promise<void>;
+        enableHandTracking(enabled: boolean): Promise<void>;
+        enablePoseDetection(enabled: boolean): Promise<void>;
+        enableBodySegmentation(enabled: boolean): Promise<void>;
+        getActiveFeatures(): CVFeature[];
+        isProcessing(): boolean;
+    };
 }
 
 export declare interface VijiAPI {
@@ -373,6 +527,7 @@ export declare interface VijiAPI {
     select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;
     text: (defaultValue: string, config: TextConfig) => TextParameter;
     number: (defaultValue: number, config: NumberConfig) => NumberParameter;
+    image: (defaultValue: null, config: ImageConfig) => ImageParameter;
     useContext(type: '2d' | 'webgl'): OffscreenCanvasRenderingContext2D | WebGLRenderingContext | WebGL2RenderingContext | null;
 }
 
@@ -448,10 +603,22 @@ export declare class VijiCore {
      * Handle parameter validation errors
      */
     private handleParameterError;
+    /**
+     * Get parameter definition by name
+     */
+    private getParameterDefinition;
     /**
      * Set a single parameter value
+     * Handles all parameter types including images intelligently
+     */
+    setParameter(name: string, value: ParameterValue | File | Blob): Promise<void>;
+    /**
+     * Internal method to handle image parameter loading and transfer
+     * Loads the image on the host side and creates TWO ImageBitmaps:
+     * - One for the host (listeners, previews)
+     * - One to transfer to the worker
      */
-    setParameter(name: string, value: ParameterValue): Promise<void>;
+    private handleImageParameter;
     /**
      * Set multiple parameter values efficiently
      */
@@ -540,13 +707,15 @@ export declare class VijiCore {
      */
     private notifyParameterListeners;
     /**
-     * Sets the frame rate to full speed (every animation frame)
+     * Sets the scene frame rate mode
+     * @param mode - 'full' for every animation frame, 'half' for every second frame
      */
-    setFullFrameRate(): Promise<void>;
+    setFrameRate(mode: FrameRateMode): Promise<void>;
     /**
-     * Sets the frame rate to half speed (every second animation frame)
+     * Sets the CV processing frame rate mode (relative to scene frame rate)
+     * @param mode - CV processing rate: 'full', 'half', 'quarter', or 'eighth' of scene rate
      */
-    setHalfFrameRate(): Promise<void>;
+    setCVFrameRate(mode: CVFrameRateMode): Promise<void>;
     /**
      * Updates the canvas resolution by sending effective dimensions to the worker
      */

package/dist/index.js

@@ -434,8 +434,9 @@ class IFrameManager {
 }
 function WorkerWrapper(options) {
   return new Worker(
-    "" + new URL("assets/viji.worker-BKsgIT1d.js", import.meta.url).href,
+    "" + new URL("assets/viji.worker-Cbdf1a6N.js", import.meta.url).href,
     {
+      type: "module",
       name: options?.name
     }
   );
@@ -460,7 +461,6 @@ class WorkerManager {
       this.postMessage("set-scene-code", { sceneCode: this.sceneCode });
       await this.initializeWorker();
       this.isInitialized = true;
-      return this.worker;
     } catch (error) {
       throw new VijiCoreError(
         `Failed to create worker: ${error}`,
@@ -599,6 +599,7 @@ class WorkerManager {
   }
   /**
    * Initializes the worker with canvas and basic setup
+   * Waits for init-response message (worker will also send separate 'ready' event)
    */
   async initializeWorker() {
     if (!this.worker) {
@@ -1695,6 +1696,7 @@ class VijiCore {
       screenRefreshRate: 60,
       effectiveRefreshRate: 60
     },
+    rendererType: "native",
     parameterCount: 0
   };
   constructor(config) {
@@ -1894,6 +1896,9 @@ class VijiCore {
     if (!this.workerManager) return;
     this.workerManager.onMessage("ready", (data) => {
       this.debugLog("Worker ready:", data);
+      if (data.rendererType !== void 0) {
+        this.stats.rendererType = data.rendererType;
+      }
     });
     this.workerManager.onMessage("error", (data) => {
       console.error("Worker error:", data);
@@ -1907,9 +1912,15 @@ class VijiCore {
         this.stats.frameRate.mode = data.frameRateMode;
         this.stats.frameRate.screenRefreshRate = data.screenRefreshRate;
       }
+      if (data.rendererType !== void 0) {
+        this.stats.rendererType = data.rendererType;
+      }
       if (data.parameterCount !== void 0) {
         this.stats.parameterCount = data.parameterCount;
       }
+      if (data.cv !== void 0) {
+        this.stats.cv = data.cv;
+      }
     });
     this.workerManager.onMessage("parameters-defined", (data) => {
       this.handleParametersDefined(data);
@@ -1963,8 +1974,20 @@ class VijiCore {
       }
     }
   }
+  /**
+   * Get parameter definition by name
+   */
+  getParameterDefinition(name) {
+    for (const group of this.parameterGroups.values()) {
+      if (group.parameters[name]) {
+        return group.parameters[name];
+      }
+    }
+    return void 0;
+  }
   /**
    * Set a single parameter value
+   * Handles all parameter types including images intelligently
    */
   async setParameter(name, value) {
     this.validateReady();
@@ -1974,6 +1997,11 @@ class VijiCore {
     if (!this.parameterValues.has(name)) {
       throw new VijiCoreError(`Unknown parameter: ${name}`, "UNKNOWN_PARAMETER");
     }
+    const paramDef = this.getParameterDefinition(name);
+    if (paramDef?.type === "image") {
+      await this.handleImageParameter(name, value);
+      return;
+    }
     const oldValue = this.parameterValues.get(name);
     this.parameterValues.set(name, value);
     if (this.workerManager) {
@@ -1986,6 +2014,64 @@ class VijiCore {
       this.notifyParameterListeners(name, value);
     }
   }
+  /**
+   * Internal method to handle image parameter loading and transfer
+   * Loads the image on the host side and creates TWO ImageBitmaps:
+   * - One for the host (listeners, previews)
+   * - One to transfer to the worker
+   */
+  async handleImageParameter(name, value) {
+    try {
+      let hostImageBitmap = null;
+      let workerImageBitmap = null;
+      if (value === null || value === void 0) {
+        hostImageBitmap = null;
+        workerImageBitmap = null;
+      } else if (typeof value === "string") {
+        const response = await fetch(value);
+        const blob = await response.blob();
+        [hostImageBitmap, workerImageBitmap] = await Promise.all([
+          createImageBitmap(blob),
+          createImageBitmap(blob)
+        ]);
+      } else if (value instanceof File || value instanceof Blob) {
+        [hostImageBitmap, workerImageBitmap] = await Promise.all([
+          createImageBitmap(value),
+          createImageBitmap(value)
+        ]);
+      } else if (value instanceof ImageBitmap) {
+        const canvas = document.createElement("canvas");
+        canvas.width = value.width;
+        canvas.height = value.height;
+        const ctx = canvas.getContext("2d");
+        ctx.drawImage(value, 0, 0);
+        hostImageBitmap = value;
+        workerImageBitmap = await createImageBitmap(canvas);
+      } else {
+        throw new Error(`Invalid value type for image parameter. Expected File, Blob, string (URL), or null.`);
+      }
+      const oldValue = this.parameterValues.get(name);
+      this.parameterValues.set(name, hostImageBitmap);
+      if (this.workerManager) {
+        const messageData = {
+          name,
+          value: workerImageBitmap
+        };
+        const transferList = workerImageBitmap ? [workerImageBitmap] : [];
+        this.workerManager.postMessage("parameter-update", messageData, transferList);
+      }
+      if (oldValue !== hostImageBitmap) {
+        this.notifyParameterListeners(name, hostImageBitmap);
+      }
+      this.debugLog(`Image parameter '${name}' ${hostImageBitmap ? "loaded" : "cleared"} (${this.instanceId})`);
+    } catch (error) {
+      throw new VijiCoreError(
+        `Failed to load image for parameter '${name}': ${error}`,
+        "IMAGE_LOAD_ERROR",
+        { error, name }
+      );
+    }
+  }
   /**
    * Set multiple parameter values efficiently
    */
@@ -2005,8 +2091,7 @@ class VijiCore {
       this.parameterValues.set(name, value);
       updates.push({
         name,
-        value,
-        timestamp: performance.now()
+        value
       });
       if (oldValue !== value) {
         changedParams.push({ name, value });
@@ -2014,8 +2099,7 @@ class VijiCore {
     }
     if (updates.length > 0 && this.workerManager) {
       this.workerManager.postMessage("parameter-batch-update", {
-        updates,
-        timestamp: performance.now()
+        updates
       });
     }
     for (const { name, value } of changedParams) {
@@ -2147,13 +2231,11 @@ class VijiCore {
     for (const [name, value] of this.parameterValues) {
       updates.push({
         name,
-        value,
-        timestamp: performance.now()
+        value
       });
     }
     this.workerManager.postMessage("parameter-batch-update", {
-      updates,
-      timestamp: performance.now()
+      updates
     });
     this.debugLog(`Synced ${updates.length} parameter values to worker`);
   }
@@ -2243,28 +2325,28 @@ class VijiCore {
     }
   }
   /**
-   * Sets the frame rate to full speed (every animation frame)
+   * Sets the scene frame rate mode
+   * @param mode - 'full' for every animation frame, 'half' for every second frame
    */
-  async setFullFrameRate() {
+  async setFrameRate(mode) {
     this.validateReady();
-    this.config.frameRateMode = "full";
+    this.config.frameRateMode = mode;
     if (this.workerManager) {
-      this.workerManager.postMessage("frame-rate-update", { mode: "full" });
+      this.workerManager.postMessage("frame-rate-update", { mode });
     }
     this.updateFrameRateStats();
-    this.debugLog(`Frame rate set to full (${this.instanceId})`);
+    this.debugLog(`Scene frame rate set to ${mode} (${this.instanceId})`);
   }
   /**
-   * Sets the frame rate to half speed (every second animation frame)
+   * Sets the CV processing frame rate mode (relative to scene frame rate)
+   * @param mode - CV processing rate: 'full', 'half', 'quarter', or 'eighth' of scene rate
    */
-  async setHalfFrameRate() {
+  async setCVFrameRate(mode) {
     this.validateReady();
-    this.config.frameRateMode = "half";
     if (this.workerManager) {
-      this.workerManager.postMessage("frame-rate-update", { mode: "half" });
+      this.workerManager.postMessage("cv-frame-rate-update", { mode });
     }
-    this.updateFrameRateStats();
-    this.debugLog(`Frame rate set to half (${this.instanceId})`);
+    this.debugLog(`CV frame rate set to ${mode} of scene rate (${this.instanceId})`);
   }
   /**
    * Updates the canvas resolution by sending effective dimensions to the worker
@@ -2525,7 +2607,7 @@ class VijiCore {
     }
   }
 }
-const VERSION = "0.2.4";
+const VERSION = "0.2.9";
 export {
   VERSION,
   VijiCore,