@viji-dev/core 0.2.7 → 0.2.8

package/dist/assets/cv-tasks.worker.js

@@ -0,0 +1,623 @@
+/**
+ * MediaPipe Tasks Vision Classic Worker
+ * 
+ * Classic worker for MediaPipe Tasks Vision processing.
+ * Uses importScripts() to load MediaPipe Tasks Vision UMD bundle.
+ */
+
+// Define CommonJS environment for MediaPipe bundle
+self.exports = {};
+self.module = { exports: {} };
+
+// Import MediaPipe Tasks Vision UMD bundle
+console.log('🔧 [CV Tasks Worker] Starting to load vision_bundle.js...');
+try {
+  importScripts('/dist/assets/vision_bundle.js');
+  console.log('✅ [CV Tasks Worker] vision_bundle.js loaded successfully');
+} catch (error) {
+  console.error('❌ [CV Tasks Worker] Failed to load vision_bundle.js:', error);
+}
+
+// Debug: Check what's available after import (disabled for production)
+// console.log('🔧 [CV Tasks Worker] Available globals after import:', Object.keys(self));
+// console.log('🔧 [CV Tasks Worker] module.exports:', self.module.exports);
+// console.log('🔧 [CV Tasks Worker] exports:', self.exports);
+
+// MediaPipe model instances
+let faceDetector = null;
+let faceLandmarker = null;
+let handLandmarker = null;
+let poseLandmarker = null;
+let imageSegmenter = null;
+
+// Vision runtime
+let vision = null;
+let isInitialized = false;
+
+// Active features tracking
+const activeFeatures = new Set();
+
+// Configuration queue to prevent race conditions
+const configQueue = [];
+let processingConfig = false;
+
+// Worker health tracking
+let workerHealthy = true;
+let memoryPressureDetected = false;
+
+// Note: No longer need reusable canvas - passing ImageBitmap directly to MediaPipe!
+
+// Debug logging
+const DEBUG = true; // Temporarily enabled to debug segmentation
+function log(...args) {
+  if (DEBUG) {
+    console.log('🔧 [CV Tasks Worker]', ...args);
+  }
+}
+
+/**
+ * Initialize MediaPipe Tasks Vision runtime
+ */
+async function initializeVision() {
+  if (isInitialized) {
+    console.log('🔧 [CV Tasks Worker] Vision already initialized, skipping');
+    return;
+  }
+  
+  try {
+    console.log('🔧 [CV Tasks Worker] Starting MediaPipe Tasks Vision initialization...');
+    
+    // Initialize the vision runtime with WASM files
+    // MediaPipe Tasks Vision expects the base path without trailing slash
+    const wasmBasePath = '/dist/assets/wasm';
+    log('WASM base path:', wasmBasePath);
+    
+    // Try different ways to access FilesetResolver
+    const FilesetResolver = self.FilesetResolver || self.module.exports.FilesetResolver || self.exports.FilesetResolver;
+    console.log('🔧 [CV Tasks Worker] FilesetResolver found:', !!FilesetResolver);
+    
+    if (!FilesetResolver) {
+      throw new Error('FilesetResolver not found in any expected location');
+    }
+    
+    vision = await FilesetResolver.forVisionTasks(wasmBasePath);
+    
+    isInitialized = true;
+    log('✅ MediaPipe Tasks Vision initialized successfully');
+  } catch (error) {
+    log('❌ Failed to initialize MediaPipe Tasks Vision:', error);
+    throw error;
+  }
+}
+
+/**
+ * Load and initialize Face Detection
+ */
+async function initializeFaceDetection() {
+  if (faceDetector) return;
+  
+  // Ensure vision runtime is initialized first
+  await initializeVision();
+  
+  try {
+    log('Loading Face Detector...');
+    
+    const options = {
+      baseOptions: {
+        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.tflite',
+        delegate: 'GPU'
+      },
+      runningMode: 'VIDEO',
+      minDetectionConfidence: 0.5,
+      minSuppressionThreshold: 0.3
+    };
+    
+    const FaceDetector = self.FaceDetector || self.module.exports.FaceDetector || self.exports.FaceDetector;
+    faceDetector = await FaceDetector.createFromOptions(vision, options);
+    log('✅ Face Detector loaded');
+  } catch (error) {
+    log('❌ Failed to load Face Detector:', error);
+    throw error;
+  }
+}
+
+/**
+ * Load and initialize Face Landmarks
+ */
+async function initializeFaceLandmarks() {
+  if (faceLandmarker) return;
+  
+  // Ensure vision runtime is initialized first
+  await initializeVision();
+  
+  try {
+    log('Loading Face Landmarker...');
+    
+    const options = {
+      baseOptions: {
+        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
+        delegate: 'GPU'
+      },
+      runningMode: 'VIDEO',
+      numFaces: 1
+    };
+    
+    const FaceLandmarker = self.FaceLandmarker || self.module.exports.FaceLandmarker || self.exports.FaceLandmarker;
+    faceLandmarker = await FaceLandmarker.createFromOptions(vision, options);
+    log('✅ Face Landmarker loaded');
+  } catch (error) {
+    log('❌ Failed to load Face Landmarker:', error);
+    throw error;
+  }
+}
+
+/**
+ * Load and initialize Hand Tracking
+ */
+async function initializeHandTracking() {
+  if (handLandmarker) return;
+  
+  // Ensure vision runtime is initialized first
+  await initializeVision();
+  
+  try {
+    log('Loading Hand Landmarker...');
+    
+    const options = {
+      baseOptions: {
+        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
+        delegate: 'GPU'
+      },
+      runningMode: 'VIDEO',
+      numHands: 2
+    };
+    
+    const HandLandmarker = self.HandLandmarker || self.module.exports.HandLandmarker || self.exports.HandLandmarker;
+    handLandmarker = await HandLandmarker.createFromOptions(vision, options);
+    log('✅ Hand Landmarker loaded');
+  } catch (error) {
+    log('❌ Failed to load Hand Landmarker:', error);
+    throw error;
+  }
+}
+
+/**
+ * Load and initialize Pose Detection
+ */
+async function initializePoseDetection() {
+  if (poseLandmarker) return;
+  
+  // Ensure vision runtime is initialized first
+  await initializeVision();
+  
+  try {
+    log('Loading Pose Landmarker...');
+    
+    const options = {
+      baseOptions: {
+        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task',
+        delegate: 'GPU'
+      },
+      runningMode: 'VIDEO',
+      numPoses: 1
+    };
+    
+    const PoseLandmarker = self.PoseLandmarker || self.module.exports.PoseLandmarker || self.exports.PoseLandmarker;
+    poseLandmarker = await PoseLandmarker.createFromOptions(vision, options);
+    log('✅ Pose Landmarker loaded');
+  } catch (error) {
+    log('❌ Failed to load Pose Landmarker:', error);
+    throw error;
+  }
+}
+
+/**
+ * Load and initialize Body Segmentation
+ */
+async function initializeBodySegmentation() {
+  if (imageSegmenter) return;
+  
+  // Ensure vision runtime is initialized first
+  await initializeVision();
+  
+  try {
+    log('Loading Image Segmenter...');
+    
+      const options = {
+        baseOptions: {
+          modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/1/selfie_segmenter.tflite',
+          delegate: 'GPU'
+        },
+        runningMode: 'IMAGE',
+        outputCategoryMask: true,
+        outputConfidenceMasks: false
+      };
+    
+    const ImageSegmenter = self.ImageSegmenter || self.module.exports.ImageSegmenter || self.exports.ImageSegmenter;
+    imageSegmenter = await ImageSegmenter.createFromOptions(vision, options);
+    log('✅ Image Segmenter loaded');
+  } catch (error) {
+    log('❌ Failed to load Image Segmenter:', error);
+    throw error;
+  }
+}
+
+/**
+ * Process video frame with active CV features
+ * @param {ImageData|ImageBitmap} imageInput - Image input (ImageData or ImageBitmap)
+ * @param {number} timestamp - Frame timestamp  
+ * @param {string[]} features - Active CV features
+ */
+async function processFrame(imageInput, timestamp, features) {
+  const results = {};
+  
+  try {
+    // Process face detection
+    if (features.includes('faceDetection') && faceDetector) {
+      const detectionResult = faceDetector.detectForVideo(imageInput, timestamp);
+      results.faces = detectionResult.detections.map((detection) => ({
+        boundingBox: {
+          // Normalize coordinates to 0-1 range to match other CV features
+          x: detection.boundingBox.originX / imageInput.width,
+          y: detection.boundingBox.originY / imageInput.height,
+          width: detection.boundingBox.width / imageInput.width,
+          height: detection.boundingBox.height / imageInput.height
+        },
+        landmarks: [],
+        expressions: {},
+        confidence: detection.categories[0]?.score || 0
+      }));
+    }
+    
+    // Process face landmarks
+    if (features.includes('faceMesh') && faceLandmarker) {
+      const landmarkResult = faceLandmarker.detectForVideo(imageInput, timestamp);
+      if (landmarkResult.faceLandmarks.length > 0) {
+        const landmarks = landmarkResult.faceLandmarks[0];
+        
+        // If no face detection results exist, create a basic face structure
+        if (!results.faces) {
+          results.faces = [{
+            boundingBox: null, // No bounding box when only mesh is enabled
+            landmarks: [],
+            expressions: {},
+            confidence: 0.8 // Default confidence for mesh-only detection
+          }];
+        }
+        
+        // Add landmarks to the first face (mesh only processes one face)
+        if (results.faces[0]) {
+          results.faces[0].landmarks = landmarks.map((landmark) => ({
+            x: landmark.x,
+            y: landmark.y,
+            z: landmark.z || 0
+          }));
+        }
+      }
+    }
+    
+    // Process hand tracking
+    if (features.includes('handTracking') && handLandmarker) {
+      const handResult = handLandmarker.detectForVideo(imageInput, timestamp);
+      results.hands = handResult.landmarks.map((landmarks, index) => ({
+        landmarks: landmarks.map((landmark) => ({
+          x: landmark.x,
+          y: landmark.y,
+          z: landmark.z || 0
+        })),
+        handedness: handResult.handednesses[index]?.[0]?.categoryName || 'Unknown',
+        confidence: handResult.handednesses[index]?.[0]?.score || 0
+      }));
+    }
+    
+    // Process pose detection
+    if (features.includes('poseDetection') && poseLandmarker) {
+      const poseResult = poseLandmarker.detectForVideo(imageInput, timestamp);
+      if (poseResult.landmarks.length > 0) {
+        results.pose = {
+          landmarks: poseResult.landmarks[0].map((landmark) => ({
+            x: landmark.x,
+            y: landmark.y,
+            z: landmark.z || 0,
+            visibility: landmark.visibility || 1
+          })),
+          worldLandmarks: poseResult.worldLandmarks?.[0]?.map((landmark) => ({
+            x: landmark.x,
+            y: landmark.y,
+            z: landmark.z || 0,
+            visibility: landmark.visibility || 1
+          })) || []
+        };
+      }
+    }
+    
+    // Process body segmentation
+    if (features.includes('bodySegmentation') && imageSegmenter) {
+      const segmentResult = imageSegmenter.segment(imageInput);
+      if (segmentResult.categoryMask) {
+        try {
+          // Extract data before closing the mask
+          results.segmentation = {
+            mask: segmentResult.categoryMask.getAsUint8Array(),
+            width: segmentResult.categoryMask.width,
+            height: segmentResult.categoryMask.height
+          };
+          
+          // Debug logging (temporary)
+          if (DEBUG) {
+            console.log('🔧 [CV Tasks Worker] Segmentation mask:', {
+              width: results.segmentation.width, 
+              height: results.segmentation.height,
+              maskSize: results.segmentation.mask.length
+            });
+          }
+        } finally {
+          // CRITICAL: Close MPMask instance to prevent resource leaks
+          segmentResult.categoryMask.close();
+        }
+      }
+    }
+    
+    return results;
+  } catch (error) {
+    log('❌ Error processing frame:', error);
+    return {};
+  }
+}
+
+// Note: Removed reusable canvas functions - no longer needed with direct ImageBitmap processing!
+
+/**
+ * Clean up WASM instance with proper memory management
+ */
+function cleanupWasmInstance(instance, featureName) {
+  if (instance) {
+    try {
+      log(`🧹 Cleaning up ${featureName} WASM instance...`);
+      instance.close();
+      
+      // Force garbage collection if available (Chrome DevTools)
+      if (typeof gc === 'function') {
+        gc();
+      }
+      
+      // Give time for WASM cleanup
+      return new Promise(resolve => {
+        setTimeout(resolve, 100);
+      });
+    } catch (error) {
+      log(`⚠️ Error cleaning up ${featureName}:`, error);
+    }
+  }
+  return Promise.resolve();
+}
+
+/**
+ * Process configuration queue sequentially
+ */
+async function processConfigQueue() {
+  if (processingConfig || configQueue.length === 0) return;
+  
+  processingConfig = true;
+  
+  try {
+    while (configQueue.length > 0) {
+      const { features, resolve, reject } = configQueue.shift();
+      
+      try {
+        await handleConfigUpdateInternal(features);
+        resolve({ configured: true, activeFeatures: Array.from(activeFeatures) });
+      } catch (error) {
+        reject(error);
+      }
+    }
+  } finally {
+    processingConfig = false;
+  }
+}
+
+/**
+ * Queue configuration update to prevent race conditions
+ */
+function queueConfigUpdate(features) {
+  return new Promise((resolve, reject) => {
+    configQueue.push({ features, resolve, reject });
+    processConfigQueue();
+  });
+}
+
+/**
+ * Handle feature configuration updates (internal)
+ */
+async function handleConfigUpdateInternal(features) {
+  if (!workerHealthy) {
+    throw new Error('Worker is in unhealthy state, restart required');
+  }
+  
+  const newFeatures = new Set(features);
+  const toEnable = features.filter(f => !activeFeatures.has(f));
+  const toDisable = Array.from(activeFeatures).filter(f => !newFeatures.has(f));
+  
+  log(`🔄 Config update: enable [${toEnable.join(', ')}], disable [${toDisable.join(', ')}]`);
+  
+  // Disable unused features first (cleanup instances)
+  const cleanupPromises = [];
+  for (const feature of toDisable) {
+    switch (feature) {
+      case 'faceDetection':
+        cleanupPromises.push(cleanupWasmInstance(faceDetector, 'FaceDetector'));
+        faceDetector = null;
+        break;
+      case 'faceMesh':
+        cleanupPromises.push(cleanupWasmInstance(faceLandmarker, 'FaceLandmarker'));
+        faceLandmarker = null;
+        break;
+      case 'handTracking':
+        cleanupPromises.push(cleanupWasmInstance(handLandmarker, 'HandLandmarker'));
+        handLandmarker = null;
+        break;
+      case 'poseDetection':
+        cleanupPromises.push(cleanupWasmInstance(poseLandmarker, 'PoseLandmarker'));
+        poseLandmarker = null;
+        break;
+      case 'bodySegmentation':
+        cleanupPromises.push(cleanupWasmInstance(imageSegmenter, 'ImageSegmenter'));
+        imageSegmenter = null;
+        break;
+    }
+    activeFeatures.delete(feature);
+    log(`🗑️ Disabled feature: ${feature}`);
+  }
+  
+  // Wait for all cleanup to complete
+  if (cleanupPromises.length > 0) {
+    await Promise.all(cleanupPromises);
+    log('✅ All cleanup completed');
+  }
+  
+  // Note: No canvas cleanup needed - using direct ImageBitmap processing!
+  
+  // Enable new features
+  for (const feature of toEnable) {
+    try {
+      switch (feature) {
+        case 'faceDetection':
+          await initializeFaceDetection();
+          break;
+        case 'faceMesh':
+          await initializeFaceLandmarks();
+          break;
+        case 'handTracking':
+          await initializeHandTracking();
+          break;
+        case 'poseDetection':
+          await initializePoseDetection();
+          break;
+        case 'bodySegmentation':
+          await initializeBodySegmentation();
+          break;
+      }
+      activeFeatures.add(feature);
+      log(`✅ Enabled feature: ${feature}`);
+    } catch (error) {
+      log(`❌ Failed to enable feature ${feature}:`, error);
+      
+      // Check if this is a memory error
+      if (error.message && error.message.includes('Out of memory')) {
+        memoryPressureDetected = true;
+        workerHealthy = false;
+        throw new Error(`Memory exhausted while enabling ${feature}. Worker restart required.`);
+      }
+      
+      throw error;
+    }
+  }
+}
+
+/**
+ * Legacy function for backward compatibility
+ */
+async function handleConfigUpdate(features) {
+  return await queueConfigUpdate(features);
+}
+
+// Message handler
+self.onmessage = async (event) => {
+  const message = event.data;
+  
+  console.log('🔧 [CV Tasks Worker] Received message:', message.type, message);
+  
+  try {
+    switch (message.type) {
+      case 'init': {
+        log('Received init message');
+        
+        try {
+          await initializeVision();
+          log('Vision runtime ready for feature loading');
+        } catch (error) {
+          log('❌ Vision runtime initialization failed:', error);
+          throw error;
+        }
+        
+        const response = {
+          type: 'result',
+          success: true,
+          data: { initialized: true }
+        };
+        self.postMessage(response);
+        break;
+      }
+      
+      case 'config': {
+        log('Received config message:', message.features);
+        
+        try {
+          const result = await handleConfigUpdate(message.features);
+          
+          const response = {
+            type: 'result',
+            success: true,
+            data: result
+          };
+          self.postMessage(response);
+        } catch (error) {
+          log('❌ Config update failed:', error);
+          
+          // Check if worker needs restart
+          if (!workerHealthy || memoryPressureDetected) {
+            const errorResponse = {
+              type: 'result',
+              success: false,
+              error: error.message,
+              restartRequired: true
+            };
+            self.postMessage(errorResponse);
+          } else {
+            throw error; // Re-throw for normal error handling
+          }
+        }
+        break;
+      }
+      
+      case 'process': {
+        try {
+          // 🚀 OPTIMIZED: Pass ImageBitmap directly to MediaPipe (no conversion!)
+          const results = await processFrame(message.bitmap, message.timestamp, message.features);
+          
+          const response = {
+            type: 'result',
+            success: true,
+            data: results
+          };
+          self.postMessage(response);
+        } finally {
+          // Clean up ImageBitmap after processing
+          if (message.bitmap && typeof message.bitmap.close === 'function') {
+            message.bitmap.close();
+          }
+        }
+        break;
+      }
+      
+      default:
+        log('❌ Unknown message type:', message.type);
+        const errorResponse = {
+          type: 'result',
+          success: false,
+          error: `Unknown message type: ${message.type}`
+        };
+        self.postMessage(errorResponse);
+    }
+  } catch (error) {
+    log('❌ Error handling message:', error);
+    const errorResponse = {
+      type: 'result',
+      success: false,
+      error: error instanceof Error ? error.message : String(error)
+    };
+    self.postMessage(errorResponse);
+  }
+};
+
+log('CV Tasks Worker initialized and ready');