@viji-dev/core 0.3.24 → 0.3.26

package/dist/artist-jsdoc.d.ts

@@ -4,29 +4,422 @@
  */
 
 /**
- * Global viji object available in artist code
- * @type {VijiAPI}
+ * Configuration for slider parameters
+ * @typedef {Object} SliderConfig
+ * @property {number} [min] - min property
+ * @property {number} [max] - max property
+ * @property {number} [step] - step property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
  */
-declare const viji;
 
 /**
- * Type alias for render function
- * @typedef {function(VijiAPI): void} Render
+ * Configuration for color parameters
+ * @typedef {Object} ColorConfig
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
  */
 
 /**
- * Type alias for setup function
- * @typedef {function(VijiAPI): void} Setup
+ * Configuration for toggle parameters
+ * @typedef {Object} ToggleConfig
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
+ */
+
+/**
+ * Configuration for select parameters
+ * @typedef {Object} SelectConfig
+ * @property {string[] | number[]} options - options property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
+ */
+
+/**
+ * Configuration for text parameters
+ * @typedef {Object} TextConfig
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
+ * @property {number} [maxLength] - maxLength property
+ */
+
+/**
+ * Configuration for number parameters
+ * @typedef {Object} NumberConfig
+ * @property {number} [min] - min property
+ * @property {number} [max] - max property
+ * @property {number} [step] - step property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
+ */
+
+/**
+ * Configuration for image parameters
+ * @typedef {Object} ImageConfig
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
+ */
+
+/**
+ * Configuration for button parameters
+ * @typedef {Object} ButtonConfig
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} [group] - group property
+ * @property {ParameterCategory} [category] - category property
+ */
+
+/**
+ * Parameter object for slider parameters
+ * @typedef {Object} SliderParameter
+ * @property {number} value - value property
+ * @property {number} min - min property
+ * @property {number} max - max property
+ * @property {number} step - step property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
+ */
+
+/**
+ * Parameter object for color parameters
+ * @typedef {Object} ColorParameter
+ * @property {string} value - value property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
  */
 
 /**
- * Artist render function - called every frame
- * @param {VijiAPI} viji - The viji API object with all capabilities
+ * Parameter object for toggle parameters
+ * @typedef {Object} ToggleParameter
+ * @property {boolean} value - value property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
  */
-declare function render(viji): void;
 
 /**
- * Artist setup function - called once at initialization
- * @param {VijiAPI} viji - The viji API object with all capabilities
+ * Parameter object for select parameters
+ * @typedef {Object} SelectParameter
+ * @property {string | number} value - value property
+ * @property {string[] | number[]} options - options property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
  */
-declare function setup(viji): void;
+
+/**
+ * Parameter object for text parameters
+ * @typedef {Object} TextParameter
+ * @property {string} value - value property
+ * @property {number} [maxLength] - maxLength property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
+ */
+
+/**
+ * Parameter object for number parameters
+ * @typedef {Object} NumberParameter
+ * @property {number} value - value property
+ * @property {number} min - min property
+ * @property {number} max - max property
+ * @property {number} step - step property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
+ */
+
+/**
+ * Parameter object for image parameters
+ * @typedef {Object} ImageParameter
+ * @property {ImageBitmap |null} value - value property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
+ */
+
+/**
+ * Parameter object for button parameters
+ * @typedef {Object} ButtonParameter
+ * @property {boolean} value - value property
+ * @property {string} label - label property
+ * @property {string} [description] - description property
+ * @property {string} group - group property
+ * @property {ParameterCategory} category - category property
+ */
+
+/**
+ * Audio analysis API - provides real-time audio data and frequency analysis
+ * @typedef {Object} AudioAPI
+ * @property {boolean} isConnected - isConnected property
+ * @property {Object} volume - volume property
+ * @property {number} volume.current - current property
+ * @property {number} volume.peak - peak property
+ * @property {number} volume.smoothed - smoothed property
+ */
+
+/**
+ * Video frame API - provides access to video stream and computer vision data
+ * @typedef {Object} VideoAPI
+ * @property {boolean} isConnected - isConnected property
+ * @property {OffscreenCanvas | ImageBitmap |null} currentFrame - currentFrame property
+ * @property {number} frameWidth - frameWidth property
+ * @property {number} frameHeight - frameHeight property
+ * @property {number} frameRate - frameRate property
+ * @property {() => ImageData |null} getFrameData - getFrameData property
+ * @property {FaceData[]} faces - faces property
+ * @property {HandData[]} hands - hands property
+ * @property {PoseData |null} pose - pose property
+ * @property {SegmentationData |null} segmentation - segmentation property
+ * @property {Object} cv - cv property
+ */
+
+/**
+ * Mouse interaction API
+ * @typedef {Object} MouseAPI
+ * @property {number} x - x property
+ * @property {number} y - y property
+ * @property {boolean} isInCanvas - isInCanvas property
+ * @property {boolean} isPressed - isPressed property
+ * @property {boolean} leftButton - leftButton property
+ * @property {boolean} rightButton - rightButton property
+ * @property {boolean} middleButton - middleButton property
+ * @property {number} deltaX - deltaX property
+ * @property {number} deltaY - deltaY property
+ * @property {number} wheelDelta - wheelDelta property
+ * @property {number} wheelX - wheelX property
+ * @property {number} wheelY - wheelY property
+ * @property {boolean} wasPressed - wasPressed property
+ * @property {boolean} wasReleased - wasReleased property
+ * @property {boolean} wasMoved - wasMoved property
+ */
+
+/**
+ * Keyboard interaction API
+ * @typedef {Object} KeyboardAPI
+ * @property {Set<string>} activeKeys - activeKeys property
+ * @property {Set<string>} pressedThisFrame - pressedThisFrame property
+ * @property {Set<string>} releasedThisFrame - releasedThisFrame property
+ * @property {string} lastKeyPressed - lastKeyPressed property
+ * @property {string} lastKeyReleased - lastKeyReleased property
+ * @property {boolean} shift - shift property
+ * @property {boolean} ctrl - ctrl property
+ * @property {boolean} alt - alt property
+ * @property {boolean} meta - meta property
+ * @property {Uint8Array} textureData - textureData property
+ */
+
+/**
+ * Touch interaction API
+ * @typedef {Object} TouchAPI
+ * @property {TouchPoint[]} points - points property
+ * @property {number} count - count property
+ * @property {TouchPoint[]} started - started property
+ * @property {TouchPoint[]} moved - moved property
+ * @property {TouchPoint[]} ended - ended property
+ * @property {TouchPoint |null} primary - primary property
+ */
+
+/**
+ * Main Viji Artist API - provides access to canvas, timing, audio, video, and interactions
+ * @typedef {Object} VijiAPI
+ * @property {OffscreenCanvas} canvas - canvas property
+ * @property {OffscreenCanvasRenderingContext2D} [ctx] - ctx property
+ * @property {WebGLRenderingContext | WebGL2RenderingContext} [gl] - gl property
+ * @property {number} width - width property
+ * @property {number} height - height property
+ * @property {number} time - time property
+ * @property {number} deltaTime - deltaTime property
+ * @property {number} frameCount - frameCount property
+ * @property {number} fps - fps property
+ * @property {AudioAPI} audio - audio property
+ * @property {VideoAPI} video - video property
+ * @property {VideoAPI[]} streams - streams property
+ * @property {MouseAPI} mouse - mouse property
+ * @property {KeyboardAPI} keyboard - keyboard property
+ * @property {TouchAPI} touches - touches property
+ * @property {PointerAPI} pointer - pointer property
+ * @property {DeviceSensorState} device - device property
+ * @property {DeviceState[]} devices - devices property
+ * @property {(defaultValue: number, config: SliderConfig) => SliderParameter} slider - slider property
+ * @property {(defaultValue: string, config: ColorConfig) => ColorParameter} color - color property
+ * @property {(defaultValue: boolean, config: ToggleConfig) => ToggleParameter} toggle - toggle property
+ * @property {(defaultValue: string | number, config: SelectConfig) => SelectParameter} select - select property
+ * @property {(defaultValue: string, config: TextConfig) => TextParameter} text - text property
+ * @property {(defaultValue: number, config: NumberConfig) => NumberParameter} number - number property
+ * @property {(defaultValue: null, config: ImageConfig) => ImageParameter} image - image property
+ * @property {(config: ButtonConfig) => ButtonParameter} button - button property
+ */
+
+/**
+ * FaceData interface
+ * @typedef {Object} FaceData
+ * @property {number} id - id property
+ * @property {Object} bounds - bounds property
+ * @property {number} bounds.x - x property
+ * @property {number} bounds.y - y property
+ * @property {number} bounds.width - width property
+ * @property {number} bounds.height - height property
+ */
+
+/**
+ * FaceBlendshapes interface
+ * @typedef {Object} FaceBlendshapes
+ * @property {number} browDownLeft - browDownLeft property
+ * @property {number} browDownRight - browDownRight property
+ * @property {number} browInnerUp - browInnerUp property
+ * @property {number} browOuterUpLeft - browOuterUpLeft property
+ * @property {number} browOuterUpRight - browOuterUpRight property
+ * @property {number} cheekPuff - cheekPuff property
+ * @property {number} cheekSquintLeft - cheekSquintLeft property
+ * @property {number} cheekSquintRight - cheekSquintRight property
+ * @property {number} eyeBlinkLeft - eyeBlinkLeft property
+ * @property {number} eyeBlinkRight - eyeBlinkRight property
+ * @property {number} eyeLookDownLeft - eyeLookDownLeft property
+ * @property {number} eyeLookDownRight - eyeLookDownRight property
+ * @property {number} eyeLookInLeft - eyeLookInLeft property
+ * @property {number} eyeLookInRight - eyeLookInRight property
+ * @property {number} eyeLookOutLeft - eyeLookOutLeft property
+ * @property {number} eyeLookOutRight - eyeLookOutRight property
+ * @property {number} eyeLookUpLeft - eyeLookUpLeft property
+ * @property {number} eyeLookUpRight - eyeLookUpRight property
+ * @property {number} eyeSquintLeft - eyeSquintLeft property
+ * @property {number} eyeSquintRight - eyeSquintRight property
+ * @property {number} eyeWideLeft - eyeWideLeft property
+ * @property {number} eyeWideRight - eyeWideRight property
+ * @property {number} jawForward - jawForward property
+ * @property {number} jawLeft - jawLeft property
+ * @property {number} jawOpen - jawOpen property
+ * @property {number} jawRight - jawRight property
+ * @property {number} mouthClose - mouthClose property
+ * @property {number} mouthDimpleLeft - mouthDimpleLeft property
+ * @property {number} mouthDimpleRight - mouthDimpleRight property
+ * @property {number} mouthFrownLeft - mouthFrownLeft property
+ * @property {number} mouthFrownRight - mouthFrownRight property
+ * @property {number} mouthFunnel - mouthFunnel property
+ * @property {number} mouthLeft - mouthLeft property
+ * @property {number} mouthLowerDownLeft - mouthLowerDownLeft property
+ * @property {number} mouthLowerDownRight - mouthLowerDownRight property
+ * @property {number} mouthPressLeft - mouthPressLeft property
+ * @property {number} mouthPressRight - mouthPressRight property
+ * @property {number} mouthPucker - mouthPucker property
+ * @property {number} mouthRight - mouthRight property
+ * @property {number} mouthRollLower - mouthRollLower property
+ * @property {number} mouthRollUpper - mouthRollUpper property
+ * @property {number} mouthShrugLower - mouthShrugLower property
+ * @property {number} mouthShrugUpper - mouthShrugUpper property
+ * @property {number} mouthSmileLeft - mouthSmileLeft property
+ * @property {number} mouthSmileRight - mouthSmileRight property
+ * @property {number} mouthStretchLeft - mouthStretchLeft property
+ * @property {number} mouthStretchRight - mouthStretchRight property
+ * @property {number} mouthUpperUpLeft - mouthUpperUpLeft property
+ * @property {number} mouthUpperUpRight - mouthUpperUpRight property
+ * @property {number} noseSneerLeft - noseSneerLeft property
+ * @property {number} noseSneerRight - noseSneerRight property
+ * @property {number} tongueOut - tongueOut property
+ */
+
+/**
+ * HandData interface
+ * @typedef {Object} HandData
+ * @property {number} id - id property
+ * @property {'left' | 'right'} handedness - handedness property
+ * @property {number} confidence - confidence property
+ * @property {Object} bounds - bounds property
+ * @property {number} bounds.x - x property
+ * @property {number} bounds.y - y property
+ * @property {number} bounds.width - width property
+ * @property {number} bounds.height - height property
+ */
+
+/**
+ * PoseData interface
+ * @typedef {Object} PoseData
+ * @property {number} confidence - confidence property
+ * @property {Object} landmarks - landmarks property
+ * @property {number} landmarks.x - x property
+ * @property {number} landmarks.y - y property
+ * @property {number} landmarks.z - z property
+ * @property {number} landmarks.visibility - visibility property
+ */
+
+/**
+ * SegmentationData interface
+ * @typedef {Object} SegmentationData
+ * @property {Uint8Array} mask - mask property
+ * @property {number} width - width property
+ * @property {number} height - height property
+ */
+
+/**
+ * DeviceMotionData interface
+ * @typedef {Object} DeviceMotionData
+ * @property {Object} acceleration - Acceleration without gravity (m/s²)
+ * @property {number |null} acceleration.x - x property
+ * @property {number |null} acceleration.y - y property
+ * @property {number |null} acceleration.z - z property
+ */
+
+/**
+ * DeviceOrientationData interface
+ * @typedef {Object} DeviceOrientationData
+ * @property {number |null} alpha - Rotation around Z-axis (0-360 degrees, compass heading)
+ * @property {number |null} beta - Rotation around X-axis (-180 to 180 degrees, front-to-back tilt)
+ * @property {number |null} gamma - Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt)
+ * @property {boolean} absolute - True if using magnetometer (compass) for absolute orientation
+ */
+
+/**
+ * DeviceSensorState interface
+ * @typedef {Object} DeviceSensorState
+ * @property {DeviceMotionData |null} motion - motion property
+ * @property {DeviceOrientationData |null} orientation - orientation property
+ */
+
+/**
+ * DeviceState interface
+ * @typedef {Object} DeviceState
+ * @property {string} id - Unique device identifier
+ * @property {string} name - User-friendly device name
+ * @property {VideoAPI |null} video - Device camera video (null if not available)
+ */
+
+/**
+ * @typedef {function(VijiAPI): void} Render
+ */
+
+/**
+ * @typedef {function(VijiAPI): void} Setup
+ */
+
+/** Global viji object available in artist code */
+declare const viji: VijiAPI;
+
+/** Artist render function - called every frame */
+declare function render(viji: VijiAPI): void;
+
+/** Artist setup function - called once at initialization */
+declare function setup(viji: VijiAPI): void;

package/dist/assets/cv-tasks.worker.js

@@ -29,7 +29,7 @@ try {
 // MediaPipe model instances
 let faceDetector = null;
 let faceLandmarker = null;
-let handLandmarker = null;
+let gestureRecognizer = null;
 let poseLandmarker = null;
 let imageSegmenter = null;
 
@@ -375,31 +375,31 @@ async function initializeFaceLandmarks() {
 }
 
 /**
- * Load and initialize Hand Tracking
+ * Load and initialize Hand Tracking via GestureRecognizer
+ * (provides landmarks + handedness + ML-based gesture classification in one call)
  */
 async function initializeHandTracking() {
-  if (handLandmarker) return;
+  if (gestureRecognizer) return;
   
-  // Ensure vision runtime is initialized first
   await initializeVision();
   
   try {
-    log('Loading Hand Landmarker...');
+    log('Loading Gesture Recognizer...');
     
     const options = {
       baseOptions: {
-        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
+        modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task',
         delegate: 'GPU'
       },
       runningMode: 'VIDEO',
       numHands: 2
     };
     
-    const HandLandmarker = self.HandLandmarker || self.module.exports.HandLandmarker || self.exports.HandLandmarker;
-    handLandmarker = await HandLandmarker.createFromOptions(vision, options);
-    log('✅ Hand Landmarker loaded');
+    const GestureRecognizer = self.GestureRecognizer || self.module.exports.GestureRecognizer || self.exports.GestureRecognizer;
+    gestureRecognizer = await GestureRecognizer.createFromOptions(vision, options);
+    log('✅ Gesture Recognizer loaded');
   } catch (error) {
-    log('❌ Failed to load Hand Landmarker:', error);
+    log('❌ Failed to load Gesture Recognizer:', error);
     throw error;
   }
 }
@@ -465,6 +465,59 @@ async function initializeBodySegmentation() {
   }
 }
 
+function computeHandBounds(landmarks) {
+  let minX = 1, minY = 1, maxX = 0, maxY = 0;
+  for (const p of landmarks) {
+    if (p.x < minX) minX = p.x;
+    if (p.y < minY) minY = p.y;
+    if (p.x > maxX) maxX = p.x;
+    if (p.y > maxY) maxY = p.y;
+  }
+  return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
+}
+
+// Map MediaPipe GestureRecognizer category names to our API field names
+const GESTURE_NAME_MAP = {
+  'Closed_Fist': 'fist',
+  'Open_Palm': 'openPalm',
+  'Victory': 'peace',
+  'Thumb_Up': 'thumbsUp',
+  'Pointing_Up': 'pointing',
+  'Thumb_Down': 'thumbsDown',
+  'ILoveYou': 'iLoveYou'
+};
+
+function mapGestures(gestureCategories) {
+  const result = { fist: 0, openPalm: 0, peace: 0, thumbsUp: 0, pointing: 0, thumbsDown: 0, iLoveYou: 0 };
+  if (!gestureCategories) return result;
+  for (const cat of gestureCategories) {
+    const key = GESTURE_NAME_MAP[cat.categoryName];
+    if (key) result[key] = cat.score;
+  }
+  return result;
+}
+
+// BlazePose landmark index groups for named body parts
+const POSE_FACE_INDICES = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
+const POSE_TORSO_INDICES = [11, 12, 23, 24];
+const POSE_LEFT_ARM_INDICES = [11, 13, 15];
+const POSE_RIGHT_ARM_INDICES = [12, 14, 16];
+const POSE_LEFT_LEG_INDICES = [23, 25, 27, 29, 31];
+const POSE_RIGHT_LEG_INDICES = [24, 26, 28, 30, 32];
+
+function extractPoseGroup(landmarks, indices) {
+  return indices.map(i => {
+    const lm = landmarks[i];
+    return lm ? { x: lm.x, y: lm.y } : { x: 0, y: 0 };
+  });
+}
+
+function computePoseConfidence(landmarks) {
+  let sum = 0;
+  for (const lm of landmarks) sum += (lm.visibility || 0);
+  return landmarks.length > 0 ? sum / landmarks.length : 0;
+}
+
 /**
  * Process video frame with active CV features
  * @param {ImageData|ImageBitmap} imageInput - Image input (ImageData or ImageBitmap)
@@ -478,7 +531,8 @@ async function processFrame(imageInput, timestamp, features) {
     // Process face detection
     if (features.includes('faceDetection') && faceDetector) {
       const detectionResult = faceDetector.detectForVideo(imageInput, timestamp);
-      results.faces = detectionResult.detections.map((detection) => ({
+      results.faces = detectionResult.detections.map((detection, index) => ({
+        id: index,
         bounds: {
           x: detection.boundingBox.originX / imageInput.width,
           y: detection.boundingBox.originY / imageInput.height,
@@ -505,6 +559,7 @@ async function processFrame(imageInput, timestamp, features) {
         
         if (!results.faces) {
           results.faces = [{
+            id: 0,
             bounds: null,
             center: null,
             landmarks: [],
@@ -525,7 +580,6 @@ async function processFrame(imageInput, timestamp, features) {
           results.faces[0].landmarks = mappedLandmarks;
           results.faces[0].headPose = computeHeadPoseFromLandmarks(landmarks);
 
-          // Populate emotion data when emotionDetection is active
           if (features.includes('emotionDetection') &&
               landmarkResult.faceBlendshapes &&
               landmarkResult.faceBlendshapes.length > 0) {
@@ -534,34 +588,55 @@ async function processFrame(imageInput, timestamp, features) {
             results.faces[0].expressions = mapBlendshapesToEmotions(bs);
           }
         }
+      } else if (!results.faces) {
+        results.faces = [];
       }
     }
     
-    // Process hand tracking
-    if (features.includes('handTracking') && handLandmarker) {
-      const handResult = handLandmarker.detectForVideo(imageInput, timestamp);
-      results.hands = handResult.landmarks.map((landmarks, index) => ({
-        landmarks: landmarks.map((landmark) => ({
+    // Process hand tracking (GestureRecognizer provides landmarks + gestures in one call)
+    if (features.includes('handTracking') && gestureRecognizer) {
+      const handResult = gestureRecognizer.recognizeForVideo(imageInput, timestamp);
+      results.hands = handResult.landmarks.map((landmarks, index) => {
+        const mapped = landmarks.map((landmark) => ({
           x: landmark.x,
           y: landmark.y,
           z: landmark.z || 0
-        })),
-        handedness: handResult.handednesses[index]?.[0]?.categoryName || 'Unknown',
-        confidence: handResult.handednesses[index]?.[0]?.score || 0
-      }));
+        }));
+
+        const rawHandedness = handResult.handednesses[index]?.[0]?.categoryName || 'unknown';
+
+        return {
+          id: index,
+          handedness: rawHandedness.toLowerCase(),
+          confidence: handResult.handednesses[index]?.[0]?.score || 0,
+          bounds: computeHandBounds(mapped),
+          landmarks: mapped,
+          palm: mapped[9],
+          gestures: mapGestures(handResult.gestures[index])
+        };
+      });
     }
     
     // Process pose detection
     if (features.includes('poseDetection') && poseLandmarker) {
       const poseResult = poseLandmarker.detectForVideo(imageInput, timestamp);
       if (poseResult.landmarks.length > 0) {
+        const poseLandmarks = poseResult.landmarks[0].map((landmark) => ({
+          x: landmark.x,
+          y: landmark.y,
+          z: landmark.z || 0,
+          visibility: landmark.visibility || 1
+        }));
+
         results.pose = {
-          landmarks: poseResult.landmarks[0].map((landmark) => ({
-            x: landmark.x,
-            y: landmark.y,
-            z: landmark.z || 0,
-            visibility: landmark.visibility || 1
-          })),
+          confidence: computePoseConfidence(poseLandmarks),
+          landmarks: poseLandmarks,
+          face: extractPoseGroup(poseLandmarks, POSE_FACE_INDICES),
+          torso: extractPoseGroup(poseLandmarks, POSE_TORSO_INDICES),
+          leftArm: extractPoseGroup(poseLandmarks, POSE_LEFT_ARM_INDICES),
+          rightArm: extractPoseGroup(poseLandmarks, POSE_RIGHT_ARM_INDICES),
+          leftLeg: extractPoseGroup(poseLandmarks, POSE_LEFT_LEG_INDICES),
+          rightLeg: extractPoseGroup(poseLandmarks, POSE_RIGHT_LEG_INDICES),
           worldLandmarks: poseResult.worldLandmarks?.[0]?.map((landmark) => ({
             x: landmark.x,
             y: landmark.y,
@@ -569,6 +644,8 @@ async function processFrame(imageInput, timestamp, features) {
             visibility: landmark.visibility || 1
           })) || []
         };
+      } else {
+        results.pose = null;
       }
     }
     
@@ -577,18 +654,17 @@ async function processFrame(imageInput, timestamp, features) {
       const segmentResult = imageSegmenter.segment(imageInput);
       if (segmentResult.categoryMask) {
         try {
-          // Extract data before closing the mask
           results.segmentation = {
             mask: segmentResult.categoryMask.getAsUint8Array(),
             width: segmentResult.categoryMask.width,
             height: segmentResult.categoryMask.height
           };
-          
           log('Segmentation mask:', results.segmentation.width, 'x', results.segmentation.height);
         } finally {
-          // CRITICAL: Close MPMask instance to prevent resource leaks
           segmentResult.categoryMask.close();
         }
+      } else {
+        results.segmentation = null;
       }
     }
     
@@ -697,8 +773,8 @@ async function handleConfigUpdateInternal(features) {
         }
         break;
       case 'handTracking':
-        cleanupPromises.push(cleanupWasmInstance(handLandmarker, 'HandLandmarker'));
-        handLandmarker = null;
+        cleanupPromises.push(cleanupWasmInstance(gestureRecognizer, 'GestureRecognizer'));
+        gestureRecognizer = null;
         break;
       case 'poseDetection':
         cleanupPromises.push(cleanupWasmInstance(poseLandmarker, 'PoseLandmarker'));