facecog-liveness-showcase 0.0.1

package/projects/facecog-liveness-verification/src/lib/services/pose-detection/openpose.service.ts

@@ -0,0 +1,287 @@
+import { Injectable } from '@angular/core';
+import * as poseDetection from '@tensorflow-models/pose-detection';
+import * as tf from '@tensorflow/tfjs-core';
+import '@tensorflow/tfjs-backend-webgl';
+
+export interface PoseKeypoints {
+  keypoints: poseDetection.Keypoint[];
+  score: number;
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class OpenposeService {
+  private detector: poseDetection.PoseDetector | null = null;
+  private isInitialized = false;
+
+  constructor() {}
+
+  /**
+   * Initialize MoveNet pose detector
+   */
+  async loadModel(): Promise<void> {
+    if (this.isInitialized) {
+      console.log('[OpenPose] Model already loaded');
+      return;
+    }
+
+    try {
+      console.log('[OpenPose] Loading MoveNet model...');
+
+      // Try WebGL first, fallback to WASM for Safari compatibility
+      try {
+        await tf.setBackend('webgl');
+        await tf.ready();
+        console.log('[OpenPose] Using WebGL backend');
+      } catch (webglError) {
+        console.warn('[OpenPose] WebGL backend failed, trying WASM for Safari compatibility:', webglError);
+        await tf.setBackend('wasm');
+        await tf.ready();
+        console.log('[OpenPose] Using WASM backend (Safari fallback)');
+      }
+
+      // Create MoveNet detector with Lightning model (faster, good for real-time)
+      const detectorConfig = {
+        modelType: poseDetection.movenet.modelType.SINGLEPOSE_LIGHTNING
+      };
+
+      this.detector = await poseDetection.createDetector(
+        poseDetection.SupportedModels.MoveNet,
+        detectorConfig
+      );
+
+      this.isInitialized = true;
+      console.log('[OpenPose] MoveNet model loaded successfully');
+    } catch (error) {
+      console.error('[OpenPose] Failed to load model:', error);
+      throw new Error('Failed to load pose detection model');
+    }
+  }
+
+  /**
+   * Detect pose in an image element
+   */
+  async detectPose(imageElement: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement): Promise<PoseKeypoints | null> {
+    if (!this.isInitialized || !this.detector) {
+      console.warn('[OpenPose] Detector not initialized, attempting to load...');
+      await this.loadModel();
+    }
+
+    try {
+      const poses = await this.detector!.estimatePoses(imageElement);
+
+      if (poses.length === 0) {
+        // Don't log every frame to reduce console spam
+        return null;
+      }
+
+      // Return the first detected pose
+      const pose = poses[0];
+
+      // Log keypoint details for debugging
+      const rightWrist = pose.keypoints[10]; // Index 10 is right_wrist in MoveNet
+      const rightShoulder = pose.keypoints[6]; // Index 6 is right_shoulder
+
+      console.log('[OpenPose] Pose detected - Score:', pose.score?.toFixed(3),
+                  'Right wrist:', rightWrist ? `(${rightWrist.x?.toFixed(3)}, ${rightWrist.y?.toFixed(3)}, score: ${rightWrist.score?.toFixed(3)})` : 'not detected',
+                  'Right shoulder:', rightShoulder ? `(${rightShoulder.x?.toFixed(3)}, ${rightShoulder.y?.toFixed(3)}, score: ${rightShoulder.score?.toFixed(3)})` : 'not detected');
+
+      return {
+        keypoints: pose.keypoints,
+        score: pose.score || 0
+      };
+    } catch (error) {
+      console.error('[OpenPose] Pose detection error:', error);
+      return null;
+    }
+  }
+
+  /**
+   * Detect pose from base64 image string
+   */
+  async detectPoseFromBase64(base64Image: string): Promise<PoseKeypoints | null> {
+    return new Promise((resolve, reject) => {
+      const img = new Image();
+
+      img.onload = async () => {
+        try {
+          const pose = await this.detectPose(img);
+          resolve(pose);
+        } catch (error) {
+          reject(error);
+        }
+      };
+
+      img.onerror = () => {
+        reject(new Error('Failed to load image from base64'));
+      };
+
+      img.src = base64Image;
+    });
+  }
+
+  /**
+   * Get specific keypoint by name
+   */
+  getKeypoint(pose: PoseKeypoints, keypointName: string): poseDetection.Keypoint | null {
+    const keypoint = pose.keypoints.find(kp => kp.name === keypointName);
+    return keypoint || null;
+  }
+
+  /**
+   * Check if a keypoint is visible (has good confidence)
+   */
+  isKeypointVisible(keypoint: poseDetection.Keypoint | null, threshold: number = 0.3): boolean {
+    if (!keypoint) return false;
+    return (keypoint.score || 0) >= threshold;
+  }
+
+  /**
+   * Get hand keypoints (wrists)
+   */
+  getHandKeypoints(pose: PoseKeypoints): {
+    leftWrist: poseDetection.Keypoint | null;
+    rightWrist: poseDetection.Keypoint | null;
+    leftWristVisible: boolean;
+    rightWristVisible: boolean;
+  } {
+    const leftWrist = this.getKeypoint(pose, 'left_wrist');
+    const rightWrist = this.getKeypoint(pose, 'right_wrist');
+
+    return {
+      leftWrist,
+      rightWrist,
+      leftWristVisible: this.isKeypointVisible(leftWrist),
+      rightWristVisible: this.isKeypointVisible(rightWrist)
+    };
+  }
+
+  /**
+   * Get face/head keypoints
+   */
+  getHeadKeypoints(pose: PoseKeypoints): {
+    nose: poseDetection.Keypoint | null;
+    leftEye: poseDetection.Keypoint | null;
+    rightEye: poseDetection.Keypoint | null;
+    leftEar: poseDetection.Keypoint | null;
+    rightEar: poseDetection.Keypoint | null;
+  } {
+    return {
+      nose: this.getKeypoint(pose, 'nose'),
+      leftEye: this.getKeypoint(pose, 'left_eye'),
+      rightEye: this.getKeypoint(pose, 'right_eye'),
+      leftEar: this.getKeypoint(pose, 'left_ear'),
+      rightEar: this.getKeypoint(pose, 'right_ear')
+    };
+  }
+
+  /**
+   * Get shoulder keypoints (useful for determining hand positions relative to body)
+   */
+  getShoulderKeypoints(pose: PoseKeypoints): {
+    leftShoulder: poseDetection.Keypoint | null;
+    rightShoulder: poseDetection.Keypoint | null;
+  } {
+    return {
+      leftShoulder: this.getKeypoint(pose, 'left_shoulder'),
+      rightShoulder: this.getKeypoint(pose, 'right_shoulder')
+    };
+  }
+
+  /**
+   * Get elbow keypoints (useful for determining arm bend/extension)
+   */
+  getElbowKeypoints(pose: PoseKeypoints): {
+    leftElbow: poseDetection.Keypoint | null;
+    rightElbow: poseDetection.Keypoint | null;
+  } {
+    return {
+      leftElbow: this.getKeypoint(pose, 'left_elbow'),
+      rightElbow: this.getKeypoint(pose, 'right_elbow')
+    };
+  }
+
+  /**
+   * Detect thumbs up gesture
+   * A thumbs up is detected when:
+   * 1. Wrist is raised above shoulder level
+   * 2. Wrist is roughly aligned with or above the elbow
+   * 3. Hand is visible with good confidence
+   */
+  detectThumbsUp(pose: PoseKeypoints): {
+    leftThumbsUp: boolean;
+    rightThumbsUp: boolean;
+    anyThumbsUp: boolean;
+    confidence: number;
+  } {
+    const hands = this.getHandKeypoints(pose);
+    const shoulders = this.getShoulderKeypoints(pose);
+    const elbows = this.getElbowKeypoints(pose);
+
+    let leftThumbsUp = false;
+    let rightThumbsUp = false;
+    let maxConfidence = 0;
+
+    // Check left hand thumbs up
+    if (hands.leftWristVisible &&
+        shoulders.leftShoulder &&
+        elbows.leftElbow &&
+        hands.leftWrist) {
+
+      const wristAboveShoulder = hands.leftWrist.y < shoulders.leftShoulder.y;
+      const wristAboveOrNearElbow = hands.leftWrist.y <= elbows.leftElbow.y + 50; // Allow some tolerance
+
+      // Wrist should be raised (above shoulder)
+      if (wristAboveShoulder && wristAboveOrNearElbow) {
+        leftThumbsUp = true;
+        maxConfidence = Math.max(maxConfidence, hands.leftWrist.score || 0);
+        console.log('[OpenPose] Left thumbs up detected!', {
+          wrist: hands.leftWrist.y,
+          shoulder: shoulders.leftShoulder.y,
+          elbow: elbows.leftElbow.y
+        });
+      }
+    }
+
+    // Check right hand thumbs up
+    if (hands.rightWristVisible &&
+        shoulders.rightShoulder &&
+        elbows.rightElbow &&
+        hands.rightWrist) {
+
+      const wristAboveShoulder = hands.rightWrist.y < shoulders.rightShoulder.y;
+      const wristAboveOrNearElbow = hands.rightWrist.y <= elbows.rightElbow.y + 50; // Allow some tolerance
+
+      // Wrist should be raised (above shoulder)
+      if (wristAboveShoulder && wristAboveOrNearElbow) {
+        rightThumbsUp = true;
+        maxConfidence = Math.max(maxConfidence, hands.rightWrist.score || 0);
+        console.log('[OpenPose] Right thumbs up detected!', {
+          wrist: hands.rightWrist.y,
+          shoulder: shoulders.rightShoulder.y,
+          elbow: elbows.rightElbow.y
+        });
+      }
+    }
+
+    return {
+      leftThumbsUp,
+      rightThumbsUp,
+      anyThumbsUp: leftThumbsUp || rightThumbsUp,
+      confidence: maxConfidence
+    };
+  }
+
+  /**
+   * Cleanup resources
+   */
+  cleanup(): void {
+    if (this.detector) {
+      this.detector.dispose();
+      this.detector = null;
+      this.isInitialized = false;
+      console.log('[OpenPose] Detector disposed');
+    }
+  }
+}

package/projects/facecog-liveness-verification/src/lib/services/pose-detection/pose-comparison.service.ts

@@ -0,0 +1,353 @@
+import { Injectable } from '@angular/core';
+import { OpenposeService, PoseKeypoints } from './openpose.service';
+import { ReferencePose } from './reference-pose.service';
+import * as poseDetection from '@tensorflow-models/pose-detection';
+
+export interface PoseComparisonResult {
+  overallScore: number;  // 0-1
+  handScore: number;     // Hand position similarity
+  bodyScore: number;     // Body pose similarity
+  matched: boolean;      // True if score > threshold
+  feedback: string;      // User feedback message
+  handsRequired: boolean; // True if reference pose has hand keypoints
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class PoseComparisonService {
+  private readonly MATCH_THRESHOLD = 0.75; // 75% similarity required
+
+  constructor(private openposeService: OpenposeService) {}
+
+  /**
+   * Compare live pose with reference pose
+   */
+  comparePoses(livePose: PoseKeypoints, referencePose: ReferencePose): PoseComparisonResult {
+    try {
+      console.log('[Pose Comparison] Comparing poses...');
+
+      // Check if reference has any hand keypoints (using lower threshold 0.15)
+      const refHands = this.getHandKeypointsWithLowerThreshold(referencePose.keypoints);
+      const handsRequired = refHands.leftWristVisible || refHands.rightWristVisible;
+
+      console.log('[Pose Comparison] Reference pose hands required:', handsRequired);
+
+      // Calculate individual scores
+      const handScore = this.calculateHandSimilarity(livePose, referencePose.keypoints);
+      const bodyScore = this.calculateBodySimilarity(livePose, referencePose.keypoints);
+
+      // Weighted overall score (hands are more important for gesture matching)
+      // If hands are required but not matched, penalize more heavily
+      let overallScore: number;
+      if (handsRequired) {
+        // Hands are 70% of score when required
+        overallScore = (handScore * 0.7) + (bodyScore * 0.3);
+      } else {
+        // Standard weighting
+        overallScore = (handScore * 0.6) + (bodyScore * 0.4);
+      }
+
+      const matched = overallScore >= this.MATCH_THRESHOLD;
+
+      // Generate feedback
+      const feedback = this.generateFeedback(overallScore, handScore, bodyScore, matched, handsRequired);
+
+      console.log(`[Pose Comparison] Hand: ${(handScore * 100).toFixed(1)}%, Body: ${(bodyScore * 100).toFixed(1)}%, Overall: ${(overallScore * 100).toFixed(1)}%, Matched: ${matched}, HandsRequired: ${handsRequired}`);
+
+      return {
+        overallScore,
+        handScore,
+        bodyScore,
+        matched,
+        feedback,
+        handsRequired
+      };
+    } catch (error) {
+      console.error('[Pose Comparison] Error comparing poses:', error);
+      return {
+        overallScore: 0,
+        handScore: 0,
+        bodyScore: 0,
+        matched: false,
+        feedback: 'Unable to compare poses',
+        handsRequired: false
+      };
+    }
+  }
+
+  /**
+   * Get hand keypoints with lower threshold (0.15 instead of 0.3)
+   */
+  private getHandKeypointsWithLowerThreshold(pose: PoseKeypoints): {
+    leftWrist: any | null;
+    rightWrist: any | null;
+    leftWristVisible: boolean;
+    rightWristVisible: boolean;
+  } {
+    const leftWrist = this.openposeService.getKeypoint(pose, 'left_wrist');
+    const rightWrist = this.openposeService.getKeypoint(pose, 'right_wrist');
+    const lowThreshold = 0.15;
+
+    return {
+      leftWrist,
+      rightWrist,
+      leftWristVisible: this.openposeService.isKeypointVisible(leftWrist, lowThreshold),
+      rightWristVisible: this.openposeService.isKeypointVisible(rightWrist, lowThreshold)
+    };
+  }
+
+  /**
+   * Calculate hand position similarity
+   */
+  private calculateHandSimilarity(livePose: PoseKeypoints, referencePose: PoseKeypoints): number {
+    // Use lower threshold (0.15) for detecting hands in both live and reference
+    const liveHands = this.getHandKeypointsWithLowerThreshold(livePose);
+    const refHands = this.getHandKeypointsWithLowerThreshold(referencePose);
+    const liveShoulders = this.openposeService.getShoulderKeypoints(livePose);
+    const refShoulders = this.openposeService.getShoulderKeypoints(referencePose);
+    const liveHead = this.openposeService.getHeadKeypoints(livePose);
+    const refHead = this.openposeService.getHeadKeypoints(referencePose);
+
+    let scores: number[] = [];
+    let leftHandScore: number | null = null;
+    let rightHandScore: number | null = null;
+
+    // Compare left hand position (relative to body center)
+    if (refHands.leftWristVisible) {
+      if (liveHands.leftWristVisible && liveHead.nose && refHead.nose &&
+          liveShoulders.leftShoulder && refShoulders.leftShoulder) {
+
+        const liveLeftHandPos = this.normalizeKeypointPosition(
+          liveHands.leftWrist!,
+          liveHead.nose,
+          liveShoulders.leftShoulder,
+          liveShoulders.rightShoulder
+        );
+
+        const refLeftHandPos = this.normalizeKeypointPosition(
+          refHands.leftWrist!,
+          refHead.nose,
+          refShoulders.leftShoulder,
+          refShoulders.rightShoulder
+        );
+
+        leftHandScore = this.calculatePositionSimilarity(liveLeftHandPos, refLeftHandPos);
+        scores.push(leftHandScore);
+      } else {
+        // Reference has left hand but live doesn't - penalize heavily
+        leftHandScore = 0.0;
+        scores.push(leftHandScore);
+      }
+    }
+
+    // Compare right hand position (relative to body center)
+    if (refHands.rightWristVisible) {
+      if (liveHands.rightWristVisible && liveHead.nose && refHead.nose &&
+          liveShoulders.rightShoulder && refShoulders.rightShoulder) {
+
+        const liveRightHandPos = this.normalizeKeypointPosition(
+          liveHands.rightWrist!,
+          liveHead.nose,
+          liveShoulders.leftShoulder,
+          liveShoulders.rightShoulder
+        );
+
+        const refRightHandPos = this.normalizeKeypointPosition(
+          refHands.rightWrist!,
+          refHead.nose,
+          refShoulders.leftShoulder,
+          refShoulders.rightShoulder
+        );
+
+        rightHandScore = this.calculatePositionSimilarity(liveRightHandPos, refRightHandPos);
+        scores.push(rightHandScore);
+      } else {
+        // Reference has right hand but live doesn't - penalize heavily
+        rightHandScore = 0.0;
+        scores.push(rightHandScore);
+      }
+    }
+
+    // If no hands visible in reference (even with lower 0.15 threshold), return full score
+    // This means the reference pose genuinely doesn't require hand matching
+    if (!refHands.leftWristVisible && !refHands.rightWristVisible) {
+      console.log('[Pose Comparison] No hands detected in reference pose (threshold 0.15) - hand matching not required');
+      return 1.0;
+    }
+
+    // If no scores calculated (shouldn't happen), return 0
+    if (scores.length === 0) {
+      return 0.0;
+    }
+
+    // IMPORTANT: Both hands must match if both are in reference
+    // If reference has both hands, BOTH must be visible and match in live pose
+    const bothHandsInReference = refHands.leftWristVisible && refHands.rightWristVisible;
+    const bothHandsInLive = liveHands.leftWristVisible && liveHands.rightWristVisible;
+
+    if (bothHandsInReference && !bothHandsInLive) {
+      console.log('[Pose Comparison] Reference has BOTH hands, but live pose only has ONE hand - FAIL');
+      return 0.0; // Hard fail if reference has both hands but live doesn't
+    }
+
+    // Average hand scores (all required hands must match)
+    const avgScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+
+    console.log(`[Pose Comparison] Hand scores - Left: ${leftHandScore?.toFixed(2) ?? 'N/A'}, Right: ${rightHandScore?.toFixed(2) ?? 'N/A'}, Avg: ${avgScore.toFixed(2)}`);
+
+    return avgScore;
+  }
+
+  /**
+   * Calculate body pose similarity (shoulders, head alignment)
+   */
+  private calculateBodySimilarity(livePose: PoseKeypoints, referencePose: PoseKeypoints): number {
+    const liveShoulders = this.openposeService.getShoulderKeypoints(livePose);
+    const refShoulders = this.openposeService.getShoulderKeypoints(referencePose);
+    const liveHead = this.openposeService.getHeadKeypoints(livePose);
+    const refHead = this.openposeService.getHeadKeypoints(referencePose);
+
+    let scores: number[] = [];
+
+    // Compare shoulder alignment (angle)
+    if (liveShoulders.leftShoulder && liveShoulders.rightShoulder &&
+        refShoulders.leftShoulder && refShoulders.rightShoulder) {
+
+      const liveShoulderAngle = this.calculateAngle(
+        liveShoulders.leftShoulder,
+        liveShoulders.rightShoulder
+      );
+
+      const refShoulderAngle = this.calculateAngle(
+        refShoulders.leftShoulder,
+        refShoulders.rightShoulder
+      );
+
+      const angleDiff = Math.abs(liveShoulderAngle - refShoulderAngle);
+      const angleSim = Math.max(0, 1 - (angleDiff / 45)); // 45 degree tolerance
+      scores.push(angleSim);
+    }
+
+    // Compare head position (relative to shoulders)
+    if (liveHead.nose && refHead.nose &&
+        liveShoulders.leftShoulder && liveShoulders.rightShoulder &&
+        refShoulders.leftShoulder && refShoulders.rightShoulder) {
+
+      const liveHeadPos = this.normalizeKeypointPosition(
+        liveHead.nose,
+        liveHead.nose,
+        liveShoulders.leftShoulder,
+        liveShoulders.rightShoulder
+      );
+
+      const refHeadPos = this.normalizeKeypointPosition(
+        refHead.nose,
+        refHead.nose,
+        refShoulders.leftShoulder,
+        refShoulders.rightShoulder
+      );
+
+      const headSim = this.calculatePositionSimilarity(liveHeadPos, refHeadPos);
+      scores.push(headSim);
+    }
+
+    // If no body keypoints visible, return 0
+    if (scores.length === 0) {
+      return 0.0;
+    }
+
+    // Average body scores
+    return scores.reduce((a, b) => a + b, 0) / scores.length;
+  }
+
+  /**
+   * Normalize keypoint position relative to body center and scale
+   */
+  private normalizeKeypointPosition(
+    keypoint: poseDetection.Keypoint,
+    referencePoint: poseDetection.Keypoint, // Usually nose/head center
+    leftShoulder: poseDetection.Keypoint | null,
+    rightShoulder: poseDetection.Keypoint | null
+  ): { x: number; y: number } {
+    // Calculate body center
+    let centerX = referencePoint.x;
+    let centerY = referencePoint.y;
+
+    // Calculate body scale (shoulder width)
+    let scale = 1.0;
+    if (leftShoulder && rightShoulder) {
+      const shoulderWidth = Math.abs(rightShoulder.x - leftShoulder.x);
+      scale = shoulderWidth || 1.0;
+
+      // Update center to midpoint between shoulders
+      centerX = (leftShoulder.x + rightShoulder.x) / 2;
+    }
+
+    // Normalize position relative to center and scale
+    return {
+      x: (keypoint.x - centerX) / scale,
+      y: (keypoint.y - centerY) / scale
+    };
+  }
+
+  /**
+   * Calculate similarity between two normalized positions
+   */
+  private calculatePositionSimilarity(pos1: { x: number; y: number }, pos2: { x: number; y: number }): number {
+    const dx = pos1.x - pos2.x;
+    const dy = pos1.y - pos2.y;
+    const distance = Math.sqrt(dx * dx + dy * dy);
+
+    // Distance threshold: 0.3 (30% of body scale)
+    const threshold = 0.3;
+    const similarity = Math.max(0, 1 - (distance / threshold));
+
+    return similarity;
+  }
+
+  /**
+   * Calculate angle between two points (in degrees)
+   */
+  private calculateAngle(point1: poseDetection.Keypoint, point2: poseDetection.Keypoint): number {
+    const dx = point2.x - point1.x;
+    const dy = point2.y - point1.y;
+    const angleRad = Math.atan2(dy, dx);
+    const angleDeg = (angleRad * 180) / Math.PI;
+    return angleDeg;
+  }
+
+  /**
+   * Generate user feedback based on scores
+   */
+  private generateFeedback(
+    overallScore: number,
+    handScore: number,
+    bodyScore: number,
+    matched: boolean,
+    handsRequired: boolean = false
+  ): string {
+    if (matched) {
+      return `Perfect match! ${Math.round(overallScore * 100)}%`;
+    }
+
+    if (overallScore >= 0.6) {
+      return `Almost there! ${Math.round(overallScore * 100)}% - Keep holding the pose`;
+    }
+
+    // If hands are required but not visible/matched
+    if (handsRequired && handScore < 0.3) {
+      return 'Show your hand in the frame - match the arm position in the reference';
+    }
+
+    if (handScore < 0.5) {
+      return 'Adjust your hand/arm position to match the reference';
+    }
+
+    if (bodyScore < 0.5) {
+      return 'Adjust your body alignment to match the reference';
+    }
+
+    return `${Math.round(overallScore * 100)}% - Try to match the reference pose`;
+  }
+}