facecog-liveness-showcase 0.0.1

package/projects/facecog-liveness-verification/src/lib/services/liveness-orchestrator.service.ts

@@ -0,0 +1,162 @@
+import { Injectable, Inject, Optional } from '@angular/core';
+import { Observable } from 'rxjs';
+import { LivenessConfigService } from './liveness-config.service';
+import { MockBackendService } from './backends/mock-backend.service';
+import { RekognitionAnalysisBackendService } from './backends/rekognition-analysis-backend.service';
+import { OpenPoseBackendService } from './backends/openpose-backend.service';
+import { FaceTecBackendService } from './backends/facetec-backend.service';
+import { AzureBackendService } from './backends/azure-backend.service';
+import { IBackendAdapter } from '../interfaces/backend-adapter.interface';
+import { IBackendHttpClient, BACKEND_HTTP_CLIENT } from '../interfaces/backend-http-client.interface';
+import { BackendResponse } from '../interfaces/backend-response.interface';
+import { MultiBackendAnalyzeResponse } from '../interfaces/multi-backend-analyze-response.interface';
+import { LivenessBackend } from '../types/liveness-backend.type';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class LivenessOrchestratorService {
+  private backends: Map<LivenessBackend, IBackendAdapter> = new Map();
+  public multiBackendResult: MultiBackendAnalyzeResponse | null = null;
+
+  constructor(
+    private configService: LivenessConfigService,
+    private mockBackend: MockBackendService,
+    private rekognitionBackend: RekognitionAnalysisBackendService,
+    private openposeBackend: OpenPoseBackendService,
+    private facetecBackend: FaceTecBackendService,
+    private azureBackend: AzureBackendService,
+    @Optional() @Inject(BACKEND_HTTP_CLIENT) private backendClient: IBackendHttpClient | null
+  ) {
+    this.initializeBackends();
+  }
+
+  private initializeBackends(): void {
+    this.backends.set('mock', this.mockBackend);
+    this.backends.set('amazon', this.rekognitionBackend);
+    this.backends.set('openpose', this.openposeBackend);
+    this.backends.set('facetec', this.facetecBackend);
+    this.backends.set('azure', this.azureBackend);
+  }
+
+  /**
+   * Verify liveness using multi-backend parallel analysis
+   * Uses configured backends from config (defaults to all if not specified)
+   */
+  verifyLiveness(imageData: string, videoBlob?: Blob): Observable<BackendResponse> {
+    const config = this.configService.getConfig();
+    const configuredBackends = config.backends || ['amazon', 'openpose', 'facetec'];
+    console.log('[Frontend Orchestrator] Configured backends:', configuredBackends);
+
+    // If no backend HTTP client configured, use mock
+    if (!this.backendClient) {
+      console.warn('[Frontend Orchestrator] No backend HTTP client configured, using mock');
+      return this.mockBackend.verifyLiveness(imageData, videoBlob);
+    }
+
+    // Use selected backends from config
+    console.log('[Frontend Orchestrator] Using multi-backend parallel analysis with backends:', configuredBackends);
+    return this.verifyWithSelectedBackends(imageData, videoBlob, configuredBackends as LivenessBackend[]);
+  }
+
+  /**
+   * Verify liveness using selected backends in parallel with voting system
+   */
+  private verifyWithSelectedBackends(imageData: string, videoBlob?: Blob, backends?: LivenessBackend[]): Observable<BackendResponse> {
+    return new Observable(observer => {
+      if (!this.backendClient) {
+        observer.error(new Error('Backend HTTP client not configured'));
+        return;
+      }
+
+      const selectedBackends = backends || ['amazon', 'openpose', 'facetec'] as LivenessBackend[];
+
+      // First create a session
+      this.backendClient.createSession('amazon').subscribe({
+        next: (sessionResponse) => {
+          const sessionId = sessionResponse.sessionId;
+
+          // Call the analyze-selected endpoint with configured backends
+          this.backendClient!.analyzeWithSelectedBackends(sessionId, imageData, selectedBackends).subscribe({
+            next: (multiResult: MultiBackendAnalyzeResponse) => {
+              // Store the multi-backend result
+              this.multiBackendResult = multiResult;
+
+              // Return the consensus result
+              const response: BackendResponse = {
+                isLive: multiResult.consensus.isLive,
+                confidence: multiResult.consensus.confidence,
+                details: multiResult
+              };
+
+              observer.next(response);
+              observer.complete();
+            },
+            error: (error: any) => {
+              console.error('Multi-backend analysis failed:', error);
+              observer.error(error);
+            }
+          });
+        },
+        error: (error: any) => {
+          console.error('Failed to create session:', error);
+          observer.error(error);
+        }
+      });
+    });
+  }
+
+  /**
+   * Verify liveness using all backends in parallel with voting system
+   * @deprecated Use verifyLiveness with configured backends instead
+   */
+  private verifyWithAllBackends(imageData: string, videoBlob?: Blob): Observable<BackendResponse> {
+    return this.verifyWithSelectedBackends(imageData, videoBlob, ['amazon', 'openpose', 'facetec']);
+  }
+
+  /**
+   * Upload video to S3
+   */
+  uploadVideoToS3(videoBlob: Blob, sessionId: string): Observable<{ success: boolean; videoUrl: string; sessionId: string }> {
+    if (!this.backendClient) {
+      return new Observable(observer => {
+        observer.error(new Error('Backend HTTP client not configured'));
+      });
+    }
+
+    return this.backendClient.uploadVideo(videoBlob, sessionId);
+  }
+
+  /**
+   * Upload a base64 image to S3 (fallback when video recording is unavailable)
+   */
+  uploadBase64ImageToS3(base64Image: string, sessionId: string): Observable<{ success: boolean; videoUrl: string; sessionId: string }> {
+    if (!this.backendClient) {
+      return new Observable(observer => {
+        observer.error(new Error('Backend HTTP client not configured'));
+      });
+    }
+
+    return this.backendClient.uploadBase64Image(base64Image, sessionId);
+  }
+
+  /**
+   * Get a specific backend adapter by name
+   */
+  getBackend(name: LivenessBackend): IBackendAdapter | undefined {
+    return this.backends.get(name);
+  }
+
+  /**
+   * Get all available backends
+   */
+  getAvailableBackends(): LivenessBackend[] {
+    const available: LivenessBackend[] = [];
+    this.backends.forEach((backend, name) => {
+      if (backend.isAvailable()) {
+        available.push(name);
+      }
+    });
+    return available;
+  }
+}

package/projects/facecog-liveness-verification/src/lib/services/pose-detection/hand-gesture-detection.service.ts

@@ -0,0 +1,315 @@
+import { Injectable } from '@angular/core';
+import { HandLandmarker, FilesetResolver, HandLandmarkerResult } from '@mediapipe/tasks-vision';
+
+export interface HandGestureResult {
+  gesture: string;
+  confidence: number;
+  handedness: 'Left' | 'Right';
+  landmarks: any[];
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class HandGestureDetectionService {
+  private handLandmarker: HandLandmarker | null = null;
+  private isInitialized = false;
+
+  constructor() {}
+
+  /**
+   * Initialize MediaPipe Hand Landmarker
+   */
+  async loadModel(): Promise<void> {
+    if (this.isInitialized) {
+      console.log('[MediaPipe Hands] Model already loaded');
+      return;
+    }
+
+    try {
+      console.log('[MediaPipe Hands] Loading Hand Landmarker model...');
+
+      const vision = await FilesetResolver.forVisionTasks(
+        'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm'
+      );
+
+      // Try GPU first, fallback to CPU for Safari compatibility
+      try {
+        this.handLandmarker = await HandLandmarker.createFromOptions(vision, {
+          baseOptions: {
+            modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
+            delegate: 'GPU'
+          },
+          runningMode: 'VIDEO',
+          numHands: 2,
+          minHandDetectionConfidence: 0.5,
+          minHandPresenceConfidence: 0.5,
+          minTrackingConfidence: 0.5
+        });
+        console.log('[MediaPipe Hands] Hand Landmarker loaded with GPU delegate');
+      } catch (gpuError) {
+        console.warn('[MediaPipe Hands] GPU delegate failed, trying CPU for Safari compatibility:', gpuError);
+        this.handLandmarker = await HandLandmarker.createFromOptions(vision, {
+          baseOptions: {
+            modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
+            delegate: 'CPU'
+          },
+          runningMode: 'VIDEO',
+          numHands: 2,
+          minHandDetectionConfidence: 0.5,
+          minHandPresenceConfidence: 0.5,
+          minTrackingConfidence: 0.5
+        });
+        console.log('[MediaPipe Hands] Hand Landmarker loaded with CPU delegate (Safari fallback)');
+      }
+
+      this.isInitialized = true;
+      console.log('[MediaPipe Hands] Hand Landmarker loaded successfully');
+    } catch (error) {
+      console.error('[MediaPipe Hands] Failed to load model:', error);
+      throw new Error('Failed to load hand detection model');
+    }
+  }
+
+  /**
+   * Detect hands in video frame
+   */
+  async detectHands(videoElement: HTMLVideoElement): Promise<HandLandmarkerResult | null> {
+    try {
+      if (!this.isInitialized || !this.handLandmarker) {
+        console.warn('[MediaPipe Hands] Model not loaded, attempting to load...');
+        await this.loadModel();
+      }
+
+      const timestamp = performance.now();
+      const result = this.handLandmarker!.detectForVideo(videoElement, timestamp);
+
+      return result && result.landmarks && result.landmarks.length > 0 ? result : null;
+    } catch (error) {
+      console.error('[MediaPipe Hands] Hand detection error:', error);
+      return null;
+    }
+  }
+
+  /**
+   * Detect specific hand gesture from landmarks
+   */
+  detectGesture(result: HandLandmarkerResult): HandGestureResult[] {
+    const gestures: HandGestureResult[] = [];
+
+    if (!result.landmarks || result.landmarks.length === 0) {
+      return gestures;
+    }
+
+    // Process each detected hand
+    for (let i = 0; i < result.landmarks.length; i++) {
+      const landmarks = result.landmarks[i];
+      const handedness = result.handedness?.[i]?.[0]?.categoryName || 'Unknown';
+
+      console.log(`[MediaPipe Hands] Raw handedness from MediaPipe: "${handedness}"`);
+
+      // Detect gesture based on finger positions
+      const gesture = this.classifyGesture(landmarks);
+
+      gestures.push({
+        gesture: gesture.name,
+        confidence: gesture.confidence,
+        handedness: handedness as 'Left' | 'Right',
+        landmarks: landmarks
+      });
+
+      console.log(`[MediaPipe Hands] Classified gesture: ${gesture.name}, Final handedness: ${handedness}`);
+    }
+
+    return gestures;
+  }
+
+  /**
+   * Classify gesture based on hand landmarks
+   */
+  private classifyGesture(landmarks: any[]): { name: string; confidence: number } {
+    // Get finger states (extended or closed)
+    const fingers = {
+      thumb: this.isThumbExtended(landmarks),
+      index: this.isFingerExtended(landmarks, 'index'),
+      middle: this.isFingerExtended(landmarks, 'middle'),
+      ring: this.isFingerExtended(landmarks, 'ring'),
+      pinky: this.isFingerExtended(landmarks, 'pinky')
+    };
+
+    const extendedCount = Object.values(fingers).filter(Boolean).length;
+
+    // Debug logging
+    console.log(`[Gesture Classification] Extended fingers: thumb=${fingers.thumb}, index=${fingers.index}, middle=${fingers.middle}, ring=${fingers.ring}, pinky=${fingers.pinky}, total=${extendedCount}`);
+
+    // Classify based on finger patterns
+    // Priority order matters! Check specific patterns before general counts.
+
+    // Thumbs Up: Only thumb extended, all others closed
+    // MUST check this FIRST with strictest conditions to avoid false positives
+    if (fingers.thumb && !fingers.index && !fingers.middle && !fingers.ring && !fingers.pinky) {
+      console.log('[Gesture Classification] Detected: thumbs_up');
+      return { name: 'thumbs_up', confidence: 0.9 };
+    }
+
+    // Fist: All fingers closed (no fingers extended)
+    // Check early to distinguish from other gestures
+    if (extendedCount === 0) {
+      console.log('[Gesture Classification] Detected: fist');
+      return { name: 'fist', confidence: 0.9 };
+    }
+
+    // OK Sign: Thumb and index tips close together (forming circle), other fingers extended
+    // MUST check BEFORE peace sign to avoid conflicts
+    if (this.isOKSign(landmarks, fingers)) {
+      console.log('[Gesture Classification] Detected: ok');
+      return { name: 'ok', confidence: 0.85 };
+    }
+
+    // Peace Sign: Index and middle extended, ring and pinky closed
+    // More lenient: allow thumb to be slightly extended
+    if (fingers.index && fingers.middle && !fingers.ring && !fingers.pinky) {
+      console.log('[Gesture Classification] Detected: peace');
+      return { name: 'peace', confidence: 0.9 };
+    }
+
+    // Three Fingers: Index + Middle + Ring extended (typical three finger gesture)
+    // Check this BEFORE palm to avoid confusion
+    if (fingers.index && fingers.middle && fingers.ring && !fingers.pinky) {
+      console.log('[Gesture Classification] Detected: three (index+middle+ring)');
+      return { name: 'three', confidence: 0.9 };
+    }
+
+    // Palm/Open Hand: All 5 fingers extended OR all 4 non-thumb fingers extended
+    // Be strict: exactly 5 fingers OR exactly 4 fingers (index, middle, ring, pinky all up)
+    const allFingersUp = extendedCount === 5;
+    const fourFingersUp = !fingers.thumb && fingers.index && fingers.middle && fingers.ring && fingers.pinky;
+
+    if (allFingersUp || fourFingersUp) {
+      console.log('[Gesture Classification] Detected: palm');
+      return { name: 'palm', confidence: 0.9 };
+    }
+
+    // Three fingers (any combination of 3 extended fingers, but not the specific patterns above)
+    if (extendedCount === 3) {
+      console.log('[Gesture Classification] Detected: three (generic 3 fingers)');
+      return { name: 'three', confidence: 0.8 };
+    }
+
+    // If we get here, it's an unknown gesture (1, 2, or 4+ fingers in unexpected patterns)
+    console.log('[Gesture Classification] Detected: unknown');
+    return { name: 'unknown', confidence: 0.5 };
+  }
+
+  /**
+   * Check if thumb is extended
+   */
+  private isThumbExtended(landmarks: any[]): boolean {
+    // Thumb: landmarks 1-4
+    // Extended if tip (4) is farther from wrist (0) than base (2)
+    const wrist = landmarks[0];
+    const thumbBase = landmarks[2];
+    const thumbTip = landmarks[4];
+
+    const baseDist = this.distance(wrist, thumbBase);
+    const tipDist = this.distance(wrist, thumbTip);
+
+    // Balanced threshold: strict enough to avoid false positives, lenient enough for detection
+    const isExtended = tipDist > baseDist * 1.25;
+
+    return isExtended;
+  }
+
+  /**
+   * Check if a finger is extended
+   */
+  private isFingerExtended(landmarks: any[], finger: 'index' | 'middle' | 'ring' | 'pinky'): boolean {
+    const fingerIndices = {
+      index: { base: 5, tip: 8, mid: 6 },
+      middle: { base: 9, tip: 12, mid: 10 },
+      ring: { base: 13, tip: 16, mid: 14 },
+      pinky: { base: 17, tip: 20, mid: 18 }
+    };
+
+    const indices = fingerIndices[finger];
+    const base = landmarks[indices.base];
+    const mid = landmarks[indices.mid];
+    const tip = landmarks[indices.tip];
+    const wrist = landmarks[0];
+
+    // Extended if tip is farther from wrist than base
+    const baseDist = this.distance(wrist, base);
+    const tipDist = this.distance(wrist, tip);
+    const midDist = this.distance(wrist, mid);
+
+    // Balanced thresholds for better accuracy
+    // Primary check: tip should be significantly farther than base
+    const distanceCheck = tipDist > baseDist * 1.12;
+
+    // Secondary check: middle joint should be at least slightly away from base
+    // This is more lenient to allow natural hand variations
+    const midCheck = midDist > baseDist * 1.02;
+
+    const isExtended = distanceCheck && midCheck;
+
+    return isExtended;
+  }
+
+  /**
+   * Check if hand is making OK sign
+   * OK sign = thumb and index tips touching (forming circle) + other 3 fingers extended
+   */
+  private isOKSign(landmarks: any[], fingers: any): boolean {
+    const thumbTip = landmarks[4];
+    const indexTip = landmarks[8];
+
+    // First check: thumb and index tips must be close together (forming the circle)
+    const dist = this.distance(thumbTip, indexTip);
+    const tipsClose = dist < 0.05; // Threshold for normalized coordinates
+
+    if (!tipsClose) {
+      return false;
+    }
+
+    // Second check: middle, ring, and pinky must be EXTENDED (not closed like thumbs up)
+    // At least 2 of the 3 fingers should be extended for a valid OK sign
+    const otherFingersExtended = [fingers.middle, fingers.ring, fingers.pinky].filter(Boolean).length;
+
+    // OK sign requires other fingers to be up (at least 2 of them)
+    // This prevents thumbs up (all fingers closed) from being detected as OK
+    const isOK = tipsClose && otherFingersExtended >= 2;
+
+    console.log(`[OK Sign Check] Tips close: ${tipsClose}, Distance: ${dist.toFixed(4)}, Other fingers extended: ${otherFingersExtended}/3, Result: ${isOK}`);
+
+    return isOK;
+  }
+
+  /**
+   * Calculate Euclidean distance between two landmarks
+   */
+  private distance(p1: any, p2: any): number {
+    const dx = p1.x - p2.x;
+    const dy = p1.y - p2.y;
+    const dz = (p1.z || 0) - (p2.z || 0);
+    return Math.sqrt(dx * dx + dy * dy + dz * dz);
+  }
+
+  /**
+   * Compare detected gesture with expected gesture
+   */
+  matchGesture(detected: string, expected: string): boolean {
+    return detected.toLowerCase() === expected.toLowerCase();
+  }
+
+  /**
+   * Cleanup resources
+   */
+  cleanup(): void {
+    if (this.handLandmarker) {
+      this.handLandmarker.close();
+      this.handLandmarker = null;
+      this.isInitialized = false;
+      console.log('[MediaPipe Hands] Hand Landmarker disposed');
+    }
+  }
+}

package/projects/facecog-liveness-verification/src/lib/services/pose-detection/index.ts

@@ -0,0 +1,5 @@
+export { OpenposeService, PoseKeypoints } from './openpose.service';
+export { HandGestureDetectionService, HandGestureResult } from './hand-gesture-detection.service';
+export { ReferencePoseService, ReferencePose } from './reference-pose.service';
+export { PoseComparisonService, PoseComparisonResult } from './pose-comparison.service';
+export { PoseMatchingService } from './pose-matching.service';