handle-and-face 1.0.0

package/README.md

@@ -0,0 +1,109 @@
+# handle-and-face 🦾👤
+
+**Improved accessibility and precise tracking for modern web applications.**
+
+`handle-and-face` is a React library that provides highly calibrated hooks for **Hand Tracking** (index finger pointer + pinch click) and **Head Tracking** (nose-driven pointer + blink click). Built on top of MediaPipe, it simplifies the integration of advanced computer vision for accessibility and interactive web experiences.
+
+---
+
+## 🇪🇸 Documentación en Español
+
+### Instalación
+
+```bash
+npm install handle-and-face
+```
+
+### Hooks Disponibles
+
+#### 1. `useHandTracking`
+
+Ideal para navegar usando el dedo índice como puntero.
+
+- **Gesto**: Junta el pulgar y el índice para hacer click.
+- **Implementación**:
+
+```javascript
+const { videoRef, canvasRef, pointer, isClicking } = useHandTracking({
+  sensitivityX: 2.5,
+  onPinch: (x, y) => console.log("Click en", x, y),
+});
+```
+
+#### 2. `useHeadTracking`
+
+Perfecto para control manos-libres y accesibilidad.
+
+- **Gesto**: Parpadea una vez para click, dos veces rápido para doble click.
+- **Implementación**:
+
+```javascript
+const { videoRef, canvasRef, pointer, isClicking } = useHeadTracking({
+  sensitivityX: 10.0,
+  onClick: (x, y) => console.log("Click"),
+  onDoubleClick: (x, y) => console.log("Doble Click"),
+});
+```
+
+---
+
+## 🇺🇸 Documentation in English
+
+### Installation
+
+```bash
+npm install handle-and-face
+```
+
+### Available Hooks
+
+#### 1. `useHandTracking`
+
+Use your index finger as a virtual mouse.
+
+- **Gesture**: Pinch thumb and index to click.
+- **Usage**:
+
+```javascript
+const { videoRef, canvasRef, pointer, isClicking } = useHandTracking({
+  sensitivityX: 2.0,
+  onPinch: (x, y) => console.log("Pinch click at", x, y),
+});
+```
+
+#### 2. `useHeadTracking`
+
+Touch-free navigation using head movement.
+
+- **Gesture**: Blink to click, rapid double-blink for double-click.
+- **Usage**:
+
+```javascript
+const { videoRef, canvasRef, pointer, isClicking } = useHeadTracking({
+  sensitivityX: 10.0,
+  onDoubleClick: (x, y) => console.log("Double blink detected"),
+});
+```
+
+---
+
+## 🛡️ Stability & Precision
+
+The library includes custom-built algorithms for:
+
+- **Dynamic Smoothing**: Reduces lag during fast movements and adds weight for precision when aiming at small targets.
+- **Blink-Freeze**: Temporarily anchors the pointer during blinks/clicks to prevent unintended cursor drift.
+- **Auto-Calibration**: Pre-tuned sensitivity values for standard laptop cameras.
+
+## ⚙️ Configuration Parameters
+
+| Parameter             | Type       | Default      | Description                                        |
+| :-------------------- | :--------- | :----------- | :------------------------------------------------- |
+| `sensitivityX/Y`      | `Number`   | `2.0 - 10.0` | Pointer speed multiplier.                          |
+| `jitterThreshold`     | `Number`   | `1.5`        | Minimum movement pixels to ignore (reduces noise). |
+| `onPinch` / `onClick` | `Function` | `null`       | Single interaction callback.                       |
+| `onDoubleClick`       | `Function` | `null`       | Double interaction callback (Head Tracking only).  |
+
+## 📄 License
+
+MIT © Starla.io

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "handle-and-face",
+  "version": "1.0.0",
+  "description": "Custom React hooks for hand and head tracking using MediaPipe. Improved accessibility and calibrated for precise web interactions.",
+  "type": "module",
+  "main": "src/index.js",
+  "files": [
+    "src",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "react",
+    "mediapipe",
+    "hand-tracking",
+    "head-tracking",
+    "accessibility",
+    "face-mesh",
+    "pinch-click",
+    "blink-detection"
+  ],
+  "author": "fernando.robles",
+  "license": "MIT",
+  "peerDependencies": {
+    "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
+    "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
+  }
+}

package/src/hooks/useHandTracking.js

@@ -0,0 +1,157 @@
+'use client';
+import { useRef, useEffect, useState, useCallback } from 'react';
+
+export function useHandTracking({
+  sensitivityX = 2.0,
+  sensitivityY = 2.0,
+  smoothing = 0.3,
+  pinchThreshold = 0.05,
+  onPinch = null
+} = {}) {
+  const videoRef = useRef(null);
+  const canvasRef = useRef(null);
+  const handsRef = useRef(null);
+  const cameraRef = useRef(null);
+  const _r = useRef({ x: 0, y: 0, initialized: false });
+
+  const [isTracking, setIsTracking] = useState(false);
+  const [pointer, setPointer] = useState({ x: 0, y: 0 });
+  const [isClicking, setIsClicking] = useState(false);
+
+  useEffect(() => {
+    const _s = [
+      'https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js',
+      'https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js',
+      'https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js'
+    ];
+    const _l = async () => {
+      for (const src of _s) {
+        if (!document.querySelector(`script[src="${src}"]`)) {
+          await new Promise((res) => {
+            const s = document.createElement('script');
+            s.src = src; s.onload = res;
+            document.body.appendChild(s);
+          });
+        }
+      }
+    };
+    _l();
+  }, []);
+
+  const onResults = useCallback((res) => {
+    if (!canvasRef.current || !res.image) return;
+    const _c = canvasRef.current;
+    if (_c.width !== res.image.width || _c.height !== res.image.height) {
+      _c.width = res.image.width; _c.height = res.image.height;
+    }
+    const _x = _c.getContext('2d');
+    _x.save(); _x.clearRect(0, 0, _c.width, _c.height);
+    _x.drawImage(res.image, 0, 0, _c.width, _c.height);
+
+    if (res.multiHandLandmarks && res.multiHandLandmarks.length > 0) {
+      const _p = res.multiHandLandmarks[0];
+      if (window.drawConnectors && window.HAND_CONNECTIONS) {
+        window.drawConnectors(_x, _p, window.HAND_CONNECTIONS, {color: '#00FF00', lineWidth: 2});
+        window.drawLandmarks(_x, _p, {color: '#FF0000', lineWidth: 1, radius: 3});
+      }
+
+      const [_i, _t] = [_p[8], _p[4]];
+      let _nx = Math.max(0, Math.min(1, ((1 - _i.x) - 0.5) * sensitivityX + 0.5));
+      let _ny = Math.max(0, Math.min(1, (_i.y - 0.5) * sensitivityY + 0.5));
+      const [_rx, _ry] = [_nx * window.innerWidth, _ny * window.innerHeight];
+
+      const _ds = Math.sqrt(Math.pow(_i.x - _t.x, 2) + Math.pow(_i.y - _t.y, 2));
+      const _act = _ds < pinchThreshold;
+
+      if (!_r.current.initialized) _r.current = { x: _rx, y: _ry, initialized: true };
+      const _f = _act ? 0.05 : smoothing;
+      _r.current.x += (_rx - _r.current.x) * _f;
+      _r.current.y += (_ry - _r.current.y) * _f;
+
+      setPointer({ x: _r.current.x, y: _r.current.y });
+
+      if (_act) {
+        setIsClicking(p => {
+          if (!p && onPinch) onPinch(_r.current.x, _r.current.y);
+          return true;
+        });
+      } else setIsClicking(false);
+    }
+    _x.restore();
+  }, [sensitivityX, sensitivityY, smoothing, pinchThreshold, onPinch]);
+
+  useEffect(() => {
+    if (isTracking && !handsRef.current && window.Hands) {
+      const hands = new window.Hands({
+        locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}`
+      });
+
+      hands.setOptions({
+        maxNumHands: 1,
+        modelComplexity: 1,
+        minDetectionConfidence: 0.5,
+        minTrackingConfidence: 0.5
+      });
+
+      hands.onResults(onResults);
+      handsRef.current = hands;
+
+      if (videoRef.current) {
+        cameraRef.current = new window.Camera(videoRef.current, {
+          onFrame: async () => {
+            if (handsRef.current && videoRef.current) {
+              await handsRef.current.send({ image: videoRef.current });
+            }
+          },
+          width: 640,
+          height: 480
+        });
+        cameraRef.current.start();
+      }
+    }
+
+    return () => {
+      if (!isTracking) {
+        if (cameraRef.current) {
+          cameraRef.current.stop();
+          cameraRef.current = null;
+        }
+        if (handsRef.current) {
+          handsRef.current.close();
+          handsRef.current = null;
+        }
+      }
+    };
+  }, [isTracking, onResults]);
+
+  const startTracking = async () => {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ video: true });
+      if (videoRef.current) {
+        videoRef.current.srcObject = stream;
+        setIsTracking(true);
+      }
+    } catch (err) {
+      console.error('Camera access error:', err);
+    }
+  };
+
+  const stopTracking = () => {
+    if (videoRef.current && videoRef.current.srcObject) {
+      videoRef.current.srcObject.getTracks().forEach(t => t.stop());
+      videoRef.current.srcObject = null;
+    }
+    setIsTracking(false);
+    _r.current.initialized = false;
+  };
+
+  return {
+    videoRef,
+    canvasRef,
+    pointer,
+    isClicking,
+    isTracking,
+    startTracking,
+    stopTracking
+  };
+}

package/src/hooks/useHeadTracking.js

@@ -0,0 +1,178 @@
+'use client';
+import { useRef, useEffect, useState, useCallback } from 'react';
+
+export function useHeadTracking({
+  sensitivityX = 10.0,
+  sensitivityY = 10.0,
+  jitterThreshold = 1.5,
+  blinkThreshold = 0.15,
+  doubleBlinkWindow = 500,
+  onDoubleClick = null,
+  onClick = null
+} = {}) {
+  const videoRef = useRef(null);
+  const canvasRef = useRef(null);
+  const _m = useRef(null);
+  const cameraRef = useRef(null);
+  
+  const _p = useRef({ x: 0, y: 0, initialized: false, history: [] });
+  const _b = useRef({ last: 0, active: false });
+
+  const [isTracking, setIsTracking] = useState(false);
+  const [pointer, setPointer] = useState({ x: 0, y: 0 });
+  const [isClicking, setIsClicking] = useState(false);
+
+  useEffect(() => {
+    const _s = [
+      'https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/face_mesh.js',
+      'https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js',
+      'https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js'
+    ];
+    const _l = async () => {
+      for (const src of _s) {
+        if (!document.querySelector(`script[src="${src}"]`)) {
+          await new Promise((res) => {
+            const s = document.createElement('script');
+            s.src = src; s.onload = res;
+            document.body.appendChild(s);
+          });
+        }
+      }
+    };
+    _l();
+  }, []);
+
+  const onResults = useCallback((res) => {
+    if (!canvasRef.current || !res.image) return;
+    const _c = canvasRef.current;
+    if (_c.width !== res.image.width || _c.height !== res.image.height) {
+      _c.width = res.image.width; _c.height = res.image.height;
+    }
+    const _x = _c.getContext('2d');
+    _x.save(); _x.clearRect(0, 0, _c.width, _c.height);
+    _x.drawImage(res.image, 0, 0, _c.width, _c.height);
+
+    if (res.multiFaceLandmarks && res.multiFaceLandmarks.length > 0) {
+      const _l = res.multiFaceLandmarks[0];
+      if (window.drawConnectors && window.FACEMESH_TESSELATION) {
+         window.drawConnectors(_x, _l, window.FACEMESH_TESSELATION, {color: '#C0C0C040', lineWidth: 1});
+         window.drawConnectors(_x, _l, window.FACEMESH_RIGHT_EYE, {color: '#FF3030'});
+         window.drawConnectors(_x, _l, window.FACEMESH_LEFT_EYE, {color: '#30FF30'});
+      }
+
+      const _n = _l[4];
+      let _nx = Math.max(0, Math.min(1, 1 - ((_n.x - 0.5) * sensitivityX + 0.5)));
+      let _ny = Math.max(0, Math.min(1, (_n.y - 0.5) * sensitivityY + 0.5));
+      const [_rx, _ry] = [_nx * window.innerWidth, _ny * window.innerHeight];
+
+      const _ear = (t, b, o, i) => Math.abs(_l[t].y - _l[b].y) / Math.abs(_l[o].x - _l[i].x);
+      const _blink = _ear(159, 145, 33, 133) < blinkThreshold && _ear(386, 374, 362, 263) < blinkThreshold;
+
+      const _now = Date.now();
+      if (_blink && !_b.current.active) {
+        const _delta = _now - _b.current.last;
+        if (_delta < doubleBlinkWindow) {
+          if (onDoubleClick) onDoubleClick(_p.current.x, _p.current.y);
+          setIsClicking(true); setTimeout(() => setIsClicking(false), 200);
+        } else {
+          if (onClick) onClick(_p.current.x, _p.current.y);
+          setIsClicking(true); setTimeout(() => setIsClicking(false), 100);
+        }
+        _b.current.last = _now;
+      }
+      _b.current.active = _blink;
+
+      const _h = _p.current.history;
+      _h.push({ x: _rx, y: _ry }); if (_h.length > 5) _h.shift();
+      const [_ax, _ay] = [_h.reduce((s, d) => s + d.x, 0) / _h.length, _h.reduce((s, d) => s + d.y, 0) / _h.length];
+
+      if (!_p.current.initialized) { _p.current.x = _ax; _p.current.y = _ay; _p.current.initialized = true; }
+
+      const [_dx, _dy] = [_ax - _p.current.x, _ay - _p.current.y];
+      const _v = Math.sqrt(_dx * _dx + _dy * _dy);
+
+      if (_v > jitterThreshold || _blink) {
+        const _sf = _blink ? 0.02 : (_v < 20 ? 0.15 : 0.35);
+        _p.current.x += _dx * _sf; _p.current.y += _dy * _sf;
+      }
+      setPointer({ x: _p.current.x, y: _p.current.y });
+    }
+    _x.restore();
+  }, [sensitivityX, sensitivityY, jitterThreshold, blinkThreshold, doubleBlinkWindow, onDoubleClick, onClick]);
+
+  useEffect(() => {
+    if (isTracking && !_m.current && window.FaceMesh) {
+      const faceMesh = new window.FaceMesh({
+        locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`
+      });
+
+      faceMesh.setOptions({
+        maxNumFaces: 1,
+        refineLandmarks: true,
+        minDetectionConfidence: 0.5,
+        minTrackingConfidence: 0.5
+      });
+
+      faceMesh.onResults(onResults);
+      _m.current = faceMesh;
+
+      if (videoRef.current) {
+        cameraRef.current = new window.Camera(videoRef.current, {
+          onFrame: async () => {
+            if (_m.current && videoRef.current) {
+              await _m.current.send({ image: videoRef.current });
+            }
+          },
+          width: 640,
+          height: 480
+        });
+        cameraRef.current.start();
+      }
+    }
+
+    return () => {
+      if (!isTracking) {
+        if (cameraRef.current) {
+          cameraRef.current.stop();
+          cameraRef.current = null;
+        }
+        if (_m.current) {
+          _m.current.close();
+          _m.current = null;
+        }
+      }
+    };
+  }, [isTracking, onResults]);
+
+  const startTracking = async () => {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ video: true });
+      if (videoRef.current) {
+        videoRef.current.srcObject = stream;
+        setIsTracking(true);
+      }
+    } catch (err) {
+      console.error('Camera access error:', err);
+    }
+  };
+
+  const stopTracking = () => {
+    if (videoRef.current && videoRef.current.srcObject) {
+      videoRef.current.srcObject.getTracks().forEach(t => t.stop());
+      videoRef.current.srcObject = null;
+    }
+    setIsTracking(false);
+    _p.current.initialized = false;
+    _p.current.history = [];
+  };
+
+  return {
+    videoRef,
+    canvasRef,
+    pointer,
+    isClicking,
+    isTracking,
+    startTracking,
+    stopTracking
+  };
+}

package/src/index.js

@@ -0,0 +1,2 @@
+export { useHandTracking } from './hooks/useHandTracking';
+export { useHeadTracking } from './hooks/useHeadTracking';