npm - mudra-skills - Versions diffs - 1.0.0 - Mend

mudra-skills 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (149) hide show

package/skills/mudra-master/mudra-xr/assets/demos/webcam_gestures.html ADDED Viewed

@@ -0,0 +1,253 @@
+<!doctype html>
+<!-- reference: demos/webcam_gestures -->
+<html lang="en">
+  <head>
+    <title>Webcam Gestures | XR Blocks</title>
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1.0, user-scalable=no"
+    />
+    <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
+    <script>
+      window.litDisableBundleWarning = true;
+    </script>
+    <script type="importmap">
+      {
+        "imports": {
+          "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
+          "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
+          "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
+          "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
+          "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
+          "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
+          "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
+          "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
+          "lit/": "https://esm.run/lit@3/",
+          "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
+          "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/",
+          "@mediapipe/tasks-vision": "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.9/+esm"
+        }
+      }
+    </script>
+  </head>
+  <body>
+    <div id="camera-container">
+      <video id="webcam-video" autoplay playsinline muted></video>
+      <canvas id="output-canvas"></canvas>
+      <div id="gesture-label">Loading...</div>
+    </div>
+    <script type="module">
+import 'xrblocks/addons/simulator/SimulatorAddons.js';
+import * as xb from 'xrblocks';
+import {
+  FilesetResolver,
+  GestureRecognizer,
+  DrawingUtils,
+} from '@mediapipe/tasks-vision';
+// --- HandTrackingService + GestureToSimulatorBridge inlined from main.js ---
+class HandTrackingService {
+  constructor() {
+    this.gestureRecognizer = null;
+    this.videoElement = document.getElementById('webcam-video');
+    this.canvasElement = document.getElementById('output-canvas');
+    this.canvasCtx = this.canvasElement.getContext('2d');
+    this.gestureLabel = document.getElementById('gesture-label');
+    this.lastVideoTime = -1;
+    this.currentGesture = 'NONE';
+  }
+  async initialize() {
+    const vision = await FilesetResolver.forVisionTasks(
+      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.9/wasm'
+    );
+    this.gestureRecognizer = await GestureRecognizer.createFromOptions(vision, {
+      baseOptions: {
+        modelAssetPath:
+          'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/latest/gesture_recognizer.task',
+        delegate: 'GPU',
+      },
+      runningMode: 'VIDEO',
+      numHands: 1,
+    });
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({video: true});
+      this.videoElement.srcObject = stream;
+      this.videoElement.addEventListener('loadeddata', () => {
+        this.gestureLabel.innerText = 'Show Hand';
+        this.predictWebcam();
+      });
+    } catch (err) {
+      console.error(err);
+      this.gestureLabel.innerText = 'Camera Error';
+    }
+  }
+  async predictWebcam() {
+    requestAnimationFrame(() => this.predictWebcam());
+    if (!this.gestureRecognizer) return;
+    if (this.videoElement.currentTime !== this.lastVideoTime) {
+      this.lastVideoTime = this.videoElement.currentTime;
+      const results = this.gestureRecognizer.recognizeForVideo(
+        this.videoElement,
+        performance.now()
+      );
+      this.drawResults(results);
+    }
+  }
+  detectCustomGestures(landmarks) {
+    const wrist = landmarks[0],
+      thumbTip = landmarks[4],
+      indexKnuckle = landmarks[5];
+    const indexTip = landmarks[8],
+      indexPip = landmarks[6];
+    const middleTip = landmarks[12],
+      middlePip = landmarks[10];
+    const ringTip = landmarks[16],
+      ringPip = landmarks[14];
+    const pinkyTip = landmarks[20],
+      pinkyPip = landmarks[18];
+    const dist = (p1, p2) =>
+      Math.sqrt(
+        Math.pow(p1.x - p2.x, 2) +
+          Math.pow(p1.y - p2.y, 2) +
+          Math.pow(p1.z - p2.z, 2)
+      );
+    const isExtended = (tip, pip) => dist(tip, wrist) > dist(pip, wrist) * 1.1;
+    const indexOpen = isExtended(indexTip, indexPip);
+    const middleOpen = isExtended(middleTip, middlePip);
+    const ringOpen = isExtended(ringTip, ringPip);
+    const pinkyOpen = isExtended(pinkyTip, pinkyPip);
+    if (!indexOpen && !middleOpen && !ringOpen && !pinkyOpen) {
+      if (thumbTip.y > indexKnuckle.y + 0.1) return 'THUMB_DOWN';
+      if (thumbTip.y < indexKnuckle.y - 0.1) return 'THUMB_UP';
+      return 'FIST';
+    }
+    const indexExtension = dist(indexTip, wrist) / dist(indexKnuckle, wrist);
+    if (dist(thumbTip, indexTip) < 0.05 && indexExtension > 1.3) {
+      return 'PINCH';
+    }
+    if (indexOpen && !middleOpen && !ringOpen && pinkyOpen) {
+      return 'ROCK';
+    }
+    return null;
+  }
+  drawResults(results) {
+    this.canvasElement.width = this.videoElement.videoWidth;
+    this.canvasElement.height = this.videoElement.videoHeight;
+    this.canvasCtx.clearRect(
+      0,
+      0,
+      this.canvasElement.width,
+      this.canvasElement.height
+    );
+    if (!results || !results.landmarks.length) {
+      this.currentGesture = 'NONE';
+      this.gestureLabel.innerText = 'No Hand';
+      this.gestureLabel.style.color = '#fff';
+      return;
+    }
+    const landmarks = results.landmarks[0];
+    const drawingUtils = new DrawingUtils(this.canvasCtx);
+    drawingUtils.drawConnectors(landmarks, GestureRecognizer.HAND_CONNECTIONS, {
+      color: '#00FF00',
+      lineWidth: 3,
+    });
+    drawingUtils.drawLandmarks(landmarks, {
+      color: '#FF0000',
+      lineWidth: 1,
+      radius: 3,
+    });
+    let gesture = this.detectCustomGestures(landmarks);
+    if (!gesture && results.gestures.length > 0) {
+      const name = results.gestures[0][0].categoryName;
+      const mapping = {
+        Pointing_Up: 'POINTING',
+        Victory: 'VICTORY',
+        Thumb_Up: 'THUMB_UP',
+        Closed_Fist: 'FIST',
+        Open_Palm: 'RELAXED',
+      };
+      gesture = mapping[name] || name;
+    }
+    gesture = gesture || 'NONE';
+    this.currentGesture = gesture;
+    this.gestureLabel.innerText = gesture.replace(/_/g, ' ');
+    this.gestureLabel.style.color =
+      gesture !== 'NONE' && gesture !== 'RELAXED' ? '#0f0' : '#ccc';
+  }
+}
+const handTracking = new HandTrackingService();
+handTracking.initialize();
+class GestureToSimulatorBridge extends xb.Script {
+  constructor() {
+    super();
+    this.lastGesture = 'NONE';
+    this.lastTriggerTime = 0;
+    this.cooldownMs = 100;
+    this.gestureToSimulatorPose = {
+      PINCH: xb.SimulatorHandPose.PINCHING,
+      FIST: xb.SimulatorHandPose.FIST,
+      THUMB_UP: xb.SimulatorHandPose.THUMBS_UP,
+      THUMB_DOWN: xb.SimulatorHandPose.THUMBS_DOWN,
+      POINTING: xb.SimulatorHandPose.POINTING,
+      VICTORY: xb.SimulatorHandPose.VICTORY,
+      ROCK: xb.SimulatorHandPose.ROCK,
+      RELAXED: xb.SimulatorHandPose.RELAXED,
+      NONE: xb.SimulatorHandPose.RELAXED,
+    };
+  }
+  update() {
+    const gesture = handTracking.currentGesture;
+    const currentTime = performance.now();
+    if (
+      gesture !== this.lastGesture &&
+      currentTime - this.lastTriggerTime >= this.cooldownMs
+    ) {
+      this.lastGesture = gesture;
+      this.lastTriggerTime = currentTime;
+      const simulatorPose = this.gestureToSimulatorPose[gesture];
+      if (simulatorPose && xb.core.simulator?.hands) {
+        xb.core.simulator.hands.setRightHandLerpPose(simulatorPose);
+      }
+    }
+  }
+}
+const options = new xb.Options();
+options.simulator.instructions.enabled = false;
+options.enableUI();
+options.simulator.defaultMode = xb.SimulatorMode.POSE;
+options.simulator.stereo.enabled = true;
+options.xrButton.alwaysAutostartSimulator = true;
+document.addEventListener('DOMContentLoaded', () => {
+  xb.add(new GestureToSimulatorBridge());
+  xb.init(options);
+});
+</script>
+  </body>
+</html>