@map-gesture-controls/core 0.1.7 → 0.1.9

package/README.md

@@ -10,13 +10,13 @@
 > Building with OpenLayers? Use [`@map-gesture-controls/ol`](https://www.npmjs.com/package/@map-gesture-controls/ol) instead. It wraps this package and adds map integration out of the box.
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/sanderdesnaijer/map-gesture-controls/main/docs/public/openlayers-gesture-control-demo.gif" alt="Screen recording of the map gesture demo: an OpenLayers map with a small webcam preview; the user pans with a fist and zooms with two open hands, all in the browser via MediaPipe." width="720" />
+  <img src="https://raw.githubusercontent.com/sanderdesnaijer/map-gesture-controls/main/docs/public/openlayers-gesture-control-demo.gif" alt="Screen recording of the map gesture demo: an OpenLayers map with a small webcam preview; the user pans with the left fist, zooms with the right fist, and rotates with both fists, all in the browser via MediaPipe." width="720" />
 </p>
 
 ## What it does
 
 - Detects hands and classifies gestures at 30+ fps using MediaPipe Hand Landmarker
-- Recognizes **fist** (pan), **open palm** (zoom), and **idle** states
+- Recognizes **fist** and **pinch** as interchangeable triggers (left = pan, right = zoom, both = rotate)
 - Manages gesture transitions with dwell timers and grace periods to avoid flickering
 - Provides a configurable webcam overlay with corner/full/hidden display modes
 - Ships fully typed TypeScript declarations
@@ -30,18 +30,26 @@ npm install @map-gesture-controls/core
 ## Quick start
 
 ```ts
-import { GestureController } from '@map-gesture-controls/core';
+import {
+  GestureController,
+  GestureStateMachine,
+  DEFAULT_TUNING_CONFIG,
+} from '@map-gesture-controls/core';
 import '@map-gesture-controls/core/style.css';
 
-const controller = new GestureController({
-  onGestureFrame(frame) {
-    // frame.hands contains detected hands with landmarks and gesture type
-    console.log(frame);
-  },
+const tuning = DEFAULT_TUNING_CONFIG;
+const stateMachine = new GestureStateMachine(tuning);
+
+const controller = new GestureController(tuning, (frame) => {
+  const output = stateMachine.update(frame);
+  // output.mode: 'idle' | 'panning' | 'zooming' | 'rotating'
+  // output.panDelta, output.zoomDelta, output.rotateDelta
+  console.log(output);
 });
 
 // Must be called from a user interaction (button click) for webcam permission
-await controller.start();
+await controller.init();
+controller.start();
 ```
 
 ## Exports
@@ -51,7 +59,8 @@ await controller.start();
 | `GestureController` | Class | Opens the webcam, runs MediaPipe detection, and emits gesture frames |
 | `GestureStateMachine` | Class | Manages gesture state transitions with dwell and grace timers |
 | `WebcamOverlay` | Class | Renders a configurable camera preview overlay |
-| `classifyGesture` | Function | Classifies a set of hand landmarks into `fist`, `openPalm`, or `none` |
+| `classifyGesture` | Function | Stateless classifier: returns `fist`, `pinch`, `openPalm`, or `none` for a set of landmarks |
+| `createHandClassifier` | Function | Returns a stateful per-hand classifier with pinch hysteresis (use this instead of `classifyGesture` in custom pipelines) |
 | `getHandSize` | Function | Computes the bounding size of a hand from its landmarks |
 | `getTwoHandDistance` | Function | Measures the distance between two detected hands |
 | `DEFAULT_WEBCAM_CONFIG` | Constant | Default webcam overlay settings |
@@ -61,12 +70,19 @@ Full TypeScript types are exported for `GestureMode`, `GestureFrame`, `DetectedH
 
 ## Gesture recognition
 
-| Gesture | Detection rule | Use case |
+Both **fist** and **pinch** trigger the same map actions — users can use whichever is more comfortable.
+
+| Gesture | Detection rule | Map action |
 | --- | --- | --- |
-| **Fist** | One hand, 3+ fingers curled | Pan / drag |
-| **Open palm** | Two hands, all fingers extended and spread | Zoom in/out |
+| **Left fist** | Left hand, 3+ fingers curled | Pan / drag |
+| **Left pinch** | Left hand, thumb and index tip within 25% of hand size (exits at 35%) | Pan / drag |
+| **Right fist** | Right hand, 3+ fingers curled | Zoom (move up = in, down = out) |
+| **Right pinch** | Right hand, thumb and index tip within 25% of hand size (exits at 35%) | Zoom (move up = in, down = out) |
+| **Both hands active** | Both hands fist or pinch (mixed is fine) | Rotate map |
 | **Idle** | Anything else | No action |
 
+Pinch detection uses hysteresis: the gesture is entered at 25% of hand size and held until fingers open beyond 35%. This prevents flickering when fingers hover near the threshold during a held pinch.
+
 Gestures are confirmed after a configurable dwell period (default 80 ms) and held through a grace period (default 150 ms) to prevent flickering when tracking briefly drops.
 
 ## Use cases

package/dist/GestureController.d.ts

@@ -15,6 +15,8 @@ export declare class GestureController {
     private onFrame;
     private tuning;
     private lastVideoTime;
+    private leftClassifier;
+    private rightClassifier;
     constructor(tuning: TuningConfig, onFrame: FrameCallback);
     /**
      * Initialise MediaPipe and request webcam access.

package/dist/GestureController.js

@@ -1,5 +1,5 @@
 import { FilesetResolver } from '@mediapipe/tasks-vision';
-import { classifyGesture } from './gestureClassifier.js';
+import { createHandClassifier } from './gestureClassifier.js';
 import { MEDIAPIPE_WASM_URL } from './constants.js';
 /**
  * GestureController
@@ -57,6 +57,19 @@ export class GestureController {
             writable: true,
             value: -1
         });
+        // One stateful classifier per hand label; persists pinch hysteresis across frames.
+        Object.defineProperty(this, "leftClassifier", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: createHandClassifier()
+        });
+        Object.defineProperty(this, "rightClassifier", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: createHandClassifier()
+        });
         this.tuning = tuning;
         this.onFrame = onFrame;
     }
@@ -146,7 +159,8 @@ export class GestureController {
             const rawLabel = handednessArr?.[0]?.categoryName;
             const label = rawLabel === 'Left' ? 'Left' : 'Right';
             const score = handednessArr?.[0]?.score ?? 0;
-            const gesture = classifyGesture(landmarks);
+            const classify = label === 'Left' ? this.leftClassifier : this.rightClassifier;
+            const gesture = classify(landmarks);
             return { handedness: label, score, landmarks, gesture };
         });
         const leftHand = hands.find((h) => h.handedness === 'Left') ?? null;

package/dist/GestureStateMachine.d.ts

@@ -3,19 +3,24 @@ export interface StateMachineOutput {
     mode: GestureMode;
     panDelta: SmoothedPoint | null;
     zoomDelta: number | null;
+    rotateDelta: number | null;
 }
 /**
- * GestureStateMachine: 3-state FSM
+ * GestureStateMachine: 4-state FSM
  *
  * Priority rules (evaluated every frame):
- *   both hands visible AND both open palm → desired = 'zooming'
- *   one hand visible AND gesture = 'fist' → desired = 'panning'
- *   otherwise                             → desired = 'idle'
+ *   both hands fist                            → desired = 'rotating'
+ *     angle of the wrist-to-wrist line; delta rotates the map
+ *   right hand fist (left absent or not fist)  → desired = 'zooming'
+ *     vertical motion of right wrist controls zoom (up = in, down = out)
+ *   left hand fist (right absent or not fist)  → desired = 'panning'
+ *     horizontal and vertical motion of left wrist pans the map
+ *   otherwise                                  → desired = 'idle'
  *
  * Transitions:
- *   idle → panning/zooming : desired stable for actionDwellMs
- *   panning/zooming → idle : desired changes, grace period releaseGraceMs,
- *                            then idle (next frame starts new dwell if needed)
+ *   idle → any active : desired stable for actionDwellMs
+ *   active → idle     : desired changes, grace period releaseGraceMs,
+ *                       then idle (next frame starts new dwell if needed)
  */
 export declare class GestureStateMachine {
     private tuning;
@@ -26,6 +31,11 @@ export declare class GestureStateMachine {
     private prevPanPos;
     private zoomSmoother;
     private prevZoomDist;
+    private rotateSmoother;
+    private prevRotateAngle;
+    private leftActiveFrames;
+    private rightActiveFrames;
+    private static readonly ESCALATION_FRAMES;
     constructor(tuning: TuningConfig);
     getMode(): GestureMode;
     update(frame: GestureFrame): StateMachineOutput;

package/dist/GestureStateMachine.js

@@ -1,4 +1,3 @@
-import { getTwoHandDistance } from './gestureClassifier.js';
 /**
  * Exponential Moving Average smoother for 2D points.
  */
@@ -65,17 +64,21 @@ class EMAScalar {
     }
 }
 /**
- * GestureStateMachine: 3-state FSM
+ * GestureStateMachine: 4-state FSM
  *
  * Priority rules (evaluated every frame):
- *   both hands visible AND both open palm → desired = 'zooming'
- *   one hand visible AND gesture = 'fist' → desired = 'panning'
- *   otherwise                             → desired = 'idle'
+ *   both hands fist                            → desired = 'rotating'
+ *     angle of the wrist-to-wrist line; delta rotates the map
+ *   right hand fist (left absent or not fist)  → desired = 'zooming'
+ *     vertical motion of right wrist controls zoom (up = in, down = out)
+ *   left hand fist (right absent or not fist)  → desired = 'panning'
+ *     horizontal and vertical motion of left wrist pans the map
+ *   otherwise                                  → desired = 'idle'
  *
  * Transitions:
- *   idle → panning/zooming : desired stable for actionDwellMs
- *   panning/zooming → idle : desired changes, grace period releaseGraceMs,
- *                            then idle (next frame starts new dwell if needed)
+ *   idle → any active : desired stable for actionDwellMs
+ *   active → idle     : desired changes, grace period releaseGraceMs,
+ *                       then idle (next frame starts new dwell if needed)
  */
 export class GestureStateMachine {
     constructor(tuning) {
@@ -127,8 +130,37 @@ export class GestureStateMachine {
             writable: true,
             value: null
         });
+        Object.defineProperty(this, "rotateSmoother", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "prevRotateAngle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        // Tracks how many consecutive frames each hand has been active.
+        // Used to require the secondary hand to be stable before escalating
+        // the mode (e.g. pan → rotate), preventing a single noisy frame from
+        // interrupting an ongoing single-hand gesture.
+        Object.defineProperty(this, "leftActiveFrames", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "rightActiveFrames", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
         this.panSmoother = new EMAPoint(tuning.smoothingAlpha);
         this.zoomSmoother = new EMAScalar(tuning.smoothingAlpha);
+        this.rotateSmoother = new EMAScalar(tuning.smoothingAlpha);
     }
     getMode() {
         return this.mode;
@@ -138,13 +170,35 @@ export class GestureStateMachine {
         const { actionDwellMs, releaseGraceMs } = this.tuning;
         const { leftHand, rightHand } = frame;
         // ── Determine desired mode for this frame ─────────────────────────────────
-        const bothOpen = leftHand !== null &&
-            rightHand !== null &&
-            leftHand.gesture === 'openPalm' &&
-            rightHand.gesture === 'openPalm';
-        const oneFist = (leftHand !== null && leftHand.gesture === 'fist' && rightHand === null) ||
-            (rightHand !== null && rightHand.gesture === 'fist' && leftHand === null);
-        const desired = bothOpen ? 'zooming' : oneFist ? 'panning' : 'idle';
+        // Both fist and pinch trigger the same modes — users can choose either.
+        const isActive = (hand) => hand !== null && (hand.gesture === 'fist' || hand.gesture === 'pinch');
+        const rightActive = isActive(rightHand);
+        const leftActive = isActive(leftHand);
+        // Track consecutive active frames per hand. Used to guard against a single
+        // noisy frame from the secondary hand escalating (or interrupting) an
+        // ongoing single-hand gesture. Counts reset to 0 the moment a hand drops.
+        this.leftActiveFrames = leftActive ? this.leftActiveFrames + 1 : 0;
+        this.rightActiveFrames = rightActive ? this.rightActiveFrames + 1 : 0;
+        // Escalation to 'rotating' requires both hands to have been active for at
+        // least ESCALATION_FRAMES consecutive frames. This prevents one brief noisy
+        // frame on the secondary hand from interrupting an ongoing pan or zoom.
+        const bothStable = this.leftActiveFrames >= GestureStateMachine.ESCALATION_FRAMES &&
+            this.rightActiveFrames >= GestureStateMachine.ESCALATION_FRAMES;
+        // Both stable → rotate; right only → zoom; left only → pan.
+        // When both hands are active but not yet stable, we preserve the current
+        // single-hand mode (panning/zooming) so the secondary hand must be held
+        // for ESCALATION_FRAMES before rotating kicks in. This prevents a single
+        // noisy frame from the secondary hand from interrupting an ongoing gesture.
+        const bothActiveButUnstable = leftActive && rightActive && !bothStable;
+        const desired = bothStable
+            ? 'rotating'
+            : bothActiveButUnstable && (this.mode === 'panning' || this.mode === 'zooming')
+                ? this.mode // hold current mode until secondary hand is confirmed stable
+                : rightActive
+                    ? 'zooming'
+                    : leftActive
+                        ? 'panning'
+                        : 'idle';
         // ── idle ──────────────────────────────────────────────────────────────────
         if (this.mode === 'idle') {
             if (desired !== 'idle') {
@@ -158,10 +212,15 @@ export class GestureStateMachine {
             else {
                 this.actionDwell = null;
             }
-            return this.buildOutput(null, null);
+            return this.buildOutput(null, null, null);
         }
         // ── panning ───────────────────────────────────────────────────────────────
         if (this.mode === 'panning') {
+            // Escalate to rotating once the right hand becomes stably active too.
+            if (bothStable) {
+                this.transitionTo('rotating');
+                return this.buildOutput(null, null, null);
+            }
             if (desired !== 'panning') {
                 if (this.releaseTimer === null) {
                     this.releaseTimer = now;
@@ -169,13 +228,13 @@ export class GestureStateMachine {
                 else if (now - this.releaseTimer >= releaseGraceMs) {
                     this.transitionTo('idle');
                 }
-                return this.buildOutput(null, null);
+                return this.buildOutput(null, null, null);
             }
             this.releaseTimer = null;
-            const fistHand = leftHand?.gesture === 'fist' ? leftHand : rightHand;
+            const fistHand = isActive(leftHand) ? leftHand : null;
             if (!fistHand) {
                 this.transitionTo('idle');
-                return this.buildOutput(null, null);
+                return this.buildOutput(null, null, null);
             }
             const wrist = fistHand.landmarks[0];
             const smooth = this.panSmoother.update(wrist.x, wrist.y);
@@ -189,10 +248,15 @@ export class GestureStateMachine {
                 }
             }
             this.prevPanPos = smooth;
-            return this.buildOutput(panDelta, null);
+            return this.buildOutput(panDelta, null, null);
         }
         // ── zooming ───────────────────────────────────────────────────────────────
         if (this.mode === 'zooming') {
+            // Escalate to rotating once the left hand becomes stably active too.
+            if (bothStable) {
+                this.transitionTo('rotating');
+                return this.buildOutput(null, null, null);
+            }
             if (desired !== 'zooming') {
                 if (this.releaseTimer === null) {
                     this.releaseTimer = now;
@@ -200,31 +264,76 @@ export class GestureStateMachine {
                 else if (now - this.releaseTimer >= releaseGraceMs) {
                     this.transitionTo('idle');
                 }
-                return this.buildOutput(null, null);
+                return this.buildOutput(null, null, null);
             }
             this.releaseTimer = null;
-            if (!leftHand || !rightHand) {
+            if (!rightHand) {
                 this.transitionTo('idle');
-                return this.buildOutput(null, null);
+                return this.buildOutput(null, null, null);
             }
-            const rawDist = getTwoHandDistance(leftHand.landmarks, rightHand.landmarks);
-            const smoothDist = this.zoomSmoother.update(rawDist);
+            // Use right wrist vertical position: moving up (lower y) = zoom in, down = zoom out
+            const wrist = rightHand.landmarks[0];
+            const smoothY = this.zoomSmoother.update(wrist.y);
             let zoomDelta = null;
             if (this.prevZoomDist !== null) {
-                const delta = smoothDist - this.prevZoomDist;
+                const delta = smoothY - this.prevZoomDist;
                 if (Math.abs(delta) > this.tuning.zoomDeadzoneRatio) {
-                    zoomDelta = delta;
+                    // Negate: moving hand up (y decreases) → zoom in (positive delta)
+                    zoomDelta = -delta;
+                }
+            }
+            this.prevZoomDist = smoothY;
+            return this.buildOutput(null, zoomDelta, null);
+        }
+        // ── rotating ─────────────────────────────────────────────────────────────
+        if (this.mode === 'rotating') {
+            if (desired !== 'rotating') {
+                if (this.releaseTimer === null) {
+                    this.releaseTimer = now;
+                }
+                else if (now - this.releaseTimer >= releaseGraceMs) {
+                    this.transitionTo('idle');
                 }
+                return this.buildOutput(null, null, null);
             }
-            this.prevZoomDist = smoothDist;
-            return this.buildOutput(null, zoomDelta);
+            this.releaseTimer = null;
+            if (!leftHand || !rightHand) {
+                this.transitionTo('idle');
+                return this.buildOutput(null, null, null);
+            }
+            // Angle of the line from left wrist to right wrist (in radians)
+            const lw = leftHand.landmarks[0];
+            const rw = rightHand.landmarks[0];
+            const rawAngle = Math.atan2(rw.y - lw.y, rw.x - lw.x);
+            const smoothAngle = this.rotateSmoother.update(rawAngle);
+            let rotateDelta = null;
+            if (this.prevRotateAngle !== null) {
+                // Wrap the delta to [-π, π] to handle the atan2 discontinuity
+                let delta = smoothAngle - this.prevRotateAngle;
+                if (delta > Math.PI)
+                    delta -= 2 * Math.PI;
+                if (delta < -Math.PI)
+                    delta += 2 * Math.PI;
+                if (Math.abs(delta) > 0.005) {
+                    rotateDelta = delta;
+                }
+            }
+            this.prevRotateAngle = smoothAngle;
+            return this.buildOutput(null, null, rotateDelta);
         }
-        return this.buildOutput(null, null);
+        return this.buildOutput(null, null, null);
     }
     transitionTo(next) {
         this.mode = next;
         this.releaseTimer = null;
         this.actionDwell = null;
+        // Reset both counters so escalation requires a fresh stable run in the new mode.
+        // Exception: keep the dominant hand's counter when entering a single-hand mode so
+        // it does not need to re-accumulate from zero if the hand was already stable.
+        if (next !== 'panning' && next !== 'rotating')
+            this.leftActiveFrames = 0;
+        if (next !== 'zooming' && next !== 'rotating')
+            this.rightActiveFrames = 0;
         if (next !== 'panning') {
             this.panSmoother.reset();
             this.prevPanPos = null;
@@ -233,11 +342,21 @@ export class GestureStateMachine {
             this.zoomSmoother.reset();
             this.prevZoomDist = null;
         }
+        if (next !== 'rotating') {
+            this.rotateSmoother.reset();
+            this.prevRotateAngle = null;
+        }
     }
-    buildOutput(panDelta, zoomDelta) {
-        return { mode: this.mode, panDelta, zoomDelta };
+    buildOutput(panDelta, zoomDelta, rotateDelta) {
+        return { mode: this.mode, panDelta, zoomDelta, rotateDelta };
     }
     reset() {
         this.transitionTo('idle');
     }
 }
+Object.defineProperty(GestureStateMachine, "ESCALATION_FRAMES", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 3
+});