@map-gesture-controls/core 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -12
- package/dist/GestureStateMachine.d.ts +14 -7
- package/dist/GestureStateMachine.js +94 -32
- package/dist/GestureStateMachine.test.js +92 -23
- package/dist/WebcamOverlay.js +1 -0
- package/dist/constants.d.ts +1 -0
- package/dist/constants.js +1 -0
- package/dist/index.js +159 -142
- package/dist/types.d.ts +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -10,13 +10,13 @@
|
|
|
10
10
|
> Building with OpenLayers? Use [`@map-gesture-controls/ol`](https://www.npmjs.com/package/@map-gesture-controls/ol) instead. It wraps this package and adds map integration out of the box.
|
|
11
11
|
|
|
12
12
|
<p align="center">
|
|
13
|
-
<img src="https://raw.githubusercontent.com/sanderdesnaijer/map-gesture-controls/main/docs/public/openlayers-gesture-control-demo.gif" alt="Screen recording of the map gesture demo: an OpenLayers map with a small webcam preview; the user pans with
|
|
13
|
+
<img src="https://raw.githubusercontent.com/sanderdesnaijer/map-gesture-controls/main/docs/public/openlayers-gesture-control-demo.gif" alt="Screen recording of the map gesture demo: an OpenLayers map with a small webcam preview; the user pans with the left fist, zooms with the right fist, and rotates with both fists, all in the browser via MediaPipe." width="720" />
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
16
|
## What it does
|
|
17
17
|
|
|
18
18
|
- Detects hands and classifies gestures at 30+ fps using MediaPipe Hand Landmarker
|
|
19
|
-
- Recognizes **fist** (pan
|
|
19
|
+
- Recognizes **fist** (left = pan, right = zoom, both = rotate) and **idle** states
|
|
20
20
|
- Manages gesture transitions with dwell timers and grace periods to avoid flickering
|
|
21
21
|
- Provides a configurable webcam overlay with corner/full/hidden display modes
|
|
22
22
|
- Ships fully typed TypeScript declarations
|
|
@@ -30,18 +30,26 @@ npm install @map-gesture-controls/core
|
|
|
30
30
|
## Quick start
|
|
31
31
|
|
|
32
32
|
```ts
|
|
33
|
-
import {
|
|
33
|
+
import {
|
|
34
|
+
GestureController,
|
|
35
|
+
GestureStateMachine,
|
|
36
|
+
DEFAULT_TUNING_CONFIG,
|
|
37
|
+
} from '@map-gesture-controls/core';
|
|
34
38
|
import '@map-gesture-controls/core/style.css';
|
|
35
39
|
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
const tuning = DEFAULT_TUNING_CONFIG;
|
|
41
|
+
const stateMachine = new GestureStateMachine(tuning);
|
|
42
|
+
|
|
43
|
+
const controller = new GestureController(tuning, (frame) => {
|
|
44
|
+
const output = stateMachine.update(frame);
|
|
45
|
+
// output.mode: 'idle' | 'panning' | 'zooming' | 'rotating'
|
|
46
|
+
// output.panDelta, output.zoomDelta, output.rotateDelta
|
|
47
|
+
console.log(output);
|
|
41
48
|
});
|
|
42
49
|
|
|
43
50
|
// Must be called from a user interaction (button click) for webcam permission
|
|
44
|
-
await controller.
|
|
51
|
+
await controller.init();
|
|
52
|
+
controller.start();
|
|
45
53
|
```
|
|
46
54
|
|
|
47
55
|
## Exports
|
|
@@ -51,7 +59,7 @@ await controller.start();
|
|
|
51
59
|
| `GestureController` | Class | Opens the webcam, runs MediaPipe detection, and emits gesture frames |
|
|
52
60
|
| `GestureStateMachine` | Class | Manages gesture state transitions with dwell and grace timers |
|
|
53
61
|
| `WebcamOverlay` | Class | Renders a configurable camera preview overlay |
|
|
54
|
-
| `classifyGesture` | Function | Classifies a set of hand landmarks into `fist
|
|
62
|
+
| `classifyGesture` | Function | Classifies a set of hand landmarks into `fist` or `none` |
|
|
55
63
|
| `getHandSize` | Function | Computes the bounding size of a hand from its landmarks |
|
|
56
64
|
| `getTwoHandDistance` | Function | Measures the distance between two detected hands |
|
|
57
65
|
| `DEFAULT_WEBCAM_CONFIG` | Constant | Default webcam overlay settings |
|
|
@@ -63,8 +71,9 @@ Full TypeScript types are exported for `GestureMode`, `GestureFrame`, `DetectedH
|
|
|
63
71
|
|
|
64
72
|
| Gesture | Detection rule | Use case |
|
|
65
73
|
| --- | --- | --- |
|
|
66
|
-
| **
|
|
67
|
-
| **
|
|
74
|
+
| **Left fist** | Left hand, 3+ fingers curled | Pan / drag |
|
|
75
|
+
| **Right fist** | Right hand, 3+ fingers curled | Zoom in/out (vertical movement) |
|
|
76
|
+
| **Both fists** | Both hands, 3+ fingers curled each | Rotate map |
|
|
68
77
|
| **Idle** | Anything else | No action |
|
|
69
78
|
|
|
70
79
|
Gestures are confirmed after a configurable dwell period (default 80 ms) and held through a grace period (default 150 ms) to prevent flickering when tracking briefly drops.
|
|
@@ -88,6 +97,8 @@ Full docs, live demos, and configuration reference at **[sanderdesnaijer.github.
|
|
|
88
97
|
|
|
89
98
|
MediaPipe WASM and the hand landmarker model are loaded from public CDNs. No video frames are sent to any server. All gesture processing happens locally in the browser.
|
|
90
99
|
|
|
100
|
+
Built by [Sander de Snaijer](https://www.sanderdesnaijer.com).
|
|
101
|
+
|
|
91
102
|
## License
|
|
92
103
|
|
|
93
104
|
MIT
|
|
@@ -3,19 +3,24 @@ export interface StateMachineOutput {
|
|
|
3
3
|
mode: GestureMode;
|
|
4
4
|
panDelta: SmoothedPoint | null;
|
|
5
5
|
zoomDelta: number | null;
|
|
6
|
+
rotateDelta: number | null;
|
|
6
7
|
}
|
|
7
8
|
/**
|
|
8
|
-
* GestureStateMachine:
|
|
9
|
+
* GestureStateMachine: 4-state FSM
|
|
9
10
|
*
|
|
10
11
|
* Priority rules (evaluated every frame):
|
|
11
|
-
* both hands
|
|
12
|
-
*
|
|
13
|
-
*
|
|
12
|
+
* both hands fist → desired = 'rotating'
|
|
13
|
+
* angle of the wrist-to-wrist line; delta rotates the map
|
|
14
|
+
* right hand fist (left absent or not fist) → desired = 'zooming'
|
|
15
|
+
* vertical motion of right wrist controls zoom (up = in, down = out)
|
|
16
|
+
* left hand fist (right absent or not fist) → desired = 'panning'
|
|
17
|
+
* horizontal and vertical motion of left wrist pans the map
|
|
18
|
+
* otherwise → desired = 'idle'
|
|
14
19
|
*
|
|
15
20
|
* Transitions:
|
|
16
|
-
* idle →
|
|
17
|
-
*
|
|
18
|
-
*
|
|
21
|
+
* idle → any active : desired stable for actionDwellMs
|
|
22
|
+
* active → idle : desired changes, grace period releaseGraceMs,
|
|
23
|
+
* then idle (next frame starts new dwell if needed)
|
|
19
24
|
*/
|
|
20
25
|
export declare class GestureStateMachine {
|
|
21
26
|
private tuning;
|
|
@@ -26,6 +31,8 @@ export declare class GestureStateMachine {
|
|
|
26
31
|
private prevPanPos;
|
|
27
32
|
private zoomSmoother;
|
|
28
33
|
private prevZoomDist;
|
|
34
|
+
private rotateSmoother;
|
|
35
|
+
private prevRotateAngle;
|
|
29
36
|
constructor(tuning: TuningConfig);
|
|
30
37
|
getMode(): GestureMode;
|
|
31
38
|
update(frame: GestureFrame): StateMachineOutput;
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { getTwoHandDistance } from './gestureClassifier.js';
|
|
2
1
|
/**
|
|
3
2
|
* Exponential Moving Average smoother for 2D points.
|
|
4
3
|
*/
|
|
@@ -65,17 +64,21 @@ class EMAScalar {
|
|
|
65
64
|
}
|
|
66
65
|
}
|
|
67
66
|
/**
|
|
68
|
-
* GestureStateMachine:
|
|
67
|
+
* GestureStateMachine: 4-state FSM
|
|
69
68
|
*
|
|
70
69
|
* Priority rules (evaluated every frame):
|
|
71
|
-
* both hands
|
|
72
|
-
*
|
|
73
|
-
*
|
|
70
|
+
* both hands fist → desired = 'rotating'
|
|
71
|
+
* angle of the wrist-to-wrist line; delta rotates the map
|
|
72
|
+
* right hand fist (left absent or not fist) → desired = 'zooming'
|
|
73
|
+
* vertical motion of right wrist controls zoom (up = in, down = out)
|
|
74
|
+
* left hand fist (right absent or not fist) → desired = 'panning'
|
|
75
|
+
* horizontal and vertical motion of left wrist pans the map
|
|
76
|
+
* otherwise → desired = 'idle'
|
|
74
77
|
*
|
|
75
78
|
* Transitions:
|
|
76
|
-
* idle →
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
+
* idle → any active : desired stable for actionDwellMs
|
|
80
|
+
* active → idle : desired changes, grace period releaseGraceMs,
|
|
81
|
+
* then idle (next frame starts new dwell if needed)
|
|
79
82
|
*/
|
|
80
83
|
export class GestureStateMachine {
|
|
81
84
|
constructor(tuning) {
|
|
@@ -127,8 +130,21 @@ export class GestureStateMachine {
|
|
|
127
130
|
writable: true,
|
|
128
131
|
value: null
|
|
129
132
|
});
|
|
133
|
+
Object.defineProperty(this, "rotateSmoother", {
|
|
134
|
+
enumerable: true,
|
|
135
|
+
configurable: true,
|
|
136
|
+
writable: true,
|
|
137
|
+
value: void 0
|
|
138
|
+
});
|
|
139
|
+
Object.defineProperty(this, "prevRotateAngle", {
|
|
140
|
+
enumerable: true,
|
|
141
|
+
configurable: true,
|
|
142
|
+
writable: true,
|
|
143
|
+
value: null
|
|
144
|
+
});
|
|
130
145
|
this.panSmoother = new EMAPoint(tuning.smoothingAlpha);
|
|
131
146
|
this.zoomSmoother = new EMAScalar(tuning.smoothingAlpha);
|
|
147
|
+
this.rotateSmoother = new EMAScalar(tuning.smoothingAlpha);
|
|
132
148
|
}
|
|
133
149
|
getMode() {
|
|
134
150
|
return this.mode;
|
|
@@ -138,13 +154,17 @@ export class GestureStateMachine {
|
|
|
138
154
|
const { actionDwellMs, releaseGraceMs } = this.tuning;
|
|
139
155
|
const { leftHand, rightHand } = frame;
|
|
140
156
|
// ── Determine desired mode for this frame ─────────────────────────────────
|
|
141
|
-
const
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
157
|
+
const rightFist = rightHand !== null && rightHand.gesture === 'fist';
|
|
158
|
+
const leftFist = leftHand !== null && leftHand.gesture === 'fist';
|
|
159
|
+
const bothFists = rightFist && leftFist;
|
|
160
|
+
// Both fists → rotate; right only → zoom; left only → pan
|
|
161
|
+
const desired = bothFists
|
|
162
|
+
? 'rotating'
|
|
163
|
+
: rightFist
|
|
164
|
+
? 'zooming'
|
|
165
|
+
: leftFist
|
|
166
|
+
? 'panning'
|
|
167
|
+
: 'idle';
|
|
148
168
|
// ── idle ──────────────────────────────────────────────────────────────────
|
|
149
169
|
if (this.mode === 'idle') {
|
|
150
170
|
if (desired !== 'idle') {
|
|
@@ -158,7 +178,7 @@ export class GestureStateMachine {
|
|
|
158
178
|
else {
|
|
159
179
|
this.actionDwell = null;
|
|
160
180
|
}
|
|
161
|
-
return this.buildOutput(null, null);
|
|
181
|
+
return this.buildOutput(null, null, null);
|
|
162
182
|
}
|
|
163
183
|
// ── panning ───────────────────────────────────────────────────────────────
|
|
164
184
|
if (this.mode === 'panning') {
|
|
@@ -169,13 +189,13 @@ export class GestureStateMachine {
|
|
|
169
189
|
else if (now - this.releaseTimer >= releaseGraceMs) {
|
|
170
190
|
this.transitionTo('idle');
|
|
171
191
|
}
|
|
172
|
-
return this.buildOutput(null, null);
|
|
192
|
+
return this.buildOutput(null, null, null);
|
|
173
193
|
}
|
|
174
194
|
this.releaseTimer = null;
|
|
175
|
-
const fistHand = leftHand?.gesture === 'fist' ? leftHand :
|
|
195
|
+
const fistHand = leftHand?.gesture === 'fist' ? leftHand : null;
|
|
176
196
|
if (!fistHand) {
|
|
177
197
|
this.transitionTo('idle');
|
|
178
|
-
return this.buildOutput(null, null);
|
|
198
|
+
return this.buildOutput(null, null, null);
|
|
179
199
|
}
|
|
180
200
|
const wrist = fistHand.landmarks[0];
|
|
181
201
|
const smooth = this.panSmoother.update(wrist.x, wrist.y);
|
|
@@ -189,7 +209,7 @@ export class GestureStateMachine {
|
|
|
189
209
|
}
|
|
190
210
|
}
|
|
191
211
|
this.prevPanPos = smooth;
|
|
192
|
-
return this.buildOutput(panDelta, null);
|
|
212
|
+
return this.buildOutput(panDelta, null, null);
|
|
193
213
|
}
|
|
194
214
|
// ── zooming ───────────────────────────────────────────────────────────────
|
|
195
215
|
if (this.mode === 'zooming') {
|
|
@@ -200,26 +220,64 @@ export class GestureStateMachine {
|
|
|
200
220
|
else if (now - this.releaseTimer >= releaseGraceMs) {
|
|
201
221
|
this.transitionTo('idle');
|
|
202
222
|
}
|
|
203
|
-
return this.buildOutput(null, null);
|
|
223
|
+
return this.buildOutput(null, null, null);
|
|
204
224
|
}
|
|
205
225
|
this.releaseTimer = null;
|
|
206
|
-
if (!
|
|
226
|
+
if (!rightHand) {
|
|
207
227
|
this.transitionTo('idle');
|
|
208
|
-
return this.buildOutput(null, null);
|
|
228
|
+
return this.buildOutput(null, null, null);
|
|
209
229
|
}
|
|
210
|
-
|
|
211
|
-
const
|
|
230
|
+
// Use right wrist vertical position: moving up (lower y) = zoom in, down = zoom out
|
|
231
|
+
const wrist = rightHand.landmarks[0];
|
|
232
|
+
const smoothY = this.zoomSmoother.update(wrist.y);
|
|
212
233
|
let zoomDelta = null;
|
|
213
234
|
if (this.prevZoomDist !== null) {
|
|
214
|
-
const delta =
|
|
235
|
+
const delta = smoothY - this.prevZoomDist;
|
|
215
236
|
if (Math.abs(delta) > this.tuning.zoomDeadzoneRatio) {
|
|
216
|
-
|
|
237
|
+
// Negate: moving hand up (y decreases) → zoom in (positive delta)
|
|
238
|
+
zoomDelta = -delta;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
this.prevZoomDist = smoothY;
|
|
242
|
+
return this.buildOutput(null, zoomDelta, null);
|
|
243
|
+
}
|
|
244
|
+
// ── rotating ─────────────────────────────────────────────────────────────
|
|
245
|
+
if (this.mode === 'rotating') {
|
|
246
|
+
if (desired !== 'rotating') {
|
|
247
|
+
if (this.releaseTimer === null) {
|
|
248
|
+
this.releaseTimer = now;
|
|
217
249
|
}
|
|
250
|
+
else if (now - this.releaseTimer >= releaseGraceMs) {
|
|
251
|
+
this.transitionTo('idle');
|
|
252
|
+
}
|
|
253
|
+
return this.buildOutput(null, null, null);
|
|
218
254
|
}
|
|
219
|
-
this.
|
|
220
|
-
|
|
255
|
+
this.releaseTimer = null;
|
|
256
|
+
if (!leftHand || !rightHand) {
|
|
257
|
+
this.transitionTo('idle');
|
|
258
|
+
return this.buildOutput(null, null, null);
|
|
259
|
+
}
|
|
260
|
+
// Angle of the line from left wrist to right wrist (in radians)
|
|
261
|
+
const lw = leftHand.landmarks[0];
|
|
262
|
+
const rw = rightHand.landmarks[0];
|
|
263
|
+
const rawAngle = Math.atan2(rw.y - lw.y, rw.x - lw.x);
|
|
264
|
+
const smoothAngle = this.rotateSmoother.update(rawAngle);
|
|
265
|
+
let rotateDelta = null;
|
|
266
|
+
if (this.prevRotateAngle !== null) {
|
|
267
|
+
// Wrap the delta to [-π, π] to handle the atan2 discontinuity
|
|
268
|
+
let delta = smoothAngle - this.prevRotateAngle;
|
|
269
|
+
if (delta > Math.PI)
|
|
270
|
+
delta -= 2 * Math.PI;
|
|
271
|
+
if (delta < -Math.PI)
|
|
272
|
+
delta += 2 * Math.PI;
|
|
273
|
+
if (Math.abs(delta) > 0.005) {
|
|
274
|
+
rotateDelta = delta;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
this.prevRotateAngle = smoothAngle;
|
|
278
|
+
return this.buildOutput(null, null, rotateDelta);
|
|
221
279
|
}
|
|
222
|
-
return this.buildOutput(null, null);
|
|
280
|
+
return this.buildOutput(null, null, null);
|
|
223
281
|
}
|
|
224
282
|
transitionTo(next) {
|
|
225
283
|
this.mode = next;
|
|
@@ -233,9 +291,13 @@ export class GestureStateMachine {
|
|
|
233
291
|
this.zoomSmoother.reset();
|
|
234
292
|
this.prevZoomDist = null;
|
|
235
293
|
}
|
|
294
|
+
if (next !== 'rotating') {
|
|
295
|
+
this.rotateSmoother.reset();
|
|
296
|
+
this.prevRotateAngle = null;
|
|
297
|
+
}
|
|
236
298
|
}
|
|
237
|
-
buildOutput(panDelta, zoomDelta) {
|
|
238
|
-
return { mode: this.mode, panDelta, zoomDelta };
|
|
299
|
+
buildOutput(panDelta, zoomDelta, rotateDelta) {
|
|
300
|
+
return { mode: this.mode, panDelta, zoomDelta, rotateDelta };
|
|
239
301
|
}
|
|
240
302
|
reset() {
|
|
241
303
|
this.transitionTo('idle');
|
|
@@ -28,6 +28,7 @@ function makeFrame(timestamp, leftHand, rightHand) {
|
|
|
28
28
|
}
|
|
29
29
|
/**
|
|
30
30
|
* Drive the FSM into 'panning' with dwell=0 (requires two identical frames).
|
|
31
|
+
* Pan = left fist only.
|
|
31
32
|
*/
|
|
32
33
|
function enterPanning(fsm, hand = makeHand('fist')) {
|
|
33
34
|
fsm.update(makeFrame(0, hand, null)); // starts dwell timer
|
|
@@ -35,8 +36,17 @@ function enterPanning(fsm, hand = makeHand('fist')) {
|
|
|
35
36
|
}
|
|
36
37
|
/**
|
|
37
38
|
* Drive the FSM into 'zooming' with dwell=0 (requires two identical frames).
|
|
39
|
+
* Zoom = right fist only.
|
|
38
40
|
*/
|
|
39
|
-
function enterZooming(fsm,
|
|
41
|
+
function enterZooming(fsm, right = makeHand('fist')) {
|
|
42
|
+
fsm.update(makeFrame(0, null, right));
|
|
43
|
+
fsm.update(makeFrame(0, null, right));
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Drive the FSM into 'rotating' with dwell=0 (requires two identical frames).
|
|
47
|
+
* Rotate = both fists.
|
|
48
|
+
*/
|
|
49
|
+
function enterRotating(fsm, left = makeHand('fist'), right = makeHand('fist')) {
|
|
40
50
|
fsm.update(makeFrame(0, left, right));
|
|
41
51
|
fsm.update(makeFrame(0, left, right));
|
|
42
52
|
}
|
|
@@ -51,22 +61,37 @@ describe('GestureStateMachine', () => {
|
|
|
51
61
|
expect(fsm.getMode()).toBe('idle');
|
|
52
62
|
});
|
|
53
63
|
// ── idle → panning ─────────────────────────────────────────────────────────
|
|
54
|
-
it('transitions to panning
|
|
64
|
+
it('transitions to panning when left fist is held (two frames, dwell=0)', () => {
|
|
55
65
|
const fistHand = makeHand('fist');
|
|
56
66
|
fsm.update(makeFrame(0, fistHand, null)); // starts dwell
|
|
57
67
|
const out = fsm.update(makeFrame(0, fistHand, null)); // 0 >= 0 → panning
|
|
58
68
|
expect(out.mode).toBe('panning');
|
|
59
69
|
});
|
|
60
70
|
// ── idle → zooming ─────────────────────────────────────────────────────────
|
|
61
|
-
it('transitions to zooming when
|
|
62
|
-
const
|
|
63
|
-
|
|
71
|
+
it('transitions to zooming when right fist is held (two frames, dwell=0)', () => {
|
|
72
|
+
const right = makeHand('fist');
|
|
73
|
+
fsm.update(makeFrame(0, null, right));
|
|
74
|
+
const out = fsm.update(makeFrame(0, null, right));
|
|
75
|
+
expect(out.mode).toBe('zooming');
|
|
76
|
+
});
|
|
77
|
+
// ── idle → rotating ────────────────────────────────────────────────────────
|
|
78
|
+
it('transitions to rotating when both fists are held (two frames, dwell=0)', () => {
|
|
79
|
+
const left = makeHand('fist');
|
|
80
|
+
const right = makeHand('fist');
|
|
64
81
|
fsm.update(makeFrame(0, left, right));
|
|
65
82
|
const out = fsm.update(makeFrame(0, left, right));
|
|
66
|
-
expect(out.mode).toBe('
|
|
83
|
+
expect(out.mode).toBe('rotating');
|
|
84
|
+
});
|
|
85
|
+
it('prefers rotating over zooming when both fists are held', () => {
|
|
86
|
+
const left = makeHand('fist');
|
|
87
|
+
const right = makeHand('fist');
|
|
88
|
+
fsm.update(makeFrame(0, left, right));
|
|
89
|
+
const out = fsm.update(makeFrame(0, left, right));
|
|
90
|
+
// both fists = rotating, not zooming
|
|
91
|
+
expect(out.mode).toBe('rotating');
|
|
67
92
|
});
|
|
68
93
|
// ── panning → idle ─────────────────────────────────────────────────────────
|
|
69
|
-
it('returns to idle when the fist is released (releaseGraceMs=0)', () => {
|
|
94
|
+
it('returns to idle when the left fist is released (releaseGraceMs=0)', () => {
|
|
70
95
|
enterPanning(fsm);
|
|
71
96
|
expect(fsm.getMode()).toBe('panning');
|
|
72
97
|
fsm.update(makeFrame(1, null, null)); // starts release timer
|
|
@@ -74,13 +99,21 @@ describe('GestureStateMachine', () => {
|
|
|
74
99
|
expect(out.mode).toBe('idle');
|
|
75
100
|
});
|
|
76
101
|
// ── zooming → idle ─────────────────────────────────────────────────────────
|
|
77
|
-
it('returns to idle when
|
|
102
|
+
it('returns to idle when right fist is released (releaseGraceMs=0)', () => {
|
|
78
103
|
enterZooming(fsm);
|
|
79
104
|
expect(fsm.getMode()).toBe('zooming');
|
|
80
105
|
fsm.update(makeFrame(1, null, null)); // starts release timer
|
|
81
106
|
const out = fsm.update(makeFrame(1, null, null)); // 0 >= 0 → idle
|
|
82
107
|
expect(out.mode).toBe('idle');
|
|
83
108
|
});
|
|
109
|
+
// ── rotating → idle ────────────────────────────────────────────────────────
|
|
110
|
+
it('returns to idle when both fists are released (releaseGraceMs=0)', () => {
|
|
111
|
+
enterRotating(fsm);
|
|
112
|
+
expect(fsm.getMode()).toBe('rotating');
|
|
113
|
+
fsm.update(makeFrame(1, null, null)); // starts release timer
|
|
114
|
+
const out = fsm.update(makeFrame(1, null, null)); // 0 >= 0 → idle
|
|
115
|
+
expect(out.mode).toBe('idle');
|
|
116
|
+
});
|
|
84
117
|
// ── panDelta output ────────────────────────────────────────────────────────
|
|
85
118
|
it('emits no panDelta on the first panning frame (no previous position)', () => {
|
|
86
119
|
const fistHand = makeHand('fist');
|
|
@@ -108,27 +141,57 @@ describe('GestureStateMachine', () => {
|
|
|
108
141
|
});
|
|
109
142
|
// ── zoomDelta output ───────────────────────────────────────────────────────
|
|
110
143
|
it('emits no zoomDelta on the first zooming frame', () => {
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
fsm.update(makeFrame(0,
|
|
114
|
-
const out = fsm.update(makeFrame(0, left, right)); // enters zooming, sets prevZoomDist
|
|
144
|
+
const right = makeHand('fist');
|
|
145
|
+
fsm.update(makeFrame(0, null, right));
|
|
146
|
+
const out = fsm.update(makeFrame(0, null, right)); // enters zooming, sets prevZoomDist
|
|
115
147
|
expect(out.zoomDelta).toBeNull();
|
|
116
148
|
});
|
|
117
|
-
it('emits zoomDelta once a previous
|
|
118
|
-
|
|
119
|
-
const lmR1 = makeLandmarks({ [LANDMARKS.
|
|
120
|
-
const
|
|
121
|
-
const lmR2 = makeLandmarks({ [LANDMARKS.INDEX_TIP]: { x: 0.7, y: 0.5, z: 0 } });
|
|
149
|
+
it('emits zoomDelta once a previous position is established', () => {
|
|
150
|
+
// Right wrist starts lower (y=0.6), then moves up (y=0.4) → zoom in (positive delta)
|
|
151
|
+
const lmR1 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.5, y: 0.6, z: 0 } });
|
|
152
|
+
const lmR2 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.5, y: 0.4, z: 0 } });
|
|
122
153
|
// Frame 1+2: dwell + transition (zooming entered on frame 2, but output comes from idle branch)
|
|
123
|
-
fsm.update(makeFrame(0,
|
|
124
|
-
fsm.update(makeFrame(0,
|
|
154
|
+
fsm.update(makeFrame(0, null, makeHand('fist', lmR1)));
|
|
155
|
+
fsm.update(makeFrame(0, null, makeHand('fist', lmR1)));
|
|
125
156
|
// Frame 3: first real zooming frame, sets prevZoomDist, no delta yet
|
|
126
|
-
fsm.update(makeFrame(1,
|
|
127
|
-
// Frame 4:
|
|
128
|
-
const out = fsm.update(makeFrame(2,
|
|
157
|
+
fsm.update(makeFrame(1, null, makeHand('fist', lmR1)));
|
|
158
|
+
// Frame 4: hand moved up → zoom in (positive delta)
|
|
159
|
+
const out = fsm.update(makeFrame(2, null, makeHand('fist', lmR2)));
|
|
129
160
|
expect(out.mode).toBe('zooming');
|
|
130
161
|
expect(out.zoomDelta).not.toBeNull();
|
|
131
|
-
expect(out.zoomDelta).toBeGreaterThan(0);
|
|
162
|
+
expect(out.zoomDelta).toBeGreaterThan(0); // hand moved up = zoom in
|
|
163
|
+
});
|
|
164
|
+
it('emits negative zoomDelta when hand moves down (zoom out)', () => {
|
|
165
|
+
const lmR1 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.5, y: 0.4, z: 0 } });
|
|
166
|
+
const lmR2 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.5, y: 0.6, z: 0 } });
|
|
167
|
+
fsm.update(makeFrame(0, null, makeHand('fist', lmR1)));
|
|
168
|
+
fsm.update(makeFrame(0, null, makeHand('fist', lmR1)));
|
|
169
|
+
fsm.update(makeFrame(1, null, makeHand('fist', lmR1)));
|
|
170
|
+
const out = fsm.update(makeFrame(2, null, makeHand('fist', lmR2)));
|
|
171
|
+
expect(out.zoomDelta).not.toBeNull();
|
|
172
|
+
expect(out.zoomDelta).toBeLessThan(0); // hand moved down = zoom out
|
|
173
|
+
});
|
|
174
|
+
// ── rotateDelta output ─────────────────────────────────────────────────────
|
|
175
|
+
it('emits no rotateDelta on the first rotating frame', () => {
|
|
176
|
+
const left = makeHand('fist');
|
|
177
|
+
const right = makeHand('fist');
|
|
178
|
+
fsm.update(makeFrame(0, left, right));
|
|
179
|
+
const out = fsm.update(makeFrame(0, left, right)); // enters rotating, sets prevRotateAngle
|
|
180
|
+
expect(out.rotateDelta).toBeNull();
|
|
181
|
+
});
|
|
182
|
+
it('emits rotateDelta once a previous angle is established', () => {
|
|
183
|
+
// Left wrist at (0.2, 0.5), right wrist at (0.8, 0.5) → angle = 0 (horizontal)
|
|
184
|
+
const lmL1 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.2, y: 0.5, z: 0 } });
|
|
185
|
+
const lmR1 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.8, y: 0.5, z: 0 } });
|
|
186
|
+
// Tilt clockwise: right wrist drops, left wrist rises → angle increases
|
|
187
|
+
const lmL2 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.2, y: 0.4, z: 0 } });
|
|
188
|
+
const lmR2 = makeLandmarks({ [LANDMARKS.WRIST]: { x: 0.8, y: 0.6, z: 0 } });
|
|
189
|
+
fsm.update(makeFrame(0, makeHand('fist', lmL1), makeHand('fist', lmR1)));
|
|
190
|
+
fsm.update(makeFrame(0, makeHand('fist', lmL1), makeHand('fist', lmR1)));
|
|
191
|
+
fsm.update(makeFrame(1, makeHand('fist', lmL1), makeHand('fist', lmR1)));
|
|
192
|
+
const out = fsm.update(makeFrame(2, makeHand('fist', lmL2), makeHand('fist', lmR2)));
|
|
193
|
+
expect(out.mode).toBe('rotating');
|
|
194
|
+
expect(out.rotateDelta).not.toBeNull();
|
|
132
195
|
});
|
|
133
196
|
// ── reset ──────────────────────────────────────────────────────────────────
|
|
134
197
|
it('reset() returns the FSM to idle', () => {
|
|
@@ -137,6 +200,12 @@ describe('GestureStateMachine', () => {
|
|
|
137
200
|
fsm.reset();
|
|
138
201
|
expect(fsm.getMode()).toBe('idle');
|
|
139
202
|
});
|
|
203
|
+
it('reset() returns the FSM to idle from rotating', () => {
|
|
204
|
+
enterRotating(fsm);
|
|
205
|
+
expect(fsm.getMode()).toBe('rotating');
|
|
206
|
+
fsm.reset();
|
|
207
|
+
expect(fsm.getMode()).toBe('idle');
|
|
208
|
+
});
|
|
140
209
|
// ── dwell timer ────────────────────────────────────────────────────────────
|
|
141
210
|
it('does NOT transition before actionDwellMs elapses', () => {
|
|
142
211
|
const slowFsm = new GestureStateMachine({ ...FAST_TUNING, actionDwellMs: 200 });
|
package/dist/WebcamOverlay.js
CHANGED
package/dist/constants.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ export declare const COLORS: {
|
|
|
19
19
|
readonly idle: "#888888";
|
|
20
20
|
readonly panning: "#00ccff";
|
|
21
21
|
readonly zooming: "#00ffcc";
|
|
22
|
+
readonly rotating: "#ff9900";
|
|
22
23
|
readonly landmark: "rgba(255,255,255,0.6)";
|
|
23
24
|
readonly connection: "rgba(255,255,255,0.3)";
|
|
24
25
|
readonly fingertipGlow: "#4488ff";
|
package/dist/constants.js
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var
|
|
4
|
-
import { FilesetResolver as
|
|
1
|
+
var E = Object.defineProperty;
|
|
2
|
+
var M = (o, t, e) => t in o ? E(o, t, { enumerable: !0, configurable: !0, writable: !0, value: e }) : o[t] = e;
|
|
3
|
+
var l = (o, t, e) => M(o, typeof t != "symbol" ? t + "" : t, e);
|
|
4
|
+
import { FilesetResolver as C } from "@mediapipe/tasks-vision";
|
|
5
5
|
const L = {
|
|
6
6
|
enabled: !0,
|
|
7
7
|
mode: "corner",
|
|
@@ -18,7 +18,7 @@ const L = {
|
|
|
18
18
|
minDetectionConfidence: 0.65,
|
|
19
19
|
minTrackingConfidence: 0.65,
|
|
20
20
|
minPresenceConfidence: 0.6
|
|
21
|
-
},
|
|
21
|
+
}, r = {
|
|
22
22
|
WRIST: 0,
|
|
23
23
|
THUMB_TIP: 4,
|
|
24
24
|
INDEX_TIP: 8,
|
|
@@ -29,75 +29,76 @@ const L = {
|
|
|
29
29
|
RING_MCP: 13,
|
|
30
30
|
PINKY_TIP: 20,
|
|
31
31
|
PINKY_MCP: 17
|
|
32
|
-
},
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
],
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
],
|
|
32
|
+
}, P = [
|
|
33
|
+
r.INDEX_TIP,
|
|
34
|
+
r.MIDDLE_TIP,
|
|
35
|
+
r.RING_TIP,
|
|
36
|
+
r.PINKY_TIP
|
|
37
|
+
], b = [
|
|
38
|
+
r.INDEX_MCP,
|
|
39
|
+
r.MIDDLE_MCP,
|
|
40
|
+
r.RING_MCP,
|
|
41
|
+
r.PINKY_MCP
|
|
42
|
+
], y = {
|
|
43
43
|
idle: "#888888",
|
|
44
44
|
panning: "#00ccff",
|
|
45
45
|
zooming: "#00ffcc",
|
|
46
|
+
rotating: "#ff9900",
|
|
46
47
|
landmark: "rgba(255,255,255,0.6)",
|
|
47
48
|
connection: "rgba(255,255,255,0.3)",
|
|
48
49
|
fingertipGlow: "#4488ff"
|
|
49
|
-
},
|
|
50
|
-
function
|
|
51
|
-
const e = o.x - t.x,
|
|
52
|
-
return Math.sqrt(e * e +
|
|
50
|
+
}, _ = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm";
|
|
51
|
+
function T(o, t) {
|
|
52
|
+
const e = o.x - t.x, i = o.y - t.y;
|
|
53
|
+
return Math.sqrt(e * e + i * i);
|
|
53
54
|
}
|
|
54
|
-
function
|
|
55
|
-
const t = o[
|
|
56
|
-
for (let
|
|
57
|
-
const d = o[
|
|
58
|
-
if (
|
|
55
|
+
function N(o) {
|
|
56
|
+
const t = o[r.WRIST];
|
|
57
|
+
for (let n = 0; n < P.length; n++) {
|
|
58
|
+
const d = o[P[n]], c = o[b[n]];
|
|
59
|
+
if (T(d, t) < T(c, t) * 0.9)
|
|
59
60
|
return !1;
|
|
60
61
|
}
|
|
61
|
-
const e =
|
|
62
|
+
const e = x(o);
|
|
62
63
|
if (e === 0) return !1;
|
|
63
|
-
const
|
|
64
|
+
const i = P.map((n) => o[n]);
|
|
64
65
|
let s = 1 / 0;
|
|
65
|
-
for (let
|
|
66
|
-
const d =
|
|
66
|
+
for (let n = 0; n < i.length - 1; n++) {
|
|
67
|
+
const d = T(i[n], i[n + 1]);
|
|
67
68
|
d < s && (s = d);
|
|
68
69
|
}
|
|
69
70
|
return s >= e * 0.18;
|
|
70
71
|
}
|
|
71
|
-
function
|
|
72
|
-
const t = o[
|
|
72
|
+
function S(o) {
|
|
73
|
+
const t = o[r.WRIST];
|
|
73
74
|
let e = 0;
|
|
74
|
-
for (let
|
|
75
|
-
const s = o[
|
|
76
|
-
|
|
75
|
+
for (let i = 0; i < P.length; i++) {
|
|
76
|
+
const s = o[P[i]], a = o[b[i]];
|
|
77
|
+
T(s, t) < T(a, t) * 1.1 && e++;
|
|
77
78
|
}
|
|
78
79
|
return e >= 3;
|
|
79
80
|
}
|
|
80
|
-
function
|
|
81
|
-
const t = o[
|
|
82
|
-
return !t || !e ? 0 :
|
|
81
|
+
function x(o) {
|
|
82
|
+
const t = o[r.WRIST], e = o[r.MIDDLE_MCP];
|
|
83
|
+
return !t || !e ? 0 : T(t, e);
|
|
83
84
|
}
|
|
84
|
-
function
|
|
85
|
-
const e = o[
|
|
86
|
-
return !e || !
|
|
85
|
+
function k(o, t) {
|
|
86
|
+
const e = o[r.INDEX_TIP], i = t[r.INDEX_TIP];
|
|
87
|
+
return !e || !i ? 0 : T(e, i);
|
|
87
88
|
}
|
|
88
|
-
function
|
|
89
|
-
return o.length < 21 ? "none" :
|
|
89
|
+
function R(o) {
|
|
90
|
+
return o.length < 21 ? "none" : S(o) ? "fist" : N(o) ? "openPalm" : "none";
|
|
90
91
|
}
|
|
91
|
-
class
|
|
92
|
+
class B {
|
|
92
93
|
constructor(t, e) {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
94
|
+
l(this, "landmarker", null);
|
|
95
|
+
l(this, "videoEl", null);
|
|
96
|
+
l(this, "stream", null);
|
|
97
|
+
l(this, "rafHandle", null);
|
|
98
|
+
l(this, "running", !1);
|
|
99
|
+
l(this, "onFrame");
|
|
100
|
+
l(this, "tuning");
|
|
101
|
+
l(this, "lastVideoTime", -1);
|
|
101
102
|
this.tuning = t, this.onFrame = e;
|
|
102
103
|
}
|
|
103
104
|
/**
|
|
@@ -105,7 +106,7 @@ class k {
|
|
|
105
106
|
* Returns the video element so the overlay can render it.
|
|
106
107
|
*/
|
|
107
108
|
async init() {
|
|
108
|
-
const t = await
|
|
109
|
+
const t = await C.forVisionTasks(_), { HandLandmarker: e } = await import("@mediapipe/tasks-vision");
|
|
109
110
|
return this.landmarker = await e.createFromOptions(t, {
|
|
110
111
|
baseOptions: {
|
|
111
112
|
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
|
@@ -118,8 +119,8 @@ class k {
|
|
|
118
119
|
minTrackingConfidence: this.tuning.minTrackingConfidence
|
|
119
120
|
}), this.videoEl = document.createElement("video"), this.videoEl.setAttribute("playsinline", ""), this.videoEl.setAttribute("autoplay", ""), this.videoEl.muted = !0, this.videoEl.width = 640, this.videoEl.height = 480, this.stream = await navigator.mediaDevices.getUserMedia({
|
|
120
121
|
video: { width: 640, height: 480, facingMode: "user" }
|
|
121
|
-
}), this.videoEl.srcObject = this.stream, await new Promise((
|
|
122
|
-
this.videoEl.addEventListener("loadeddata", () =>
|
|
122
|
+
}), this.videoEl.srcObject = this.stream, await new Promise((i) => {
|
|
123
|
+
this.videoEl.addEventListener("loadeddata", () => i(), { once: !0 });
|
|
123
124
|
}), this.videoEl;
|
|
124
125
|
}
|
|
125
126
|
start() {
|
|
@@ -130,7 +131,7 @@ class k {
|
|
|
130
131
|
}
|
|
131
132
|
destroy() {
|
|
132
133
|
var t, e;
|
|
133
|
-
this.stop(), (t = this.stream) == null || t.getTracks().forEach((
|
|
134
|
+
this.stop(), (t = this.stream) == null || t.getTracks().forEach((i) => i.stop()), (e = this.landmarker) == null || e.close(), this.landmarker = null, this.videoEl = null, this.stream = null;
|
|
134
135
|
}
|
|
135
136
|
loop() {
|
|
136
137
|
this.running && (this.rafHandle = requestAnimationFrame(() => this.loop()), this.processFrame());
|
|
@@ -138,33 +139,33 @@ class k {
|
|
|
138
139
|
processFrame() {
|
|
139
140
|
const t = this.videoEl, e = this.landmarker;
|
|
140
141
|
if (!t || !e || t.readyState < 2) return;
|
|
141
|
-
const
|
|
142
|
+
const i = performance.now();
|
|
142
143
|
if (t.currentTime === this.lastVideoTime) return;
|
|
143
144
|
this.lastVideoTime = t.currentTime;
|
|
144
145
|
let s;
|
|
145
146
|
try {
|
|
146
|
-
s = e.detectForVideo(t,
|
|
147
|
+
s = e.detectForVideo(t, i);
|
|
147
148
|
} catch {
|
|
148
149
|
return;
|
|
149
150
|
}
|
|
150
|
-
const
|
|
151
|
-
this.onFrame(
|
|
151
|
+
const a = this.buildFrame(s, i);
|
|
152
|
+
this.onFrame(a);
|
|
152
153
|
}
|
|
153
154
|
buildFrame(t, e) {
|
|
154
|
-
const
|
|
155
|
-
var
|
|
156
|
-
const c = t.handedness[d],
|
|
157
|
-
return { handedness:
|
|
158
|
-
}), s =
|
|
159
|
-
return { timestamp: e, hands:
|
|
155
|
+
const i = t.landmarks.map((n, d) => {
|
|
156
|
+
var p, g;
|
|
157
|
+
const c = t.handedness[d], h = ((p = c == null ? void 0 : c[0]) == null ? void 0 : p.categoryName) === "Left" ? "Left" : "Right", m = ((g = c == null ? void 0 : c[0]) == null ? void 0 : g.score) ?? 0, f = R(n);
|
|
158
|
+
return { handedness: h, score: m, landmarks: n, gesture: f };
|
|
159
|
+
}), s = i.find((n) => n.handedness === "Left") ?? null, a = i.find((n) => n.handedness === "Right") ?? null;
|
|
160
|
+
return { timestamp: e, hands: i, leftHand: s, rightHand: a };
|
|
160
161
|
}
|
|
161
162
|
getVideoElement() {
|
|
162
163
|
return this.videoEl;
|
|
163
164
|
}
|
|
164
165
|
}
|
|
165
|
-
class
|
|
166
|
+
class A {
|
|
166
167
|
constructor(t) {
|
|
167
|
-
|
|
168
|
+
l(this, "value", null);
|
|
168
169
|
this.alpha = t;
|
|
169
170
|
}
|
|
170
171
|
update(t, e) {
|
|
@@ -177,9 +178,9 @@ class R {
|
|
|
177
178
|
this.value = null;
|
|
178
179
|
}
|
|
179
180
|
}
|
|
180
|
-
class
|
|
181
|
+
class D {
|
|
181
182
|
constructor(t) {
|
|
182
|
-
|
|
183
|
+
l(this, "value", null);
|
|
183
184
|
this.alpha = t;
|
|
184
185
|
}
|
|
185
186
|
update(t) {
|
|
@@ -189,65 +190,80 @@ class F {
|
|
|
189
190
|
this.value = null;
|
|
190
191
|
}
|
|
191
192
|
}
|
|
192
|
-
class
|
|
193
|
+
class U {
|
|
193
194
|
constructor(t) {
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
this
|
|
195
|
+
l(this, "mode", "idle");
|
|
196
|
+
l(this, "actionDwell", null);
|
|
197
|
+
l(this, "releaseTimer", null);
|
|
198
|
+
l(this, "panSmoother");
|
|
199
|
+
l(this, "prevPanPos", null);
|
|
200
|
+
l(this, "zoomSmoother");
|
|
201
|
+
l(this, "prevZoomDist", null);
|
|
202
|
+
l(this, "rotateSmoother");
|
|
203
|
+
l(this, "prevRotateAngle", null);
|
|
204
|
+
this.tuning = t, this.panSmoother = new A(t.smoothingAlpha), this.zoomSmoother = new D(t.smoothingAlpha), this.rotateSmoother = new D(t.smoothingAlpha);
|
|
202
205
|
}
|
|
203
206
|
getMode() {
|
|
204
207
|
return this.mode;
|
|
205
208
|
}
|
|
206
209
|
update(t) {
|
|
207
|
-
const e = t.timestamp, { actionDwellMs:
|
|
210
|
+
const e = t.timestamp, { actionDwellMs: i, releaseGraceMs: s } = this.tuning, { leftHand: a, rightHand: n } = t, d = n !== null && n.gesture === "fist", c = a !== null && a.gesture === "fist", h = d && c ? "rotating" : d ? "zooming" : c ? "panning" : "idle";
|
|
208
211
|
if (this.mode === "idle")
|
|
209
|
-
return
|
|
212
|
+
return h !== "idle" ? this.actionDwell === null || this.actionDwell.gesture !== h ? this.actionDwell = { gesture: h, startMs: e } : e - this.actionDwell.startMs >= i && this.transitionTo(h) : this.actionDwell = null, this.buildOutput(null, null, null);
|
|
210
213
|
if (this.mode === "panning") {
|
|
211
|
-
if (
|
|
212
|
-
return this.releaseTimer === null ? this.releaseTimer = e : e - this.releaseTimer >= s && this.transitionTo("idle"), this.buildOutput(null, null);
|
|
214
|
+
if (h !== "panning")
|
|
215
|
+
return this.releaseTimer === null ? this.releaseTimer = e : e - this.releaseTimer >= s && this.transitionTo("idle"), this.buildOutput(null, null, null);
|
|
213
216
|
this.releaseTimer = null;
|
|
214
|
-
const
|
|
215
|
-
if (!
|
|
216
|
-
return this.transitionTo("idle"), this.buildOutput(null, null);
|
|
217
|
-
const
|
|
218
|
-
let
|
|
217
|
+
const m = (a == null ? void 0 : a.gesture) === "fist" ? a : null;
|
|
218
|
+
if (!m)
|
|
219
|
+
return this.transitionTo("idle"), this.buildOutput(null, null, null);
|
|
220
|
+
const f = m.landmarks[0], p = this.panSmoother.update(f.x, f.y);
|
|
221
|
+
let g = null;
|
|
219
222
|
if (this.prevPanPos !== null) {
|
|
220
|
-
const
|
|
221
|
-
(Math.abs(
|
|
223
|
+
const I = p.x - this.prevPanPos.x, v = p.y - this.prevPanPos.y, w = this.tuning.panDeadzonePx / 640;
|
|
224
|
+
(Math.abs(I) > w || Math.abs(v) > w) && (g = { x: I, y: v });
|
|
222
225
|
}
|
|
223
|
-
return this.prevPanPos = p, this.buildOutput(
|
|
226
|
+
return this.prevPanPos = p, this.buildOutput(g, null, null);
|
|
224
227
|
}
|
|
225
228
|
if (this.mode === "zooming") {
|
|
226
|
-
if (
|
|
227
|
-
return this.releaseTimer === null ? this.releaseTimer = e : e - this.releaseTimer >= s && this.transitionTo("idle"), this.buildOutput(null, null);
|
|
228
|
-
if (this.releaseTimer = null, !
|
|
229
|
-
return this.transitionTo("idle"), this.buildOutput(null, null);
|
|
230
|
-
const
|
|
229
|
+
if (h !== "zooming")
|
|
230
|
+
return this.releaseTimer === null ? this.releaseTimer = e : e - this.releaseTimer >= s && this.transitionTo("idle"), this.buildOutput(null, null, null);
|
|
231
|
+
if (this.releaseTimer = null, !n)
|
|
232
|
+
return this.transitionTo("idle"), this.buildOutput(null, null, null);
|
|
233
|
+
const m = n.landmarks[0], f = this.zoomSmoother.update(m.y);
|
|
231
234
|
let p = null;
|
|
232
235
|
if (this.prevZoomDist !== null) {
|
|
233
|
-
const
|
|
234
|
-
Math.abs(
|
|
236
|
+
const g = f - this.prevZoomDist;
|
|
237
|
+
Math.abs(g) > this.tuning.zoomDeadzoneRatio && (p = -g);
|
|
235
238
|
}
|
|
236
|
-
return this.prevZoomDist =
|
|
239
|
+
return this.prevZoomDist = f, this.buildOutput(null, p, null);
|
|
237
240
|
}
|
|
238
|
-
|
|
241
|
+
if (this.mode === "rotating") {
|
|
242
|
+
if (h !== "rotating")
|
|
243
|
+
return this.releaseTimer === null ? this.releaseTimer = e : e - this.releaseTimer >= s && this.transitionTo("idle"), this.buildOutput(null, null, null);
|
|
244
|
+
if (this.releaseTimer = null, !a || !n)
|
|
245
|
+
return this.transitionTo("idle"), this.buildOutput(null, null, null);
|
|
246
|
+
const m = a.landmarks[0], f = n.landmarks[0], p = Math.atan2(f.y - m.y, f.x - m.x), g = this.rotateSmoother.update(p);
|
|
247
|
+
let I = null;
|
|
248
|
+
if (this.prevRotateAngle !== null) {
|
|
249
|
+
let v = g - this.prevRotateAngle;
|
|
250
|
+
v > Math.PI && (v -= 2 * Math.PI), v < -Math.PI && (v += 2 * Math.PI), Math.abs(v) > 5e-3 && (I = v);
|
|
251
|
+
}
|
|
252
|
+
return this.prevRotateAngle = g, this.buildOutput(null, null, I);
|
|
253
|
+
}
|
|
254
|
+
return this.buildOutput(null, null, null);
|
|
239
255
|
}
|
|
240
256
|
transitionTo(t) {
|
|
241
|
-
this.mode = t, this.releaseTimer = null, this.actionDwell = null, t !== "panning" && (this.panSmoother.reset(), this.prevPanPos = null), t !== "zooming" && (this.zoomSmoother.reset(), this.prevZoomDist = null);
|
|
257
|
+
this.mode = t, this.releaseTimer = null, this.actionDwell = null, t !== "panning" && (this.panSmoother.reset(), this.prevPanPos = null), t !== "zooming" && (this.zoomSmoother.reset(), this.prevZoomDist = null), t !== "rotating" && (this.rotateSmoother.reset(), this.prevRotateAngle = null);
|
|
242
258
|
}
|
|
243
|
-
buildOutput(t, e) {
|
|
244
|
-
return { mode: this.mode, panDelta: t, zoomDelta: e };
|
|
259
|
+
buildOutput(t, e, i) {
|
|
260
|
+
return { mode: this.mode, panDelta: t, zoomDelta: e, rotateDelta: i };
|
|
245
261
|
}
|
|
246
262
|
reset() {
|
|
247
263
|
this.transitionTo("idle");
|
|
248
264
|
}
|
|
249
265
|
}
|
|
250
|
-
const
|
|
266
|
+
const F = [
|
|
251
267
|
[0, 1],
|
|
252
268
|
[1, 2],
|
|
253
269
|
[2, 3],
|
|
@@ -277,20 +293,20 @@ const O = [
|
|
|
277
293
|
[9, 13],
|
|
278
294
|
[13, 17]
|
|
279
295
|
// palm cross
|
|
280
|
-
],
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
296
|
+
], O = [
|
|
297
|
+
r.THUMB_TIP,
|
|
298
|
+
r.INDEX_TIP,
|
|
299
|
+
r.MIDDLE_TIP,
|
|
300
|
+
r.RING_TIP,
|
|
301
|
+
r.PINKY_TIP
|
|
286
302
|
];
|
|
287
|
-
class
|
|
303
|
+
class V {
|
|
288
304
|
constructor(t) {
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
305
|
+
l(this, "container");
|
|
306
|
+
l(this, "canvas");
|
|
307
|
+
l(this, "ctx");
|
|
308
|
+
l(this, "badge");
|
|
309
|
+
l(this, "config");
|
|
294
310
|
this.config = t, this.container = document.createElement("div"), this.container.className = "ol-gesture-overlay", this.applyContainerStyles(), this.canvas = document.createElement("canvas"), this.canvas.className = "ol-gesture-canvas", this.canvas.style.cssText = "position:absolute;top:0;left:0;width:100%;height:100%;pointer-events:none;", this.badge = document.createElement("div"), this.badge.className = "ol-gesture-badge ol-gesture-badge--idle", this.badge.textContent = "Idle", this.container.appendChild(this.canvas), this.container.appendChild(this.badge);
|
|
295
311
|
const e = this.canvas.getContext("2d");
|
|
296
312
|
if (!e) throw new Error("Cannot get 2D canvas context");
|
|
@@ -311,35 +327,36 @@ class U {
|
|
|
311
327
|
/** Called each frame with the latest gesture frame and mode. */
|
|
312
328
|
render(t, e) {
|
|
313
329
|
this.updateBadge(e);
|
|
314
|
-
const
|
|
315
|
-
if (this.canvas.width =
|
|
316
|
-
for (const
|
|
317
|
-
this.drawSkeleton(
|
|
318
|
-
}
|
|
319
|
-
drawSkeleton(t, e,
|
|
320
|
-
const { ctx: s } = this,
|
|
321
|
-
s.strokeStyle =
|
|
322
|
-
for (const [
|
|
323
|
-
!t[
|
|
324
|
-
for (let
|
|
325
|
-
const
|
|
326
|
-
s.beginPath(), s.arc(d(
|
|
330
|
+
const i = this.config.width, s = this.config.height;
|
|
331
|
+
if (this.canvas.width = i, this.canvas.height = s, this.ctx.clearRect(0, 0, i, s), t !== null)
|
|
332
|
+
for (const a of t.hands)
|
|
333
|
+
this.drawSkeleton(a.landmarks, e, a.gesture === "fist");
|
|
334
|
+
}
|
|
335
|
+
drawSkeleton(t, e, i) {
|
|
336
|
+
const { ctx: s } = this, a = this.config.width, n = this.config.height, d = (u) => (1 - u.x) * a, c = (u) => u.y * n;
|
|
337
|
+
s.strokeStyle = y.connection, s.lineWidth = 1.5;
|
|
338
|
+
for (const [u, h] of F)
|
|
339
|
+
!t[u] || !t[h] || (s.beginPath(), s.moveTo(d(t[u]), c(t[u])), s.lineTo(d(t[h]), c(t[h])), s.stroke());
|
|
340
|
+
for (let u = 0; u < t.length; u++) {
|
|
341
|
+
const h = t[u], m = O.includes(u), f = e !== "idle" && m ? y.fingertipGlow : y.landmark;
|
|
342
|
+
s.beginPath(), s.arc(d(h), c(h), m ? 5 : 3, 0, Math.PI * 2), s.fillStyle = f, s.fill(), e !== "idle" && m && (s.shadowBlur = i ? 12 : 6, s.shadowColor = y.fingertipGlow, s.fill(), s.shadowBlur = 0);
|
|
327
343
|
}
|
|
328
344
|
}
|
|
329
345
|
updateBadge(t) {
|
|
330
346
|
const e = {
|
|
331
347
|
idle: "Idle",
|
|
332
348
|
panning: "Pan",
|
|
333
|
-
zooming: "Zoom"
|
|
349
|
+
zooming: "Zoom",
|
|
350
|
+
rotating: "Rotate"
|
|
334
351
|
};
|
|
335
352
|
this.badge.textContent = e[t], this.badge.className = `ol-gesture-badge ol-gesture-badge--${t}`;
|
|
336
353
|
}
|
|
337
354
|
applyContainerStyles() {
|
|
338
|
-
const { mode: t, position: e, width:
|
|
339
|
-
if (this.container.style.cssText = "", this.container.style.position = "fixed", this.container.style.zIndex = "9999", this.container.style.overflow = "hidden", this.container.style.borderRadius = "8px", this.container.style.opacity = String(
|
|
340
|
-
this.container.style.width = `${
|
|
341
|
-
const
|
|
342
|
-
e === "bottom-right" ? (this.container.style.bottom =
|
|
355
|
+
const { mode: t, position: e, width: i, height: s, opacity: a } = this.config;
|
|
356
|
+
if (this.container.style.cssText = "", this.container.style.position = "fixed", this.container.style.zIndex = "9999", this.container.style.overflow = "hidden", this.container.style.borderRadius = "8px", this.container.style.opacity = String(a), this.container.style.display = t === "hidden" ? "none" : "block", t === "corner") {
|
|
357
|
+
this.container.style.width = `${i}px`, this.container.style.height = `${s}px`;
|
|
358
|
+
const n = "16px";
|
|
359
|
+
e === "bottom-right" ? (this.container.style.bottom = n, this.container.style.right = n) : e === "bottom-left" ? (this.container.style.bottom = n, this.container.style.left = n) : e === "top-right" ? (this.container.style.top = n, this.container.style.right = n) : (this.container.style.top = n, this.container.style.left = n);
|
|
343
360
|
} else t === "full" && (this.container.style.top = "0", this.container.style.left = "0", this.container.style.width = "100vw", this.container.style.height = "100vh", this.container.style.borderRadius = "0");
|
|
344
361
|
}
|
|
345
362
|
updateConfig(t) {
|
|
@@ -347,14 +364,14 @@ class U {
|
|
|
347
364
|
}
|
|
348
365
|
}
|
|
349
366
|
export {
|
|
350
|
-
|
|
367
|
+
y as COLORS,
|
|
351
368
|
H as DEFAULT_TUNING_CONFIG,
|
|
352
369
|
L as DEFAULT_WEBCAM_CONFIG,
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
370
|
+
B as GestureController,
|
|
371
|
+
U as GestureStateMachine,
|
|
372
|
+
r as LANDMARKS,
|
|
373
|
+
V as WebcamOverlay,
|
|
374
|
+
R as classifyGesture,
|
|
375
|
+
x as getHandSize,
|
|
376
|
+
k as getTwoHandDistance
|
|
360
377
|
};
|
package/dist/types.d.ts
CHANGED