@map-gesture-controls/core 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/GestureController.d.ts +32 -0
- package/dist/GestureController.js +159 -0
- package/dist/GestureStateMachine.d.ts +35 -0
- package/dist/GestureStateMachine.js +243 -0
- package/dist/GestureStateMachine.test.d.ts +1 -0
- package/dist/GestureStateMachine.test.js +163 -0
- package/dist/WebcamOverlay.d.ts +30 -0
- package/dist/WebcamOverlay.js +194 -0
- package/dist/constants.d.ts +26 -0
- package/dist/constants.js +54 -0
- package/dist/gestureClassifier.d.ts +22 -0
- package/dist/gestureClassifier.js +110 -0
- package/dist/gestureClassifier.test.d.ts +1 -0
- package/dist/gestureClassifier.test.js +98 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +360 -0
- package/dist/types.d.ts +47 -0
- package/dist/types.js +1 -0
- package/package.json +39 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { GestureFrame, TuningConfig } from './types.js';
|
|
2
|
+
type FrameCallback = (frame: GestureFrame) => void;
|
|
3
|
+
/**
|
|
4
|
+
* GestureController
|
|
5
|
+
*
|
|
6
|
+
* Manages the webcam stream and MediaPipe HandLandmarker inference loop.
|
|
7
|
+
* Calls `onFrame` with classified hand data every animation frame.
|
|
8
|
+
*/
|
|
9
|
+
export declare class GestureController {
|
|
10
|
+
private landmarker;
|
|
11
|
+
private videoEl;
|
|
12
|
+
private stream;
|
|
13
|
+
private rafHandle;
|
|
14
|
+
private running;
|
|
15
|
+
private onFrame;
|
|
16
|
+
private tuning;
|
|
17
|
+
private lastVideoTime;
|
|
18
|
+
constructor(tuning: TuningConfig, onFrame: FrameCallback);
|
|
19
|
+
/**
|
|
20
|
+
* Initialise MediaPipe and request webcam access.
|
|
21
|
+
* Returns the video element so the overlay can render it.
|
|
22
|
+
*/
|
|
23
|
+
init(): Promise<HTMLVideoElement>;
|
|
24
|
+
start(): void;
|
|
25
|
+
stop(): void;
|
|
26
|
+
destroy(): void;
|
|
27
|
+
private loop;
|
|
28
|
+
private processFrame;
|
|
29
|
+
private buildFrame;
|
|
30
|
+
getVideoElement(): HTMLVideoElement | null;
|
|
31
|
+
}
|
|
32
|
+
export {};
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { FilesetResolver } from '@mediapipe/tasks-vision';
|
|
2
|
+
import { classifyGesture } from './gestureClassifier.js';
|
|
3
|
+
import { MEDIAPIPE_WASM_URL } from './constants.js';
|
|
4
|
+
/**
|
|
5
|
+
* GestureController
|
|
6
|
+
*
|
|
7
|
+
* Manages the webcam stream and MediaPipe HandLandmarker inference loop.
|
|
8
|
+
* Calls `onFrame` with classified hand data every animation frame.
|
|
9
|
+
*/
|
|
10
|
+
export class GestureController {
|
|
11
|
+
constructor(tuning, onFrame) {
|
|
12
|
+
Object.defineProperty(this, "landmarker", {
|
|
13
|
+
enumerable: true,
|
|
14
|
+
configurable: true,
|
|
15
|
+
writable: true,
|
|
16
|
+
value: null
|
|
17
|
+
});
|
|
18
|
+
Object.defineProperty(this, "videoEl", {
|
|
19
|
+
enumerable: true,
|
|
20
|
+
configurable: true,
|
|
21
|
+
writable: true,
|
|
22
|
+
value: null
|
|
23
|
+
});
|
|
24
|
+
Object.defineProperty(this, "stream", {
|
|
25
|
+
enumerable: true,
|
|
26
|
+
configurable: true,
|
|
27
|
+
writable: true,
|
|
28
|
+
value: null
|
|
29
|
+
});
|
|
30
|
+
Object.defineProperty(this, "rafHandle", {
|
|
31
|
+
enumerable: true,
|
|
32
|
+
configurable: true,
|
|
33
|
+
writable: true,
|
|
34
|
+
value: null
|
|
35
|
+
});
|
|
36
|
+
Object.defineProperty(this, "running", {
|
|
37
|
+
enumerable: true,
|
|
38
|
+
configurable: true,
|
|
39
|
+
writable: true,
|
|
40
|
+
value: false
|
|
41
|
+
});
|
|
42
|
+
Object.defineProperty(this, "onFrame", {
|
|
43
|
+
enumerable: true,
|
|
44
|
+
configurable: true,
|
|
45
|
+
writable: true,
|
|
46
|
+
value: void 0
|
|
47
|
+
});
|
|
48
|
+
Object.defineProperty(this, "tuning", {
|
|
49
|
+
enumerable: true,
|
|
50
|
+
configurable: true,
|
|
51
|
+
writable: true,
|
|
52
|
+
value: void 0
|
|
53
|
+
});
|
|
54
|
+
Object.defineProperty(this, "lastVideoTime", {
|
|
55
|
+
enumerable: true,
|
|
56
|
+
configurable: true,
|
|
57
|
+
writable: true,
|
|
58
|
+
value: -1
|
|
59
|
+
});
|
|
60
|
+
this.tuning = tuning;
|
|
61
|
+
this.onFrame = onFrame;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Initialise MediaPipe and request webcam access.
|
|
65
|
+
* Returns the video element so the overlay can render it.
|
|
66
|
+
*/
|
|
67
|
+
async init() {
|
|
68
|
+
const vision = await FilesetResolver.forVisionTasks(MEDIAPIPE_WASM_URL);
|
|
69
|
+
// Dynamic import to avoid bundling issues
|
|
70
|
+
const { HandLandmarker } = await import('@mediapipe/tasks-vision');
|
|
71
|
+
this.landmarker = await HandLandmarker.createFromOptions(vision, {
|
|
72
|
+
baseOptions: {
|
|
73
|
+
modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
|
|
74
|
+
delegate: 'GPU',
|
|
75
|
+
},
|
|
76
|
+
runningMode: 'VIDEO',
|
|
77
|
+
numHands: 2,
|
|
78
|
+
minHandDetectionConfidence: this.tuning.minDetectionConfidence,
|
|
79
|
+
minHandPresenceConfidence: this.tuning.minPresenceConfidence,
|
|
80
|
+
minTrackingConfidence: this.tuning.minTrackingConfidence,
|
|
81
|
+
});
|
|
82
|
+
this.videoEl = document.createElement('video');
|
|
83
|
+
this.videoEl.setAttribute('playsinline', '');
|
|
84
|
+
this.videoEl.setAttribute('autoplay', '');
|
|
85
|
+
this.videoEl.muted = true;
|
|
86
|
+
this.videoEl.width = 640;
|
|
87
|
+
this.videoEl.height = 480;
|
|
88
|
+
this.stream = await navigator.mediaDevices.getUserMedia({
|
|
89
|
+
video: { width: 640, height: 480, facingMode: 'user' },
|
|
90
|
+
});
|
|
91
|
+
this.videoEl.srcObject = this.stream;
|
|
92
|
+
await new Promise((resolve) => {
|
|
93
|
+
this.videoEl.addEventListener('loadeddata', () => resolve(), { once: true });
|
|
94
|
+
});
|
|
95
|
+
return this.videoEl;
|
|
96
|
+
}
|
|
97
|
+
start() {
|
|
98
|
+
if (this.running)
|
|
99
|
+
return;
|
|
100
|
+
this.running = true;
|
|
101
|
+
this.loop();
|
|
102
|
+
}
|
|
103
|
+
stop() {
|
|
104
|
+
this.running = false;
|
|
105
|
+
if (this.rafHandle !== null) {
|
|
106
|
+
cancelAnimationFrame(this.rafHandle);
|
|
107
|
+
this.rafHandle = null;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
destroy() {
|
|
111
|
+
this.stop();
|
|
112
|
+
this.stream?.getTracks().forEach((t) => t.stop());
|
|
113
|
+
this.landmarker?.close();
|
|
114
|
+
this.landmarker = null;
|
|
115
|
+
this.videoEl = null;
|
|
116
|
+
this.stream = null;
|
|
117
|
+
}
|
|
118
|
+
loop() {
|
|
119
|
+
if (!this.running)
|
|
120
|
+
return;
|
|
121
|
+
this.rafHandle = requestAnimationFrame(() => this.loop());
|
|
122
|
+
this.processFrame();
|
|
123
|
+
}
|
|
124
|
+
processFrame() {
|
|
125
|
+
const video = this.videoEl;
|
|
126
|
+
const landmarker = this.landmarker;
|
|
127
|
+
if (!video || !landmarker || video.readyState < 2)
|
|
128
|
+
return;
|
|
129
|
+
const nowMs = performance.now();
|
|
130
|
+
if (video.currentTime === this.lastVideoTime)
|
|
131
|
+
return;
|
|
132
|
+
this.lastVideoTime = video.currentTime;
|
|
133
|
+
let result;
|
|
134
|
+
try {
|
|
135
|
+
result = landmarker.detectForVideo(video, nowMs);
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
const frame = this.buildFrame(result, nowMs);
|
|
141
|
+
this.onFrame(frame);
|
|
142
|
+
}
|
|
143
|
+
buildFrame(result, timestamp) {
|
|
144
|
+
const hands = result.landmarks.map((landmarks, i) => {
|
|
145
|
+
const handednessArr = result.handedness[i];
|
|
146
|
+
const rawLabel = handednessArr?.[0]?.categoryName;
|
|
147
|
+
const label = rawLabel === 'Left' ? 'Left' : 'Right';
|
|
148
|
+
const score = handednessArr?.[0]?.score ?? 0;
|
|
149
|
+
const gesture = classifyGesture(landmarks);
|
|
150
|
+
return { handedness: label, score, landmarks, gesture };
|
|
151
|
+
});
|
|
152
|
+
const leftHand = hands.find((h) => h.handedness === 'Left') ?? null;
|
|
153
|
+
const rightHand = hands.find((h) => h.handedness === 'Right') ?? null;
|
|
154
|
+
return { timestamp, hands, leftHand, rightHand };
|
|
155
|
+
}
|
|
156
|
+
getVideoElement() {
|
|
157
|
+
return this.videoEl;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { GestureMode, GestureFrame, TuningConfig, SmoothedPoint } from './types.js';
|
|
2
|
+
export interface StateMachineOutput {
|
|
3
|
+
mode: GestureMode;
|
|
4
|
+
panDelta: SmoothedPoint | null;
|
|
5
|
+
zoomDelta: number | null;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* GestureStateMachine — 3-state FSM
|
|
9
|
+
*
|
|
10
|
+
* Priority rules (evaluated every frame):
|
|
11
|
+
* both hands visible AND both open palm → desired = 'zooming'
|
|
12
|
+
* one hand visible AND gesture = 'fist' → desired = 'panning'
|
|
13
|
+
* otherwise → desired = 'idle'
|
|
14
|
+
*
|
|
15
|
+
* Transitions:
|
|
16
|
+
* idle → panning/zooming : desired stable for actionDwellMs
|
|
17
|
+
* panning/zooming → idle : desired changes, grace period releaseGraceMs,
|
|
18
|
+
* then idle (next frame starts new dwell if needed)
|
|
19
|
+
*/
|
|
20
|
+
export declare class GestureStateMachine {
|
|
21
|
+
private tuning;
|
|
22
|
+
private mode;
|
|
23
|
+
private actionDwell;
|
|
24
|
+
private releaseTimer;
|
|
25
|
+
private panSmoother;
|
|
26
|
+
private prevPanPos;
|
|
27
|
+
private zoomSmoother;
|
|
28
|
+
private prevZoomDist;
|
|
29
|
+
constructor(tuning: TuningConfig);
|
|
30
|
+
getMode(): GestureMode;
|
|
31
|
+
update(frame: GestureFrame): StateMachineOutput;
|
|
32
|
+
private transitionTo;
|
|
33
|
+
private buildOutput;
|
|
34
|
+
reset(): void;
|
|
35
|
+
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { getTwoHandDistance } from './gestureClassifier.js';
|
|
2
|
+
/**
|
|
3
|
+
* Exponential Moving Average smoother for 2D points.
|
|
4
|
+
*/
|
|
5
|
+
class EMAPoint {
|
|
6
|
+
constructor(alpha) {
|
|
7
|
+
Object.defineProperty(this, "alpha", {
|
|
8
|
+
enumerable: true,
|
|
9
|
+
configurable: true,
|
|
10
|
+
writable: true,
|
|
11
|
+
value: alpha
|
|
12
|
+
});
|
|
13
|
+
Object.defineProperty(this, "value", {
|
|
14
|
+
enumerable: true,
|
|
15
|
+
configurable: true,
|
|
16
|
+
writable: true,
|
|
17
|
+
value: null
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
update(x, y) {
|
|
21
|
+
if (this.value === null) {
|
|
22
|
+
this.value = { x, y };
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
this.value = {
|
|
26
|
+
x: this.alpha * x + (1 - this.alpha) * this.value.x,
|
|
27
|
+
y: this.alpha * y + (1 - this.alpha) * this.value.y,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
return this.value;
|
|
31
|
+
}
|
|
32
|
+
reset() {
|
|
33
|
+
this.value = null;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Exponential Moving Average smoother for a scalar value.
|
|
38
|
+
*/
|
|
39
|
+
class EMAScalar {
|
|
40
|
+
constructor(alpha) {
|
|
41
|
+
Object.defineProperty(this, "alpha", {
|
|
42
|
+
enumerable: true,
|
|
43
|
+
configurable: true,
|
|
44
|
+
writable: true,
|
|
45
|
+
value: alpha
|
|
46
|
+
});
|
|
47
|
+
Object.defineProperty(this, "value", {
|
|
48
|
+
enumerable: true,
|
|
49
|
+
configurable: true,
|
|
50
|
+
writable: true,
|
|
51
|
+
value: null
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
update(v) {
|
|
55
|
+
if (this.value === null) {
|
|
56
|
+
this.value = v;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
this.value = this.alpha * v + (1 - this.alpha) * this.value;
|
|
60
|
+
}
|
|
61
|
+
return this.value;
|
|
62
|
+
}
|
|
63
|
+
reset() {
|
|
64
|
+
this.value = null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* GestureStateMachine — 3-state FSM
|
|
69
|
+
*
|
|
70
|
+
* Priority rules (evaluated every frame):
|
|
71
|
+
* both hands visible AND both open palm → desired = 'zooming'
|
|
72
|
+
* one hand visible AND gesture = 'fist' → desired = 'panning'
|
|
73
|
+
* otherwise → desired = 'idle'
|
|
74
|
+
*
|
|
75
|
+
* Transitions:
|
|
76
|
+
* idle → panning/zooming : desired stable for actionDwellMs
|
|
77
|
+
* panning/zooming → idle : desired changes, grace period releaseGraceMs,
|
|
78
|
+
* then idle (next frame starts new dwell if needed)
|
|
79
|
+
*/
|
|
80
|
+
export class GestureStateMachine {
|
|
81
|
+
constructor(tuning) {
|
|
82
|
+
Object.defineProperty(this, "tuning", {
|
|
83
|
+
enumerable: true,
|
|
84
|
+
configurable: true,
|
|
85
|
+
writable: true,
|
|
86
|
+
value: tuning
|
|
87
|
+
});
|
|
88
|
+
Object.defineProperty(this, "mode", {
|
|
89
|
+
enumerable: true,
|
|
90
|
+
configurable: true,
|
|
91
|
+
writable: true,
|
|
92
|
+
value: 'idle'
|
|
93
|
+
});
|
|
94
|
+
Object.defineProperty(this, "actionDwell", {
|
|
95
|
+
enumerable: true,
|
|
96
|
+
configurable: true,
|
|
97
|
+
writable: true,
|
|
98
|
+
value: null
|
|
99
|
+
});
|
|
100
|
+
Object.defineProperty(this, "releaseTimer", {
|
|
101
|
+
enumerable: true,
|
|
102
|
+
configurable: true,
|
|
103
|
+
writable: true,
|
|
104
|
+
value: null
|
|
105
|
+
});
|
|
106
|
+
Object.defineProperty(this, "panSmoother", {
|
|
107
|
+
enumerable: true,
|
|
108
|
+
configurable: true,
|
|
109
|
+
writable: true,
|
|
110
|
+
value: void 0
|
|
111
|
+
});
|
|
112
|
+
Object.defineProperty(this, "prevPanPos", {
|
|
113
|
+
enumerable: true,
|
|
114
|
+
configurable: true,
|
|
115
|
+
writable: true,
|
|
116
|
+
value: null
|
|
117
|
+
});
|
|
118
|
+
Object.defineProperty(this, "zoomSmoother", {
|
|
119
|
+
enumerable: true,
|
|
120
|
+
configurable: true,
|
|
121
|
+
writable: true,
|
|
122
|
+
value: void 0
|
|
123
|
+
});
|
|
124
|
+
Object.defineProperty(this, "prevZoomDist", {
|
|
125
|
+
enumerable: true,
|
|
126
|
+
configurable: true,
|
|
127
|
+
writable: true,
|
|
128
|
+
value: null
|
|
129
|
+
});
|
|
130
|
+
this.panSmoother = new EMAPoint(tuning.smoothingAlpha);
|
|
131
|
+
this.zoomSmoother = new EMAScalar(tuning.smoothingAlpha);
|
|
132
|
+
}
|
|
133
|
+
getMode() {
|
|
134
|
+
return this.mode;
|
|
135
|
+
}
|
|
136
|
+
update(frame) {
|
|
137
|
+
const now = frame.timestamp;
|
|
138
|
+
const { actionDwellMs, releaseGraceMs } = this.tuning;
|
|
139
|
+
const { leftHand, rightHand } = frame;
|
|
140
|
+
// ── Determine desired mode for this frame ─────────────────────────────────
|
|
141
|
+
const bothOpen = leftHand !== null &&
|
|
142
|
+
rightHand !== null &&
|
|
143
|
+
leftHand.gesture === 'openPalm' &&
|
|
144
|
+
rightHand.gesture === 'openPalm';
|
|
145
|
+
const oneFist = (leftHand !== null && leftHand.gesture === 'fist' && rightHand === null) ||
|
|
146
|
+
(rightHand !== null && rightHand.gesture === 'fist' && leftHand === null);
|
|
147
|
+
const desired = bothOpen ? 'zooming' : oneFist ? 'panning' : 'idle';
|
|
148
|
+
// ── idle ──────────────────────────────────────────────────────────────────
|
|
149
|
+
if (this.mode === 'idle') {
|
|
150
|
+
if (desired !== 'idle') {
|
|
151
|
+
if (this.actionDwell === null || this.actionDwell.gesture !== desired) {
|
|
152
|
+
this.actionDwell = { gesture: desired, startMs: now };
|
|
153
|
+
}
|
|
154
|
+
else if (now - this.actionDwell.startMs >= actionDwellMs) {
|
|
155
|
+
this.transitionTo(desired);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
this.actionDwell = null;
|
|
160
|
+
}
|
|
161
|
+
return this.buildOutput(null, null);
|
|
162
|
+
}
|
|
163
|
+
// ── panning ───────────────────────────────────────────────────────────────
|
|
164
|
+
if (this.mode === 'panning') {
|
|
165
|
+
if (desired !== 'panning') {
|
|
166
|
+
if (this.releaseTimer === null) {
|
|
167
|
+
this.releaseTimer = now;
|
|
168
|
+
}
|
|
169
|
+
else if (now - this.releaseTimer >= releaseGraceMs) {
|
|
170
|
+
this.transitionTo('idle');
|
|
171
|
+
}
|
|
172
|
+
return this.buildOutput(null, null);
|
|
173
|
+
}
|
|
174
|
+
this.releaseTimer = null;
|
|
175
|
+
const fistHand = leftHand?.gesture === 'fist' ? leftHand : rightHand;
|
|
176
|
+
if (!fistHand) {
|
|
177
|
+
this.transitionTo('idle');
|
|
178
|
+
return this.buildOutput(null, null);
|
|
179
|
+
}
|
|
180
|
+
const wrist = fistHand.landmarks[0];
|
|
181
|
+
const smooth = this.panSmoother.update(wrist.x, wrist.y);
|
|
182
|
+
let panDelta = null;
|
|
183
|
+
if (this.prevPanPos !== null) {
|
|
184
|
+
const rawDx = smooth.x - this.prevPanPos.x;
|
|
185
|
+
const rawDy = smooth.y - this.prevPanPos.y;
|
|
186
|
+
const deadzone = this.tuning.panDeadzonePx / 640;
|
|
187
|
+
if (Math.abs(rawDx) > deadzone || Math.abs(rawDy) > deadzone) {
|
|
188
|
+
panDelta = { x: rawDx, y: rawDy };
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
this.prevPanPos = smooth;
|
|
192
|
+
return this.buildOutput(panDelta, null);
|
|
193
|
+
}
|
|
194
|
+
// ── zooming ───────────────────────────────────────────────────────────────
|
|
195
|
+
if (this.mode === 'zooming') {
|
|
196
|
+
if (desired !== 'zooming') {
|
|
197
|
+
if (this.releaseTimer === null) {
|
|
198
|
+
this.releaseTimer = now;
|
|
199
|
+
}
|
|
200
|
+
else if (now - this.releaseTimer >= releaseGraceMs) {
|
|
201
|
+
this.transitionTo('idle');
|
|
202
|
+
}
|
|
203
|
+
return this.buildOutput(null, null);
|
|
204
|
+
}
|
|
205
|
+
this.releaseTimer = null;
|
|
206
|
+
if (!leftHand || !rightHand) {
|
|
207
|
+
this.transitionTo('idle');
|
|
208
|
+
return this.buildOutput(null, null);
|
|
209
|
+
}
|
|
210
|
+
const rawDist = getTwoHandDistance(leftHand.landmarks, rightHand.landmarks);
|
|
211
|
+
const smoothDist = this.zoomSmoother.update(rawDist);
|
|
212
|
+
let zoomDelta = null;
|
|
213
|
+
if (this.prevZoomDist !== null) {
|
|
214
|
+
const delta = smoothDist - this.prevZoomDist;
|
|
215
|
+
if (Math.abs(delta) > this.tuning.zoomDeadzoneRatio) {
|
|
216
|
+
zoomDelta = delta;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
this.prevZoomDist = smoothDist;
|
|
220
|
+
return this.buildOutput(null, zoomDelta);
|
|
221
|
+
}
|
|
222
|
+
return this.buildOutput(null, null);
|
|
223
|
+
}
|
|
224
|
+
transitionTo(next) {
|
|
225
|
+
this.mode = next;
|
|
226
|
+
this.releaseTimer = null;
|
|
227
|
+
this.actionDwell = null;
|
|
228
|
+
if (next !== 'panning') {
|
|
229
|
+
this.panSmoother.reset();
|
|
230
|
+
this.prevPanPos = null;
|
|
231
|
+
}
|
|
232
|
+
if (next !== 'zooming') {
|
|
233
|
+
this.zoomSmoother.reset();
|
|
234
|
+
this.prevZoomDist = null;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
buildOutput(panDelta, zoomDelta) {
|
|
238
|
+
return { mode: this.mode, panDelta, zoomDelta };
|
|
239
|
+
}
|
|
240
|
+
reset() {
|
|
241
|
+
this.transitionTo('idle');
|
|
242
|
+
}
|
|
243
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach } from 'vitest';
|
|
2
|
+
import { GestureStateMachine } from './GestureStateMachine.js';
|
|
3
|
+
import { LANDMARKS } from './constants.js';
|
|
4
|
+
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
|
5
|
+
const FAST_TUNING = {
|
|
6
|
+
actionDwellMs: 0, // dwell=0: transition fires on the SECOND frame (first sets timer)
|
|
7
|
+
releaseGraceMs: 0, // grace=0: idle on the SECOND frame after release
|
|
8
|
+
panDeadzonePx: 0,
|
|
9
|
+
zoomDeadzoneRatio: 0,
|
|
10
|
+
smoothingAlpha: 1, // no smoothing — raw values pass through
|
|
11
|
+
minDetectionConfidence: 0.65,
|
|
12
|
+
minTrackingConfidence: 0.65,
|
|
13
|
+
minPresenceConfidence: 0.60,
|
|
14
|
+
};
|
|
15
|
+
function makeLandmarks(overrides = {}) {
|
|
16
|
+
const lm = Array.from({ length: 21 }, () => ({ x: 0.5, y: 0.5, z: 0 }));
|
|
17
|
+
for (const [idx, vals] of Object.entries(overrides)) {
|
|
18
|
+
lm[Number(idx)] = { ...lm[Number(idx)], ...vals };
|
|
19
|
+
}
|
|
20
|
+
return lm;
|
|
21
|
+
}
|
|
22
|
+
function makeHand(gesture, landmarks = makeLandmarks()) {
|
|
23
|
+
return { handedness: 'Right', score: 1, landmarks, gesture };
|
|
24
|
+
}
|
|
25
|
+
function makeFrame(timestamp, leftHand, rightHand) {
|
|
26
|
+
const hands = [leftHand, rightHand].filter(Boolean);
|
|
27
|
+
return { timestamp, hands, leftHand, rightHand };
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Drive the FSM into 'panning' with dwell=0 (requires two identical frames).
|
|
31
|
+
*/
|
|
32
|
+
function enterPanning(fsm, hand = makeHand('fist')) {
|
|
33
|
+
fsm.update(makeFrame(0, hand, null)); // starts dwell timer
|
|
34
|
+
fsm.update(makeFrame(0, hand, null)); // dwell elapsed → panning
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Drive the FSM into 'zooming' with dwell=0 (requires two identical frames).
|
|
38
|
+
*/
|
|
39
|
+
function enterZooming(fsm, left = makeHand('openPalm'), right = makeHand('openPalm')) {
|
|
40
|
+
fsm.update(makeFrame(0, left, right));
|
|
41
|
+
fsm.update(makeFrame(0, left, right));
|
|
42
|
+
}
|
|
43
|
+
// ─── Tests ────────────────────────────────────────────────────────────────────
|
|
44
|
+
describe('GestureStateMachine', () => {
|
|
45
|
+
let fsm;
|
|
46
|
+
beforeEach(() => {
|
|
47
|
+
fsm = new GestureStateMachine(FAST_TUNING);
|
|
48
|
+
});
|
|
49
|
+
// ── Initial state ──────────────────────────────────────────────────────────
|
|
50
|
+
it('starts in idle mode', () => {
|
|
51
|
+
expect(fsm.getMode()).toBe('idle');
|
|
52
|
+
});
|
|
53
|
+
// ── idle → panning ─────────────────────────────────────────────────────────
|
|
54
|
+
it('transitions to panning after the dwell elapses (two frames, dwell=0)', () => {
|
|
55
|
+
const fistHand = makeHand('fist');
|
|
56
|
+
fsm.update(makeFrame(0, fistHand, null)); // starts dwell
|
|
57
|
+
const out = fsm.update(makeFrame(0, fistHand, null)); // 0 >= 0 → panning
|
|
58
|
+
expect(out.mode).toBe('panning');
|
|
59
|
+
});
|
|
60
|
+
// ── idle → zooming ─────────────────────────────────────────────────────────
|
|
61
|
+
it('transitions to zooming when two open palms are held (two frames, dwell=0)', () => {
|
|
62
|
+
const left = makeHand('openPalm');
|
|
63
|
+
const right = makeHand('openPalm');
|
|
64
|
+
fsm.update(makeFrame(0, left, right));
|
|
65
|
+
const out = fsm.update(makeFrame(0, left, right));
|
|
66
|
+
expect(out.mode).toBe('zooming');
|
|
67
|
+
});
|
|
68
|
+
// ── panning → idle ─────────────────────────────────────────────────────────
|
|
69
|
+
it('returns to idle when the fist is released (releaseGraceMs=0)', () => {
|
|
70
|
+
enterPanning(fsm);
|
|
71
|
+
expect(fsm.getMode()).toBe('panning');
|
|
72
|
+
fsm.update(makeFrame(1, null, null)); // starts release timer
|
|
73
|
+
const out = fsm.update(makeFrame(1, null, null)); // 0 >= 0 → idle
|
|
74
|
+
expect(out.mode).toBe('idle');
|
|
75
|
+
});
|
|
76
|
+
// ── zooming → idle ─────────────────────────────────────────────────────────
|
|
77
|
+
it('returns to idle when open palms are released (releaseGraceMs=0)', () => {
|
|
78
|
+
enterZooming(fsm);
|
|
79
|
+
expect(fsm.getMode()).toBe('zooming');
|
|
80
|
+
fsm.update(makeFrame(1, null, null)); // starts release timer
|
|
81
|
+
const out = fsm.update(makeFrame(1, null, null)); // 0 >= 0 → idle
|
|
82
|
+
expect(out.mode).toBe('idle');
|
|
83
|
+
});
|
|
84
|
+
// ── panDelta output ────────────────────────────────────────────────────────
|
|
85
|
+
it('emits no panDelta on the first panning frame (no previous position)', () => {
|
|
86
|
+
const fistHand = makeHand('fist');
|
|
87
|
+
fsm.update(makeFrame(0, fistHand, null)); // dwell starts
|
|
88
|
+
const out = fsm.update(makeFrame(0, fistHand, null)); // enters panning, sets prevPos
|
|
89
|
+
expect(out.panDelta).toBeNull();
|
|
90
|
+
});
|
|
91
|
+
it('emits panDelta once a previous position is established', () => {
|
|
92
|
+
const wristPos1 = { x: 0.3, y: 0.4, z: 0 };
|
|
93
|
+
const wristPos2 = { x: 0.35, y: 0.45, z: 0 };
|
|
94
|
+
const lm1 = makeLandmarks({ [LANDMARKS.WRIST]: wristPos1 });
|
|
95
|
+
const lm2 = makeLandmarks({ [LANDMARKS.WRIST]: wristPos2 });
|
|
96
|
+
// Frame 1: start dwell (idle)
|
|
97
|
+
fsm.update(makeFrame(0, makeHand('fist', lm1), null));
|
|
98
|
+
// Frame 2: transition fires → panning, but buildOutput returns from idle branch (panDelta=null)
|
|
99
|
+
fsm.update(makeFrame(0, makeHand('fist', lm1), null));
|
|
100
|
+
// Frame 3: first real panning frame — sets prevPanPos = wristPos1, no delta yet
|
|
101
|
+
fsm.update(makeFrame(1, makeHand('fist', lm1), null));
|
|
102
|
+
// Frame 4: second panning frame — emits delta
|
|
103
|
+
const out = fsm.update(makeFrame(2, makeHand('fist', lm2), null));
|
|
104
|
+
expect(out.mode).toBe('panning');
|
|
105
|
+
expect(out.panDelta).not.toBeNull();
|
|
106
|
+
expect(out.panDelta.x).toBeCloseTo(wristPos2.x - wristPos1.x);
|
|
107
|
+
expect(out.panDelta.y).toBeCloseTo(wristPos2.y - wristPos1.y);
|
|
108
|
+
});
|
|
109
|
+
// ── zoomDelta output ───────────────────────────────────────────────────────
|
|
110
|
+
it('emits no zoomDelta on the first zooming frame', () => {
|
|
111
|
+
const left = makeHand('openPalm');
|
|
112
|
+
const right = makeHand('openPalm');
|
|
113
|
+
fsm.update(makeFrame(0, left, right));
|
|
114
|
+
const out = fsm.update(makeFrame(0, left, right)); // enters zooming, sets prevZoomDist
|
|
115
|
+
expect(out.zoomDelta).toBeNull();
|
|
116
|
+
});
|
|
117
|
+
it('emits zoomDelta once a previous distance is established', () => {
|
|
118
|
+
const lmL1 = makeLandmarks({ [LANDMARKS.INDEX_TIP]: { x: 0.4, y: 0.5, z: 0 } });
|
|
119
|
+
const lmR1 = makeLandmarks({ [LANDMARKS.INDEX_TIP]: { x: 0.6, y: 0.5, z: 0 } });
|
|
120
|
+
const lmL2 = makeLandmarks({ [LANDMARKS.INDEX_TIP]: { x: 0.3, y: 0.5, z: 0 } });
|
|
121
|
+
const lmR2 = makeLandmarks({ [LANDMARKS.INDEX_TIP]: { x: 0.7, y: 0.5, z: 0 } });
|
|
122
|
+
// Frame 1+2: dwell + transition (zooming entered on frame 2, but output comes from idle branch)
|
|
123
|
+
fsm.update(makeFrame(0, makeHand('openPalm', lmL1), makeHand('openPalm', lmR1)));
|
|
124
|
+
fsm.update(makeFrame(0, makeHand('openPalm', lmL1), makeHand('openPalm', lmR1)));
|
|
125
|
+
// Frame 3: first real zooming frame — sets prevZoomDist, no delta yet
|
|
126
|
+
fsm.update(makeFrame(1, makeHand('openPalm', lmL1), makeHand('openPalm', lmR1)));
|
|
127
|
+
// Frame 4: wider spread → positive delta
|
|
128
|
+
const out = fsm.update(makeFrame(2, makeHand('openPalm', lmL2), makeHand('openPalm', lmR2)));
|
|
129
|
+
expect(out.mode).toBe('zooming');
|
|
130
|
+
expect(out.zoomDelta).not.toBeNull();
|
|
131
|
+
expect(out.zoomDelta).toBeGreaterThan(0);
|
|
132
|
+
});
|
|
133
|
+
// ── reset ──────────────────────────────────────────────────────────────────
|
|
134
|
+
it('reset() returns the FSM to idle', () => {
|
|
135
|
+
enterPanning(fsm);
|
|
136
|
+
expect(fsm.getMode()).toBe('panning');
|
|
137
|
+
fsm.reset();
|
|
138
|
+
expect(fsm.getMode()).toBe('idle');
|
|
139
|
+
});
|
|
140
|
+
// ── dwell timer ────────────────────────────────────────────────────────────
|
|
141
|
+
it('does NOT transition before actionDwellMs elapses', () => {
|
|
142
|
+
const slowFsm = new GestureStateMachine({ ...FAST_TUNING, actionDwellMs: 200 });
|
|
143
|
+
const fistHand = makeHand('fist');
|
|
144
|
+
const out1 = slowFsm.update(makeFrame(0, fistHand, null));
|
|
145
|
+
expect(out1.mode).toBe('idle');
|
|
146
|
+
const out2 = slowFsm.update(makeFrame(100, fistHand, null)); // 100 ms < 200 ms
|
|
147
|
+
expect(out2.mode).toBe('idle');
|
|
148
|
+
const out3 = slowFsm.update(makeFrame(200, fistHand, null)); // 200 ms >= 200 ms
|
|
149
|
+
expect(out3.mode).toBe('panning');
|
|
150
|
+
});
|
|
151
|
+
// ── release grace period ───────────────────────────────────────────────────
|
|
152
|
+
it('stays in panning during release grace period', () => {
|
|
153
|
+
const graceFsm = new GestureStateMachine({ ...FAST_TUNING, releaseGraceMs: 100 });
|
|
154
|
+
enterPanning(graceFsm);
|
|
155
|
+
expect(graceFsm.getMode()).toBe('panning');
|
|
156
|
+
const out1 = graceFsm.update(makeFrame(1, null, null)); // grace starts
|
|
157
|
+
expect(out1.mode).toBe('panning');
|
|
158
|
+
const out2 = graceFsm.update(makeFrame(50, null, null)); // 50 ms < 100 ms
|
|
159
|
+
expect(out2.mode).toBe('panning');
|
|
160
|
+
const out3 = graceFsm.update(makeFrame(101, null, null)); // 101 ms >= 100 ms
|
|
161
|
+
expect(out3.mode).toBe('idle');
|
|
162
|
+
});
|
|
163
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { GestureFrame, GestureMode, WebcamConfig } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* WebcamOverlay
|
|
4
|
+
*
|
|
5
|
+
* Manages a DOM container with:
|
|
6
|
+
* - <video> element (webcam feed)
|
|
7
|
+
* - <canvas> for landmark drawing
|
|
8
|
+
* - mode badge
|
|
9
|
+
*
|
|
10
|
+
* Supports 'corner', 'full', and 'hidden' modes.
|
|
11
|
+
*/
|
|
12
|
+
export declare class WebcamOverlay {
|
|
13
|
+
private container;
|
|
14
|
+
private canvas;
|
|
15
|
+
private ctx;
|
|
16
|
+
private badge;
|
|
17
|
+
private config;
|
|
18
|
+
constructor(config: WebcamConfig);
|
|
19
|
+
/** Attach video element produced by GestureController */
|
|
20
|
+
attachVideo(video: HTMLVideoElement): void;
|
|
21
|
+
/** Mount the overlay into the given parent (usually document.body or map container) */
|
|
22
|
+
mount(parent: HTMLElement): void;
|
|
23
|
+
unmount(): void;
|
|
24
|
+
/** Called each frame with the latest gesture frame and mode. */
|
|
25
|
+
render(frame: GestureFrame | null, mode: GestureMode): void;
|
|
26
|
+
private drawSkeleton;
|
|
27
|
+
private updateBadge;
|
|
28
|
+
private applyContainerStyles;
|
|
29
|
+
updateConfig(config: Partial<WebcamConfig>): void;
|
|
30
|
+
}
|