mudra-skills 3.1.1 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/skills/mudra-master/SKILL.md +1 -1
- package/skills/mudra-master/mudra-preview/SKILL.md +9 -9
- package/skills/mudra-master/mudra-preview/assets/ar-menu.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/document-scroller.html +2 -2
- package/skills/mudra-master/mudra-preview/assets/drum-machine.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/emg-visualizer.html +6 -6
- package/skills/mudra-master/mudra-preview/assets/generative-art.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/gesture-assistant.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/gesture-speech.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/hands-free-desktop.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/model-rotator.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/mudra-duel.html +48 -48
- package/skills/mudra-master/mudra-preview/assets/mudra-monitor.html +55 -55
- package/skills/mudra-master/mudra-preview/assets/mudra-ultimate-template.html +13 -13
- package/skills/mudra-master/mudra-preview/assets/music-sequencer.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/neural-pong.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/neural-snake.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/presentation-controller.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/pressure-painter.html +4 -4
- package/skills/mudra-master/mudra-preview/assets/runner.html +11 -11
- package/skills/mudra-master/mudra-preview/assets/smart-home.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/space-invaders.html +1 -1
- package/skills/mudra-master/mudra-preview/assets/waterful-ring-toss.html +1 -1
- package/skills/mudra-master/mudra-preview/references/agent_protocol.json +34 -34
- package/skills/mudra-master/mudra-preview/references/prompt.md +182 -182
- package/skills/mudra-master/mudra-xr/SKILL.md +7 -7
- package/skills/mudra-master/mudra-xr/references/agent_protocol.json +20 -20
- package/skills/mudra-master/mudra-xr/references/prompt.md +28 -28
- package/skills/mudra-preview/SKILL.md +9 -9
- package/skills/mudra-preview/assets/ar-menu.html +1 -1
- package/skills/mudra-preview/assets/document-scroller.html +2 -2
- package/skills/mudra-preview/assets/drum-machine.html +1 -1
- package/skills/mudra-preview/assets/emg-visualizer.html +6 -6
- package/skills/mudra-preview/assets/generative-art.html +1 -1
- package/skills/mudra-preview/assets/gesture-assistant.html +1 -1
- package/skills/mudra-preview/assets/gesture-speech.html +1 -1
- package/skills/mudra-preview/assets/hands-free-desktop.html +1 -1
- package/skills/mudra-preview/assets/model-rotator.html +1 -1
- package/skills/mudra-preview/assets/mudra-duel.html +48 -48
- package/skills/mudra-preview/assets/mudra-monitor.html +55 -55
- package/skills/mudra-preview/assets/mudra-ultimate-template.html +13 -13
- package/skills/mudra-preview/assets/music-sequencer.html +1 -1
- package/skills/mudra-preview/assets/neural-pong.html +1 -1
- package/skills/mudra-preview/assets/neural-snake.html +1 -1
- package/skills/mudra-preview/assets/presentation-controller.html +1 -1
- package/skills/mudra-preview/assets/pressure-painter.html +4 -4
- package/skills/mudra-preview/assets/runner.html +11 -11
- package/skills/mudra-preview/assets/smart-home.html +1 -1
- package/skills/mudra-preview/assets/space-invaders.html +1 -1
- package/skills/mudra-preview/assets/waterful-ring-toss.html +1 -1
- package/skills/mudra-preview/references/agent_protocol.json +34 -34
- package/skills/mudra-preview/references/prompt.md +182 -182
- package/skills/mudra-xr/SKILL.md +7 -7
- package/skills/mudra-xr/references/agent_protocol.json +20 -20
- package/skills/mudra-xr/references/prompt.md +28 -28
package/README.md
CHANGED
|
@@ -83,7 +83,7 @@ Every generated app includes:
|
|
|
83
83
|
| `nav_direction` | discrete | swipe direction (up/down/left/right) |
|
|
84
84
|
| `imu_acc` | analog | accelerometer (x, y, z) |
|
|
85
85
|
| `imu_gyro` | analog | gyroscope (x, y, z) |
|
|
86
|
-
| `
|
|
86
|
+
| `emg` | analog | 3-channel bio signal |
|
|
87
87
|
|
|
88
88
|
## Requirements
|
|
89
89
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: mudra-master
|
|
3
|
-
version: 3.1.
|
|
3
|
+
version: 3.1.2
|
|
4
4
|
description: Master router for Mudra Band app builds. Use when the user describes an app, experience, prototype, timer, counter, or tool to build with the Mudra Band but has not made clear whether they want a 2D (flat/screen) or 3D (spatial/XR/VR/AR) version. This skill classifies the prompt deterministically and hands off the original prompt verbatim to either mudra-preview (2D) or mudra-xr (3D), asking exactly one 2D-or-3D disambiguation question only when the dimension cannot be inferred. Do not use for pure code edits, repo questions, or any request that is not a Mudra-app build.
|
|
5
5
|
---
|
|
6
6
|
# Mudra Master Skill
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: mudra-preview
|
|
3
|
-
version: 3.1.
|
|
4
|
-
description: Generate a working Mudra Band interactive app preview as a single-file HTML. Use when the user describes a Mudra-controlled experience (gesture, pressure, navigation, IMU,
|
|
3
|
+
version: 3.1.2
|
|
4
|
+
description: Generate a working Mudra Band interactive app preview as a single-file HTML. Use when the user describes a Mudra-controlled experience (gesture, pressure, navigation, IMU, EMG), wants to prototype a Mudra Link app, or asks to build/preview a Mudra app.
|
|
5
5
|
---
|
|
6
6
|
|
|
7
7
|
# Mudra Preview
|
|
@@ -25,7 +25,7 @@ no banner, no modal.
|
|
|
25
25
|
other variant.
|
|
26
26
|
|
|
27
27
|
**Mandatory feature :** Connection state MUST reflect
|
|
28
|
-
the **band**, not the WebSocket. The
|
|
28
|
+
the **band**, not the WebSocket. The Link service accepts socket
|
|
29
29
|
connections even when no band is paired. Every generated app MUST:
|
|
30
30
|
1. Send `{command:"get_status"}` immediately in `ws.onopen` (no waiting for any
|
|
31
31
|
server-initiated frame — the new server sends none).
|
|
@@ -75,7 +75,7 @@ Every generated app MUST include a **compact, always-visible simulator panel** w
|
|
|
75
75
|
| `button` | `Press`, `Release` |
|
|
76
76
|
| `imu_acc` | `Tilt X`, `Tilt Y`, `Tilt Z` (each fires a 5-frame burst at ±2 m/s²) |
|
|
77
77
|
| `imu_gyro` | `Rot X`, `Rot Y`, `Rot Z` (each fires a 5-frame burst at ±10 deg/s) |
|
|
78
|
-
| `
|
|
78
|
+
| `emg` | `Spike` (injects a burst of elevated samples on all 3 channels) |
|
|
79
79
|
|
|
80
80
|
**How each button must fire**
|
|
81
81
|
- `gesture` buttons → `ws.send(JSON.stringify({ command: 'trigger_gesture', data: { type: '<name>' } }))`. This works both when connected to the real band (round-trips through the service) and when the mock is active (mock echoes it back).
|
|
@@ -127,13 +127,13 @@ Rules:
|
|
|
127
127
|
pick the one that fits the interaction (discrete swipes →
|
|
128
128
|
`nav_direction`; continuous cursor/scroll → `navigation`). Never wire
|
|
129
129
|
both into the same app.
|
|
130
|
-
- **IMU+Biometric bundle (`imu_acc` + `imu_gyro` + `
|
|
130
|
+
- **IMU+Biometric bundle (`imu_acc` + `imu_gyro` + `emg`) is
|
|
131
131
|
inseparable.** If the concept needs any one of them, subscribe to all
|
|
132
132
|
three. The bundle is **mutually exclusive** with `navigation` and
|
|
133
133
|
`nav_direction` — pick directional motion OR the IMU+Biometric bundle,
|
|
134
134
|
never both.
|
|
135
135
|
- Other gesture subtypes (`twist`, `double_twist`, etc.) and other
|
|
136
|
-
signals (`button`, `imu_acc`, `imu_gyro`, `
|
|
136
|
+
signals (`button`, `imu_acc`, `imu_gyro`, `emg`) are **off by
|
|
137
137
|
default**. Only include them when the user's prompt names them,
|
|
138
138
|
names a synonym from the Signal Inference table in
|
|
139
139
|
`references/prompt.md` § "Signal Inference Reference", or describes
|
|
@@ -151,9 +151,9 @@ Rules:
|
|
|
151
151
|
- Mode toggle (Manual / Mudra) is **mandatory** in every generated app — see `references/prompt.md` § "Mode Toggle (Manual / Mudra) — Required"
|
|
152
152
|
- Lazy WS lifecycle: open on Manual→Mudra, close on Mudra→Manual. Manual mode opens NO WebSocket.
|
|
153
153
|
- Subscribe one signal per command: `{ "command": "subscribe", "signal": "<name>" }` — singular `signal`, never `signals`, never an array
|
|
154
|
-
- Motion modes are mutually exclusive: Pointer (`navigation`+`button`) / Direction (`nav_direction`) / IMU+Biometric (`imu_acc`+`imu_gyro`+`
|
|
155
|
-
- IMU+Biometric bundle: `imu_acc`, `imu_gyro`, `
|
|
154
|
+
- Motion modes are mutually exclusive: Pointer (`navigation`+`button`) / Direction (`nav_direction`) / IMU+Biometric (`imu_acc`+`imu_gyro`+`emg`, always all three together)
|
|
155
|
+
- IMU+Biometric bundle: `imu_acc`, `imu_gyro`, `emg` always subscribed together — never partially. The bundle is mutually exclusive with `navigation` and `nav_direction`.
|
|
156
156
|
- `gesture` and `pressure` are mutually exclusive — never combine them
|
|
157
|
-
- `button` combines freely with `gesture`, `pressure`, `
|
|
157
|
+
- `button` combines freely with `gesture`, `pressure`, `emg`, `imu_acc`, `imu_gyro` (subject to the Pointer/Direction/IMU motion-mode XOR — `button` belongs to Pointer mode and never combines with `nav_direction`).
|
|
158
158
|
- **Navigation sensitivity is gentle by default**: keyboard `step = 3`, sim button `±3`, cursor multiplier `0.002`. Raise only when the prompt explicitly asks for fast/snappy movement. See `references/prompt.md` § "Navigation sensitivity defaults".
|
|
159
159
|
- Canonical protocol JSON: `references/agent_protocol.json`
|
|
@@ -154,7 +154,7 @@ function triggerGesture(type) {
|
|
|
154
154
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
-
/* ── Simulation (when
|
|
157
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
158
158
|
function startSim() {
|
|
159
159
|
if (simTimer) return;
|
|
160
160
|
simulating = true;
|
|
@@ -177,7 +177,7 @@ function triggerGesture(type) {
|
|
|
177
177
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
-
/* ── Simulation (when
|
|
180
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
181
181
|
function startSim() {
|
|
182
182
|
if (simTimer) return;
|
|
183
183
|
simulating = true;
|
|
@@ -250,7 +250,7 @@ const sections = [
|
|
|
250
250
|
{
|
|
251
251
|
title: 'WebSocket Protocol v2',
|
|
252
252
|
paragraphs: [
|
|
253
|
-
'The Mudra
|
|
253
|
+
'The Mudra Link application exposes a WebSocket server on localhost port 8766. Client applications connect and subscribe to specific signals they need: gesture, pressure, navigation, imu_acc, imu_gyro, emg, button, and battery.',
|
|
254
254
|
'Each signal is subscribed individually using the command format. Data arrives as typed JSON messages with the signal type and a data payload containing the relevant values and a timestamp.',
|
|
255
255
|
'Connection management follows standard WebSocket patterns with automatic reconnection. Applications should implement simulation mode to generate realistic synthetic data for development without a physical device.',
|
|
256
256
|
'The protocol supports bidirectional communication. The trigger_gesture command allows applications to programmatically trigger gesture events, useful for testing and accessibility features.'
|
|
@@ -328,7 +328,7 @@ class MudraWebSocket {
|
|
|
328
328
|
this._useMock = true;
|
|
329
329
|
setTimeout(() => {
|
|
330
330
|
this._trigger('open', {});
|
|
331
|
-
this._emit({ type: 'connection_status', data: { status: 'connected', message: 'Mudra
|
|
331
|
+
this._emit({ type: 'connection_status', data: { status: 'connected', message: 'Mudra Link ready (simulated)' }, timestamp: Date.now() });
|
|
332
332
|
}, 100);
|
|
333
333
|
// Gesture: every 4s (slowed down so it's not distracting)
|
|
334
334
|
this._timers.push(setInterval(() => {
|
|
@@ -124,7 +124,7 @@
|
|
|
124
124
|
<script>
|
|
125
125
|
/* ── Mudra Protocol Connection (v2) ────────────────────────── */
|
|
126
126
|
const WS_URL = 'ws://127.0.0.1:8766';
|
|
127
|
-
const SIGNALS = ['
|
|
127
|
+
const SIGNALS = ['emg', 'gesture', 'pressure'];
|
|
128
128
|
|
|
129
129
|
let ws, reconnectTimer, simTimer;
|
|
130
130
|
let wsOpen = false, deviceReady = false, simulating = true;
|
|
@@ -170,16 +170,16 @@ function triggerGesture(type) {
|
|
|
170
170
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
-
/* ── Simulation (when
|
|
173
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
174
174
|
function startSim() {
|
|
175
175
|
if (simTimer) return;
|
|
176
176
|
simulating = true;
|
|
177
177
|
let t = 0;
|
|
178
178
|
simTimer = setInterval(() => {
|
|
179
179
|
t += 0.05;
|
|
180
|
-
// Generate realistic
|
|
180
|
+
// Generate realistic EMG data with varying amplitude and frequency
|
|
181
181
|
const burstEnvelope = 0.5 + 0.5 * Math.sin(t * 0.15);
|
|
182
|
-
emit('
|
|
182
|
+
emit('emg', {
|
|
183
183
|
values: [
|
|
184
184
|
burstEnvelope * (0.4 * Math.sin(t * 3.7 + Math.sin(t * 0.8)) + 0.15 * Math.sin(t * 11.3) + (Math.random() - 0.5) * 0.2),
|
|
185
185
|
burstEnvelope * (0.35 * Math.cos(t * 4.1 + Math.cos(t * 0.6)) + 0.12 * Math.sin(t * 13.7) + (Math.random() - 0.5) * 0.18),
|
|
@@ -195,7 +195,7 @@ function startSim() {
|
|
|
195
195
|
const g = ['tap', 'double_tap', 'twist', 'double_twist'];
|
|
196
196
|
emit('gesture', { type: g[Math.floor(Math.random() * g.length)], confidence: 0.7 + Math.random() * 0.3, timestamp: Date.now() });
|
|
197
197
|
}
|
|
198
|
-
}, 16); // Higher frequency for
|
|
198
|
+
}, 16); // Higher frequency for EMG data
|
|
199
199
|
}
|
|
200
200
|
|
|
201
201
|
function stopSim() { clearInterval(simTimer); simTimer = null; simulating = false; }
|
|
@@ -237,7 +237,7 @@ resizeCanvases();
|
|
|
237
237
|
window.addEventListener('resize', resizeCanvases);
|
|
238
238
|
|
|
239
239
|
/* ── Signal Handlers ───────────────────────────────────────── */
|
|
240
|
-
on('
|
|
240
|
+
on('emg', (d) => {
|
|
241
241
|
if (frozen) return;
|
|
242
242
|
const vals = d.values || [0, 0, 0];
|
|
243
243
|
for (let ch = 0; ch < 3; ch++) {
|
|
@@ -139,7 +139,7 @@ function triggerGesture(type) {
|
|
|
139
139
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
140
140
|
}
|
|
141
141
|
|
|
142
|
-
/* ── Simulation (when
|
|
142
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
143
143
|
function startSim() {
|
|
144
144
|
if (simTimer) return;
|
|
145
145
|
simulating = true;
|
|
@@ -163,7 +163,7 @@ function triggerGesture(type) {
|
|
|
163
163
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
164
164
|
}
|
|
165
165
|
|
|
166
|
-
/* ── Simulation (when
|
|
166
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
167
167
|
function startSim() {
|
|
168
168
|
if (simTimer) return;
|
|
169
169
|
simulating = true;
|
|
@@ -165,7 +165,7 @@ function triggerGesture(type) {
|
|
|
165
165
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
-
/* ── Simulation (when
|
|
168
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
169
169
|
function startSim() {
|
|
170
170
|
if (simTimer) return;
|
|
171
171
|
simulating = true;
|
|
@@ -174,7 +174,7 @@ function triggerGesture(type) {
|
|
|
174
174
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
/* ── Simulation (when
|
|
177
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
178
178
|
function startSim() {
|
|
179
179
|
if (simTimer) return;
|
|
180
180
|
simulating = true;
|
|
@@ -183,7 +183,7 @@ function triggerGesture(type) {
|
|
|
183
183
|
else emit('gesture', { type, confidence: 0.95, timestamp: Date.now() });
|
|
184
184
|
}
|
|
185
185
|
|
|
186
|
-
/* ── Simulation (when
|
|
186
|
+
/* ── Simulation (when Link not reachable) ─────────────── */
|
|
187
187
|
function startSim() {
|
|
188
188
|
if (simTimer) return;
|
|
189
189
|
simulating = true;
|
|
@@ -211,10 +211,10 @@
|
|
|
211
211
|
.sensor-vals { display: flex; gap: 12px; font-family: 'Menlo', monospace; font-size: 0.82rem; }
|
|
212
212
|
.sv-x { color: #ff6b6b; } .sv-y { color: #4ecdc4; } .sv-z { color: #ffe66d; }
|
|
213
213
|
.sensor-canvas { width: 100%; height: 70px; background: rgba(0,0,0,0.2); border-radius: 8px; }
|
|
214
|
-
.
|
|
215
|
-
.
|
|
216
|
-
.
|
|
217
|
-
.
|
|
214
|
+
.emg-wave-canvas { width: 100%; height: 70px; border-radius: 6px; background: rgba(0,0,0,0.2); }
|
|
215
|
+
.emg-legend { display: flex; justify-content: center; gap: 14px; margin-top: 5px; font-size: 0.72rem; }
|
|
216
|
+
.emg-legend-item { display: flex; align-items: center; gap: 4px; }
|
|
217
|
+
.emg-legend-color { width: 12px; height: 3px; border-radius: 2px; }
|
|
218
218
|
|
|
219
219
|
.live-recog { text-align: center; padding: 10px; }
|
|
220
220
|
.live-icon { font-size: 2rem; margin-bottom: 4px; }
|
|
@@ -629,12 +629,12 @@
|
|
|
629
629
|
<canvas id="canvasGyro" class="sensor-canvas"></canvas>
|
|
630
630
|
</div>
|
|
631
631
|
<div class="sensor-card">
|
|
632
|
-
<div class="sensor-header"><span class="sensor-title">
|
|
633
|
-
<canvas id="canvasSnc" class="
|
|
634
|
-
<div class="
|
|
635
|
-
<div class="
|
|
636
|
-
<div class="
|
|
637
|
-
<div class="
|
|
632
|
+
<div class="sensor-header"><span class="sensor-title">EMG (Muscle Activity)</span></div>
|
|
633
|
+
<canvas id="canvasSnc" class="emg-wave-canvas"></canvas>
|
|
634
|
+
<div class="emg-legend">
|
|
635
|
+
<div class="emg-legend-item"><div class="emg-legend-color" style="background:#00d4ff;"></div><span>CH1</span></div>
|
|
636
|
+
<div class="emg-legend-item"><div class="emg-legend-color" style="background:#22c55e;"></div><span>CH2</span></div>
|
|
637
|
+
<div class="emg-legend-item"><div class="emg-legend-color" style="background:#ff6b6b;"></div><span>CH3</span></div>
|
|
638
638
|
</div>
|
|
639
639
|
</div>
|
|
640
640
|
<div class="sensor-card">
|
|
@@ -855,12 +855,12 @@
|
|
|
855
855
|
<canvas id="canvasGyroDebug" class="sensor-canvas"></canvas>
|
|
856
856
|
</div>
|
|
857
857
|
<div class="sensor-card">
|
|
858
|
-
<div class="sensor-header"><span class="sensor-title">
|
|
859
|
-
<canvas id="canvasSncDebug" class="
|
|
860
|
-
<div class="
|
|
861
|
-
<div class="
|
|
862
|
-
<div class="
|
|
863
|
-
<div class="
|
|
858
|
+
<div class="sensor-header"><span class="sensor-title">EMG (Muscle Activity)</span></div>
|
|
859
|
+
<canvas id="canvasSncDebug" class="emg-wave-canvas"></canvas>
|
|
860
|
+
<div class="emg-legend">
|
|
861
|
+
<div class="emg-legend-item"><div class="emg-legend-color" style="background:#00d4ff;"></div><span>CH1</span></div>
|
|
862
|
+
<div class="emg-legend-item"><div class="emg-legend-color" style="background:#22c55e;"></div><span>CH2</span></div>
|
|
863
|
+
<div class="emg-legend-item"><div class="emg-legend-color" style="background:#ff6b6b;"></div><span>CH3</span></div>
|
|
864
864
|
</div>
|
|
865
865
|
</div>
|
|
866
866
|
<div class="sensor-card" style="text-align:center;">
|
|
@@ -943,7 +943,7 @@
|
|
|
943
943
|
<div style="margin-top:6px;"><strong>Readings:</strong></div>
|
|
944
944
|
<div>Acc: <span id="dbgAcc">-</span></div>
|
|
945
945
|
<div>Gyro: <span id="dbgGyro">-</span></div>
|
|
946
|
-
<div>
|
|
946
|
+
<div>EMG: <span id="dbgSnc">-</span></div>
|
|
947
947
|
</div>
|
|
948
948
|
|
|
949
949
|
<button class="fab fab-debug" id="fabDebug">D</button>
|
|
@@ -953,12 +953,12 @@
|
|
|
953
953
|
<script>
|
|
954
954
|
// ============================================================
|
|
955
955
|
// MUDRA DUEL v2 — Neural Network Gesture Recognition
|
|
956
|
-
// Signals: imu_acc + imu_gyro +
|
|
956
|
+
// Signals: imu_acc + imu_gyro + emg + gesture
|
|
957
957
|
// 3 gestures: rock, paper, scissors (no idle)
|
|
958
958
|
// ============================================================
|
|
959
959
|
|
|
960
960
|
const WS_URL = "ws://127.0.0.1:8766";
|
|
961
|
-
const
|
|
961
|
+
const EMG_BUF = 500;
|
|
962
962
|
const GESTURES = ["rock", "paper", "scissors"];
|
|
963
963
|
const ICONS = { rock: "\u{270A}", paper: "\u{270B}", scissors: "\u{270C}" };
|
|
964
964
|
const INFO = {
|
|
@@ -974,13 +974,13 @@
|
|
|
974
974
|
curGestIdx: 0,
|
|
975
975
|
gestureData: { rock: { samples: [] }, paper: { samples: [] }, scissors: { samples: [] } },
|
|
976
976
|
isRecording: false,
|
|
977
|
-
recBuf: { imu: [],
|
|
977
|
+
recBuf: { imu: [], emg: [] },
|
|
978
978
|
samplesReq: 5,
|
|
979
979
|
accHist: { x:[], y:[], z:[] }, gyroHist: { x:[], y:[], z:[] },
|
|
980
980
|
sncBufs: [[], [], []],
|
|
981
981
|
curAcc: [0,0,0], curGyro: [0,0,0], curSnc: [0,0,0],
|
|
982
982
|
imuSeq: 0, sncSeq: 0, // incremented on each new WebSocket reading
|
|
983
|
-
inferBuf: { imu: [],
|
|
983
|
+
inferBuf: { imu: [], emg: [] }, // rolling buffer for live inference (~1.5s)
|
|
984
984
|
pScore: 0, aScore: 0, roundsToWin: 3, detectionMs: 1500,
|
|
985
985
|
isPlaying: false, roundHistory: [],
|
|
986
986
|
usage: { rock:0, paper:0, scissors:0 },
|
|
@@ -1005,7 +1005,7 @@
|
|
|
1005
1005
|
let quickPlayGesture = null;
|
|
1006
1006
|
|
|
1007
1007
|
// ============ MODEL (sklearn-trained, JS inference) ============
|
|
1008
|
-
// Features: acc(mean3+std3+mag2=8) + gyro(mean3+std3=6) +
|
|
1008
|
+
// Features: acc(mean3+std3+mag2=8) + gyro(mean3+std3=6) + emg(mean3+std3+rms3+min3+max3+range3=18) = 32
|
|
1009
1009
|
const FEAT_DIM = 32;
|
|
1010
1010
|
const TRAINER_URL = "http://127.0.0.1:8767";
|
|
1011
1011
|
|
|
@@ -1238,8 +1238,8 @@
|
|
|
1238
1238
|
f[idx++] = gyroS.mean[0]; f[idx++] = gyroS.mean[1]; f[idx++] = gyroS.mean[2];
|
|
1239
1239
|
f[idx++] = gyroS.std[0]; f[idx++] = gyroS.std[1]; f[idx++] = gyroS.std[2];
|
|
1240
1240
|
|
|
1241
|
-
//
|
|
1242
|
-
//
|
|
1241
|
+
// EMG: mean(3) + std(3) + rms(3) + min(3) + max(3) + range(3) = 18
|
|
1242
|
+
// EMG is the KEY signal for hand pose — extract rich features
|
|
1243
1243
|
const sncS = statsRich3D(sncData);
|
|
1244
1244
|
f[idx++] = sncS.mean[0]; f[idx++] = sncS.mean[1]; f[idx++] = sncS.mean[2];
|
|
1245
1245
|
f[idx++] = sncS.std[0]; f[idx++] = sncS.std[1]; f[idx++] = sncS.std[2];
|
|
@@ -1255,15 +1255,15 @@
|
|
|
1255
1255
|
function extractSampleFeatures(sample) {
|
|
1256
1256
|
const accData = sample.imu_data.accelerometer;
|
|
1257
1257
|
const gyroData = sample.imu_data.gyroscope;
|
|
1258
|
-
const sncData = sample.
|
|
1258
|
+
const sncData = sample.emg_data.channels;
|
|
1259
1259
|
return extractFeatures(accData, gyroData, sncData);
|
|
1260
1260
|
}
|
|
1261
1261
|
|
|
1262
|
-
// Unified extraction from live buffer { imu: [{acc, gyro}],
|
|
1262
|
+
// Unified extraction from live buffer { imu: [{acc, gyro}], emg: [[c1,c2,c3]] }
|
|
1263
1263
|
function extractLiveFeatures(buffer) {
|
|
1264
1264
|
const accData = buffer.imu.map(d => d.acc);
|
|
1265
1265
|
const gyroData = buffer.imu.map(d => d.gyro);
|
|
1266
|
-
const sncData = buffer.
|
|
1266
|
+
const sncData = buffer.emg;
|
|
1267
1267
|
return extractFeatures(accData, gyroData, sncData);
|
|
1268
1268
|
}
|
|
1269
1269
|
|
|
@@ -1432,13 +1432,13 @@
|
|
|
1432
1432
|
if (msg.type === "connection_status") { S.deviceConnected = msg.data?.status === "connected"; updateStatus(); return; }
|
|
1433
1433
|
if (msg.type === "imu_acc") handleImu("acc", msg.data);
|
|
1434
1434
|
if (msg.type === "imu_gyro") handleImu("gyro", msg.data);
|
|
1435
|
-
if (msg.type === "
|
|
1435
|
+
if (msg.type === "emg") handleEmg(msg.data);
|
|
1436
1436
|
if (msg.type === "gesture") handleMudraGesture(msg.data);
|
|
1437
1437
|
};
|
|
1438
1438
|
}
|
|
1439
1439
|
|
|
1440
1440
|
function subscribeSignals() {
|
|
1441
|
-
["imu_acc", "imu_gyro", "
|
|
1441
|
+
["imu_acc", "imu_gyro", "emg", "gesture"].forEach(signal => send({ command: "subscribe", signal }));
|
|
1442
1442
|
}
|
|
1443
1443
|
|
|
1444
1444
|
function send(p) { if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify(p)); }
|
|
@@ -1477,12 +1477,12 @@
|
|
|
1477
1477
|
if (S.isRecording) S.recBuf.imu.push({ timestamp: Date.now(), type, values: [...vals] });
|
|
1478
1478
|
}
|
|
1479
1479
|
|
|
1480
|
-
function
|
|
1480
|
+
function handleEmg(data) {
|
|
1481
1481
|
const channels = Array.isArray(data?.values) && data.values.length === 3 ? data.values : [[0],[0],[0]];
|
|
1482
1482
|
for (let i = 0; i < 3; i++) {
|
|
1483
1483
|
const ch = Array.isArray(channels[i]) ? channels[i] : [0];
|
|
1484
1484
|
S.sncBufs[i].push(...ch);
|
|
1485
|
-
if (S.sncBufs[i].length >
|
|
1485
|
+
if (S.sncBufs[i].length > EMG_BUF) S.sncBufs[i].splice(0, S.sncBufs[i].length - EMG_BUF);
|
|
1486
1486
|
}
|
|
1487
1487
|
S.curSnc = S.sncBufs.map(buf => Number(buf[buf.length - 1] || 0));
|
|
1488
1488
|
S.sncSeq++;
|
|
@@ -1491,7 +1491,7 @@
|
|
|
1491
1491
|
// Record ALL samples from this batch, not just the latest
|
|
1492
1492
|
const batchLen = Math.max(...channels.map(ch => Array.isArray(ch) ? ch.length : 1));
|
|
1493
1493
|
for (let s = 0; s < batchLen; s++) {
|
|
1494
|
-
S.recBuf.
|
|
1494
|
+
S.recBuf.emg.push({
|
|
1495
1495
|
timestamp: Date.now(),
|
|
1496
1496
|
values: [
|
|
1497
1497
|
Number(Array.isArray(channels[0]) ? (channels[0][s] ?? channels[0][channels[0].length-1]) : 0),
|
|
@@ -1609,13 +1609,13 @@
|
|
|
1609
1609
|
if (dots[curLen]) dots[curLen].classList.add("recording");
|
|
1610
1610
|
|
|
1611
1611
|
await sleep(300);
|
|
1612
|
-
S.isRecording = true; S.recBuf = { imu: [],
|
|
1612
|
+
S.isRecording = true; S.recBuf = { imu: [], emg: [] };
|
|
1613
1613
|
await sleep(1500);
|
|
1614
1614
|
S.isRecording = false;
|
|
1615
1615
|
btn.textContent = "Record Sample"; btn.classList.remove("recording"); btn.disabled = false;
|
|
1616
1616
|
Sfx.playRecordStop();
|
|
1617
1617
|
|
|
1618
|
-
if (S.recBuf.imu.length >= 10 || S.recBuf.
|
|
1618
|
+
if (S.recBuf.imu.length >= 10 || S.recBuf.emg.length >= 10) {
|
|
1619
1619
|
const stab = calcStability(S.recBuf);
|
|
1620
1620
|
if (stab < 0.3) { toast(`Too much movement! (${(stab*100).toFixed(0)}%)`, "error"); updateCalUI(); return; }
|
|
1621
1621
|
S.gestureData[g].samples.push({
|
|
@@ -1625,7 +1625,7 @@
|
|
|
1625
1625
|
accelerometer: S.recBuf.imu.filter(d => d.type === "acc").map(d => d.values),
|
|
1626
1626
|
gyroscope: S.recBuf.imu.filter(d => d.type === "gyro").map(d => d.values)
|
|
1627
1627
|
},
|
|
1628
|
-
|
|
1628
|
+
emg_data: { channels: S.recBuf.emg.map(d => d.values) }
|
|
1629
1629
|
});
|
|
1630
1630
|
const qt = stab > 0.7 ? "Excellent!" : stab > 0.5 ? "Good" : "OK";
|
|
1631
1631
|
toast(`Sample ${S.gestureData[g].samples.length} recorded! ${qt}`, "success");
|
|
@@ -1638,7 +1638,7 @@
|
|
|
1638
1638
|
function calcStability(buf) {
|
|
1639
1639
|
const accD = buf.imu.filter(d => d.type === "acc").map(d => d.values);
|
|
1640
1640
|
const gyroD = buf.imu.filter(d => d.type === "gyro").map(d => d.values);
|
|
1641
|
-
const sncD = buf.
|
|
1641
|
+
const sncD = buf.emg.map(d => d.values);
|
|
1642
1642
|
if (accD.length < 5) return 0.5;
|
|
1643
1643
|
const accVar = variance3D(accD);
|
|
1644
1644
|
const gyroVar = gyroD.length >= 5 ? variance3D(gyroD) : 0;
|
|
@@ -1760,7 +1760,7 @@
|
|
|
1760
1760
|
const gyroE = S.recBuf.imu.filter(d => d.type === "gyro");
|
|
1761
1761
|
return {
|
|
1762
1762
|
imu: accE.map((a, i) => ({ acc: [...a.values], gyro: gyroE[i] ? [...gyroE[i].values] : [0,0,0] })),
|
|
1763
|
-
|
|
1763
|
+
emg: S.recBuf.emg.map(d => [...d.values])
|
|
1764
1764
|
};
|
|
1765
1765
|
};
|
|
1766
1766
|
|
|
@@ -1790,7 +1790,7 @@
|
|
|
1790
1790
|
// Start recording once ready period is over
|
|
1791
1791
|
if (el >= readyMs && !S.isRecording) {
|
|
1792
1792
|
S.isRecording = true;
|
|
1793
|
-
S.recBuf = { imu: [],
|
|
1793
|
+
S.recBuf = { imu: [], emg: [] };
|
|
1794
1794
|
}
|
|
1795
1795
|
|
|
1796
1796
|
if (el < readyMs + detectMs) {
|
|
@@ -1873,7 +1873,7 @@
|
|
|
1873
1873
|
|
|
1874
1874
|
// Use event-driven recording (same as calibration & debug) for full data density
|
|
1875
1875
|
S.isRecording = true;
|
|
1876
|
-
S.recBuf = { imu: [],
|
|
1876
|
+
S.recBuf = { imu: [], emg: [] };
|
|
1877
1877
|
|
|
1878
1878
|
// Helper: build merged buffer from recBuf for prediction
|
|
1879
1879
|
const buildBuf = () => {
|
|
@@ -1881,7 +1881,7 @@
|
|
|
1881
1881
|
const gyroE = S.recBuf.imu.filter(d => d.type === "gyro");
|
|
1882
1882
|
return {
|
|
1883
1883
|
imu: accE.map((a, i) => ({ acc: [...a.values], gyro: gyroE[i] ? [...gyroE[i].values] : [0,0,0] })),
|
|
1884
|
-
|
|
1884
|
+
emg: S.recBuf.emg.map(d => [...d.values])
|
|
1885
1885
|
};
|
|
1886
1886
|
};
|
|
1887
1887
|
|
|
@@ -1984,7 +1984,7 @@
|
|
|
1984
1984
|
accelerometer: lastRoundRecBuf.imu.filter(d => d.type === "acc").map(d => d.values),
|
|
1985
1985
|
gyroscope: lastRoundRecBuf.imu.filter(d => d.type === "gyro").map(d => d.values)
|
|
1986
1986
|
},
|
|
1987
|
-
|
|
1987
|
+
emg_data: { channels: lastRoundRecBuf.emg.map(d => d.values) }
|
|
1988
1988
|
});
|
|
1989
1989
|
S.debugNewSamples = (S.debugNewSamples || 0) + 1;
|
|
1990
1990
|
|
|
@@ -2116,8 +2116,8 @@
|
|
|
2116
2116
|
inferLastImuSeq = S.imuSeq;
|
|
2117
2117
|
}
|
|
2118
2118
|
if (S.sncSeq !== inferLastSncSeq) {
|
|
2119
|
-
S.inferBuf.
|
|
2120
|
-
if (S.inferBuf.
|
|
2119
|
+
S.inferBuf.emg.push([...S.curSnc]);
|
|
2120
|
+
if (S.inferBuf.emg.length > INFER_BUF_MAX) S.inferBuf.emg.shift();
|
|
2121
2121
|
inferLastSncSeq = S.sncSeq;
|
|
2122
2122
|
}
|
|
2123
2123
|
renderIMU(accCtx, S.accHist, accCanvas);
|
|
@@ -2629,7 +2629,7 @@
|
|
|
2629
2629
|
|
|
2630
2630
|
// Record 1.5s of sensor data (same mechanism as calibration)
|
|
2631
2631
|
S.isRecording = true;
|
|
2632
|
-
S.recBuf = { imu: [],
|
|
2632
|
+
S.recBuf = { imu: [], emg: [] };
|
|
2633
2633
|
await sleep(1500);
|
|
2634
2634
|
S.isRecording = false;
|
|
2635
2635
|
|
|
@@ -2639,7 +2639,7 @@
|
|
|
2639
2639
|
Sfx.playRecordStop();
|
|
2640
2640
|
|
|
2641
2641
|
// Need enough data
|
|
2642
|
-
if (S.recBuf.imu.length < 10 && S.recBuf.
|
|
2642
|
+
if (S.recBuf.imu.length < 10 && S.recBuf.emg.length < 10) {
|
|
2643
2643
|
toast("Not enough sensor data. Check connection.", "error");
|
|
2644
2644
|
S.debugIsCapturing = false;
|
|
2645
2645
|
S.debugPhase = 'live';
|
|
@@ -2647,7 +2647,7 @@
|
|
|
2647
2647
|
}
|
|
2648
2648
|
|
|
2649
2649
|
// Build merged buffer: recBuf stores separate {type:"acc"} and {type:"gyro"} entries
|
|
2650
|
-
// predictGesture expects {imu: [{acc, gyro}],
|
|
2650
|
+
// predictGesture expects {imu: [{acc, gyro}], emg: [[c1,c2,c3]]}
|
|
2651
2651
|
const accEntries = S.recBuf.imu.filter(d => d.type === "acc");
|
|
2652
2652
|
const gyroEntries = S.recBuf.imu.filter(d => d.type === "gyro");
|
|
2653
2653
|
const mergedBuf = {
|
|
@@ -2655,7 +2655,7 @@
|
|
|
2655
2655
|
acc: [...a.values],
|
|
2656
2656
|
gyro: gyroEntries[i] ? [...gyroEntries[i].values] : [0, 0, 0]
|
|
2657
2657
|
})),
|
|
2658
|
-
|
|
2658
|
+
emg: S.recBuf.emg.map(d => [...d.values])
|
|
2659
2659
|
};
|
|
2660
2660
|
|
|
2661
2661
|
// Run prediction on captured buffer
|
|
@@ -2722,7 +2722,7 @@
|
|
|
2722
2722
|
accelerometer: recBuf.imu.filter(d => d.type === "acc").map(d => d.values),
|
|
2723
2723
|
gyroscope: recBuf.imu.filter(d => d.type === "gyro").map(d => d.values)
|
|
2724
2724
|
},
|
|
2725
|
-
|
|
2725
|
+
emg_data: { channels: recBuf.emg.map(d => d.values) }
|
|
2726
2726
|
});
|
|
2727
2727
|
|
|
2728
2728
|
S.debugNewSamples++;
|