mudra-skills 3.1.2 → 3.2.0

package/skills/mudra-xr/references/agent_protocol.json

@@ -74,7 +74,7 @@
         "sprint",
         "charge"
       ],
-      "pressure": [
+      "direct_pressure": [
         "slide",
         "volume",
         "size",
@@ -83,7 +83,31 @@
         "opacity",
         "brush",
         "zoom",
-        "analog"
+        "analog",
+        "pressure",
+        "press harder",
+        "force"
+      ],
+      "pinch_pressure": [
+        "pinch",
+        "squeeze",
+        "pinch and hold",
+        "tap then squeeze",
+        "grab and scale",
+        "pinch-to-zoom",
+        "hold to charge"
+      ],
+      "imu_quaternion": [
+        "hand orientation",
+        "wrist orientation",
+        "absolute orientation",
+        "aim",
+        "point at",
+        "heading",
+        "which way the hand is pointing",
+        "roll pitch yaw",
+        "quaternion",
+        "1:1 rotation"
       ],
       "navigation": [
         "move",
@@ -98,10 +122,9 @@
       ],
       "imu_acc+imu_gyro": [
         "tilt",
-        "orientation",
+        "shake",
+        "acceleration",
         "angle",
-        "rotate",
-        "3D",
         "balance",
         "level"
       ],
@@ -120,7 +143,9 @@
         "nerve"
       ]
     },
-    "when_ambiguous": "If the user's concept could use either navigation or IMU, ask which fits better. Also ask when two motion modes (Pointer / Direction / IMU) map equally well — do not silently pick. Default to 'direction' when motion language is present but no clear winner."
+    "pressure_mode_rule": "There is no signal called 'pressure' — that was the old name for pinch_pressure. Pick exactly one of direct_pressure or pinch_pressure. direct_pressure is the NEW continuous ungated stream (always on, no gating; default). Requires firmware 6.0.12.11 and above. pinch_pressure is the ORIGINAL tap-to-release filtered stream (values stream only between tap and release) and works on older firmware. Choose pinch_pressure only for explicit commit-then-modulate interactions (grab-and-scale, pinch-to-zoom, hold-to-charge). Do not ask the user which mode — infer it.",
+    "orientation_rule": "For aiming, heading, pose gating, or 1:1 rotation of a mesh, use imu_quaternion. It is absolute and drift-free, needs no fusion on your side, belongs to no bundle, and is exempt from the Pointer/Direction/IMU motion-mode XOR — so it combines with navigation and nav_direction. Requires firmware 6.0.12.11 and above. Reach for imu_acc + imu_gyro only when the concept wants raw acceleration or shake.",
+    "when_ambiguous": "If the user's concept could use either navigation or IMU, ask which fits better. Also ask when two motion modes (Pointer / Direction / IMU) map equally well — do not silently pick. Default to 'direction' when motion language is present but no clear winner. Note: pure orientation is not an ambiguity — use imu_quaternion, which conflicts with nothing."
   },
   "creative_proposals": {
     "description": "When the user's concept maps to one signal, propose a complementary one that respects motion-mode exclusivity. Keep it to one sentence.",
@@ -177,9 +202,9 @@
         "Forward gesture.double_tap → onSelectEnd for release"
       ]
     },
-    "pressure": {
-      "description": "Finger pressure (0-100%)",
-      "use_for": "Analog control (scale, zoom, opacity, intensity)",
+    "direct_pressure": {
+      "description": "Finger pressure 0–100, normalized 0–1. NEW continuous ungated stream — always on, no tap/release gating. The default pressure signal.",
+      "use_for": "Analog control (scale, zoom, opacity, intensity) that is live whenever the finger is pressing, with no gesture gate",
       "examples": [
         "Brush size",
         "Zoom level",
@@ -188,7 +213,7 @@
         "Mesh scale"
       ],
       "data_format": {
-        "type": "pressure",
+        "type": "direct_pressure",
         "data": {
           "value": 50,
           "normalized": 0.5,
@@ -197,8 +222,70 @@
         "timestamp": 1234567890
       },
       "notes": [
+        "Finger pressure 0–100, normalized 0–1",
         "value: 0-100 integer",
-        "normalized: 0.0-1.0 float (convenient for scale, hue, opacity)"
+        "normalized: 0.0-1.0 float (convenient for scale, hue, opacity)",
+        "Mutually exclusive with pinch_pressure — subscribe to exactly one",
+        "Requires firmware 6.0.12.11 and above.",
+        "'pressure' is not a valid signal name. That was the old name for pinch_pressure. direct_pressure is a new signal, not a rename."
+      ]
+    },
+    "pinch_pressure": {
+      "description": "Finger pressure 0–100, normalized 0–1. ORIGINAL tap-to-release filtered stream. Formerly named 'pressure'. Values stream only between tap and release: streaming starts on tap, the value falls off on release, and streaming stops until the next tap.",
+      "use_for": "Analog control that only begins once the user has committed with a pinch",
+      "examples": [
+        "Pinch a mesh, then squeeze to scale it",
+        "Pinch and hold to charge a throw",
+        "Pinch-to-zoom on a virtual screen",
+        "Grab-and-drag with force-sensitive resistance"
+      ],
+      "data_format": {
+        "type": "pinch_pressure",
+        "data": {
+          "value": 50,
+          "normalized": 0.5,
+          "timestamp": 1234567890
+        },
+        "timestamp": 1234567890
+      },
+      "notes": [
+        "Finger pressure 0–100, normalized 0–1",
+        "Identical payload to direct_pressure — only the acquisition window differs",
+        "Values stream only between tap and release; expect a run of frames bracketed by rest",
+        "Mutually exclusive with direct_pressure — subscribe to exactly one",
+        "Works on older firmware — not gated on 6.0.12.11",
+        "In XR this pairs naturally with a latched target: pinch to grab an object, squeeze to modulate it, release to commit"
+      ]
+    },
+    "imu_quaternion": {
+      "display_name": "Hand Orientation",
+      "description": "Absolute hand orientation as a stream of unit quaternions [w, x, y, z]",
+      "use_for": "Aiming, ray direction, 1:1 mesh rotation, heading, pose gating — no drift, no fusion needed",
+      "examples": [
+        "Aim a ray/reticle by pointing the hand",
+        "Rotate a model 1:1 with the wrist",
+        "Orient a virtual screen to face the hand",
+        "Palm-down / palm-up pose gate",
+        "Compass heading in a spatial HUD"
+      ],
+      "data_format": {
+        "type": "imu_quaternion",
+        "data": {
+          "values": [[0.7071, 0.0, 0.7071, 0.0], [0.706, 0.01, 0.708, 0.0]],
+          "frequency": 50,
+          "frequency_std": 0.4,
+          "timestamp": 1234567890
+        },
+        "timestamp": 1234567890
+      },
+      "notes": [
+        "Requires firmware 6.0.12.11 and above.",
+        "CRITICAL SHAPE DIFFERENCE: values is a LIST OF SAMPLES, each a 4-element [w, x, y, z] array. One nesting level deeper than imu_acc / imu_gyro, whose values is three flat per-axis arrays. Never assume the shapes match.",
+        "Read the latest sample per frame: const [w, x, y, z] = msg.data.values.at(-1)",
+        "Each sample is a unit quaternion (w² + x² + y² + z² ≈ 1.0). Scattered norms mean you are reading at the wrong depth.",
+        "three.js argument order differs: new THREE.Quaternion(x, y, z, w)",
+        "Prefer object.quaternion.slerp(incoming, 0.2) over a hard set, to absorb packet jitter",
+        "Standalone signal. NOT part of the imu_acc + imu_gyro + emg bundle and exempt from the Pointer/Direction/IMU motion-mode XOR — it combines with navigation and nav_direction."
       ]
     },
     "imu_acc": {
@@ -404,24 +491,41 @@
       }
     },
     "xor_rules": [
-      "gesture XOR pressure (never both)",
+      "gesture XOR pressure (never both) — applies to direct_pressure and pinch_pressure alike",
+      "direct_pressure XOR pinch_pressure (exactly one pressure signal per app — they are mutually exclusive)",
       "navigation XOR nav_direction (never both)",
       "(navigation OR nav_direction) XOR imu_biometric_mode (never combine pointer/direction with the IMU+Biometric bundle)",
-      "imu_acc, imu_gyro, emg are bundled — partial subscriptions are forbidden"
+      "imu_acc, imu_gyro, emg are bundled — partial subscriptions are forbidden",
+      "imu_quaternion is exempt from every rule above — it is standalone and combines with anything"
     ],
     "can_combine": [
-      "gesture OR pressure",
+      "gesture OR one pressure mode (direct_pressure or pinch_pressure)",
       "button (with pointer/direction/imu_biometric/none)",
-      "ONE OF: pointer_mode OR direction_mode OR imu_biometric_mode"
+      "ONE OF: pointer_mode OR direction_mode OR imu_biometric_mode",
+      "imu_quaternion + ANY of the above, including navigation and nav_direction"
     ],
     "cannot_combine": [
       {
         "signals": [
           "gesture",
-          "pressure"
+          "direct_pressure"
+        ],
+        "reason": "XOR rule - pick one"
+      },
+      {
+        "signals": [
+          "gesture",
+          "pinch_pressure"
         ],
         "reason": "XOR rule - pick one"
       },
+      {
+        "signals": [
+          "direct_pressure",
+          "pinch_pressure"
+        ],
+        "reason": "Mutually exclusive — pick direct_pressure (new continuous stream) or pinch_pressure (original tap-to-release stream)"
+      },
       {
         "signals": [
           "navigation",
@@ -479,7 +583,7 @@
         "reason": "Button is part of pointer mode - cannot mix with direction mode"
       }
     ],
-    "guidance": "Three mutually exclusive motion groups: (1) Pointer mode: navigation+button for continuous cursor/drag/pan, (2) Direction mode: nav_direction for discrete swipe-like gestures, (3) IMU+Biometric mode: imu_acc+imu_gyro+emg as an inseparable bundle for orientation/tilt/rotation/biometric. Pick ONE per app. gesture XOR pressure; button combines freely (subject to mode rules)."
+    "guidance": "Three mutually exclusive motion groups: (1) Pointer mode: navigation+button for continuous cursor/drag/pan, (2) Direction mode: nav_direction for discrete swipe-like gestures, (3) IMU+Biometric mode: imu_acc+imu_gyro+emg as an inseparable bundle for shake/acceleration/biometrics. Pick ONE per app. gesture XOR pressure, and direct_pressure XOR pinch_pressure; button combines freely (subject to mode rules). Hand Orientation (imu_quaternion) is a FOURTH, non-exclusive option that sits outside this scheme — if the app needs aiming or 1:1 rotation, add imu_quaternion to whichever motion mode you picked rather than treating it as a conflict. imu_quaternion: Requires firmware 6.0.12.11 and above. direct_pressure: Requires firmware 6.0.12.11 and above. pinch_pressure works on older firmware."
   },
   "websocket_api": {
     "url": "ws://127.0.0.1:8766",
@@ -524,7 +628,7 @@
         },
         "example": {
           "command": "unsubscribe",
-          "signal": "pressure"
+          "signal": "direct_pressure"
         }
       },
       "get_subscriptions": {
@@ -590,10 +694,25 @@
         "reason": "Raw WebSocket has no mock fallback; MudraClient handles real + simulated paths transparently"
       },
       {
-        "wrong": "{command: 'subscribe', signals: ['gesture', 'pressure']}",
+        "wrong": "{command: 'subscribe', signals: ['gesture', 'navigation']}",
         "correct": "Send separate subscribe commands — one per signal",
         "reason": "Parameter is 'signal' (singular), not 'signals'"
       },
+      {
+        "wrong": "{command: 'subscribe', signal: 'pressure'}",
+        "correct": "Subscribe to 'direct_pressure' (default) or 'pinch_pressure' — never both",
+        "reason": "'pressure' is the old name for pinch_pressure and is no longer a valid signal. The original tap-to-release stream is pinch_pressure; direct_pressure is a new continuous ungated stream. The server returns invalid_signal and the app silently receives nothing. The frame type mirrors the signal name, so handlers keyed on 'pressure' never fire either."
+      },
+      {
+        "wrong": "const [w, x, y, z] = msg.data.values  // on an imu_quaternion frame",
+        "correct": "const [w, x, y, z] = msg.data.values.at(-1)",
+        "reason": "imu_quaternion values is a LIST of [w,x,y,z] samples, one nesting level deeper than imu_acc / imu_gyro. Read flat, you get whole samples where components belong and the rotation is nonsense."
+      },
+      {
+        "wrong": "Subscribing to imu_acc + imu_gyro + emg to derive where the hand is pointing",
+        "correct": "Subscribe to imu_quaternion alone",
+        "reason": "imu_quaternion is already fused and drift-free, and unlike the bundle it can coexist with navigation / nav_direction — so it does not force a motion-mode choice on the app."
+      },
       {
         "wrong": "Instantiating MudraClient inside init()",
         "correct": "Instantiate MudraClient at module scope, outside the xb.Script class",
@@ -814,14 +933,36 @@
       ]
     },
     "pressure_analog": {
-      "description": "Analog control via finger pressure — brush size, zoom, opacity",
-      "signal": "pressure",
+      "description": "NEW continuous ungated pressure stream — brush size, zoom, opacity. Requires firmware 6.0.12.11 and above.",
+      "signal": "direct_pressure",
       "implementation": [
         "Map normalized (0..1) to your range",
         "Smooth with rolling average (3-5 samples)",
         "Visual feedback: scale, color hue, bar indicator"
       ]
     },
+    "pinch_and_squeeze": {
+      "description": "ORIGINAL tap-to-release filtered stream — commit with a pinch, then modulate with force. Works on older firmware.",
+      "signal": "pinch_pressure",
+      "implementation": [
+        "Latch the target mesh on the first non-zero frame — that is the grab",
+        "Map normalized (0..1) to the modulated property while the pinch is held",
+        "Treat the return to ~0 as release — commit the final value and clear the latch",
+        "Visual feedback: highlight the latched mesh plus a force meter, so the user sees what they grabbed and how hard"
+      ]
+    },
+    "hand_orientation": {
+      "description": "Absolute hand orientation — aiming a ray, 1:1 mesh rotation, pose gating, heading. Requires firmware 6.0.12.11 and above.",
+      "signal": "imu_quaternion",
+      "implementation": [
+        "Read the latest sample each frame: const [w, x, y, z] = msg.data.values.at(-1)",
+        "Build the three.js quaternion with swapped argument order: new THREE.Quaternion(x, y, z, w)",
+        "Slerp toward it (factor ~0.2) rather than setting it hard, to absorb packet jitter",
+        "For aiming, apply the quaternion to a forward vector (0, 0, -1) and raycast along the result",
+        "Latch a reference quaternion on gesture.tap so the user can zero the pose in a comfortable arm position, then apply incoming rotations relative to it",
+        "Sanity check while developing: w² + x² + y² + z² should be ≈ 1.0"
+      ]
+    },
     "emg_time_series": {
       "description": "EMG visualization, muscle-activity overlays",
       "signal": "emg",

package/skills/mudra-xr/references/prompt.md

@@ -17,19 +17,36 @@ ws://127.0.0.1:8766
 Always construct the connection through `MudraClient` (Section 4).
 Never use raw `new WebSocket(...)`.
 
-### Eight canonical signals
+### Ten canonical signals
 
 | Signal | Category | Description |
 |--------|----------|-------------|
 | `gesture` | Discrete | Hand gesture events (tap, double_tap, twist, double_twist) |
 | `button` | Discrete | Button hold / release |
-| `pressure` | Analog | Finger pressure 0–100, normalized 0–1 |
+| `direct_pressure` | Analog | Finger pressure 0–100, normalized 0–1. **New** continuous ungated stream (always on). **Default.** Requires firmware 6.0.12.11 and above. |
+| `pinch_pressure` | Analog | Finger pressure 0–100, normalized 0–1. **Original** tap-to-release filtered stream. Works on older firmware. |
 | `navigation` | Motion (Pointer) | Continuous delta_x / delta_y cursor movement |
 | `nav_direction` | Motion (Direction) | Discrete directional swipes: None, Right, Left, Up, Down, Roll Left, Roll Right |
 | `imu_acc` | Motion (IMU) | Accelerometer values [x, y, z] m/s², frequency 1125 Hz |
 | `imu_gyro` | Motion (IMU) | Gyroscope values [x, y, z] deg/s, frequency 1125 Hz |
+| `imu_quaternion` | Orientation (standalone) | **Hand Orientation** — absolute unit quaternions. `values` is a **list** of `[w, x, y, z]` samples. Requires firmware 6.0.12.11 and above. |
 | `emg` | Biometric | 3 de-interleaved channel arrays [[ch1], [ch2], [ch3]] |
 
+**`pressure` is not a signal name.** It was the old name for
+`pinch_pressure`. Both signals are Finger pressure 0–100, normalized 0–1.
+`direct_pressure` is a **new** continuous ungated
+stream, not a split of the old signal. Sending `pressure` returns
+`invalid_signal` and the app receives nothing. Subscribe to exactly one
+pressure signal. Default to `direct_pressure`. Requires firmware 6.0.12.11 and above. Choose `pinch_pressure` only for explicit
+commit-then-modulate interactions (grab-and-scale, pinch-to-zoom,
+hold-to-charge). `pinch_pressure` works on older firmware.
+
+**`imu_quaternion` is exempt from the motion-mode XOR** in Section 8 — it
+is standalone and combines with any other signal, including `navigation`
+and `nav_direction`. Prefer it over the IMU bundle whenever the app needs
+absolute orientation (aiming, ray direction, 1:1 mesh rotation, pose
+gating) rather than raw acceleration. Requires firmware 6.0.12.11 and above.
+
 
 ### Subscription handshake
 
@@ -38,11 +55,15 @@ Send one command per signal — never use plural `signals`, arrays, or batch com
 ```js
 // CORRECT
 ws.send(JSON.stringify({ command: 'subscribe', signal: 'gesture' }));
-ws.send(JSON.stringify({ command: 'subscribe', signal: 'pressure' }));
+ws.send(JSON.stringify({ command: 'subscribe', signal: 'direct_pressure' }));
+ws.send(JSON.stringify({ command: 'subscribe', signal: 'imu_quaternion' }));
 
 // WRONG — never do this
-ws.send(JSON.stringify({ command: 'subscribe', signals: ['gesture', 'pressure'] }));
-ws.send(JSON.stringify({ command: 'subscribe', signal: ['gesture', 'pressure'] }));
+ws.send(JSON.stringify({ command: 'subscribe', signals: ['gesture', 'navigation'] }));
+ws.send(JSON.stringify({ command: 'subscribe', signal: ['gesture', 'navigation'] }));
+
+// WRONG — 'pressure' is the old name for pinch_pressure, not a signal
+ws.send(JSON.stringify({ command: 'subscribe', signal: 'pressure' }));
 ```
 
 ### Full command surface
@@ -60,8 +81,8 @@ ws.send(JSON.stringify({ command: 'subscribe', signal: ['gesture', 'pressure'] }
 // button
 { type: 'button', data: { state: 'pressed'|'released', timestamp }, timestamp }
 
-// pressure
-{ type: 'pressure', data: { value: 0–100, normalized: 0–1, timestamp }, timestamp }
+// direct_pressure / pinch_pressure — same payload; pick exactly one
+{ type: 'direct_pressure'|'pinch_pressure', data: { value: 0–100, normalized: 0–1, timestamp }, timestamp }
 
 // navigation
 { type: 'navigation', data: { delta_x: number, delta_y: number, timestamp }, timestamp }
@@ -79,7 +100,7 @@ ws.send(JSON.stringify({ command: 'subscribe', signal: ['gesture', 'pressure'] }
 { type: 'emg', data: { values: [[ch1_samples], [ch2_samples], [ch3_samples]], frequency: number, frequency_std: number, timestamp }, timestamp }
 
 // status — response to get_status command
-{ type: 'status', data: { device: { name, address, battery, charging, firmware, serial_number, hand, state, firmware_config: { target, active } }, subscriptions: { emg, imu_acc, imu_gyro, pressure, gesture, navigation, nav_direction, button } }, timestamp }
+{ type: 'status', data: { device: { name, address, battery, charging, firmware, serial_number, hand, state, firmware_config: { target, active } }, subscriptions: { emg, imu_acc, imu_gyro, imu_quaternion, direct_pressure, pinch_pressure, gesture, navigation, nav_direction, button } }, timestamp }
 
 // subscription_status — response to subscribe/unsubscribe
 { type: 'subscription_status', data: { signal: string, subscribed: boolean }, timestamp }
@@ -630,6 +651,20 @@ mudra.subscribe('emg');                   // missing imu_acc and imu_gyro
 mudra.subscribe('imu_acc');               // missing imu_gyro and emg
 ```
 
+### Hand Orientation — `imu_quaternion` (standalone)
+
+Exempt from every XOR in this section. It belongs to no bundle, requires
+no motion mode, and combines with any other signal — including
+`navigation` and `nav_direction`. Use it for aiming, ray direction, 1:1
+mesh rotation, pose gating, and heading. Requires firmware 6.0.12.11 and above.
+
+### Pressure — `direct_pressure` vs `pinch_pressure`
+
+Pick exactly one. There is no bare `pressure` signal.
+
+- `direct_pressure` — Finger pressure 0–100, normalized 0–1. **New** continuous ungated stream (always on, no tap/release gating). Default. Requires firmware 6.0.12.11 and above.
+- `pinch_pressure` — Finger pressure 0–100, normalized 0–1. **Original** tap-to-release filtered stream (the former `pressure` signal). Values stream only between tap and release. Works on older firmware.
+
 ### XOR rules (all non-negotiable)
 
 1. **Gesture ⊕ Pressure** — an app may use `gesture` OR `pressure`, never both.