@pollen-robotics/reachy-mini-sdk 1.7.3-main.7457b2b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +188 -0
- package/LICENSE +201 -0
- package/README.md +129 -0
- package/host/APP_AUTHOR_GUIDE.md +646 -0
- package/host/README.md +155 -0
- package/host/SPEC.md +618 -0
- package/host/dist/ReachyHost.d.ts +39 -0
- package/host/dist/ReachyHost.d.ts.map +1 -0
- package/host/dist/assets/index.d.ts +16 -0
- package/host/dist/assets/index.d.ts.map +1 -0
- package/host/dist/chunks/index-C3B3FuBC.js +402 -0
- package/host/dist/chunks/index-C3B3FuBC.js.map +1 -0
- package/host/dist/chunks/mountHost-8f-laxwI.js +48280 -0
- package/host/dist/chunks/mountHost-8f-laxwI.js.map +1 -0
- package/host/dist/chunks/reachy-mini-sdk-T3Zr7Hwc.js +1726 -0
- package/host/dist/chunks/reachy-mini-sdk-T3Zr7Hwc.js.map +1 -0
- package/host/dist/components/ConnectingView.d.ts +13 -0
- package/host/dist/components/ConnectingView.d.ts.map +1 -0
- package/host/dist/components/EmbedFrame.d.ts +16 -0
- package/host/dist/components/EmbedFrame.d.ts.map +1 -0
- package/host/dist/components/ErrorView.d.ts +21 -0
- package/host/dist/components/ErrorView.d.ts.map +1 -0
- package/host/dist/components/LeavingView.d.ts +33 -0
- package/host/dist/components/LeavingView.d.ts.map +1 -0
- package/host/dist/components/PickerView.d.ts +51 -0
- package/host/dist/components/PickerView.d.ts.map +1 -0
- package/host/dist/components/ReachyHostShell.d.ts +21 -0
- package/host/dist/components/ReachyHostShell.d.ts.map +1 -0
- package/host/dist/components/SignInView.d.ts +32 -0
- package/host/dist/components/SignInView.d.ts.map +1 -0
- package/host/dist/components/StepsProgressIndicator.d.ts +41 -0
- package/host/dist/components/StepsProgressIndicator.d.ts.map +1 -0
- package/host/dist/components/TopBar.d.ts +52 -0
- package/host/dist/components/TopBar.d.ts.map +1 -0
- package/host/dist/components/WelcomeBackOverlay.d.ts +28 -0
- package/host/dist/components/WelcomeBackOverlay.d.ts.map +1 -0
- package/host/dist/embed/index.d.ts +150 -0
- package/host/dist/embed/index.d.ts.map +1 -0
- package/host/dist/entry/auto.d.ts +12 -0
- package/host/dist/entry/auto.d.ts.map +1 -0
- package/host/dist/entry/auto.js +7 -0
- package/host/dist/entry/auto.js.map +1 -0
- package/host/dist/entry/embed.d.ts +12 -0
- package/host/dist/entry/embed.d.ts.map +1 -0
- package/host/dist/entry/embed.js +7 -0
- package/host/dist/entry/embed.js.map +1 -0
- package/host/dist/hooks/useHfProfile.d.ts +7 -0
- package/host/dist/hooks/useHfProfile.d.ts.map +1 -0
- package/host/dist/hooks/useHostBridge.d.ts +37 -0
- package/host/dist/hooks/useHostBridge.d.ts.map +1 -0
- package/host/dist/hooks/useOAuth.d.ts +16 -0
- package/host/dist/hooks/useOAuth.d.ts.map +1 -0
- package/host/dist/hooks/useRobots.d.ts +17 -0
- package/host/dist/hooks/useRobots.d.ts.map +1 -0
- package/host/dist/hooks/useSdk.d.ts +16 -0
- package/host/dist/hooks/useSdk.d.ts.map +1 -0
- package/host/dist/index.d.ts +25 -0
- package/host/dist/index.d.ts.map +1 -0
- package/host/dist/index.js +15 -0
- package/host/dist/index.js.map +1 -0
- package/host/dist/lib/centralListener.d.ts +73 -0
- package/host/dist/lib/centralListener.d.ts.map +1 -0
- package/host/dist/lib/centralRest.d.ts +35 -0
- package/host/dist/lib/centralRest.d.ts.map +1 -0
- package/host/dist/lib/protocol.d.ts +230 -0
- package/host/dist/lib/protocol.d.ts.map +1 -0
- package/host/dist/lib/protocol.js +48 -0
- package/host/dist/lib/protocol.js.map +1 -0
- package/host/dist/lib/sdk-types.d.ts +46 -0
- package/host/dist/lib/sdk-types.d.ts.map +1 -0
- package/host/dist/lib/settings.d.ts +69 -0
- package/host/dist/lib/settings.d.ts.map +1 -0
- package/host/dist/lib/signalingUrl.d.ts +28 -0
- package/host/dist/lib/signalingUrl.d.ts.map +1 -0
- package/host/dist/lib/theme.d.ts +4 -0
- package/host/dist/lib/theme.d.ts.map +1 -0
- package/host/dist/lib/themeMode.d.ts +9 -0
- package/host/dist/lib/themeMode.d.ts.map +1 -0
- package/host/dist/lib/tokens.d.ts +64 -0
- package/host/dist/lib/tokens.d.ts.map +1 -0
- package/host/dist/mountHost.d.ts +36 -0
- package/host/dist/mountHost.d.ts.map +1 -0
- package/package.json +109 -0
- package/reachy-mini-sdk.d.ts +285 -0
- package/reachy-mini-sdk.js +2625 -0
|
@@ -0,0 +1,2625 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* reachy-mini.js — Browser SDK for controlling a Reachy Mini robot over WebRTC.
|
|
3
|
+
* https://github.com/pollen-robotics/reachy-mini
|
|
4
|
+
*
|
|
5
|
+
* QUICK START
|
|
6
|
+
* ───────────
|
|
7
|
+
* import { ReachyMini } from "./reachy-mini-sdk.js";
|
|
8
|
+
* const robot = new ReachyMini();
|
|
9
|
+
*
|
|
10
|
+
* // 1. Auth (HuggingFace OAuth — required for the signaling server)
|
|
11
|
+
* if (!await robot.authenticate()) { robot.login(); return; }
|
|
12
|
+
*
|
|
13
|
+
* // 2. Connect to signaling server (SSE)
|
|
14
|
+
* await robot.connect();
|
|
15
|
+
*
|
|
16
|
+
* // 3. Pick a robot once the list arrives
|
|
17
|
+
* robot.addEventListener("robotsChanged", (e) => {
|
|
18
|
+
* const robots = e.detail.robots; // [{ id, meta: { name } }, ...]
|
|
19
|
+
* });
|
|
20
|
+
*
|
|
21
|
+
* // 4. Start a WebRTC session (resolves when video + data channel ready)
|
|
22
|
+
* const detach = robot.attachVideo(document.querySelector("video"));
|
|
23
|
+
* await robot.startSession(robotId);
|
|
24
|
+
*
|
|
25
|
+
* // 5. Send commands — degree-friendly helpers, all built on setTarget()
|
|
26
|
+
* robot.setHeadRpyDeg(0, 10, -5); // roll, pitch, yaw in degrees
|
|
27
|
+
* robot.setAntennasDeg(30, -30); // right, left in degrees
|
|
28
|
+
* robot.setBodyYawDeg(15); // body yaw in degrees
|
|
29
|
+
*
|
|
30
|
+
* // …or compose an atomic update in raw wire units (full SE(3); no XYZ loss):
|
|
31
|
+
* robot.setTarget({
|
|
32
|
+
* head: rpyToMatrix(0, 10, -5).flat(), // number[16] flat row-major 4×4
|
|
33
|
+
* antennas: [degToRad(30), degToRad(-30)],
|
|
34
|
+
* body_yaw: degToRad(15),
|
|
35
|
+
* });
|
|
36
|
+
* robot.playSound("wake_up.wav"); // filename on robot
|
|
37
|
+
* const ver = await robot.getVersion(); // e.g. "1.5.1"
|
|
38
|
+
*
|
|
39
|
+
* // …or play a recorded move (motion + optional audio) on the daemon's
|
|
40
|
+
* // local clock — smooth on wireless robots, single-clock A/V sync.
|
|
41
|
+
* await robot.playMove(motion, { audioBlob, audioLeadMs: -100 });
|
|
42
|
+
* // robot.cancelMove() to stop early.
|
|
43
|
+
* //
|
|
44
|
+
* // For record-time flows that need the same audio pipeline at both
|
|
45
|
+
* // capture and replay:
|
|
46
|
+
* // const audioId = await robot.uploadAudio(blob);
|
|
47
|
+
* // await robot.playUploadedAudio(audioId); // sync anchor
|
|
48
|
+
* // // …start your motion capture now…
|
|
49
|
+
* // robot.cancelAudio(); // when capture stops
|
|
50
|
+
*
|
|
51
|
+
* // 6. Receive live state (emitted every ~500 ms while streaming; call
|
|
52
|
+
* // robot.requestState() yourself for higher rates — see its JSDoc).
|
|
53
|
+
* // State payload is the daemon's raw wire shape — use the math
|
|
54
|
+
* // utilities exported from this module for degree conversions.
|
|
55
|
+
* robot.addEventListener("state", (e) => {
|
|
56
|
+
* const { head, antennas, body_yaw, motor_mode, is_move_running } = e.detail;
|
|
57
|
+
* // head: number[16] — flat row-major 4×4 (full SE(3))
|
|
58
|
+
* // antennas: [rightRad, leftRad]
|
|
59
|
+
* // body_yaw: number — radians
|
|
60
|
+
* // motor_mode: "enabled" | "disabled" | "gravity_compensation"
|
|
61
|
+
* // is_move_running: boolean
|
|
62
|
+
* // For human-friendly head RPY:
|
|
63
|
+
* // const rpy = matrixToRpy(head); // { roll, pitch, yaw } in degrees
|
|
64
|
+
* });
|
|
65
|
+
*
|
|
66
|
+
* // 7. Audio controls
|
|
67
|
+
* robot.setAudioMuted(false); // unmute robot speaker (muted by default)
|
|
68
|
+
* robot.setMicMuted(false); // unmute your mic → robot speaker (if supported)
|
|
69
|
+
*
|
|
70
|
+
* // 8. Cleanup
|
|
71
|
+
* detach(); // remove video binding
|
|
72
|
+
* await robot.stopSession(); // back to 'connected'
|
|
73
|
+
* robot.disconnect(); // back to 'disconnected' (keeps auth)
|
|
74
|
+
* robot.logout(); // clear HF credentials too
|
|
75
|
+
*
|
|
76
|
+
*
|
|
77
|
+
* STATE MACHINE
|
|
78
|
+
* ─────────────
|
|
79
|
+
* 'disconnected' ──connect()──▸ 'connected' ──startSession()──▸ 'streaming'
|
|
80
|
+
* ▴ disconnect() ▴ stopSession()
|
|
81
|
+
* └─────────────────────────────┘
|
|
82
|
+
*
|
|
83
|
+
*
|
|
84
|
+
* CONSTRUCTOR OPTIONS
|
|
85
|
+
* ───────────────────
|
|
86
|
+
* new ReachyMini({
|
|
87
|
+
* signalingUrl: string, // default: "https://pollen-robotics-reachy-mini-central.hf.space"
|
|
88
|
+
* enableMicrophone: boolean, // default: true — acquire mic for bidirectional audio
|
|
89
|
+
* videoJitterBufferTargetMs: number, // default: 0 — receiver-side jitter buffer hint, ms
|
|
90
|
+
* // 0 = "render ASAP" (teleop). Spec range [0, 4000].
|
|
91
|
+
* // Raise (100–400) on flaky links to trade latency for resilience.
|
|
92
|
+
* autoStartFromUrl: boolean, // default: false — when true AND the URL carries a `robot_peer_id` hint,
|
|
93
|
+
* // auto-call `startSession(preselectedRobotId)` after
|
|
94
|
+
* // `connect()` resolves and that robot appears online.
|
|
95
|
+
* // One-shot per page load; suits iframe-embedded apps
|
|
96
|
+
* // that want zero-tap entry from the host shell.
|
|
97
|
+
* })
|
|
98
|
+
*
|
|
99
|
+
*
|
|
100
|
+
* READ-ONLY PROPERTIES
|
|
101
|
+
* ────────────────────
|
|
102
|
+
* .state "disconnected" | "connected" | "streaming"
|
|
103
|
+
* .robots Array<{ id: string, meta: { name: string } }>
|
|
104
|
+
* .robotState Mirror of the latest "state" event detail —
|
|
105
|
+
* { head: number[16], antennas: [rightRad, leftRad],
|
|
106
|
+
* body_yaw, motor_mode, is_move_running }
|
|
107
|
+
* (fields only present once the daemon sends them;
|
|
108
|
+
* see EVENTS below)
|
|
109
|
+
* .username string | null — HF username after authenticate()
|
|
110
|
+
* .isAuthenticated boolean — true if a valid HF token is available
|
|
111
|
+
* .micSupported boolean — true if robot offers bidirectional audio
|
|
112
|
+
* .micMuted boolean — your microphone mute state
|
|
113
|
+
* .audioMuted boolean — robot speaker mute state (local)
|
|
114
|
+
* .preselectedRobotId string | null — peer id from `?robot_peer_id=` /
|
|
115
|
+
* `#robot_peer_id=`; null if absent.
|
|
116
|
+
* Use it to skip your robot picker
|
|
117
|
+
* when a host iframe (e.g. the
|
|
118
|
+
* Reachy Mini mobile shell) embeds
|
|
119
|
+
* this app.
|
|
120
|
+
* .isEmbedded boolean — true iff `preselectedRobotId !==
|
|
121
|
+
* null`. Branch your UX on this:
|
|
122
|
+
* when true, hide the robot picker
|
|
123
|
+
* and your sign-in screen (the
|
|
124
|
+
* host has already handled both).
|
|
125
|
+
*
|
|
126
|
+
*
|
|
127
|
+
* EVENTS (EventTarget — use addEventListener)
|
|
128
|
+
* ──────────────────────────────────────────────
|
|
129
|
+
* "connected" { peerId: string }
|
|
130
|
+
* "disconnected" { reason: string }
|
|
131
|
+
* "robotsChanged" { robots: Array<{ id, meta }> }
|
|
132
|
+
* "streaming" { sessionId: string, robotId: string }
|
|
133
|
+
* "sessionStopped" { reason: string }
|
|
134
|
+
* "state" { head: number[16], // flat row-major 4×4, when daemon sends head_pose
|
|
135
|
+
* antennas: [rightRad, leftRad], // when daemon sends antennas
|
|
136
|
+
* body_yaw: number, // radians, when daemon sends body_yaw
|
|
137
|
+
* motor_mode: string, // when daemon sends motor_mode
|
|
138
|
+
* is_move_running: boolean } // when daemon sends is_move_running
|
|
139
|
+
* "videoTrack" { track: MediaStreamTrack, stream: MediaStream }
|
|
140
|
+
* "micSupported" { supported: boolean }
|
|
141
|
+
* "error" { source: "signaling"|"webrtc"|"robot", error: Error|string }
|
|
142
|
+
*
|
|
143
|
+
*
|
|
144
|
+
* EXPORTS
|
|
145
|
+
* ───────
|
|
146
|
+
* export default ReachyMini;
|
|
147
|
+
* export { ReachyMini, rpyToMatrix, matrixToRpy, degToRad, radToDeg };
|
|
148
|
+
*/
|
|
149
|
+
|
|
150
|
+
import {
|
|
151
|
+
oauthHandleRedirectIfPresent,
|
|
152
|
+
oauthLoginUrl,
|
|
153
|
+
} from "@huggingface/hub";
|
|
154
|
+
|
|
155
|
+
// ─── Math utilities ──────────────────────────────────────────────────────────
|
|
156
|
+
|
|
157
|
+
/** @param {number} deg @returns {number} */
|
|
158
|
+
export function degToRad(deg) { return deg * Math.PI / 180; }
|
|
159
|
+
|
|
160
|
+
/** @param {number} rad @returns {number} */
|
|
161
|
+
export function radToDeg(rad) { return rad * 180 / Math.PI; }
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Roll/pitch/yaw (degrees) → 4×4 rotation matrix (ZYX convention).
|
|
165
|
+
* This is the wire format for the robot's `set_target` command.
|
|
166
|
+
* @param {number} rollDeg @param {number} pitchDeg @param {number} yawDeg
|
|
167
|
+
* @returns {number[][]} 4×4 matrix
|
|
168
|
+
*/
|
|
169
|
+
export function rpyToMatrix(rollDeg, pitchDeg, yawDeg) {
|
|
170
|
+
const r = degToRad(rollDeg), p = degToRad(pitchDeg), y = degToRad(yawDeg);
|
|
171
|
+
const cy = Math.cos(y), sy = Math.sin(y);
|
|
172
|
+
const cp = Math.cos(p), sp = Math.sin(p);
|
|
173
|
+
const cr = Math.cos(r), sr = Math.sin(r);
|
|
174
|
+
return [
|
|
175
|
+
[cy * cp, cy * sp * sr - sy * cr, cy * sp * cr + sy * sr, 0],
|
|
176
|
+
[sy * cp, sy * sp * sr + cy * cr, sy * sp * cr - cy * sr, 0],
|
|
177
|
+
[-sp, cp * sr, cp * cr, 0],
|
|
178
|
+
[0, 0, 0, 1],
|
|
179
|
+
];
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Rotation matrix (3×3 or 4×4) → { roll, pitch, yaw } in degrees.
|
|
184
|
+
* @param {number[][]} m @returns {{ roll: number, pitch: number, yaw: number }}
|
|
185
|
+
*/
|
|
186
|
+
export function matrixToRpy(m) {
|
|
187
|
+
return {
|
|
188
|
+
roll: radToDeg(Math.atan2(m[2][1], m[2][2])),
|
|
189
|
+
pitch: radToDeg(Math.asin(-m[2][0])),
|
|
190
|
+
yaw: radToDeg(Math.atan2(m[1][0], m[0][0])),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ─── Internal helpers ────────────────────────────────────────────────────────
|
|
195
|
+
|
|
196
|
+
/** Clamp a volume to [0, 100] and round to integer — mirrors the server-side
|
|
197
|
+
* Field(..., ge=0, le=100) validator so calling setVolume(150) doesn't 400. */
|
|
198
|
+
function clampVolume(v) {
|
|
199
|
+
const n = Math.round(Number(v) || 0);
|
|
200
|
+
return Math.max(0, Math.min(100, n));
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Pick up HuggingFace credentials passed via the URL fragment and move them
|
|
205
|
+
* into `sessionStorage`, where `authenticate()` looks them up.
|
|
206
|
+
*
|
|
207
|
+
* This is the bridge that lets a host page (e.g. the Reachy Mini mobile
|
|
208
|
+
* app, or the vibe-coder preview iframe) embed a Space hosting a SDK
|
|
209
|
+
* consumer despite `X-Frame-Options: SAMEORIGIN` on `huggingface.co/login`:
|
|
210
|
+
* the host already holds a valid token (through its own OAuth flow) and
|
|
211
|
+
* appends it to the iframe URL as
|
|
212
|
+
*
|
|
213
|
+
* #hf_token=<jwt>&hf_username=<handle>&hf_token_expires=<iso>
|
|
214
|
+
*
|
|
215
|
+
* Fragments are NOT sent over HTTP, so the credentials never leak to
|
|
216
|
+
* the HF Space backend or to intermediate proxies.
|
|
217
|
+
*
|
|
218
|
+
* Why all three keys: `authenticate()`'s cache check requires the token,
|
|
219
|
+
* the username AND a future expiry to ALL be present in `sessionStorage`,
|
|
220
|
+
* otherwise it returns `false` and the app falls through to a full OAuth
|
|
221
|
+
* round-trip — which can't complete inside an iframe.
|
|
222
|
+
*
|
|
223
|
+
* Called once from the top of `authenticate()` so SDK consumers don't
|
|
224
|
+
* need any boilerplate of their own. We clear the fragment right after
|
|
225
|
+
* reading it so a page reload does not keep the credentials visible in
|
|
226
|
+
* the address bar.
|
|
227
|
+
*
|
|
228
|
+
* No-op when:
|
|
229
|
+
* - there is no `window` (SSR / Worker contexts),
|
|
230
|
+
* - the URL has no fragment,
|
|
231
|
+
* - the fragment carries no `hf_token` (other apps may use the
|
|
232
|
+
* fragment for theme / route / etc.; we leave those alone).
|
|
233
|
+
*/
|
|
234
|
+
function consumeFragmentCredentials() {
|
|
235
|
+
if (typeof window === 'undefined' || !window.location.hash) return;
|
|
236
|
+
const raw = window.location.hash.startsWith('#')
|
|
237
|
+
? window.location.hash.slice(1)
|
|
238
|
+
: window.location.hash;
|
|
239
|
+
let params;
|
|
240
|
+
try { params = new URLSearchParams(raw); } catch (_e) { return; }
|
|
241
|
+
const token = params.get('hf_token');
|
|
242
|
+
if (!token) return;
|
|
243
|
+
// `hf_username` is required by the cache check. Hosts that haven't
|
|
244
|
+
// resolved the user's HF handle yet may pass a literal "user"
|
|
245
|
+
// placeholder; the SDK only uses the value for display and never
|
|
246
|
+
// round-trips it server-side, so the placeholder is harmless.
|
|
247
|
+
const username = params.get('hf_username') || 'user';
|
|
248
|
+
// `hf_token_expires` is a far-future ISO date for personal access
|
|
249
|
+
// tokens (no real expiry). Hosts typically synthesise ~1 year out;
|
|
250
|
+
// we accept whatever was sent and fall back to "1 year from now"
|
|
251
|
+
// if the parameter is missing or unparseable, so a partial fragment
|
|
252
|
+
// still gets the user logged in.
|
|
253
|
+
const expiresParam = params.get('hf_token_expires');
|
|
254
|
+
const expires =
|
|
255
|
+
expiresParam && !Number.isNaN(new Date(expiresParam).getTime())
|
|
256
|
+
? expiresParam
|
|
257
|
+
: new Date(Date.now() + 365 * 24 * 60 * 60 * 1000).toISOString();
|
|
258
|
+
try {
|
|
259
|
+
sessionStorage.setItem('hf_token', token);
|
|
260
|
+
sessionStorage.setItem('hf_username', username);
|
|
261
|
+
sessionStorage.setItem('hf_token_expires', expires);
|
|
262
|
+
} catch (err) {
|
|
263
|
+
console.warn('[reachy-mini] could not persist pre-seeded HF credentials:', err);
|
|
264
|
+
}
|
|
265
|
+
// Strip the auth keys from the address bar but keep any other hash
|
|
266
|
+
// params the app or SDK might care about (theme, embedded, …).
|
|
267
|
+
params.delete('hf_token');
|
|
268
|
+
params.delete('hf_username');
|
|
269
|
+
params.delete('hf_token_expires');
|
|
270
|
+
const remaining = params.toString();
|
|
271
|
+
const cleanUrl =
|
|
272
|
+
window.location.pathname +
|
|
273
|
+
window.location.search +
|
|
274
|
+
(remaining ? '#' + remaining : '');
|
|
275
|
+
try { window.history.replaceState(null, '', cleanUrl); } catch (_e) {}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Pick up a preselected robot peer id from the URL.
|
|
280
|
+
*
|
|
281
|
+
* Looked up in this order:
|
|
282
|
+
* 1. URL fragment (`#robot_peer_id=<peerId>`)
|
|
283
|
+
* 2. URL query (`?robot_peer_id=<peerId>`)
|
|
284
|
+
*
|
|
285
|
+
* Both spellings are accepted because:
|
|
286
|
+
* - the Reachy Mini mobile shell sends it in the query today,
|
|
287
|
+
* - the vibe-coder preview / future hosts may prefer the fragment for
|
|
288
|
+
* symmetry with `consumeFragmentCredentials`,
|
|
289
|
+
* - the value is NOT a secret (peer ids are public on the central
|
|
290
|
+
* signaling server's robot listing) so query is fine.
|
|
291
|
+
*
|
|
292
|
+
* Returns `null` when no peer id is found in either location, when there
|
|
293
|
+
* is no `window` (SSR / Worker context), or on parse error. Unlike
|
|
294
|
+
* credentials, we do NOT strip the param from the URL: the value is
|
|
295
|
+
* harmless to keep visible and removing it would break tools that read
|
|
296
|
+
* the URL for context.
|
|
297
|
+
*
|
|
298
|
+
* @returns {string|null}
|
|
299
|
+
*/
|
|
300
|
+
function readPreselectedRobotIdFromUrl() {
|
|
301
|
+
if (typeof window === 'undefined') return null;
|
|
302
|
+
// 1. Fragment (`#robot_peer_id=…`).
|
|
303
|
+
if (window.location.hash) {
|
|
304
|
+
const raw = window.location.hash.startsWith('#')
|
|
305
|
+
? window.location.hash.slice(1)
|
|
306
|
+
: window.location.hash;
|
|
307
|
+
try {
|
|
308
|
+
const params = new URLSearchParams(raw);
|
|
309
|
+
const fromHash = params.get('robot_peer_id');
|
|
310
|
+
if (fromHash) return fromHash;
|
|
311
|
+
} catch (_e) { /* malformed fragment — fall through */ }
|
|
312
|
+
}
|
|
313
|
+
// 2. Query (`?robot_peer_id=…`).
|
|
314
|
+
if (window.location.search) {
|
|
315
|
+
try {
|
|
316
|
+
const params = new URLSearchParams(window.location.search);
|
|
317
|
+
const fromQuery = params.get('robot_peer_id');
|
|
318
|
+
if (fromQuery) return fromQuery;
|
|
319
|
+
} catch (_e) { /* malformed query — fall through */ }
|
|
320
|
+
}
|
|
321
|
+
return null;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Check if the audio m= section of an SDP has a=sendrecv (bidirectional audio). */
|
|
325
|
+
function sdpHasAudioSendRecv(sdp) {
|
|
326
|
+
const lines = sdp.split('\r\n');
|
|
327
|
+
let inAudio = false;
|
|
328
|
+
for (const line of lines) {
|
|
329
|
+
if (line.startsWith('m=audio')) inAudio = true;
|
|
330
|
+
else if (line.startsWith('m=')) inAudio = false;
|
|
331
|
+
if (inAudio && line === 'a=sendrecv') return true;
|
|
332
|
+
}
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// ─── Daemon-side upload helpers ──────────────────────────────────────────────
|
|
337
|
+
// Wire-level constants and pure helpers used by playMove / uploadAudio /
|
|
338
|
+
// playUploadedAudio. Private to this module — apps call the public methods.
|
|
339
|
+
|
|
340
|
+
// Conservative per-message size for the data channel. 16 KB is the cross-
|
|
341
|
+
// browser safe ceiling; we slice payloads at 12 KB and let the JSON envelope
|
|
342
|
+
// add ~80 bytes.
|
|
343
|
+
const UPLOAD_CHUNK_SIZE = 12 * 1024;
|
|
344
|
+
// Backpressure thresholds: pause sending if `bufferedAmount` climbs over the
|
|
345
|
+
// high watermark; resume once it drains below the low watermark. WebRTC's
|
|
346
|
+
// SCTP can buffer plenty, but spiking it to tens of megabytes degrades every
|
|
347
|
+
// other channel on the same peer connection.
|
|
348
|
+
const UPLOAD_BUFFERED_HIGH_WATER = 1 * 1024 * 1024;
|
|
349
|
+
const UPLOAD_BUFFERED_LOW_WATER = 512 * 1024;
|
|
350
|
+
|
|
351
|
+
function hasCompressionStream() {
|
|
352
|
+
return typeof CompressionStream !== "undefined";
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/** Cheap unique upload id; collision odds within a session are negligible. */
|
|
356
|
+
function makeUploadId() {
|
|
357
|
+
return "u" + Math.random().toString(36).slice(2, 11)
|
|
358
|
+
+ Date.now().toString(36);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/** Base64-encode a Uint8Array, chunking to avoid call-stack overflow on multi-MB blobs. */
|
|
362
|
+
function bytesToBase64(bytes) {
|
|
363
|
+
let str = "";
|
|
364
|
+
const STEP = 0x8000;
|
|
365
|
+
for (let i = 0; i < bytes.length; i += STEP) {
|
|
366
|
+
str += String.fromCharCode.apply(null, bytes.subarray(i, i + STEP));
|
|
367
|
+
}
|
|
368
|
+
return btoa(str);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** Base64(gzip(utf8(s))) via the browser CompressionStream API. */
|
|
372
|
+
async function gzipBase64(jsonStr) {
|
|
373
|
+
const enc = new TextEncoder().encode(jsonStr);
|
|
374
|
+
const compressed = await new Response(
|
|
375
|
+
new Blob([enc]).stream().pipeThrough(new CompressionStream("gzip"))
|
|
376
|
+
).arrayBuffer();
|
|
377
|
+
return bytesToBase64(new Uint8Array(compressed));
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// ─── ReachyMini class ────────────────────────────────────────────────────────
|
|
381
|
+
|
|
382
|
+
export class ReachyMini extends EventTarget {
|
|
383
|
+
|
|
384
|
+
/** @param {{ signalingUrl?: string, enableMicrophone?: boolean, clientId?: string, appName?: string, videoJitterBufferTargetMs?: number, autoStartFromUrl?: boolean }} [options] */
|
|
385
|
+
constructor(options = {}) {
|
|
386
|
+
super();
|
|
387
|
+
this._signalingUrl = options.signalingUrl || 'https://pollen-robotics-reachy-mini-central.hf.space';
|
|
388
|
+
this._enableMicrophone = options.enableMicrophone !== false;
|
|
389
|
+
this._clientId = options.clientId || null;
|
|
390
|
+
this._appName = options.appName || 'unknown';
|
|
391
|
+
// Hint to the receiver's WebRTC jitter buffer (ms). 0 = "render ASAP",
|
|
392
|
+
// appropriate for teleop. Spec range [0, 4000]. Browsers that don't
|
|
393
|
+
// implement RTCRtpReceiver.jitterBufferTarget fall back to default
|
|
394
|
+
// buffering (~150-200 ms).
|
|
395
|
+
this._videoJitterBufferTargetMs = options.videoJitterBufferTargetMs ?? 0;
|
|
396
|
+
// When true AND the URL carried a `robot_peer_id` hint at
|
|
397
|
+
// construction (so `preselectedRobotId !== null`), the SDK
|
|
398
|
+
// auto-calls `startSession(preselectedRobotId)` as soon as
|
|
399
|
+
// that robot appears in the central's robot list after the
|
|
400
|
+
// app's own `connect()` resolves. Lets host-iframe-embedded
|
|
401
|
+
// consumers (mobile shell, vibe-coder preview) skip their
|
|
402
|
+
// robot picker AND skip the manual `startSession` call —
|
|
403
|
+
// they just `await robot.connect()` and receive a `streaming`
|
|
404
|
+
// event when the SDK has dialed in. One-shot: a manual
|
|
405
|
+
// `stopSession()` followed by another `startSession()` is
|
|
406
|
+
// not auto-replayed. Default `false` keeps the standalone
|
|
407
|
+
// Space behavior unchanged.
|
|
408
|
+
this._autoStartFromUrl = options.autoStartFromUrl === true;
|
|
409
|
+
this._autoStartAttempted = false;
|
|
410
|
+
|
|
411
|
+
this._state = 'disconnected'; // 'disconnected' | 'connected' | 'streaming'
|
|
412
|
+
this._robots = []; // latest robot list from signaling
|
|
413
|
+
this._robotState = {}; // populated from daemon state events (wire shape)
|
|
414
|
+
|
|
415
|
+
// Preselected robot peer id read from the URL at construction
|
|
416
|
+
// time. When a host iframe (typically the Reachy Mini mobile
|
|
417
|
+
// shell) embeds an SDK consumer, it appends the peer id of the
|
|
418
|
+
// robot it's already connected to via
|
|
419
|
+
// `?robot_peer_id=…` (or `#robot_peer_id=…`). Apps can read
|
|
420
|
+
// `robot.preselectedRobotId` and call `startSession(id)`
|
|
421
|
+
// directly to skip their robot picker. Captured ONCE at
|
|
422
|
+
// construction so subsequent URL changes (history navigation,
|
|
423
|
+
// hash mutations from `consumeFragmentCredentials`) don't move
|
|
424
|
+
// the target out from under the consumer.
|
|
425
|
+
this._preselectedRobotId = readPreselectedRobotIdFromUrl();
|
|
426
|
+
|
|
427
|
+
// Auth
|
|
428
|
+
this._token = null;
|
|
429
|
+
this._username = null;
|
|
430
|
+
this._tokenExpires = null;
|
|
431
|
+
|
|
432
|
+
// Signaling
|
|
433
|
+
this._peerId = null;
|
|
434
|
+
this._sseAbortController = null;
|
|
435
|
+
|
|
436
|
+
// WebRTC
|
|
437
|
+
this._pc = null; // RTCPeerConnection
|
|
438
|
+
this._dc = null; // RTCDataChannel (robot commands)
|
|
439
|
+
this._sessionId = null;
|
|
440
|
+
this._selectedRobotId = null;
|
|
441
|
+
|
|
442
|
+
// Audio
|
|
443
|
+
this._micStream = null; // MediaStream from getUserMedia
|
|
444
|
+
this._micMuted = true;
|
|
445
|
+
this._audioMuted = true;
|
|
446
|
+
this._micSupported = false; // set after SDP negotiation
|
|
447
|
+
|
|
448
|
+
// Timers
|
|
449
|
+
this._latencyMonitorId = null;
|
|
450
|
+
this._stateRefreshInterval = null;
|
|
451
|
+
|
|
452
|
+
// getVersion() / getHardwareId() promise plumbing
|
|
453
|
+
this._versionResolve = null;
|
|
454
|
+
this._hardwareIdResolve = null;
|
|
455
|
+
|
|
456
|
+
// Volume getter/setter promise plumbing (get_volume / set_volume).
|
|
457
|
+
// Speaker and microphone are tracked separately so two in-flight
|
|
458
|
+
// requests can't collide on the same slot.
|
|
459
|
+
this._volumeResolve = null;
|
|
460
|
+
this._micVolumeResolve = null;
|
|
461
|
+
|
|
462
|
+
// subscribeLogs(): a Set of {onLine, onError} subscribers. The
|
|
463
|
+
// first add sends `subscribe_logs`; removing the last sends
|
|
464
|
+
// `unsubscribe_logs`. We keep a single daemon-side stream and
|
|
465
|
+
// fan out to local subscribers in `_handleRobotMessage`.
|
|
466
|
+
this._logSubscribers = new Set();
|
|
467
|
+
|
|
468
|
+
// Pending one-shot broadcast waiters used by playMove /
|
|
469
|
+
// playUploadedAudio. Each entry is a { predicate, resolve, timer }
|
|
470
|
+
// installed by `_waitForBroadcast`. Dispatched in `_handleRobotMessage`
|
|
471
|
+
// when the daemon broadcasts a matching {type, upload_id, ...}
|
|
472
|
+
// event. LIFO order so the most recently registered waiter wins
|
|
473
|
+
// on duplicates (rare but possible during reconnect).
|
|
474
|
+
this._broadcastWaiters = [];
|
|
475
|
+
|
|
476
|
+
// upload_id of the in-flight playMove / playUploadedAudio so
|
|
477
|
+
// the parameter-less `cancelMove()` / `cancelAudio()` calls
|
|
478
|
+
// can target the right run. The daemon now requires upload_id
|
|
479
|
+
// on cancels (otherwise back-to-back plays would cross-cancel).
|
|
480
|
+
// Apps that hold the id themselves can pass it explicitly
|
|
481
|
+
// instead — see the cancelMove / cancelAudio JSDoc.
|
|
482
|
+
this._activeMoveUploadId = null;
|
|
483
|
+
this._activeAudioUploadId = null;
|
|
484
|
+
|
|
485
|
+
// startSession() promise plumbing
|
|
486
|
+
this._sessionResolve = null;
|
|
487
|
+
this._sessionReject = null;
|
|
488
|
+
this._iceConnected = false;
|
|
489
|
+
this._dcOpen = false;
|
|
490
|
+
|
|
491
|
+
// Motion-completion plumbing for wakeUp() / gotoSleep().
|
|
492
|
+
//
|
|
493
|
+
// The daemon's data-channel handler dispatches `wake_up` and
|
|
494
|
+
// `goto_sleep` as async tasks and replies with
|
|
495
|
+
// `{status: "ok", command, completed: true}` when the trajectory
|
|
496
|
+
// ACTUALLY finishes (or with `{error, command}` on failure). We
|
|
497
|
+
// surface that as a Promise so callers can `await robot.gotoSleep()`
|
|
498
|
+
// and chain `setMotorMode('disabled')` without racing the
|
|
499
|
+
// trajectory player.
|
|
500
|
+
//
|
|
501
|
+
// Queues, not single slots: the data-channel protocol has no
|
|
502
|
+
// request IDs, but FIFO ordering is guaranteed by the daemon's
|
|
503
|
+
// serialised dispatcher, so the N-th response matches the N-th
|
|
504
|
+
// request. A queue makes back-to-back calls (e.g. teardown firing
|
|
505
|
+
// on top of an in-flight wake_up) safe; a single slot would
|
|
506
|
+
// silently drop the earlier awaiter.
|
|
507
|
+
this._pendingMotionCompletions = {
|
|
508
|
+
wake_up: [],
|
|
509
|
+
goto_sleep: [],
|
|
510
|
+
};
|
|
511
|
+
|
|
512
|
+
// Set by attachVideo()
|
|
513
|
+
this._videoElement = null;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// ─── Read-only properties ────────────────────────────────────────────
|
|
517
|
+
|
|
518
|
+
/** @returns {"disconnected"|"connected"|"streaming"} */
|
|
519
|
+
get state() { return this._state; }
|
|
520
|
+
|
|
521
|
+
/** @returns {Array<{id: string, meta: {name: string}}>} */
|
|
522
|
+
get robots() { return this._robots; }
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* Latest robot state (same shape as the "state" event detail).
|
|
526
|
+
* Mirrors the daemon's wire format — fields appear only once the
|
|
527
|
+
* daemon has sent the corresponding source field. Use the exported
|
|
528
|
+
* math utilities (``matrixToRpy``, ``radToDeg``) for human units.
|
|
529
|
+
* @returns {{
|
|
530
|
+
* head?: number[],
|
|
531
|
+
* antennas?: number[],
|
|
532
|
+
* body_yaw?: number,
|
|
533
|
+
* motor_mode?: "enabled"|"disabled"|"gravity_compensation",
|
|
534
|
+
* is_move_running?: boolean,
|
|
535
|
+
* }}
|
|
536
|
+
*/
|
|
537
|
+
get robotState() { return this._robotState; }
|
|
538
|
+
|
|
539
|
+
/** @returns {string|null} HuggingFace username, set after authenticate(). */
|
|
540
|
+
get username() { return this._username; }
|
|
541
|
+
|
|
542
|
+
/** @returns {boolean} True if a valid HF token is available. */
|
|
543
|
+
get isAuthenticated() { return !!this._token; }
|
|
544
|
+
|
|
545
|
+
/** @returns {boolean} True if the robot's SDP offered bidirectional audio. */
|
|
546
|
+
get micSupported() { return this._micSupported; }
|
|
547
|
+
|
|
548
|
+
/** @returns {boolean} */
|
|
549
|
+
get micMuted() { return this._micMuted; }
|
|
550
|
+
|
|
551
|
+
/** @returns {boolean} */
|
|
552
|
+
get audioMuted() { return this._audioMuted; }
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Peer id of the robot the embedding host wants this session to
|
|
556
|
+
* target, captured from the URL at construction time. Apps that
|
|
557
|
+
* want to support iframe-embedding without forcing the user to
|
|
558
|
+
* re-pick a robot read this and pass it straight to
|
|
559
|
+
* `startSession()` once `connect()` resolves:
|
|
560
|
+
*
|
|
561
|
+
* await robot.connect();
|
|
562
|
+
* await robot.startSession(robot.preselectedRobotId ?? pickedId);
|
|
563
|
+
*
|
|
564
|
+
* Returns `null` when the URL carries no `robot_peer_id` (typical
|
|
565
|
+
* standalone Space load). The value is also exposed on the
|
|
566
|
+
* "robotsChanged" payload via the `meta` sidecar for
|
|
567
|
+
* convenience, but the most direct read is right here.
|
|
568
|
+
*
|
|
569
|
+
* @returns {string|null}
|
|
570
|
+
*/
|
|
571
|
+
get preselectedRobotId() { return this._preselectedRobotId; }
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Convenience flag for apps that want to branch their UX on
|
|
575
|
+
* "am I embedded in a host shell?". True iff the URL carried a
|
|
576
|
+
* `robot_peer_id` hint at construction time (which only happens
|
|
577
|
+
* when a host iframe — mobile shell, vibe-coder preview, etc. —
|
|
578
|
+
* is the parent). Apps typically use it to skip their robot
|
|
579
|
+
* picker and their sign-in screen, since both are duplicated
|
|
580
|
+
* work the host has already done.
|
|
581
|
+
*
|
|
582
|
+
* @returns {boolean}
|
|
583
|
+
*/
|
|
584
|
+
get isEmbedded() { return this._preselectedRobotId !== null; }
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Internal: try to honour the `autoStartFromUrl` constructor
|
|
588
|
+
* option. Called from the signaling-message handler after every
|
|
589
|
+
* `robotsChanged` emit, so a robot that comes online after the
|
|
590
|
+
* SDK is already `connected` still triggers the auto-start.
|
|
591
|
+
* No-op unless `autoStartFromUrl` is set, the URL carries a
|
|
592
|
+
* preselect, the SDK is `connected`, the preselected robot is
|
|
593
|
+
* in the latest list, and we haven't already attempted in this
|
|
594
|
+
* page load. Errors are swallowed to a `console.warn` — the
|
|
595
|
+
* normal `startSession` rejection / `sessionRejected` event
|
|
596
|
+
* still fires for app-level handling.
|
|
597
|
+
*
|
|
598
|
+
* Defers the actual `startSession()` call by one macrotask
|
|
599
|
+
* (`setTimeout(..., 0)`) so it runs OUTSIDE the
|
|
600
|
+
* `_handleSignalingMessage` callstack that just processed the
|
|
601
|
+
* `'list'` message. Reproduced on Android WebView: firing
|
|
602
|
+
* `startSession` synchronously inside the SSE handler races the
|
|
603
|
+
* daemon's setup, leading to a connected-but-no-keyframe state
|
|
604
|
+
* where the receiver eternally NACKs and the iframe shows a
|
|
605
|
+
* black <video>. The macrotask-deferral is the minimum nudge
|
|
606
|
+
* that consistently resolves the race in our reproduction; if
|
|
607
|
+
* it ever proves insufficient on slower hardware, bump to
|
|
608
|
+
* a small explicit delay (e.g. 250 ms).
|
|
609
|
+
*/
|
|
610
|
+
_maybeAutoStart() {
|
|
611
|
+
if (!this._autoStartFromUrl) return;
|
|
612
|
+
if (this._autoStartAttempted) return;
|
|
613
|
+
if (!this._preselectedRobotId) return;
|
|
614
|
+
if (this._state !== 'connected') return;
|
|
615
|
+
const match = this._robots.find((r) => r.id === this._preselectedRobotId);
|
|
616
|
+
if (!match) return;
|
|
617
|
+
this._autoStartAttempted = true;
|
|
618
|
+
const peerId = this._preselectedRobotId;
|
|
619
|
+
setTimeout(() => {
|
|
620
|
+
// Re-check state in case a manual stopSession / disconnect
|
|
621
|
+
// landed between the schedule and the fire.
|
|
622
|
+
if (this._state !== 'connected') return;
|
|
623
|
+
this.startSession(peerId).catch((err) => {
|
|
624
|
+
console.warn('[reachy-mini] autoStartFromUrl: startSession rejected:', err);
|
|
625
|
+
});
|
|
626
|
+
}, 0);
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
// ─── Auth ────────────────────────────────────────────────────────────
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Check for a valid HuggingFace token.
|
|
633
|
+
*
|
|
634
|
+
* Resolution order:
|
|
635
|
+
* 1. URL fragment hand-off (`#hf_token=…&hf_username=…&hf_token_expires=…`).
|
|
636
|
+
* A host iframe — typically the Reachy Mini mobile app or a
|
|
637
|
+
* vibe-coder preview — can pass credentials through the URL
|
|
638
|
+
* fragment to bypass HF's `X-Frame-Options: SAMEORIGIN` block
|
|
639
|
+
* on `huggingface.co/login`. Seeded into `sessionStorage` and
|
|
640
|
+
* then stripped from the address bar so a page reload does not
|
|
641
|
+
* keep the credentials visible.
|
|
642
|
+
* 2. OAuth redirect callback (standalone Space, first sign-in).
|
|
643
|
+
* 3. `sessionStorage` cache (subsequent loads in any context).
|
|
644
|
+
*
|
|
645
|
+
* @returns {Promise<boolean>} true → token ready, false → call login()
|
|
646
|
+
*/
|
|
647
|
+
async authenticate() {
|
|
648
|
+
try {
|
|
649
|
+
// 1. Iframe hand-off. No-op when the URL has no fragment or
|
|
650
|
+
// the fragment carries no `hf_token`, so this is free on
|
|
651
|
+
// standalone Space loads.
|
|
652
|
+
consumeFragmentCredentials();
|
|
653
|
+
|
|
654
|
+
// 2. OAuth redirect callback.
|
|
655
|
+
const result = await oauthHandleRedirectIfPresent();
|
|
656
|
+
if (result) {
|
|
657
|
+
this._username = result.userInfo.preferred_username || result.userInfo.name;
|
|
658
|
+
this._token = result.accessToken;
|
|
659
|
+
this._tokenExpires = result.accessTokenExpiresAt;
|
|
660
|
+
sessionStorage.setItem('hf_token', this._token);
|
|
661
|
+
sessionStorage.setItem('hf_username', this._username);
|
|
662
|
+
sessionStorage.setItem('hf_token_expires', this._tokenExpires);
|
|
663
|
+
return true;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
// 3. sessionStorage cache. Both paths above also write here,
|
|
667
|
+
// so this is the canonical lookup for any subsequent call.
|
|
668
|
+
const t = sessionStorage.getItem('hf_token');
|
|
669
|
+
const u = sessionStorage.getItem('hf_username');
|
|
670
|
+
const e = sessionStorage.getItem('hf_token_expires');
|
|
671
|
+
if (t && u && e && new Date(e) > new Date()) {
|
|
672
|
+
this._token = t;
|
|
673
|
+
this._username = u;
|
|
674
|
+
this._tokenExpires = e;
|
|
675
|
+
return true;
|
|
676
|
+
}
|
|
677
|
+
return false;
|
|
678
|
+
} catch (e) {
|
|
679
|
+
console.error('Auth error:', e);
|
|
680
|
+
return false;
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/** Redirect the browser to the HuggingFace OAuth login page. */
|
|
685
|
+
async login() {
|
|
686
|
+
const opts = {};
|
|
687
|
+
if (this._clientId) opts.clientId = this._clientId;
|
|
688
|
+
window.location.href = await oauthLoginUrl(opts);
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/** Clear stored HF credentials and disconnect everything. */
|
|
692
|
+
logout() {
|
|
693
|
+
sessionStorage.removeItem('hf_token');
|
|
694
|
+
sessionStorage.removeItem('hf_username');
|
|
695
|
+
sessionStorage.removeItem('hf_token_expires');
|
|
696
|
+
this._username = null;
|
|
697
|
+
this._tokenExpires = null;
|
|
698
|
+
this.disconnect();
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// ─── Lifecycle ───────────────────────────────────────────────────────
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Open SSE signaling connection. Resolves once the server sends `welcome`.
|
|
705
|
+
* Emits "robotsChanged" as robots come and go.
|
|
706
|
+
* @param {string} [token] — HF access token. Omit to use the one from authenticate().
|
|
707
|
+
* @returns {Promise<void>}
|
|
708
|
+
*/
|
|
709
|
+
async connect(token) {
|
|
710
|
+
if (this._state !== 'disconnected') throw new Error('Already connected');
|
|
711
|
+
if (token) this._token = token;
|
|
712
|
+
if (!this._token) throw new Error('No token — call authenticate() first or pass a token');
|
|
713
|
+
this._sseAbortController = new AbortController();
|
|
714
|
+
|
|
715
|
+
let res;
|
|
716
|
+
try {
|
|
717
|
+
// Token goes in the Authorization header, not the URL —
|
|
718
|
+
// keeps it out of DevTools Network tab, browser history,
|
|
719
|
+
// Referer, and any server/proxy access log. We use fetch
|
|
720
|
+
// + manual stream reader (below) rather than EventSource
|
|
721
|
+
// specifically to allow custom headers.
|
|
722
|
+
res = await fetch(
|
|
723
|
+
`${this._signalingUrl}/events`,
|
|
724
|
+
{
|
|
725
|
+
signal: this._sseAbortController.signal,
|
|
726
|
+
headers: { 'Authorization': `Bearer ${this._token}` },
|
|
727
|
+
},
|
|
728
|
+
);
|
|
729
|
+
} catch (e) {
|
|
730
|
+
this._sseAbortController = null;
|
|
731
|
+
throw e;
|
|
732
|
+
}
|
|
733
|
+
if (!res.ok) {
|
|
734
|
+
this._sseAbortController = null;
|
|
735
|
+
throw new Error(`HTTP ${res.status}`);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
return new Promise((resolve, reject) => {
|
|
739
|
+
let welcomed = false;
|
|
740
|
+
const reader = res.body.getReader();
|
|
741
|
+
const decoder = new TextDecoder();
|
|
742
|
+
let buffer = '';
|
|
743
|
+
|
|
744
|
+
const readLoop = async () => {
|
|
745
|
+
try {
|
|
746
|
+
while (true) {
|
|
747
|
+
const { done, value } = await reader.read();
|
|
748
|
+
if (done) break;
|
|
749
|
+
buffer += decoder.decode(value, { stream: true });
|
|
750
|
+
const lines = buffer.split('\n');
|
|
751
|
+
buffer = lines.pop();
|
|
752
|
+
for (const line of lines) {
|
|
753
|
+
if (!line.startsWith('data:')) continue;
|
|
754
|
+
try {
|
|
755
|
+
const msg = JSON.parse(line.slice(5).trim());
|
|
756
|
+
if (!welcomed && msg.type === 'welcome') {
|
|
757
|
+
welcomed = true;
|
|
758
|
+
this._peerId = msg.peerId;
|
|
759
|
+
this._state = 'connected';
|
|
760
|
+
await this._sendToServer({
|
|
761
|
+
type: 'setPeerStatus',
|
|
762
|
+
roles: ['listener'],
|
|
763
|
+
meta: { name: this._appName },
|
|
764
|
+
});
|
|
765
|
+
this._emit('connected', { peerId: msg.peerId });
|
|
766
|
+
resolve();
|
|
767
|
+
}
|
|
768
|
+
this._handleSignalingMessage(msg);
|
|
769
|
+
} catch (_) { /* malformed JSON — skip */ }
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
} catch (e) {
|
|
773
|
+
if (e.name !== 'AbortError') {
|
|
774
|
+
this._emit('error', { source: 'signaling', error: e });
|
|
775
|
+
}
|
|
776
|
+
if (!welcomed) { reject(e); return; }
|
|
777
|
+
}
|
|
778
|
+
// SSE stream ended (server closed or network drop)
|
|
779
|
+
if (this._state !== 'disconnected') {
|
|
780
|
+
this._state = 'disconnected';
|
|
781
|
+
this._emit('disconnected', { reason: 'SSE closed' });
|
|
782
|
+
}
|
|
783
|
+
if (!welcomed) reject(new Error('Connection closed before welcome'));
|
|
784
|
+
};
|
|
785
|
+
|
|
786
|
+
readLoop();
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
/**
|
|
791
|
+
* One-shot bring-up: auth → SSE connect → robot selection → session →
|
|
792
|
+
* wake up. The all-in-one entry point that captures the common
|
|
793
|
+
* "embed *or* standalone, just get me streaming" flow so each
|
|
794
|
+
* consumer does not have to re-implement it.
|
|
795
|
+
*
|
|
796
|
+
* What it does, in order:
|
|
797
|
+
* 1. **Auth.** If `this._token` is not set, calls `authenticate()`
|
|
798
|
+
* (which honours the iframe URL-fragment hand-off, the OAuth
|
|
799
|
+
* redirect callback, and the `sessionStorage` cache). Throws if
|
|
800
|
+
* none yield a token — the consumer should call `login()` and
|
|
801
|
+
* retry after the redirect. Pass an explicit `token` to skip
|
|
802
|
+
* `authenticate()` entirely.
|
|
803
|
+
* 2. **Connect.** If `state === 'disconnected'`, opens the SSE
|
|
804
|
+
* signaling channel.
|
|
805
|
+
* 3. **Pick a robot.**
|
|
806
|
+
* - **Embed mode** (`this.isEmbedded`): uses
|
|
807
|
+
* `this._preselectedRobotId` from the URL. No picker callback
|
|
808
|
+
* invoked; we briefly wait for that robot to appear in the
|
|
809
|
+
* SSE list, then proceed.
|
|
810
|
+
* - **Standalone**: GETs `/api/robot-status` for the owner's
|
|
811
|
+
* robots with busy state, dedupes by `install_id`, sorts by
|
|
812
|
+
* freshness. If `autoPickIfSingle` and exactly one free, picks
|
|
813
|
+
* it. Else calls the consumer-supplied `pickRobot(robots)`
|
|
814
|
+
* callback. Throws if neither yields an id.
|
|
815
|
+
* 4. **Start session.** Awaits `startSession(robotId)` (ICE + DC).
|
|
816
|
+
* 5. **Wake up.** Awaits `ensureAwake()` so sliders don't silently
|
|
817
|
+
* no-op against a torque-off robot.
|
|
818
|
+
*
|
|
819
|
+
* @param {{
|
|
820
|
+
* token?: string, // skip authenticate(); use this raw HF token
|
|
821
|
+
* pickRobot?: (robots: Array<{
|
|
822
|
+
* id: string,
|
|
823
|
+
* name: string|null,
|
|
824
|
+
* busy: boolean,
|
|
825
|
+
* activeApp: string|null,
|
|
826
|
+
* meta: object,
|
|
827
|
+
* lastSeenAgeSeconds: number|null,
|
|
828
|
+
* }>) => Promise<string|null>, // called only in standalone, multi-robot case
|
|
829
|
+
* autoPickIfSingle?: boolean, // default true — skip the callback when 1 free robot
|
|
830
|
+
* filterBusy?: boolean, // default true — hide busy robots from the picker
|
|
831
|
+
* wakeOnConnect?: boolean, // default true — call ensureAwake() after startSession
|
|
832
|
+
* }} [options]
|
|
833
|
+
* @returns {Promise<{
|
|
834
|
+
* robotId: string,
|
|
835
|
+
* robotName: string|null,
|
|
836
|
+
* isEmbedded: boolean,
|
|
837
|
+
* alreadyStreaming?: boolean,
|
|
838
|
+
* }>}
|
|
839
|
+
*/
|
|
840
|
+
async autoConnect(options = {}) {
|
|
841
|
+
const {
|
|
842
|
+
token = null,
|
|
843
|
+
pickRobot = null,
|
|
844
|
+
autoPickIfSingle = true,
|
|
845
|
+
filterBusy = true,
|
|
846
|
+
wakeOnConnect = true,
|
|
847
|
+
} = options;
|
|
848
|
+
|
|
849
|
+
// Idempotent fast-path: caller invoked autoConnect() on an
|
|
850
|
+
// already-streaming session (e.g. on a route change inside an
|
|
851
|
+
// SPA). Return the current selection rather than tearing down.
|
|
852
|
+
if (this._state === 'streaming') {
|
|
853
|
+
const cur = this._robots?.find((r) => r.id === this._selectedRobotId);
|
|
854
|
+
return {
|
|
855
|
+
robotId: this._selectedRobotId,
|
|
856
|
+
robotName: cur?.meta?.name ?? null,
|
|
857
|
+
isEmbedded: this.isEmbedded,
|
|
858
|
+
alreadyStreaming: true,
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
// autoConnect takes over the bring-up — disable the SDK's
|
|
863
|
+
// own `autoStartFromUrl` so the two paths don't race and
|
|
864
|
+
// both call `startSession()` against the same preselected
|
|
865
|
+
// robot. The race used to manifest as central rejecting the
|
|
866
|
+
// second attempt with "Robot is busy: <appName>" — the
|
|
867
|
+
// appName being our own first attempt. Restored on the way
|
|
868
|
+
// out so a later `stopSession()` followed by a fresh
|
|
869
|
+
// listener attach still benefits from auto-start.
|
|
870
|
+
const _prevAutoStartFromUrl = this._autoStartFromUrl;
|
|
871
|
+
this._autoStartFromUrl = false;
|
|
872
|
+
|
|
873
|
+
try {
|
|
874
|
+
// 1. Auth.
|
|
875
|
+
if (token) {
|
|
876
|
+
this._token = token;
|
|
877
|
+
} else if (!this._token) {
|
|
878
|
+
const ok = await this.authenticate();
|
|
879
|
+
if (!ok) {
|
|
880
|
+
// login() does a full page redirect; we don't trigger
|
|
881
|
+
// it here so the consumer can decide (a desktop tray
|
|
882
|
+
// wants different recovery than a standalone Space).
|
|
883
|
+
throw new Error('Not authenticated — call login() or pass a token');
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// 2. SSE connect.
|
|
888
|
+
if (this._state === 'disconnected') {
|
|
889
|
+
await this.connect();
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
// 3. Resolve the target robot.
|
|
893
|
+
let robotId;
|
|
894
|
+
let robotName = null;
|
|
895
|
+
if (this.isEmbedded) {
|
|
896
|
+
robotId = this._preselectedRobotId;
|
|
897
|
+
// Wait briefly for the preselected robot to surface in the
|
|
898
|
+
// SSE list. Best-effort: if it never shows we still try
|
|
899
|
+
// startSession() — central may know about a robot the SSE
|
|
900
|
+
// list pushes only a moment later.
|
|
901
|
+
try {
|
|
902
|
+
await this._waitForRobotInList(robotId, 5000);
|
|
903
|
+
} catch (_) { /* fall through */ }
|
|
904
|
+
const found = this._robots?.find((r) => r.id === robotId);
|
|
905
|
+
robotName = found?.meta?.name ?? null;
|
|
906
|
+
} else {
|
|
907
|
+
const robots = await this._fetchOwnedRobots({ filterBusy });
|
|
908
|
+
if (robots.length === 0) {
|
|
909
|
+
throw new Error('No reachable robots');
|
|
910
|
+
}
|
|
911
|
+
if (autoPickIfSingle && robots.length === 1 && !robots[0].busy) {
|
|
912
|
+
robotId = robots[0].id;
|
|
913
|
+
robotName = robots[0].name;
|
|
914
|
+
} else if (pickRobot) {
|
|
915
|
+
const picked = await pickRobot(robots);
|
|
916
|
+
if (!picked) throw new Error('Robot selection cancelled');
|
|
917
|
+
robotId = picked;
|
|
918
|
+
robotName = robots.find((r) => r.id === picked)?.name ?? null;
|
|
919
|
+
} else {
|
|
920
|
+
throw new Error(
|
|
921
|
+
'Multiple robots available — pass a pickRobot callback to autoConnect()',
|
|
922
|
+
);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
// 4. Session.
|
|
927
|
+
await this.startSession(robotId);
|
|
928
|
+
|
|
929
|
+
// 5. Wake.
|
|
930
|
+
if (wakeOnConnect && typeof this.ensureAwake === 'function') {
|
|
931
|
+
try { await this.ensureAwake(); }
|
|
932
|
+
catch (e) { console.warn('[reachy-mini] autoConnect: ensureAwake failed:', e); }
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
return { robotId, robotName, isEmbedded: this.isEmbedded };
|
|
936
|
+
} finally {
|
|
937
|
+
this._autoStartFromUrl = _prevAutoStartFromUrl;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
/**
|
|
942
|
+
* Fetch the caller's robots with busy state, deduped + sorted.
|
|
943
|
+
* One-shot snapshot — no live subscription. Falls back to the SSE
|
|
944
|
+
* `_robots` cache if `/api/robot-status` is unavailable (older
|
|
945
|
+
* central deployments don't expose it).
|
|
946
|
+
*
|
|
947
|
+
* Dedup: same physical robot can appear twice transiently after a
|
|
948
|
+
* daemon reinstall (new peerId, same install_id). Last-writer-wins
|
|
949
|
+
* on `install_id`, then `hardware_id`, then `peerId` (= no dedup).
|
|
950
|
+
*
|
|
951
|
+
* @returns {Promise<Array<{
|
|
952
|
+
* id: string,
|
|
953
|
+
* name: string|null,
|
|
954
|
+
* busy: boolean,
|
|
955
|
+
* activeApp: string|null,
|
|
956
|
+
* meta: object,
|
|
957
|
+
* lastSeenAgeSeconds: number|null,
|
|
958
|
+
* }>>}
|
|
959
|
+
*/
|
|
960
|
+
async _fetchOwnedRobots({ filterBusy = true } = {}) {
|
|
961
|
+
try {
|
|
962
|
+
const res = await fetch(`${this._signalingUrl}/api/robot-status`, {
|
|
963
|
+
headers: { 'Authorization': `Bearer ${this._token}` },
|
|
964
|
+
});
|
|
965
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
966
|
+
const json = await res.json();
|
|
967
|
+
const seen = new Map(); // dedup key → projected robot
|
|
968
|
+
for (const r of (json.robots || [])) {
|
|
969
|
+
if (filterBusy && r.busy) continue;
|
|
970
|
+
const key = r.meta?.install_id ?? r.meta?.hardware_id ?? r.peerId;
|
|
971
|
+
seen.set(key, {
|
|
972
|
+
id: r.peerId,
|
|
973
|
+
name: r.robotName ?? r.meta?.name ?? null,
|
|
974
|
+
busy: !!r.busy,
|
|
975
|
+
activeApp: r.activeApp ?? null,
|
|
976
|
+
meta: r.meta ?? {},
|
|
977
|
+
lastSeenAgeSeconds: r.last_seen_age_seconds ?? null,
|
|
978
|
+
});
|
|
979
|
+
}
|
|
980
|
+
return Array.from(seen.values()).sort(
|
|
981
|
+
(a, b) => (a.lastSeenAgeSeconds ?? Infinity) - (b.lastSeenAgeSeconds ?? Infinity),
|
|
982
|
+
);
|
|
983
|
+
} catch (e) {
|
|
984
|
+
console.warn('[reachy-mini] /api/robot-status unavailable, using SSE list:', e);
|
|
985
|
+
return (this._robots || []).map((r) => ({
|
|
986
|
+
id: r.id,
|
|
987
|
+
name: r.meta?.name ?? null,
|
|
988
|
+
busy: false, // unknown — SSE list does not carry busy state
|
|
989
|
+
activeApp: null,
|
|
990
|
+
meta: r.meta ?? {},
|
|
991
|
+
lastSeenAgeSeconds: null,
|
|
992
|
+
}));
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
/**
|
|
997
|
+
* Resolve once `robotId` appears in `_robots`, or reject after
|
|
998
|
+
* `timeoutMs`. Used by `autoConnect()`'s embed branch so the preselected
|
|
999
|
+
* robot has a chance to surface from the first SSE `list` push before
|
|
1000
|
+
* `startSession()` is fired.
|
|
1001
|
+
*/
|
|
1002
|
+
_waitForRobotInList(robotId, timeoutMs) {
|
|
1003
|
+
if (this._robots?.find((r) => r.id === robotId)) return Promise.resolve();
|
|
1004
|
+
return new Promise((resolve, reject) => {
|
|
1005
|
+
const onChange = () => {
|
|
1006
|
+
if (this._robots?.find((r) => r.id === robotId)) {
|
|
1007
|
+
this.removeEventListener('robotsChanged', onChange);
|
|
1008
|
+
clearTimeout(timeoutId);
|
|
1009
|
+
resolve();
|
|
1010
|
+
}
|
|
1011
|
+
};
|
|
1012
|
+
const timeoutId = setTimeout(() => {
|
|
1013
|
+
this.removeEventListener('robotsChanged', onChange);
|
|
1014
|
+
reject(new Error(`Timeout waiting for robot ${robotId} in list`));
|
|
1015
|
+
}, timeoutMs);
|
|
1016
|
+
this.addEventListener('robotsChanged', onChange);
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* Start a WebRTC session with the given robot.
|
|
1022
|
+
* Acquires the microphone (if enabled), negotiates SDP, and waits for
|
|
1023
|
+
* both ICE connection and data channel to be ready before resolving.
|
|
1024
|
+
* Emits "videoTrack" when the robot's camera stream arrives.
|
|
1025
|
+
* Emits "micSupported" once SDP negotiation reveals whether the robot
|
|
1026
|
+
* accepts bidirectional audio.
|
|
1027
|
+
* @param {string} robotId — one of the ids from the robots list
|
|
1028
|
+
* @returns {Promise<void>}
|
|
1029
|
+
*/
|
|
1030
|
+
async startSession(robotId) {
|
|
1031
|
+
if (this._state !== 'connected') throw new Error('Not connected');
|
|
1032
|
+
this._selectedRobotId = robotId;
|
|
1033
|
+
this._iceConnected = false;
|
|
1034
|
+
this._dcOpen = false;
|
|
1035
|
+
this._micSupported = false;
|
|
1036
|
+
// Buffer for ICE candidates that arrive before the SDP
|
|
1037
|
+
// exchange completes (see _handlePeerMessage). Reset on every
|
|
1038
|
+
// fresh session so stale candidates from a previous attempt
|
|
1039
|
+
// don't get applied to a new RTCPeerConnection.
|
|
1040
|
+
this._pendingRemoteIce = [];
|
|
1041
|
+
|
|
1042
|
+
// Acquire mic eagerly so the browser permission prompt appears now,
|
|
1043
|
+
// but tracks stay disabled (muted) until the user explicitly unmutes.
|
|
1044
|
+
if (this._enableMicrophone) {
|
|
1045
|
+
try {
|
|
1046
|
+
this._micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
1047
|
+
this._micStream.getAudioTracks().forEach(t => { t.enabled = false; });
|
|
1048
|
+
this._micMuted = true;
|
|
1049
|
+
} catch (e) {
|
|
1050
|
+
console.warn('Microphone not available:', e);
|
|
1051
|
+
// Fall back to a silent placeholder track. We MUST add an
|
|
1052
|
+
// audio track before createAnswer or the answer SDP comes
|
|
1053
|
+
// back as recvonly for audio - which negotiates the audio
|
|
1054
|
+
// SENDER side off the wire entirely. A host that wants to
|
|
1055
|
+
// later inject a different audio source (e.g. a synthesised
|
|
1056
|
+
// AI voice via replaceTrack on the sender) needs a live
|
|
1057
|
+
// sendrecv slot, even if the initial track is silent.
|
|
1058
|
+
try {
|
|
1059
|
+
const ctx = new (window.AudioContext || window.webkitAudioContext)();
|
|
1060
|
+
const dst = ctx.createMediaStreamDestination();
|
|
1061
|
+
// A muted oscillator keeps the track "alive" without
|
|
1062
|
+
// emitting any audible signal.
|
|
1063
|
+
const osc = ctx.createOscillator();
|
|
1064
|
+
const gain = ctx.createGain();
|
|
1065
|
+
gain.gain.value = 0;
|
|
1066
|
+
osc.connect(gain).connect(dst);
|
|
1067
|
+
osc.start();
|
|
1068
|
+
this._micStream = dst.stream;
|
|
1069
|
+
this._micStream.getAudioTracks().forEach(t => { t.enabled = false; });
|
|
1070
|
+
this._micMuted = true;
|
|
1071
|
+
this._silentMicFallback = { ctx, osc };
|
|
1072
|
+
} catch (fallbackErr) {
|
|
1073
|
+
console.warn('Silent mic fallback failed:', fallbackErr);
|
|
1074
|
+
this._micStream = null;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
this._pc = new RTCPeerConnection({
|
|
1080
|
+
iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
|
|
1081
|
+
});
|
|
1082
|
+
|
|
1083
|
+
return new Promise((resolve, reject) => {
|
|
1084
|
+
this._sessionResolve = resolve;
|
|
1085
|
+
this._sessionReject = reject;
|
|
1086
|
+
|
|
1087
|
+
this._pc.ontrack = (e) => {
|
|
1088
|
+
if (e.track.kind === 'video') {
|
|
1089
|
+
// Tell the receiver's jitter buffer to minimise its hold
|
|
1090
|
+
// time. Both properties target the same internal buffer;
|
|
1091
|
+
// browsers ignore whichever they don't implement.
|
|
1092
|
+
const ms = this._videoJitterBufferTargetMs;
|
|
1093
|
+
try { e.receiver.jitterBufferTarget = ms; } catch (_) {}
|
|
1094
|
+
try { e.receiver.playoutDelayHint = ms / 1000; } catch (_) {}
|
|
1095
|
+
this._emit('videoTrack', { track: e.track, stream: e.streams[0] });
|
|
1096
|
+
}
|
|
1097
|
+
};
|
|
1098
|
+
|
|
1099
|
+
this._pc.onicecandidate = async (e) => {
|
|
1100
|
+
if (e.candidate && this._sessionId) {
|
|
1101
|
+
await this._sendToServer({
|
|
1102
|
+
type: 'peer',
|
|
1103
|
+
sessionId: this._sessionId,
|
|
1104
|
+
ice: {
|
|
1105
|
+
candidate: e.candidate.candidate,
|
|
1106
|
+
sdpMLineIndex: e.candidate.sdpMLineIndex,
|
|
1107
|
+
sdpMid: e.candidate.sdpMid,
|
|
1108
|
+
},
|
|
1109
|
+
});
|
|
1110
|
+
}
|
|
1111
|
+
};
|
|
1112
|
+
|
|
1113
|
+
this._pc.oniceconnectionstatechange = () => {
|
|
1114
|
+
const s = this._pc?.iceConnectionState;
|
|
1115
|
+
if (!s) return;
|
|
1116
|
+
if (s === 'connected' || s === 'completed') {
|
|
1117
|
+
this._iceConnected = true;
|
|
1118
|
+
this._checkSessionReady();
|
|
1119
|
+
} else if (s === 'failed') {
|
|
1120
|
+
const err = new Error('ICE connection failed');
|
|
1121
|
+
if (this._sessionReject) {
|
|
1122
|
+
this._sessionReject(err);
|
|
1123
|
+
this._sessionResolve = null;
|
|
1124
|
+
this._sessionReject = null;
|
|
1125
|
+
}
|
|
1126
|
+
this._emit('error', { source: 'webrtc', error: err });
|
|
1127
|
+
} else if (s === 'disconnected') {
|
|
1128
|
+
this._emit('error', { source: 'webrtc', error: new Error('ICE disconnected') });
|
|
1129
|
+
}
|
|
1130
|
+
};
|
|
1131
|
+
|
|
1132
|
+
this._pc.ondatachannel = (e) => {
|
|
1133
|
+
this._dc = e.channel;
|
|
1134
|
+
this._dc.onopen = () => {
|
|
1135
|
+
this._dcOpen = true;
|
|
1136
|
+
this._checkSessionReady();
|
|
1137
|
+
};
|
|
1138
|
+
this._dc.onmessage = (ev) => this._handleRobotMessage(JSON.parse(ev.data));
|
|
1139
|
+
};
|
|
1140
|
+
|
|
1141
|
+
this._sendToServer({ type: 'startSession', peerId: robotId }).then((r) => {
|
|
1142
|
+
if (r?.type === 'sessionRejected') {
|
|
1143
|
+
this._failSessionRejected(r);
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
if (r?.sessionId) this._sessionId = r.sessionId;
|
|
1147
|
+
});
|
|
1148
|
+
});
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
/**
|
|
1152
|
+
* Internal: handle a sessionRejected response from central.
|
|
1153
|
+
* Releases resources allocated by startSession() (RTCPeerConnection,
|
|
1154
|
+
* microphone stream) and rejects the pending startSession() promise
|
|
1155
|
+
* with an Error carrying `.reason` and `.activeApp`.
|
|
1156
|
+
*
|
|
1157
|
+
* Called from both the POST-response path (primary) and the SSE
|
|
1158
|
+
* handler (defensive, in case the server changes).
|
|
1159
|
+
*/
|
|
1160
|
+
_failSessionRejected(msg) {
|
|
1161
|
+
const err = new Error(
|
|
1162
|
+
msg.reason === 'robot_busy'
|
|
1163
|
+
? `Robot is busy: "${msg.activeApp || 'another app'}" is already connected`
|
|
1164
|
+
: `Session rejected: ${msg.reason || 'unknown reason'}`
|
|
1165
|
+
);
|
|
1166
|
+
err.reason = msg.reason;
|
|
1167
|
+
err.activeApp = msg.activeApp;
|
|
1168
|
+
|
|
1169
|
+
// Release resources allocated optimistically in startSession()
|
|
1170
|
+
// before we knew the server would refuse.
|
|
1171
|
+
if (this._pc) { this._pc.close(); this._pc = null; }
|
|
1172
|
+
if (this._micStream) { this._micStream.getTracks().forEach(t => t.stop()); this._micStream = null; }
|
|
1173
|
+
this._iceConnected = false;
|
|
1174
|
+
this._dcOpen = false;
|
|
1175
|
+
this._micMuted = true;
|
|
1176
|
+
this._micSupported = false;
|
|
1177
|
+
|
|
1178
|
+
this._emit('sessionRejected', { reason: msg.reason, activeApp: msg.activeApp });
|
|
1179
|
+
|
|
1180
|
+
if (this._sessionReject) {
|
|
1181
|
+
const reject = this._sessionReject;
|
|
1182
|
+
this._sessionResolve = null;
|
|
1183
|
+
this._sessionReject = null;
|
|
1184
|
+
reject(err);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
/**
|
|
1189
|
+
* End the WebRTC session. Returns to "connected" state so you can
|
|
1190
|
+
* startSession() again with the same or a different robot.
|
|
1191
|
+
* @returns {Promise<void>}
|
|
1192
|
+
*/
|
|
1193
|
+
async stopSession() {
|
|
1194
|
+
if (this._versionResolve) { this._versionResolve(null); this._versionResolve = null; }
|
|
1195
|
+
if (this._hardwareIdResolve) { this._hardwareIdResolve(null); this._hardwareIdResolve = null; }
|
|
1196
|
+
if (this._volumeResolve) { this._volumeResolve(null); this._volumeResolve = null; }
|
|
1197
|
+
if (this._micVolumeResolve) { this._micVolumeResolve(null); this._micVolumeResolve = null; }
|
|
1198
|
+
// Drop any active log subscribers — the daemon-side subprocess
|
|
1199
|
+
// is torn down on peer-disconnect, so resubscribing across a
|
|
1200
|
+
// reconnect requires a fresh subscribeLogs() call from the
|
|
1201
|
+
// consumer.
|
|
1202
|
+
this._logSubscribers.clear();
|
|
1203
|
+
// Drain any in-flight wakeUp() / gotoSleep() awaiters before
|
|
1204
|
+
// the data channel is killed below, so callers don't sit on a
|
|
1205
|
+
// promise that can never resolve.
|
|
1206
|
+
this._rejectPendingMotionCompletions(new Error('Session stopped'));
|
|
1207
|
+
if (this._sessionReject) {
|
|
1208
|
+
this._sessionReject(new Error('Session stopped'));
|
|
1209
|
+
this._sessionResolve = null;
|
|
1210
|
+
this._sessionReject = null;
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
if (this._stateRefreshInterval) { clearInterval(this._stateRefreshInterval); this._stateRefreshInterval = null; }
|
|
1214
|
+
if (this._latencyMonitorId) { clearInterval(this._latencyMonitorId); this._latencyMonitorId = null; }
|
|
1215
|
+
|
|
1216
|
+
if (this._sessionId) {
|
|
1217
|
+
await this._sendToServer({ type: 'endSession', sessionId: this._sessionId });
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
if (this._micStream) { this._micStream.getTracks().forEach(t => t.stop()); this._micStream = null; }
|
|
1221
|
+
this._micMuted = true;
|
|
1222
|
+
this._micSupported = false;
|
|
1223
|
+
|
|
1224
|
+
if (this._pc) { this._pc.close(); this._pc = null; }
|
|
1225
|
+
if (this._dc) { this._dc.close(); this._dc = null; }
|
|
1226
|
+
|
|
1227
|
+
this._sessionId = null;
|
|
1228
|
+
this._iceConnected = false;
|
|
1229
|
+
this._dcOpen = false;
|
|
1230
|
+
|
|
1231
|
+
const wasStreaming = this._state === 'streaming';
|
|
1232
|
+
if (wasStreaming) {
|
|
1233
|
+
this._state = 'connected';
|
|
1234
|
+
this._emit('sessionStopped', { reason: 'user' });
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
/**
|
|
1239
|
+
* Full teardown — abort SSE, close WebRTC.
|
|
1240
|
+
* Auth state is preserved (call logout() to also clear credentials).
|
|
1241
|
+
*/
|
|
1242
|
+
disconnect() {
|
|
1243
|
+
if (this._sseAbortController) { this._sseAbortController.abort(); this._sseAbortController = null; }
|
|
1244
|
+
|
|
1245
|
+
if (this._versionResolve) { this._versionResolve(null); this._versionResolve = null; }
|
|
1246
|
+
if (this._hardwareIdResolve) { this._hardwareIdResolve(null); this._hardwareIdResolve = null; }
|
|
1247
|
+
if (this._volumeResolve) { this._volumeResolve(null); this._volumeResolve = null; }
|
|
1248
|
+
if (this._micVolumeResolve) { this._micVolumeResolve(null); this._micVolumeResolve = null; }
|
|
1249
|
+
this._logSubscribers.clear();
|
|
1250
|
+
// Same rationale as in stopSession(): drain pending motion
|
|
1251
|
+
// awaiters before tearing down the data channel.
|
|
1252
|
+
this._rejectPendingMotionCompletions(new Error('Disconnected'));
|
|
1253
|
+
if (this._sessionReject) {
|
|
1254
|
+
this._sessionReject(new Error('Disconnected'));
|
|
1255
|
+
this._sessionResolve = null;
|
|
1256
|
+
this._sessionReject = null;
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
if (this._stateRefreshInterval) { clearInterval(this._stateRefreshInterval); this._stateRefreshInterval = null; }
|
|
1260
|
+
if (this._latencyMonitorId) { clearInterval(this._latencyMonitorId); this._latencyMonitorId = null; }
|
|
1261
|
+
|
|
1262
|
+
if (this._sessionId && this._token) {
|
|
1263
|
+
this._sendToServer({ type: 'endSession', sessionId: this._sessionId }); // fire-and-forget
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
if (this._micStream) { this._micStream.getTracks().forEach(t => t.stop()); this._micStream = null; }
|
|
1267
|
+
if (this._pc) { this._pc.close(); this._pc = null; }
|
|
1268
|
+
if (this._dc) { this._dc.close(); this._dc = null; }
|
|
1269
|
+
|
|
1270
|
+
this._sessionId = null;
|
|
1271
|
+
this._micMuted = true;
|
|
1272
|
+
this._micSupported = false;
|
|
1273
|
+
this._iceConnected = false;
|
|
1274
|
+
this._dcOpen = false;
|
|
1275
|
+
this._peerId = null;
|
|
1276
|
+
this._robots = [];
|
|
1277
|
+
this._state = 'disconnected';
|
|
1278
|
+
this._emit('disconnected', { reason: 'user' });
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
// ─── Commands ────────────────────────────────────────────────────────
|
|
1282
|
+
// All return false if the data channel is not open, true if sent.
|
|
1283
|
+
|
|
1284
|
+
/**
|
|
1285
|
+
* Send a target pose to the robot. Wire-shape, raw units only —
|
|
1286
|
+
* single source of truth for motion commands. Every field is
|
|
1287
|
+
* optional; omitted fields leave the daemon's previous target
|
|
1288
|
+
* unchanged, so partial updates compose naturally.
|
|
1289
|
+
*
|
|
1290
|
+
* For human units (degrees), use the ``setHeadRpyDeg`` /
|
|
1291
|
+
* ``setAntennasDeg`` / ``setBodyYawDeg`` thin wrappers below.
|
|
1292
|
+
*
|
|
1293
|
+
* @param {object} [target]
|
|
1294
|
+
* @param {number[]} [target.head] 16-element flat row-major 4×4
|
|
1295
|
+
* matrix (full SE(3); preserves translation, no XYZ loss).
|
|
1296
|
+
* @param {number[]} [target.antennas] ``[rightRad, leftRad]``.
|
|
1297
|
+
* @param {number} [target.body_yaw] Body yaw in radians.
|
|
1298
|
+
* @returns {boolean} false if the data channel is not open.
|
|
1299
|
+
* @throws {TypeError} if any provided field has the wrong shape or
|
|
1300
|
+
* contains a non-finite value (NaN, Infinity). Validation runs at
|
|
1301
|
+
* the JS boundary so caller mistakes surface with a stack trace
|
|
1302
|
+
* pointing to the call site, not as a confusing daemon-side error.
|
|
1303
|
+
*/
|
|
1304
|
+
setTarget({ head, antennas, body_yaw } = {}) {
|
|
1305
|
+
const cmd = { type: "set_full_target" };
|
|
1306
|
+
if (head !== undefined) {
|
|
1307
|
+
if (!Array.isArray(head) || head.length !== 16
|
|
1308
|
+
|| !head.every((n) => Number.isFinite(n))) {
|
|
1309
|
+
throw new TypeError(
|
|
1310
|
+
'setTarget: head must be a 16-element flat row-major 4×4 matrix '
|
|
1311
|
+
+ `of finite numbers; got ${Array.isArray(head) ? `Array(${head.length})` : typeof head}`
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
cmd.head = head;
|
|
1315
|
+
}
|
|
1316
|
+
if (antennas !== undefined) {
|
|
1317
|
+
if (!Array.isArray(antennas) || antennas.length !== 2
|
|
1318
|
+
|| !antennas.every((n) => Number.isFinite(n))) {
|
|
1319
|
+
throw new TypeError(
|
|
1320
|
+
'setTarget: antennas must be [rightRad, leftRad] (2 finite numbers); '
|
|
1321
|
+
+ `got ${Array.isArray(antennas) ? `Array(${antennas.length})` : typeof antennas}`
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
cmd.antennas = antennas;
|
|
1325
|
+
}
|
|
1326
|
+
if (body_yaw !== undefined) {
|
|
1327
|
+
if (!Number.isFinite(body_yaw)) {
|
|
1328
|
+
throw new TypeError(
|
|
1329
|
+
`setTarget: body_yaw must be a finite number (radians); got ${body_yaw}`
|
|
1330
|
+
);
|
|
1331
|
+
}
|
|
1332
|
+
cmd.body_yaw = body_yaw;
|
|
1333
|
+
}
|
|
1334
|
+
return this._sendCommand(cmd);
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
/**
|
|
1338
|
+
* Smooth daemon-side interpolation to a target pose over
|
|
1339
|
+
* ``duration`` seconds. Mirrors ``setTarget``'s wire shape (head
|
|
1340
|
+
* is a 16-element flat row-major 4×4, antennas are
|
|
1341
|
+
* ``[rightRad, leftRad]``, body_yaw is radians) and adds a
|
|
1342
|
+
* required ``duration`` field. The daemon dispatches the command
|
|
1343
|
+
* to its lerp planner instead of jumping to the target.
|
|
1344
|
+
*
|
|
1345
|
+
* Use this for one-shot smooth approaches to an arbitrary pose
|
|
1346
|
+
* (e.g. soft-return-to-base after recording, or pre-positioning
|
|
1347
|
+
* before a streamed playback). For continuous streamed motion,
|
|
1348
|
+
* use ``setTarget`` and lerp client-side.
|
|
1349
|
+
*
|
|
1350
|
+
* @param {{head?: number[], antennas?: number[], body_yaw?: number, duration: number}} args
|
|
1351
|
+
* @returns {boolean} false if the data channel is not open.
|
|
1352
|
+
* @throws {TypeError} if any provided field has the wrong shape
|
|
1353
|
+
* or contains a non-finite value (NaN, Infinity), or if
|
|
1354
|
+
* ``duration`` is missing or non-positive.
|
|
1355
|
+
*/
|
|
1356
|
+
gotoTarget({ head, antennas, body_yaw, duration } = {}) {
|
|
1357
|
+
const cmd = { type: "goto_target" };
|
|
1358
|
+
if (head !== undefined) {
|
|
1359
|
+
if (!Array.isArray(head) || head.length !== 16
|
|
1360
|
+
|| !head.every((n) => Number.isFinite(n))) {
|
|
1361
|
+
throw new TypeError(
|
|
1362
|
+
'gotoTarget: head must be a 16-element flat row-major 4×4 matrix '
|
|
1363
|
+
+ `of finite numbers; got ${Array.isArray(head) ? `Array(${head.length})` : typeof head}`
|
|
1364
|
+
);
|
|
1365
|
+
}
|
|
1366
|
+
cmd.head = head;
|
|
1367
|
+
}
|
|
1368
|
+
if (antennas !== undefined) {
|
|
1369
|
+
if (!Array.isArray(antennas) || antennas.length !== 2
|
|
1370
|
+
|| !antennas.every((n) => Number.isFinite(n))) {
|
|
1371
|
+
throw new TypeError(
|
|
1372
|
+
'gotoTarget: antennas must be [rightRad, leftRad] (2 finite numbers); '
|
|
1373
|
+
+ `got ${Array.isArray(antennas) ? `Array(${antennas.length})` : typeof antennas}`
|
|
1374
|
+
);
|
|
1375
|
+
}
|
|
1376
|
+
cmd.antennas = antennas;
|
|
1377
|
+
}
|
|
1378
|
+
if (body_yaw !== undefined) {
|
|
1379
|
+
if (!Number.isFinite(body_yaw)) {
|
|
1380
|
+
throw new TypeError(
|
|
1381
|
+
`gotoTarget: body_yaw must be a finite number (radians); got ${body_yaw}`
|
|
1382
|
+
);
|
|
1383
|
+
}
|
|
1384
|
+
cmd.body_yaw = body_yaw;
|
|
1385
|
+
}
|
|
1386
|
+
if (!Number.isFinite(duration) || duration <= 0) {
|
|
1387
|
+
throw new TypeError(
|
|
1388
|
+
`gotoTarget: duration must be a positive finite number (seconds); got ${duration}`
|
|
1389
|
+
);
|
|
1390
|
+
}
|
|
1391
|
+
cmd.duration = duration;
|
|
1392
|
+
return this._sendCommand(cmd);
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
/**
|
|
1396
|
+
* Set head orientation from roll/pitch/yaw in degrees.
|
|
1397
|
+
* Convenience wrapper over ``setTarget``.
|
|
1398
|
+
* @param {number} rollDeg @param {number} pitchDeg @param {number} yawDeg
|
|
1399
|
+
* @returns {boolean}
|
|
1400
|
+
*/
|
|
1401
|
+
setHeadRpyDeg(rollDeg, pitchDeg, yawDeg) {
|
|
1402
|
+
return this.setTarget({ head: rpyToMatrix(rollDeg, pitchDeg, yawDeg).flat() });
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
/**
|
|
1406
|
+
* Set antenna positions from degrees.
|
|
1407
|
+
* Convenience wrapper over ``setTarget``.
|
|
1408
|
+
* @param {number} rightDeg @param {number} leftDeg
|
|
1409
|
+
* @returns {boolean}
|
|
1410
|
+
*/
|
|
1411
|
+
setAntennasDeg(rightDeg, leftDeg) {
|
|
1412
|
+
return this.setTarget({ antennas: [degToRad(rightDeg), degToRad(leftDeg)] });
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
/**
|
|
1416
|
+
* Set body yaw from degrees.
|
|
1417
|
+
* Convenience wrapper over ``setTarget``.
|
|
1418
|
+
* @param {number} yawDeg
|
|
1419
|
+
* @returns {boolean}
|
|
1420
|
+
*/
|
|
1421
|
+
setBodyYawDeg(yawDeg) {
|
|
1422
|
+
return this.setTarget({ body_yaw: degToRad(yawDeg) });
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
/**
|
|
1426
|
+
* Play a sound file on the robot.
|
|
1427
|
+
* @param {string} file — filename available on the robot (e.g. "wake_up.wav")
|
|
1428
|
+
* @returns {boolean}
|
|
1429
|
+
*/
|
|
1430
|
+
playSound(file) {
|
|
1431
|
+
return this._sendCommand({ type: "play_sound", file });
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
/**
|
|
1435
|
+
* Set the motor control mode.
|
|
1436
|
+
*
|
|
1437
|
+
* @param {"enabled"|"disabled"|"gravity_compensation"} mode
|
|
1438
|
+
* - "enabled" torque on, position-controlled.
|
|
1439
|
+
* - "disabled" torque off; the robot is backdrivable
|
|
1440
|
+
* and will not hold any pose.
|
|
1441
|
+
* - "gravity_compensation" torque on in current-control mode;
|
|
1442
|
+
* motors actively cancel gravity so the
|
|
1443
|
+
* robot is easy to move by hand.
|
|
1444
|
+
* @returns {boolean} false if the data channel is not open.
|
|
1445
|
+
*/
|
|
1446
|
+
setMotorMode(mode) {
|
|
1447
|
+
return this._sendCommand({ type: "set_motor_mode", mode });
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
/**
|
|
1451
|
+
* Toggle torque on/off, optionally per-motor.
|
|
1452
|
+
*
|
|
1453
|
+
* @param {boolean} on
|
|
1454
|
+
* @param {string[]} [ids] motor names (e.g. ["left_antenna"]). When
|
|
1455
|
+
* omitted, applies globally — equivalent to setMotorMode("enabled"
|
|
1456
|
+
* | "disabled").
|
|
1457
|
+
* @returns {boolean} false if the data channel is not open.
|
|
1458
|
+
*/
|
|
1459
|
+
setMotorTorque(on, ids = null) {
|
|
1460
|
+
return this._sendCommand({ type: "set_torque", on, ids });
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
/**
|
|
1464
|
+
* Play the wake-up animation (full head/antennas trajectory on the
|
|
1465
|
+
* robot, ~1-3 s depending on the starting head pose) and resolve
|
|
1466
|
+
* when the daemon reports the trajectory player has actually
|
|
1467
|
+
* finished.
|
|
1468
|
+
*
|
|
1469
|
+
* This helper sends a ``set_motor_mode: "enabled"`` command *before*
|
|
1470
|
+
* the ``wake_up`` command so the animation actually moves the motors.
|
|
1471
|
+
* The robot's ``wake_up`` handler does not touch motor mode itself;
|
|
1472
|
+
* if torque is off when the trajectory runs, the commanded positions
|
|
1473
|
+
* are silently ignored and the robot stays limp. Both commands
|
|
1474
|
+
* travel over the same data channel so ordering at the backend is
|
|
1475
|
+
* preserved.
|
|
1476
|
+
*
|
|
1477
|
+
* The returned promise resolves on the daemon's
|
|
1478
|
+
* ``{command: "wake_up", completed: true}`` response (sent after
|
|
1479
|
+
* the trajectory player is fully done, not just when the command
|
|
1480
|
+
* is enqueued). Lets a UI overlay (e.g. the host's "Wake-up" step)
|
|
1481
|
+
* stay up for exactly the right duration, and lets callers chain
|
|
1482
|
+
* setup that depends on the head being in the awake pose without
|
|
1483
|
+
* racing the trajectory.
|
|
1484
|
+
*
|
|
1485
|
+
* Semantics match the REST endpoint ``POST /api/move/play/wake_up``
|
|
1486
|
+
* plus the LAN convention of enabling motors before playing motion
|
|
1487
|
+
* trajectories.
|
|
1488
|
+
*
|
|
1489
|
+
* @param {object} [options]
|
|
1490
|
+
* @param {number} [options.timeoutMs=8000] hard upper bound; the
|
|
1491
|
+
* promise rejects with a TimeoutError-shaped Error if the daemon
|
|
1492
|
+
* stops responding (e.g. data channel went down mid-animation
|
|
1493
|
+
* without firing close events).
|
|
1494
|
+
* @returns {Promise<void>}
|
|
1495
|
+
*/
|
|
1496
|
+
wakeUp({ timeoutMs = 8000 } = {}) {
|
|
1497
|
+
this._sendCommand({ type: "set_motor_mode", mode: "enabled" });
|
|
1498
|
+
return this._sendCommandAwaitCompletion("wake_up", timeoutMs);
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
/**
|
|
1502
|
+
* Play the goto-sleep animation and resolve when the daemon reports
|
|
1503
|
+
* the trajectory player has finished. See ``wakeUp`` for the
|
|
1504
|
+
* completion-signal rationale.
|
|
1505
|
+
*
|
|
1506
|
+
* Does NOT touch motor mode: the daemon's ``goto_sleep`` handler
|
|
1507
|
+
* manages the transition out of torque on its own (motors must stay
|
|
1508
|
+
* powered during the trajectory to move into the sleep pose, then
|
|
1509
|
+
* are typically disabled by the daemon once the pose is reached).
|
|
1510
|
+
*
|
|
1511
|
+
* The awaitable form lets callers chain ``setMotorMode('disabled')``
|
|
1512
|
+
* AFTER the trajectory lands instead of racing it, which previously
|
|
1513
|
+
* caused the head to drop mid-animation when consumers tore down
|
|
1514
|
+
* too eagerly.
|
|
1515
|
+
*
|
|
1516
|
+
* Semantics match ``POST /api/move/play/goto_sleep`` and the
|
|
1517
|
+
* ``"goto_sleep"`` WebRTC command.
|
|
1518
|
+
*
|
|
1519
|
+
* @param {object} [options]
|
|
1520
|
+
* @param {number} [options.timeoutMs=8000]
|
|
1521
|
+
* @returns {Promise<void>}
|
|
1522
|
+
*/
|
|
1523
|
+
gotoSleep({ timeoutMs = 8000 } = {}) {
|
|
1524
|
+
return this._sendCommandAwaitCompletion("goto_sleep", timeoutMs);
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
/**
|
|
1528
|
+
* Internal: send a motion command and resolve when the daemon's
|
|
1529
|
+
* matching ``{command, completed: true}`` response lands.
|
|
1530
|
+
*
|
|
1531
|
+
* Pushes one entry onto ``_pendingMotionCompletions[command]``; the
|
|
1532
|
+
* data-channel reader (``_handleRobotMessage``) shifts the oldest
|
|
1533
|
+
* entry off the queue when a response arrives, which preserves the
|
|
1534
|
+
* FIFO matching that the daemon's serialised dispatcher relies on.
|
|
1535
|
+
*
|
|
1536
|
+
* Rejects immediately if the data channel is not open; the underlying
|
|
1537
|
+
* ``_sendCommand`` returns false in that case and we never enqueue an
|
|
1538
|
+
* awaiter that the daemon could never reach.
|
|
1539
|
+
*
|
|
1540
|
+
* @param {"wake_up"|"goto_sleep"} command
|
|
1541
|
+
* @param {number} timeoutMs
|
|
1542
|
+
* @returns {Promise<void>}
|
|
1543
|
+
*/
|
|
1544
|
+
_sendCommandAwaitCompletion(command, timeoutMs) {
|
|
1545
|
+
if (!this._sendCommand({ type: command })) {
|
|
1546
|
+
return Promise.reject(new Error(`${command}: data channel not open`));
|
|
1547
|
+
}
|
|
1548
|
+
return new Promise((resolve, reject) => {
|
|
1549
|
+
const entry = {
|
|
1550
|
+
resolve,
|
|
1551
|
+
reject,
|
|
1552
|
+
timer: setTimeout(() => {
|
|
1553
|
+
const queue = this._pendingMotionCompletions[command];
|
|
1554
|
+
const idx = queue.indexOf(entry);
|
|
1555
|
+
if (idx !== -1) queue.splice(idx, 1);
|
|
1556
|
+
reject(new Error(`${command} timed out after ${timeoutMs}ms`));
|
|
1557
|
+
}, timeoutMs),
|
|
1558
|
+
};
|
|
1559
|
+
this._pendingMotionCompletions[command].push(entry);
|
|
1560
|
+
});
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
/**
|
|
1564
|
+
* Internal: drain every pending motion-completion resolver with the
|
|
1565
|
+
* given error. Called by ``stopSession()`` and ``disconnect()`` so a
|
|
1566
|
+
* teardown that interrupts an in-flight ``gotoSleep`` does not leave
|
|
1567
|
+
* the caller awaiting forever.
|
|
1568
|
+
*/
|
|
1569
|
+
_rejectPendingMotionCompletions(error) {
|
|
1570
|
+
for (const command of Object.keys(this._pendingMotionCompletions)) {
|
|
1571
|
+
const queue = this._pendingMotionCompletions[command];
|
|
1572
|
+
while (queue.length) {
|
|
1573
|
+
const entry = queue.shift();
|
|
1574
|
+
clearTimeout(entry.timer);
|
|
1575
|
+
entry.reject(error);
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
/**
|
|
1581
|
+
* Whether the robot's motors are currently powered (the "awake" state).
|
|
1582
|
+
*
|
|
1583
|
+
* Reads ``motor_mode`` from the last state event. Both ``"enabled"``
|
|
1584
|
+
* and ``"gravity_compensation"`` count as awake: in gravity-comp the
|
|
1585
|
+
* motors are actively holding the arm against gravity, so the robot
|
|
1586
|
+
* is *not* limp and playing wake_up on top would fight the user.
|
|
1587
|
+
* Only ``"disabled"`` (true sleep) is considered not-awake.
|
|
1588
|
+
*
|
|
1589
|
+
* Returns ``false`` before the first state event arrives (typical
|
|
1590
|
+
* right after ``startSession()``). Use ``ensureAwake()`` if you want
|
|
1591
|
+
* to wait for the first state before deciding.
|
|
1592
|
+
*
|
|
1593
|
+
* @returns {boolean}
|
|
1594
|
+
*/
|
|
1595
|
+
isAwake() {
|
|
1596
|
+
const mode = this._robotState?.motor_mode;
|
|
1597
|
+
return mode === "enabled" || mode === "gravity_compensation";
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* Wake the robot up if it is currently asleep, otherwise no-op.
|
|
1602
|
+
*
|
|
1603
|
+
* Intended as the first line of any app after ``startSession()``
|
|
1604
|
+
* resolves — robots are often left in the sleep pose (torque off,
|
|
1605
|
+
* head resting on the base) and commanded positions are silently
|
|
1606
|
+
* ignored in that state.
|
|
1607
|
+
*
|
|
1608
|
+
* If no state event has arrived yet, waits up to ``timeoutMs`` for
|
|
1609
|
+
* one before deciding. If still no state, falls back to sending
|
|
1610
|
+
* ``wakeUp()`` (safe: the daemon's wake_up handler is idempotent
|
|
1611
|
+
* at the motion level — it moves to the awake pose from wherever
|
|
1612
|
+
* the head currently is).
|
|
1613
|
+
*
|
|
1614
|
+
* @param {number} [timeoutMs=1000] how long to wait for the first
|
|
1615
|
+
* state event before falling through to wakeUp().
|
|
1616
|
+
* @returns {Promise<boolean>} true if the robot is awake afterwards.
|
|
1617
|
+
*/
|
|
1618
|
+
async ensureAwake(timeoutMs = 1000) {
|
|
1619
|
+
if (this._robotState?.motor_mode === undefined) {
|
|
1620
|
+
await new Promise((resolve) => {
|
|
1621
|
+
const done = () => {
|
|
1622
|
+
this.removeEventListener('state', done);
|
|
1623
|
+
clearTimeout(timer);
|
|
1624
|
+
resolve();
|
|
1625
|
+
};
|
|
1626
|
+
const timer = setTimeout(done, timeoutMs);
|
|
1627
|
+
this.addEventListener('state', done);
|
|
1628
|
+
this.requestState();
|
|
1629
|
+
});
|
|
1630
|
+
}
|
|
1631
|
+
if (this.isAwake()) return true;
|
|
1632
|
+
// wakeUp() now returns a Promise. Fire-and-forget here — we
|
|
1633
|
+
// intentionally do not await the trajectory completion, the
|
|
1634
|
+
// caller of ensureAwake() decides whether to block on the
|
|
1635
|
+
// animation. Catch the rejection so a teardown that interrupts
|
|
1636
|
+
// the wake doesn't surface an unhandledrejection event from
|
|
1637
|
+
// this internal helper.
|
|
1638
|
+
this.wakeUp().catch(() => { /* swallow: caller may have torn down */ });
|
|
1639
|
+
return true;
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
/**
|
|
1643
|
+
* Request the daemon version.
|
|
1644
|
+
* Resolves with the version string (or null if unavailable).
|
|
1645
|
+
* @returns {Promise<string|null>}
|
|
1646
|
+
*/
|
|
1647
|
+
getVersion() {
|
|
1648
|
+
return new Promise((resolve, reject) => {
|
|
1649
|
+
if (!this._dc || this._dc.readyState !== 'open') {
|
|
1650
|
+
reject(new Error('Data channel not open'));
|
|
1651
|
+
return;
|
|
1652
|
+
}
|
|
1653
|
+
if (this._versionResolve) {
|
|
1654
|
+
this._versionResolve(null);
|
|
1655
|
+
}
|
|
1656
|
+
this._versionResolve = resolve;
|
|
1657
|
+
this._sendCommand({ type: "get_version" });
|
|
1658
|
+
});
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
/**
|
|
1662
|
+
* Request the robot's unique hardware ID — the Pollen audio device's
|
|
1663
|
+
* USB serial. Same value across Lite and Wireless variants, stable
|
|
1664
|
+
* across reboots and OS reinstalls. Useful for fleet management,
|
|
1665
|
+
* per-robot calibration cache keys, or identifying which physical
|
|
1666
|
+
* robot a session is bound to.
|
|
1667
|
+
* Resolves with the hardware ID string (or null if no robot is
|
|
1668
|
+
* attached, e.g. the daemon is running on a developer machine).
|
|
1669
|
+
* @returns {Promise<string|null>}
|
|
1670
|
+
*/
|
|
1671
|
+
getHardwareId() {
|
|
1672
|
+
return new Promise((resolve, reject) => {
|
|
1673
|
+
if (!this._dc || this._dc.readyState !== 'open') {
|
|
1674
|
+
reject(new Error('Data channel not open'));
|
|
1675
|
+
return;
|
|
1676
|
+
}
|
|
1677
|
+
if (this._hardwareIdResolve) {
|
|
1678
|
+
this._hardwareIdResolve(null);
|
|
1679
|
+
}
|
|
1680
|
+
this._hardwareIdResolve = resolve;
|
|
1681
|
+
this._sendCommand({ type: "get_hardware_id" });
|
|
1682
|
+
});
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
/**
|
|
1686
|
+
* Query the current speaker volume (0-100).
|
|
1687
|
+
* Resolves with null if volume control is unavailable (platform unsupported
|
|
1688
|
+
* or audio stack down).
|
|
1689
|
+
* @returns {Promise<number|null>}
|
|
1690
|
+
*/
|
|
1691
|
+
getVolume() {
|
|
1692
|
+
return this._volumeRoundtrip({ type: "get_volume" }, "_volumeResolve");
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
/**
|
|
1696
|
+
* Set the speaker volume (0-100). Persists for the next connection
|
|
1697
|
+
* (same semantics as the REST /api/volume/set endpoint).
|
|
1698
|
+
* Resolves with the applied volume, or null on failure.
|
|
1699
|
+
* @param {number} volume 0-100
|
|
1700
|
+
* @returns {Promise<number|null>}
|
|
1701
|
+
*/
|
|
1702
|
+
setVolume(volume) {
|
|
1703
|
+
return this._volumeRoundtrip(
|
|
1704
|
+
{ type: "set_volume", volume: clampVolume(volume) },
|
|
1705
|
+
"_volumeResolve",
|
|
1706
|
+
);
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
/**
|
|
1710
|
+
* Query the current microphone input volume (0-100).
|
|
1711
|
+
* @returns {Promise<number|null>}
|
|
1712
|
+
*/
|
|
1713
|
+
getMicrophoneVolume() {
|
|
1714
|
+
return this._volumeRoundtrip(
|
|
1715
|
+
{ type: "get_microphone_volume" },
|
|
1716
|
+
"_micVolumeResolve",
|
|
1717
|
+
);
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
/**
|
|
1721
|
+
* Set the microphone input volume (0-100). Persists across sessions.
|
|
1722
|
+
* @param {number} volume 0-100
|
|
1723
|
+
* @returns {Promise<number|null>}
|
|
1724
|
+
*/
|
|
1725
|
+
setMicrophoneVolume(volume) {
|
|
1726
|
+
return this._volumeRoundtrip(
|
|
1727
|
+
{ type: "set_microphone_volume", volume: clampVolume(volume) },
|
|
1728
|
+
"_micVolumeResolve",
|
|
1729
|
+
);
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
/**
|
|
1733
|
+
* Internal: send a volume command and await the single-slot response.
|
|
1734
|
+
* The slot name selects which resolver (speaker vs mic) owns the
|
|
1735
|
+
* pending request so the two can be in-flight concurrently without
|
|
1736
|
+
* collision.
|
|
1737
|
+
*/
|
|
1738
|
+
_volumeRoundtrip(command, slot) {
|
|
1739
|
+
return new Promise((resolve, reject) => {
|
|
1740
|
+
if (!this._dc || this._dc.readyState !== 'open') {
|
|
1741
|
+
reject(new Error('Data channel not open'));
|
|
1742
|
+
return;
|
|
1743
|
+
}
|
|
1744
|
+
// If a previous request on the same slot is still pending,
|
|
1745
|
+
// resolve it to null so its caller doesn't hang forever.
|
|
1746
|
+
if (this[slot]) this[slot](null);
|
|
1747
|
+
this[slot] = resolve;
|
|
1748
|
+
this._sendCommand(command);
|
|
1749
|
+
});
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
/**
|
|
1753
|
+
* Send an arbitrary JSON command over the data channel.
|
|
1754
|
+
* @param {object} data @returns {boolean}
|
|
1755
|
+
*/
|
|
1756
|
+
sendRaw(data) {
|
|
1757
|
+
return this._sendCommand(data);
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
/**
|
|
1761
|
+
* Subscribe to the daemon's `journalctl -u reachy-mini-daemon`
|
|
1762
|
+
* stream over the WebRTC data channel.
|
|
1763
|
+
*
|
|
1764
|
+
* One daemon-side subprocess is shared across all local subscribers:
|
|
1765
|
+
* the first call sends `subscribe_logs`, removing the last subscriber
|
|
1766
|
+
* sends `unsubscribe_logs`. Calling the returned `unsubscribe()`
|
|
1767
|
+
* twice is a no-op.
|
|
1768
|
+
*
|
|
1769
|
+
* @param {{
|
|
1770
|
+
* onLine: (entry: { timestamp: string, line: string }) => void,
|
|
1771
|
+
* onError?: (error: string) => void,
|
|
1772
|
+
* }} options
|
|
1773
|
+
* @returns {() => void} unsubscribe
|
|
1774
|
+
*/
|
|
1775
|
+
subscribeLogs({ onLine, onError } = {}) {
|
|
1776
|
+
if (typeof onLine !== 'function') {
|
|
1777
|
+
throw new TypeError('subscribeLogs: onLine callback is required');
|
|
1778
|
+
}
|
|
1779
|
+
const sub = { onLine, onError };
|
|
1780
|
+
const wasEmpty = this._logSubscribers.size === 0;
|
|
1781
|
+
this._logSubscribers.add(sub);
|
|
1782
|
+
if (wasEmpty) this._sendCommand({ type: 'subscribe_logs' });
|
|
1783
|
+
|
|
1784
|
+
let detached = false;
|
|
1785
|
+
return () => {
|
|
1786
|
+
if (detached) return;
|
|
1787
|
+
detached = true;
|
|
1788
|
+
this._logSubscribers.delete(sub);
|
|
1789
|
+
if (this._logSubscribers.size === 0) {
|
|
1790
|
+
this._sendCommand({ type: 'unsubscribe_logs' });
|
|
1791
|
+
}
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
/**
|
|
1796
|
+
* Request a state snapshot. The response arrives as a "state" event.
|
|
1797
|
+
* Called automatically every 500 ms while streaming.
|
|
1798
|
+
*
|
|
1799
|
+
* Safe to call at a higher rate if you need faster telemetry: e.g.
|
|
1800
|
+
* ``setInterval(() => robot.requestState(), 20)`` for ~50 Hz, or drive
|
|
1801
|
+
* it from a ``requestAnimationFrame`` loop for display-rate updates.
|
|
1802
|
+
* On LAN the daemon can sustain ~90-100 Hz round-trips over the
|
|
1803
|
+
* datachannel; over the internet expect the WebRTC path's RTT to
|
|
1804
|
+
* dominate. The built-in 500 ms poll keeps running in parallel — it
|
|
1805
|
+
* is harmless, as state responses are idempotent.
|
|
1806
|
+
*
|
|
1807
|
+
* @returns {boolean}
|
|
1808
|
+
*/
|
|
1809
|
+
requestState() {
|
|
1810
|
+
return this._sendCommand({ type: "get_state" });
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
// ─── Audio ───────────────────────────────────────────────────────────
|
|
1814
|
+
|
|
1815
|
+
/**
|
|
1816
|
+
* Mute/unmute the robot's audio playback (speaker) locally.
|
|
1817
|
+
* Audio is muted by default — browsers require a user gesture to unmute.
|
|
1818
|
+
* @param {boolean} muted
|
|
1819
|
+
*/
|
|
1820
|
+
setAudioMuted(muted) {
|
|
1821
|
+
this._audioMuted = muted;
|
|
1822
|
+
if (this._videoElement) this._videoElement.muted = muted;
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
/**
|
|
1826
|
+
* Mute/unmute your microphone. Only works if micSupported is true.
|
|
1827
|
+
* Mic is muted by default even after acquisition.
|
|
1828
|
+
* @param {boolean} muted
|
|
1829
|
+
*/
|
|
1830
|
+
setMicMuted(muted) {
|
|
1831
|
+
this._micMuted = muted;
|
|
1832
|
+
if (this._micStream) {
|
|
1833
|
+
this._micStream.getAudioTracks().forEach(t => { t.enabled = !muted; });
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
// ─── Video helper ────────────────────────────────────────────────────
|
|
1838
|
+
|
|
1839
|
+
/**
|
|
1840
|
+
* Bind a `<video>` element to this robot's stream.
|
|
1841
|
+
* Call before startSession(). Sets srcObject when the video track arrives,
|
|
1842
|
+
* applies audio mute state, and runs a latency monitor that snaps to the
|
|
1843
|
+
* live edge if the buffer grows > 0.5 s.
|
|
1844
|
+
*
|
|
1845
|
+
* @param {HTMLVideoElement} videoElement
|
|
1846
|
+
* @returns {() => void} cleanup function — call to detach video and stop monitoring
|
|
1847
|
+
*/
|
|
1848
|
+
attachVideo(videoElement) {
|
|
1849
|
+
this._videoElement = videoElement;
|
|
1850
|
+
videoElement.muted = this._audioMuted;
|
|
1851
|
+
|
|
1852
|
+
const onVideoTrack = (e) => {
|
|
1853
|
+
videoElement.srcObject = e.detail.stream;
|
|
1854
|
+
videoElement.playsInline = true;
|
|
1855
|
+
if ('requestVideoFrameCallback' in videoElement) {
|
|
1856
|
+
this._startLatencyMonitor(videoElement);
|
|
1857
|
+
}
|
|
1858
|
+
};
|
|
1859
|
+
|
|
1860
|
+
const onSessionStopped = () => { videoElement.srcObject = null; };
|
|
1861
|
+
|
|
1862
|
+
this.addEventListener('videoTrack', onVideoTrack);
|
|
1863
|
+
this.addEventListener('sessionStopped', onSessionStopped);
|
|
1864
|
+
|
|
1865
|
+
return () => {
|
|
1866
|
+
this.removeEventListener('videoTrack', onVideoTrack);
|
|
1867
|
+
this.removeEventListener('sessionStopped', onSessionStopped);
|
|
1868
|
+
if (this._latencyMonitorId) { clearInterval(this._latencyMonitorId); this._latencyMonitorId = null; }
|
|
1869
|
+
videoElement.srcObject = null;
|
|
1870
|
+
this._videoElement = null;
|
|
1871
|
+
};
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
// ─── Daemon-side recorded-move playback ──────────────────────────────
|
|
1875
|
+
//
|
|
1876
|
+
// These methods talk to the daemon's `feature/daemon-side-move-upload`
|
|
1877
|
+
// protocol: motion (and optional audio) are uploaded once over the
|
|
1878
|
+
// data channel, then the daemon's play_move loop runs at the requested
|
|
1879
|
+
// frequency server-side — no per-frame WebRTC round-trip, smooth on
|
|
1880
|
+
// wireless robots. Audio, when present, plays on the same daemon-side
|
|
1881
|
+
// GStreamer pipeline so motion and sound share a single clock.
|
|
1882
|
+
//
|
|
1883
|
+
// For record-time flows that need the SAME audio pipeline at capture
|
|
1884
|
+
// and replay (so pipeline latency cancels), use uploadAudio +
|
|
1885
|
+
// playUploadedAudio to play audio standalone with a sync anchor.
|
|
1886
|
+
|
|
1887
|
+
/**
|
|
1888
|
+
* Upload a recorded move (and optionally its audio) and play it on
|
|
1889
|
+
* the daemon's local clock. Resolves when playback ends with the
|
|
1890
|
+
* daemon's final broadcast — `{finished: true}`, `{cancelled: true}`,
|
|
1891
|
+
* or `{error: string}`.
|
|
1892
|
+
*
|
|
1893
|
+
* `audioLeadMs` shifts audio relative to motion:
|
|
1894
|
+
* - Positive: audio fires N ms BEFORE motion (compensates motor pickup).
|
|
1895
|
+
* - Negative: motion fires N ms BEFORE audio (compensates pipeline warmup).
|
|
1896
|
+
* - Default `-100` is the empirically-measured system-wide constant
|
|
1897
|
+
* (combined motor + GStreamer playbin warmup); tune per setup only
|
|
1898
|
+
* if you've measured a different value.
|
|
1899
|
+
*
|
|
1900
|
+
* @param {{ time: number[], set_target_data: object[] }} motion
|
|
1901
|
+
* @param {object} [opts]
|
|
1902
|
+
* @param {Blob} [opts.audioBlob] - canonical 16 kHz mono PCM WAV
|
|
1903
|
+
* @param {number} [opts.audioLeadMs=-100]
|
|
1904
|
+
* @param {string} [opts.description="move"]
|
|
1905
|
+
* @param {"gzip+base64"|"json"} [opts.encoding="gzip+base64"]
|
|
1906
|
+
* @param {number} [opts.playFrequency=100]
|
|
1907
|
+
* @param {number} [opts.initialGotoDuration=0]
|
|
1908
|
+
* @param {number} [opts.startTimeoutMs=8000]
|
|
1909
|
+
* @param {(p: { phase: string, sent?: number, total?: number,
|
|
1910
|
+
* bytes?: number, encoding?: string,
|
|
1911
|
+
* duration_s?: number }) => void} [opts.onProgress]
|
|
1912
|
+
* @param {(s: { duration_s: number, has_audio: boolean }) => void} [opts.onStarted]
|
|
1913
|
+
* @returns {Promise<{ finished?: boolean, cancelled?: boolean,
|
|
1914
|
+
* error?: string, has_audio?: boolean }>}
|
|
1915
|
+
*/
|
|
1916
|
+
async playMove(motion, {
|
|
1917
|
+
audioBlob = null,
|
|
1918
|
+
audioLeadMs = -100,
|
|
1919
|
+
description = "move",
|
|
1920
|
+
encoding = "gzip+base64",
|
|
1921
|
+
playFrequency = 100,
|
|
1922
|
+
initialGotoDuration = 0,
|
|
1923
|
+
startTimeoutMs = 8000,
|
|
1924
|
+
onProgress = () => {},
|
|
1925
|
+
onStarted = () => {},
|
|
1926
|
+
} = {}) {
|
|
1927
|
+
if (!this._dc || this._dc.readyState !== "open") {
|
|
1928
|
+
throw new Error("data channel not open");
|
|
1929
|
+
}
|
|
1930
|
+
if (!motion?.time?.length || !motion?.set_target_data?.length) {
|
|
1931
|
+
throw new Error("playMove: motion must have time + set_target_data");
|
|
1932
|
+
}
|
|
1933
|
+
const uploadId = makeUploadId();
|
|
1934
|
+
// Publish the id so `cancelMove()` without args targets this
|
|
1935
|
+
// run. Cleared in finally to avoid stale cancels biting the
|
|
1936
|
+
// next run.
|
|
1937
|
+
this._activeMoveUploadId = uploadId;
|
|
1938
|
+
|
|
1939
|
+
// Encode the move payload. gzip+base64 typically compresses
|
|
1940
|
+
// recorded-move JSON ~3× thanks to repeated float patterns;
|
|
1941
|
+
// falls back to plain JSON if CompressionStream is missing.
|
|
1942
|
+
const moveDict = {
|
|
1943
|
+
description,
|
|
1944
|
+
time: motion.time,
|
|
1945
|
+
set_target_data: motion.set_target_data,
|
|
1946
|
+
};
|
|
1947
|
+
const jsonStr = JSON.stringify(moveDict);
|
|
1948
|
+
let payload;
|
|
1949
|
+
let effectiveEncoding;
|
|
1950
|
+
if (encoding === "gzip+base64" && hasCompressionStream()) {
|
|
1951
|
+
payload = await gzipBase64(jsonStr);
|
|
1952
|
+
effectiveEncoding = "gzip+base64";
|
|
1953
|
+
} else {
|
|
1954
|
+
payload = jsonStr;
|
|
1955
|
+
effectiveEncoding = "json";
|
|
1956
|
+
}
|
|
1957
|
+
const totalChunks = Math.ceil(payload.length / UPLOAD_CHUNK_SIZE) || 1;
|
|
1958
|
+
|
|
1959
|
+
onProgress({
|
|
1960
|
+
phase: "starting",
|
|
1961
|
+
sent: 0,
|
|
1962
|
+
total: totalChunks,
|
|
1963
|
+
bytes: payload.length,
|
|
1964
|
+
encoding: effectiveEncoding,
|
|
1965
|
+
});
|
|
1966
|
+
|
|
1967
|
+
// 1. Open the move slot.
|
|
1968
|
+
this._sendCommand({
|
|
1969
|
+
type: "upload_move_start",
|
|
1970
|
+
upload_id: uploadId,
|
|
1971
|
+
total_chunks: totalChunks,
|
|
1972
|
+
description,
|
|
1973
|
+
encoding: effectiveEncoding,
|
|
1974
|
+
});
|
|
1975
|
+
// 2. Pipeline motion chunks. No per-chunk acks; pace on
|
|
1976
|
+
// bufferedAmount so a long song doesn't blow up the channel.
|
|
1977
|
+
for (let i = 0; i < totalChunks; i++) {
|
|
1978
|
+
if (this._dc.bufferedAmount > UPLOAD_BUFFERED_HIGH_WATER) {
|
|
1979
|
+
await this._awaitDataChannelDrain();
|
|
1980
|
+
}
|
|
1981
|
+
const start = i * UPLOAD_CHUNK_SIZE;
|
|
1982
|
+
this._sendCommand({
|
|
1983
|
+
type: "upload_move_chunk",
|
|
1984
|
+
upload_id: uploadId,
|
|
1985
|
+
chunk_index: i,
|
|
1986
|
+
chunk: payload.slice(start, start + UPLOAD_CHUNK_SIZE),
|
|
1987
|
+
});
|
|
1988
|
+
onProgress({ phase: "upload", sent: i + 1, total: totalChunks });
|
|
1989
|
+
}
|
|
1990
|
+
// 3. Close the slot. Daemon parses synchronously.
|
|
1991
|
+
this._sendCommand({ type: "upload_move_finish", upload_id: uploadId });
|
|
1992
|
+
onProgress({ phase: "uploaded", sent: totalChunks, total: totalChunks });
|
|
1993
|
+
|
|
1994
|
+
// 3b. Optional audio: pipelined under the SAME upload_id so the
|
|
1995
|
+
// daemon pairs it with the move at play time. Raw WAV bytes
|
|
1996
|
+
// are base64-encoded (no gzip; PCM compresses poorly).
|
|
1997
|
+
if (audioBlob) {
|
|
1998
|
+
const rawBytes = new Uint8Array(await audioBlob.arrayBuffer());
|
|
1999
|
+
const audioB64 = bytesToBase64(rawBytes);
|
|
2000
|
+
const audioTotal = Math.ceil(audioB64.length / UPLOAD_CHUNK_SIZE) || 1;
|
|
2001
|
+
onProgress({
|
|
2002
|
+
phase: "audio-starting",
|
|
2003
|
+
sent: 0,
|
|
2004
|
+
total: audioTotal,
|
|
2005
|
+
bytes: audioB64.length,
|
|
2006
|
+
});
|
|
2007
|
+
this._sendCommand({
|
|
2008
|
+
type: "upload_audio_start",
|
|
2009
|
+
upload_id: uploadId,
|
|
2010
|
+
total_chunks: audioTotal,
|
|
2011
|
+
encoding: "wav-base64",
|
|
2012
|
+
description,
|
|
2013
|
+
});
|
|
2014
|
+
for (let i = 0; i < audioTotal; i++) {
|
|
2015
|
+
if (this._dc.bufferedAmount > UPLOAD_BUFFERED_HIGH_WATER) {
|
|
2016
|
+
await this._awaitDataChannelDrain();
|
|
2017
|
+
}
|
|
2018
|
+
const start = i * UPLOAD_CHUNK_SIZE;
|
|
2019
|
+
this._sendCommand({
|
|
2020
|
+
type: "upload_audio_chunk",
|
|
2021
|
+
upload_id: uploadId,
|
|
2022
|
+
chunk_index: i,
|
|
2023
|
+
chunk: audioB64.slice(start, start + UPLOAD_CHUNK_SIZE),
|
|
2024
|
+
});
|
|
2025
|
+
onProgress({ phase: "audio-upload", sent: i + 1, total: audioTotal });
|
|
2026
|
+
}
|
|
2027
|
+
this._sendCommand({ type: "upload_audio_finish", upload_id: uploadId });
|
|
2028
|
+
onProgress({ phase: "audio-uploaded", sent: audioTotal, total: audioTotal });
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
// 4. Trigger playback; await the daemon's "started" broadcast.
|
|
2032
|
+
this._sendCommand({
|
|
2033
|
+
type: "play_uploaded_move",
|
|
2034
|
+
upload_id: uploadId,
|
|
2035
|
+
play_frequency: playFrequency,
|
|
2036
|
+
initial_goto_duration: initialGotoDuration,
|
|
2037
|
+
audio_lead_ms: audioLeadMs,
|
|
2038
|
+
});
|
|
2039
|
+
let startedAck;
|
|
2040
|
+
try {
|
|
2041
|
+
startedAck = await this._waitForBroadcast(
|
|
2042
|
+
(m) =>
|
|
2043
|
+
m?.type === "play_uploaded_move"
|
|
2044
|
+
&& m?.upload_id === uploadId
|
|
2045
|
+
&& (m.started === true || typeof m.error === "string"),
|
|
2046
|
+
{ timeoutMs: startTimeoutMs, debugLabel: "play_uploaded_move started" },
|
|
2047
|
+
);
|
|
2048
|
+
} catch (e) {
|
|
2049
|
+
throw new Error(
|
|
2050
|
+
"Daemon did not respond to play_uploaded_move "
|
|
2051
|
+
+ "(requires the reachy_mini daemon with feature/daemon-side-move-upload). "
|
|
2052
|
+
+ `Underlying: ${e.message}`,
|
|
2053
|
+
);
|
|
2054
|
+
}
|
|
2055
|
+
if (typeof startedAck.error === "string") {
|
|
2056
|
+
throw new Error(`play_uploaded_move: ${startedAck.error}`);
|
|
2057
|
+
}
|
|
2058
|
+
try {
|
|
2059
|
+
onStarted({
|
|
2060
|
+
duration_s: startedAck.duration_s,
|
|
2061
|
+
has_audio: startedAck.has_audio === true,
|
|
2062
|
+
});
|
|
2063
|
+
} catch (e) {
|
|
2064
|
+
// onStarted is user code — never let it abort playback.
|
|
2065
|
+
console.warn("playMove.onStarted threw:", e);
|
|
2066
|
+
}
|
|
2067
|
+
onProgress({ phase: "playing", duration_s: startedAck.duration_s });
|
|
2068
|
+
|
|
2069
|
+
// 5. Wait for the final broadcast.
|
|
2070
|
+
const final = await this._waitForBroadcast(
|
|
2071
|
+
(m) =>
|
|
2072
|
+
m?.type === "play_uploaded_move"
|
|
2073
|
+
&& m?.upload_id === uploadId
|
|
2074
|
+
&& (m.finished === true
|
|
2075
|
+
|| m.cancelled === true
|
|
2076
|
+
|| typeof m.error === "string"),
|
|
2077
|
+
{
|
|
2078
|
+
timeoutMs: (startedAck.duration_s + 30) * 1000,
|
|
2079
|
+
debugLabel: "play_uploaded_move final",
|
|
2080
|
+
},
|
|
2081
|
+
);
|
|
2082
|
+
// Release the "current move id" pointer so the next no-arg
|
|
2083
|
+
// cancelMove() doesn't target an already-ended run. Guarded
|
|
2084
|
+
// against the unlikely case that a concurrent playMove has
|
|
2085
|
+
// already overwritten it.
|
|
2086
|
+
if (this._activeMoveUploadId === uploadId) {
|
|
2087
|
+
this._activeMoveUploadId = null;
|
|
2088
|
+
}
|
|
2089
|
+
return final;
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
/**
|
|
2093
|
+
* Cancel an in-flight `playMove`. Fire-and-forget; the daemon
|
|
2094
|
+
* broadcasts the cancelled event which `playMove` resolves with.
|
|
2095
|
+
*
|
|
2096
|
+
* Pass `uploadId` explicitly to target a specific run; defaults to
|
|
2097
|
+
* the most recent in-flight `playMove`. The daemon now scopes
|
|
2098
|
+
* cancels by upload_id so two back-to-back plays can't cross-cancel
|
|
2099
|
+
* each other — a cancel with no live target is a no-op.
|
|
2100
|
+
*
|
|
2101
|
+
* @param {string} [uploadId] - optional; defaults to the active playMove id
|
|
2102
|
+
* @returns {boolean} false if the data channel isn't open or no run to target
|
|
2103
|
+
*/
|
|
2104
|
+
cancelMove(uploadId = null) {
|
|
2105
|
+
const id = uploadId ?? this._activeMoveUploadId;
|
|
2106
|
+
if (!id) return false;
|
|
2107
|
+
return this._sendCommand({ type: "cancel_move", upload_id: id });
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
/**
|
|
2111
|
+
* Upload audio to the daemon as a standalone slot (no motion attached).
|
|
2112
|
+
* Used by recording flows that want the SAME audio pipeline at record
|
|
2113
|
+
* time and play time — pipeline latency cancels, so a single per-system
|
|
2114
|
+
* `audioLeadMs` is enough for sync.
|
|
2115
|
+
*
|
|
2116
|
+
* @param {Blob} audioBlob - canonical 16 kHz mono PCM WAV
|
|
2117
|
+
* @param {object} [opts]
|
|
2118
|
+
* @param {string} [opts.description="audio"]
|
|
2119
|
+
* @param {(p: { phase: string, sent?: number, total?: number,
|
|
2120
|
+
* bytes?: number }) => void} [opts.onProgress]
|
|
2121
|
+
* @returns {Promise<string>} uploadId — pair with playUploadedAudio
|
|
2122
|
+
*/
|
|
2123
|
+
async uploadAudio(audioBlob, { description = "audio", onProgress = () => {} } = {}) {
|
|
2124
|
+
if (!this._dc || this._dc.readyState !== "open") {
|
|
2125
|
+
throw new Error("data channel not open");
|
|
2126
|
+
}
|
|
2127
|
+
if (!(audioBlob instanceof Blob)) {
|
|
2128
|
+
throw new TypeError("uploadAudio: expected a Blob");
|
|
2129
|
+
}
|
|
2130
|
+
const uploadId = makeUploadId();
|
|
2131
|
+
const rawBytes = new Uint8Array(await audioBlob.arrayBuffer());
|
|
2132
|
+
const audioB64 = bytesToBase64(rawBytes);
|
|
2133
|
+
const total = Math.ceil(audioB64.length / UPLOAD_CHUNK_SIZE) || 1;
|
|
2134
|
+
onProgress({ phase: "audio-starting", sent: 0, total, bytes: audioB64.length });
|
|
2135
|
+
this._sendCommand({
|
|
2136
|
+
type: "upload_audio_start",
|
|
2137
|
+
upload_id: uploadId,
|
|
2138
|
+
total_chunks: total,
|
|
2139
|
+
encoding: "wav-base64",
|
|
2140
|
+
description,
|
|
2141
|
+
});
|
|
2142
|
+
for (let i = 0; i < total; i++) {
|
|
2143
|
+
if (this._dc.bufferedAmount > UPLOAD_BUFFERED_HIGH_WATER) {
|
|
2144
|
+
await this._awaitDataChannelDrain();
|
|
2145
|
+
}
|
|
2146
|
+
const start = i * UPLOAD_CHUNK_SIZE;
|
|
2147
|
+
this._sendCommand({
|
|
2148
|
+
type: "upload_audio_chunk",
|
|
2149
|
+
upload_id: uploadId,
|
|
2150
|
+
chunk_index: i,
|
|
2151
|
+
chunk: audioB64.slice(start, start + UPLOAD_CHUNK_SIZE),
|
|
2152
|
+
});
|
|
2153
|
+
onProgress({ phase: "audio-upload", sent: i + 1, total });
|
|
2154
|
+
}
|
|
2155
|
+
this._sendCommand({ type: "upload_audio_finish", upload_id: uploadId });
|
|
2156
|
+
onProgress({ phase: "audio-uploaded", sent: total, total });
|
|
2157
|
+
return uploadId;
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
/**
|
|
2161
|
+
* Trigger daemon-side playback of a previously-uploaded audio.
|
|
2162
|
+
* Resolves when the daemon broadcasts the "started" event — this is
|
|
2163
|
+
* the sync anchor callers use as t=0 for related capture.
|
|
2164
|
+
*
|
|
2165
|
+
* The daemon does NOT emit a finished event for standalone audio;
|
|
2166
|
+
* callers know the duration from the WAV header and send
|
|
2167
|
+
* `cancelAudio()` when they're done (e.g. recording stopped).
|
|
2168
|
+
*
|
|
2169
|
+
* @param {string} uploadId
|
|
2170
|
+
* @param {object} [opts]
|
|
2171
|
+
* @param {number} [opts.timeoutMs=8000]
|
|
2172
|
+
* @returns {Promise<{ started: true }>}
|
|
2173
|
+
*/
|
|
2174
|
+
async playUploadedAudio(uploadId, { timeoutMs = 8000 } = {}) {
|
|
2175
|
+
if (!this._dc || this._dc.readyState !== "open") {
|
|
2176
|
+
throw new Error("data channel not open");
|
|
2177
|
+
}
|
|
2178
|
+
const waiter = this._waitForBroadcast(
|
|
2179
|
+
(m) =>
|
|
2180
|
+
m?.type === "play_uploaded_audio"
|
|
2181
|
+
&& m?.upload_id === uploadId
|
|
2182
|
+
&& (m.started === true || typeof m.error === "string"),
|
|
2183
|
+
{ timeoutMs, debugLabel: "play_uploaded_audio started" },
|
|
2184
|
+
);
|
|
2185
|
+
this._sendCommand({ type: "play_uploaded_audio", upload_id: uploadId });
|
|
2186
|
+
const ack = await waiter;
|
|
2187
|
+
if (typeof ack.error === "string") throw new Error(ack.error);
|
|
2188
|
+
// Publish the id so a no-arg `cancelAudio()` targets this run.
|
|
2189
|
+
// The daemon has no "audio ended" event, so the id stays set
|
|
2190
|
+
// until either cancelAudio() is called or playUploadedAudio()
|
|
2191
|
+
// is called again with a different id.
|
|
2192
|
+
this._activeAudioUploadId = uploadId;
|
|
2193
|
+
return ack;
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
/**
|
|
2197
|
+
* Cancel an in-flight `playUploadedAudio`. Fire-and-forget.
|
|
2198
|
+
*
|
|
2199
|
+
* Pass `uploadId` explicitly to target a specific run; defaults
|
|
2200
|
+
* to the most recent `playUploadedAudio`. The daemon scopes
|
|
2201
|
+
* cancels by upload_id so a stale cancel won't kill the audio
|
|
2202
|
+
* attached to a concurrently-running `playMove`.
|
|
2203
|
+
*
|
|
2204
|
+
* @param {string} [uploadId] - optional; defaults to the active playUploadedAudio id
|
|
2205
|
+
* @returns {boolean} false if the data channel isn't open or no run to target
|
|
2206
|
+
*/
|
|
2207
|
+
cancelAudio(uploadId = null) {
|
|
2208
|
+
const id = uploadId ?? this._activeAudioUploadId;
|
|
2209
|
+
if (!id) return false;
|
|
2210
|
+
if (this._activeAudioUploadId === id) {
|
|
2211
|
+
this._activeAudioUploadId = null;
|
|
2212
|
+
}
|
|
2213
|
+
return this._sendCommand({ type: "cancel_audio", upload_id: id });
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
// ─── Private ─────────────────────────────────────────────────────────
|
|
2217
|
+
|
|
2218
|
+
_emit(name, detail) {
|
|
2219
|
+
this.dispatchEvent(new CustomEvent(name, { detail }));
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
/**
|
|
2223
|
+
* Register a one-shot waiter for a daemon broadcast event. Resolves
|
|
2224
|
+
* with the matching payload, rejects on `timeoutMs`. Used internally
|
|
2225
|
+
* by `playMove` / `playUploadedAudio`.
|
|
2226
|
+
*/
|
|
2227
|
+
_waitForBroadcast(predicate, { timeoutMs = 5000, debugLabel = "" } = {}) {
|
|
2228
|
+
return new Promise((resolve, reject) => {
|
|
2229
|
+
const slot = { predicate, resolve };
|
|
2230
|
+
slot.timer = setTimeout(() => {
|
|
2231
|
+
const i = this._broadcastWaiters.indexOf(slot);
|
|
2232
|
+
if (i !== -1) this._broadcastWaiters.splice(i, 1);
|
|
2233
|
+
reject(new Error(`broadcast timeout (${timeoutMs} ms): ${debugLabel}`));
|
|
2234
|
+
}, timeoutMs);
|
|
2235
|
+
this._broadcastWaiters.push(slot);
|
|
2236
|
+
});
|
|
2237
|
+
}
|
|
2238
|
+
|
|
2239
|
+
/**
|
|
2240
|
+
* Wait until `_dc.bufferedAmount` drops below the low watermark. Polls
|
|
2241
|
+
* at ~30 ms; browsers don't expose a uniform event for SCTP data channels
|
|
2242
|
+
* (`onbufferedamountlow` is patchy across engines).
|
|
2243
|
+
*/
|
|
2244
|
+
async _awaitDataChannelDrain() {
|
|
2245
|
+
while (this._dc && this._dc.bufferedAmount > UPLOAD_BUFFERED_LOW_WATER) {
|
|
2246
|
+
await new Promise((r) => setTimeout(r, 30));
|
|
2247
|
+
if (!this._dc || this._dc.readyState !== "open") {
|
|
2248
|
+
throw new Error("data channel closed mid-upload");
|
|
2249
|
+
}
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2253
|
+
async _sendToServer(message) {
|
|
2254
|
+
// Mirrors connect()'s guard — a missing token between connect and
|
|
2255
|
+
// send (e.g. logout mid-session) would otherwise produce
|
|
2256
|
+
// "Authorization: Bearer undefined", which central correctly 401s
|
|
2257
|
+
// but silently returns null to the caller, hiding the real cause.
|
|
2258
|
+
if (!this._token) throw new Error('No token — authenticate() first');
|
|
2259
|
+
try {
|
|
2260
|
+
// Token in Authorization header, not URL — same reasoning as
|
|
2261
|
+
// connect()'s SSE fetch: never put secrets in URLs.
|
|
2262
|
+
const res = await fetch(`${this._signalingUrl}/send`, {
|
|
2263
|
+
method: 'POST',
|
|
2264
|
+
headers: {
|
|
2265
|
+
'Content-Type': 'application/json',
|
|
2266
|
+
'Authorization': `Bearer ${this._token}`,
|
|
2267
|
+
},
|
|
2268
|
+
body: JSON.stringify(message),
|
|
2269
|
+
});
|
|
2270
|
+
if (!res.ok) {
|
|
2271
|
+
// Surface 4xx/5xx with the rejected message type. The
|
|
2272
|
+
// browser already logs "Failed to load resource: <status>"
|
|
2273
|
+
// but never says which call produced it, which makes
|
|
2274
|
+
// tardy `peer`/`endSession`/`setPeerStatus` races
|
|
2275
|
+
// (typical after a session has been torn down) hard to
|
|
2276
|
+
// diagnose. Returning null preserves the historical
|
|
2277
|
+
// contract for callers that only care about the success
|
|
2278
|
+
// path.
|
|
2279
|
+
let body = '';
|
|
2280
|
+
try { body = await res.text(); } catch { /* ignore */ }
|
|
2281
|
+
console.warn(
|
|
2282
|
+
`[reachy-mini] /send rejected (${res.status}) for type=${message?.type}; body=${body || '<empty>'}`,
|
|
2283
|
+
);
|
|
2284
|
+
return null;
|
|
2285
|
+
}
|
|
2286
|
+
return await res.json();
|
|
2287
|
+
} catch (e) {
|
|
2288
|
+
console.error('Send error:', e);
|
|
2289
|
+
return null;
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2293
|
+
_sendCommand(cmd) {
|
|
2294
|
+
if (!this._dc || this._dc.readyState !== 'open') return false;
|
|
2295
|
+
this._dc.send(JSON.stringify(cmd));
|
|
2296
|
+
return true;
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
/** Resolves the startSession() promise once both ICE and datachannel are ready. */
|
|
2300
|
+
_checkSessionReady() {
|
|
2301
|
+
if (this._iceConnected && this._dcOpen && this._sessionResolve) {
|
|
2302
|
+
this._state = 'streaming';
|
|
2303
|
+
this.requestState();
|
|
2304
|
+
this._stateRefreshInterval = setInterval(() => this.requestState(), 500);
|
|
2305
|
+
this._emit('streaming', { sessionId: this._sessionId, robotId: this._selectedRobotId });
|
|
2306
|
+
this._sessionResolve();
|
|
2307
|
+
this._sessionResolve = null;
|
|
2308
|
+
this._sessionReject = null;
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
async _handleSignalingMessage(msg) {
|
|
2313
|
+
switch (msg.type) {
|
|
2314
|
+
case 'welcome':
|
|
2315
|
+
break; // handled in connect()
|
|
2316
|
+
case 'list':
|
|
2317
|
+
this._robots = msg.producers || [];
|
|
2318
|
+
this._emit('robotsChanged', { robots: this._robots });
|
|
2319
|
+
this._maybeAutoStart();
|
|
2320
|
+
break;
|
|
2321
|
+
case 'peerStatusChanged': {
|
|
2322
|
+
const list = await this._sendToServer({ type: 'list' });
|
|
2323
|
+
if (list?.producers) {
|
|
2324
|
+
this._robots = list.producers;
|
|
2325
|
+
this._emit('robotsChanged', { robots: this._robots });
|
|
2326
|
+
this._maybeAutoStart();
|
|
2327
|
+
}
|
|
2328
|
+
break;
|
|
2329
|
+
}
|
|
2330
|
+
case 'sessionStarted':
|
|
2331
|
+
this._sessionId = msg.sessionId;
|
|
2332
|
+
break;
|
|
2333
|
+
case 'sessionRejected':
|
|
2334
|
+
// Defensive: the current central server returns sessionRejected
|
|
2335
|
+
// as the direct POST response (handled at the startSession() call
|
|
2336
|
+
// site). This branch is a safety net in case the server ever
|
|
2337
|
+
// pushes the rejection over SSE instead.
|
|
2338
|
+
this._failSessionRejected(msg);
|
|
2339
|
+
break;
|
|
2340
|
+
case 'endSession':
|
|
2341
|
+
this._handleEndSession(msg);
|
|
2342
|
+
break;
|
|
2343
|
+
case 'peer':
|
|
2344
|
+
this._handlePeerMessage(msg);
|
|
2345
|
+
break;
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
/**
|
|
2350
|
+
* Internal: handle an endSession pushed from central.
|
|
2351
|
+
*
|
|
2352
|
+
* Two user-visible scenarios converge here:
|
|
2353
|
+
*
|
|
2354
|
+
* 1. Pending startSession — central accepted our request but the
|
|
2355
|
+
* robot-side relay then refused (e.g. a local Python app holds
|
|
2356
|
+
* the daemon's robot lock). Without this handler, the client's
|
|
2357
|
+
* startSession() promise would hang forever because ICE/datachannel
|
|
2358
|
+
* never come up.
|
|
2359
|
+
* 2. Streaming was evicted — a local Python app started on the robot
|
|
2360
|
+
* while we were connected, so the relay tore down our session.
|
|
2361
|
+
*
|
|
2362
|
+
* The ``reason`` is forwarded verbatim from the relay through central:
|
|
2363
|
+
* - "robot_busy_local_app": daemon lock held by a local Python app
|
|
2364
|
+
* (refused before any media negotiation).
|
|
2365
|
+
* - "local_app_started": a local Python app started mid-session and
|
|
2366
|
+
* evicted us.
|
|
2367
|
+
* - "robot_busy_local": relay's safety-net for stale/concurrent
|
|
2368
|
+
* sessions (should be rare).
|
|
2369
|
+
*/
|
|
2370
|
+
_handleEndSession(msg) {
|
|
2371
|
+
const reason = msg.reason;
|
|
2372
|
+
const friendly = reason === 'robot_busy_local_app'
|
|
2373
|
+
? 'Robot is busy: a local Python app is running'
|
|
2374
|
+
: reason === 'local_app_started'
|
|
2375
|
+
? 'Disconnected: a local Python app started on the robot'
|
|
2376
|
+
: reason === 'robot_busy_local'
|
|
2377
|
+
? 'Robot is busy: another session is already active'
|
|
2378
|
+
: null;
|
|
2379
|
+
|
|
2380
|
+
// Case 1: a startSession() is still pending — reject its promise
|
|
2381
|
+
// with the same error shape as sessionRejected so app code can
|
|
2382
|
+
// treat both paths identically.
|
|
2383
|
+
if (this._sessionReject) {
|
|
2384
|
+
const err = new Error(
|
|
2385
|
+
friendly || `Session ended before it could start: ${reason || 'unknown reason'}`
|
|
2386
|
+
);
|
|
2387
|
+
err.reason = reason;
|
|
2388
|
+
this._emit('sessionRejected', { reason, activeApp: null });
|
|
2389
|
+
// Release resources allocated optimistically by startSession().
|
|
2390
|
+
if (this._pc) { this._pc.close(); this._pc = null; }
|
|
2391
|
+
if (this._micStream) { this._micStream.getTracks().forEach(t => t.stop()); this._micStream = null; }
|
|
2392
|
+
this._iceConnected = false;
|
|
2393
|
+
this._dcOpen = false;
|
|
2394
|
+
this._micMuted = true;
|
|
2395
|
+
this._micSupported = false;
|
|
2396
|
+
const reject = this._sessionReject;
|
|
2397
|
+
this._sessionResolve = null;
|
|
2398
|
+
this._sessionReject = null;
|
|
2399
|
+
reject(err);
|
|
2400
|
+
return;
|
|
2401
|
+
}
|
|
2402
|
+
|
|
2403
|
+
// Case 2: we were streaming and got kicked. Leave cleanup of _pc,
|
|
2404
|
+
// _dc, mic and timers to stopSession() so the event listener path
|
|
2405
|
+
// matches the user-initiated stop path exactly.
|
|
2406
|
+
if (this._state === 'streaming') {
|
|
2407
|
+
this._emit('sessionStopped', {
|
|
2408
|
+
reason: reason || 'remote_end',
|
|
2409
|
+
message: friendly,
|
|
2410
|
+
});
|
|
2411
|
+
// Fire-and-forget: we just react to what the server already
|
|
2412
|
+
// decided. stopSession() sends its own endSession back but
|
|
2413
|
+
// central has already dropped the session, so the echo is
|
|
2414
|
+
// harmless.
|
|
2415
|
+
this.stopSession().catch(() => { });
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
async _handlePeerMessage(msg) {
|
|
2420
|
+
if (!this._pc) return;
|
|
2421
|
+
try {
|
|
2422
|
+
if (msg.sdp) {
|
|
2423
|
+
const sdp = msg.sdp;
|
|
2424
|
+
if (sdp.type === 'offer') {
|
|
2425
|
+
const supportsMic = sdpHasAudioSendRecv(sdp.sdp);
|
|
2426
|
+
this._micSupported = supportsMic;
|
|
2427
|
+
this._emit('micSupported', { supported: supportsMic });
|
|
2428
|
+
|
|
2429
|
+
// Mic track must be added BEFORE setRemoteDescription so the
|
|
2430
|
+
// generated answer naturally includes sendrecv for audio.
|
|
2431
|
+
if (supportsMic && this._micStream) {
|
|
2432
|
+
for (const track of this._micStream.getAudioTracks()) {
|
|
2433
|
+
this._pc.addTrack(track, this._micStream);
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
await this._pc.setRemoteDescription(new RTCSessionDescription(sdp));
|
|
2438
|
+
const answer = await this._pc.createAnswer();
|
|
2439
|
+
await this._pc.setLocalDescription(answer);
|
|
2440
|
+
await this._sendToServer({
|
|
2441
|
+
type: 'peer',
|
|
2442
|
+
sessionId: this._sessionId,
|
|
2443
|
+
sdp: { type: 'answer', sdp: answer.sdp },
|
|
2444
|
+
});
|
|
2445
|
+
} else {
|
|
2446
|
+
await this._pc.setRemoteDescription(new RTCSessionDescription(sdp));
|
|
2447
|
+
}
|
|
2448
|
+
// Replay any ICE candidates that arrived before the
|
|
2449
|
+
// SDP exchange completed (see the buffering branch
|
|
2450
|
+
// below for context).
|
|
2451
|
+
const pending = this._pendingRemoteIce;
|
|
2452
|
+
if (pending && pending.length) {
|
|
2453
|
+
this._pendingRemoteIce = [];
|
|
2454
|
+
for (const ice of pending) {
|
|
2455
|
+
try {
|
|
2456
|
+
await this._pc.addIceCandidate(new RTCIceCandidate(ice));
|
|
2457
|
+
} catch (err) {
|
|
2458
|
+
console.warn('[reachy-mini] buffered ICE candidate rejected:', err);
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
if (msg.ice) {
|
|
2464
|
+
// Safari (and the iOS WKWebView Tauri ships on) rejects
|
|
2465
|
+
// empty candidate strings with `OperationError: Expect
|
|
2466
|
+
// line: candidate:<candidate-str>`. The signaling
|
|
2467
|
+
// server uses an empty string as the end-of-candidates
|
|
2468
|
+
// marker (legal per the WebRTC spec but optional).
|
|
2469
|
+
// Chrome / Firefox swallow it silently; we mirror that
|
|
2470
|
+
// here so the iOS WebView stops surfacing the noise as
|
|
2471
|
+
// a robot-side WebRTC error event.
|
|
2472
|
+
if (!msg.ice.candidate) return;
|
|
2473
|
+
if (this._pc.remoteDescription) {
|
|
2474
|
+
await this._pc.addIceCandidate(new RTCIceCandidate(msg.ice));
|
|
2475
|
+
} else {
|
|
2476
|
+
// The signaling transport (SSE through central) is
|
|
2477
|
+
// not strictly ordered across the offer / ICE
|
|
2478
|
+
// streams when the SDK runs inside a cross-origin
|
|
2479
|
+
// iframe or in Safari / iOS WKWebView: the first
|
|
2480
|
+
// ICE candidates can land before the offer SDP
|
|
2481
|
+
// does. Calling addIceCandidate before
|
|
2482
|
+
// setRemoteDescription throws
|
|
2483
|
+
// `InvalidStateError: The remote description was
|
|
2484
|
+
// null` and the candidate is silently lost,
|
|
2485
|
+
// sometimes wedging ICE altogether. Buffer here
|
|
2486
|
+
// and replay above as soon as the offer has been
|
|
2487
|
+
// applied.
|
|
2488
|
+
if (!this._pendingRemoteIce) this._pendingRemoteIce = [];
|
|
2489
|
+
this._pendingRemoteIce.push(msg.ice);
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
} catch (e) {
|
|
2493
|
+
console.error('WebRTC error:', e);
|
|
2494
|
+
this._emit('error', { source: 'webrtc', error: e });
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
/** Parse robot messages and dispatch. */
|
|
2499
|
+
_handleRobotMessage(data) {
|
|
2500
|
+
if ('version' in data && this._versionResolve) {
|
|
2501
|
+
this._versionResolve(data.version);
|
|
2502
|
+
this._versionResolve = null;
|
|
2503
|
+
return;
|
|
2504
|
+
}
|
|
2505
|
+
if ('hardware_id' in data && this._hardwareIdResolve) {
|
|
2506
|
+
this._hardwareIdResolve(data.hardware_id);
|
|
2507
|
+
this._hardwareIdResolve = null;
|
|
2508
|
+
return;
|
|
2509
|
+
}
|
|
2510
|
+
// Volume responses. Backend tags each response with `command` so we
|
|
2511
|
+
// know which pending resolver (speaker vs mic) to fulfil. If a
|
|
2512
|
+
// response arrives with no matching pending request (e.g. stale
|
|
2513
|
+
// after reconnect), just ignore it — the data channel protocol
|
|
2514
|
+
// has no multiplexing, so unmatched replies are expected.
|
|
2515
|
+
if (data.command === 'get_volume' || data.command === 'set_volume') {
|
|
2516
|
+
if (this._volumeResolve) {
|
|
2517
|
+
this._volumeResolve(data.status === 'error' ? null : data.volume);
|
|
2518
|
+
this._volumeResolve = null;
|
|
2519
|
+
}
|
|
2520
|
+
return;
|
|
2521
|
+
}
|
|
2522
|
+
if (data.command === 'get_microphone_volume' || data.command === 'set_microphone_volume') {
|
|
2523
|
+
if (this._micVolumeResolve) {
|
|
2524
|
+
this._micVolumeResolve(data.status === 'error' ? null : data.volume);
|
|
2525
|
+
this._micVolumeResolve = null;
|
|
2526
|
+
}
|
|
2527
|
+
return;
|
|
2528
|
+
}
|
|
2529
|
+
// Motion completion responses. The daemon emits
|
|
2530
|
+
// `{status: "ok", command: "wake_up"|"goto_sleep", completed: true}`
|
|
2531
|
+
// after the trajectory player is fully done, or `{error, command}`
|
|
2532
|
+
// on failure. Route them to the FIFO queue of pending awaiters
|
|
2533
|
+
// populated by `_sendCommandAwaitCompletion`; the N-th response
|
|
2534
|
+
// matches the N-th request thanks to the daemon's serialised
|
|
2535
|
+
// dispatcher.
|
|
2536
|
+
if (
|
|
2537
|
+
(data.command === 'wake_up' || data.command === 'goto_sleep') &&
|
|
2538
|
+
this._pendingMotionCompletions &&
|
|
2539
|
+
this._pendingMotionCompletions[data.command]
|
|
2540
|
+
) {
|
|
2541
|
+
const queue = this._pendingMotionCompletions[data.command];
|
|
2542
|
+
if (data.completed === true && queue.length > 0) {
|
|
2543
|
+
const entry = queue.shift();
|
|
2544
|
+
clearTimeout(entry.timer);
|
|
2545
|
+
entry.resolve();
|
|
2546
|
+
return;
|
|
2547
|
+
}
|
|
2548
|
+
if (data.error && queue.length > 0) {
|
|
2549
|
+
const entry = queue.shift();
|
|
2550
|
+
clearTimeout(entry.timer);
|
|
2551
|
+
entry.reject(new Error(`${data.command}: ${data.error}`));
|
|
2552
|
+
return;
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
if (data.type === 'log_line') {
|
|
2556
|
+
for (const sub of this._logSubscribers) {
|
|
2557
|
+
try {
|
|
2558
|
+
sub.onLine({ timestamp: data.timestamp, line: data.line });
|
|
2559
|
+
} catch (e) {
|
|
2560
|
+
console.error('subscribeLogs onLine threw:', e);
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
return;
|
|
2564
|
+
}
|
|
2565
|
+
if (data.type === 'log_stream_error') {
|
|
2566
|
+
for (const sub of this._logSubscribers) {
|
|
2567
|
+
if (typeof sub.onError === 'function') {
|
|
2568
|
+
try { sub.onError(data.error); }
|
|
2569
|
+
catch (e) { console.error('subscribeLogs onError threw:', e); }
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
return;
|
|
2573
|
+
}
|
|
2574
|
+
if (data.state) {
|
|
2575
|
+
const s = data.state;
|
|
2576
|
+
// Wire-shape pass-through. The daemon ships the head pose as a
|
|
2577
|
+
// nested 4×4 (numpy tolist()); we flatten to 16 numbers so
|
|
2578
|
+
// consumers can hand it straight to WebGL / Three.js / trajectory
|
|
2579
|
+
// logs. Everything else is forwarded as-is.
|
|
2580
|
+
if (s.head_pose) this._robotState.head = s.head_pose.flat();
|
|
2581
|
+
if (s.antennas) this._robotState.antennas = [s.antennas[0], s.antennas[1]];
|
|
2582
|
+
if (typeof s.body_yaw === 'number') this._robotState.body_yaw = s.body_yaw;
|
|
2583
|
+
if (s.motor_mode) this._robotState.motor_mode = s.motor_mode;
|
|
2584
|
+
if (typeof s.is_move_running === 'boolean') this._robotState.is_move_running = s.is_move_running;
|
|
2585
|
+
this._emit('state', { ...this._robotState });
|
|
2586
|
+
}
|
|
2587
|
+
if (data.error) {
|
|
2588
|
+
this._emit('error', { source: 'robot', error: data.error });
|
|
2589
|
+
}
|
|
2590
|
+
// Daemon-side upload/play broadcasts: dispatch to any waiter
|
|
2591
|
+
// whose predicate matches. Iterating in reverse keeps the
|
|
2592
|
+
// newest registration first (FIFO across same-predicate
|
|
2593
|
+
// duplicates would yield the wrong upload on a stale resend).
|
|
2594
|
+
if (this._broadcastWaiters.length > 0) {
|
|
2595
|
+
for (let i = this._broadcastWaiters.length - 1; i >= 0; i--) {
|
|
2596
|
+
const slot = this._broadcastWaiters[i];
|
|
2597
|
+
if (slot.predicate(data)) {
|
|
2598
|
+
this._broadcastWaiters.splice(i, 1);
|
|
2599
|
+
clearTimeout(slot.timer);
|
|
2600
|
+
slot.resolve(data);
|
|
2601
|
+
return;
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
2604
|
+
}
|
|
2605
|
+
}
|
|
2606
|
+
|
|
2607
|
+
/** Snap video playback to live edge if buffered lag exceeds 0.5 s. */
|
|
2608
|
+
_startLatencyMonitor(video) {
|
|
2609
|
+
if (this._latencyMonitorId) clearInterval(this._latencyMonitorId);
|
|
2610
|
+
this._latencyMonitorId = setInterval(() => {
|
|
2611
|
+
if (!video.srcObject || video.paused) return;
|
|
2612
|
+
const buf = video.buffered;
|
|
2613
|
+
if (buf.length > 0) {
|
|
2614
|
+
const end = buf.end(buf.length - 1);
|
|
2615
|
+
const lag = end - video.currentTime;
|
|
2616
|
+
if (lag > 0.5) {
|
|
2617
|
+
console.log(`Latency correction: was ${lag.toFixed(2)}s behind`);
|
|
2618
|
+
video.currentTime = end - 0.1;
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
}, 2000);
|
|
2622
|
+
}
|
|
2623
|
+
}
|
|
2624
|
+
|
|
2625
|
+
export default ReachyMini;
|