@pollen-robotics/reachy-mini-sdk 1.7.3 → 1.8.0-main.54c2dcd

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/APP_CREATION_GUIDE.md +1795 -0
  2. package/README.md +65 -5
  3. package/dist/animation/distance.d.ts +87 -0
  4. package/dist/animation/distance.d.ts.map +1 -0
  5. package/dist/animation/distance.js +140 -0
  6. package/dist/animation/distance.js.map +1 -0
  7. package/dist/animation/index.d.ts +34 -0
  8. package/dist/animation/index.d.ts.map +1 -0
  9. package/dist/animation/index.js +33 -0
  10. package/dist/animation/index.js.map +1 -0
  11. package/dist/animation/pose.d.ts +40 -0
  12. package/dist/animation/pose.d.ts.map +1 -0
  13. package/dist/animation/pose.js +15 -0
  14. package/dist/animation/pose.js.map +1 -0
  15. package/dist/animation/presets.d.ts +85 -0
  16. package/dist/animation/presets.d.ts.map +1 -0
  17. package/dist/animation/presets.js +82 -0
  18. package/dist/animation/presets.js.map +1 -0
  19. package/dist/animation/safe-return.d.ts +90 -0
  20. package/dist/animation/safe-return.d.ts.map +1 -0
  21. package/dist/animation/safe-return.js +123 -0
  22. package/dist/animation/safe-return.js.map +1 -0
  23. package/dist/lib/math.d.ts +23 -0
  24. package/dist/lib/math.d.ts.map +1 -0
  25. package/dist/lib/math.js +40 -0
  26. package/dist/lib/math.js.map +1 -0
  27. package/dist/lib/reachy-mini.d.ts +193 -0
  28. package/dist/lib/reachy-mini.d.ts.map +1 -0
  29. package/dist/lib/reachy-mini.js +1495 -0
  30. package/dist/lib/reachy-mini.js.map +1 -0
  31. package/dist/lib/types.d.ts +384 -0
  32. package/dist/lib/types.d.ts.map +1 -0
  33. package/dist/lib/types.js +9 -0
  34. package/dist/lib/types.js.map +1 -0
  35. package/dist/lib/upload-helpers.d.ts +18 -0
  36. package/dist/lib/upload-helpers.d.ts.map +1 -0
  37. package/dist/lib/upload-helpers.js +44 -0
  38. package/dist/lib/upload-helpers.js.map +1 -0
  39. package/dist/lib/url-helpers.d.ts +63 -0
  40. package/dist/lib/url-helpers.d.ts.map +1 -0
  41. package/dist/lib/url-helpers.js +153 -0
  42. package/dist/lib/url-helpers.js.map +1 -0
  43. package/dist/reachy-mini-sdk.d.ts +69 -0
  44. package/dist/reachy-mini-sdk.d.ts.map +1 -0
  45. package/dist/reachy-mini-sdk.js +68 -0
  46. package/dist/reachy-mini-sdk.js.map +1 -0
  47. package/host/README.md +157 -0
  48. package/host/dist/ReachyHost.d.ts +40 -0
  49. package/host/dist/ReachyHost.d.ts.map +1 -0
  50. package/host/dist/assets/index.d.ts +16 -0
  51. package/host/dist/assets/index.d.ts.map +1 -0
  52. package/host/dist/chunks/index-CyLPysJS.js +400 -0
  53. package/host/dist/chunks/mountHost-zdJ99_oZ.js +48392 -0
  54. package/host/dist/chunks/reachy-mini-BV9nAnC1.js +1283 -0
  55. package/host/dist/components/ConnectingView.d.ts +13 -0
  56. package/host/dist/components/ConnectingView.d.ts.map +1 -0
  57. package/host/dist/components/EmbedFrame.d.ts +16 -0
  58. package/host/dist/components/EmbedFrame.d.ts.map +1 -0
  59. package/host/dist/components/ErrorView.d.ts +21 -0
  60. package/host/dist/components/ErrorView.d.ts.map +1 -0
  61. package/host/dist/components/LeavingView.d.ts +33 -0
  62. package/host/dist/components/LeavingView.d.ts.map +1 -0
  63. package/host/dist/components/PickerView.d.ts +51 -0
  64. package/host/dist/components/PickerView.d.ts.map +1 -0
  65. package/host/dist/components/PostOAuthSplash.d.ts +25 -0
  66. package/host/dist/components/PostOAuthSplash.d.ts.map +1 -0
  67. package/host/dist/components/ReachyHostShell.d.ts +21 -0
  68. package/host/dist/components/ReachyHostShell.d.ts.map +1 -0
  69. package/host/dist/components/SignInView.d.ts +32 -0
  70. package/host/dist/components/SignInView.d.ts.map +1 -0
  71. package/host/dist/components/StepsProgressIndicator.d.ts +41 -0
  72. package/host/dist/components/StepsProgressIndicator.d.ts.map +1 -0
  73. package/host/dist/components/TopBar.d.ts +52 -0
  74. package/host/dist/components/TopBar.d.ts.map +1 -0
  75. package/host/dist/components/WelcomeBackOverlay.d.ts +28 -0
  76. package/host/dist/components/WelcomeBackOverlay.d.ts.map +1 -0
  77. package/host/dist/embed/index.d.ts +152 -0
  78. package/host/dist/embed/index.d.ts.map +1 -0
  79. package/host/dist/entry/auto.d.ts +12 -0
  80. package/host/dist/entry/auto.d.ts.map +1 -0
  81. package/host/dist/entry/auto.js +6 -0
  82. package/host/dist/entry/embed.d.ts +12 -0
  83. package/host/dist/entry/embed.d.ts.map +1 -0
  84. package/host/dist/entry/embed.js +6 -0
  85. package/host/dist/hooks/useHfProfile.d.ts +7 -0
  86. package/host/dist/hooks/useHfProfile.d.ts.map +1 -0
  87. package/host/dist/hooks/useHostBridge.d.ts +37 -0
  88. package/host/dist/hooks/useHostBridge.d.ts.map +1 -0
  89. package/host/dist/hooks/useOAuth.d.ts +16 -0
  90. package/host/dist/hooks/useOAuth.d.ts.map +1 -0
  91. package/host/dist/hooks/useRobots.d.ts +17 -0
  92. package/host/dist/hooks/useRobots.d.ts.map +1 -0
  93. package/host/dist/hooks/useSdk.d.ts +16 -0
  94. package/host/dist/hooks/useSdk.d.ts.map +1 -0
  95. package/host/dist/index.d.ts +25 -0
  96. package/host/dist/index.d.ts.map +1 -0
  97. package/host/dist/index.js +14 -0
  98. package/host/dist/lib/centralListener.d.ts +73 -0
  99. package/host/dist/lib/centralListener.d.ts.map +1 -0
  100. package/host/dist/lib/centralRest.d.ts +35 -0
  101. package/host/dist/lib/centralRest.d.ts.map +1 -0
  102. package/host/dist/lib/protocol.d.ts +230 -0
  103. package/host/dist/lib/protocol.d.ts.map +1 -0
  104. package/host/dist/lib/protocol.js +47 -0
  105. package/host/dist/lib/sdk-types.d.ts +46 -0
  106. package/host/dist/lib/sdk-types.d.ts.map +1 -0
  107. package/host/dist/lib/settings.d.ts +70 -0
  108. package/host/dist/lib/settings.d.ts.map +1 -0
  109. package/host/dist/lib/signalingUrl.d.ts +28 -0
  110. package/host/dist/lib/signalingUrl.d.ts.map +1 -0
  111. package/host/dist/lib/theme.d.ts +4 -0
  112. package/host/dist/lib/theme.d.ts.map +1 -0
  113. package/host/dist/lib/themeMode.d.ts +9 -0
  114. package/host/dist/lib/themeMode.d.ts.map +1 -0
  115. package/host/dist/lib/tokens.d.ts +64 -0
  116. package/host/dist/lib/tokens.d.ts.map +1 -0
  117. package/host/dist/mountHost.d.ts +36 -0
  118. package/host/dist/mountHost.d.ts.map +1 -0
  119. package/package.json +79 -11
  120. package/reachy-mini-sdk.js +0 -2013
@@ -0,0 +1,1495 @@
1
+ /**
2
+ * ReachyMini — Browser SDK for controlling a Reachy Mini robot over WebRTC.
3
+ * See `../reachy-mini-sdk.ts` for the package's barrel and the README for
4
+ * a quick-start guide.
5
+ */
6
+ import { oauthHandleRedirectIfPresent, oauthLoginUrl, } from '@huggingface/hub';
7
+ import { degToRad, rpyToMatrix } from './math.js';
8
+ import { consumeFragmentCredentials, readPreselectedRobotIdFromUrl, sdpHasAudioSendRecv, } from './url-helpers.js';
9
+ import { UPLOAD_CHUNK_SIZE, UPLOAD_BUFFERED_HIGH_WATER, UPLOAD_BUFFERED_LOW_WATER, hasCompressionStream, makeUploadId, bytesToBase64, gzipBase64, clampVolume, } from './upload-helpers.js';
10
+ export class ReachyMini extends EventTarget {
11
+ // ─── Config ──────────────────────────────────────────────────────────
12
+ _signalingUrl;
13
+ _clientId;
14
+ _appName;
15
+ _videoJitterBufferTargetMs;
16
+ _autoStartFromUrl;
17
+ _autoStartAttempted;
18
+ // ─── Public-ish state mirrors ────────────────────────────────────────
19
+ _state = 'disconnected';
20
+ _robots = [];
21
+ _robotState = {};
22
+ _preselectedRobotId;
23
+ // ─── Auth ────────────────────────────────────────────────────────────
24
+ _token = null;
25
+ _username = null;
26
+ _tokenExpires = null;
27
+ // ─── Signaling ───────────────────────────────────────────────────────
28
+ _sseAbortController = null;
29
+ // ─── WebRTC ──────────────────────────────────────────────────────────
30
+ _pc = null;
31
+ _dc = null;
32
+ _sessionId = null;
33
+ _selectedRobotId = null;
34
+ _pendingRemoteIce = [];
35
+ // ─── Audio ───────────────────────────────────────────────────────────
36
+ _micStream = null;
37
+ _micMuted = true;
38
+ _audioMuted = true;
39
+ _micSupported = false;
40
+ // ─── Timers ──────────────────────────────────────────────────────────
41
+ _latencyMonitorId = null;
42
+ _stateRefreshInterval = null;
43
+ // ─── Single-slot promise resolvers ───────────────────────────────────
44
+ _versionResolve = null;
45
+ _hardwareIdResolve = null;
46
+ _volumeResolve = null;
47
+ _micVolumeResolve = null;
48
+ // applyAudioConfig() / readAudioParameter() share the same single-slot
49
+ // pattern as the volume helpers. Separate slots so the two can be
50
+ // in-flight concurrently without collision.
51
+ _applyAudioConfigResolve = null;
52
+ _readAudioParameterResolve = null;
53
+ // ─── Log subscribers ─────────────────────────────────────────────────
54
+ _logSubscribers = new Set();
55
+ // ─── Broadcast waiters (playMove / playUploadedAudio) ────────────────
56
+ _broadcastWaiters = [];
57
+ // ─── Active upload ids for no-arg cancels ────────────────────────────
58
+ _activeMoveUploadId = null;
59
+ _activeAudioUploadId = null;
60
+ // ─── Session promise plumbing ────────────────────────────────────────
61
+ _sessionResolve = null;
62
+ _sessionReject = null;
63
+ _iceConnected = false;
64
+ _dcOpen = false;
65
+ // ─── Motion completion plumbing (wake_up / goto_sleep) ───────────────
66
+ _pendingMotionCompletions = {
67
+ wake_up: [],
68
+ goto_sleep: [],
69
+ };
70
+ // ─── Video element ───────────────────────────────────────────────────
71
+ _videoElement = null;
72
+ constructor(options = {}) {
73
+ super();
74
+ this._signalingUrl = options.signalingUrl || 'https://pollen-robotics-reachy-mini-central.hf.space';
75
+ // `enableMicrophone` is intentionally NOT stored: the SDK no longer
76
+ // calls getUserMedia (see startSession). Apps that still pass it for
77
+ // backward compatibility have their value silently ignored — matches
78
+ // the @deprecated annotation on the option type.
79
+ this._clientId = options.clientId || null;
80
+ this._appName = options.appName || 'unknown';
81
+ this._videoJitterBufferTargetMs = options.videoJitterBufferTargetMs ?? 0;
82
+ this._autoStartFromUrl = options.autoStartFromUrl === true;
83
+ this._autoStartAttempted = false;
84
+ this._preselectedRobotId = readPreselectedRobotIdFromUrl();
85
+ }
86
+ // ─── Read-only properties ────────────────────────────────────────────
87
+ get state() { return this._state; }
88
+ get robots() { return this._robots; }
89
+ get robotState() { return this._robotState; }
90
+ get username() { return this._username; }
91
+ get isAuthenticated() { return !!this._token; }
92
+ get micSupported() { return this._micSupported; }
93
+ get micMuted() { return this._micMuted; }
94
+ get audioMuted() { return this._audioMuted; }
95
+ get preselectedRobotId() { return this._preselectedRobotId; }
96
+ get isEmbedded() { return this._preselectedRobotId !== null; }
97
+ /**
98
+ * Internal: try to honour the `autoStartFromUrl` constructor
99
+ * option. Called from the signaling-message handler after every
100
+ * `robotsChanged` emit, so a robot that comes online after the
101
+ * SDK is already `connected` still triggers the auto-start.
102
+ */
103
+ _maybeAutoStart() {
104
+ if (!this._autoStartFromUrl)
105
+ return;
106
+ if (this._autoStartAttempted)
107
+ return;
108
+ if (!this._preselectedRobotId)
109
+ return;
110
+ if (this._state !== 'connected')
111
+ return;
112
+ const match = this._robots.find((r) => r.id === this._preselectedRobotId);
113
+ if (!match)
114
+ return;
115
+ this._autoStartAttempted = true;
116
+ const peerId = this._preselectedRobotId;
117
+ setTimeout(() => {
118
+ if (this._state !== 'connected')
119
+ return;
120
+ this.startSession(peerId).catch((err) => {
121
+ console.warn('[reachy-mini] autoStartFromUrl: startSession rejected:', err);
122
+ });
123
+ }, 0);
124
+ }
125
+ // ─── Auth ────────────────────────────────────────────────────────────
126
+ async authenticate() {
127
+ try {
128
+ consumeFragmentCredentials();
129
+ const result = (await oauthHandleRedirectIfPresent());
130
+ if (result) {
131
+ this._username = result.userInfo.preferred_username || result.userInfo.name || null;
132
+ this._token = result.accessToken;
133
+ this._tokenExpires = result.accessTokenExpiresAt;
134
+ sessionStorage.setItem('hf_token', this._token);
135
+ sessionStorage.setItem('hf_username', this._username ?? '');
136
+ sessionStorage.setItem('hf_token_expires', typeof this._tokenExpires === 'string'
137
+ ? this._tokenExpires
138
+ : this._tokenExpires.toISOString());
139
+ return true;
140
+ }
141
+ const t = sessionStorage.getItem('hf_token');
142
+ const u = sessionStorage.getItem('hf_username');
143
+ const e = sessionStorage.getItem('hf_token_expires');
144
+ if (t && u && e && new Date(e) > new Date()) {
145
+ this._token = t;
146
+ this._username = u;
147
+ this._tokenExpires = e;
148
+ return true;
149
+ }
150
+ return false;
151
+ }
152
+ catch (e) {
153
+ console.error('Auth error:', e);
154
+ return false;
155
+ }
156
+ }
157
+ async login() {
158
+ const opts = {};
159
+ if (this._clientId)
160
+ opts.clientId = this._clientId;
161
+ window.location.href = await oauthLoginUrl(opts);
162
+ }
163
+ logout() {
164
+ sessionStorage.removeItem('hf_token');
165
+ sessionStorage.removeItem('hf_username');
166
+ sessionStorage.removeItem('hf_token_expires');
167
+ this._username = null;
168
+ this._tokenExpires = null;
169
+ this.disconnect();
170
+ }
171
+ // ─── Lifecycle ───────────────────────────────────────────────────────
172
+ async connect(token) {
173
+ if (this._state !== 'disconnected')
174
+ throw new Error('Already connected');
175
+ if (token)
176
+ this._token = token;
177
+ if (!this._token)
178
+ throw new Error('No token — call authenticate() first or pass a token');
179
+ this._sseAbortController = new AbortController();
180
+ let res;
181
+ try {
182
+ res = await fetch(`${this._signalingUrl}/events`, {
183
+ signal: this._sseAbortController.signal,
184
+ headers: { 'Authorization': `Bearer ${this._token}` },
185
+ });
186
+ }
187
+ catch (e) {
188
+ this._sseAbortController = null;
189
+ throw e;
190
+ }
191
+ if (!res.ok) {
192
+ this._sseAbortController = null;
193
+ throw new Error(`HTTP ${res.status}`);
194
+ }
195
+ return new Promise((resolve, reject) => {
196
+ let welcomed = false;
197
+ const reader = res.body.getReader();
198
+ const decoder = new TextDecoder();
199
+ let buffer = '';
200
+ const readLoop = async () => {
201
+ try {
202
+ while (true) {
203
+ const { done, value } = await reader.read();
204
+ if (done)
205
+ break;
206
+ buffer += decoder.decode(value, { stream: true });
207
+ const lines = buffer.split('\n');
208
+ buffer = lines.pop() ?? '';
209
+ for (const line of lines) {
210
+ if (!line.startsWith('data:'))
211
+ continue;
212
+ try {
213
+ const msg = JSON.parse(line.slice(5).trim());
214
+ if (!welcomed && msg.type === 'welcome' && msg.peerId) {
215
+ welcomed = true;
216
+ this._state = 'connected';
217
+ await this._sendToServer({
218
+ type: 'setPeerStatus',
219
+ roles: ['listener'],
220
+ meta: { name: this._appName },
221
+ });
222
+ this._emit('connected', { peerId: msg.peerId });
223
+ resolve();
224
+ }
225
+ this._handleSignalingMessage(msg);
226
+ }
227
+ catch { /* malformed JSON — skip */ }
228
+ }
229
+ }
230
+ }
231
+ catch (e) {
232
+ if (e.name !== 'AbortError') {
233
+ this._emit('error', { source: 'signaling', error: e });
234
+ }
235
+ if (!welcomed) {
236
+ reject(e);
237
+ return;
238
+ }
239
+ }
240
+ if (this._state !== 'disconnected') {
241
+ this._state = 'disconnected';
242
+ this._emit('disconnected', { reason: 'SSE closed' });
243
+ }
244
+ if (!welcomed)
245
+ reject(new Error('Connection closed before welcome'));
246
+ };
247
+ readLoop();
248
+ });
249
+ }
250
+ async autoConnect(options = {}) {
251
+ const { token, pickRobot, autoPickIfSingle = true, filterBusy = true, wakeOnConnect = true, } = options;
252
+ if (this._state === 'streaming') {
253
+ const cur = this._robots?.find((r) => r.id === this._selectedRobotId);
254
+ return {
255
+ robotId: this._selectedRobotId,
256
+ robotName: cur?.meta?.name ?? null,
257
+ isEmbedded: this.isEmbedded,
258
+ alreadyStreaming: true,
259
+ };
260
+ }
261
+ const _prevAutoStartFromUrl = this._autoStartFromUrl;
262
+ this._autoStartFromUrl = false;
263
+ try {
264
+ if (token) {
265
+ this._token = token;
266
+ }
267
+ else if (!this._token) {
268
+ const ok = await this.authenticate();
269
+ if (!ok) {
270
+ throw new Error('Not authenticated — call login() or pass a token');
271
+ }
272
+ }
273
+ if (this._state === 'disconnected') {
274
+ await this.connect();
275
+ }
276
+ let robotId;
277
+ let robotName = null;
278
+ if (this.isEmbedded) {
279
+ robotId = this._preselectedRobotId;
280
+ try {
281
+ await this._waitForRobotInList(robotId, 5000);
282
+ }
283
+ catch { /* fall through */ }
284
+ const found = this._robots?.find((r) => r.id === robotId);
285
+ robotName = found?.meta?.name ?? null;
286
+ }
287
+ else {
288
+ const robots = await this._fetchOwnedRobots({ filterBusy });
289
+ if (robots.length === 0) {
290
+ throw new Error('No reachable robots');
291
+ }
292
+ if (autoPickIfSingle && robots.length === 1 && !robots[0].busy) {
293
+ robotId = robots[0].id;
294
+ robotName = robots[0].name;
295
+ }
296
+ else if (pickRobot) {
297
+ const picked = await pickRobot(robots);
298
+ if (!picked)
299
+ throw new Error('Robot selection cancelled');
300
+ robotId = picked;
301
+ robotName = robots.find((r) => r.id === picked)?.name ?? null;
302
+ }
303
+ else {
304
+ throw new Error('Multiple robots available — pass a pickRobot callback to autoConnect()');
305
+ }
306
+ }
307
+ await this.startSession(robotId);
308
+ if (wakeOnConnect && typeof this.ensureAwake === 'function') {
309
+ try {
310
+ await this.ensureAwake();
311
+ }
312
+ catch (e) {
313
+ console.warn('[reachy-mini] autoConnect: ensureAwake failed:', e);
314
+ }
315
+ }
316
+ return { robotId, robotName, isEmbedded: this.isEmbedded };
317
+ }
318
+ finally {
319
+ this._autoStartFromUrl = _prevAutoStartFromUrl;
320
+ }
321
+ }
322
+ async _fetchOwnedRobots({ filterBusy = true } = {}) {
323
+ try {
324
+ const res = await fetch(`${this._signalingUrl}/api/robot-status`, {
325
+ headers: { 'Authorization': `Bearer ${this._token}` },
326
+ });
327
+ if (!res.ok)
328
+ throw new Error(`HTTP ${res.status}`);
329
+ const json = await res.json();
330
+ const seen = new Map();
331
+ for (const r of (json.robots || [])) {
332
+ if (filterBusy && r.busy)
333
+ continue;
334
+ const key = r.meta?.install_id ?? r.meta?.hardware_id ?? r.peerId;
335
+ seen.set(key, {
336
+ id: r.peerId,
337
+ name: r.robotName ?? r.meta?.name ?? null,
338
+ busy: !!r.busy,
339
+ activeApp: r.activeApp ?? null,
340
+ meta: (r.meta ?? {}),
341
+ lastSeenAgeSeconds: r.last_seen_age_seconds ?? null,
342
+ });
343
+ }
344
+ return Array.from(seen.values()).sort((a, b) => (a.lastSeenAgeSeconds ?? Infinity) - (b.lastSeenAgeSeconds ?? Infinity));
345
+ }
346
+ catch (e) {
347
+ console.warn('[reachy-mini] /api/robot-status unavailable, using SSE list:', e);
348
+ return (this._robots || []).map((r) => ({
349
+ id: r.id,
350
+ name: r.meta?.name ?? null,
351
+ busy: false,
352
+ activeApp: null,
353
+ meta: (r.meta ?? {}),
354
+ lastSeenAgeSeconds: null,
355
+ }));
356
+ }
357
+ }
358
+ _waitForRobotInList(robotId, timeoutMs) {
359
+ if (this._robots?.find((r) => r.id === robotId))
360
+ return Promise.resolve();
361
+ return new Promise((resolve, reject) => {
362
+ const onChange = () => {
363
+ if (this._robots?.find((r) => r.id === robotId)) {
364
+ this.removeEventListener('robotsChanged', onChange);
365
+ clearTimeout(timeoutId);
366
+ resolve();
367
+ }
368
+ };
369
+ const timeoutId = setTimeout(() => {
370
+ this.removeEventListener('robotsChanged', onChange);
371
+ reject(new Error(`Timeout waiting for robot ${robotId} in list`));
372
+ }, timeoutMs);
373
+ this.addEventListener('robotsChanged', onChange);
374
+ });
375
+ }
376
+ async startSession(robotId) {
377
+ if (this._state !== 'connected')
378
+ throw new Error('Not connected');
379
+ this._selectedRobotId = robotId;
380
+ this._iceConnected = false;
381
+ this._dcOpen = false;
382
+ this._micSupported = false;
383
+ this._pendingRemoteIce = [];
384
+ // Silent placeholder audio track for the WebRTC audio sender.
385
+ // The SDK does NOT call navigator.mediaDevices.getUserMedia — the
386
+ // user's microphone is the app's responsibility. WebRTC needs a
387
+ // sendrecv audio sender for robot-speaker output to work, so we
388
+ // always set up a 0-gain oscillator → MediaStreamDestination as
389
+ // the initial track. Apps that want to send actual audio (TTS,
390
+ // prerecorded files, the user's mic for teleop, …) do so by
391
+ // calling sender.replaceTrack() on the audio sender exposed via
392
+ // this._pc after the `streaming` event fires.
393
+ try {
394
+ // Safari (and the iOS WKWebView Tauri ships on) exposes
395
+ // AudioContext only under the `webkitAudioContext` prefix.
396
+ // Narrow once, locally, so we don't sprinkle vendor casts
397
+ // through the code.
398
+ const w = window;
399
+ const Ctx = w.AudioContext ?? w.webkitAudioContext;
400
+ if (!Ctx)
401
+ throw new Error('AudioContext not supported');
402
+ const ctx = new Ctx();
403
+ const dst = ctx.createMediaStreamDestination();
404
+ const osc = ctx.createOscillator();
405
+ const gain = ctx.createGain();
406
+ gain.gain.value = 0;
407
+ osc.connect(gain).connect(dst);
408
+ osc.start();
409
+ const stream = dst.stream;
410
+ stream.getAudioTracks().forEach((t) => { t.enabled = false; });
411
+ this._micStream = stream;
412
+ this._micMuted = true;
413
+ }
414
+ catch (e) {
415
+ console.warn('Audio sender placeholder setup failed:', e);
416
+ this._micStream = null;
417
+ }
418
+ this._pc = new RTCPeerConnection({
419
+ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }],
420
+ });
421
+ return new Promise((resolve, reject) => {
422
+ this._sessionResolve = resolve;
423
+ this._sessionReject = reject;
424
+ this._pc.ontrack = (e) => {
425
+ if (e.track.kind === 'video') {
426
+ const ms = this._videoJitterBufferTargetMs;
427
+ try {
428
+ e.receiver
429
+ .jitterBufferTarget = ms;
430
+ }
431
+ catch { /* ignore */ }
432
+ try {
433
+ e.receiver
434
+ .playoutDelayHint = ms / 1000;
435
+ }
436
+ catch { /* ignore */ }
437
+ this._emit('videoTrack', { track: e.track, stream: e.streams[0] });
438
+ }
439
+ };
440
+ this._pc.onicecandidate = async (e) => {
441
+ if (e.candidate && this._sessionId) {
442
+ await this._sendToServer({
443
+ type: 'peer',
444
+ sessionId: this._sessionId,
445
+ ice: {
446
+ candidate: e.candidate.candidate,
447
+ sdpMLineIndex: e.candidate.sdpMLineIndex,
448
+ sdpMid: e.candidate.sdpMid,
449
+ },
450
+ });
451
+ }
452
+ };
453
+ this._pc.oniceconnectionstatechange = () => {
454
+ const s = this._pc?.iceConnectionState;
455
+ if (!s)
456
+ return;
457
+ if (s === 'connected' || s === 'completed') {
458
+ this._iceConnected = true;
459
+ this._checkSessionReady();
460
+ }
461
+ else if (s === 'failed') {
462
+ const err = new Error('ICE connection failed');
463
+ if (this._sessionReject) {
464
+ this._sessionReject(err);
465
+ this._sessionResolve = null;
466
+ this._sessionReject = null;
467
+ }
468
+ this._emit('error', { source: 'webrtc', error: err });
469
+ }
470
+ else if (s === 'disconnected') {
471
+ this._emit('error', { source: 'webrtc', error: new Error('ICE disconnected') });
472
+ }
473
+ };
474
+ this._pc.ondatachannel = (e) => {
475
+ this._dc = e.channel;
476
+ this._dc.onopen = () => {
477
+ this._dcOpen = true;
478
+ this._checkSessionReady();
479
+ };
480
+ this._dc.onmessage = (ev) => this._handleRobotMessage(JSON.parse(ev.data));
481
+ };
482
+ this._sendToServer({ type: 'startSession', peerId: robotId }).then((r) => {
483
+ if (r?.type === 'sessionRejected') {
484
+ this._failSessionRejected(r);
485
+ return;
486
+ }
487
+ if (r?.sessionId)
488
+ this._sessionId = r.sessionId;
489
+ });
490
+ });
491
+ }
492
+ _failSessionRejected(msg) {
493
+ const err = new Error(msg.reason === 'robot_busy'
494
+ ? `Robot is busy: "${msg.activeApp || 'another app'}" is already connected`
495
+ : `Session rejected: ${msg.reason || 'unknown reason'}`);
496
+ err.reason = msg.reason ?? null;
497
+ err.activeApp = msg.activeApp ?? null;
498
+ if (this._pc) {
499
+ this._pc.close();
500
+ this._pc = null;
501
+ }
502
+ if (this._micStream) {
503
+ this._micStream.getTracks().forEach((t) => t.stop());
504
+ this._micStream = null;
505
+ }
506
+ this._iceConnected = false;
507
+ this._dcOpen = false;
508
+ this._micMuted = true;
509
+ this._micSupported = false;
510
+ this._emit('sessionRejected', { reason: msg.reason, activeApp: msg.activeApp });
511
+ if (this._sessionReject) {
512
+ const reject = this._sessionReject;
513
+ this._sessionResolve = null;
514
+ this._sessionReject = null;
515
+ reject(err);
516
+ }
517
+ }
518
+ async stopSession() {
519
+ if (this._versionResolve) {
520
+ this._versionResolve(null);
521
+ this._versionResolve = null;
522
+ }
523
+ if (this._hardwareIdResolve) {
524
+ this._hardwareIdResolve(null);
525
+ this._hardwareIdResolve = null;
526
+ }
527
+ if (this._volumeResolve) {
528
+ this._volumeResolve(null);
529
+ this._volumeResolve = null;
530
+ }
531
+ if (this._micVolumeResolve) {
532
+ this._micVolumeResolve(null);
533
+ this._micVolumeResolve = null;
534
+ }
535
+ if (this._applyAudioConfigResolve) {
536
+ this._applyAudioConfigResolve(false);
537
+ this._applyAudioConfigResolve = null;
538
+ }
539
+ if (this._readAudioParameterResolve) {
540
+ this._readAudioParameterResolve(null);
541
+ this._readAudioParameterResolve = null;
542
+ }
543
+ this._logSubscribers.clear();
544
+ this._rejectPendingMotionCompletions(new Error('Session stopped'));
545
+ if (this._sessionReject) {
546
+ this._sessionReject(new Error('Session stopped'));
547
+ this._sessionResolve = null;
548
+ this._sessionReject = null;
549
+ }
550
+ if (this._stateRefreshInterval) {
551
+ clearInterval(this._stateRefreshInterval);
552
+ this._stateRefreshInterval = null;
553
+ }
554
+ if (this._latencyMonitorId) {
555
+ clearInterval(this._latencyMonitorId);
556
+ this._latencyMonitorId = null;
557
+ }
558
+ if (this._sessionId) {
559
+ await this._sendToServer({ type: 'endSession', sessionId: this._sessionId });
560
+ }
561
+ if (this._micStream) {
562
+ this._micStream.getTracks().forEach((t) => t.stop());
563
+ this._micStream = null;
564
+ }
565
+ this._micMuted = true;
566
+ this._micSupported = false;
567
+ if (this._pc) {
568
+ this._pc.close();
569
+ this._pc = null;
570
+ }
571
+ if (this._dc) {
572
+ this._dc.close();
573
+ this._dc = null;
574
+ }
575
+ this._sessionId = null;
576
+ this._iceConnected = false;
577
+ this._dcOpen = false;
578
+ const wasStreaming = this._state === 'streaming';
579
+ if (wasStreaming) {
580
+ this._state = 'connected';
581
+ this._emit('sessionStopped', { reason: 'user' });
582
+ }
583
+ }
584
+ disconnect() {
585
+ if (this._sseAbortController) {
586
+ this._sseAbortController.abort();
587
+ this._sseAbortController = null;
588
+ }
589
+ if (this._versionResolve) {
590
+ this._versionResolve(null);
591
+ this._versionResolve = null;
592
+ }
593
+ if (this._hardwareIdResolve) {
594
+ this._hardwareIdResolve(null);
595
+ this._hardwareIdResolve = null;
596
+ }
597
+ if (this._volumeResolve) {
598
+ this._volumeResolve(null);
599
+ this._volumeResolve = null;
600
+ }
601
+ if (this._micVolumeResolve) {
602
+ this._micVolumeResolve(null);
603
+ this._micVolumeResolve = null;
604
+ }
605
+ if (this._applyAudioConfigResolve) {
606
+ this._applyAudioConfigResolve(false);
607
+ this._applyAudioConfigResolve = null;
608
+ }
609
+ if (this._readAudioParameterResolve) {
610
+ this._readAudioParameterResolve(null);
611
+ this._readAudioParameterResolve = null;
612
+ }
613
+ this._logSubscribers.clear();
614
+ this._rejectPendingMotionCompletions(new Error('Disconnected'));
615
+ if (this._sessionReject) {
616
+ this._sessionReject(new Error('Disconnected'));
617
+ this._sessionResolve = null;
618
+ this._sessionReject = null;
619
+ }
620
+ if (this._stateRefreshInterval) {
621
+ clearInterval(this._stateRefreshInterval);
622
+ this._stateRefreshInterval = null;
623
+ }
624
+ if (this._latencyMonitorId) {
625
+ clearInterval(this._latencyMonitorId);
626
+ this._latencyMonitorId = null;
627
+ }
628
+ if (this._sessionId && this._token) {
629
+ this._sendToServer({ type: 'endSession', sessionId: this._sessionId });
630
+ }
631
+ if (this._micStream) {
632
+ this._micStream.getTracks().forEach((t) => t.stop());
633
+ this._micStream = null;
634
+ }
635
+ if (this._pc) {
636
+ this._pc.close();
637
+ this._pc = null;
638
+ }
639
+ if (this._dc) {
640
+ this._dc.close();
641
+ this._dc = null;
642
+ }
643
+ this._sessionId = null;
644
+ this._micMuted = true;
645
+ this._micSupported = false;
646
+ this._iceConnected = false;
647
+ this._dcOpen = false;
648
+ this._robots = [];
649
+ this._state = 'disconnected';
650
+ this._emit('disconnected', { reason: 'user' });
651
+ }
652
+ // ─── Commands ────────────────────────────────────────────────────────
653
+ /**
654
+ * Atomic raw-units pose update over the data channel. Channels you
655
+ * omit are held at their last commanded value (per-axis, independent).
656
+ *
657
+ * **Head pose is in the WORLD frame.** The daemon's IK splits the
658
+ * requested head world-yaw between body rotation and the stewart
659
+ * platform, subject to the mechanical limit
660
+ * `|head_yaw_world − body_yaw| ≤ 65°`.
661
+ *
662
+ * **If you want the head to FOLLOW the body** (tank-style rotation):
663
+ * a `setTarget({ body_yaw })` on its own does NOT rotate the head —
664
+ * the head's commanded world yaw is unchanged, so its gaze stays
665
+ * pinned in world frame while the body turns under it. To make the
666
+ * head turn with the body, include a `head` matrix in the SAME call
667
+ * with the body-yaw delta added to the head RPY's yaw:
668
+ *
669
+ * ```ts
670
+ * // Body-yaw drag handler: tank-couple the head so it follows.
671
+ * const delta = newBodyDeg - lastCommandedBodyDeg;
672
+ * const nextHeadYaw = lastCommandedHeadYawDeg + delta;
673
+ * robot.setTarget({
674
+ * head: rpyToMatrix(headRoll, headPitch, nextHeadYaw).flat(),
675
+ * body_yaw: degToRad(newBodyDeg),
676
+ * });
677
+ * lastCommandedHeadYawDeg = nextHeadYaw;
678
+ * lastCommandedBodyDeg = newBodyDeg;
679
+ * ```
680
+ *
681
+ * **Baseline must be the last COMMANDED value, not telemetry.** For
682
+ * continuous-input controllers (slider drag, joystick), do not use
683
+ * `state.head` from the `state` event as the baseline for incremental
684
+ * commands — telemetry lags one WebRTC round-trip, so cumulative
685
+ * deltas computed against it stall (every iteration in a rapid drag
686
+ * adds the same `delta` to the same stale baseline → the head fails
687
+ * to keep up). Track the last-commanded RPY in your own buffer.
688
+ *
689
+ * @param head Flat row-major 4×4 matrix (16 finite numbers) in
690
+ * the world frame. Omit to hold the previous head target.
691
+ * @param antennas `[rightRad, leftRad]` (radians). Omit to hold.
692
+ * @param body_yaw Signed radians. Omit to hold.
693
+ * @returns `true` if the command was queued on the data channel,
694
+ * `false` if the channel is not open.
695
+ */
696
+ setTarget({ head, antennas, body_yaw } = {}) {
697
+ const cmd = { type: 'set_full_target' };
698
+ if (head !== undefined) {
699
+ if (!Array.isArray(head) || head.length !== 16
700
+ || !head.every((n) => Number.isFinite(n))) {
701
+ throw new TypeError('setTarget: head must be a 16-element flat row-major 4×4 matrix '
702
+ + `of finite numbers; got ${Array.isArray(head) ? `Array(${head.length})` : typeof head}`);
703
+ }
704
+ cmd.head = head;
705
+ }
706
+ if (antennas !== undefined) {
707
+ if (!Array.isArray(antennas) || antennas.length !== 2
708
+ || !antennas.every((n) => Number.isFinite(n))) {
709
+ throw new TypeError('setTarget: antennas must be [rightRad, leftRad] (2 finite numbers); '
710
+ + `got ${Array.isArray(antennas) ? `Array(${antennas.length})` : typeof antennas}`);
711
+ }
712
+ cmd.antennas = antennas;
713
+ }
714
+ if (body_yaw !== undefined) {
715
+ if (!Number.isFinite(body_yaw)) {
716
+ throw new TypeError(`setTarget: body_yaw must be a finite number (radians); got ${body_yaw}`);
717
+ }
718
+ cmd.body_yaw = body_yaw;
719
+ }
720
+ return this._sendCommand(cmd);
721
+ }
722
+ gotoTarget({ head, antennas, body_yaw, duration }) {
723
+ const cmd = { type: 'goto_target' };
724
+ if (head !== undefined) {
725
+ if (!Array.isArray(head) || head.length !== 16
726
+ || !head.every((n) => Number.isFinite(n))) {
727
+ throw new TypeError('gotoTarget: head must be a 16-element flat row-major 4×4 matrix '
728
+ + `of finite numbers; got ${Array.isArray(head) ? `Array(${head.length})` : typeof head}`);
729
+ }
730
+ cmd.head = head;
731
+ }
732
+ if (antennas !== undefined) {
733
+ if (!Array.isArray(antennas) || antennas.length !== 2
734
+ || !antennas.every((n) => Number.isFinite(n))) {
735
+ throw new TypeError('gotoTarget: antennas must be [rightRad, leftRad] (2 finite numbers); '
736
+ + `got ${Array.isArray(antennas) ? `Array(${antennas.length})` : typeof antennas}`);
737
+ }
738
+ cmd.antennas = antennas;
739
+ }
740
+ if (body_yaw !== undefined) {
741
+ if (!Number.isFinite(body_yaw)) {
742
+ throw new TypeError(`gotoTarget: body_yaw must be a finite number (radians); got ${body_yaw}`);
743
+ }
744
+ cmd.body_yaw = body_yaw;
745
+ }
746
+ if (!Number.isFinite(duration) || duration <= 0) {
747
+ throw new TypeError(`gotoTarget: duration must be a positive finite number (seconds); got ${duration}`);
748
+ }
749
+ cmd.duration = duration;
750
+ return this._sendCommand(cmd);
751
+ }
752
+ setHeadRpyDeg(rollDeg, pitchDeg, yawDeg) {
753
+ return this.setTarget({ head: rpyToMatrix(rollDeg, pitchDeg, yawDeg).flat() });
754
+ }
755
+ setAntennasDeg(rightDeg, leftDeg) {
756
+ return this.setTarget({ antennas: [degToRad(rightDeg), degToRad(leftDeg)] });
757
+ }
758
+ setBodyYawDeg(yawDeg) {
759
+ return this.setTarget({ body_yaw: degToRad(yawDeg) });
760
+ }
761
+ playSound(file) {
762
+ return this._sendCommand({ type: 'play_sound', file });
763
+ }
764
+ setMotorMode(mode) {
765
+ return this._sendCommand({ type: 'set_motor_mode', mode });
766
+ }
767
+ setMotorTorque(on, ids = null) {
768
+ return this._sendCommand({ type: 'set_torque', on, ids });
769
+ }
770
+ wakeUp({ timeoutMs = 8000 } = {}) {
771
+ this._sendCommand({ type: 'set_motor_mode', mode: 'enabled' });
772
+ return this._sendCommandAwaitCompletion('wake_up', timeoutMs);
773
+ }
774
+ gotoSleep({ timeoutMs = 8000 } = {}) {
775
+ return this._sendCommandAwaitCompletion('goto_sleep', timeoutMs);
776
+ }
777
+ _sendCommandAwaitCompletion(command, timeoutMs) {
778
+ if (!this._sendCommand({ type: command })) {
779
+ return Promise.reject(new Error(`${command}: data channel not open`));
780
+ }
781
+ return new Promise((resolve, reject) => {
782
+ const entry = {
783
+ resolve,
784
+ reject,
785
+ timer: setTimeout(() => {
786
+ const queue = this._pendingMotionCompletions[command];
787
+ const idx = queue.indexOf(entry);
788
+ if (idx !== -1)
789
+ queue.splice(idx, 1);
790
+ reject(new Error(`${command} timed out after ${timeoutMs}ms`));
791
+ }, timeoutMs),
792
+ };
793
+ this._pendingMotionCompletions[command].push(entry);
794
+ });
795
+ }
796
+ _rejectPendingMotionCompletions(error) {
797
+ for (const command of Object.keys(this._pendingMotionCompletions)) {
798
+ const queue = this._pendingMotionCompletions[command];
799
+ while (queue.length) {
800
+ const entry = queue.shift();
801
+ clearTimeout(entry.timer);
802
+ entry.reject(error);
803
+ }
804
+ }
805
+ }
806
+ isAwake() {
807
+ const mode = this._robotState?.motor_mode;
808
+ return mode === 'enabled' || mode === 'gravity_compensation';
809
+ }
810
+ async ensureAwake(timeoutMs = 1000) {
811
+ if (this._robotState?.motor_mode === undefined) {
812
+ await new Promise((resolve) => {
813
+ const done = () => {
814
+ this.removeEventListener('state', done);
815
+ clearTimeout(timer);
816
+ resolve();
817
+ };
818
+ const timer = setTimeout(done, timeoutMs);
819
+ this.addEventListener('state', done);
820
+ this.requestState();
821
+ });
822
+ }
823
+ if (this.isAwake())
824
+ return true;
825
+ this.wakeUp().catch(() => { });
826
+ return true;
827
+ }
828
+ getVersion() {
829
+ return new Promise((resolve, reject) => {
830
+ if (!this._dc || this._dc.readyState !== 'open') {
831
+ reject(new Error('Data channel not open'));
832
+ return;
833
+ }
834
+ if (this._versionResolve) {
835
+ this._versionResolve(null);
836
+ }
837
+ this._versionResolve = resolve;
838
+ this._sendCommand({ type: 'get_version' });
839
+ });
840
+ }
841
+ getHardwareId() {
842
+ return new Promise((resolve, reject) => {
843
+ if (!this._dc || this._dc.readyState !== 'open') {
844
+ reject(new Error('Data channel not open'));
845
+ return;
846
+ }
847
+ if (this._hardwareIdResolve) {
848
+ this._hardwareIdResolve(null);
849
+ }
850
+ this._hardwareIdResolve = resolve;
851
+ this._sendCommand({ type: 'get_hardware_id' });
852
+ });
853
+ }
854
+ getVolume() {
855
+ return this._slotRoundtrip(() => this._volumeResolve, (next) => { this._volumeResolve = next; }, { type: 'get_volume' });
856
+ }
857
+ setVolume(volume) {
858
+ return this._slotRoundtrip(() => this._volumeResolve, (next) => { this._volumeResolve = next; }, { type: 'set_volume', volume: clampVolume(volume) });
859
+ }
860
+ getMicrophoneVolume() {
861
+ return this._slotRoundtrip(() => this._micVolumeResolve, (next) => { this._micVolumeResolve = next; }, { type: 'get_microphone_volume' });
862
+ }
863
+ setMicrophoneVolume(volume) {
864
+ return this._slotRoundtrip(() => this._micVolumeResolve, (next) => { this._micVolumeResolve = next; }, { type: 'set_microphone_volume', volume: clampVolume(volume) });
865
+ }
866
+ applyAudioConfig(config, { verify = true } = {}) {
867
+ return this._slotRoundtrip(() => this._applyAudioConfigResolve, (next) => { this._applyAudioConfigResolve = next; }, { type: 'apply_audio_config', config, verify }).then((v) => v === true);
868
+ }
869
+ readAudioParameter(name) {
870
+ return this._slotRoundtrip(() => this._readAudioParameterResolve, (next) => { this._readAudioParameterResolve = next; }, { type: 'read_audio_parameter', name });
871
+ }
872
+ /**
873
+ * Internal: send a command and await the matching daemon response in a
874
+ * named single-resolver slot. Used by the volume helpers and the
875
+ * XVF3800 audio-config helpers — every one of them has a strict
876
+ * request/response shape where a single in-flight call per slot is
877
+ * sufficient. If a previous request on the same slot is still
878
+ * pending when a new one comes in, the older promise is resolved to
879
+ * `null` so its caller doesn't hang forever.
880
+ *
881
+ * Slot access is passed in as getter/setter closures rather than a
882
+ * key into `this`: that keeps the helper fully generic-checked (T is
883
+ * inferred from the slot's resolver type at each call site) with no
884
+ * indexed-property casts.
885
+ */
886
+ _slotRoundtrip(getSlot, setSlot, command) {
887
+ return new Promise((resolve, reject) => {
888
+ if (!this._dc || this._dc.readyState !== 'open') {
889
+ reject(new Error('Data channel not open'));
890
+ return;
891
+ }
892
+ const prev = getSlot();
893
+ if (prev)
894
+ prev(null);
895
+ setSlot(resolve);
896
+ this._sendCommand(command);
897
+ });
898
+ }
899
+ sendRaw(data) {
900
+ return this._sendCommand(data);
901
+ }
902
+ subscribeLogs({ onLine, onError }) {
903
+ if (typeof onLine !== 'function') {
904
+ throw new TypeError('subscribeLogs: onLine callback is required');
905
+ }
906
+ const sub = { onLine, onError };
907
+ const wasEmpty = this._logSubscribers.size === 0;
908
+ this._logSubscribers.add(sub);
909
+ if (wasEmpty)
910
+ this._sendCommand({ type: 'subscribe_logs' });
911
+ let detached = false;
912
+ return () => {
913
+ if (detached)
914
+ return;
915
+ detached = true;
916
+ this._logSubscribers.delete(sub);
917
+ if (this._logSubscribers.size === 0) {
918
+ this._sendCommand({ type: 'unsubscribe_logs' });
919
+ }
920
+ };
921
+ }
922
+ requestState() {
923
+ return this._sendCommand({ type: 'get_state' });
924
+ }
925
+ // ─── Audio ───────────────────────────────────────────────────────────
926
+ setAudioMuted(muted) {
927
+ this._audioMuted = muted;
928
+ if (this._videoElement)
929
+ this._videoElement.muted = muted;
930
+ }
931
+ setMicMuted(muted) {
932
+ this._micMuted = muted;
933
+ if (this._micStream) {
934
+ this._micStream.getAudioTracks().forEach((t) => { t.enabled = !muted; });
935
+ }
936
+ }
937
+ // ─── Video helper ────────────────────────────────────────────────────
938
+ attachVideo(videoElement) {
939
+ this._videoElement = videoElement;
940
+ videoElement.muted = this._audioMuted;
941
+ const onVideoTrack = (e) => {
942
+ const ev = e;
943
+ videoElement.srcObject = ev.detail.stream;
944
+ videoElement.playsInline = true;
945
+ if ('requestVideoFrameCallback' in videoElement) {
946
+ this._startLatencyMonitor(videoElement);
947
+ }
948
+ };
949
+ const onSessionStopped = () => { videoElement.srcObject = null; };
950
+ this.addEventListener('videoTrack', onVideoTrack);
951
+ this.addEventListener('sessionStopped', onSessionStopped);
952
+ return () => {
953
+ this.removeEventListener('videoTrack', onVideoTrack);
954
+ this.removeEventListener('sessionStopped', onSessionStopped);
955
+ if (this._latencyMonitorId) {
956
+ clearInterval(this._latencyMonitorId);
957
+ this._latencyMonitorId = null;
958
+ }
959
+ videoElement.srcObject = null;
960
+ this._videoElement = null;
961
+ };
962
+ }
963
+ // ─── Daemon-side recorded-move playback ──────────────────────────────
964
+ async playMove(motion, { audioBlob = null, audioLeadMs = -100, description = 'move', encoding = 'gzip+base64', playFrequency = 100, initialGotoDuration = 0, startTimeoutMs = 8000, onProgress = () => { }, onStarted = () => { }, } = {}) {
965
+ if (!this._dc || this._dc.readyState !== 'open') {
966
+ throw new Error('data channel not open');
967
+ }
968
+ if (!motion?.time?.length || !motion?.set_target_data?.length) {
969
+ throw new Error('playMove: motion must have time + set_target_data');
970
+ }
971
+ const uploadId = makeUploadId();
972
+ this._activeMoveUploadId = uploadId;
973
+ const moveDict = {
974
+ description,
975
+ time: motion.time,
976
+ set_target_data: motion.set_target_data,
977
+ };
978
+ const jsonStr = JSON.stringify(moveDict);
979
+ let payload;
980
+ let effectiveEncoding;
981
+ if (encoding === 'gzip+base64' && hasCompressionStream()) {
982
+ payload = await gzipBase64(jsonStr);
983
+ effectiveEncoding = 'gzip+base64';
984
+ }
985
+ else {
986
+ payload = jsonStr;
987
+ effectiveEncoding = 'json';
988
+ }
989
+ const totalChunks = Math.ceil(payload.length / UPLOAD_CHUNK_SIZE) || 1;
990
+ onProgress({
991
+ phase: 'starting',
992
+ sent: 0,
993
+ total: totalChunks,
994
+ bytes: payload.length,
995
+ encoding: effectiveEncoding,
996
+ });
997
+ this._sendCommand({
998
+ type: 'upload_move_start',
999
+ upload_id: uploadId,
1000
+ total_chunks: totalChunks,
1001
+ description,
1002
+ encoding: effectiveEncoding,
1003
+ });
1004
+ for (let i = 0; i < totalChunks; i++) {
1005
+ if (this._dc.bufferedAmount > UPLOAD_BUFFERED_HIGH_WATER) {
1006
+ await this._awaitDataChannelDrain();
1007
+ }
1008
+ const start = i * UPLOAD_CHUNK_SIZE;
1009
+ this._sendCommand({
1010
+ type: 'upload_move_chunk',
1011
+ upload_id: uploadId,
1012
+ chunk_index: i,
1013
+ chunk: payload.slice(start, start + UPLOAD_CHUNK_SIZE),
1014
+ });
1015
+ onProgress({ phase: 'upload', sent: i + 1, total: totalChunks });
1016
+ }
1017
+ this._sendCommand({ type: 'upload_move_finish', upload_id: uploadId });
1018
+ onProgress({ phase: 'uploaded', sent: totalChunks, total: totalChunks });
1019
+ if (audioBlob) {
1020
+ const rawBytes = new Uint8Array(await audioBlob.arrayBuffer());
1021
+ const audioB64 = bytesToBase64(rawBytes);
1022
+ const audioTotal = Math.ceil(audioB64.length / UPLOAD_CHUNK_SIZE) || 1;
1023
+ onProgress({
1024
+ phase: 'audio-starting',
1025
+ sent: 0,
1026
+ total: audioTotal,
1027
+ bytes: audioB64.length,
1028
+ });
1029
+ this._sendCommand({
1030
+ type: 'upload_audio_start',
1031
+ upload_id: uploadId,
1032
+ total_chunks: audioTotal,
1033
+ encoding: 'wav-base64',
1034
+ description,
1035
+ });
1036
+ for (let i = 0; i < audioTotal; i++) {
1037
+ if (this._dc.bufferedAmount > UPLOAD_BUFFERED_HIGH_WATER) {
1038
+ await this._awaitDataChannelDrain();
1039
+ }
1040
+ const start = i * UPLOAD_CHUNK_SIZE;
1041
+ this._sendCommand({
1042
+ type: 'upload_audio_chunk',
1043
+ upload_id: uploadId,
1044
+ chunk_index: i,
1045
+ chunk: audioB64.slice(start, start + UPLOAD_CHUNK_SIZE),
1046
+ });
1047
+ onProgress({ phase: 'audio-upload', sent: i + 1, total: audioTotal });
1048
+ }
1049
+ this._sendCommand({ type: 'upload_audio_finish', upload_id: uploadId });
1050
+ onProgress({ phase: 'audio-uploaded', sent: audioTotal, total: audioTotal });
1051
+ }
1052
+ this._sendCommand({
1053
+ type: 'play_uploaded_move',
1054
+ upload_id: uploadId,
1055
+ play_frequency: playFrequency,
1056
+ initial_goto_duration: initialGotoDuration,
1057
+ audio_lead_ms: audioLeadMs,
1058
+ });
1059
+ let startedAck;
1060
+ try {
1061
+ startedAck = await this._waitForBroadcast((m) => m?.type === 'play_uploaded_move'
1062
+ && m?.upload_id === uploadId
1063
+ && (m.started === true || typeof m.error === 'string'), { timeoutMs: startTimeoutMs, debugLabel: 'play_uploaded_move started' });
1064
+ }
1065
+ catch (e) {
1066
+ throw new Error('Daemon did not respond to play_uploaded_move '
1067
+ + '(requires the reachy_mini daemon with feature/daemon-side-move-upload). '
1068
+ + `Underlying: ${e.message}`);
1069
+ }
1070
+ if (typeof startedAck.error === 'string') {
1071
+ throw new Error(`play_uploaded_move: ${startedAck.error}`);
1072
+ }
1073
+ try {
1074
+ onStarted({
1075
+ duration_s: startedAck.duration_s,
1076
+ has_audio: startedAck.has_audio === true,
1077
+ });
1078
+ }
1079
+ catch (e) {
1080
+ console.warn('playMove.onStarted threw:', e);
1081
+ }
1082
+ onProgress({ phase: 'playing', duration_s: startedAck.duration_s });
1083
+ const final = await this._waitForBroadcast((m) => m?.type === 'play_uploaded_move'
1084
+ && m?.upload_id === uploadId
1085
+ && (m.finished === true
1086
+ || m.cancelled === true
1087
+ || typeof m.error === 'string'), {
1088
+ timeoutMs: (startedAck.duration_s + 30) * 1000,
1089
+ debugLabel: 'play_uploaded_move final',
1090
+ });
1091
+ if (this._activeMoveUploadId === uploadId) {
1092
+ this._activeMoveUploadId = null;
1093
+ }
1094
+ return final;
1095
+ }
1096
+ cancelMove(uploadId = null) {
1097
+ const id = uploadId ?? this._activeMoveUploadId;
1098
+ if (!id)
1099
+ return false;
1100
+ return this._sendCommand({ type: 'cancel_move', upload_id: id });
1101
+ }
1102
+ async uploadAudio(audioBlob, { description = 'audio', onProgress = () => { } } = {}) {
1103
+ if (!this._dc || this._dc.readyState !== 'open') {
1104
+ throw new Error('data channel not open');
1105
+ }
1106
+ if (!(audioBlob instanceof Blob)) {
1107
+ throw new TypeError('uploadAudio: expected a Blob');
1108
+ }
1109
+ const uploadId = makeUploadId();
1110
+ const rawBytes = new Uint8Array(await audioBlob.arrayBuffer());
1111
+ const audioB64 = bytesToBase64(rawBytes);
1112
+ const total = Math.ceil(audioB64.length / UPLOAD_CHUNK_SIZE) || 1;
1113
+ onProgress({ phase: 'audio-starting', sent: 0, total, bytes: audioB64.length });
1114
+ this._sendCommand({
1115
+ type: 'upload_audio_start',
1116
+ upload_id: uploadId,
1117
+ total_chunks: total,
1118
+ encoding: 'wav-base64',
1119
+ description,
1120
+ });
1121
+ for (let i = 0; i < total; i++) {
1122
+ if (this._dc.bufferedAmount > UPLOAD_BUFFERED_HIGH_WATER) {
1123
+ await this._awaitDataChannelDrain();
1124
+ }
1125
+ const start = i * UPLOAD_CHUNK_SIZE;
1126
+ this._sendCommand({
1127
+ type: 'upload_audio_chunk',
1128
+ upload_id: uploadId,
1129
+ chunk_index: i,
1130
+ chunk: audioB64.slice(start, start + UPLOAD_CHUNK_SIZE),
1131
+ });
1132
+ onProgress({ phase: 'audio-upload', sent: i + 1, total });
1133
+ }
1134
+ this._sendCommand({ type: 'upload_audio_finish', upload_id: uploadId });
1135
+ onProgress({ phase: 'audio-uploaded', sent: total, total });
1136
+ return uploadId;
1137
+ }
1138
+ async playUploadedAudio(uploadId, { timeoutMs = 8000 } = {}) {
1139
+ if (!this._dc || this._dc.readyState !== 'open') {
1140
+ throw new Error('data channel not open');
1141
+ }
1142
+ const waiter = this._waitForBroadcast((m) => m?.type === 'play_uploaded_audio'
1143
+ && m?.upload_id === uploadId
1144
+ && (m.started === true || typeof m.error === 'string'), { timeoutMs, debugLabel: 'play_uploaded_audio started' });
1145
+ this._sendCommand({ type: 'play_uploaded_audio', upload_id: uploadId });
1146
+ const ack = await waiter;
1147
+ if (typeof ack.error === 'string')
1148
+ throw new Error(ack.error);
1149
+ this._activeAudioUploadId = uploadId;
1150
+ return ack;
1151
+ }
1152
+ cancelAudio(uploadId = null) {
1153
+ const id = uploadId ?? this._activeAudioUploadId;
1154
+ if (!id)
1155
+ return false;
1156
+ if (this._activeAudioUploadId === id) {
1157
+ this._activeAudioUploadId = null;
1158
+ }
1159
+ return this._sendCommand({ type: 'cancel_audio', upload_id: id });
1160
+ }
1161
+ // ─── Private ─────────────────────────────────────────────────────────
1162
+ _emit(name, detail) {
1163
+ this.dispatchEvent(new CustomEvent(name, { detail }));
1164
+ }
1165
+ _waitForBroadcast(predicate, { timeoutMs = 5000, debugLabel = '' } = {}) {
1166
+ return new Promise((resolve, reject) => {
1167
+ const slot = {
1168
+ predicate,
1169
+ resolve,
1170
+ timer: setTimeout(() => {
1171
+ const i = this._broadcastWaiters.indexOf(slot);
1172
+ if (i !== -1)
1173
+ this._broadcastWaiters.splice(i, 1);
1174
+ reject(new Error(`broadcast timeout (${timeoutMs} ms): ${debugLabel}`));
1175
+ }, timeoutMs),
1176
+ };
1177
+ this._broadcastWaiters.push(slot);
1178
+ });
1179
+ }
1180
+ async _awaitDataChannelDrain() {
1181
+ while (this._dc && this._dc.bufferedAmount > UPLOAD_BUFFERED_LOW_WATER) {
1182
+ await new Promise((r) => setTimeout(r, 30));
1183
+ if (!this._dc || this._dc.readyState !== 'open') {
1184
+ throw new Error('data channel closed mid-upload');
1185
+ }
1186
+ }
1187
+ }
1188
+ async _sendToServer(message) {
1189
+ if (!this._token)
1190
+ throw new Error('No token — authenticate() first');
1191
+ try {
1192
+ const res = await fetch(`${this._signalingUrl}/send`, {
1193
+ method: 'POST',
1194
+ headers: {
1195
+ 'Content-Type': 'application/json',
1196
+ 'Authorization': `Bearer ${this._token}`,
1197
+ },
1198
+ body: JSON.stringify(message),
1199
+ });
1200
+ if (!res.ok) {
1201
+ let body = '';
1202
+ try {
1203
+ body = await res.text();
1204
+ }
1205
+ catch { /* ignore */ }
1206
+ console.warn(`[reachy-mini] /send rejected (${res.status}) for type=${message?.type}; body=${body || '<empty>'}`);
1207
+ return null;
1208
+ }
1209
+ return await res.json();
1210
+ }
1211
+ catch (e) {
1212
+ console.error('Send error:', e);
1213
+ return null;
1214
+ }
1215
+ }
1216
+ _sendCommand(cmd) {
1217
+ if (!this._dc || this._dc.readyState !== 'open')
1218
+ return false;
1219
+ this._dc.send(JSON.stringify(cmd));
1220
+ return true;
1221
+ }
1222
+ _checkSessionReady() {
1223
+ if (this._iceConnected && this._dcOpen && this._sessionResolve) {
1224
+ this._state = 'streaming';
1225
+ this.requestState();
1226
+ this._stateRefreshInterval = setInterval(() => this.requestState(), 500);
1227
+ this._emit('streaming', { sessionId: this._sessionId, robotId: this._selectedRobotId });
1228
+ this._sessionResolve();
1229
+ this._sessionResolve = null;
1230
+ this._sessionReject = null;
1231
+ }
1232
+ }
1233
+ async _handleSignalingMessage(msg) {
1234
+ switch (msg.type) {
1235
+ case 'welcome':
1236
+ break;
1237
+ case 'list':
1238
+ this._robots = msg.producers || [];
1239
+ this._emit('robotsChanged', { robots: this._robots });
1240
+ this._maybeAutoStart();
1241
+ break;
1242
+ case 'peerStatusChanged': {
1243
+ const list = await this._sendToServer({ type: 'list' });
1244
+ if (list?.producers) {
1245
+ this._robots = list.producers;
1246
+ this._emit('robotsChanged', { robots: this._robots });
1247
+ this._maybeAutoStart();
1248
+ }
1249
+ break;
1250
+ }
1251
+ case 'sessionStarted':
1252
+ this._sessionId = msg.sessionId ?? null;
1253
+ break;
1254
+ case 'sessionRejected':
1255
+ this._failSessionRejected(msg);
1256
+ break;
1257
+ case 'endSession':
1258
+ this._handleEndSession(msg);
1259
+ break;
1260
+ case 'peer':
1261
+ this._handlePeerMessage(msg);
1262
+ break;
1263
+ }
1264
+ }
1265
+ _handleEndSession(msg) {
1266
+ const reason = msg.reason;
1267
+ const friendly = reason === 'robot_busy_local_app'
1268
+ ? 'Robot is busy: a local Python app is running'
1269
+ : reason === 'local_app_started'
1270
+ ? 'Disconnected: a local Python app started on the robot'
1271
+ : reason === 'robot_busy_local'
1272
+ ? 'Robot is busy: another session is already active'
1273
+ : null;
1274
+ if (this._sessionReject) {
1275
+ const err = new Error(friendly || `Session ended before it could start: ${reason || 'unknown reason'}`);
1276
+ err.reason = reason ?? null;
1277
+ this._emit('sessionRejected', { reason, activeApp: null });
1278
+ if (this._pc) {
1279
+ this._pc.close();
1280
+ this._pc = null;
1281
+ }
1282
+ if (this._micStream) {
1283
+ this._micStream.getTracks().forEach((t) => t.stop());
1284
+ this._micStream = null;
1285
+ }
1286
+ this._iceConnected = false;
1287
+ this._dcOpen = false;
1288
+ this._micMuted = true;
1289
+ this._micSupported = false;
1290
+ const reject = this._sessionReject;
1291
+ this._sessionResolve = null;
1292
+ this._sessionReject = null;
1293
+ reject(err);
1294
+ return;
1295
+ }
1296
+ if (this._state === 'streaming') {
1297
+ this._emit('sessionStopped', {
1298
+ reason: reason || 'remote_end',
1299
+ message: friendly,
1300
+ });
1301
+ this.stopSession().catch(() => { });
1302
+ }
1303
+ }
1304
+ async _handlePeerMessage(msg) {
1305
+ if (!this._pc)
1306
+ return;
1307
+ try {
1308
+ if (msg.sdp) {
1309
+ const sdp = msg.sdp;
1310
+ if (sdp.type === 'offer') {
1311
+ const supportsMic = sdpHasAudioSendRecv(sdp.sdp);
1312
+ this._micSupported = supportsMic;
1313
+ this._emit('micSupported', { supported: supportsMic });
1314
+ if (supportsMic && this._micStream) {
1315
+ for (const track of this._micStream.getAudioTracks()) {
1316
+ this._pc.addTrack(track, this._micStream);
1317
+ }
1318
+ }
1319
+ await this._pc.setRemoteDescription(new RTCSessionDescription(sdp));
1320
+ const answer = await this._pc.createAnswer();
1321
+ await this._pc.setLocalDescription(answer);
1322
+ await this._sendToServer({
1323
+ type: 'peer',
1324
+ sessionId: this._sessionId,
1325
+ sdp: { type: 'answer', sdp: answer.sdp },
1326
+ });
1327
+ }
1328
+ else {
1329
+ await this._pc.setRemoteDescription(new RTCSessionDescription(sdp));
1330
+ }
1331
+ const pending = this._pendingRemoteIce;
1332
+ if (pending && pending.length) {
1333
+ this._pendingRemoteIce = [];
1334
+ for (const ice of pending) {
1335
+ try {
1336
+ await this._pc.addIceCandidate(new RTCIceCandidate(ice));
1337
+ }
1338
+ catch (err) {
1339
+ console.warn('[reachy-mini] buffered ICE candidate rejected:', err);
1340
+ }
1341
+ }
1342
+ }
1343
+ }
1344
+ if (msg.ice) {
1345
+ // Safari (and the iOS WKWebView Tauri ships on) rejects
1346
+ // empty candidate strings with `OperationError: Expect
1347
+ // line: candidate:<candidate-str>`. The signaling
1348
+ // server uses an empty string as the end-of-candidates
1349
+ // marker (legal per the WebRTC spec but optional).
1350
+ if (!msg.ice.candidate)
1351
+ return;
1352
+ if (this._pc.remoteDescription) {
1353
+ await this._pc.addIceCandidate(new RTCIceCandidate(msg.ice));
1354
+ }
1355
+ else {
1356
+ if (!this._pendingRemoteIce)
1357
+ this._pendingRemoteIce = [];
1358
+ this._pendingRemoteIce.push(msg.ice);
1359
+ }
1360
+ }
1361
+ }
1362
+ catch (e) {
1363
+ console.error('WebRTC error:', e);
1364
+ this._emit('error', { source: 'webrtc', error: e });
1365
+ }
1366
+ }
1367
+ _handleRobotMessage(data) {
1368
+ if ('version' in data && this._versionResolve) {
1369
+ this._versionResolve(data.version);
1370
+ this._versionResolve = null;
1371
+ return;
1372
+ }
1373
+ if ('hardware_id' in data && this._hardwareIdResolve) {
1374
+ this._hardwareIdResolve(data.hardware_id);
1375
+ this._hardwareIdResolve = null;
1376
+ return;
1377
+ }
1378
+ if (data.command === 'get_volume' || data.command === 'set_volume') {
1379
+ if (this._volumeResolve) {
1380
+ this._volumeResolve(data.status === 'error' ? null : data.volume);
1381
+ this._volumeResolve = null;
1382
+ }
1383
+ return;
1384
+ }
1385
+ if (data.command === 'get_microphone_volume' || data.command === 'set_microphone_volume') {
1386
+ if (this._micVolumeResolve) {
1387
+ this._micVolumeResolve(data.status === 'error' ? null : data.volume);
1388
+ this._micVolumeResolve = null;
1389
+ }
1390
+ return;
1391
+ }
1392
+ if (data.command === 'apply_audio_config') {
1393
+ if (this._applyAudioConfigResolve) {
1394
+ this._applyAudioConfigResolve(data.error ? false : !!data.applied);
1395
+ this._applyAudioConfigResolve = null;
1396
+ }
1397
+ return;
1398
+ }
1399
+ if (data.command === 'read_audio_parameter') {
1400
+ if (this._readAudioParameterResolve) {
1401
+ this._readAudioParameterResolve(data.error ? null : (data.values ?? null));
1402
+ this._readAudioParameterResolve = null;
1403
+ }
1404
+ return;
1405
+ }
1406
+ if ((data.command === 'wake_up' || data.command === 'goto_sleep')
1407
+ && this._pendingMotionCompletions
1408
+ && this._pendingMotionCompletions[data.command]) {
1409
+ const queue = this._pendingMotionCompletions[data.command];
1410
+ if (data.completed === true && queue.length > 0) {
1411
+ const entry = queue.shift();
1412
+ clearTimeout(entry.timer);
1413
+ entry.resolve();
1414
+ return;
1415
+ }
1416
+ if (data.error && queue.length > 0) {
1417
+ const entry = queue.shift();
1418
+ clearTimeout(entry.timer);
1419
+ entry.reject(new Error(`${data.command}: ${data.error}`));
1420
+ return;
1421
+ }
1422
+ }
1423
+ if (data.type === 'log_line') {
1424
+ for (const sub of this._logSubscribers) {
1425
+ try {
1426
+ sub.onLine({ timestamp: data.timestamp, line: data.line });
1427
+ }
1428
+ catch (e) {
1429
+ console.error('subscribeLogs onLine threw:', e);
1430
+ }
1431
+ }
1432
+ return;
1433
+ }
1434
+ if (data.type === 'log_stream_error') {
1435
+ for (const sub of this._logSubscribers) {
1436
+ if (typeof sub.onError === 'function') {
1437
+ try {
1438
+ sub.onError(data.error);
1439
+ }
1440
+ catch (e) {
1441
+ console.error('subscribeLogs onError threw:', e);
1442
+ }
1443
+ }
1444
+ }
1445
+ return;
1446
+ }
1447
+ if (data.state) {
1448
+ const s = data.state;
1449
+ if (s.head_pose)
1450
+ this._robotState.head = s.head_pose.flat();
1451
+ if (s.antennas)
1452
+ this._robotState.antennas = [s.antennas[0], s.antennas[1]];
1453
+ if (typeof s.body_yaw === 'number')
1454
+ this._robotState.body_yaw = s.body_yaw;
1455
+ if (s.motor_mode)
1456
+ this._robotState.motor_mode = s.motor_mode;
1457
+ if (typeof s.is_move_running === 'boolean')
1458
+ this._robotState.is_move_running = s.is_move_running;
1459
+ this._emit('state', { ...this._robotState });
1460
+ }
1461
+ if (data.error) {
1462
+ this._emit('error', { source: 'robot', error: data.error });
1463
+ }
1464
+ if (this._broadcastWaiters.length > 0) {
1465
+ for (let i = this._broadcastWaiters.length - 1; i >= 0; i--) {
1466
+ const slot = this._broadcastWaiters[i];
1467
+ if (slot.predicate(data)) {
1468
+ this._broadcastWaiters.splice(i, 1);
1469
+ clearTimeout(slot.timer);
1470
+ slot.resolve(data);
1471
+ return;
1472
+ }
1473
+ }
1474
+ }
1475
+ }
1476
+ /** Snap video playback to live edge if buffered lag exceeds 0.5 s. */
1477
+ _startLatencyMonitor(video) {
1478
+ if (this._latencyMonitorId)
1479
+ clearInterval(this._latencyMonitorId);
1480
+ this._latencyMonitorId = setInterval(() => {
1481
+ if (!video.srcObject || video.paused)
1482
+ return;
1483
+ const buf = video.buffered;
1484
+ if (buf.length > 0) {
1485
+ const end = buf.end(buf.length - 1);
1486
+ const lag = end - video.currentTime;
1487
+ if (lag > 0.5) {
1488
+ console.log(`Latency correction: was ${lag.toFixed(2)}s behind`);
1489
+ video.currentTime = end - 0.1;
1490
+ }
1491
+ }
1492
+ }, 2000);
1493
+ }
1494
+ }
1495
+ //# sourceMappingURL=reachy-mini.js.map