neoagent 2.4.0 → 2.4.1-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE +619 -21
  2. package/README.md +1 -1
  3. package/extensions/chrome-browser/background.mjs +19 -7
  4. package/extensions/chrome-browser/icons/icon128.png +0 -0
  5. package/extensions/chrome-browser/icons/icon16.png +0 -0
  6. package/extensions/chrome-browser/icons/icon48.png +0 -0
  7. package/extensions/chrome-browser/icons/logo.svg +12 -0
  8. package/extensions/chrome-browser/manifest.json +13 -2
  9. package/extensions/chrome-browser/popup.css +5 -0
  10. package/extensions/chrome-browser/popup.html +7 -5
  11. package/extensions/chrome-browser/popup.js +16 -7
  12. package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +391 -0
  13. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
  14. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
  15. package/flutter_app/lib/main.dart +1 -0
  16. package/flutter_app/lib/main_controller.dart +156 -3
  17. package/flutter_app/lib/main_devices.dart +485 -119
  18. package/flutter_app/lib/main_settings.dart +289 -30
  19. package/flutter_app/lib/src/backend_client.dart +89 -0
  20. package/flutter_app/lib/src/desktop_companion_actions.dart +144 -0
  21. package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
  22. package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
  23. package/flutter_app/lib/src/stream_renderer.dart +286 -0
  24. package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
  25. package/package.json +2 -2
  26. package/server/guest_agent.js +19 -1
  27. package/server/http/routes.js +191 -0
  28. package/server/http/socket.js +1 -1
  29. package/server/index.js +4 -1
  30. package/server/public/.last_build_id +1 -1
  31. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  32. package/server/public/flutter_bootstrap.js +1 -1
  33. package/server/public/main.dart.js +73834 -72596
  34. package/server/routes/browser.js +14 -0
  35. package/server/routes/browser_extension.js +21 -4
  36. package/server/routes/desktop.js +10 -0
  37. package/server/routes/settings.js +4 -0
  38. package/server/routes/stream.js +187 -0
  39. package/server/services/ai/tools.js +40 -29
  40. package/server/services/android/controller.js +41 -2
  41. package/server/services/browser/controller.js +34 -0
  42. package/server/services/browser/extension/manifest.js +33 -0
  43. package/server/services/browser/extension/provider.js +12 -6
  44. package/server/services/browser/extension/registry.js +188 -18
  45. package/server/services/desktop/gateway.js +28 -3
  46. package/server/services/desktop/protocol.js +34 -0
  47. package/server/services/desktop/provider.js +25 -0
  48. package/server/services/desktop/registry.js +92 -10
  49. package/server/services/manager.js +19 -2
  50. package/server/services/runtime/backends/local-vm.js +6 -0
  51. package/server/services/runtime/docker-vm-manager.js +26 -3
  52. package/server/services/runtime/manager.js +36 -5
  53. package/server/services/runtime/settings.js +17 -0
  54. package/server/services/streaming/android-stream.js +298 -0
  55. package/server/services/streaming/browser-stream.js +87 -0
  56. package/server/services/streaming/stream-hub.js +231 -0
  57. package/server/services/websocket.js +73 -0
@@ -8,6 +8,8 @@ const {
8
8
 
9
9
  const DEFAULT_PAIRING_TTL_MS = 10 * 60 * 1000;
10
10
  const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
11
+ const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
12
+ const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
11
13
 
12
14
  function sha256(value) {
13
15
  return crypto.createHash('sha256').update(String(value || '')).digest('hex');
@@ -42,9 +44,20 @@ class BrowserExtensionRegistry {
42
44
  this.db = options.db || db;
43
45
  this.commandTimeoutMs = Number(options.commandTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_COMMAND_TIMEOUT_MS || DEFAULT_COMMAND_TIMEOUT_MS);
44
46
  this.pairingTtlMs = Number(options.pairingTtlMs || process.env.NEOAGENT_BROWSER_EXTENSION_PAIRING_TTL_MS || DEFAULT_PAIRING_TTL_MS);
47
+ this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_INTERVAL_MS || DEFAULT_HEARTBEAT_INTERVAL_MS);
48
+ this.heartbeatTimeoutMs = Number(options.heartbeatTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_TIMEOUT_MS || DEFAULT_HEARTBEAT_TIMEOUT_MS);
45
49
  this.connectionsByUser = new Map();
46
50
  }
47
51
 
52
+ #getUserConnections(userId, create = false) {
53
+ const key = String(userId || '').trim();
54
+ if (!key) return null;
55
+ if (!this.connectionsByUser.has(key) && create) {
56
+ this.connectionsByUser.set(key, new Map());
57
+ }
58
+ return this.connectionsByUser.get(key) || null;
59
+ }
60
+
48
61
  createPairingRequest(options = {}) {
49
62
  const pairingId = crypto.randomUUID();
50
63
  const pairingSecret = randomSecret(48);
@@ -129,16 +142,42 @@ class BrowserExtensionRegistry {
129
142
  };
130
143
 
131
144
  const tx = this.db.transaction(() => {
145
+ // Find other active tokens with the same name for this user.
146
+ // If they are currently offline, revoke them to clean up old installs/duplicates.
147
+ const duplicates = this.db.prepare(
148
+ `SELECT id FROM browser_extension_tokens
149
+ WHERE user_id = ? AND name = ? AND status = 'active'`
150
+ ).all(row.user_id, extensionName);
151
+
152
+ for (const dup of duplicates) {
153
+ if (!this.isConnected(row.user_id, dup.id)) {
154
+ this.db.prepare(
155
+ `UPDATE browser_extension_tokens
156
+ SET status = 'revoked', revoked_at = datetime('now')
157
+ WHERE id = ?`
158
+ ).run(dup.id);
159
+ }
160
+ }
161
+
132
162
  this.db.prepare(
133
163
  `UPDATE browser_extension_pairing_requests
134
164
  SET status = 'claimed', claimed_at = datetime('now')
135
165
  WHERE id = ?`
136
166
  ).run(row.id);
167
+
137
168
  this.db.prepare(
138
169
  `INSERT INTO browser_extension_tokens (
139
170
  id, user_id, token_hash, name, status, metadata_json
140
171
  ) VALUES (?, ?, ?, ?, 'active', ?)`
141
172
  ).run(tokenId, row.user_id, sha256(token), extensionName, safeJson(metadata));
173
+
174
+ // Auto-select the newly paired token on claim
175
+ const write = this.db.prepare(
176
+ `INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
177
+ ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
178
+ );
179
+ write.run(row.user_id, 'browser_extension_token_id', tokenId);
180
+ write.run(row.user_id, 'selected_browser_extension_token_id', tokenId);
142
181
  });
143
182
  tx();
144
183
 
@@ -164,7 +203,8 @@ class BrowserExtensionRegistry {
164
203
 
165
204
  registerConnection(tokenRow, ws, meta = {}) {
166
205
  const userId = String(tokenRow.user_id);
167
- const existing = this.connectionsByUser.get(userId);
206
+ const userMap = this.#getUserConnections(userId, true);
207
+ const existing = userMap.get(tokenRow.id);
168
208
  if (existing && existing.ws !== ws) {
169
209
  existing.close('replaced by a newer extension connection');
170
210
  }
@@ -176,8 +216,10 @@ class BrowserExtensionRegistry {
176
216
  tokenId: tokenRow.id,
177
217
  meta: { ...tokenRow.metadata, ...meta },
178
218
  timeoutMs: this.commandTimeoutMs,
219
+ heartbeatIntervalMs: this.heartbeatIntervalMs,
220
+ heartbeatTimeoutMs: this.heartbeatTimeoutMs,
179
221
  });
180
- this.connectionsByUser.set(userId, connection);
222
+ userMap.set(tokenRow.id, connection);
181
223
  this.db.prepare(
182
224
  `UPDATE browser_extension_tokens
183
225
  SET last_connected_at = datetime('now'), last_seen_at = datetime('now')
@@ -188,21 +230,93 @@ class BrowserExtensionRegistry {
188
230
 
189
231
  unregisterConnection(connection) {
190
232
  const userId = String(connection.userId);
191
- if (this.connectionsByUser.get(userId) === connection) {
192
- this.connectionsByUser.delete(userId);
233
+ const userMap = this.#getUserConnections(userId);
234
+ if (userMap?.get(connection.tokenId) === connection) {
235
+ userMap.delete(connection.tokenId);
236
+ if (userMap.size === 0) {
237
+ this.connectionsByUser.delete(userId);
238
+ }
193
239
  }
194
240
  }
195
241
 
196
- getConnection(userId) {
197
- return this.connectionsByUser.get(String(userId));
242
+ getSelectedTokenId(userId) {
243
+ const value = this.db.prepare(
244
+ `SELECT value FROM user_settings WHERE user_id = ? AND key = ?`
245
+ ).get(userId, 'browser_extension_token_id')?.value || null;
246
+ const normalized = String(value || '').trim();
247
+ if (!normalized || normalized === 'null') return null;
248
+ const existing = this.db.prepare(
249
+ `SELECT id FROM browser_extension_tokens WHERE user_id = ? AND id = ? AND status = 'active'`
250
+ ).get(userId, normalized);
251
+ if (!existing) return null;
252
+ return normalized;
198
253
  }
199
254
 
200
- isConnected(userId) {
201
- return Boolean(this.getConnection(userId)?.isOpen());
255
+ setSelectedTokenId(userId, tokenId) {
256
+ const normalized = String(tokenId || '').trim();
257
+ if (!normalized) {
258
+ const error = new Error('Browser extension token id is required.');
259
+ error.status = 400;
260
+ throw error;
261
+ }
262
+ const existing = this.db.prepare(
263
+ `SELECT id FROM browser_extension_tokens WHERE user_id = ? AND id = ? AND status = 'active'`
264
+ ).get(userId, normalized);
265
+ if (!existing) {
266
+ const error = new Error('Browser extension device not found.');
267
+ error.status = 404;
268
+ throw error;
269
+ }
270
+ const write = this.db.prepare(
271
+ `INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
272
+ ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
273
+ );
274
+ write.run(userId, 'browser_extension_token_id', normalized);
275
+ write.run(userId, 'selected_browser_extension_token_id', normalized);
276
+ return { success: true, selectedTokenId: normalized };
277
+ }
278
+
279
+ getConnection(userId, tokenId = null) {
280
+ const userMap = this.#getUserConnections(userId);
281
+ if (!userMap) return null;
282
+ const explicit = String(tokenId || '').trim();
283
+ if (explicit) return userMap.get(explicit) || null;
284
+
285
+ const selected = this.getSelectedTokenId(userId);
286
+ if (selected && userMap.get(selected)?.isOpen()) {
287
+ return userMap.get(selected);
288
+ }
289
+
290
+ // Auto-select online connection if selected is offline/unset
291
+ const online = Array.from(userMap.values()).filter((connection) => connection.isOpen());
292
+ if (online.length > 0) {
293
+ online.sort((a, b) => String(b.connectedAt || '').localeCompare(String(a.connectedAt || '')));
294
+ const activeConnection = online[0];
295
+ try {
296
+ const write = this.db.prepare(
297
+ `INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
298
+ ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
299
+ );
300
+ write.run(userId, 'browser_extension_token_id', activeConnection.tokenId);
301
+ write.run(userId, 'selected_browser_extension_token_id', activeConnection.tokenId);
302
+ } catch (err) {
303
+ console.error('[Registry] failed to auto-select online extension', err);
304
+ }
305
+ return activeConnection;
306
+ }
307
+
308
+ if (selected) {
309
+ return userMap.get(selected) || null;
310
+ }
311
+ return null;
312
+ }
313
+
314
+ isConnected(userId, tokenId = null) {
315
+ return Boolean(this.getConnection(userId, tokenId)?.isOpen());
202
316
  }
203
317
 
204
318
  async dispatch(userId, command, payload = {}, options = {}) {
205
- const connection = this.getConnection(userId);
319
+ const connection = this.getConnection(userId, options.tokenId || payload.tokenId || null);
206
320
  if (!connection || !connection.isOpen()) {
207
321
  throw new ExtensionBrowserUnavailableError();
208
322
  }
@@ -214,20 +328,35 @@ class BrowserExtensionRegistry {
214
328
  }
215
329
 
216
330
  getStatus(userId) {
217
- const connected = this.getConnection(userId);
331
+ const userMap = this.#getUserConnections(userId);
332
+ const selectedTokenId = this.getSelectedTokenId(userId);
333
+ const connected = selectedTokenId
334
+ ? userMap?.get(selectedTokenId)
335
+ : this.getConnection(userId);
336
+ const effectiveSelectedTokenId = selectedTokenId || connected?.tokenId || null;
218
337
  const tokens = this.db.prepare(
219
338
  `SELECT id, name, status, last_connected_at, last_seen_at, revoked_at, created_at, metadata_json
220
339
  FROM browser_extension_tokens
221
340
  WHERE user_id = ?
222
341
  ORDER BY created_at DESC`
223
- ).all(userId).map((row) => ({
224
- ...row,
225
- metadata: parseJson(row.metadata_json),
226
- metadata_json: undefined,
227
- }));
342
+ ).all(userId).map((row) => {
343
+ const connection = userMap?.get(row.id) || null;
344
+ return {
345
+ ...row,
346
+ tokenId: row.id,
347
+ deviceId: row.id,
348
+ connected: Boolean(connection?.isOpen()),
349
+ online: Boolean(connection?.isOpen()),
350
+ selected: row.id === effectiveSelectedTokenId,
351
+ metadata: parseJson(row.metadata_json),
352
+ connectedMeta: connection?.meta || null,
353
+ metadata_json: undefined,
354
+ };
355
+ });
228
356
  return {
229
357
  connected: Boolean(connected?.isOpen()),
230
358
  activeTokenId: connected?.tokenId || null,
359
+ selectedTokenId: effectiveSelectedTokenId,
231
360
  tokens,
232
361
  connectedMeta: connected?.meta || null,
233
362
  };
@@ -249,7 +378,7 @@ class BrowserExtensionRegistry {
249
378
  ).run(userId);
250
379
  }
251
380
 
252
- const connection = this.getConnection(userId);
381
+ const connection = this.getConnection(userId, targetTokenId);
253
382
  if (connection && (!targetTokenId || connection.tokenId === targetTokenId)) {
254
383
  connection.close('extension token revoked');
255
384
  }
@@ -258,25 +387,38 @@ class BrowserExtensionRegistry {
258
387
 
259
388
  closeAll() {
260
389
  for (const connection of this.connectionsByUser.values()) {
261
- connection.close('server shutdown');
390
+ if (connection instanceof Map) {
391
+ for (const nested of connection.values()) {
392
+ nested.close('server shutdown');
393
+ }
394
+ } else {
395
+ connection.close('server shutdown');
396
+ }
262
397
  }
263
398
  this.connectionsByUser.clear();
264
399
  }
265
400
  }
266
401
 
267
402
  class ExtensionBrowserConnection {
268
- constructor({ registry, ws, userId, tokenId, meta, timeoutMs }) {
403
+ constructor({ registry, ws, userId, tokenId, meta, timeoutMs, heartbeatIntervalMs, heartbeatTimeoutMs }) {
269
404
  this.registry = registry;
270
405
  this.ws = ws;
271
406
  this.userId = userId;
272
407
  this.tokenId = tokenId;
273
408
  this.meta = meta || {};
274
409
  this.timeoutMs = timeoutMs;
410
+ this.heartbeatIntervalMs = heartbeatIntervalMs;
411
+ this.heartbeatTimeoutMs = heartbeatTimeoutMs;
275
412
  this.pending = new Map();
413
+ this.connectedAt = new Date().toISOString();
414
+ this.lastPongAt = Date.now();
415
+ this.heartbeatTimer = null;
276
416
 
277
417
  ws.on('message', (data) => this.#handleMessage(data));
418
+ ws.on('pong', () => { this.lastPongAt = Date.now(); });
278
419
  ws.on('close', () => this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.')));
279
420
  ws.on('error', (error) => this.#closePending(error));
421
+ this.#startHeartbeat();
280
422
  }
281
423
 
282
424
  isOpen() {
@@ -317,6 +459,7 @@ class ExtensionBrowserConnection {
317
459
  }
318
460
 
319
461
  #handleMessage(data) {
462
+ this.lastPongAt = Date.now();
320
463
  let message;
321
464
  try {
322
465
  message = parseExtensionMessage(data);
@@ -339,12 +482,39 @@ class ExtensionBrowserConnection {
339
482
 
340
483
  #closePending(error) {
341
484
  this.registry.unregisterConnection(this);
485
+ if (this.heartbeatTimer) {
486
+ clearInterval(this.heartbeatTimer);
487
+ this.heartbeatTimer = null;
488
+ }
342
489
  for (const pending of this.pending.values()) {
343
490
  clearTimeout(pending.timer);
344
491
  pending.reject(error);
345
492
  }
346
493
  this.pending.clear();
347
494
  }
495
+
496
+ #startHeartbeat() {
497
+ const intervalMs = Number(this.heartbeatIntervalMs);
498
+ const timeoutMs = Number(this.heartbeatTimeoutMs);
499
+ if (!Number.isFinite(intervalMs) || intervalMs <= 0) return;
500
+ this.heartbeatTimer = setInterval(() => {
501
+ if (!this.isOpen()) {
502
+ this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.'));
503
+ return;
504
+ }
505
+ if (Number.isFinite(timeoutMs) && timeoutMs > 0 && Date.now() - this.lastPongAt > timeoutMs) {
506
+ try { this.ws.terminate(); } catch {}
507
+ this.#closePending(new ExtensionBrowserUnavailableError('Extension browser heartbeat timed out.'));
508
+ return;
509
+ }
510
+ try {
511
+ this.ws.ping();
512
+ } catch (error) {
513
+ this.#closePending(error);
514
+ }
515
+ }, intervalMs);
516
+ this.heartbeatTimer.unref?.();
517
+ }
348
518
  }
349
519
 
350
520
  module.exports = {
@@ -1,5 +1,11 @@
1
1
  const { WebSocketServer } = require('ws');
2
- const { DESKTOP_COMPANION_WS_PATH, parseDesktopMessage } = require('./protocol');
2
+ const {
3
+ DESKTOP_COMPANION_WS_PATH,
4
+ FRAME_TYPE_VIDEO,
5
+ MAX_DESKTOP_STREAM_FRAME_BYTES,
6
+ parseBinaryFrame,
7
+ parseDesktopMessage,
8
+ } = require('./protocol');
3
9
  const {
4
10
  assertDesktopHelloAuth,
5
11
  isDesktopCompanionHello,
@@ -72,8 +78,11 @@ function createUpgradeThrottleObserver() {
72
78
  return { record, snapshot };
73
79
  }
74
80
 
75
- function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware) {
76
- const wss = new WebSocketServer({ noServer: true });
81
+ function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware, streamHub = null) {
82
+ const wss = new WebSocketServer({
83
+ noServer: true,
84
+ maxPayload: MAX_DESKTOP_STREAM_FRAME_BYTES,
85
+ });
77
86
  const upgradeAttempts = new Map();
78
87
  const upgradeThrottleObserver = createUpgradeThrottleObserver();
79
88
 
@@ -173,6 +182,22 @@ function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware) {
173
182
  device,
174
183
  }));
175
184
  ws.on('message', (nextData) => {
185
+ const activeStreamHub = streamHub || app?.locals?.streamHub || null;
186
+ if (
187
+ activeStreamHub
188
+ && Buffer.isBuffer(nextData)
189
+ && nextData.length > 10
190
+ && nextData[0] === FRAME_TYPE_VIDEO
191
+ ) {
192
+ const frame = parseBinaryFrame(nextData);
193
+ if (frame) {
194
+ activeStreamHub.handleFrame(req.session.userId, device.deviceId, {
195
+ ...frame,
196
+ platform: 'desktop',
197
+ });
198
+ }
199
+ return;
200
+ }
176
201
  let parsed;
177
202
  try {
178
203
  parsed = parseDesktopMessage(nextData);
@@ -3,6 +3,8 @@ const DESKTOP_COMPANION_WS_PATH = '/api/desktop/ws';
3
3
  const DESKTOP_COMMANDS = Object.freeze({
4
4
  GET_STATUS: 'getStatus',
5
5
  CAPTURE_FRAME: 'captureFrame',
6
+ STREAM_START: 'startStream',
7
+ STREAM_STOP: 'stopStream',
6
8
  OBSERVE: 'observe',
7
9
  CLICK: 'click',
8
10
  DRAG: 'drag',
@@ -14,9 +16,14 @@ const DESKTOP_COMMANDS = Object.freeze({
14
16
  SELECT_DISPLAY: 'selectDisplay',
15
17
  GET_TREE: 'getTree',
16
18
  PAUSE_CONTROL: 'pauseControl',
19
+ EXECUTE_COMMAND: 'executeCommand',
17
20
  PING: 'ping',
21
+ MOUSE_MOVE: 'mouseMove',
18
22
  });
19
23
 
24
+ const FRAME_TYPE_VIDEO = 0x01;
25
+ const MAX_DESKTOP_STREAM_FRAME_BYTES = 8 * 1024 * 1024;
26
+
20
27
  class DesktopCompanionUnavailableError extends Error {
21
28
  constructor(message = 'Desktop companion is not connected.') {
22
29
  super(message);
@@ -54,11 +61,38 @@ function parseDesktopMessage(data) {
54
61
  }
55
62
  }
56
63
 
64
+ function parseBinaryFrame(buffer) {
65
+ if (
66
+ !Buffer.isBuffer(buffer)
67
+ || buffer.length <= 10
68
+ || buffer.length > MAX_DESKTOP_STREAM_FRAME_BYTES
69
+ || buffer[0] !== FRAME_TYPE_VIDEO
70
+ ) {
71
+ return null;
72
+ }
73
+ const jpeg = buffer.subarray(10);
74
+ const hasJpegMarkers = jpeg.length >= 4
75
+ && jpeg[0] === 0xff
76
+ && jpeg[1] === 0xd8
77
+ && jpeg[jpeg.length - 2] === 0xff
78
+ && jpeg[jpeg.length - 1] === 0xd9;
79
+ if (!hasJpegMarkers) return null;
80
+ return {
81
+ seq: buffer.readUInt32BE(1),
82
+ ts: buffer.readUInt32BE(5),
83
+ flags: buffer.readUInt8(9),
84
+ jpeg,
85
+ };
86
+ }
87
+
57
88
  module.exports = {
58
89
  DESKTOP_COMPANION_WS_PATH,
59
90
  DESKTOP_COMMANDS,
91
+ FRAME_TYPE_VIDEO,
92
+ MAX_DESKTOP_STREAM_FRAME_BYTES,
60
93
  DesktopCompanionUnavailableError,
61
94
  DesktopCompanionSelectionError,
62
95
  createDesktopCommandMessage,
96
+ parseBinaryFrame,
63
97
  parseDesktopMessage,
64
98
  };
@@ -130,6 +130,16 @@ class DesktopProvider {
130
130
  return this._dispatch(DESKTOP_COMMANDS.CAPTURE_FRAME, options);
131
131
  }
132
132
 
133
+ startStream(options = {}) {
134
+ this._assertReady();
135
+ return this.registry.startStream(this.userId, options.deviceId || null, options);
136
+ }
137
+
138
+ stopStream(options = {}) {
139
+ this._assertReady();
140
+ return this.registry.stopStream(this.userId, options.deviceId || null);
141
+ }
142
+
133
143
  observe(options = {}) {
134
144
  return this._dispatch(DESKTOP_COMMANDS.OBSERVE, options);
135
145
  }
@@ -138,6 +148,10 @@ class DesktopProvider {
138
148
  return this._dispatch(DESKTOP_COMMANDS.CLICK, { ...options, x, y });
139
149
  }
140
150
 
151
+ mouseMove(x, y, options = {}) {
152
+ return this._dispatch(DESKTOP_COMMANDS.MOUSE_MOVE, { ...options, x, y });
153
+ }
154
+
141
155
  drag(options = {}) {
142
156
  return this._dispatch(DESKTOP_COMMANDS.DRAG, options);
143
157
  }
@@ -169,6 +183,17 @@ class DesktopProvider {
169
183
  getAccessibilityTree(options = {}) {
170
184
  return this._dispatch(DESKTOP_COMMANDS.GET_TREE, options);
171
185
  }
186
+
187
+ executeCommand(command, options = {}) {
188
+ return this._dispatch(DESKTOP_COMMANDS.EXECUTE_COMMAND, {
189
+ command,
190
+ cwd: options.cwd || null,
191
+ timeout: options.timeout || null,
192
+ stdin_input: options.stdinInput || null,
193
+ pty: options.pty === true,
194
+ inputs: options.inputs || [],
195
+ });
196
+ }
172
197
  }
173
198
 
174
199
  module.exports = {
@@ -1,6 +1,8 @@
1
1
  const crypto = require('crypto');
2
2
  const db = require('../../db/database');
3
3
  const {
4
+ DESKTOP_COMMANDS,
5
+ FRAME_TYPE_VIDEO,
4
6
  DesktopCompanionSelectionError,
5
7
  DesktopCompanionUnavailableError,
6
8
  createDesktopCommandMessage,
@@ -127,6 +129,16 @@ class DesktopCompanionRegistry {
127
129
  now,
128
130
  );
129
131
 
132
+ // Remove stale offline entries for the same machine (e.g. after a re-install
133
+ // that generated a new device_id but kept the same hostname).
134
+ const hostname = hello.hostname ? String(hello.hostname).trim() : null;
135
+ if (hostname) {
136
+ this.db.prepare(
137
+ `DELETE FROM desktop_companion_devices
138
+ WHERE user_id = ? AND hostname = ? AND device_id != ? AND status = 'offline'`
139
+ ).run(userId, hostname, hello.deviceId);
140
+ }
141
+
130
142
  return this.getDeviceRecordByDeviceId(userId, hello.deviceId);
131
143
  }
132
144
 
@@ -172,9 +184,6 @@ class DesktopCompanionRegistry {
172
184
  const record = this._upsertDeviceRecord(userId, hello, sessionId);
173
185
  const userMap = this._getUserMap(userId, true);
174
186
  const existing = userMap.get(record.deviceId);
175
- if (existing && existing.ws !== ws) {
176
- existing.close('replaced by a newer desktop companion connection');
177
- }
178
187
 
179
188
  const connection = new DesktopCompanionConnection({
180
189
  registry: this,
@@ -192,7 +201,16 @@ class DesktopCompanionRegistry {
192
201
  },
193
202
  timeoutMs: this.commandTimeoutMs,
194
203
  });
204
+ // Install the new connection in the map BEFORE closing the old one.
205
+ // This ensures that when the old socket's async 'close' event fires and
206
+ // calls unregisterConnection, it sees the new connection as the owner and
207
+ // skips the DB status='offline' write — preventing a false offline report.
195
208
  userMap.set(record.deviceId, connection);
209
+
210
+ if (existing && existing.ws !== ws) {
211
+ existing.close('replaced by a newer desktop companion connection');
212
+ }
213
+
196
214
  return {
197
215
  connection,
198
216
  device: this.getDeviceRecordByDeviceId(userId, record.deviceId),
@@ -201,17 +219,21 @@ class DesktopCompanionRegistry {
201
219
 
202
220
  unregisterConnection(connection) {
203
221
  const userMap = this._getUserMap(connection.userId);
204
- if (userMap && userMap.get(connection.deviceId) === connection) {
222
+ const isOwner = userMap != null && userMap.get(connection.deviceId) === connection;
223
+ if (isOwner) {
205
224
  userMap.delete(connection.deviceId);
206
225
  if (userMap.size === 0) {
207
226
  this.connectionsByUser.delete(String(connection.userId));
208
227
  }
228
+ // Only mark offline in the DB when this connection is still the active owner.
229
+ // If a newer connection has already taken over (reconnect race), its
230
+ // _upsertDeviceRecord already wrote status='online' and we must not clobber it.
231
+ this.db.prepare(
232
+ `UPDATE desktop_companion_devices
233
+ SET status = 'offline', updated_at = datetime('now')
234
+ WHERE user_id = ? AND device_id = ?`
235
+ ).run(connection.userId, connection.deviceId);
209
236
  }
210
- this.db.prepare(
211
- `UPDATE desktop_companion_devices
212
- SET status = 'offline', updated_at = datetime('now')
213
- WHERE user_id = ? AND device_id = ?`
214
- ).run(connection.userId, connection.deviceId);
215
237
  }
216
238
 
217
239
  touchConnection(userId, deviceId, patch = {}) {
@@ -253,6 +275,11 @@ class DesktopCompanionRegistry {
253
275
  return this.getDeviceRecordByDeviceId(userId, deviceId);
254
276
  }
255
277
 
278
+ isConnected(userId) {
279
+ const userMap = this._getUserMap(userId);
280
+ return userMap != null && userMap.size > 0;
281
+ }
282
+
256
283
  getConnection(userId, deviceId) {
257
284
  const userMap = this._getUserMap(userId);
258
285
  if (!userMap) return null;
@@ -348,12 +375,58 @@ class DesktopCompanionRegistry {
348
375
  };
349
376
  }
350
377
 
378
+ async startStream(userId, deviceId, options = {}) {
379
+ const device = this.resolveDevice(userId, deviceId);
380
+ const connection = this.getConnection(userId, device.deviceId);
381
+ if (!connection || !connection.isOpen()) {
382
+ throw new DesktopCompanionUnavailableError();
383
+ }
384
+ const result = await connection.sendCommand(DESKTOP_COMMANDS.STREAM_START, {
385
+ fps: options.fps,
386
+ quality: options.quality,
387
+ displayId: options.displayId || device.activeDisplayId || null,
388
+ }, options);
389
+ connection._streaming = true;
390
+ return {
391
+ ...result,
392
+ success: result?.success !== false,
393
+ deviceId: device.deviceId,
394
+ device: this.getDeviceRecordByDeviceId(userId, device.deviceId),
395
+ };
396
+ }
397
+
398
+ async stopStream(userId, deviceId) {
399
+ const device = this.resolveDevice(userId, deviceId);
400
+ const connection = this.getConnection(userId, device.deviceId);
401
+ if (!connection || !connection.isOpen()) {
402
+ throw new DesktopCompanionUnavailableError();
403
+ }
404
+ const result = await connection.sendCommand(DESKTOP_COMMANDS.STREAM_STOP, {});
405
+ connection._streaming = false;
406
+ return {
407
+ ...result,
408
+ success: result?.success !== false,
409
+ deviceId: device.deviceId,
410
+ device: this.getDeviceRecordByDeviceId(userId, device.deviceId),
411
+ };
412
+ }
413
+
351
414
  getStatus(userId) {
352
415
  const devices = this.listDevices(userId);
353
416
  const onlineDevices = devices.filter((device) => device.online);
417
+ let selectedDeviceId = this.getSelectedDeviceId(userId);
418
+
419
+ // Auto-select the most-recently-online device when there is no valid selection.
420
+ // listDevices returns online devices first, ordered by last_seen_at DESC.
421
+ const selectionIsOnline = selectedDeviceId && onlineDevices.some((d) => d.deviceId === selectedDeviceId);
422
+ if (!selectionIsOnline && onlineDevices.length > 0) {
423
+ selectedDeviceId = onlineDevices[0].deviceId;
424
+ this.setSelectedDeviceId(userId, selectedDeviceId);
425
+ }
426
+
354
427
  return {
355
428
  connected: onlineDevices.length > 0,
356
- selectedDeviceId: this.getSelectedDeviceId(userId),
429
+ selectedDeviceId,
357
430
  onlineCount: onlineDevices.length,
358
431
  devices,
359
432
  };
@@ -446,6 +519,8 @@ class DesktopCompanionConnection {
446
519
  this.ws.close(1000, String(reason || 'closing').slice(0, 120));
447
520
  }
448
521
  } catch {}
522
+ // unregisterConnection is intentionally called here (not inside _closePending)
523
+ // so it runs synchronously before any async ws 'close' event can fire.
449
524
  this.registry.unregisterConnection(this);
450
525
  this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.'));
451
526
  }
@@ -474,6 +549,9 @@ class DesktopCompanionConnection {
474
549
  }
475
550
 
476
551
  _handleMessage(data) {
552
+ if (Buffer.isBuffer(data) && data.length > 0 && data[0] === FRAME_TYPE_VIDEO) {
553
+ return;
554
+ }
477
555
  let message;
478
556
  try {
479
557
  message = parseDesktopMessage(data);
@@ -512,6 +590,10 @@ class DesktopCompanionConnection {
512
590
  pending.reject(error);
513
591
  }
514
592
  this.pending.clear();
593
+ // unregisterConnection is idempotent (ownership-checked) so calling it
594
+ // here is safe whether we arrived via close(), the ws 'close' event, or
595
+ // both. It ensures natural socket drops (no explicit close() call) still
596
+ // mark the device offline.
515
597
  this.registry.unregisterConnection(this);
516
598
  }
517
599
  }