neoagent 2.4.0 → 2.4.1-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +619 -21
- package/README.md +1 -1
- package/extensions/chrome-browser/background.mjs +19 -7
- package/extensions/chrome-browser/icons/icon128.png +0 -0
- package/extensions/chrome-browser/icons/icon16.png +0 -0
- package/extensions/chrome-browser/icons/icon48.png +0 -0
- package/extensions/chrome-browser/icons/logo.svg +12 -0
- package/extensions/chrome-browser/manifest.json +13 -2
- package/extensions/chrome-browser/popup.css +5 -0
- package/extensions/chrome-browser/popup.html +7 -5
- package/extensions/chrome-browser/popup.js +16 -7
- package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +391 -0
- package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
- package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
- package/flutter_app/lib/main.dart +1 -0
- package/flutter_app/lib/main_controller.dart +156 -3
- package/flutter_app/lib/main_devices.dart +485 -119
- package/flutter_app/lib/main_settings.dart +289 -30
- package/flutter_app/lib/src/backend_client.dart +89 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +144 -0
- package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
- package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
- package/flutter_app/lib/src/stream_renderer.dart +286 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
- package/package.json +2 -2
- package/server/guest_agent.js +19 -1
- package/server/http/routes.js +191 -0
- package/server/http/socket.js +1 -1
- package/server/index.js +4 -1
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +73834 -72596
- package/server/routes/browser.js +14 -0
- package/server/routes/browser_extension.js +21 -4
- package/server/routes/desktop.js +10 -0
- package/server/routes/settings.js +4 -0
- package/server/routes/stream.js +187 -0
- package/server/services/ai/tools.js +40 -29
- package/server/services/android/controller.js +41 -2
- package/server/services/browser/controller.js +34 -0
- package/server/services/browser/extension/manifest.js +33 -0
- package/server/services/browser/extension/provider.js +12 -6
- package/server/services/browser/extension/registry.js +188 -18
- package/server/services/desktop/gateway.js +28 -3
- package/server/services/desktop/protocol.js +34 -0
- package/server/services/desktop/provider.js +25 -0
- package/server/services/desktop/registry.js +92 -10
- package/server/services/manager.js +19 -2
- package/server/services/runtime/backends/local-vm.js +6 -0
- package/server/services/runtime/docker-vm-manager.js +26 -3
- package/server/services/runtime/manager.js +36 -5
- package/server/services/runtime/settings.js +17 -0
- package/server/services/streaming/android-stream.js +298 -0
- package/server/services/streaming/browser-stream.js +87 -0
- package/server/services/streaming/stream-hub.js +231 -0
- package/server/services/websocket.js +73 -0
|
@@ -8,6 +8,8 @@ const {
|
|
|
8
8
|
|
|
9
9
|
const DEFAULT_PAIRING_TTL_MS = 10 * 60 * 1000;
|
|
10
10
|
const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
|
|
11
|
+
const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
|
|
12
|
+
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
|
|
11
13
|
|
|
12
14
|
function sha256(value) {
|
|
13
15
|
return crypto.createHash('sha256').update(String(value || '')).digest('hex');
|
|
@@ -42,9 +44,20 @@ class BrowserExtensionRegistry {
|
|
|
42
44
|
this.db = options.db || db;
|
|
43
45
|
this.commandTimeoutMs = Number(options.commandTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_COMMAND_TIMEOUT_MS || DEFAULT_COMMAND_TIMEOUT_MS);
|
|
44
46
|
this.pairingTtlMs = Number(options.pairingTtlMs || process.env.NEOAGENT_BROWSER_EXTENSION_PAIRING_TTL_MS || DEFAULT_PAIRING_TTL_MS);
|
|
47
|
+
this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_INTERVAL_MS || DEFAULT_HEARTBEAT_INTERVAL_MS);
|
|
48
|
+
this.heartbeatTimeoutMs = Number(options.heartbeatTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_TIMEOUT_MS || DEFAULT_HEARTBEAT_TIMEOUT_MS);
|
|
45
49
|
this.connectionsByUser = new Map();
|
|
46
50
|
}
|
|
47
51
|
|
|
52
|
+
#getUserConnections(userId, create = false) {
|
|
53
|
+
const key = String(userId || '').trim();
|
|
54
|
+
if (!key) return null;
|
|
55
|
+
if (!this.connectionsByUser.has(key) && create) {
|
|
56
|
+
this.connectionsByUser.set(key, new Map());
|
|
57
|
+
}
|
|
58
|
+
return this.connectionsByUser.get(key) || null;
|
|
59
|
+
}
|
|
60
|
+
|
|
48
61
|
createPairingRequest(options = {}) {
|
|
49
62
|
const pairingId = crypto.randomUUID();
|
|
50
63
|
const pairingSecret = randomSecret(48);
|
|
@@ -129,16 +142,42 @@ class BrowserExtensionRegistry {
|
|
|
129
142
|
};
|
|
130
143
|
|
|
131
144
|
const tx = this.db.transaction(() => {
|
|
145
|
+
// Find other active tokens with the same name for this user.
|
|
146
|
+
// If they are currently offline, revoke them to clean up old installs/duplicates.
|
|
147
|
+
const duplicates = this.db.prepare(
|
|
148
|
+
`SELECT id FROM browser_extension_tokens
|
|
149
|
+
WHERE user_id = ? AND name = ? AND status = 'active'`
|
|
150
|
+
).all(row.user_id, extensionName);
|
|
151
|
+
|
|
152
|
+
for (const dup of duplicates) {
|
|
153
|
+
if (!this.isConnected(row.user_id, dup.id)) {
|
|
154
|
+
this.db.prepare(
|
|
155
|
+
`UPDATE browser_extension_tokens
|
|
156
|
+
SET status = 'revoked', revoked_at = datetime('now')
|
|
157
|
+
WHERE id = ?`
|
|
158
|
+
).run(dup.id);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
132
162
|
this.db.prepare(
|
|
133
163
|
`UPDATE browser_extension_pairing_requests
|
|
134
164
|
SET status = 'claimed', claimed_at = datetime('now')
|
|
135
165
|
WHERE id = ?`
|
|
136
166
|
).run(row.id);
|
|
167
|
+
|
|
137
168
|
this.db.prepare(
|
|
138
169
|
`INSERT INTO browser_extension_tokens (
|
|
139
170
|
id, user_id, token_hash, name, status, metadata_json
|
|
140
171
|
) VALUES (?, ?, ?, ?, 'active', ?)`
|
|
141
172
|
).run(tokenId, row.user_id, sha256(token), extensionName, safeJson(metadata));
|
|
173
|
+
|
|
174
|
+
// Auto-select the newly paired token on claim
|
|
175
|
+
const write = this.db.prepare(
|
|
176
|
+
`INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
|
|
177
|
+
ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
|
|
178
|
+
);
|
|
179
|
+
write.run(row.user_id, 'browser_extension_token_id', tokenId);
|
|
180
|
+
write.run(row.user_id, 'selected_browser_extension_token_id', tokenId);
|
|
142
181
|
});
|
|
143
182
|
tx();
|
|
144
183
|
|
|
@@ -164,7 +203,8 @@ class BrowserExtensionRegistry {
|
|
|
164
203
|
|
|
165
204
|
registerConnection(tokenRow, ws, meta = {}) {
|
|
166
205
|
const userId = String(tokenRow.user_id);
|
|
167
|
-
const
|
|
206
|
+
const userMap = this.#getUserConnections(userId, true);
|
|
207
|
+
const existing = userMap.get(tokenRow.id);
|
|
168
208
|
if (existing && existing.ws !== ws) {
|
|
169
209
|
existing.close('replaced by a newer extension connection');
|
|
170
210
|
}
|
|
@@ -176,8 +216,10 @@ class BrowserExtensionRegistry {
|
|
|
176
216
|
tokenId: tokenRow.id,
|
|
177
217
|
meta: { ...tokenRow.metadata, ...meta },
|
|
178
218
|
timeoutMs: this.commandTimeoutMs,
|
|
219
|
+
heartbeatIntervalMs: this.heartbeatIntervalMs,
|
|
220
|
+
heartbeatTimeoutMs: this.heartbeatTimeoutMs,
|
|
179
221
|
});
|
|
180
|
-
|
|
222
|
+
userMap.set(tokenRow.id, connection);
|
|
181
223
|
this.db.prepare(
|
|
182
224
|
`UPDATE browser_extension_tokens
|
|
183
225
|
SET last_connected_at = datetime('now'), last_seen_at = datetime('now')
|
|
@@ -188,21 +230,93 @@ class BrowserExtensionRegistry {
|
|
|
188
230
|
|
|
189
231
|
unregisterConnection(connection) {
|
|
190
232
|
const userId = String(connection.userId);
|
|
191
|
-
|
|
192
|
-
|
|
233
|
+
const userMap = this.#getUserConnections(userId);
|
|
234
|
+
if (userMap?.get(connection.tokenId) === connection) {
|
|
235
|
+
userMap.delete(connection.tokenId);
|
|
236
|
+
if (userMap.size === 0) {
|
|
237
|
+
this.connectionsByUser.delete(userId);
|
|
238
|
+
}
|
|
193
239
|
}
|
|
194
240
|
}
|
|
195
241
|
|
|
196
|
-
|
|
197
|
-
|
|
242
|
+
getSelectedTokenId(userId) {
|
|
243
|
+
const value = this.db.prepare(
|
|
244
|
+
`SELECT value FROM user_settings WHERE user_id = ? AND key = ?`
|
|
245
|
+
).get(userId, 'browser_extension_token_id')?.value || null;
|
|
246
|
+
const normalized = String(value || '').trim();
|
|
247
|
+
if (!normalized || normalized === 'null') return null;
|
|
248
|
+
const existing = this.db.prepare(
|
|
249
|
+
`SELECT id FROM browser_extension_tokens WHERE user_id = ? AND id = ? AND status = 'active'`
|
|
250
|
+
).get(userId, normalized);
|
|
251
|
+
if (!existing) return null;
|
|
252
|
+
return normalized;
|
|
198
253
|
}
|
|
199
254
|
|
|
200
|
-
|
|
201
|
-
|
|
255
|
+
setSelectedTokenId(userId, tokenId) {
|
|
256
|
+
const normalized = String(tokenId || '').trim();
|
|
257
|
+
if (!normalized) {
|
|
258
|
+
const error = new Error('Browser extension token id is required.');
|
|
259
|
+
error.status = 400;
|
|
260
|
+
throw error;
|
|
261
|
+
}
|
|
262
|
+
const existing = this.db.prepare(
|
|
263
|
+
`SELECT id FROM browser_extension_tokens WHERE user_id = ? AND id = ? AND status = 'active'`
|
|
264
|
+
).get(userId, normalized);
|
|
265
|
+
if (!existing) {
|
|
266
|
+
const error = new Error('Browser extension device not found.');
|
|
267
|
+
error.status = 404;
|
|
268
|
+
throw error;
|
|
269
|
+
}
|
|
270
|
+
const write = this.db.prepare(
|
|
271
|
+
`INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
|
|
272
|
+
ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
|
|
273
|
+
);
|
|
274
|
+
write.run(userId, 'browser_extension_token_id', normalized);
|
|
275
|
+
write.run(userId, 'selected_browser_extension_token_id', normalized);
|
|
276
|
+
return { success: true, selectedTokenId: normalized };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
getConnection(userId, tokenId = null) {
|
|
280
|
+
const userMap = this.#getUserConnections(userId);
|
|
281
|
+
if (!userMap) return null;
|
|
282
|
+
const explicit = String(tokenId || '').trim();
|
|
283
|
+
if (explicit) return userMap.get(explicit) || null;
|
|
284
|
+
|
|
285
|
+
const selected = this.getSelectedTokenId(userId);
|
|
286
|
+
if (selected && userMap.get(selected)?.isOpen()) {
|
|
287
|
+
return userMap.get(selected);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Auto-select online connection if selected is offline/unset
|
|
291
|
+
const online = Array.from(userMap.values()).filter((connection) => connection.isOpen());
|
|
292
|
+
if (online.length > 0) {
|
|
293
|
+
online.sort((a, b) => String(b.connectedAt || '').localeCompare(String(a.connectedAt || '')));
|
|
294
|
+
const activeConnection = online[0];
|
|
295
|
+
try {
|
|
296
|
+
const write = this.db.prepare(
|
|
297
|
+
`INSERT INTO user_settings (user_id, key, value) VALUES (?, ?, ?)
|
|
298
|
+
ON CONFLICT(user_id, key) DO UPDATE SET value = excluded.value`
|
|
299
|
+
);
|
|
300
|
+
write.run(userId, 'browser_extension_token_id', activeConnection.tokenId);
|
|
301
|
+
write.run(userId, 'selected_browser_extension_token_id', activeConnection.tokenId);
|
|
302
|
+
} catch (err) {
|
|
303
|
+
console.error('[Registry] failed to auto-select online extension', err);
|
|
304
|
+
}
|
|
305
|
+
return activeConnection;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
if (selected) {
|
|
309
|
+
return userMap.get(selected) || null;
|
|
310
|
+
}
|
|
311
|
+
return null;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
isConnected(userId, tokenId = null) {
|
|
315
|
+
return Boolean(this.getConnection(userId, tokenId)?.isOpen());
|
|
202
316
|
}
|
|
203
317
|
|
|
204
318
|
async dispatch(userId, command, payload = {}, options = {}) {
|
|
205
|
-
const connection = this.getConnection(userId);
|
|
319
|
+
const connection = this.getConnection(userId, options.tokenId || payload.tokenId || null);
|
|
206
320
|
if (!connection || !connection.isOpen()) {
|
|
207
321
|
throw new ExtensionBrowserUnavailableError();
|
|
208
322
|
}
|
|
@@ -214,20 +328,35 @@ class BrowserExtensionRegistry {
|
|
|
214
328
|
}
|
|
215
329
|
|
|
216
330
|
getStatus(userId) {
|
|
217
|
-
const
|
|
331
|
+
const userMap = this.#getUserConnections(userId);
|
|
332
|
+
const selectedTokenId = this.getSelectedTokenId(userId);
|
|
333
|
+
const connected = selectedTokenId
|
|
334
|
+
? userMap?.get(selectedTokenId)
|
|
335
|
+
: this.getConnection(userId);
|
|
336
|
+
const effectiveSelectedTokenId = selectedTokenId || connected?.tokenId || null;
|
|
218
337
|
const tokens = this.db.prepare(
|
|
219
338
|
`SELECT id, name, status, last_connected_at, last_seen_at, revoked_at, created_at, metadata_json
|
|
220
339
|
FROM browser_extension_tokens
|
|
221
340
|
WHERE user_id = ?
|
|
222
341
|
ORDER BY created_at DESC`
|
|
223
|
-
).all(userId).map((row) =>
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
342
|
+
).all(userId).map((row) => {
|
|
343
|
+
const connection = userMap?.get(row.id) || null;
|
|
344
|
+
return {
|
|
345
|
+
...row,
|
|
346
|
+
tokenId: row.id,
|
|
347
|
+
deviceId: row.id,
|
|
348
|
+
connected: Boolean(connection?.isOpen()),
|
|
349
|
+
online: Boolean(connection?.isOpen()),
|
|
350
|
+
selected: row.id === effectiveSelectedTokenId,
|
|
351
|
+
metadata: parseJson(row.metadata_json),
|
|
352
|
+
connectedMeta: connection?.meta || null,
|
|
353
|
+
metadata_json: undefined,
|
|
354
|
+
};
|
|
355
|
+
});
|
|
228
356
|
return {
|
|
229
357
|
connected: Boolean(connected?.isOpen()),
|
|
230
358
|
activeTokenId: connected?.tokenId || null,
|
|
359
|
+
selectedTokenId: effectiveSelectedTokenId,
|
|
231
360
|
tokens,
|
|
232
361
|
connectedMeta: connected?.meta || null,
|
|
233
362
|
};
|
|
@@ -249,7 +378,7 @@ class BrowserExtensionRegistry {
|
|
|
249
378
|
).run(userId);
|
|
250
379
|
}
|
|
251
380
|
|
|
252
|
-
const connection = this.getConnection(userId);
|
|
381
|
+
const connection = this.getConnection(userId, targetTokenId);
|
|
253
382
|
if (connection && (!targetTokenId || connection.tokenId === targetTokenId)) {
|
|
254
383
|
connection.close('extension token revoked');
|
|
255
384
|
}
|
|
@@ -258,25 +387,38 @@ class BrowserExtensionRegistry {
|
|
|
258
387
|
|
|
259
388
|
closeAll() {
|
|
260
389
|
for (const connection of this.connectionsByUser.values()) {
|
|
261
|
-
connection
|
|
390
|
+
if (connection instanceof Map) {
|
|
391
|
+
for (const nested of connection.values()) {
|
|
392
|
+
nested.close('server shutdown');
|
|
393
|
+
}
|
|
394
|
+
} else {
|
|
395
|
+
connection.close('server shutdown');
|
|
396
|
+
}
|
|
262
397
|
}
|
|
263
398
|
this.connectionsByUser.clear();
|
|
264
399
|
}
|
|
265
400
|
}
|
|
266
401
|
|
|
267
402
|
class ExtensionBrowserConnection {
|
|
268
|
-
constructor({ registry, ws, userId, tokenId, meta, timeoutMs }) {
|
|
403
|
+
constructor({ registry, ws, userId, tokenId, meta, timeoutMs, heartbeatIntervalMs, heartbeatTimeoutMs }) {
|
|
269
404
|
this.registry = registry;
|
|
270
405
|
this.ws = ws;
|
|
271
406
|
this.userId = userId;
|
|
272
407
|
this.tokenId = tokenId;
|
|
273
408
|
this.meta = meta || {};
|
|
274
409
|
this.timeoutMs = timeoutMs;
|
|
410
|
+
this.heartbeatIntervalMs = heartbeatIntervalMs;
|
|
411
|
+
this.heartbeatTimeoutMs = heartbeatTimeoutMs;
|
|
275
412
|
this.pending = new Map();
|
|
413
|
+
this.connectedAt = new Date().toISOString();
|
|
414
|
+
this.lastPongAt = Date.now();
|
|
415
|
+
this.heartbeatTimer = null;
|
|
276
416
|
|
|
277
417
|
ws.on('message', (data) => this.#handleMessage(data));
|
|
418
|
+
ws.on('pong', () => { this.lastPongAt = Date.now(); });
|
|
278
419
|
ws.on('close', () => this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.')));
|
|
279
420
|
ws.on('error', (error) => this.#closePending(error));
|
|
421
|
+
this.#startHeartbeat();
|
|
280
422
|
}
|
|
281
423
|
|
|
282
424
|
isOpen() {
|
|
@@ -317,6 +459,7 @@ class ExtensionBrowserConnection {
|
|
|
317
459
|
}
|
|
318
460
|
|
|
319
461
|
#handleMessage(data) {
|
|
462
|
+
this.lastPongAt = Date.now();
|
|
320
463
|
let message;
|
|
321
464
|
try {
|
|
322
465
|
message = parseExtensionMessage(data);
|
|
@@ -339,12 +482,39 @@ class ExtensionBrowserConnection {
|
|
|
339
482
|
|
|
340
483
|
#closePending(error) {
|
|
341
484
|
this.registry.unregisterConnection(this);
|
|
485
|
+
if (this.heartbeatTimer) {
|
|
486
|
+
clearInterval(this.heartbeatTimer);
|
|
487
|
+
this.heartbeatTimer = null;
|
|
488
|
+
}
|
|
342
489
|
for (const pending of this.pending.values()) {
|
|
343
490
|
clearTimeout(pending.timer);
|
|
344
491
|
pending.reject(error);
|
|
345
492
|
}
|
|
346
493
|
this.pending.clear();
|
|
347
494
|
}
|
|
495
|
+
|
|
496
|
+
#startHeartbeat() {
|
|
497
|
+
const intervalMs = Number(this.heartbeatIntervalMs);
|
|
498
|
+
const timeoutMs = Number(this.heartbeatTimeoutMs);
|
|
499
|
+
if (!Number.isFinite(intervalMs) || intervalMs <= 0) return;
|
|
500
|
+
this.heartbeatTimer = setInterval(() => {
|
|
501
|
+
if (!this.isOpen()) {
|
|
502
|
+
this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.'));
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
if (Number.isFinite(timeoutMs) && timeoutMs > 0 && Date.now() - this.lastPongAt > timeoutMs) {
|
|
506
|
+
try { this.ws.terminate(); } catch {}
|
|
507
|
+
this.#closePending(new ExtensionBrowserUnavailableError('Extension browser heartbeat timed out.'));
|
|
508
|
+
return;
|
|
509
|
+
}
|
|
510
|
+
try {
|
|
511
|
+
this.ws.ping();
|
|
512
|
+
} catch (error) {
|
|
513
|
+
this.#closePending(error);
|
|
514
|
+
}
|
|
515
|
+
}, intervalMs);
|
|
516
|
+
this.heartbeatTimer.unref?.();
|
|
517
|
+
}
|
|
348
518
|
}
|
|
349
519
|
|
|
350
520
|
module.exports = {
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
const { WebSocketServer } = require('ws');
|
|
2
|
-
const {
|
|
2
|
+
const {
|
|
3
|
+
DESKTOP_COMPANION_WS_PATH,
|
|
4
|
+
FRAME_TYPE_VIDEO,
|
|
5
|
+
MAX_DESKTOP_STREAM_FRAME_BYTES,
|
|
6
|
+
parseBinaryFrame,
|
|
7
|
+
parseDesktopMessage,
|
|
8
|
+
} = require('./protocol');
|
|
3
9
|
const {
|
|
4
10
|
assertDesktopHelloAuth,
|
|
5
11
|
isDesktopCompanionHello,
|
|
@@ -72,8 +78,11 @@ function createUpgradeThrottleObserver() {
|
|
|
72
78
|
return { record, snapshot };
|
|
73
79
|
}
|
|
74
80
|
|
|
75
|
-
function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware) {
|
|
76
|
-
const wss = new WebSocketServer({
|
|
81
|
+
function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware, streamHub = null) {
|
|
82
|
+
const wss = new WebSocketServer({
|
|
83
|
+
noServer: true,
|
|
84
|
+
maxPayload: MAX_DESKTOP_STREAM_FRAME_BYTES,
|
|
85
|
+
});
|
|
77
86
|
const upgradeAttempts = new Map();
|
|
78
87
|
const upgradeThrottleObserver = createUpgradeThrottleObserver();
|
|
79
88
|
|
|
@@ -173,6 +182,22 @@ function bindDesktopCompanionGateway(httpServer, app, sessionMiddleware) {
|
|
|
173
182
|
device,
|
|
174
183
|
}));
|
|
175
184
|
ws.on('message', (nextData) => {
|
|
185
|
+
const activeStreamHub = streamHub || app?.locals?.streamHub || null;
|
|
186
|
+
if (
|
|
187
|
+
activeStreamHub
|
|
188
|
+
&& Buffer.isBuffer(nextData)
|
|
189
|
+
&& nextData.length > 10
|
|
190
|
+
&& nextData[0] === FRAME_TYPE_VIDEO
|
|
191
|
+
) {
|
|
192
|
+
const frame = parseBinaryFrame(nextData);
|
|
193
|
+
if (frame) {
|
|
194
|
+
activeStreamHub.handleFrame(req.session.userId, device.deviceId, {
|
|
195
|
+
...frame,
|
|
196
|
+
platform: 'desktop',
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
176
201
|
let parsed;
|
|
177
202
|
try {
|
|
178
203
|
parsed = parseDesktopMessage(nextData);
|
|
@@ -3,6 +3,8 @@ const DESKTOP_COMPANION_WS_PATH = '/api/desktop/ws';
|
|
|
3
3
|
const DESKTOP_COMMANDS = Object.freeze({
|
|
4
4
|
GET_STATUS: 'getStatus',
|
|
5
5
|
CAPTURE_FRAME: 'captureFrame',
|
|
6
|
+
STREAM_START: 'startStream',
|
|
7
|
+
STREAM_STOP: 'stopStream',
|
|
6
8
|
OBSERVE: 'observe',
|
|
7
9
|
CLICK: 'click',
|
|
8
10
|
DRAG: 'drag',
|
|
@@ -14,9 +16,14 @@ const DESKTOP_COMMANDS = Object.freeze({
|
|
|
14
16
|
SELECT_DISPLAY: 'selectDisplay',
|
|
15
17
|
GET_TREE: 'getTree',
|
|
16
18
|
PAUSE_CONTROL: 'pauseControl',
|
|
19
|
+
EXECUTE_COMMAND: 'executeCommand',
|
|
17
20
|
PING: 'ping',
|
|
21
|
+
MOUSE_MOVE: 'mouseMove',
|
|
18
22
|
});
|
|
19
23
|
|
|
24
|
+
const FRAME_TYPE_VIDEO = 0x01;
|
|
25
|
+
const MAX_DESKTOP_STREAM_FRAME_BYTES = 8 * 1024 * 1024;
|
|
26
|
+
|
|
20
27
|
class DesktopCompanionUnavailableError extends Error {
|
|
21
28
|
constructor(message = 'Desktop companion is not connected.') {
|
|
22
29
|
super(message);
|
|
@@ -54,11 +61,38 @@ function parseDesktopMessage(data) {
|
|
|
54
61
|
}
|
|
55
62
|
}
|
|
56
63
|
|
|
64
|
+
function parseBinaryFrame(buffer) {
|
|
65
|
+
if (
|
|
66
|
+
!Buffer.isBuffer(buffer)
|
|
67
|
+
|| buffer.length <= 10
|
|
68
|
+
|| buffer.length > MAX_DESKTOP_STREAM_FRAME_BYTES
|
|
69
|
+
|| buffer[0] !== FRAME_TYPE_VIDEO
|
|
70
|
+
) {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
const jpeg = buffer.subarray(10);
|
|
74
|
+
const hasJpegMarkers = jpeg.length >= 4
|
|
75
|
+
&& jpeg[0] === 0xff
|
|
76
|
+
&& jpeg[1] === 0xd8
|
|
77
|
+
&& jpeg[jpeg.length - 2] === 0xff
|
|
78
|
+
&& jpeg[jpeg.length - 1] === 0xd9;
|
|
79
|
+
if (!hasJpegMarkers) return null;
|
|
80
|
+
return {
|
|
81
|
+
seq: buffer.readUInt32BE(1),
|
|
82
|
+
ts: buffer.readUInt32BE(5),
|
|
83
|
+
flags: buffer.readUInt8(9),
|
|
84
|
+
jpeg,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
57
88
|
module.exports = {
|
|
58
89
|
DESKTOP_COMPANION_WS_PATH,
|
|
59
90
|
DESKTOP_COMMANDS,
|
|
91
|
+
FRAME_TYPE_VIDEO,
|
|
92
|
+
MAX_DESKTOP_STREAM_FRAME_BYTES,
|
|
60
93
|
DesktopCompanionUnavailableError,
|
|
61
94
|
DesktopCompanionSelectionError,
|
|
62
95
|
createDesktopCommandMessage,
|
|
96
|
+
parseBinaryFrame,
|
|
63
97
|
parseDesktopMessage,
|
|
64
98
|
};
|
|
@@ -130,6 +130,16 @@ class DesktopProvider {
|
|
|
130
130
|
return this._dispatch(DESKTOP_COMMANDS.CAPTURE_FRAME, options);
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
startStream(options = {}) {
|
|
134
|
+
this._assertReady();
|
|
135
|
+
return this.registry.startStream(this.userId, options.deviceId || null, options);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
stopStream(options = {}) {
|
|
139
|
+
this._assertReady();
|
|
140
|
+
return this.registry.stopStream(this.userId, options.deviceId || null);
|
|
141
|
+
}
|
|
142
|
+
|
|
133
143
|
observe(options = {}) {
|
|
134
144
|
return this._dispatch(DESKTOP_COMMANDS.OBSERVE, options);
|
|
135
145
|
}
|
|
@@ -138,6 +148,10 @@ class DesktopProvider {
|
|
|
138
148
|
return this._dispatch(DESKTOP_COMMANDS.CLICK, { ...options, x, y });
|
|
139
149
|
}
|
|
140
150
|
|
|
151
|
+
mouseMove(x, y, options = {}) {
|
|
152
|
+
return this._dispatch(DESKTOP_COMMANDS.MOUSE_MOVE, { ...options, x, y });
|
|
153
|
+
}
|
|
154
|
+
|
|
141
155
|
drag(options = {}) {
|
|
142
156
|
return this._dispatch(DESKTOP_COMMANDS.DRAG, options);
|
|
143
157
|
}
|
|
@@ -169,6 +183,17 @@ class DesktopProvider {
|
|
|
169
183
|
getAccessibilityTree(options = {}) {
|
|
170
184
|
return this._dispatch(DESKTOP_COMMANDS.GET_TREE, options);
|
|
171
185
|
}
|
|
186
|
+
|
|
187
|
+
executeCommand(command, options = {}) {
|
|
188
|
+
return this._dispatch(DESKTOP_COMMANDS.EXECUTE_COMMAND, {
|
|
189
|
+
command,
|
|
190
|
+
cwd: options.cwd || null,
|
|
191
|
+
timeout: options.timeout || null,
|
|
192
|
+
stdin_input: options.stdinInput || null,
|
|
193
|
+
pty: options.pty === true,
|
|
194
|
+
inputs: options.inputs || [],
|
|
195
|
+
});
|
|
196
|
+
}
|
|
172
197
|
}
|
|
173
198
|
|
|
174
199
|
module.exports = {
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
const crypto = require('crypto');
|
|
2
2
|
const db = require('../../db/database');
|
|
3
3
|
const {
|
|
4
|
+
DESKTOP_COMMANDS,
|
|
5
|
+
FRAME_TYPE_VIDEO,
|
|
4
6
|
DesktopCompanionSelectionError,
|
|
5
7
|
DesktopCompanionUnavailableError,
|
|
6
8
|
createDesktopCommandMessage,
|
|
@@ -127,6 +129,16 @@ class DesktopCompanionRegistry {
|
|
|
127
129
|
now,
|
|
128
130
|
);
|
|
129
131
|
|
|
132
|
+
// Remove stale offline entries for the same machine (e.g. after a re-install
|
|
133
|
+
// that generated a new device_id but kept the same hostname).
|
|
134
|
+
const hostname = hello.hostname ? String(hello.hostname).trim() : null;
|
|
135
|
+
if (hostname) {
|
|
136
|
+
this.db.prepare(
|
|
137
|
+
`DELETE FROM desktop_companion_devices
|
|
138
|
+
WHERE user_id = ? AND hostname = ? AND device_id != ? AND status = 'offline'`
|
|
139
|
+
).run(userId, hostname, hello.deviceId);
|
|
140
|
+
}
|
|
141
|
+
|
|
130
142
|
return this.getDeviceRecordByDeviceId(userId, hello.deviceId);
|
|
131
143
|
}
|
|
132
144
|
|
|
@@ -172,9 +184,6 @@ class DesktopCompanionRegistry {
|
|
|
172
184
|
const record = this._upsertDeviceRecord(userId, hello, sessionId);
|
|
173
185
|
const userMap = this._getUserMap(userId, true);
|
|
174
186
|
const existing = userMap.get(record.deviceId);
|
|
175
|
-
if (existing && existing.ws !== ws) {
|
|
176
|
-
existing.close('replaced by a newer desktop companion connection');
|
|
177
|
-
}
|
|
178
187
|
|
|
179
188
|
const connection = new DesktopCompanionConnection({
|
|
180
189
|
registry: this,
|
|
@@ -192,7 +201,16 @@ class DesktopCompanionRegistry {
|
|
|
192
201
|
},
|
|
193
202
|
timeoutMs: this.commandTimeoutMs,
|
|
194
203
|
});
|
|
204
|
+
// Install the new connection in the map BEFORE closing the old one.
|
|
205
|
+
// This ensures that when the old socket's async 'close' event fires and
|
|
206
|
+
// calls unregisterConnection, it sees the new connection as the owner and
|
|
207
|
+
// skips the DB status='offline' write — preventing a false offline report.
|
|
195
208
|
userMap.set(record.deviceId, connection);
|
|
209
|
+
|
|
210
|
+
if (existing && existing.ws !== ws) {
|
|
211
|
+
existing.close('replaced by a newer desktop companion connection');
|
|
212
|
+
}
|
|
213
|
+
|
|
196
214
|
return {
|
|
197
215
|
connection,
|
|
198
216
|
device: this.getDeviceRecordByDeviceId(userId, record.deviceId),
|
|
@@ -201,17 +219,21 @@ class DesktopCompanionRegistry {
|
|
|
201
219
|
|
|
202
220
|
unregisterConnection(connection) {
|
|
203
221
|
const userMap = this._getUserMap(connection.userId);
|
|
204
|
-
|
|
222
|
+
const isOwner = userMap != null && userMap.get(connection.deviceId) === connection;
|
|
223
|
+
if (isOwner) {
|
|
205
224
|
userMap.delete(connection.deviceId);
|
|
206
225
|
if (userMap.size === 0) {
|
|
207
226
|
this.connectionsByUser.delete(String(connection.userId));
|
|
208
227
|
}
|
|
228
|
+
// Only mark offline in the DB when this connection is still the active owner.
|
|
229
|
+
// If a newer connection has already taken over (reconnect race), its
|
|
230
|
+
// _upsertDeviceRecord already wrote status='online' and we must not clobber it.
|
|
231
|
+
this.db.prepare(
|
|
232
|
+
`UPDATE desktop_companion_devices
|
|
233
|
+
SET status = 'offline', updated_at = datetime('now')
|
|
234
|
+
WHERE user_id = ? AND device_id = ?`
|
|
235
|
+
).run(connection.userId, connection.deviceId);
|
|
209
236
|
}
|
|
210
|
-
this.db.prepare(
|
|
211
|
-
`UPDATE desktop_companion_devices
|
|
212
|
-
SET status = 'offline', updated_at = datetime('now')
|
|
213
|
-
WHERE user_id = ? AND device_id = ?`
|
|
214
|
-
).run(connection.userId, connection.deviceId);
|
|
215
237
|
}
|
|
216
238
|
|
|
217
239
|
touchConnection(userId, deviceId, patch = {}) {
|
|
@@ -253,6 +275,11 @@ class DesktopCompanionRegistry {
|
|
|
253
275
|
return this.getDeviceRecordByDeviceId(userId, deviceId);
|
|
254
276
|
}
|
|
255
277
|
|
|
278
|
+
isConnected(userId) {
|
|
279
|
+
const userMap = this._getUserMap(userId);
|
|
280
|
+
return userMap != null && userMap.size > 0;
|
|
281
|
+
}
|
|
282
|
+
|
|
256
283
|
getConnection(userId, deviceId) {
|
|
257
284
|
const userMap = this._getUserMap(userId);
|
|
258
285
|
if (!userMap) return null;
|
|
@@ -348,12 +375,58 @@ class DesktopCompanionRegistry {
|
|
|
348
375
|
};
|
|
349
376
|
}
|
|
350
377
|
|
|
378
|
+
async startStream(userId, deviceId, options = {}) {
|
|
379
|
+
const device = this.resolveDevice(userId, deviceId);
|
|
380
|
+
const connection = this.getConnection(userId, device.deviceId);
|
|
381
|
+
if (!connection || !connection.isOpen()) {
|
|
382
|
+
throw new DesktopCompanionUnavailableError();
|
|
383
|
+
}
|
|
384
|
+
const result = await connection.sendCommand(DESKTOP_COMMANDS.STREAM_START, {
|
|
385
|
+
fps: options.fps,
|
|
386
|
+
quality: options.quality,
|
|
387
|
+
displayId: options.displayId || device.activeDisplayId || null,
|
|
388
|
+
}, options);
|
|
389
|
+
connection._streaming = true;
|
|
390
|
+
return {
|
|
391
|
+
...result,
|
|
392
|
+
success: result?.success !== false,
|
|
393
|
+
deviceId: device.deviceId,
|
|
394
|
+
device: this.getDeviceRecordByDeviceId(userId, device.deviceId),
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
async stopStream(userId, deviceId) {
|
|
399
|
+
const device = this.resolveDevice(userId, deviceId);
|
|
400
|
+
const connection = this.getConnection(userId, device.deviceId);
|
|
401
|
+
if (!connection || !connection.isOpen()) {
|
|
402
|
+
throw new DesktopCompanionUnavailableError();
|
|
403
|
+
}
|
|
404
|
+
const result = await connection.sendCommand(DESKTOP_COMMANDS.STREAM_STOP, {});
|
|
405
|
+
connection._streaming = false;
|
|
406
|
+
return {
|
|
407
|
+
...result,
|
|
408
|
+
success: result?.success !== false,
|
|
409
|
+
deviceId: device.deviceId,
|
|
410
|
+
device: this.getDeviceRecordByDeviceId(userId, device.deviceId),
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
351
414
|
getStatus(userId) {
|
|
352
415
|
const devices = this.listDevices(userId);
|
|
353
416
|
const onlineDevices = devices.filter((device) => device.online);
|
|
417
|
+
let selectedDeviceId = this.getSelectedDeviceId(userId);
|
|
418
|
+
|
|
419
|
+
// Auto-select the most-recently-online device when there is no valid selection.
|
|
420
|
+
// listDevices returns online devices first, ordered by last_seen_at DESC.
|
|
421
|
+
const selectionIsOnline = selectedDeviceId && onlineDevices.some((d) => d.deviceId === selectedDeviceId);
|
|
422
|
+
if (!selectionIsOnline && onlineDevices.length > 0) {
|
|
423
|
+
selectedDeviceId = onlineDevices[0].deviceId;
|
|
424
|
+
this.setSelectedDeviceId(userId, selectedDeviceId);
|
|
425
|
+
}
|
|
426
|
+
|
|
354
427
|
return {
|
|
355
428
|
connected: onlineDevices.length > 0,
|
|
356
|
-
selectedDeviceId
|
|
429
|
+
selectedDeviceId,
|
|
357
430
|
onlineCount: onlineDevices.length,
|
|
358
431
|
devices,
|
|
359
432
|
};
|
|
@@ -446,6 +519,8 @@ class DesktopCompanionConnection {
|
|
|
446
519
|
this.ws.close(1000, String(reason || 'closing').slice(0, 120));
|
|
447
520
|
}
|
|
448
521
|
} catch {}
|
|
522
|
+
// unregisterConnection is intentionally called here (not inside _closePending)
|
|
523
|
+
// so it runs synchronously before any async ws 'close' event can fire.
|
|
449
524
|
this.registry.unregisterConnection(this);
|
|
450
525
|
this._closePending(new DesktopCompanionUnavailableError('Desktop companion disconnected.'));
|
|
451
526
|
}
|
|
@@ -474,6 +549,9 @@ class DesktopCompanionConnection {
|
|
|
474
549
|
}
|
|
475
550
|
|
|
476
551
|
_handleMessage(data) {
|
|
552
|
+
if (Buffer.isBuffer(data) && data.length > 0 && data[0] === FRAME_TYPE_VIDEO) {
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
477
555
|
let message;
|
|
478
556
|
try {
|
|
479
557
|
message = parseDesktopMessage(data);
|
|
@@ -512,6 +590,10 @@ class DesktopCompanionConnection {
|
|
|
512
590
|
pending.reject(error);
|
|
513
591
|
}
|
|
514
592
|
this.pending.clear();
|
|
593
|
+
// unregisterConnection is idempotent (ownership-checked) so calling it
|
|
594
|
+
// here is safe whether we arrived via close(), the ws 'close' event, or
|
|
595
|
+
// both. It ensures natural socket drops (no explicit close() call) still
|
|
596
|
+
// mark the device offline.
|
|
515
597
|
this.registry.unregisterConnection(this);
|
|
516
598
|
}
|
|
517
599
|
}
|