neoagent 2.4.0 → 2.4.1-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE +619 -21
  2. package/README.md +1 -1
  3. package/extensions/chrome-browser/background.mjs +19 -7
  4. package/extensions/chrome-browser/icons/icon128.png +0 -0
  5. package/extensions/chrome-browser/icons/icon16.png +0 -0
  6. package/extensions/chrome-browser/icons/icon48.png +0 -0
  7. package/extensions/chrome-browser/icons/logo.svg +12 -0
  8. package/extensions/chrome-browser/manifest.json +13 -2
  9. package/extensions/chrome-browser/popup.css +5 -0
  10. package/extensions/chrome-browser/popup.html +7 -5
  11. package/extensions/chrome-browser/popup.js +16 -7
  12. package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +721 -0
  13. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
  14. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
  15. package/flutter_app/lib/main.dart +1 -0
  16. package/flutter_app/lib/main_controller.dart +156 -3
  17. package/flutter_app/lib/main_devices.dart +485 -119
  18. package/flutter_app/lib/main_settings.dart +289 -30
  19. package/flutter_app/lib/src/backend_client.dart +89 -0
  20. package/flutter_app/lib/src/desktop_companion_actions.dart +153 -3
  21. package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
  22. package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
  23. package/flutter_app/lib/src/stream_renderer.dart +286 -0
  24. package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
  25. package/package.json +2 -2
  26. package/server/guest_agent.js +19 -1
  27. package/server/http/routes.js +191 -0
  28. package/server/http/socket.js +1 -1
  29. package/server/index.js +4 -1
  30. package/server/public/.last_build_id +1 -1
  31. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  32. package/server/public/flutter_bootstrap.js +1 -1
  33. package/server/public/main.dart.js +75438 -74005
  34. package/server/routes/browser.js +14 -0
  35. package/server/routes/browser_extension.js +21 -4
  36. package/server/routes/desktop.js +10 -0
  37. package/server/routes/settings.js +4 -0
  38. package/server/routes/stream.js +187 -0
  39. package/server/services/ai/tools.js +40 -29
  40. package/server/services/android/controller.js +41 -2
  41. package/server/services/browser/controller.js +34 -0
  42. package/server/services/browser/extension/manifest.js +33 -0
  43. package/server/services/browser/extension/provider.js +12 -6
  44. package/server/services/browser/extension/registry.js +188 -18
  45. package/server/services/desktop/gateway.js +28 -3
  46. package/server/services/desktop/protocol.js +34 -0
  47. package/server/services/desktop/provider.js +25 -0
  48. package/server/services/desktop/registry.js +92 -10
  49. package/server/services/manager.js +19 -2
  50. package/server/services/runtime/backends/local-vm.js +6 -0
  51. package/server/services/runtime/docker-vm-manager.js +26 -3
  52. package/server/services/runtime/manager.js +36 -5
  53. package/server/services/runtime/settings.js +17 -0
  54. package/server/services/streaming/android-stream.js +298 -0
  55. package/server/services/streaming/browser-stream.js +87 -0
  56. package/server/services/streaming/stream-hub.js +231 -0
  57. package/server/services/websocket.js +73 -0
@@ -0,0 +1,286 @@
1
+ import 'dart:async';
2
+ import 'dart:typed_data';
3
+
4
+ import 'package:flutter/material.dart';
5
+ import 'package:socket_io_client/socket_io_client.dart' as io;
6
+
7
+ class StreamRenderer extends StatefulWidget {
8
+ const StreamRenderer({
9
+ super.key,
10
+ required this.socket,
11
+ required this.deviceId,
12
+ required this.platform,
13
+ this.remoteResolution,
14
+ this.onTap,
15
+ this.onSwipe,
16
+ this.onType,
17
+ this.onHover,
18
+ this.fit = BoxFit.contain,
19
+ this.alignment = Alignment.center,
20
+ });
21
+
22
+ final io.Socket socket;
23
+ final String deviceId;
24
+ final String platform;
25
+ final Size? remoteResolution;
26
+ final void Function(double x, double y)? onTap;
27
+ final void Function(double x1, double y1, double x2, double y2)? onSwipe;
28
+ final void Function(String text)? onType;
29
+ final void Function(double x, double y)? onHover;
30
+ final BoxFit fit;
31
+ final Alignment alignment;
32
+
33
+ @override
34
+ State<StreamRenderer> createState() => _StreamRendererState();
35
+ }
36
+
37
+ class _StreamRendererState extends State<StreamRenderer> {
38
+ Uint8List? _frame;
39
+ Size? _frameSize;
40
+ ImageStream? _imageStream;
41
+ ImageStreamListener? _imageListener;
42
+ Offset? _dragStart;
43
+ Offset? _dragEnd;
44
+
45
+ Timer? _hoverThrottleTimer;
46
+ Offset? _pendingHoverOffset;
47
+ DateTime _lastHoverTime = DateTime.fromMillisecondsSinceEpoch(0);
48
+
49
+ @override
50
+ void initState() {
51
+ super.initState();
52
+ widget.socket.on('stream:frame', _onFrame);
53
+ widget.socket.emit('stream:subscribe', <String, Object?>{
54
+ 'deviceId': widget.deviceId,
55
+ 'platform': widget.platform,
56
+ });
57
+ }
58
+
59
+ @override
60
+ void didUpdateWidget(StreamRenderer oldWidget) {
61
+ super.didUpdateWidget(oldWidget);
62
+ if (oldWidget.socket == widget.socket &&
63
+ oldWidget.deviceId == widget.deviceId &&
64
+ oldWidget.platform == widget.platform) {
65
+ return;
66
+ }
67
+ _frameSize = null;
68
+ _detachImageListener();
69
+ oldWidget.socket.off('stream:frame', _onFrame);
70
+ oldWidget.socket.emit('stream:unsubscribe', <String, Object?>{
71
+ 'deviceId': oldWidget.deviceId,
72
+ 'platform': oldWidget.platform,
73
+ });
74
+ widget.socket.on('stream:frame', _onFrame);
75
+ widget.socket.emit('stream:subscribe', <String, Object?>{
76
+ 'deviceId': widget.deviceId,
77
+ 'platform': widget.platform,
78
+ });
79
+ }
80
+
81
+ void _onFrame(dynamic data) {
82
+ Object? meta;
83
+ Object? bytes;
84
+ if (data is List && data.length >= 2) {
85
+ meta = data[0];
86
+ bytes = data[1];
87
+ }
88
+ if (meta is Map) {
89
+ final frameDeviceId = meta['deviceId']?.toString() ?? '';
90
+ final framePlatform = meta['platform']?.toString() ?? '';
91
+ if (frameDeviceId.isNotEmpty && frameDeviceId != widget.deviceId) {
92
+ return;
93
+ }
94
+ if (framePlatform.isNotEmpty && framePlatform != widget.platform) {
95
+ return;
96
+ }
97
+ }
98
+ final frame = switch (bytes) {
99
+ Uint8List value => value,
100
+ List<int> value => Uint8List.fromList(value),
101
+ ByteBuffer value => Uint8List.view(value),
102
+ _ => null,
103
+ };
104
+ if (frame == null || frame.isEmpty || !mounted) return;
105
+ if (_frameSize == null) {
106
+ _resolveFrameSize(frame);
107
+ }
108
+ setState(() => _frame = frame);
109
+ }
110
+
111
+ void _resolveFrameSize(Uint8List frame) {
112
+ _detachImageListener();
113
+ final provider = MemoryImage(frame);
114
+ final stream = provider.resolve(const ImageConfiguration());
115
+ final listener = ImageStreamListener((image, _) {
116
+ if (!mounted || _frameSize != null) return;
117
+ setState(() {
118
+ _frameSize = Size(
119
+ image.image.width.toDouble(),
120
+ image.image.height.toDouble(),
121
+ );
122
+ });
123
+ _detachImageListener();
124
+ });
125
+ _imageStream = stream;
126
+ _imageListener = listener;
127
+ stream.addListener(listener);
128
+ }
129
+
130
+ void _detachImageListener() {
131
+ if (_imageStream != null && _imageListener != null) {
132
+ _imageStream!.removeListener(_imageListener!);
133
+ }
134
+ _imageStream = null;
135
+ _imageListener = null;
136
+ }
137
+
138
+ @override
139
+ Widget build(BuildContext context) {
140
+ final frame = _frame;
141
+ if (frame == null) {
142
+ return const Center(child: CircularProgressIndicator());
143
+ }
144
+ return LayoutBuilder(
145
+ builder: (context, constraints) {
146
+ return MouseRegion(
147
+ onHover: widget.onHover == null
148
+ ? null
149
+ : (event) => _handleHoverEvent(event.localPosition, constraints.biggest),
150
+ child: GestureDetector(
151
+ behavior: HitTestBehavior.opaque,
152
+ onTapDown: widget.onTap == null
153
+ ? null
154
+ : (details) => _handleTap(details, constraints.biggest),
155
+ onPanStart: widget.onSwipe == null
156
+ ? null
157
+ : (details) {
158
+ _dragStart = details.localPosition;
159
+ _dragEnd = details.localPosition;
160
+ },
161
+ onPanUpdate: widget.onSwipe == null
162
+ ? null
163
+ : (details) {
164
+ _dragEnd = details.localPosition;
165
+ },
166
+ onPanEnd: widget.onSwipe == null
167
+ ? null
168
+ : (_) {
169
+ final start = _dragStart;
170
+ final end = _dragEnd;
171
+ _dragStart = null;
172
+ _dragEnd = null;
173
+ if (start == null ||
174
+ end == null ||
175
+ (start - end).distance < 12) {
176
+ return;
177
+ }
178
+ _handleSwipe(start, end, constraints.biggest);
179
+ },
180
+ child: Image.memory(
181
+ frame,
182
+ gaplessPlayback: true,
183
+ fit: widget.fit,
184
+ alignment: widget.alignment,
185
+ width: constraints.maxWidth,
186
+ height: constraints.maxHeight,
187
+ ),
188
+ ),
189
+ );
190
+ },
191
+ );
192
+ }
193
+
194
+ void _handleHoverEvent(Offset localPosition, Size boxSize) {
195
+ if (widget.onHover == null) return;
196
+ _pendingHoverOffset = localPosition;
197
+ final now = DateTime.now();
198
+ final elapsed = now.difference(_lastHoverTime);
199
+ const throttleDuration = Duration(milliseconds: 70);
200
+
201
+ if (elapsed >= throttleDuration) {
202
+ _sendPendingHover(boxSize);
203
+ } else {
204
+ _hoverThrottleTimer ??= Timer(throttleDuration - elapsed, () {
205
+ _hoverThrottleTimer = null;
206
+ _sendPendingHover(boxSize);
207
+ });
208
+ }
209
+ }
210
+
211
+ void _sendPendingHover(Size boxSize) {
212
+ final offset = _pendingHoverOffset;
213
+ if (offset == null) return;
214
+ _pendingHoverOffset = null;
215
+ _lastHoverTime = DateTime.now();
216
+
217
+ final point = _mapToRemote(offset, boxSize);
218
+ if (point != null) {
219
+ widget.onHover?.call(point.dx, point.dy);
220
+ }
221
+ }
222
+
223
+ void _handleTap(TapDownDetails details, Size boxSize) {
224
+ final point = _mapToRemote(details.localPosition, boxSize);
225
+ if (point == null) {
226
+ return;
227
+ }
228
+ widget.onTap?.call(point.dx, point.dy);
229
+ }
230
+
231
+ void _handleSwipe(Offset start, Offset end, Size boxSize) {
232
+ final mappedStart = _mapToRemote(start, boxSize);
233
+ final mappedEnd = _mapToRemote(end, boxSize);
234
+ if (mappedStart == null || mappedEnd == null) {
235
+ return;
236
+ }
237
+ widget.onSwipe?.call(
238
+ mappedStart.dx,
239
+ mappedStart.dy,
240
+ mappedEnd.dx,
241
+ mappedEnd.dy,
242
+ );
243
+ }
244
+
245
+ Offset? _mapToRemote(Offset localPosition, Size boxSize) {
246
+ final remote = widget.remoteResolution ?? _frameSize;
247
+ if (remote == null ||
248
+ remote.width <= 0 ||
249
+ remote.height <= 0 ||
250
+ boxSize.width <= 0 ||
251
+ boxSize.height <= 0) {
252
+ return null;
253
+ }
254
+ final remoteSize = Size(remote.width, remote.height);
255
+ final fitted = applyBoxFit(widget.fit, remoteSize, boxSize);
256
+ final sourceRect = widget.alignment.inscribe(
257
+ fitted.source,
258
+ Offset.zero & remoteSize,
259
+ );
260
+ final destRect = widget.alignment.inscribe(
261
+ fitted.destination,
262
+ Offset.zero & boxSize,
263
+ );
264
+ if (!destRect.contains(localPosition)) {
265
+ return null;
266
+ }
267
+ final localX = localPosition.dx - destRect.left;
268
+ final localY = localPosition.dy - destRect.top;
269
+ return Offset(
270
+ sourceRect.left + localX * fitted.source.width / destRect.width,
271
+ sourceRect.top + localY * fitted.source.height / destRect.height,
272
+ );
273
+ }
274
+
275
+ @override
276
+ void dispose() {
277
+ _hoverThrottleTimer?.cancel();
278
+ widget.socket.emit('stream:unsubscribe', <String, Object?>{
279
+ 'deviceId': widget.deviceId,
280
+ 'platform': widget.platform,
281
+ });
282
+ widget.socket.off('stream:frame', _onFrame);
283
+ _detachImageListener();
284
+ super.dispose();
285
+ }
286
+ }
@@ -106,6 +106,36 @@ final class DesktopCompanionNativePlugin: NSObject {
106
106
  displayId: displayId
107
107
  )
108
108
  result(nil)
109
+ case "mouseMove":
110
+ guard isAccessibilityTrusted() else {
111
+ result(
112
+ FlutterError(
113
+ code: "accessibility_permission_denied",
114
+ message: "Accessibility permission is required for input control.",
115
+ details: nil
116
+ )
117
+ )
118
+ return
119
+ }
120
+ guard let arguments = call.arguments as? [String: Any],
121
+ let x = arguments["x"] as? NSNumber,
122
+ let y = arguments["y"] as? NSNumber else {
123
+ result(
124
+ FlutterError(
125
+ code: "invalid_arguments",
126
+ message: "Missing mouseMove coordinates.",
127
+ details: nil
128
+ )
129
+ )
130
+ return
131
+ }
132
+ let displayId = arguments["displayId"] as? String
133
+ performMouseMove(
134
+ x: x.doubleValue,
135
+ y: y.doubleValue,
136
+ displayId: displayId
137
+ )
138
+ result(nil)
109
139
  case "drag":
110
140
  guard isAccessibilityTrusted() else {
111
141
  result(
@@ -316,7 +346,18 @@ final class DesktopCompanionNativePlugin: NSObject {
316
346
  }
317
347
 
318
348
  private func isAccessibilityTrusted() -> Bool {
319
- AXIsProcessTrusted()
349
+ if AXIsProcessTrusted() { return true }
350
+ // AXIsProcessTrusted() may cache false on macOS 14+ after a System Settings grant.
351
+ // Probe with a live AX read: .apiDisabled is the error returned when the process
352
+ // lacks accessibility permission (AXError has no .notTrusted case in the macOS SDK).
353
+ let sysElement = AXUIElementCreateSystemWide()
354
+ var value: CFTypeRef?
355
+ let status = AXUIElementCopyAttributeValue(
356
+ sysElement,
357
+ kAXFocusedApplicationAttribute as CFString,
358
+ &value
359
+ )
360
+ return status != .apiDisabled
320
361
  }
321
362
 
322
363
  private func resolveDisplayId(_ raw: String?) -> CGDirectDisplayID {
@@ -406,6 +447,20 @@ final class DesktopCompanionNativePlugin: NSObject {
406
447
  up.post(tap: .cghidEventTap)
407
448
  }
408
449
 
450
+ private func performMouseMove(x: Double, y: Double, displayId: String?) {
451
+ let point = nativePointForCapturedPixel(x: x, y: y, displayId: displayId)
452
+ CGWarpMouseCursorPosition(point)
453
+ guard let moveEvent = CGEvent(
454
+ mouseEventSource: nil,
455
+ mouseType: .mouseMoved,
456
+ mouseCursorPosition: point,
457
+ mouseButton: .left
458
+ ) else {
459
+ return
460
+ }
461
+ moveEvent.post(tap: .cghidEventTap)
462
+ }
463
+
409
464
  private func performDrag(
410
465
  x1: Double,
411
466
  y1: Double,
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "neoagent",
3
- "version": "2.4.0",
3
+ "version": "2.4.1-beta.11",
4
4
  "description": "Proactive personal AI agent with no limits",
5
- "license": "MIT",
5
+ "license": "AGPL-3.0-only",
6
6
  "main": "server/index.js",
7
7
  "engines": {
8
8
  "node": ">=20"
@@ -62,7 +62,18 @@ function isInsideAllowedRoots(targetPath) {
62
62
  }
63
63
 
64
64
  function requireToken(req, res, next) {
65
- next();
65
+ if (!AUTH_TOKEN) {
66
+ // Token not configured in this environment — allow but unauthenticated.
67
+ // Pass NEOAGENT_VM_GUEST_TOKEN to the container to enforce auth.
68
+ return next();
69
+ }
70
+ const header = String(req.headers?.authorization || '').trim();
71
+ const prefix = 'Bearer ';
72
+ const provided = header.startsWith(prefix) ? header.slice(prefix.length).trim() : '';
73
+ if (!provided || provided !== AUTH_TOKEN) {
74
+ return res.status(401).json({ error: 'Unauthorized.' });
75
+ }
76
+ return next();
66
77
  }
67
78
 
68
79
  function sanitizeError(err) {
@@ -184,6 +195,13 @@ function requireCapability(controller, name) {
184
195
  app.post('/browser/launch', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').launch(req.body || {})));
185
196
  app.post('/browser/navigate', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').navigate(req.body?.url, req.body || {})));
186
197
  app.post('/browser/screenshot', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').screenshot(req.body || {})));
198
+ app.post('/browser/screenshot-jpeg', async (req, res) => handle(res, async () => {
199
+ const jpeg = await requireCapability(browserController, 'browser').screenshotJpeg(req.body?.quality, req.body || {});
200
+ return {
201
+ contentType: 'image/jpeg',
202
+ contentBase64: Buffer.from(jpeg).toString('base64'),
203
+ };
204
+ }));
187
205
  app.post('/browser/click', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
188
206
  app.post('/browser/click-point', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
189
207
  app.post('/browser/fill', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));
@@ -25,6 +25,7 @@ const routeRegistry = [
25
25
  { basePath: '/api/browser-extension', modulePath: '../routes/browser_extension' },
26
26
  { basePath: '/api/android', modulePath: '../routes/android' },
27
27
  { basePath: '/api/desktop', modulePath: '../routes/desktop' },
28
+ { basePath: '/api/stream', modulePath: '../routes/stream' },
28
29
  { basePath: '/api/recordings', modulePath: '../routes/recordings' },
29
30
  { basePath: '/api/social-video', modulePath: '../routes/social_video' },
30
31
  { basePath: '/api/voice-assistant', modulePath: '../routes/voice_assistant' },
@@ -67,6 +68,196 @@ function registerApiRoutes(app) {
67
68
  });
68
69
  });
69
70
 
71
+ app.get('/api/system/health-check', requireAuth, async (req, res) => {
72
+ const userId = req.session?.userId;
73
+ const runtimeManager = req.app?.locals?.runtimeManager;
74
+ const desktopRegistry = req.app?.locals?.desktopCompanionRegistry;
75
+ const extensionRegistry = req.app?.locals?.browserExtensionRegistry;
76
+ const results = [];
77
+
78
+ // 1. Backend connectivity — trivially true if we got here.
79
+ results.push({ id: 'backend', label: 'Backend server', passed: true, detail: 'Reachable' });
80
+
81
+ // 2. Cloud VM runtime availability.
82
+ const runtimeValidation = getRuntimeValidation(runtimeManager);
83
+ const runtimeReady = Boolean(runtimeValidation?.ready);
84
+ results.push({
85
+ id: 'vm_runtime',
86
+ label: 'Cloud VM runtime',
87
+ passed: runtimeReady,
88
+ detail: runtimeReady ? 'Available' : String(runtimeValidation?.issues?.[0] || 'Not configured'),
89
+ });
90
+
91
+ // 3. Cloud VM CLI execution — actually run a command.
92
+ if (runtimeManager && typeof runtimeManager.executeCommand === 'function') {
93
+ try {
94
+ const cmdResult = await runtimeManager.executeCommand(userId, 'echo "health_check_ok"', { timeout: 15000 });
95
+ const exitOk = cmdResult?.exitCode === 0;
96
+ const outputOk = String(cmdResult?.stdout || '').includes('health_check_ok');
97
+ results.push({
98
+ id: 'vm_cli',
99
+ label: 'Cloud VM — command execution',
100
+ passed: exitOk && outputOk,
101
+ detail: exitOk && outputOk
102
+ ? 'Commands running'
103
+ : `Exit ${cmdResult?.exitCode ?? '?'}: ${String(cmdResult?.stderr || cmdResult?.stdout || '').slice(0, 120)}`,
104
+ });
105
+ } catch (err) {
106
+ results.push({ id: 'vm_cli', label: 'Cloud VM — command execution', passed: false, detail: String(err?.message || err).slice(0, 120) });
107
+ }
108
+ } else {
109
+ results.push({ id: 'vm_cli', label: 'Cloud VM — command execution', passed: false, detail: 'VM runtime unavailable' });
110
+ }
111
+
112
+ // 4. Desktop companion (macOS app / remote device) connectivity + permissions.
113
+ if (desktopRegistry) {
114
+ try {
115
+ const desktopStatus = desktopRegistry.getStatus(userId);
116
+ const connected = Boolean(desktopStatus?.connected);
117
+ results.push({
118
+ id: 'desktop_connected',
119
+ label: 'Desktop companion',
120
+ passed: connected,
121
+ detail: connected
122
+ ? `${desktopStatus.onlineCount} device${desktopStatus.onlineCount !== 1 ? 's' : ''} connected`
123
+ : 'No device connected — open the desktop app',
124
+ });
125
+
126
+ if (connected && Array.isArray(desktopStatus?.devices)) {
127
+ const onlineDevice = desktopStatus.devices.find((d) => d.online && !d.revokedAt);
128
+ const perms = onlineDevice?.permissions || {};
129
+ const screenOk = Boolean(perms.screenCapture || perms.screen_capture);
130
+ const inputOk = Boolean(perms.accessibility || perms.inputControl || perms.input_control);
131
+ results.push({
132
+ id: 'desktop_screen',
133
+ label: 'Desktop — screen capture',
134
+ passed: screenOk,
135
+ detail: screenOk ? 'Granted' : 'Not granted — open System Settings › Privacy › Screen Recording',
136
+ });
137
+ results.push({
138
+ id: 'desktop_input',
139
+ label: 'Desktop — input control',
140
+ passed: inputOk,
141
+ detail: inputOk ? 'Granted' : 'Not granted — open System Settings › Privacy › Accessibility',
142
+ });
143
+ }
144
+ } catch (err) {
145
+ results.push({ id: 'desktop_connected', label: 'Desktop companion', passed: false, detail: String(err?.message || err).slice(0, 120) });
146
+ }
147
+ }
148
+
149
+ // 5. Chrome extension connectivity.
150
+ if (extensionRegistry) {
151
+ try {
152
+ const extStatus = extensionRegistry.getStatus(userId);
153
+ const extConnected = Boolean(extStatus?.connected);
154
+ results.push({
155
+ id: 'chrome_extension',
156
+ label: 'Chrome extension',
157
+ passed: extConnected,
158
+ detail: extConnected ? 'Connected' : 'Not connected — install the NeoAgent extension in Chrome',
159
+ });
160
+ } catch (err) {
161
+ results.push({ id: 'chrome_extension', label: 'Chrome extension', passed: false, detail: String(err?.message || err).slice(0, 120) });
162
+ }
163
+ }
164
+
165
+ const allPassed = results.every((r) => r.passed);
166
+ res.json({ passed: allPassed, results });
167
+ });
168
+
169
+ // Targeted runtime self-tests — one check per endpoint so the UI can embed
170
+ // results inline next to the relevant settings control.
171
+
172
+ app.get('/api/system/test/cli', requireAuth, async (req, res) => {
173
+ const userId = req.session?.userId;
174
+ const runtimeManager = req.app?.locals?.runtimeManager;
175
+ if (!runtimeManager || typeof runtimeManager.executeCommand !== 'function') {
176
+ return res.json({ passed: false, backendUsed: 'vm', detail: 'Runtime not configured on this server.' });
177
+ }
178
+ // Note: executeCommand always routes through the VM backend regardless of
179
+ // the cli_backend setting — desktop CLI routing is not yet implemented.
180
+ try {
181
+ const result = await runtimeManager.executeCommand(userId, 'echo "cli_test_ok"', { timeout: 15000 });
182
+ const exitOk = result?.exitCode === 0;
183
+ const outputOk = String(result?.stdout || '').includes('cli_test_ok');
184
+ return res.json({
185
+ passed: exitOk && outputOk,
186
+ backendUsed: 'vm',
187
+ detail: exitOk && outputOk
188
+ ? 'Command executed successfully'
189
+ : `Exit ${result?.exitCode ?? '?'}: ${String(result?.stderr || result?.stdout || '').slice(0, 120)}`,
190
+ });
191
+ } catch (err) {
192
+ return res.json({ passed: false, backendUsed: 'vm', detail: String(err?.message || err).slice(0, 120) });
193
+ }
194
+ });
195
+
196
+ app.get('/api/system/test/extension', requireAuth, (req, res) => {
197
+ const userId = req.session?.userId;
198
+ const extensionRegistry = req.app?.locals?.browserExtensionRegistry;
199
+ if (!extensionRegistry) {
200
+ return res.json({ passed: false, detail: 'Extension registry not available on this server.' });
201
+ }
202
+ try {
203
+ const status = extensionRegistry.getStatus(userId);
204
+ const connected = Boolean(status?.connected);
205
+ return res.json({
206
+ passed: connected,
207
+ detail: connected ? 'Extension is connected and live' : 'Extension is not connected',
208
+ tokenId: status?.activeTokenId || null,
209
+ meta: status?.connectedMeta || null,
210
+ });
211
+ } catch (err) {
212
+ return res.json({ passed: false, detail: String(err?.message || err).slice(0, 120) });
213
+ }
214
+ });
215
+
216
+ app.get('/api/system/test/desktop', requireAuth, (req, res) => {
217
+ const userId = req.session?.userId;
218
+ const desktopRegistry = req.app?.locals?.desktopCompanionRegistry;
219
+ if (!desktopRegistry) {
220
+ return res.json({ passed: false, detail: 'Desktop registry not available on this server.' });
221
+ }
222
+ try {
223
+ const status = desktopRegistry.getStatus(userId);
224
+ const connected = Boolean(status?.connected);
225
+ const devices = Array.isArray(status?.devices)
226
+ ? status.devices.filter((d) => d.online && !d.revokedAt)
227
+ : [];
228
+ const selected = status?.selectedDeviceId || null;
229
+ const activeDevice = selected
230
+ ? devices.find((d) => d.deviceId === selected)
231
+ : devices.length === 1 ? devices[0] : null;
232
+ const perms = activeDevice?.permissions || {};
233
+ const screenOk = Boolean(perms.screenCapture || perms.screen_capture);
234
+ const inputOk = Boolean(perms.accessibility || perms.inputControl || perms.input_control);
235
+ return res.json({
236
+ passed: connected,
237
+ connected,
238
+ onlineCount: devices.length,
239
+ selectedDeviceId: selected,
240
+ activeDevice: activeDevice ? {
241
+ deviceId: activeDevice.deviceId,
242
+ label: activeDevice.label || activeDevice.hostname || activeDevice.deviceId,
243
+ platform: activeDevice.platform || null,
244
+ paused: activeDevice.paused || false,
245
+ permissions: { screenCapture: screenOk, inputControl: inputOk },
246
+ } : null,
247
+ multipleOnline: devices.length > 1 && !activeDevice,
248
+ detail: !connected
249
+ ? 'No device connected'
250
+ : devices.length > 1 && !activeDevice
251
+ ? `${devices.length} devices online — select one in Desktop settings`
252
+ : activeDevice?.paused
253
+ ? `${activeDevice.label || 'Device'} is paused`
254
+ : `${activeDevice?.label || 'Device'} connected`,
255
+ });
256
+ } catch (err) {
257
+ return res.json({ passed: false, detail: String(err?.message || err).slice(0, 120) });
258
+ }
259
+ });
260
+
70
261
  app.get('/api/version', requireAuth, (req, res) => {
71
262
  res.json(getVersionInfo());
72
263
  });
@@ -8,7 +8,7 @@ function createSocketServer(httpServer, { validateOrigin }) {
8
8
  pingInterval: Number(process.env.NEOAGENT_SOCKET_PING_INTERVAL_MS || 25000),
9
9
  pingTimeout: Number(process.env.NEOAGENT_SOCKET_PING_TIMEOUT_MS || 20000),
10
10
  connectTimeout: Number(process.env.NEOAGENT_SOCKET_CONNECT_TIMEOUT_MS || 10000),
11
- maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES || 1_000_000),
11
+ maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES || 8 * 1024 * 1024),
12
12
  cors: {
13
13
  origin(origin, callback) {
14
14
  return validateOrigin(origin, callback, { allowMissingOrigin: true });
package/server/index.js CHANGED
@@ -34,6 +34,7 @@ const { startServices, stopServices } = require('./services/manager');
34
34
  const { bindBrowserExtensionGateway } = require('./services/browser/extension/gateway');
35
35
  const { bindDesktopCompanionGateway } = require('./services/desktop/gateway');
36
36
  const { bindWearableGateway } = require('./services/wearable/gateway');
37
+ const { StreamHub } = require('./services/streaming/stream-hub');
37
38
 
38
39
  function parseBooleanFlag(value, fallback = false) {
39
40
  const normalized = String(value || '').trim().toLowerCase();
@@ -89,6 +90,8 @@ const app = express();
89
90
  app.disable('x-powered-by');
90
91
  const httpServer = createServer(app);
91
92
  const io = createSocketServer(httpServer, { validateOrigin });
93
+ const streamHub = new StreamHub(io);
94
+ app.locals.streamHub = streamHub;
92
95
  app.locals.httpRuntimeConfig = {
93
96
  secureCookies: SECURE_COOKIES,
94
97
  trustProxy: TRUST_PROXY,
@@ -112,7 +115,7 @@ registerApiRoutes(app);
112
115
  registerStaticRoutes(app);
113
116
  registerErrorHandler(app);
114
117
  bindBrowserExtensionGateway(httpServer, app);
115
- bindDesktopCompanionGateway(httpServer, app, sessionMiddleware);
118
+ bindDesktopCompanionGateway(httpServer, app, sessionMiddleware, streamHub);
116
119
  bindWearableGateway(httpServer, app, sessionMiddleware);
117
120
 
118
121
  let shuttingDown = false;
@@ -1 +1 @@
1
- 18f90bb9a4d3bcea22e501f688e05b77
1
+ 178383462d4c7125ce032b3783b45564
@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"4c525dac5ebe5971c5708ef73558ed8edcf4a3
37
37
 
38
38
  _flutter.loader.load({
39
39
  serviceWorkerSettings: {
40
- serviceWorkerVersion: "1600220817" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
40
+ serviceWorkerVersion: "559469558" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
41
41
  }
42
42
  });