neoagent 2.4.0 → 2.4.1-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +619 -21
- package/README.md +1 -1
- package/extensions/chrome-browser/background.mjs +19 -7
- package/extensions/chrome-browser/icons/icon128.png +0 -0
- package/extensions/chrome-browser/icons/icon16.png +0 -0
- package/extensions/chrome-browser/icons/icon48.png +0 -0
- package/extensions/chrome-browser/icons/logo.svg +12 -0
- package/extensions/chrome-browser/manifest.json +13 -2
- package/extensions/chrome-browser/popup.css +5 -0
- package/extensions/chrome-browser/popup.html +7 -5
- package/extensions/chrome-browser/popup.js +16 -7
- package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +721 -0
- package/flutter_app/lib/features/onboarding/onboarding_shell.dart +6 -0
- package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +1 -1
- package/flutter_app/lib/main.dart +1 -0
- package/flutter_app/lib/main_controller.dart +156 -3
- package/flutter_app/lib/main_devices.dart +485 -119
- package/flutter_app/lib/main_settings.dart +289 -30
- package/flutter_app/lib/src/backend_client.dart +89 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +153 -3
- package/flutter_app/lib/src/desktop_companion_io.dart +145 -4
- package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
- package/flutter_app/lib/src/stream_renderer.dart +286 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +56 -1
- package/package.json +2 -2
- package/server/guest_agent.js +19 -1
- package/server/http/routes.js +191 -0
- package/server/http/socket.js +1 -1
- package/server/index.js +4 -1
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +75438 -74005
- package/server/routes/browser.js +14 -0
- package/server/routes/browser_extension.js +21 -4
- package/server/routes/desktop.js +10 -0
- package/server/routes/settings.js +4 -0
- package/server/routes/stream.js +187 -0
- package/server/services/ai/tools.js +40 -29
- package/server/services/android/controller.js +41 -2
- package/server/services/browser/controller.js +34 -0
- package/server/services/browser/extension/manifest.js +33 -0
- package/server/services/browser/extension/provider.js +12 -6
- package/server/services/browser/extension/registry.js +188 -18
- package/server/services/desktop/gateway.js +28 -3
- package/server/services/desktop/protocol.js +34 -0
- package/server/services/desktop/provider.js +25 -0
- package/server/services/desktop/registry.js +92 -10
- package/server/services/manager.js +19 -2
- package/server/services/runtime/backends/local-vm.js +6 -0
- package/server/services/runtime/docker-vm-manager.js +26 -3
- package/server/services/runtime/manager.js +36 -5
- package/server/services/runtime/settings.js +17 -0
- package/server/services/streaming/android-stream.js +298 -0
- package/server/services/streaming/browser-stream.js +87 -0
- package/server/services/streaming/stream-hub.js +231 -0
- package/server/services/websocket.js +73 -0
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import 'dart:async';
|
|
2
|
+
import 'dart:typed_data';
|
|
3
|
+
|
|
4
|
+
import 'package:flutter/material.dart';
|
|
5
|
+
import 'package:socket_io_client/socket_io_client.dart' as io;
|
|
6
|
+
|
|
7
|
+
class StreamRenderer extends StatefulWidget {
|
|
8
|
+
const StreamRenderer({
|
|
9
|
+
super.key,
|
|
10
|
+
required this.socket,
|
|
11
|
+
required this.deviceId,
|
|
12
|
+
required this.platform,
|
|
13
|
+
this.remoteResolution,
|
|
14
|
+
this.onTap,
|
|
15
|
+
this.onSwipe,
|
|
16
|
+
this.onType,
|
|
17
|
+
this.onHover,
|
|
18
|
+
this.fit = BoxFit.contain,
|
|
19
|
+
this.alignment = Alignment.center,
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
final io.Socket socket;
|
|
23
|
+
final String deviceId;
|
|
24
|
+
final String platform;
|
|
25
|
+
final Size? remoteResolution;
|
|
26
|
+
final void Function(double x, double y)? onTap;
|
|
27
|
+
final void Function(double x1, double y1, double x2, double y2)? onSwipe;
|
|
28
|
+
final void Function(String text)? onType;
|
|
29
|
+
final void Function(double x, double y)? onHover;
|
|
30
|
+
final BoxFit fit;
|
|
31
|
+
final Alignment alignment;
|
|
32
|
+
|
|
33
|
+
@override
|
|
34
|
+
State<StreamRenderer> createState() => _StreamRendererState();
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
class _StreamRendererState extends State<StreamRenderer> {
|
|
38
|
+
Uint8List? _frame;
|
|
39
|
+
Size? _frameSize;
|
|
40
|
+
ImageStream? _imageStream;
|
|
41
|
+
ImageStreamListener? _imageListener;
|
|
42
|
+
Offset? _dragStart;
|
|
43
|
+
Offset? _dragEnd;
|
|
44
|
+
|
|
45
|
+
Timer? _hoverThrottleTimer;
|
|
46
|
+
Offset? _pendingHoverOffset;
|
|
47
|
+
DateTime _lastHoverTime = DateTime.fromMillisecondsSinceEpoch(0);
|
|
48
|
+
|
|
49
|
+
@override
|
|
50
|
+
void initState() {
|
|
51
|
+
super.initState();
|
|
52
|
+
widget.socket.on('stream:frame', _onFrame);
|
|
53
|
+
widget.socket.emit('stream:subscribe', <String, Object?>{
|
|
54
|
+
'deviceId': widget.deviceId,
|
|
55
|
+
'platform': widget.platform,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
@override
|
|
60
|
+
void didUpdateWidget(StreamRenderer oldWidget) {
|
|
61
|
+
super.didUpdateWidget(oldWidget);
|
|
62
|
+
if (oldWidget.socket == widget.socket &&
|
|
63
|
+
oldWidget.deviceId == widget.deviceId &&
|
|
64
|
+
oldWidget.platform == widget.platform) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
_frameSize = null;
|
|
68
|
+
_detachImageListener();
|
|
69
|
+
oldWidget.socket.off('stream:frame', _onFrame);
|
|
70
|
+
oldWidget.socket.emit('stream:unsubscribe', <String, Object?>{
|
|
71
|
+
'deviceId': oldWidget.deviceId,
|
|
72
|
+
'platform': oldWidget.platform,
|
|
73
|
+
});
|
|
74
|
+
widget.socket.on('stream:frame', _onFrame);
|
|
75
|
+
widget.socket.emit('stream:subscribe', <String, Object?>{
|
|
76
|
+
'deviceId': widget.deviceId,
|
|
77
|
+
'platform': widget.platform,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
void _onFrame(dynamic data) {
|
|
82
|
+
Object? meta;
|
|
83
|
+
Object? bytes;
|
|
84
|
+
if (data is List && data.length >= 2) {
|
|
85
|
+
meta = data[0];
|
|
86
|
+
bytes = data[1];
|
|
87
|
+
}
|
|
88
|
+
if (meta is Map) {
|
|
89
|
+
final frameDeviceId = meta['deviceId']?.toString() ?? '';
|
|
90
|
+
final framePlatform = meta['platform']?.toString() ?? '';
|
|
91
|
+
if (frameDeviceId.isNotEmpty && frameDeviceId != widget.deviceId) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
if (framePlatform.isNotEmpty && framePlatform != widget.platform) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
final frame = switch (bytes) {
|
|
99
|
+
Uint8List value => value,
|
|
100
|
+
List<int> value => Uint8List.fromList(value),
|
|
101
|
+
ByteBuffer value => Uint8List.view(value),
|
|
102
|
+
_ => null,
|
|
103
|
+
};
|
|
104
|
+
if (frame == null || frame.isEmpty || !mounted) return;
|
|
105
|
+
if (_frameSize == null) {
|
|
106
|
+
_resolveFrameSize(frame);
|
|
107
|
+
}
|
|
108
|
+
setState(() => _frame = frame);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
void _resolveFrameSize(Uint8List frame) {
|
|
112
|
+
_detachImageListener();
|
|
113
|
+
final provider = MemoryImage(frame);
|
|
114
|
+
final stream = provider.resolve(const ImageConfiguration());
|
|
115
|
+
final listener = ImageStreamListener((image, _) {
|
|
116
|
+
if (!mounted || _frameSize != null) return;
|
|
117
|
+
setState(() {
|
|
118
|
+
_frameSize = Size(
|
|
119
|
+
image.image.width.toDouble(),
|
|
120
|
+
image.image.height.toDouble(),
|
|
121
|
+
);
|
|
122
|
+
});
|
|
123
|
+
_detachImageListener();
|
|
124
|
+
});
|
|
125
|
+
_imageStream = stream;
|
|
126
|
+
_imageListener = listener;
|
|
127
|
+
stream.addListener(listener);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
void _detachImageListener() {
|
|
131
|
+
if (_imageStream != null && _imageListener != null) {
|
|
132
|
+
_imageStream!.removeListener(_imageListener!);
|
|
133
|
+
}
|
|
134
|
+
_imageStream = null;
|
|
135
|
+
_imageListener = null;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
@override
|
|
139
|
+
Widget build(BuildContext context) {
|
|
140
|
+
final frame = _frame;
|
|
141
|
+
if (frame == null) {
|
|
142
|
+
return const Center(child: CircularProgressIndicator());
|
|
143
|
+
}
|
|
144
|
+
return LayoutBuilder(
|
|
145
|
+
builder: (context, constraints) {
|
|
146
|
+
return MouseRegion(
|
|
147
|
+
onHover: widget.onHover == null
|
|
148
|
+
? null
|
|
149
|
+
: (event) => _handleHoverEvent(event.localPosition, constraints.biggest),
|
|
150
|
+
child: GestureDetector(
|
|
151
|
+
behavior: HitTestBehavior.opaque,
|
|
152
|
+
onTapDown: widget.onTap == null
|
|
153
|
+
? null
|
|
154
|
+
: (details) => _handleTap(details, constraints.biggest),
|
|
155
|
+
onPanStart: widget.onSwipe == null
|
|
156
|
+
? null
|
|
157
|
+
: (details) {
|
|
158
|
+
_dragStart = details.localPosition;
|
|
159
|
+
_dragEnd = details.localPosition;
|
|
160
|
+
},
|
|
161
|
+
onPanUpdate: widget.onSwipe == null
|
|
162
|
+
? null
|
|
163
|
+
: (details) {
|
|
164
|
+
_dragEnd = details.localPosition;
|
|
165
|
+
},
|
|
166
|
+
onPanEnd: widget.onSwipe == null
|
|
167
|
+
? null
|
|
168
|
+
: (_) {
|
|
169
|
+
final start = _dragStart;
|
|
170
|
+
final end = _dragEnd;
|
|
171
|
+
_dragStart = null;
|
|
172
|
+
_dragEnd = null;
|
|
173
|
+
if (start == null ||
|
|
174
|
+
end == null ||
|
|
175
|
+
(start - end).distance < 12) {
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
_handleSwipe(start, end, constraints.biggest);
|
|
179
|
+
},
|
|
180
|
+
child: Image.memory(
|
|
181
|
+
frame,
|
|
182
|
+
gaplessPlayback: true,
|
|
183
|
+
fit: widget.fit,
|
|
184
|
+
alignment: widget.alignment,
|
|
185
|
+
width: constraints.maxWidth,
|
|
186
|
+
height: constraints.maxHeight,
|
|
187
|
+
),
|
|
188
|
+
),
|
|
189
|
+
);
|
|
190
|
+
},
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
void _handleHoverEvent(Offset localPosition, Size boxSize) {
|
|
195
|
+
if (widget.onHover == null) return;
|
|
196
|
+
_pendingHoverOffset = localPosition;
|
|
197
|
+
final now = DateTime.now();
|
|
198
|
+
final elapsed = now.difference(_lastHoverTime);
|
|
199
|
+
const throttleDuration = Duration(milliseconds: 70);
|
|
200
|
+
|
|
201
|
+
if (elapsed >= throttleDuration) {
|
|
202
|
+
_sendPendingHover(boxSize);
|
|
203
|
+
} else {
|
|
204
|
+
_hoverThrottleTimer ??= Timer(throttleDuration - elapsed, () {
|
|
205
|
+
_hoverThrottleTimer = null;
|
|
206
|
+
_sendPendingHover(boxSize);
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
void _sendPendingHover(Size boxSize) {
|
|
212
|
+
final offset = _pendingHoverOffset;
|
|
213
|
+
if (offset == null) return;
|
|
214
|
+
_pendingHoverOffset = null;
|
|
215
|
+
_lastHoverTime = DateTime.now();
|
|
216
|
+
|
|
217
|
+
final point = _mapToRemote(offset, boxSize);
|
|
218
|
+
if (point != null) {
|
|
219
|
+
widget.onHover?.call(point.dx, point.dy);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
void _handleTap(TapDownDetails details, Size boxSize) {
|
|
224
|
+
final point = _mapToRemote(details.localPosition, boxSize);
|
|
225
|
+
if (point == null) {
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
widget.onTap?.call(point.dx, point.dy);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
void _handleSwipe(Offset start, Offset end, Size boxSize) {
|
|
232
|
+
final mappedStart = _mapToRemote(start, boxSize);
|
|
233
|
+
final mappedEnd = _mapToRemote(end, boxSize);
|
|
234
|
+
if (mappedStart == null || mappedEnd == null) {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
widget.onSwipe?.call(
|
|
238
|
+
mappedStart.dx,
|
|
239
|
+
mappedStart.dy,
|
|
240
|
+
mappedEnd.dx,
|
|
241
|
+
mappedEnd.dy,
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
Offset? _mapToRemote(Offset localPosition, Size boxSize) {
|
|
246
|
+
final remote = widget.remoteResolution ?? _frameSize;
|
|
247
|
+
if (remote == null ||
|
|
248
|
+
remote.width <= 0 ||
|
|
249
|
+
remote.height <= 0 ||
|
|
250
|
+
boxSize.width <= 0 ||
|
|
251
|
+
boxSize.height <= 0) {
|
|
252
|
+
return null;
|
|
253
|
+
}
|
|
254
|
+
final remoteSize = Size(remote.width, remote.height);
|
|
255
|
+
final fitted = applyBoxFit(widget.fit, remoteSize, boxSize);
|
|
256
|
+
final sourceRect = widget.alignment.inscribe(
|
|
257
|
+
fitted.source,
|
|
258
|
+
Offset.zero & remoteSize,
|
|
259
|
+
);
|
|
260
|
+
final destRect = widget.alignment.inscribe(
|
|
261
|
+
fitted.destination,
|
|
262
|
+
Offset.zero & boxSize,
|
|
263
|
+
);
|
|
264
|
+
if (!destRect.contains(localPosition)) {
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
267
|
+
final localX = localPosition.dx - destRect.left;
|
|
268
|
+
final localY = localPosition.dy - destRect.top;
|
|
269
|
+
return Offset(
|
|
270
|
+
sourceRect.left + localX * fitted.source.width / destRect.width,
|
|
271
|
+
sourceRect.top + localY * fitted.source.height / destRect.height,
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
@override
|
|
276
|
+
void dispose() {
|
|
277
|
+
_hoverThrottleTimer?.cancel();
|
|
278
|
+
widget.socket.emit('stream:unsubscribe', <String, Object?>{
|
|
279
|
+
'deviceId': widget.deviceId,
|
|
280
|
+
'platform': widget.platform,
|
|
281
|
+
});
|
|
282
|
+
widget.socket.off('stream:frame', _onFrame);
|
|
283
|
+
_detachImageListener();
|
|
284
|
+
super.dispose();
|
|
285
|
+
}
|
|
286
|
+
}
|
|
@@ -106,6 +106,36 @@ final class DesktopCompanionNativePlugin: NSObject {
|
|
|
106
106
|
displayId: displayId
|
|
107
107
|
)
|
|
108
108
|
result(nil)
|
|
109
|
+
case "mouseMove":
|
|
110
|
+
guard isAccessibilityTrusted() else {
|
|
111
|
+
result(
|
|
112
|
+
FlutterError(
|
|
113
|
+
code: "accessibility_permission_denied",
|
|
114
|
+
message: "Accessibility permission is required for input control.",
|
|
115
|
+
details: nil
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
return
|
|
119
|
+
}
|
|
120
|
+
guard let arguments = call.arguments as? [String: Any],
|
|
121
|
+
let x = arguments["x"] as? NSNumber,
|
|
122
|
+
let y = arguments["y"] as? NSNumber else {
|
|
123
|
+
result(
|
|
124
|
+
FlutterError(
|
|
125
|
+
code: "invalid_arguments",
|
|
126
|
+
message: "Missing mouseMove coordinates.",
|
|
127
|
+
details: nil
|
|
128
|
+
)
|
|
129
|
+
)
|
|
130
|
+
return
|
|
131
|
+
}
|
|
132
|
+
let displayId = arguments["displayId"] as? String
|
|
133
|
+
performMouseMove(
|
|
134
|
+
x: x.doubleValue,
|
|
135
|
+
y: y.doubleValue,
|
|
136
|
+
displayId: displayId
|
|
137
|
+
)
|
|
138
|
+
result(nil)
|
|
109
139
|
case "drag":
|
|
110
140
|
guard isAccessibilityTrusted() else {
|
|
111
141
|
result(
|
|
@@ -316,7 +346,18 @@ final class DesktopCompanionNativePlugin: NSObject {
|
|
|
316
346
|
}
|
|
317
347
|
|
|
318
348
|
private func isAccessibilityTrusted() -> Bool {
|
|
319
|
-
AXIsProcessTrusted()
|
|
349
|
+
if AXIsProcessTrusted() { return true }
|
|
350
|
+
// AXIsProcessTrusted() may cache false on macOS 14+ after a System Settings grant.
|
|
351
|
+
// Probe with a live AX read: .apiDisabled is the error returned when the process
|
|
352
|
+
// lacks accessibility permission (AXError has no .notTrusted case in the macOS SDK).
|
|
353
|
+
let sysElement = AXUIElementCreateSystemWide()
|
|
354
|
+
var value: CFTypeRef?
|
|
355
|
+
let status = AXUIElementCopyAttributeValue(
|
|
356
|
+
sysElement,
|
|
357
|
+
kAXFocusedApplicationAttribute as CFString,
|
|
358
|
+
&value
|
|
359
|
+
)
|
|
360
|
+
return status != .apiDisabled
|
|
320
361
|
}
|
|
321
362
|
|
|
322
363
|
private func resolveDisplayId(_ raw: String?) -> CGDirectDisplayID {
|
|
@@ -406,6 +447,20 @@ final class DesktopCompanionNativePlugin: NSObject {
|
|
|
406
447
|
up.post(tap: .cghidEventTap)
|
|
407
448
|
}
|
|
408
449
|
|
|
450
|
+
private func performMouseMove(x: Double, y: Double, displayId: String?) {
|
|
451
|
+
let point = nativePointForCapturedPixel(x: x, y: y, displayId: displayId)
|
|
452
|
+
CGWarpMouseCursorPosition(point)
|
|
453
|
+
guard let moveEvent = CGEvent(
|
|
454
|
+
mouseEventSource: nil,
|
|
455
|
+
mouseType: .mouseMoved,
|
|
456
|
+
mouseCursorPosition: point,
|
|
457
|
+
mouseButton: .left
|
|
458
|
+
) else {
|
|
459
|
+
return
|
|
460
|
+
}
|
|
461
|
+
moveEvent.post(tap: .cghidEventTap)
|
|
462
|
+
}
|
|
463
|
+
|
|
409
464
|
private func performDrag(
|
|
410
465
|
x1: Double,
|
|
411
466
|
y1: Double,
|
package/package.json
CHANGED
package/server/guest_agent.js
CHANGED
|
@@ -62,7 +62,18 @@ function isInsideAllowedRoots(targetPath) {
|
|
|
62
62
|
}
|
|
63
63
|
|
|
64
64
|
function requireToken(req, res, next) {
|
|
65
|
-
|
|
65
|
+
if (!AUTH_TOKEN) {
|
|
66
|
+
// Token not configured in this environment — allow but unauthenticated.
|
|
67
|
+
// Pass NEOAGENT_VM_GUEST_TOKEN to the container to enforce auth.
|
|
68
|
+
return next();
|
|
69
|
+
}
|
|
70
|
+
const header = String(req.headers?.authorization || '').trim();
|
|
71
|
+
const prefix = 'Bearer ';
|
|
72
|
+
const provided = header.startsWith(prefix) ? header.slice(prefix.length).trim() : '';
|
|
73
|
+
if (!provided || provided !== AUTH_TOKEN) {
|
|
74
|
+
return res.status(401).json({ error: 'Unauthorized.' });
|
|
75
|
+
}
|
|
76
|
+
return next();
|
|
66
77
|
}
|
|
67
78
|
|
|
68
79
|
function sanitizeError(err) {
|
|
@@ -184,6 +195,13 @@ function requireCapability(controller, name) {
|
|
|
184
195
|
app.post('/browser/launch', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').launch(req.body || {})));
|
|
185
196
|
app.post('/browser/navigate', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').navigate(req.body?.url, req.body || {})));
|
|
186
197
|
app.post('/browser/screenshot', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').screenshot(req.body || {})));
|
|
198
|
+
app.post('/browser/screenshot-jpeg', async (req, res) => handle(res, async () => {
|
|
199
|
+
const jpeg = await requireCapability(browserController, 'browser').screenshotJpeg(req.body?.quality, req.body || {});
|
|
200
|
+
return {
|
|
201
|
+
contentType: 'image/jpeg',
|
|
202
|
+
contentBase64: Buffer.from(jpeg).toString('base64'),
|
|
203
|
+
};
|
|
204
|
+
}));
|
|
187
205
|
app.post('/browser/click', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
|
|
188
206
|
app.post('/browser/click-point', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
|
|
189
207
|
app.post('/browser/fill', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));
|
package/server/http/routes.js
CHANGED
|
@@ -25,6 +25,7 @@ const routeRegistry = [
|
|
|
25
25
|
{ basePath: '/api/browser-extension', modulePath: '../routes/browser_extension' },
|
|
26
26
|
{ basePath: '/api/android', modulePath: '../routes/android' },
|
|
27
27
|
{ basePath: '/api/desktop', modulePath: '../routes/desktop' },
|
|
28
|
+
{ basePath: '/api/stream', modulePath: '../routes/stream' },
|
|
28
29
|
{ basePath: '/api/recordings', modulePath: '../routes/recordings' },
|
|
29
30
|
{ basePath: '/api/social-video', modulePath: '../routes/social_video' },
|
|
30
31
|
{ basePath: '/api/voice-assistant', modulePath: '../routes/voice_assistant' },
|
|
@@ -67,6 +68,196 @@ function registerApiRoutes(app) {
|
|
|
67
68
|
});
|
|
68
69
|
});
|
|
69
70
|
|
|
71
|
+
app.get('/api/system/health-check', requireAuth, async (req, res) => {
|
|
72
|
+
const userId = req.session?.userId;
|
|
73
|
+
const runtimeManager = req.app?.locals?.runtimeManager;
|
|
74
|
+
const desktopRegistry = req.app?.locals?.desktopCompanionRegistry;
|
|
75
|
+
const extensionRegistry = req.app?.locals?.browserExtensionRegistry;
|
|
76
|
+
const results = [];
|
|
77
|
+
|
|
78
|
+
// 1. Backend connectivity — trivially true if we got here.
|
|
79
|
+
results.push({ id: 'backend', label: 'Backend server', passed: true, detail: 'Reachable' });
|
|
80
|
+
|
|
81
|
+
// 2. Cloud VM runtime availability.
|
|
82
|
+
const runtimeValidation = getRuntimeValidation(runtimeManager);
|
|
83
|
+
const runtimeReady = Boolean(runtimeValidation?.ready);
|
|
84
|
+
results.push({
|
|
85
|
+
id: 'vm_runtime',
|
|
86
|
+
label: 'Cloud VM runtime',
|
|
87
|
+
passed: runtimeReady,
|
|
88
|
+
detail: runtimeReady ? 'Available' : String(runtimeValidation?.issues?.[0] || 'Not configured'),
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
// 3. Cloud VM CLI execution — actually run a command.
|
|
92
|
+
if (runtimeManager && typeof runtimeManager.executeCommand === 'function') {
|
|
93
|
+
try {
|
|
94
|
+
const cmdResult = await runtimeManager.executeCommand(userId, 'echo "health_check_ok"', { timeout: 15000 });
|
|
95
|
+
const exitOk = cmdResult?.exitCode === 0;
|
|
96
|
+
const outputOk = String(cmdResult?.stdout || '').includes('health_check_ok');
|
|
97
|
+
results.push({
|
|
98
|
+
id: 'vm_cli',
|
|
99
|
+
label: 'Cloud VM — command execution',
|
|
100
|
+
passed: exitOk && outputOk,
|
|
101
|
+
detail: exitOk && outputOk
|
|
102
|
+
? 'Commands running'
|
|
103
|
+
: `Exit ${cmdResult?.exitCode ?? '?'}: ${String(cmdResult?.stderr || cmdResult?.stdout || '').slice(0, 120)}`,
|
|
104
|
+
});
|
|
105
|
+
} catch (err) {
|
|
106
|
+
results.push({ id: 'vm_cli', label: 'Cloud VM — command execution', passed: false, detail: String(err?.message || err).slice(0, 120) });
|
|
107
|
+
}
|
|
108
|
+
} else {
|
|
109
|
+
results.push({ id: 'vm_cli', label: 'Cloud VM — command execution', passed: false, detail: 'VM runtime unavailable' });
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// 4. Desktop companion (macOS app / remote device) connectivity + permissions.
|
|
113
|
+
if (desktopRegistry) {
|
|
114
|
+
try {
|
|
115
|
+
const desktopStatus = desktopRegistry.getStatus(userId);
|
|
116
|
+
const connected = Boolean(desktopStatus?.connected);
|
|
117
|
+
results.push({
|
|
118
|
+
id: 'desktop_connected',
|
|
119
|
+
label: 'Desktop companion',
|
|
120
|
+
passed: connected,
|
|
121
|
+
detail: connected
|
|
122
|
+
? `${desktopStatus.onlineCount} device${desktopStatus.onlineCount !== 1 ? 's' : ''} connected`
|
|
123
|
+
: 'No device connected — open the desktop app',
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
if (connected && Array.isArray(desktopStatus?.devices)) {
|
|
127
|
+
const onlineDevice = desktopStatus.devices.find((d) => d.online && !d.revokedAt);
|
|
128
|
+
const perms = onlineDevice?.permissions || {};
|
|
129
|
+
const screenOk = Boolean(perms.screenCapture || perms.screen_capture);
|
|
130
|
+
const inputOk = Boolean(perms.accessibility || perms.inputControl || perms.input_control);
|
|
131
|
+
results.push({
|
|
132
|
+
id: 'desktop_screen',
|
|
133
|
+
label: 'Desktop — screen capture',
|
|
134
|
+
passed: screenOk,
|
|
135
|
+
detail: screenOk ? 'Granted' : 'Not granted — open System Settings › Privacy › Screen Recording',
|
|
136
|
+
});
|
|
137
|
+
results.push({
|
|
138
|
+
id: 'desktop_input',
|
|
139
|
+
label: 'Desktop — input control',
|
|
140
|
+
passed: inputOk,
|
|
141
|
+
detail: inputOk ? 'Granted' : 'Not granted — open System Settings › Privacy › Accessibility',
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
} catch (err) {
|
|
145
|
+
results.push({ id: 'desktop_connected', label: 'Desktop companion', passed: false, detail: String(err?.message || err).slice(0, 120) });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// 5. Chrome extension connectivity.
|
|
150
|
+
if (extensionRegistry) {
|
|
151
|
+
try {
|
|
152
|
+
const extStatus = extensionRegistry.getStatus(userId);
|
|
153
|
+
const extConnected = Boolean(extStatus?.connected);
|
|
154
|
+
results.push({
|
|
155
|
+
id: 'chrome_extension',
|
|
156
|
+
label: 'Chrome extension',
|
|
157
|
+
passed: extConnected,
|
|
158
|
+
detail: extConnected ? 'Connected' : 'Not connected — install the NeoAgent extension in Chrome',
|
|
159
|
+
});
|
|
160
|
+
} catch (err) {
|
|
161
|
+
results.push({ id: 'chrome_extension', label: 'Chrome extension', passed: false, detail: String(err?.message || err).slice(0, 120) });
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const allPassed = results.every((r) => r.passed);
|
|
166
|
+
res.json({ passed: allPassed, results });
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
// Targeted runtime self-tests — one check per endpoint so the UI can embed
|
|
170
|
+
// results inline next to the relevant settings control.
|
|
171
|
+
|
|
172
|
+
app.get('/api/system/test/cli', requireAuth, async (req, res) => {
|
|
173
|
+
const userId = req.session?.userId;
|
|
174
|
+
const runtimeManager = req.app?.locals?.runtimeManager;
|
|
175
|
+
if (!runtimeManager || typeof runtimeManager.executeCommand !== 'function') {
|
|
176
|
+
return res.json({ passed: false, backendUsed: 'vm', detail: 'Runtime not configured on this server.' });
|
|
177
|
+
}
|
|
178
|
+
// Note: executeCommand always routes through the VM backend regardless of
|
|
179
|
+
// the cli_backend setting — desktop CLI routing is not yet implemented.
|
|
180
|
+
try {
|
|
181
|
+
const result = await runtimeManager.executeCommand(userId, 'echo "cli_test_ok"', { timeout: 15000 });
|
|
182
|
+
const exitOk = result?.exitCode === 0;
|
|
183
|
+
const outputOk = String(result?.stdout || '').includes('cli_test_ok');
|
|
184
|
+
return res.json({
|
|
185
|
+
passed: exitOk && outputOk,
|
|
186
|
+
backendUsed: 'vm',
|
|
187
|
+
detail: exitOk && outputOk
|
|
188
|
+
? 'Command executed successfully'
|
|
189
|
+
: `Exit ${result?.exitCode ?? '?'}: ${String(result?.stderr || result?.stdout || '').slice(0, 120)}`,
|
|
190
|
+
});
|
|
191
|
+
} catch (err) {
|
|
192
|
+
return res.json({ passed: false, backendUsed: 'vm', detail: String(err?.message || err).slice(0, 120) });
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
app.get('/api/system/test/extension', requireAuth, (req, res) => {
|
|
197
|
+
const userId = req.session?.userId;
|
|
198
|
+
const extensionRegistry = req.app?.locals?.browserExtensionRegistry;
|
|
199
|
+
if (!extensionRegistry) {
|
|
200
|
+
return res.json({ passed: false, detail: 'Extension registry not available on this server.' });
|
|
201
|
+
}
|
|
202
|
+
try {
|
|
203
|
+
const status = extensionRegistry.getStatus(userId);
|
|
204
|
+
const connected = Boolean(status?.connected);
|
|
205
|
+
return res.json({
|
|
206
|
+
passed: connected,
|
|
207
|
+
detail: connected ? 'Extension is connected and live' : 'Extension is not connected',
|
|
208
|
+
tokenId: status?.activeTokenId || null,
|
|
209
|
+
meta: status?.connectedMeta || null,
|
|
210
|
+
});
|
|
211
|
+
} catch (err) {
|
|
212
|
+
return res.json({ passed: false, detail: String(err?.message || err).slice(0, 120) });
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
app.get('/api/system/test/desktop', requireAuth, (req, res) => {
|
|
217
|
+
const userId = req.session?.userId;
|
|
218
|
+
const desktopRegistry = req.app?.locals?.desktopCompanionRegistry;
|
|
219
|
+
if (!desktopRegistry) {
|
|
220
|
+
return res.json({ passed: false, detail: 'Desktop registry not available on this server.' });
|
|
221
|
+
}
|
|
222
|
+
try {
|
|
223
|
+
const status = desktopRegistry.getStatus(userId);
|
|
224
|
+
const connected = Boolean(status?.connected);
|
|
225
|
+
const devices = Array.isArray(status?.devices)
|
|
226
|
+
? status.devices.filter((d) => d.online && !d.revokedAt)
|
|
227
|
+
: [];
|
|
228
|
+
const selected = status?.selectedDeviceId || null;
|
|
229
|
+
const activeDevice = selected
|
|
230
|
+
? devices.find((d) => d.deviceId === selected)
|
|
231
|
+
: devices.length === 1 ? devices[0] : null;
|
|
232
|
+
const perms = activeDevice?.permissions || {};
|
|
233
|
+
const screenOk = Boolean(perms.screenCapture || perms.screen_capture);
|
|
234
|
+
const inputOk = Boolean(perms.accessibility || perms.inputControl || perms.input_control);
|
|
235
|
+
return res.json({
|
|
236
|
+
passed: connected,
|
|
237
|
+
connected,
|
|
238
|
+
onlineCount: devices.length,
|
|
239
|
+
selectedDeviceId: selected,
|
|
240
|
+
activeDevice: activeDevice ? {
|
|
241
|
+
deviceId: activeDevice.deviceId,
|
|
242
|
+
label: activeDevice.label || activeDevice.hostname || activeDevice.deviceId,
|
|
243
|
+
platform: activeDevice.platform || null,
|
|
244
|
+
paused: activeDevice.paused || false,
|
|
245
|
+
permissions: { screenCapture: screenOk, inputControl: inputOk },
|
|
246
|
+
} : null,
|
|
247
|
+
multipleOnline: devices.length > 1 && !activeDevice,
|
|
248
|
+
detail: !connected
|
|
249
|
+
? 'No device connected'
|
|
250
|
+
: devices.length > 1 && !activeDevice
|
|
251
|
+
? `${devices.length} devices online — select one in Desktop settings`
|
|
252
|
+
: activeDevice?.paused
|
|
253
|
+
? `${activeDevice.label || 'Device'} is paused`
|
|
254
|
+
: `${activeDevice?.label || 'Device'} connected`,
|
|
255
|
+
});
|
|
256
|
+
} catch (err) {
|
|
257
|
+
return res.json({ passed: false, detail: String(err?.message || err).slice(0, 120) });
|
|
258
|
+
}
|
|
259
|
+
});
|
|
260
|
+
|
|
70
261
|
app.get('/api/version', requireAuth, (req, res) => {
|
|
71
262
|
res.json(getVersionInfo());
|
|
72
263
|
});
|
package/server/http/socket.js
CHANGED
|
@@ -8,7 +8,7 @@ function createSocketServer(httpServer, { validateOrigin }) {
|
|
|
8
8
|
pingInterval: Number(process.env.NEOAGENT_SOCKET_PING_INTERVAL_MS || 25000),
|
|
9
9
|
pingTimeout: Number(process.env.NEOAGENT_SOCKET_PING_TIMEOUT_MS || 20000),
|
|
10
10
|
connectTimeout: Number(process.env.NEOAGENT_SOCKET_CONNECT_TIMEOUT_MS || 10000),
|
|
11
|
-
maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES ||
|
|
11
|
+
maxHttpBufferSize: Number(process.env.NEOAGENT_SOCKET_MAX_HTTP_BUFFER_BYTES || 8 * 1024 * 1024),
|
|
12
12
|
cors: {
|
|
13
13
|
origin(origin, callback) {
|
|
14
14
|
return validateOrigin(origin, callback, { allowMissingOrigin: true });
|
package/server/index.js
CHANGED
|
@@ -34,6 +34,7 @@ const { startServices, stopServices } = require('./services/manager');
|
|
|
34
34
|
const { bindBrowserExtensionGateway } = require('./services/browser/extension/gateway');
|
|
35
35
|
const { bindDesktopCompanionGateway } = require('./services/desktop/gateway');
|
|
36
36
|
const { bindWearableGateway } = require('./services/wearable/gateway');
|
|
37
|
+
const { StreamHub } = require('./services/streaming/stream-hub');
|
|
37
38
|
|
|
38
39
|
function parseBooleanFlag(value, fallback = false) {
|
|
39
40
|
const normalized = String(value || '').trim().toLowerCase();
|
|
@@ -89,6 +90,8 @@ const app = express();
|
|
|
89
90
|
app.disable('x-powered-by');
|
|
90
91
|
const httpServer = createServer(app);
|
|
91
92
|
const io = createSocketServer(httpServer, { validateOrigin });
|
|
93
|
+
const streamHub = new StreamHub(io);
|
|
94
|
+
app.locals.streamHub = streamHub;
|
|
92
95
|
app.locals.httpRuntimeConfig = {
|
|
93
96
|
secureCookies: SECURE_COOKIES,
|
|
94
97
|
trustProxy: TRUST_PROXY,
|
|
@@ -112,7 +115,7 @@ registerApiRoutes(app);
|
|
|
112
115
|
registerStaticRoutes(app);
|
|
113
116
|
registerErrorHandler(app);
|
|
114
117
|
bindBrowserExtensionGateway(httpServer, app);
|
|
115
|
-
bindDesktopCompanionGateway(httpServer, app, sessionMiddleware);
|
|
118
|
+
bindDesktopCompanionGateway(httpServer, app, sessionMiddleware, streamHub);
|
|
116
119
|
bindWearableGateway(httpServer, app, sessionMiddleware);
|
|
117
120
|
|
|
118
121
|
let shuttingDown = false;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
178383462d4c7125ce032b3783b45564
|
|
Binary file
|
|
@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"4c525dac5ebe5971c5708ef73558ed8edcf4a3
|
|
|
37
37
|
|
|
38
38
|
_flutter.loader.load({
|
|
39
39
|
serviceWorkerSettings: {
|
|
40
|
-
serviceWorkerVersion: "
|
|
40
|
+
serviceWorkerVersion: "559469558" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
|
|
41
41
|
}
|
|
42
42
|
});
|