neoagent 2.4.0 → 2.4.1-beta.10

package/flutter_app/lib/src/desktop_companion_io.dart

@@ -25,6 +25,18 @@ class DesktopCompanionManager extends ChangeNotifier {
   final DesktopCompanionActions _actions;
   WebSocket? _socket;
   Timer? _reconnectTimer;
+  Timer? _streamTimer;
+  bool _streamCaptureInFlight = false;
+  // Set true while a click / drag / scroll / typeText / pressKey command is
+  // being executed.  _captureAndSendBinaryFrame respects this flag so it does
+  // not compete with the input command for the native bridge or the WebSocket
+  // send buffer, and a fresh frame is forced immediately after the action.
+  bool _inputCommandInFlight = false;
+  int _frameSeq = 0;
+  int _streamGeneration = 0;
+  // Tracks the current stream quality so the forced post-input capture can use
+  // the same setting without re-parsing the original startStream payload.
+  int _currentStreamQuality = 80;
 
   String _backendUrl = '';
   String _sessionCookie = '';
@@ -52,7 +64,8 @@ class DesktopCompanionManager extends ChangeNotifier {
 
   Future<void> bootstrap(SharedPreferences prefs) async {
     _enabled = prefs.getBool(desktopCompanionEnabledPrefsKey) ?? false;
-    _paused = prefs.getBool(desktopCompanionPausedPrefsKey) ?? false;
+    // Always start unpaused — paused state must not carry over across restarts.
+    _paused = false;
     _label =
         prefs.getString(desktopCompanionLabelPrefsKey)?.trim() ??
         _defaultLabel();
@@ -116,7 +129,6 @@ class DesktopCompanionManager extends ChangeNotifier {
 
   Future<void> setPaused(bool value, SharedPreferences prefs) async {
     _paused = value;
-    await prefs.setBool(desktopCompanionPausedPrefsKey, value);
     notifyListeners();
     if (_connected) {
       await _sendEvent('statusChanged', <String, Object?>{'paused': value});
@@ -126,6 +138,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   Future<void> disconnect() async {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _stopStreaming();
     _connecting = false;
     _connected = false;
     final socket = _socket;
@@ -281,6 +294,19 @@ class DesktopCompanionManager extends ChangeNotifier {
     }
   }
 
+  // Commands that interact with the remote machine's input system.  While one
+  // of these is executing we pause frame captures so the WebSocket send buffer
+  // is clear for the result message, and to avoid the native bridge being busy
+  // with a screenshot when the click/drag/etc. needs to run.
+  static const _inputCommands = <String>{
+    'click',
+    'mouseMove',
+    'drag',
+    'scroll',
+    'typeText',
+    'pressKey',
+  };
+
   Future<void> _handleCommand(Map<String, Object?> message) async {
     final id = message['id']?.toString() ?? '';
     final command = message['command']?.toString() ?? '';
@@ -289,6 +315,10 @@ class DesktopCompanionManager extends ChangeNotifier {
             (key, value) => MapEntry(key.toString(), value),
           )
         : const <String, Object?>{};
+
+    final isInput = _inputCommands.contains(command);
+    if (isInput) _inputCommandInFlight = true;
+
     try {
       final response = await _dispatchCommand(command, payload);
       _socket?.add(
@@ -299,6 +329,18 @@ class DesktopCompanionManager extends ChangeNotifier {
           'payload': response,
         }),
       );
+      // Immediately capture a fresh frame after an input action so the user
+      // sees the result of their interaction without waiting for the next
+      // timer tick.
+      if (isInput && _streamTimer != null && _connected) {
+        unawaited(
+          _captureAndSendBinaryFrame(
+            _currentStreamQuality,
+            _streamGeneration,
+            forced: true,
+          ),
+        );
+      }
     } catch (error) {
       _socket?.add(
         jsonEncode(<String, Object?>{
@@ -308,6 +350,8 @@ class DesktopCompanionManager extends ChangeNotifier {
           'error': '$error',
         }),
       );
+    } finally {
+      if (isInput) _inputCommandInFlight = false;
     }
   }
 
@@ -327,6 +371,10 @@ class DesktopCompanionManager extends ChangeNotifier {
         );
       case 'captureFrame':
         return _actions.captureFrame(activeDisplayId: _activeDisplayId);
+      case 'startStream':
+        return _startStreaming(payload);
+      case 'stopStream':
+        return _stopStreaming();
       case 'observe':
         return _actions.observe(
           includeTree: payload['includeTree'] == true,
@@ -339,6 +387,12 @@ class DesktopCompanionManager extends ChangeNotifier {
           button: payload['button']?.toString() ?? 'left',
           displayId: _activeDisplayId,
         );
+      case 'mouseMove':
+        return _actions.mouseMove(
+          x: (payload['x'] as num?)?.round() ?? 0,
+          y: (payload['y'] as num?)?.round() ?? 0,
+          displayId: _activeDisplayId,
+        );
       case 'drag':
         return _actions.drag(
           x1: (payload['x1'] as num?)?.round() ?? 0,
@@ -387,10 +441,15 @@ class DesktopCompanionManager extends ChangeNotifier {
       case 'pauseControl':
         final paused = payload['paused'] != false;
         _paused = paused;
-        final prefs = await SharedPreferences.getInstance();
-        await prefs.setBool(desktopCompanionPausedPrefsKey, paused);
         notifyListeners();
         return <String, Object?>{'success': true, 'paused': _paused};
+      case 'executeCommand':
+        return _actions.executeShellCommand(
+          command: payload['command']?.toString() ?? '',
+          cwd: payload['cwd']?.toString(),
+          timeoutMs: (payload['timeout'] as num?)?.toInt(),
+          stdinInput: payload['stdin_input']?.toString(),
+        );
       case 'ping':
         return <String, Object?>{'pong': true};
       default:
@@ -399,6 +458,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   }
 
   void _handleSocketClosed() {
+    _stopStreaming();
     _socket = null;
     _connecting = false;
     _connected = false;
@@ -410,6 +470,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   void dispose() {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _stopStreaming();
     _connecting = false;
     _connected = false;
     _enabled = false;
@@ -443,6 +504,86 @@ class DesktopCompanionManager extends ChangeNotifier {
     );
   }
 
+  Future<Map<String, Object?>> _startStreaming(
+    Map<String, Object?> payload,
+  ) async {
+    _streamTimer?.cancel();
+    final generation = ++_streamGeneration;
+    final fps = ((payload['fps'] as num?)?.round() ?? 15).clamp(1, 20);
+    final quality = ((payload['quality'] as num?)?.round() ?? 80).clamp(30, 95);
+    final displayId = payload['displayId']?.toString().trim();
+    if (displayId != null && displayId.isNotEmpty) {
+      _activeDisplayId = displayId;
+    }
+    final interval = Duration(milliseconds: max(1, (1000 / fps).floor()));
+    _frameSeq = 0;
+    _currentStreamQuality = quality;
+    _streamTimer = Timer.periodic(interval, (_) {
+      unawaited(_captureAndSendBinaryFrame(quality, generation));
+    });
+    unawaited(_captureAndSendBinaryFrame(quality, generation));
+    return <String, Object?>{
+      'success': true,
+      'fps': fps,
+      'quality': quality,
+      'displayId': _activeDisplayId,
+    };
+  }
+
+  Map<String, Object?> _stopStreaming() {
+    _streamTimer?.cancel();
+    _streamTimer = null;
+    _streamGeneration++;
+    _streamCaptureInFlight = false;
+    return <String, Object?>{'success': true};
+  }
+
+  Future<void> _captureAndSendBinaryFrame(
+    int quality,
+    int generation, {
+    bool forced = false,
+  }) async {
+    final socket = _socket;
+    if (socket == null ||
+        !_connected ||
+        _streamCaptureInFlight ||
+        generation != _streamGeneration) {
+      return;
+    }
+    // If an input command is actively running, skip this frame unless we were
+    // explicitly forced (i.e. this IS the post-input refresh capture).
+    if (!forced && _inputCommandInFlight) return;
+    _streamCaptureInFlight = true;
+    try {
+      final snapshot = await _actions.captureSnapshot(
+        activeDisplayId: _activeDisplayId,
+      );
+      if (snapshot == null) return;
+      final jpeg = await _actions.compressToJpeg(snapshot, quality);
+      if (jpeg.isEmpty) return;
+      if (!_connected || generation != _streamGeneration || _socket != socket) {
+        return;
+      }
+      final frame = Uint8List(10 + jpeg.length);
+      final header = ByteData.sublistView(frame, 0, 10);
+      header.setUint8(0, 0x01);
+      header.setUint32(1, _frameSeq++ & 0xffffffff, Endian.big);
+      header.setUint32(
+        5,
+        DateTime.now().millisecondsSinceEpoch & 0xffffffff,
+        Endian.big,
+      );
+      header.setUint8(9, 0x01);
+      frame.setRange(10, frame.length, jpeg);
+      socket.add(frame);
+    } catch (error) {
+      _errorMessage = 'Desktop stream capture failed: $error';
+      notifyListeners();
+    } finally {
+      _streamCaptureInFlight = false;
+    }
+  }
+
   Future<void> _openMacPermissionSettings(String key) async {
     final uri = switch (key) {
       'screencapture' =>

package/flutter_app/lib/src/desktop_native_bridge.dart

@@ -44,6 +44,19 @@ class DesktopNativeBridge {
     });
   }
 
+  Future<void> mouseMove({
+    required int x,
+    required int y,
+    String? displayId,
+  }) {
+    return _channel.invokeMethod<void>('mouseMove', <String, Object?>{
+      'x': x,
+      'y': y,
+      if (displayId != null && displayId.trim().isNotEmpty)
+        'displayId': displayId.trim(),
+    });
+  }
+
   Future<void> drag({
     required int x1,
     required int y1,

package/flutter_app/lib/src/stream_renderer.dart

@@ -0,0 +1,286 @@
+import 'dart:async';
+import 'dart:typed_data';
+
+import 'package:flutter/material.dart';
+import 'package:socket_io_client/socket_io_client.dart' as io;
+
+class StreamRenderer extends StatefulWidget {
+  const StreamRenderer({
+    super.key,
+    required this.socket,
+    required this.deviceId,
+    required this.platform,
+    this.remoteResolution,
+    this.onTap,
+    this.onSwipe,
+    this.onType,
+    this.onHover,
+    this.fit = BoxFit.contain,
+    this.alignment = Alignment.center,
+  });
+
+  final io.Socket socket;
+  final String deviceId;
+  final String platform;
+  final Size? remoteResolution;
+  final void Function(double x, double y)? onTap;
+  final void Function(double x1, double y1, double x2, double y2)? onSwipe;
+  final void Function(String text)? onType;
+  final void Function(double x, double y)? onHover;
+  final BoxFit fit;
+  final Alignment alignment;
+
+  @override
+  State<StreamRenderer> createState() => _StreamRendererState();
+}
+
+class _StreamRendererState extends State<StreamRenderer> {
+  Uint8List? _frame;
+  Size? _frameSize;
+  ImageStream? _imageStream;
+  ImageStreamListener? _imageListener;
+  Offset? _dragStart;
+  Offset? _dragEnd;
+
+  Timer? _hoverThrottleTimer;
+  Offset? _pendingHoverOffset;
+  DateTime _lastHoverTime = DateTime.fromMillisecondsSinceEpoch(0);
+
+  @override
+  void initState() {
+    super.initState();
+    widget.socket.on('stream:frame', _onFrame);
+    widget.socket.emit('stream:subscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+  }
+
+  @override
+  void didUpdateWidget(StreamRenderer oldWidget) {
+    super.didUpdateWidget(oldWidget);
+    if (oldWidget.socket == widget.socket &&
+        oldWidget.deviceId == widget.deviceId &&
+        oldWidget.platform == widget.platform) {
+      return;
+    }
+    _frameSize = null;
+    _detachImageListener();
+    oldWidget.socket.off('stream:frame', _onFrame);
+    oldWidget.socket.emit('stream:unsubscribe', <String, Object?>{
+      'deviceId': oldWidget.deviceId,
+      'platform': oldWidget.platform,
+    });
+    widget.socket.on('stream:frame', _onFrame);
+    widget.socket.emit('stream:subscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+  }
+
+  void _onFrame(dynamic data) {
+    Object? meta;
+    Object? bytes;
+    if (data is List && data.length >= 2) {
+      meta = data[0];
+      bytes = data[1];
+    }
+    if (meta is Map) {
+      final frameDeviceId = meta['deviceId']?.toString() ?? '';
+      final framePlatform = meta['platform']?.toString() ?? '';
+      if (frameDeviceId.isNotEmpty && frameDeviceId != widget.deviceId) {
+        return;
+      }
+      if (framePlatform.isNotEmpty && framePlatform != widget.platform) {
+        return;
+      }
+    }
+    final frame = switch (bytes) {
+      Uint8List value => value,
+      List<int> value => Uint8List.fromList(value),
+      ByteBuffer value => Uint8List.view(value),
+      _ => null,
+    };
+    if (frame == null || frame.isEmpty || !mounted) return;
+    if (_frameSize == null) {
+      _resolveFrameSize(frame);
+    }
+    setState(() => _frame = frame);
+  }
+
+  void _resolveFrameSize(Uint8List frame) {
+    _detachImageListener();
+    final provider = MemoryImage(frame);
+    final stream = provider.resolve(const ImageConfiguration());
+    final listener = ImageStreamListener((image, _) {
+      if (!mounted || _frameSize != null) return;
+      setState(() {
+        _frameSize = Size(
+          image.image.width.toDouble(),
+          image.image.height.toDouble(),
+        );
+      });
+      _detachImageListener();
+    });
+    _imageStream = stream;
+    _imageListener = listener;
+    stream.addListener(listener);
+  }
+
+  void _detachImageListener() {
+    if (_imageStream != null && _imageListener != null) {
+      _imageStream!.removeListener(_imageListener!);
+    }
+    _imageStream = null;
+    _imageListener = null;
+  }
+
+  @override
+  Widget build(BuildContext context) {
+    final frame = _frame;
+    if (frame == null) {
+      return const Center(child: CircularProgressIndicator());
+    }
+    return LayoutBuilder(
+      builder: (context, constraints) {
+        return MouseRegion(
+          onHover: widget.onHover == null
+              ? null
+              : (event) => _handleHoverEvent(event.localPosition, constraints.biggest),
+          child: GestureDetector(
+            behavior: HitTestBehavior.opaque,
+            onTapDown: widget.onTap == null
+                ? null
+                : (details) => _handleTap(details, constraints.biggest),
+            onPanStart: widget.onSwipe == null
+                ? null
+                : (details) {
+                    _dragStart = details.localPosition;
+                    _dragEnd = details.localPosition;
+                  },
+            onPanUpdate: widget.onSwipe == null
+                ? null
+                : (details) {
+                    _dragEnd = details.localPosition;
+                  },
+            onPanEnd: widget.onSwipe == null
+                ? null
+                : (_) {
+                    final start = _dragStart;
+                    final end = _dragEnd;
+                    _dragStart = null;
+                    _dragEnd = null;
+                    if (start == null ||
+                        end == null ||
+                        (start - end).distance < 12) {
+                      return;
+                    }
+                    _handleSwipe(start, end, constraints.biggest);
+                  },
+            child: Image.memory(
+              frame,
+              gaplessPlayback: true,
+              fit: widget.fit,
+              alignment: widget.alignment,
+              width: constraints.maxWidth,
+              height: constraints.maxHeight,
+            ),
+          ),
+        );
+      },
+    );
+  }
+
+  void _handleHoverEvent(Offset localPosition, Size boxSize) {
+    if (widget.onHover == null) return;
+    _pendingHoverOffset = localPosition;
+    final now = DateTime.now();
+    final elapsed = now.difference(_lastHoverTime);
+    const throttleDuration = Duration(milliseconds: 70);
+    
+    if (elapsed >= throttleDuration) {
+      _sendPendingHover(boxSize);
+    } else {
+      _hoverThrottleTimer ??= Timer(throttleDuration - elapsed, () {
+        _hoverThrottleTimer = null;
+        _sendPendingHover(boxSize);
+      });
+    }
+  }
+
+  void _sendPendingHover(Size boxSize) {
+    final offset = _pendingHoverOffset;
+    if (offset == null) return;
+    _pendingHoverOffset = null;
+    _lastHoverTime = DateTime.now();
+    
+    final point = _mapToRemote(offset, boxSize);
+    if (point != null) {
+      widget.onHover?.call(point.dx, point.dy);
+    }
+  }
+
+  void _handleTap(TapDownDetails details, Size boxSize) {
+    final point = _mapToRemote(details.localPosition, boxSize);
+    if (point == null) {
+      return;
+    }
+    widget.onTap?.call(point.dx, point.dy);
+  }
+
+  void _handleSwipe(Offset start, Offset end, Size boxSize) {
+    final mappedStart = _mapToRemote(start, boxSize);
+    final mappedEnd = _mapToRemote(end, boxSize);
+    if (mappedStart == null || mappedEnd == null) {
+      return;
+    }
+    widget.onSwipe?.call(
+      mappedStart.dx,
+      mappedStart.dy,
+      mappedEnd.dx,
+      mappedEnd.dy,
+    );
+  }
+
+  Offset? _mapToRemote(Offset localPosition, Size boxSize) {
+    final remote = widget.remoteResolution ?? _frameSize;
+    if (remote == null ||
+        remote.width <= 0 ||
+        remote.height <= 0 ||
+        boxSize.width <= 0 ||
+        boxSize.height <= 0) {
+      return null;
+    }
+    final remoteSize = Size(remote.width, remote.height);
+    final fitted = applyBoxFit(widget.fit, remoteSize, boxSize);
+    final sourceRect = widget.alignment.inscribe(
+      fitted.source,
+      Offset.zero & remoteSize,
+    );
+    final destRect = widget.alignment.inscribe(
+      fitted.destination,
+      Offset.zero & boxSize,
+    );
+    if (!destRect.contains(localPosition)) {
+      return null;
+    }
+    final localX = localPosition.dx - destRect.left;
+    final localY = localPosition.dy - destRect.top;
+    return Offset(
+      sourceRect.left + localX * fitted.source.width / destRect.width,
+      sourceRect.top + localY * fitted.source.height / destRect.height,
+    );
+  }
+
+  @override
+  void dispose() {
+    _hoverThrottleTimer?.cancel();
+    widget.socket.emit('stream:unsubscribe', <String, Object?>{
+      'deviceId': widget.deviceId,
+      'platform': widget.platform,
+    });
+    widget.socket.off('stream:frame', _onFrame);
+    _detachImageListener();
+    super.dispose();
+  }
+}

package/flutter_app/macos/Runner/AppDelegate.swift

@@ -106,6 +106,36 @@ final class DesktopCompanionNativePlugin: NSObject {
         displayId: displayId
       )
       result(nil)
+    case "mouseMove":
+      guard isAccessibilityTrusted() else {
+        result(
+          FlutterError(
+            code: "accessibility_permission_denied",
+            message: "Accessibility permission is required for input control.",
+            details: nil
+          )
+        )
+        return
+      }
+      guard let arguments = call.arguments as? [String: Any],
+            let x = arguments["x"] as? NSNumber,
+            let y = arguments["y"] as? NSNumber else {
+        result(
+          FlutterError(
+            code: "invalid_arguments",
+            message: "Missing mouseMove coordinates.",
+            details: nil
+          )
+        )
+        return
+      }
+      let displayId = arguments["displayId"] as? String
+      performMouseMove(
+        x: x.doubleValue,
+        y: y.doubleValue,
+        displayId: displayId
+      )
+      result(nil)
     case "drag":
       guard isAccessibilityTrusted() else {
         result(
@@ -316,7 +346,18 @@ final class DesktopCompanionNativePlugin: NSObject {
   }
 
   private func isAccessibilityTrusted() -> Bool {
-    AXIsProcessTrusted()
+    if AXIsProcessTrusted() { return true }
+    // AXIsProcessTrusted() may cache false on macOS 14+ after a System Settings grant.
+    // Probe with a live AX read: .apiDisabled is the error returned when the process
+    // lacks accessibility permission (AXError has no .notTrusted case in the macOS SDK).
+    let sysElement = AXUIElementCreateSystemWide()
+    var value: CFTypeRef?
+    let status = AXUIElementCopyAttributeValue(
+      sysElement,
+      kAXFocusedApplicationAttribute as CFString,
+      &value
+    )
+    return status != .apiDisabled
   }
 
   private func resolveDisplayId(_ raw: String?) -> CGDirectDisplayID {
@@ -406,6 +447,20 @@ final class DesktopCompanionNativePlugin: NSObject {
     up.post(tap: .cghidEventTap)
   }
 
+  private func performMouseMove(x: Double, y: Double, displayId: String?) {
+    let point = nativePointForCapturedPixel(x: x, y: y, displayId: displayId)
+    CGWarpMouseCursorPosition(point)
+    guard let moveEvent = CGEvent(
+      mouseEventSource: nil,
+      mouseType: .mouseMoved,
+      mouseCursorPosition: point,
+      mouseButton: .left
+    ) else {
+      return
+    }
+    moveEvent.post(tap: .cghidEventTap)
+  }
+
   private func performDrag(
     x1: Double,
     y1: Double,

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "neoagent",
-  "version": "2.4.0",
+  "version": "2.4.1-beta.10",
   "description": "Proactive personal AI agent with no limits",
-  "license": "MIT",
+  "license": "AGPL-3.0-only",
   "main": "server/index.js",
   "engines": {
     "node": ">=20"

package/server/guest_agent.js

@@ -62,7 +62,18 @@ function isInsideAllowedRoots(targetPath) {
 }
 
 function requireToken(req, res, next) {
-  next();
+  if (!AUTH_TOKEN) {
+    // Token not configured in this environment — allow but unauthenticated.
+    // Pass NEOAGENT_VM_GUEST_TOKEN to the container to enforce auth.
+    return next();
+  }
+  const header = String(req.headers?.authorization || '').trim();
+  const prefix = 'Bearer ';
+  const provided = header.startsWith(prefix) ? header.slice(prefix.length).trim() : '';
+  if (!provided || provided !== AUTH_TOKEN) {
+    return res.status(401).json({ error: 'Unauthorized.' });
+  }
+  return next();
 }
 
 function sanitizeError(err) {
@@ -184,6 +195,13 @@ function requireCapability(controller, name) {
 app.post('/browser/launch', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').launch(req.body || {})));
 app.post('/browser/navigate', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').navigate(req.body?.url, req.body || {})));
 app.post('/browser/screenshot', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').screenshot(req.body || {})));
+app.post('/browser/screenshot-jpeg', async (req, res) => handle(res, async () => {
+  const jpeg = await requireCapability(browserController, 'browser').screenshotJpeg(req.body?.quality, req.body || {});
+  return {
+    contentType: 'image/jpeg',
+    contentBase64: Buffer.from(jpeg).toString('base64'),
+  };
+}));
 app.post('/browser/click', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
 app.post('/browser/click-point', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
 app.post('/browser/fill', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));