neoagent 2.4.0 → 2.4.1-beta.11

package/flutter_app/lib/src/desktop_companion_actions.dart

@@ -1,3 +1,4 @@
+import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
@@ -8,6 +9,20 @@ import 'package:package_info_plus/package_info_plus.dart';
 import 'desktop_native_bridge.dart';
 import 'desktop_screen_capture.dart';
 
+// ─── Isolate helpers for JPEG compression ────────────────────────────────────
+// `compressToJpeg` offloads the CPU-intensive pure-Dart PNG→JPEG conversion
+// to a background isolate via `compute()` so the main isolate's event loop
+// stays free to process incoming WebSocket commands (click, drag, etc.)
+// immediately, rather than queuing behind a 300–600 ms compression job.
+
+typedef _JpegArgs = ({Uint8List bytes, int quality});
+
+Uint8List _compressJpegInIsolate(_JpegArgs args) {
+  final decoded = img.decodeImage(args.bytes);
+  if (decoded == null) return args.bytes;
+  return Uint8List.fromList(img.encodeJpg(decoded, quality: args.quality));
+}
+
 class DesktopCompanionSnapshot {
   const DesktopCompanionSnapshot({
     required this.screenshotBase64,
@@ -85,9 +100,15 @@ class DesktopCompanionActions {
       if (bytes is! Uint8List || bytes.isEmpty) {
         return null;
       }
-      final decoded = img.decodeImage(bytes);
-      final width = (frame['width'] as num?)?.round() ?? decoded?.width ?? 0;
-      final height = (frame['height'] as num?)?.round() ?? decoded?.height ?? 0;
+      // Prefer dimensions reported by the native bridge; only fall back to a
+      // pure-Dart image decode (which is slow) when the bridge omits them.
+      final nativeWidth = (frame['width'] as num?)?.round();
+      final nativeHeight = (frame['height'] as num?)?.round();
+      final decoded = (nativeWidth == null || nativeHeight == null)
+          ? img.decodeImage(bytes)
+          : null;
+      final width = nativeWidth ?? decoded?.width ?? 0;
+      final height = nativeHeight ?? decoded?.height ?? 0;
       final displays = _normalizeDisplays(
         frame['displays'],
         fallbackDisplayId:
@@ -176,6 +197,21 @@ class DesktopCompanionActions {
     };
   }
 
+  Future<Uint8List> compressToJpeg(
+    DesktopCompanionSnapshot snapshot,
+    int quality,
+  ) async {
+    final raw = _decodeScreenshotBytes(snapshot.screenshotBase64);
+    // Already JPEG — return immediately without any heavy work on this isolate.
+    if (_looksLikeJpeg(raw)) return raw;
+    // Run the pure-Dart PNG decode + JPEG encode in a background isolate so the
+    // main isolate's event loop stays responsive for incoming commands.
+    return compute(
+      _compressJpegInIsolate,
+      (bytes: raw, quality: quality.clamp(30, 95)),
+    );
+  }
+
   Future<Map<String, Object?>> observe({
     bool includeTree = false,
     String? activeDisplayId,
@@ -222,6 +258,32 @@ class DesktopCompanionActions {
     return <String, Object?>{'success': true, 'x': x, 'y': y, 'button': button};
   }
 
+  Future<Map<String, Object?>> mouseMove({
+    required int x,
+    required int y,
+    String? displayId,
+  }) async {
+    await _assertInputSupported('mouseMove');
+    if (_usesNativeDesktopBridge) {
+      await _nativeBridge.mouseMove(
+        x: x,
+        y: y,
+        displayId: displayId,
+      );
+    } else if (defaultTargetPlatform == TargetPlatform.linux) {
+      await _run(
+        _ShellCommand('xdotool', <String>[
+          'mousemove',
+          '$x',
+          '$y',
+        ]),
+      );
+    } else {
+      throw Exception('mouseMove is not supported on this platform.');
+    }
+    return <String, Object?>{'success': true, 'x': x, 'y': y};
+  }
+
   Future<Map<String, Object?>> drag({
     required int x1,
     required int y1,
@@ -396,6 +458,77 @@ class DesktopCompanionActions {
     };
   }
 
+  Future<Map<String, Object?>> executeShellCommand({
+    required String command,
+    String? cwd,
+    int? timeoutMs,
+    String? stdinInput,
+  }) async {
+    final shell = Platform.isWindows ? 'cmd.exe' : (Platform.environment['SHELL'] ?? '/bin/sh');
+    final args = Platform.isWindows ? <String>['/c', command] : <String>['-lc', command];
+    final workingDir = cwd?.trim().isNotEmpty == true ? cwd : Platform.environment['HOME'];
+    final startedAt = DateTime.now();
+
+    final process = await Process.start(
+      shell,
+      args,
+      workingDirectory: workingDir,
+      runInShell: false,
+    );
+
+    if (stdinInput != null && stdinInput.isNotEmpty) {
+      process.stdin.write(stdinInput);
+      await process.stdin.close();
+    } else {
+      unawaited(process.stdin.close());
+    }
+
+    const maxChars = 50000;
+    final stdoutBuf = StringBuffer();
+    final stderrBuf = StringBuffer();
+
+    final stdoutSub = process.stdout.transform(utf8.decoder).listen((data) {
+      stdoutBuf.write(data);
+    });
+    final stderrSub = process.stderr.transform(utf8.decoder).listen((data) {
+      stderrBuf.write(data);
+    });
+
+    final effectiveTimeout = Duration(
+      milliseconds: (timeoutMs != null && timeoutMs > 0) ? timeoutMs : 15 * 60 * 1000,
+    );
+
+    bool timedOut = false;
+    int? exitCode;
+    try {
+      exitCode = await process.exitCode.timeout(effectiveTimeout);
+    } on TimeoutException {
+      timedOut = true;
+      process.kill(ProcessSignal.sigterm);
+      exitCode = null;
+    }
+
+    await stdoutSub.cancel();
+    await stderrSub.cancel();
+
+    String trimOutput(StringBuffer buf) {
+      final s = buf.toString().trim();
+      return s.length > maxChars ? '${s.substring(0, maxChars)}\n...[truncated, ${s.length} total chars]' : s;
+    }
+
+    return <String, Object?>{
+      'exitCode': exitCode,
+      'stdout': trimOutput(stdoutBuf),
+      'stderr': trimOutput(stderrBuf),
+      'timedOut': timedOut,
+      'killed': timedOut,
+      'durationMs': DateTime.now().difference(startedAt).inMilliseconds,
+      'command': command,
+      'cwd': workingDir,
+      'backend': 'desktop-companion',
+    };
+  }
+
   Future<Map<String, Object?>> _capabilities({
     Map<String, Object?>? platformStatus,
   }) async {
@@ -516,6 +649,23 @@ class DesktopCompanionActions {
     }
   }
 
+  Uint8List _decodeScreenshotBytes(String screenshotBase64) {
+    final trimmed = screenshotBase64.trim();
+    final commaIndex = trimmed.indexOf(',');
+    final encoded = trimmed.startsWith('data:image/') && commaIndex >= 0
+        ? trimmed.substring(commaIndex + 1)
+        : trimmed;
+    return Uint8List.fromList(base64Decode(encoded));
+  }
+
+  bool _looksLikeJpeg(Uint8List bytes) {
+    return bytes.length >= 4 &&
+        bytes[0] == 0xff &&
+        bytes[1] == 0xd8 &&
+        bytes[bytes.length - 2] == 0xff &&
+        bytes[bytes.length - 1] == 0xd9;
+  }
+
   String _normalizeMouseButton(String button) {
     final value = button.trim().toLowerCase();
     if (value == 'left' || value == 'right' || value == 'middle') {

package/flutter_app/lib/src/desktop_companion_io.dart

@@ -25,6 +25,18 @@ class DesktopCompanionManager extends ChangeNotifier {
   final DesktopCompanionActions _actions;
   WebSocket? _socket;
   Timer? _reconnectTimer;
+  Timer? _streamTimer;
+  bool _streamCaptureInFlight = false;
+  // Set true while a click / drag / scroll / typeText / pressKey command is
+  // being executed.  _captureAndSendBinaryFrame respects this flag so it does
+  // not compete with the input command for the native bridge or the WebSocket
+  // send buffer, and a fresh frame is forced immediately after the action.
+  bool _inputCommandInFlight = false;
+  int _frameSeq = 0;
+  int _streamGeneration = 0;
+  // Tracks the current stream quality so the forced post-input capture can use
+  // the same setting without re-parsing the original startStream payload.
+  int _currentStreamQuality = 80;
 
   String _backendUrl = '';
   String _sessionCookie = '';
@@ -52,7 +64,8 @@ class DesktopCompanionManager extends ChangeNotifier {
 
   Future<void> bootstrap(SharedPreferences prefs) async {
     _enabled = prefs.getBool(desktopCompanionEnabledPrefsKey) ?? false;
-    _paused = prefs.getBool(desktopCompanionPausedPrefsKey) ?? false;
+    // Always start unpaused — paused state must not carry over across restarts.
+    _paused = false;
     _label =
         prefs.getString(desktopCompanionLabelPrefsKey)?.trim() ??
         _defaultLabel();
@@ -116,7 +129,6 @@ class DesktopCompanionManager extends ChangeNotifier {
 
   Future<void> setPaused(bool value, SharedPreferences prefs) async {
     _paused = value;
-    await prefs.setBool(desktopCompanionPausedPrefsKey, value);
     notifyListeners();
     if (_connected) {
       await _sendEvent('statusChanged', <String, Object?>{'paused': value});
@@ -126,6 +138,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   Future<void> disconnect() async {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _stopStreaming();
     _connecting = false;
     _connected = false;
     final socket = _socket;
@@ -281,6 +294,19 @@ class DesktopCompanionManager extends ChangeNotifier {
     }
   }
 
+  // Commands that interact with the remote machine's input system.  While one
+  // of these is executing we pause frame captures so the WebSocket send buffer
+  // is clear for the result message, and to avoid the native bridge being busy
+  // with a screenshot when the click/drag/etc. needs to run.
+  static const _inputCommands = <String>{
+    'click',
+    'mouseMove',
+    'drag',
+    'scroll',
+    'typeText',
+    'pressKey',
+  };
+
   Future<void> _handleCommand(Map<String, Object?> message) async {
     final id = message['id']?.toString() ?? '';
     final command = message['command']?.toString() ?? '';
@@ -289,6 +315,10 @@ class DesktopCompanionManager extends ChangeNotifier {
             (key, value) => MapEntry(key.toString(), value),
           )
         : const <String, Object?>{};
+
+    final isInput = _inputCommands.contains(command);
+    if (isInput) _inputCommandInFlight = true;
+
     try {
       final response = await _dispatchCommand(command, payload);
       _socket?.add(
@@ -299,6 +329,18 @@ class DesktopCompanionManager extends ChangeNotifier {
           'payload': response,
         }),
       );
+      // Immediately capture a fresh frame after an input action so the user
+      // sees the result of their interaction without waiting for the next
+      // timer tick.
+      if (isInput && _streamTimer != null && _connected) {
+        unawaited(
+          _captureAndSendBinaryFrame(
+            _currentStreamQuality,
+            _streamGeneration,
+            forced: true,
+          ),
+        );
+      }
     } catch (error) {
       _socket?.add(
         jsonEncode(<String, Object?>{
@@ -308,6 +350,8 @@ class DesktopCompanionManager extends ChangeNotifier {
           'error': '$error',
         }),
       );
+    } finally {
+      if (isInput) _inputCommandInFlight = false;
     }
   }
 
@@ -327,6 +371,10 @@ class DesktopCompanionManager extends ChangeNotifier {
         );
       case 'captureFrame':
         return _actions.captureFrame(activeDisplayId: _activeDisplayId);
+      case 'startStream':
+        return _startStreaming(payload);
+      case 'stopStream':
+        return _stopStreaming();
       case 'observe':
         return _actions.observe(
           includeTree: payload['includeTree'] == true,
@@ -339,6 +387,12 @@ class DesktopCompanionManager extends ChangeNotifier {
           button: payload['button']?.toString() ?? 'left',
           displayId: _activeDisplayId,
         );
+      case 'mouseMove':
+        return _actions.mouseMove(
+          x: (payload['x'] as num?)?.round() ?? 0,
+          y: (payload['y'] as num?)?.round() ?? 0,
+          displayId: _activeDisplayId,
+        );
       case 'drag':
         return _actions.drag(
           x1: (payload['x1'] as num?)?.round() ?? 0,
@@ -387,10 +441,15 @@ class DesktopCompanionManager extends ChangeNotifier {
       case 'pauseControl':
         final paused = payload['paused'] != false;
         _paused = paused;
-        final prefs = await SharedPreferences.getInstance();
-        await prefs.setBool(desktopCompanionPausedPrefsKey, paused);
         notifyListeners();
         return <String, Object?>{'success': true, 'paused': _paused};
+      case 'executeCommand':
+        return _actions.executeShellCommand(
+          command: payload['command']?.toString() ?? '',
+          cwd: payload['cwd']?.toString(),
+          timeoutMs: (payload['timeout'] as num?)?.toInt(),
+          stdinInput: payload['stdin_input']?.toString(),
+        );
       case 'ping':
         return <String, Object?>{'pong': true};
       default:
@@ -399,6 +458,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   }
 
   void _handleSocketClosed() {
+    _stopStreaming();
     _socket = null;
     _connecting = false;
     _connected = false;
@@ -410,6 +470,7 @@ class DesktopCompanionManager extends ChangeNotifier {
   void dispose() {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _stopStreaming();
     _connecting = false;
     _connected = false;
     _enabled = false;
@@ -443,6 +504,86 @@ class DesktopCompanionManager extends ChangeNotifier {
     );
   }
 
+  Future<Map<String, Object?>> _startStreaming(
+    Map<String, Object?> payload,
+  ) async {
+    _streamTimer?.cancel();
+    final generation = ++_streamGeneration;
+    final fps = ((payload['fps'] as num?)?.round() ?? 15).clamp(1, 20);
+    final quality = ((payload['quality'] as num?)?.round() ?? 80).clamp(30, 95);
+    final displayId = payload['displayId']?.toString().trim();
+    if (displayId != null && displayId.isNotEmpty) {
+      _activeDisplayId = displayId;
+    }
+    final interval = Duration(milliseconds: max(1, (1000 / fps).floor()));
+    _frameSeq = 0;
+    _currentStreamQuality = quality;
+    _streamTimer = Timer.periodic(interval, (_) {
+      unawaited(_captureAndSendBinaryFrame(quality, generation));
+    });
+    unawaited(_captureAndSendBinaryFrame(quality, generation));
+    return <String, Object?>{
+      'success': true,
+      'fps': fps,
+      'quality': quality,
+      'displayId': _activeDisplayId,
+    };
+  }
+
+  Map<String, Object?> _stopStreaming() {
+    _streamTimer?.cancel();
+    _streamTimer = null;
+    _streamGeneration++;
+    _streamCaptureInFlight = false;
+    return <String, Object?>{'success': true};
+  }
+
+  Future<void> _captureAndSendBinaryFrame(
+    int quality,
+    int generation, {
+    bool forced = false,
+  }) async {
+    final socket = _socket;
+    if (socket == null ||
+        !_connected ||
+        _streamCaptureInFlight ||
+        generation != _streamGeneration) {
+      return;
+    }
+    // If an input command is actively running, skip this frame unless we were
+    // explicitly forced (i.e. this IS the post-input refresh capture).
+    if (!forced && _inputCommandInFlight) return;
+    _streamCaptureInFlight = true;
+    try {
+      final snapshot = await _actions.captureSnapshot(
+        activeDisplayId: _activeDisplayId,
+      );
+      if (snapshot == null) return;
+      final jpeg = await _actions.compressToJpeg(snapshot, quality);
+      if (jpeg.isEmpty) return;
+      if (!_connected || generation != _streamGeneration || _socket != socket) {
+        return;
+      }
+      final frame = Uint8List(10 + jpeg.length);
+      final header = ByteData.sublistView(frame, 0, 10);
+      header.setUint8(0, 0x01);
+      header.setUint32(1, _frameSeq++ & 0xffffffff, Endian.big);
+      header.setUint32(
+        5,
+        DateTime.now().millisecondsSinceEpoch & 0xffffffff,
+        Endian.big,
+      );
+      header.setUint8(9, 0x01);
+      frame.setRange(10, frame.length, jpeg);
+      socket.add(frame);
+    } catch (error) {
+      _errorMessage = 'Desktop stream capture failed: $error';
+      notifyListeners();
+    } finally {
+      _streamCaptureInFlight = false;
+    }
+  }
+
   Future<void> _openMacPermissionSettings(String key) async {
     final uri = switch (key) {
       'screencapture' =>

package/flutter_app/lib/src/desktop_native_bridge.dart

@@ -44,6 +44,19 @@ class DesktopNativeBridge {
     });
   }
 
+  Future<void> mouseMove({
+    required int x,
+    required int y,
+    String? displayId,
+  }) {
+    return _channel.invokeMethod<void>('mouseMove', <String, Object?>{
+      'x': x,
+      'y': y,
+      if (displayId != null && displayId.trim().isNotEmpty)
+        'displayId': displayId.trim(),
+    });
+  }
+
   Future<void> drag({
     required int x1,
     required int y1,