neoagent 2.3.1-beta.6 → 2.3.1-beta.60

package/flutter_app/lib/src/desktop_companion.dart

@@ -0,0 +1,2 @@
+export 'desktop_companion_stub.dart'
+    if (dart.library.io) 'desktop_companion_io.dart';

package/flutter_app/lib/src/desktop_companion_actions.dart

@@ -0,0 +1,586 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter/foundation.dart';
+import 'package:image/image.dart' as img;
+import 'package:package_info_plus/package_info_plus.dart';
+
+import 'desktop_native_bridge.dart';
+import 'desktop_screen_capture.dart';
+
+class DesktopCompanionSnapshot {
+  const DesktopCompanionSnapshot({
+    required this.screenshotBase64,
+    required this.contentType,
+    required this.width,
+    required this.height,
+    required this.displays,
+    required this.activeDisplayId,
+  });
+
+  final String screenshotBase64;
+  final String contentType;
+  final int width;
+  final int height;
+  final List<Map<String, Object?>> displays;
+  final String activeDisplayId;
+}
+
+class DesktopCompanionActions {
+  DesktopCompanionActions({required DesktopScreenCapture screenCapture})
+    : _screenCapture = screenCapture;
+
+  final DesktopScreenCapture _screenCapture;
+  final DesktopNativeBridge _nativeBridge = DesktopNativeBridge();
+
+  bool get isCaptureSupported => _screenCapture.isSupported;
+
+  Future<Map<String, Object?>> buildHello({
+    required String deviceId,
+    required String activationId,
+    required String label,
+    required bool companionEnabled,
+    required bool paused,
+    String? activeDisplayId,
+  }) async {
+    final platformStatus = await _platformStatus();
+    final capabilities = await _capabilities(platformStatus: platformStatus);
+    final snapshot = await _safeSnapshotForStatus(
+      activeDisplayId: activeDisplayId,
+      platformStatus: platformStatus,
+    );
+    final packageInfo = await PackageInfo.fromPlatform();
+    return <String, Object?>{
+      'deviceId': deviceId,
+      'activationId': activationId,
+      'label': label,
+      'hostname': _localHostname(),
+      'platform': defaultTargetPlatform.name,
+      'platformVersion': Platform.operatingSystemVersion,
+      'appVersion': packageInfo.version,
+      'companionEnabled': companionEnabled,
+      'paused': paused,
+      'permissions': _permissions(capabilities, platformStatus: platformStatus),
+      'capabilities': capabilities,
+      'displays': snapshot?.displays ?? const <Map<String, Object?>>[],
+      'activeDisplayId':
+          snapshot?.activeDisplayId ??
+          platformStatus['activeDisplayId']?.toString() ??
+          activeDisplayId ??
+          'primary',
+      'metadata': <String, Object?>{
+        'captureSupported': _screenCapture.isSupported,
+      },
+    };
+  }
+
+  Future<DesktopCompanionSnapshot?> captureSnapshot({
+    String? activeDisplayId,
+  }) async {
+    if (_usesNativeDesktopBridge) {
+      final frame = await _nativeBridge.captureFrame(
+        displayId: activeDisplayId,
+      );
+      final bytes = frame['bytes'];
+      if (bytes is! Uint8List || bytes.isEmpty) {
+        return null;
+      }
+      final decoded = img.decodeImage(bytes);
+      final width = (frame['width'] as num?)?.round() ?? decoded?.width ?? 0;
+      final height = (frame['height'] as num?)?.round() ?? decoded?.height ?? 0;
+      final displays = _normalizeDisplays(
+        frame['displays'],
+        fallbackDisplayId:
+            frame['displayId']?.toString() ?? activeDisplayId ?? 'primary',
+        width: width,
+        height: height,
+      );
+      return DesktopCompanionSnapshot(
+        screenshotBase64: base64Encode(bytes),
+        contentType: frame['mimeType']?.toString() ?? 'image/png',
+        width: width,
+        height: height,
+        displays: displays,
+        activeDisplayId:
+            frame['displayId']?.toString() ?? activeDisplayId ?? 'primary',
+      );
+    }
+
+    final capture = await _screenCapture.captureCurrentScreen();
+    if (capture == null || capture.bytes.isEmpty) return null;
+    final bytes = Uint8List.fromList(capture.bytes);
+    final decoded = img.decodeImage(bytes);
+    final width = decoded?.width ?? 0;
+    final height = decoded?.height ?? 0;
+    return DesktopCompanionSnapshot(
+      screenshotBase64: base64Encode(bytes),
+      contentType: capture.mimeType,
+      width: width,
+      height: height,
+      activeDisplayId: activeDisplayId ?? 'primary',
+      displays: <Map<String, Object?>>[
+        <String, Object?>{
+          'id': activeDisplayId ?? 'primary',
+          'label': 'Primary Display',
+          'width': width,
+          'height': height,
+          'scaleFactor': 1,
+          'primary': true,
+        },
+      ],
+    );
+  }
+
+  Future<Map<String, Object?>> getStatus({
+    required String label,
+    required bool paused,
+    String? activeDisplayId,
+  }) async {
+    final platformStatus = await _platformStatus();
+    final capabilities = await _capabilities(platformStatus: platformStatus);
+    final snapshot = await _safeSnapshotForStatus(
+      activeDisplayId: activeDisplayId,
+      platformStatus: platformStatus,
+    );
+    return <String, Object?>{
+      'paused': paused,
+      'label': label,
+      'activeDisplayId':
+          snapshot?.activeDisplayId ??
+          platformStatus['activeDisplayId']?.toString() ??
+          activeDisplayId ??
+          'primary',
+      'displays': snapshot?.displays ?? const <Map<String, Object?>>[],
+      'permissions': _permissions(capabilities, platformStatus: platformStatus),
+      'capabilities': capabilities,
+      if (platformStatus['frontmostApp'] != null)
+        'frontmostApp': platformStatus['frontmostApp'],
+      if (platformStatus['frontmostWindowTitle'] != null)
+        'frontmostWindowTitle': platformStatus['frontmostWindowTitle'],
+    };
+  }
+
+  Future<Map<String, Object?>> captureFrame({String? activeDisplayId}) async {
+    final snapshot = await captureSnapshot(activeDisplayId: activeDisplayId);
+    if (snapshot == null) {
+      throw Exception('Desktop capture is not available on this platform.');
+    }
+    return <String, Object?>{
+      'screenshotBase64': snapshot.screenshotBase64,
+      'contentType': snapshot.contentType,
+      'width': snapshot.width,
+      'height': snapshot.height,
+      'displayId': snapshot.activeDisplayId,
+      'displays': snapshot.displays,
+      'capturedAt': DateTime.now().toUtc().toIso8601String(),
+    };
+  }
+
+  Future<Map<String, Object?>> observe({
+    bool includeTree = false,
+    String? activeDisplayId,
+  }) async {
+    final result = await captureFrame(activeDisplayId: activeDisplayId);
+    return <String, Object?>{
+      ...result,
+      'tree': includeTree ? const <Map<String, Object?>>[] : null,
+      'treeSupported': false,
+    };
+  }
+
+  Future<Map<String, Object?>> click({
+    required int x,
+    required int y,
+    String button = 'left',
+    String? displayId,
+  }) async {
+    await _assertInputSupported('click');
+    final normalizedButton = _normalizeMouseButton(button);
+    if (_usesNativeDesktopBridge) {
+      await _nativeBridge.click(
+        x: x,
+        y: y,
+        button: normalizedButton,
+        displayId: displayId,
+      );
+    } else if (defaultTargetPlatform == TargetPlatform.linux) {
+      final buttonCode = normalizedButton == 'right'
+          ? '3'
+          : (normalizedButton == 'middle' ? '2' : '1');
+      await _run(
+        _ShellCommand('xdotool', <String>[
+          'mousemove',
+          '$x',
+          '$y',
+          'click',
+          buttonCode,
+        ]),
+      );
+    } else {
+      throw Exception('click is not supported on this platform.');
+    }
+    return <String, Object?>{'success': true, 'x': x, 'y': y, 'button': button};
+  }
+
+  Future<Map<String, Object?>> drag({
+    required int x1,
+    required int y1,
+    required int x2,
+    required int y2,
+    int durationMs = 280,
+    String? displayId,
+  }) async {
+    await _assertInputSupported('drag');
+    if (_usesNativeDesktopBridge) {
+      await _nativeBridge.drag(
+        x1: x1,
+        y1: y1,
+        x2: x2,
+        y2: y2,
+        durationMs: durationMs,
+        displayId: displayId,
+      );
+    } else if (defaultTargetPlatform == TargetPlatform.linux) {
+      await _run(_ShellCommand('xdotool', <String>['mousemove', '$x1', '$y1']));
+      await _run(_ShellCommand('xdotool', const <String>['mousedown', '1']));
+      await Future<void>.delayed(
+        Duration(milliseconds: durationMs.clamp(40, 2000)),
+      );
+      await _run(_ShellCommand('xdotool', <String>['mousemove', '$x2', '$y2']));
+      await _run(_ShellCommand('xdotool', const <String>['mouseup', '1']));
+    } else {
+      throw Exception('drag is not supported on this platform.');
+    }
+    return <String, Object?>{
+      'success': true,
+      'x1': x1,
+      'y1': y1,
+      'x2': x2,
+      'y2': y2,
+      'durationMs': durationMs,
+    };
+  }
+
+  Future<Map<String, Object?>> scroll({
+    int deltaX = 0,
+    int deltaY = 0,
+    String? displayId,
+  }) async {
+    await _assertInputSupported('scroll');
+    if (deltaY == 0 && deltaX == 0) {
+      return <String, Object?>{
+        'success': true,
+        'deltaX': deltaX,
+        'deltaY': deltaY,
+      };
+    }
+    if (_usesNativeDesktopBridge) {
+      await _nativeBridge.scroll(
+        deltaX: deltaX,
+        deltaY: deltaY,
+        displayId: displayId,
+      );
+    } else if (defaultTargetPlatform == TargetPlatform.linux) {
+      if (deltaY != 0) {
+        final clicks = (deltaY.abs() / 120).ceil().clamp(1, 12);
+        final button = deltaY < 0 ? '5' : '4';
+        await _run(
+          _ShellCommand('xdotool', <String>[
+            'click',
+            '--repeat',
+            '$clicks',
+            button,
+          ]),
+        );
+      }
+      if (deltaX != 0) {
+        final clicks = (deltaX.abs() / 120).ceil().clamp(1, 12);
+        final button = deltaX < 0 ? '6' : '7';
+        await _run(
+          _ShellCommand('xdotool', <String>[
+            'click',
+            '--repeat',
+            '$clicks',
+            button,
+          ]),
+        );
+      }
+    } else {
+      throw Exception('scroll is not supported on this platform.');
+    }
+    return <String, Object?>{
+      'success': true,
+      'deltaX': deltaX,
+      'deltaY': deltaY,
+    };
+  }
+
+  Future<Map<String, Object?>> typeText({
+    required String text,
+    bool pressEnter = false,
+  }) async {
+    await _assertInputSupported('type text');
+    if (_usesNativeDesktopBridge) {
+      await _nativeBridge.typeText(text: text, pressEnter: pressEnter);
+    } else if (defaultTargetPlatform == TargetPlatform.linux) {
+      if (text.isNotEmpty) {
+        await _run(
+          _ShellCommand('xdotool', <String>[
+            'type',
+            '--delay',
+            '1',
+            '--',
+            text,
+          ]),
+        );
+      }
+      if (pressEnter) {
+        await _run(_ShellCommand('xdotool', const <String>['key', 'Return']));
+      }
+    } else {
+      throw Exception('type text is not supported on this platform.');
+    }
+    return <String, Object?>{
+      'success': true,
+      'textLength': text.length,
+      'pressEnter': pressEnter,
+    };
+  }
+
+  Future<Map<String, Object?>> pressKey({required String key}) async {
+    await _assertInputSupported('press keys');
+    if (_usesNativeDesktopBridge) {
+      await _nativeBridge.pressKey(key);
+    } else if (defaultTargetPlatform == TargetPlatform.linux) {
+      final normalized = key.trim();
+      if (normalized.isEmpty) {
+        throw Exception('Key is required.');
+      }
+      await _run(_ShellCommand('xdotool', <String>['key', normalized]));
+    } else {
+      throw Exception('press keys is not supported on this platform.');
+    }
+    return <String, Object?>{'success': true, 'key': key};
+  }
+
+  Future<Map<String, Object?>> launchApp({required String app}) async {
+    if (app.trim().isEmpty) {
+      throw Exception('App name is required.');
+    }
+    final command = switch (defaultTargetPlatform) {
+      TargetPlatform.macOS => _ShellCommand('open', <String>['-a', app]),
+      TargetPlatform.windows => _ShellCommand('powershell', <String>[
+        '-NoProfile',
+        '-NonInteractive',
+        '-Command',
+        'Start-Process -FilePath ${_psQuote(app)}',
+      ]),
+      TargetPlatform.linux => _ShellCommand('sh', <String>[
+        '-lc',
+        'if command -v ${_shQuote(app)} >/dev/null 2>&1; then ${_shQuote(app)} >/dev/null 2>&1 & disown; else gtk-launch ${_shQuote(app)} >/dev/null 2>&1 & disown; fi',
+      ]),
+      TargetPlatform.android ||
+      TargetPlatform.iOS ||
+      TargetPlatform.fuchsia => throw Exception(
+        'Launching desktop apps is not supported on this platform.',
+      ),
+    };
+    await _run(command);
+    return <String, Object?>{'success': true, 'app': app};
+  }
+
+  Future<Map<String, Object?>> getTree() async {
+    return <String, Object?>{
+      'supported': false,
+      'nodes': const <Map<String, Object?>>[],
+    };
+  }
+
+  Future<Map<String, Object?>> _capabilities({
+    Map<String, Object?>? platformStatus,
+  }) async {
+    final status = platformStatus ?? await _platformStatus();
+    final inputSupported = await _inputSupported(platformStatus: status);
+    final permissions = status['permissions'];
+    final screenCapturePermission = permissions is Map
+        ? permissions['screenCapture']?.toString()
+        : null;
+    return <String, Object?>{
+      'screenshot':
+          _screenCapture.isSupported &&
+          screenCapturePermission != 'required' &&
+          screenCapturePermission != 'unsupported',
+      'click': inputSupported,
+      'drag': inputSupported,
+      'scroll': inputSupported,
+      'typeText': inputSupported,
+      'pressKey': inputSupported,
+      'launchApp': _isDesktopPlatform,
+      'accessibilityTree': false,
+    };
+  }
+
+  Map<String, Object?> _permissions(
+    Map<String, Object?> capabilities, {
+    Map<String, Object?>? platformStatus,
+  }) {
+    final status = platformStatus ?? const <String, Object?>{};
+    final reportedPermissions = status['permissions'];
+    if (reportedPermissions is Map) {
+      return reportedPermissions.map(
+        (key, value) => MapEntry(key.toString(), value),
+      );
+    }
+    final inputAvailable = capabilities['click'] == true;
+    return <String, Object?>{
+      'screenCapture': _screenCapture.isSupported ? 'available' : 'unsupported',
+      'inputControl': inputAvailable ? 'available' : 'unsupported',
+      'accessibility': defaultTargetPlatform == TargetPlatform.windows
+          ? 'available'
+          : 'unsupported',
+    };
+  }
+
+  Future<bool> _inputSupported({Map<String, Object?>? platformStatus}) async {
+    switch (defaultTargetPlatform) {
+      case TargetPlatform.macOS:
+        final status = platformStatus ?? await _platformStatus();
+        final permissions = status['permissions'];
+        if (permissions is Map) {
+          return permissions['inputControl'] == 'available' &&
+              permissions['accessibility'] == 'available';
+        }
+        return false;
+      case TargetPlatform.windows:
+        return true;
+      case TargetPlatform.linux:
+        try {
+          final result = await Process.run('sh', <String>[
+            '-lc',
+            'command -v xdotool >/dev/null 2>&1',
+          ]);
+          return result.exitCode == 0;
+        } catch (_) {
+          return false;
+        }
+      case TargetPlatform.android:
+      case TargetPlatform.iOS:
+      case TargetPlatform.fuchsia:
+        return false;
+    }
+  }
+
+  Future<Map<String, Object?>> _platformStatus() async {
+    if (_usesNativeDesktopBridge) {
+      return await _nativeBridge.getStatus();
+    }
+    return const <String, Object?>{};
+  }
+
+  bool get _usesNativeDesktopBridge =>
+      defaultTargetPlatform == TargetPlatform.macOS ||
+      defaultTargetPlatform == TargetPlatform.windows;
+
+  bool get _isDesktopPlatform =>
+      defaultTargetPlatform == TargetPlatform.macOS ||
+      defaultTargetPlatform == TargetPlatform.windows ||
+      defaultTargetPlatform == TargetPlatform.linux;
+
+  Future<void> _assertInputSupported(String action) async {
+    final supported = await _inputSupported();
+    if (!supported) {
+      throw Exception(
+        '$action is not available on ${defaultTargetPlatform.name} (missing runtime permission or dependency).',
+      );
+    }
+  }
+
+  Future<DesktopCompanionSnapshot?> _safeSnapshotForStatus({
+    required String? activeDisplayId,
+    required Map<String, Object?> platformStatus,
+  }) async {
+    final permissions = _permissions(
+      const <String, Object?>{},
+      platformStatus: platformStatus,
+    );
+    final screenCaptureState =
+        permissions['screenCapture']?.toString().toLowerCase() ?? 'unknown';
+    if (screenCaptureState == 'required' ||
+        screenCaptureState == 'unsupported') {
+      return null;
+    }
+    try {
+      return await captureSnapshot(activeDisplayId: activeDisplayId);
+    } catch (_) {
+      return null;
+    }
+  }
+
+  String _normalizeMouseButton(String button) {
+    final value = button.trim().toLowerCase();
+    if (value == 'left' || value == 'right' || value == 'middle') {
+      return value;
+    }
+    return 'left';
+  }
+
+  List<Map<String, Object?>> _normalizeDisplays(
+    Object? raw, {
+    required String fallbackDisplayId,
+    required int width,
+    required int height,
+  }) {
+    if (raw is List) {
+      final displays = raw
+          .whereType<Map>()
+          .map(
+            (item) => item.map((key, value) => MapEntry(key.toString(), value)),
+          )
+          .toList(growable: false);
+      if (displays.isNotEmpty) {
+        return displays;
+      }
+    }
+    return <Map<String, Object?>>[
+      <String, Object?>{
+        'id': fallbackDisplayId,
+        'label': 'Primary Display',
+        'width': width,
+        'height': height,
+        'scaleFactor': 1,
+        'primary': true,
+      },
+    ];
+  }
+
+  Future<void> _run(_ShellCommand command) async {
+    final result = await Process.run(command.command, command.args);
+    if (result.exitCode != 0) {
+      final stderr = result.stderr?.toString().trim();
+      final stdout = result.stdout?.toString().trim();
+      final details = stderr?.isNotEmpty == true
+          ? stderr
+          : (stdout?.isNotEmpty == true ? stdout : 'unknown error');
+      throw Exception('Command failed (${command.command}): $details');
+    }
+  }
+}
+
+class _ShellCommand {
+  const _ShellCommand(this.command, this.args);
+
+  final String command;
+  final List<String> args;
+}
+
+String _shQuote(String value) => "'${value.replaceAll("'", "'\"'\"'")}'";
+
+String _psQuote(String value) => "'${value.replaceAll("'", "''")}'";
+
+String _localHostname() {
+  final host = Platform.localHostname.trim();
+  if (host.isNotEmpty) {
+    return host;
+  }
+  return defaultTargetPlatform.name;
+}