neoagent 2.4.1-beta.8 → 2.4.1

package/flutter_app/lib/src/desktop_companion_io.dart

@@ -25,10 +25,19 @@ class DesktopCompanionManager extends ChangeNotifier {
   final DesktopCompanionActions _actions;
   WebSocket? _socket;
   Timer? _reconnectTimer;
+  Timer? _connectionWatchdogTimer;
   Timer? _streamTimer;
   bool _streamCaptureInFlight = false;
+  // Set true while a click / drag / scroll / typeText / pressKey command is
+  // being executed.  _captureAndSendBinaryFrame respects this flag so it does
+  // not compete with the input command for the native bridge or the WebSocket
+  // send buffer, and a fresh frame is forced immediately after the action.
+  bool _inputCommandInFlight = false;
   int _frameSeq = 0;
   int _streamGeneration = 0;
+  // Tracks the current stream quality so the forced post-input capture can use
+  // the same setting without re-parsing the original startStream payload.
+  int _currentStreamQuality = 80;
 
   String _backendUrl = '';
   String _sessionCookie = '';
@@ -86,6 +95,7 @@ class DesktopCompanionManager extends ChangeNotifier {
       await disconnect();
       return;
     }
+    _ensureConnectionWatchdog();
     await _ensureConnected();
   }
 
@@ -130,6 +140,8 @@ class DesktopCompanionManager extends ChangeNotifier {
   Future<void> disconnect() async {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _connectionWatchdogTimer?.cancel();
+    _connectionWatchdogTimer = null;
     _stopStreaming();
     _connecting = false;
     _connected = false;
@@ -216,6 +228,7 @@ class DesktopCompanionManager extends ChangeNotifier {
         uri.toString(),
         headers: <String, Object>{'Cookie': _sessionCookie},
       );
+      socket.pingInterval = const Duration(seconds: 25);
       _socket = socket;
       socket.listen(
         _handleMessage,
@@ -286,6 +299,19 @@ class DesktopCompanionManager extends ChangeNotifier {
     }
   }
 
+  // Commands that interact with the remote machine's input system.  While one
+  // of these is executing we pause frame captures so the WebSocket send buffer
+  // is clear for the result message, and to avoid the native bridge being busy
+  // with a screenshot when the click/drag/etc. needs to run.
+  static const _inputCommands = <String>{
+    'click',
+    'mouseMove',
+    'drag',
+    'scroll',
+    'typeText',
+    'pressKey',
+  };
+
   Future<void> _handleCommand(Map<String, Object?> message) async {
     final id = message['id']?.toString() ?? '';
     final command = message['command']?.toString() ?? '';
@@ -294,6 +320,10 @@ class DesktopCompanionManager extends ChangeNotifier {
             (key, value) => MapEntry(key.toString(), value),
           )
         : const <String, Object?>{};
+
+    final isInput = _inputCommands.contains(command);
+    if (isInput) _inputCommandInFlight = true;
+
     try {
       final response = await _dispatchCommand(command, payload);
       _socket?.add(
@@ -304,6 +334,18 @@ class DesktopCompanionManager extends ChangeNotifier {
           'payload': response,
         }),
       );
+      // Immediately capture a fresh frame after an input action so the user
+      // sees the result of their interaction without waiting for the next
+      // timer tick.
+      if (isInput && _streamTimer != null && _connected) {
+        unawaited(
+          _captureAndSendBinaryFrame(
+            _currentStreamQuality,
+            _streamGeneration,
+            forced: true,
+          ),
+        );
+      }
     } catch (error) {
       _socket?.add(
         jsonEncode(<String, Object?>{
@@ -313,6 +355,8 @@ class DesktopCompanionManager extends ChangeNotifier {
           'error': '$error',
         }),
       );
+    } finally {
+      if (isInput) _inputCommandInFlight = false;
     }
   }
 
@@ -348,6 +392,12 @@ class DesktopCompanionManager extends ChangeNotifier {
           button: payload['button']?.toString() ?? 'left',
           displayId: _activeDisplayId,
         );
+      case 'mouseMove':
+        return _actions.mouseMove(
+          x: (payload['x'] as num?)?.round() ?? 0,
+          y: (payload['y'] as num?)?.round() ?? 0,
+          displayId: _activeDisplayId,
+        );
       case 'drag':
         return _actions.drag(
           x1: (payload['x1'] as num?)?.round() ?? 0,
@@ -425,6 +475,8 @@ class DesktopCompanionManager extends ChangeNotifier {
   void dispose() {
     _reconnectTimer?.cancel();
     _reconnectTimer = null;
+    _connectionWatchdogTimer?.cancel();
+    _connectionWatchdogTimer = null;
     _stopStreaming();
     _connecting = false;
     _connected = false;
@@ -441,12 +493,28 @@ class DesktopCompanionManager extends ChangeNotifier {
 
   void _scheduleReconnect() {
     if (!_enabled || !_authenticated || _sessionCookie.isEmpty) return;
+    _ensureConnectionWatchdog();
     _reconnectTimer?.cancel();
     _reconnectTimer = Timer(const Duration(seconds: 5), () {
       unawaited(_ensureConnected());
     });
   }
 
+  void _ensureConnectionWatchdog() {
+    if (!_enabled || !_authenticated || _sessionCookie.isEmpty) return;
+    if (_connectionWatchdogTimer != null) return;
+    _connectionWatchdogTimer = Timer.periodic(const Duration(seconds: 30), (_) {
+      if (!_enabled || !_authenticated || _sessionCookie.isEmpty) {
+        _connectionWatchdogTimer?.cancel();
+        _connectionWatchdogTimer = null;
+        return;
+      }
+      if (!_connected && !_connecting) {
+        unawaited(_ensureConnected());
+      }
+    });
+  }
+
   Future<void> _sendEvent(String event, Map<String, Object?> payload) async {
     final socket = _socket;
     if (socket == null || !_connected) return;
@@ -472,6 +540,7 @@ class DesktopCompanionManager extends ChangeNotifier {
     }
     final interval = Duration(milliseconds: max(1, (1000 / fps).floor()));
     _frameSeq = 0;
+    _currentStreamQuality = quality;
     _streamTimer = Timer.periodic(interval, (_) {
       unawaited(_captureAndSendBinaryFrame(quality, generation));
     });
@@ -492,7 +561,11 @@ class DesktopCompanionManager extends ChangeNotifier {
     return <String, Object?>{'success': true};
   }
 
-  Future<void> _captureAndSendBinaryFrame(int quality, int generation) async {
+  Future<void> _captureAndSendBinaryFrame(
+    int quality,
+    int generation, {
+    bool forced = false,
+  }) async {
     final socket = _socket;
     if (socket == null ||
         !_connected ||
@@ -500,6 +573,9 @@ class DesktopCompanionManager extends ChangeNotifier {
         generation != _streamGeneration) {
       return;
     }
+    // If an input command is actively running, skip this frame unless we were
+    // explicitly forced (i.e. this IS the post-input refresh capture).
+    if (!forced && _inputCommandInFlight) return;
     _streamCaptureInFlight = true;
     try {
       final snapshot = await _actions.captureSnapshot(

package/flutter_app/lib/src/desktop_native_bridge.dart

@@ -44,6 +44,19 @@ class DesktopNativeBridge {
     });
   }
 
+  Future<void> mouseMove({
+    required int x,
+    required int y,
+    String? displayId,
+  }) {
+    return _channel.invokeMethod<void>('mouseMove', <String, Object?>{
+      'x': x,
+      'y': y,
+      if (displayId != null && displayId.trim().isNotEmpty)
+        'displayId': displayId.trim(),
+    });
+  }
+
   Future<void> drag({
     required int x1,
     required int y1,

package/flutter_app/lib/src/security/password_strength.dart

@@ -0,0 +1,84 @@
+class PasswordStrengthEvaluation {
+  const PasswordStrengthEvaluation({
+    required this.score,
+    required this.containsUserInfo,
+    required this.obviousPattern,
+    required this.hasMinimumLength,
+  });
+
+  final int score;
+  final bool containsUserInfo;
+  final bool obviousPattern;
+  final bool hasMinimumLength;
+}
+
+bool hasSequentialPattern(String input) {
+  if (input.length < 4) return false;
+  
+  // check for ascending/descending sequences of length >= 4
+  for (int i = 0; i <= input.length - 4; i++) {
+    bool asc = true;
+    bool desc = true;
+    for (int j = 0; j < 3; j++) {
+      if (input.codeUnitAt(i + j + 1) != input.codeUnitAt(i + j) + 1) asc = false;
+      if (input.codeUnitAt(i + j + 1) != input.codeUnitAt(i + j) - 1) desc = false;
+    }
+    if (asc || desc) return true;
+  }
+  return false;
+}
+
+PasswordStrengthEvaluation evaluatePasswordStrength({
+  required String password,
+  String username = '',
+  String email = '',
+}) {
+  final value = password.trim();
+  if (value.isEmpty) {
+    return const PasswordStrengthEvaluation(
+      score: 0,
+      containsUserInfo: false,
+      obviousPattern: false,
+      hasMinimumLength: false,
+    );
+  }
+
+  final lower = RegExp(r'[a-z]').hasMatch(value);
+  final upper = RegExp(r'[A-Z]').hasMatch(value);
+  final digits = RegExp(r'[0-9]').hasMatch(value);
+  final symbols = RegExp(r'[^A-Za-z0-9]').hasMatch(value);
+  final variety = <bool>[lower, upper, digits, symbols]
+      .where((item) => item)
+      .length;
+  final normalized = value.toLowerCase();
+  final userHints = <String>{
+    username.trim().toLowerCase(),
+    email.trim().toLowerCase(),
+    email.trim().toLowerCase().split('@').first,
+  }.where((item) => item.length >= 3);
+  final containsUserInfo = userHints.any(normalized.contains);
+  final obviousPattern =
+      RegExp(r'(.)\1\1').hasMatch(value) ||
+      normalized.contains('password') ||
+      normalized.contains('123456') ||
+      normalized.contains('qwerty') ||
+      normalized.contains('letmein') ||
+      normalized.contains('welcome') ||
+      normalized.contains('admin') ||
+      normalized.contains('neoagent') ||
+      hasSequentialPattern(normalized);
+
+  var score = 0;
+  if (value.length >= 8) score += 1;
+  if (value.length >= 12) score += 1;
+  if (variety >= 3) score += 1;
+  if (variety == 4 || value.length >= 16) score += 1;
+  if (containsUserInfo || obviousPattern) score -= 1;
+
+  return PasswordStrengthEvaluation(
+    score: score.clamp(0, 4),
+    containsUserInfo: containsUserInfo,
+    obviousPattern: obviousPattern,
+    hasMinimumLength: value.length >= 8,
+  );
+}

package/flutter_app/lib/src/stream_renderer.dart

@@ -1,3 +1,4 @@
+import 'dart:async';
 import 'dart:typed_data';
 
 import 'package:flutter/material.dart';
@@ -9,19 +10,31 @@ class StreamRenderer extends StatefulWidget {
     required this.socket,
     required this.deviceId,
     required this.platform,
-    required this.remoteResolution,
+    this.remoteResolution,
     this.onTap,
+    this.onSwipe,
     this.onType,
+    this.onHover,
+    this.onFirstFrame,
+    this.onFrameTimeout,
+    this.firstFrameTimeout = const Duration(seconds: 8),
     this.fit = BoxFit.contain,
+    this.alignment = Alignment.center,
   });
 
   final io.Socket socket;
   final String deviceId;
   final String platform;
-  final Size remoteResolution;
+  final Size? remoteResolution;
   final void Function(double x, double y)? onTap;
+  final void Function(double x1, double y1, double x2, double y2)? onSwipe;
   final void Function(String text)? onType;
+  final void Function(double x, double y)? onHover;
+  final VoidCallback? onFirstFrame;
+  final VoidCallback? onFrameTimeout;
+  final Duration firstFrameTimeout;
   final BoxFit fit;
+  final Alignment alignment;
 
   @override
   State<StreamRenderer> createState() => _StreamRendererState();
@@ -29,6 +42,17 @@ class StreamRenderer extends StatefulWidget {
 
 class _StreamRendererState extends State<StreamRenderer> {
   Uint8List? _frame;
+  Size? _frameSize;
+  ImageStream? _imageStream;
+  ImageStreamListener? _imageListener;
+  Offset? _dragStart;
+  Offset? _dragEnd;
+
+  Timer? _hoverThrottleTimer;
+  Timer? _firstFrameTimer;
+  bool _firstFrameTimedOut = false;
+  Offset? _pendingHoverOffset;
+  DateTime _lastHoverTime = DateTime.fromMillisecondsSinceEpoch(0);
 
   @override
   void initState() {
@@ -38,6 +62,7 @@ class _StreamRendererState extends State<StreamRenderer> {
       'deviceId': widget.deviceId,
       'platform': widget.platform,
     });
+    _startFirstFrameTimer();
   }
 
   @override
@@ -48,6 +73,10 @@ class _StreamRendererState extends State<StreamRenderer> {
         oldWidget.platform == widget.platform) {
       return;
     }
+    _frameSize = null;
+    _frame = null;
+    _detachImageListener();
+    _firstFrameTimer?.cancel();
     oldWidget.socket.off('stream:frame', _onFrame);
     oldWidget.socket.emit('stream:unsubscribe', <String, Object?>{
       'deviceId': oldWidget.deviceId,
@@ -58,6 +87,24 @@ class _StreamRendererState extends State<StreamRenderer> {
       'deviceId': widget.deviceId,
       'platform': widget.platform,
     });
+    _startFirstFrameTimer();
+  }
+
+  void _startFirstFrameTimer() {
+    _firstFrameTimer?.cancel();
+    _firstFrameTimedOut = false;
+    if (widget.onFrameTimeout == null ||
+        widget.firstFrameTimeout <= Duration.zero ||
+        _frame != null) {
+      return;
+    }
+    _firstFrameTimer = Timer(widget.firstFrameTimeout, () {
+      if (!mounted || _frame != null) {
+        return;
+      }
+      _firstFrameTimedOut = true;
+      widget.onFrameTimeout?.call();
+    });
   }
 
   void _onFrame(dynamic data) {
@@ -84,9 +131,47 @@ class _StreamRendererState extends State<StreamRenderer> {
       _ => null,
     };
     if (frame == null || frame.isEmpty || !mounted) return;
+    final hadFrame = _frame != null;
+    if (!hadFrame) {
+      _firstFrameTimer?.cancel();
+      _firstFrameTimer = null;
+      if (!_firstFrameTimedOut) {
+        widget.onFirstFrame?.call();
+      }
+    }
+    if (_frameSize == null) {
+      _resolveFrameSize(frame);
+    }
     setState(() => _frame = frame);
   }
 
+  void _resolveFrameSize(Uint8List frame) {
+    _detachImageListener();
+    final provider = MemoryImage(frame);
+    final stream = provider.resolve(const ImageConfiguration());
+    final listener = ImageStreamListener((image, _) {
+      if (!mounted || _frameSize != null) return;
+      setState(() {
+        _frameSize = Size(
+          image.image.width.toDouble(),
+          image.image.height.toDouble(),
+        );
+      });
+      _detachImageListener();
+    });
+    _imageStream = stream;
+    _imageListener = listener;
+    stream.addListener(listener);
+  }
+
+  void _detachImageListener() {
+    if (_imageStream != null && _imageListener != null) {
+      _imageStream!.removeListener(_imageListener!);
+    }
+    _imageStream = null;
+    _imageListener = null;
+  }
+
   @override
   Widget build(BuildContext context) {
     final frame = _frame;
@@ -95,61 +180,146 @@ class _StreamRendererState extends State<StreamRenderer> {
     }
     return LayoutBuilder(
       builder: (context, constraints) {
-        return GestureDetector(
-          behavior: HitTestBehavior.opaque,
-          onTapDown: widget.onTap == null
+        return MouseRegion(
+          onHover: widget.onHover == null
               ? null
-              : (details) => _handleTap(details, constraints.biggest),
-          child: Image.memory(
-            frame,
-            gaplessPlayback: true,
-            fit: widget.fit,
-            width: constraints.maxWidth,
-            height: constraints.maxHeight,
+              : (event) =>
+                    _handleHoverEvent(event.localPosition, constraints.biggest),
+          child: GestureDetector(
+            behavior: HitTestBehavior.opaque,
+            onTapDown: widget.onTap == null
+                ? null
+                : (details) => _handleTap(details, constraints.biggest),
+            onPanStart: widget.onSwipe == null
+                ? null
+                : (details) {
+                    _dragStart = details.localPosition;
+                    _dragEnd = details.localPosition;
+                  },
+            onPanUpdate: widget.onSwipe == null
+                ? null
+                : (details) {
+                    _dragEnd = details.localPosition;
+                  },
+            onPanEnd: widget.onSwipe == null
+                ? null
+                : (_) {
+                    final start = _dragStart;
+                    final end = _dragEnd;
+                    _dragStart = null;
+                    _dragEnd = null;
+                    if (start == null ||
+                        end == null ||
+                        (start - end).distance < 12) {
+                      return;
+                    }
+                    _handleSwipe(start, end, constraints.biggest);
+                  },
+            child: Image.memory(
+              frame,
+              gaplessPlayback: true,
+              fit: widget.fit,
+              alignment: widget.alignment,
+              width: constraints.maxWidth,
+              height: constraints.maxHeight,
+            ),
           ),
         );
       },
     );
   }
 
+  void _handleHoverEvent(Offset localPosition, Size boxSize) {
+    if (widget.onHover == null) return;
+    _pendingHoverOffset = localPosition;
+    final now = DateTime.now();
+    final elapsed = now.difference(_lastHoverTime);
+    const throttleDuration = Duration(milliseconds: 70);
+
+    if (elapsed >= throttleDuration) {
+      _sendPendingHover(boxSize);
+    } else {
+      _hoverThrottleTimer ??= Timer(throttleDuration - elapsed, () {
+        _hoverThrottleTimer = null;
+        _sendPendingHover(boxSize);
+      });
+    }
+  }
+
+  void _sendPendingHover(Size boxSize) {
+    final offset = _pendingHoverOffset;
+    if (offset == null) return;
+    _pendingHoverOffset = null;
+    _lastHoverTime = DateTime.now();
+
+    final point = _mapToRemote(offset, boxSize);
+    if (point != null) {
+      widget.onHover?.call(point.dx, point.dy);
+    }
+  }
+
   void _handleTap(TapDownDetails details, Size boxSize) {
-    final remote = widget.remoteResolution;
-    if (remote.width <= 0 || remote.height <= 0 || boxSize.width <= 0 || boxSize.height <= 0) {
+    final point = _mapToRemote(details.localPosition, boxSize);
+    if (point == null) {
       return;
     }
-    final imageAspect = remote.width / remote.height;
-    final boxAspect = boxSize.width / boxSize.height;
-    double renderWidth;
-    double renderHeight;
-    double offsetX = 0;
-    double offsetY = 0;
-    if (boxAspect > imageAspect) {
-      renderHeight = boxSize.height;
-      renderWidth = renderHeight * imageAspect;
-      offsetX = (boxSize.width - renderWidth) / 2;
-    } else {
-      renderWidth = boxSize.width;
-      renderHeight = renderWidth / imageAspect;
-      offsetY = (boxSize.height - renderHeight) / 2;
-    }
-    final localX = details.localPosition.dx - offsetX;
-    final localY = details.localPosition.dy - offsetY;
-    if (localX < 0 || localY < 0 || localX > renderWidth || localY > renderHeight) {
+    widget.onTap?.call(point.dx, point.dy);
+  }
+
+  void _handleSwipe(Offset start, Offset end, Size boxSize) {
+    final mappedStart = _mapToRemote(start, boxSize);
+    final mappedEnd = _mapToRemote(end, boxSize);
+    if (mappedStart == null || mappedEnd == null) {
       return;
     }
-    widget.onTap?.call(
-      localX * remote.width / renderWidth,
-      localY * remote.height / renderHeight,
+    widget.onSwipe?.call(
+      mappedStart.dx,
+      mappedStart.dy,
+      mappedEnd.dx,
+      mappedEnd.dy,
+    );
+  }
+
+  Offset? _mapToRemote(Offset localPosition, Size boxSize) {
+    final remote = widget.remoteResolution ?? _frameSize;
+    if (remote == null ||
+        remote.width <= 0 ||
+        remote.height <= 0 ||
+        boxSize.width <= 0 ||
+        boxSize.height <= 0) {
+      return null;
+    }
+    final remoteSize = Size(remote.width, remote.height);
+    final fitted = applyBoxFit(widget.fit, remoteSize, boxSize);
+    final sourceRect = widget.alignment.inscribe(
+      fitted.source,
+      Offset.zero & remoteSize,
+    );
+    final destRect = widget.alignment.inscribe(
+      fitted.destination,
+      Offset.zero & boxSize,
+    );
+    if (!destRect.contains(localPosition)) {
+      return null;
+    }
+    final localX = localPosition.dx - destRect.left;
+    final localY = localPosition.dy - destRect.top;
+    return Offset(
+      sourceRect.left + localX * fitted.source.width / destRect.width,
+      sourceRect.top + localY * fitted.source.height / destRect.height,
     );
   }
 
   @override
   void dispose() {
+    _hoverThrottleTimer?.cancel();
+    _firstFrameTimer?.cancel();
     widget.socket.emit('stream:unsubscribe', <String, Object?>{
       'deviceId': widget.deviceId,
       'platform': widget.platform,
     });
     widget.socket.off('stream:frame', _onFrame);
+    _detachImageListener();
     super.dispose();
   }
 }