neoagent 2.4.0 → 2.4.1-beta.11

package/flutter_app/lib/features/onboarding/onboarding_shell.dart

@@ -2,6 +2,7 @@ import 'package:flutter/material.dart';
 import '../../main.dart';
 import 'onboarding_video_step.dart';
 import 'onboarding_welcome_step.dart';
+import 'onboarding_companion_step.dart';
 import 'onboarding_messaging_step.dart';
 import 'onboarding_model_step.dart';
 
@@ -45,6 +46,10 @@ class _OnboardingShellState extends State<OnboardingShell> {
         children: <Widget>[
           OnboardingVideoStep(onComplete: _nextStep),
           OnboardingWelcomeStep(onNext: _nextStep),
+          OnboardingCompanionStep(
+            onNext: _nextStep,
+            controller: widget.controller,
+          ),
           OnboardingMessagingStep(
             onNext: _nextStep,
             controller: widget.controller,
@@ -55,3 +60,4 @@ class _OnboardingShellState extends State<OnboardingShell> {
     );
   }
 }
+

package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart

@@ -11,7 +11,7 @@ class OnboardingWelcomeStep extends StatelessWidget {
   @override
   Widget build(BuildContext context) {
     return OnboardingScaffold(
-      step: 1,
+      step: 0,
       totalSteps: 4,
       eyebrow: 'WELCOME',
       title: 'Welcome to\nNeoOS',

package/flutter_app/lib/main.dart

@@ -39,6 +39,7 @@ import 'src/messaging_access_summary.dart';
 import 'src/oauth_launcher.dart';
 import 'src/recording_bridge.dart';
 import 'src/recording_payloads.dart';
+import 'src/stream_renderer.dart';
 import 'src/theme/palette.dart';
 import 'src/web_app_update_monitor.dart';
 import 'src/widget_bridge.dart';

package/flutter_app/lib/main_controller.dart

@@ -105,6 +105,8 @@ class NeoAgentController extends ChangeNotifier {
   bool _analyticsConsentResolved = false;
   bool _analyticsConsentGranted = false;
 
+  io.Socket? get streamSocket => socketConnected ? _socket : null;
+
   bool hasUser = true;
   bool registrationOpen = false;
   bool serviceEmailConfigured = false;
@@ -168,6 +170,8 @@ class NeoAgentController extends ChangeNotifier {
   List<McpServerItem> mcpServers = const <McpServerItem>[];
   Map<String, dynamic> browserRuntime = const <String, dynamic>{};
   Map<String, dynamic> browserExtensionStatus = const <String, dynamic>{};
+  List<Map<String, dynamic>> browserExtensionTokens =
+      const <Map<String, dynamic>>[];
   Map<String, dynamic> androidRuntime = const <String, dynamic>{};
   Map<String, dynamic> desktopRuntime = const <String, dynamic>{};
   List<String> androidInstalledApps = const <String>[];
@@ -176,6 +180,7 @@ class NeoAgentController extends ChangeNotifier {
   List<Map<String, dynamic>> desktopDisplays = const <Map<String, dynamic>>[];
   Map<String, dynamic> desktopPermissions = const <String, dynamic>{};
   String? selectedDesktopDeviceId;
+  String? selectedBrowserExtensionTokenId;
   String? browserScreenshotPath;
   String? androidScreenshotPath;
   String? desktopScreenshotPath;
@@ -893,8 +898,9 @@ class NeoAgentController extends ChangeNotifier {
       // On web, require explicit opt-in (GDPR). On native, consent is implicit
       // unless the user has previously declined.
       _analyticsConsentResolved = consentState != null || !kIsWeb;
-      _analyticsConsentGranted =
-          kIsWeb ? consentState == true : consentState != false;
+      _analyticsConsentGranted = kIsWeb
+          ? consentState == true
+          : consentState != false;
       await _analytics.initialize(
         token: token,
         consentGranted: _analyticsConsentGranted,
@@ -1040,6 +1046,15 @@ class NeoAgentController extends ChangeNotifier {
     }
   }
 
+  Future<Map<String, dynamic>> testCliRuntime() =>
+      _backendClient.testCli(backendUrl);
+
+  Future<Map<String, dynamic>> testBrowserExtension() =>
+      _backendClient.testExtension(backendUrl);
+
+  Future<Map<String, dynamic>> testDesktopCompanion() =>
+      _backendClient.testDesktop(backendUrl);
+
   Future<void> openAppUpdate() async {
     final release = availableAppUpdate;
     if (release == null || isOpeningAppUpdate) {
@@ -1644,6 +1659,7 @@ class NeoAgentController extends ChangeNotifier {
     mcpServers = const <McpServerItem>[];
     browserRuntime = const <String, dynamic>{};
     browserExtensionStatus = const <String, dynamic>{};
+    browserExtensionTokens = const <Map<String, dynamic>>[];
     androidRuntime = const <String, dynamic>{};
     desktopRuntime = const <String, dynamic>{};
     androidInstalledApps = const <String>[];
@@ -1652,6 +1668,7 @@ class NeoAgentController extends ChangeNotifier {
     desktopDisplays = const <Map<String, dynamic>>[];
     desktopPermissions = const <String, dynamic>{};
     selectedDesktopDeviceId = null;
+    selectedBrowserExtensionTokenId = null;
     browserScreenshotPath = null;
     androidScreenshotPath = null;
     desktopScreenshotPath = null;
@@ -2333,6 +2350,13 @@ class NeoAgentController extends ChangeNotifier {
       browserExtensionStatus = Map<String, dynamic>.from(
         browserExtensionResponse,
       );
+      browserExtensionTokens = _jsonMapList(
+        browserExtensionStatus['tokens'],
+        fallbackToMapValues: true,
+      );
+      selectedBrowserExtensionTokenId = _optionalIdFrom(
+        browserExtensionStatus['selectedTokenId'],
+      );
       androidRuntime = Map<String, dynamic>.from(androidResponse);
       desktopRuntime = Map<String, dynamic>.from(desktopResponse);
       selectedDesktopDeviceId =
@@ -2428,6 +2452,11 @@ class NeoAgentController extends ChangeNotifier {
     return parsed;
   }
 
+  String? _optionalIdFrom(dynamic value) {
+    final normalized = value?.toString().trim() ?? '';
+    return normalized.isEmpty || normalized == 'null' ? null : normalized;
+  }
+
   Future<void> refreshRunsOnly() async {
     try {
       final runsResponse = await _backendClient.fetchRuns(
@@ -2731,6 +2760,13 @@ class NeoAgentController extends ChangeNotifier {
       browserExtensionStatus = Map<String, dynamic>.from(
         browserExtensionResponse,
       );
+      browserExtensionTokens = _jsonMapList(
+        browserExtensionStatus['tokens'],
+        fallbackToMapValues: true,
+      );
+      selectedBrowserExtensionTokenId = _optionalIdFrom(
+        browserExtensionStatus['selectedTokenId'],
+      );
       androidRuntime = Map<String, dynamic>.from(androidResponse);
       desktopRuntime = Map<String, dynamic>.from(desktopResponse);
       selectedDesktopDeviceId =
@@ -2758,6 +2794,13 @@ class NeoAgentController extends ChangeNotifier {
         backendUrl,
       );
       browserExtensionStatus = Map<String, dynamic>.from(response);
+      browserExtensionTokens = _jsonMapList(
+        browserExtensionStatus['tokens'],
+        fallbackToMapValues: true,
+      );
+      selectedBrowserExtensionTokenId = _optionalIdFrom(
+        browserExtensionStatus['selectedTokenId'],
+      );
       notifyListeners();
     } catch (error) {
       errorMessage = _friendlyErrorMessage(error);
@@ -2902,6 +2945,15 @@ class NeoAgentController extends ChangeNotifier {
     );
   }
 
+  Future<void> hoverBrowserPointRuntime({
+    required int x,
+    required int y,
+  }) async {
+    try {
+      await _backendClient.hoverBrowserPoint(backendUrl, x: x, y: y);
+    } catch (_) {}
+  }
+
   Future<void> fillBrowserRuntime({
     required String selector,
     required String value,
@@ -3027,6 +3079,40 @@ class NeoAgentController extends ChangeNotifier {
     } catch (_) {}
   }
 
+  Future<void> startStreamRuntime({
+    required String platform,
+    required String deviceId,
+    int fps = 10,
+    int quality = 70,
+  }) async {
+    final normalizedDeviceId = deviceId.trim();
+    if (normalizedDeviceId.isEmpty) {
+      return;
+    }
+    await _backendClient.startStream(
+      backendUrl,
+      platform: platform,
+      deviceId: normalizedDeviceId,
+      fps: fps,
+      quality: quality,
+    );
+  }
+
+  Future<void> stopStreamRuntime({
+    required String platform,
+    required String deviceId,
+  }) async {
+    final normalizedDeviceId = deviceId.trim();
+    if (normalizedDeviceId.isEmpty) {
+      return;
+    }
+    await _backendClient.stopStream(
+      backendUrl,
+      platform: platform,
+      deviceId: normalizedDeviceId,
+    );
+  }
+
   Future<void> dumpAndroidUiRuntime() async {
     await _runDeviceAction(
       () => _backendClient.dumpAndroidUi(backendUrl),
@@ -3194,6 +3280,44 @@ class NeoAgentController extends ChangeNotifier {
     await refreshDesktopFrameRuntime();
   }
 
+  Future<void> selectBrowserExtensionRuntime(String tokenId) async {
+    isRunningDeviceAction = true;
+    errorMessage = null;
+    notifyListeners();
+    try {
+      final response = await _backendClient.selectBrowserExtensionToken(
+        backendUrl,
+        tokenId: tokenId,
+      );
+      final status = response['status'] is Map
+          ? Map<String, dynamic>.from(response['status'] as Map)
+          : await _backendClient.fetchBrowserExtensionStatus(backendUrl);
+      browserExtensionStatus = Map<String, dynamic>.from(status);
+      browserExtensionTokens = _jsonMapList(
+        browserExtensionStatus['tokens'],
+        fallbackToMapValues: true,
+      );
+      selectedBrowserExtensionTokenId = _optionalIdFrom(
+        browserExtensionStatus['selectedTokenId'],
+      );
+      if (selectedBrowserExtensionTokenId != null) {
+        settings = <String, dynamic>{
+          ...settings,
+          'browser_extension_token_id': selectedBrowserExtensionTokenId,
+          'selected_browser_extension_token_id':
+              selectedBrowserExtensionTokenId,
+        };
+      }
+      browserScreenshotPath = null;
+      await refreshBrowserFrameRuntime();
+    } catch (error) {
+      errorMessage = _friendlyErrorMessage(error);
+    } finally {
+      isRunningDeviceAction = false;
+      notifyListeners();
+    }
+  }
+
   Future<void> openDesktopSelectionRuntime() async {
     await refreshDevices();
     errorMessage =
@@ -3272,6 +3396,17 @@ class NeoAgentController extends ChangeNotifier {
     );
   }
 
+  Future<void> hoverDesktopRuntime({required int x, required int y}) async {
+    try {
+      await _backendClient.hoverDesktop(
+        backendUrl,
+        deviceId: selectedDesktopDeviceId,
+        x: x,
+        y: y,
+      );
+    } catch (_) {}
+  }
+
   Future<void> dragDesktopRuntime({
     required int x1,
     required int y1,
@@ -4564,7 +4699,9 @@ class NeoAgentController extends ChangeNotifier {
 
   Future<void> saveSettings({
     required String browserBackend,
+    String? browserExtensionTokenId,
     required String cliBackend,
+    String? cliDesktopDeviceId,
     required bool smarterSelector,
     required List<String> enabledModels,
     required String defaultChatModel,
@@ -4593,7 +4730,11 @@ class NeoAgentController extends ChangeNotifier {
     final payload = <String, dynamic>{
       'headless_browser': true,
       'browser_backend': browserBackend,
+      if (browserExtensionTokenId != null)
+        'browser_extension_token_id': browserExtensionTokenId,
       'cli_backend': cliBackend,
+      if (cliDesktopDeviceId != null)
+        'cli_desktop_device_id': cliDesktopDeviceId,
       'smarter_model_selector': smarterSelector,
       'enabled_models': enabledModels,
       'default_chat_model': defaultChatModel,
@@ -6131,6 +6272,11 @@ class NeoAgentController extends ChangeNotifier {
   String get cliBackend =>
       settings['cli_backend']?.toString().trim().toLowerCase() ?? 'vm';
 
+  String? get cliDesktopDeviceId {
+    final v = settings['cli_desktop_device_id']?.toString().trim();
+    return (v == null || v.isEmpty) ? null : v;
+  }
+
   String get cloudBrowserBackend {
     final browser = browserBackend;
     final profile = settings['runtime_profile']
@@ -6154,6 +6300,11 @@ class NeoAgentController extends ChangeNotifier {
   bool get browserExtensionConnected =>
       browserExtensionStatus['connected'] == true;
 
+  String? get browserExtensionTokenId {
+    final v = settings['browser_extension_token_id']?.toString().trim();
+    return (v == null || v.isEmpty || v == 'null') ? null : v;
+  }
+
   bool get smarterSelector => settings['smarter_model_selector'] != false;
 
   Map<String, AiProviderConfig> get aiProviderConfigs {
@@ -6336,7 +6487,9 @@ class NeoAgentController extends ChangeNotifier {
 
   List<ChatEntry> get visibleChatMessages {
     final entries = <ChatEntry>[...chatMessages];
-    if (isSendingMessage && activeRun != null && streamingAssistant.trim().isEmpty) {
+    if (isSendingMessage &&
+        activeRun != null &&
+        streamingAssistant.trim().isEmpty) {
       entries.add(
         ChatEntry(
           id: '',