neoagent 2.3.1-beta.6 → 2.3.1-beta.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (312) hide show
  1. package/.env.example +45 -0
  2. package/docs/capabilities.md +2 -2
  3. package/docs/configuration.md +6 -5
  4. package/docs/hardware.md +1 -1
  5. package/docs/integrations.md +2 -3
  6. package/flutter_app/.metadata +42 -0
  7. package/flutter_app/README.md +21 -0
  8. package/flutter_app/analysis_options.yaml +32 -0
  9. package/flutter_app/android/app/build.gradle.kts +111 -0
  10. package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
  11. package/flutter_app/android/app/src/main/AndroidManifest.xml +171 -0
  12. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +804 -0
  13. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/auto/NeoAgentCarAppService.kt +114 -0
  14. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
  15. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
  16. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
  17. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
  18. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
  19. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
  20. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
  21. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
  22. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
  23. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
  24. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnection.kt +118 -0
  25. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnectionService.kt +86 -0
  26. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
  27. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
  28. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
  29. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
  30. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
  31. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
  32. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
  33. package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
  34. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
  35. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
  36. package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
  37. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
  38. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
  39. package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
  40. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  41. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  42. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  43. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  44. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  45. package/flutter_app/android/app/src/main/res/values/arrays.xml +8 -0
  46. package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
  47. package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
  48. package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
  49. package/flutter_app/android/app/src/main/res/xml/automotive_app_desc.xml +4 -0
  50. package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
  51. package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
  52. package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
  53. package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
  54. package/flutter_app/android/build.gradle.kts +24 -0
  55. package/flutter_app/android/ci-release.keystore +0 -0
  56. package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
  57. package/flutter_app/android/gradle.properties +3 -0
  58. package/flutter_app/android/key.properties +4 -0
  59. package/flutter_app/android/settings.gradle.kts +26 -0
  60. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  61. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  62. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  63. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  64. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  65. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  66. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  67. package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
  68. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  69. package/flutter_app/lib/features/location/location_service.dart +119 -0
  70. package/flutter_app/lib/features/notifications/notification_interceptor.dart +97 -0
  71. package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +571 -0
  72. package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +275 -0
  73. package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +366 -0
  74. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +57 -0
  75. package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +218 -0
  76. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +46 -0
  77. package/flutter_app/lib/main.dart +103 -0
  78. package/flutter_app/lib/main_account_settings.dart +1250 -0
  79. package/flutter_app/lib/main_admin.dart +886 -0
  80. package/flutter_app/lib/main_app_shell.dart +2134 -0
  81. package/flutter_app/lib/main_chat.dart +3316 -0
  82. package/flutter_app/lib/main_controller.dart +6836 -0
  83. package/flutter_app/lib/main_devices.dart +2396 -0
  84. package/flutter_app/lib/main_integrations.dart +909 -0
  85. package/flutter_app/lib/main_launcher.dart +959 -0
  86. package/flutter_app/lib/main_launcher_entry.dart +5 -0
  87. package/flutter_app/lib/main_models.dart +3539 -0
  88. package/flutter_app/lib/main_navigation.dart +188 -0
  89. package/flutter_app/lib/main_operations.dart +4851 -0
  90. package/flutter_app/lib/main_recordings.dart +920 -0
  91. package/flutter_app/lib/main_runtime.dart +838 -0
  92. package/flutter_app/lib/main_settings.dart +2040 -0
  93. package/flutter_app/lib/main_shared.dart +3147 -0
  94. package/flutter_app/lib/main_theme.dart +257 -0
  95. package/flutter_app/lib/main_voice_assistant.dart +957 -0
  96. package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
  97. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
  98. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
  99. package/flutter_app/lib/src/android_app_installer.dart +22 -0
  100. package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
  101. package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
  102. package/flutter_app/lib/src/android_auto_bridge.dart +59 -0
  103. package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
  104. package/flutter_app/lib/src/app_launch_bridge.dart +29 -0
  105. package/flutter_app/lib/src/app_release_updater.dart +511 -0
  106. package/flutter_app/lib/src/backend_client.dart +1837 -0
  107. package/flutter_app/lib/src/desktop_companion.dart +2 -0
  108. package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
  109. package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
  110. package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
  111. package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
  112. package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
  113. package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
  114. package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
  115. package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
  116. package/flutter_app/lib/src/health_bridge.dart +136 -0
  117. package/flutter_app/lib/src/live_voice_capture.dart +85 -0
  118. package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
  119. package/flutter_app/lib/src/network/app_http_client.dart +53 -0
  120. package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
  121. package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
  122. package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
  123. package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
  124. package/flutter_app/lib/src/oauth_launcher.dart +33 -0
  125. package/flutter_app/lib/src/oauth_launcher_io.dart +95 -0
  126. package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
  127. package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
  128. package/flutter_app/lib/src/recording_bridge.dart +232 -0
  129. package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
  130. package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
  131. package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
  132. package/flutter_app/lib/src/recording_payloads.dart +86 -0
  133. package/flutter_app/lib/src/theme/palette.dart +81 -0
  134. package/flutter_app/lib/src/web_app_update_monitor.dart +17 -0
  135. package/flutter_app/lib/src/web_app_update_monitor_stub.dart +24 -0
  136. package/flutter_app/lib/src/web_app_update_monitor_web.dart +123 -0
  137. package/flutter_app/lib/src/widget_bridge.dart +49 -0
  138. package/flutter_app/linux/CMakeLists.txt +128 -0
  139. package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
  140. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +47 -0
  141. package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
  142. package/flutter_app/linux/flutter/generated_plugins.cmake +32 -0
  143. package/flutter_app/linux/runner/CMakeLists.txt +26 -0
  144. package/flutter_app/linux/runner/main.cc +6 -0
  145. package/flutter_app/linux/runner/my_application.cc +144 -0
  146. package/flutter_app/linux/runner/my_application.h +18 -0
  147. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  148. package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
  149. package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
  150. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +44 -0
  151. package/flutter_app/macos/Podfile +42 -0
  152. package/flutter_app/macos/Podfile.lock +87 -0
  153. package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
  154. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
  155. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  156. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  157. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  158. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  159. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  160. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  161. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  162. package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
  163. package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
  164. package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
  165. package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
  166. package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
  167. package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
  168. package/flutter_app/macos/Runner/Info.plist +36 -0
  169. package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
  170. package/flutter_app/macos/Runner/Release.entitlements +12 -0
  171. package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
  172. package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  173. package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
  174. package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
  175. package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  176. package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
  177. package/flutter_app/patch_strings.py +12 -0
  178. package/flutter_app/pubspec.lock +1224 -0
  179. package/flutter_app/pubspec.yaml +57 -0
  180. package/flutter_app/test/messaging_access_summary_test.dart +22 -0
  181. package/flutter_app/test/recording_payloads_test.dart +53 -0
  182. package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
  183. package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
  184. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
  185. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
  186. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
  187. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
  188. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
  189. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
  190. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
  191. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
  192. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
  193. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
  194. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
  195. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
  196. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
  197. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
  198. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
  199. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
  200. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
  201. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
  202. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
  203. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
  204. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
  205. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
  206. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
  207. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
  208. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
  209. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
  210. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
  211. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
  212. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
  213. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
  214. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
  215. package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
  216. package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
  217. package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
  218. package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
  219. package/flutter_app/tool/generate_desktop_branding.py +219 -0
  220. package/flutter_app/web/favicon.png +0 -0
  221. package/flutter_app/web/favicon.svg +12 -0
  222. package/flutter_app/web/icons/Icon-192.png +0 -0
  223. package/flutter_app/web/icons/Icon-512.png +0 -0
  224. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  225. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  226. package/flutter_app/web/index.html +39 -0
  227. package/flutter_app/web/manifest.json +35 -0
  228. package/flutter_app/windows/CMakeLists.txt +108 -0
  229. package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
  230. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +50 -0
  231. package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
  232. package/flutter_app/windows/flutter/generated_plugins.cmake +36 -0
  233. package/flutter_app/windows/runner/CMakeLists.txt +41 -0
  234. package/flutter_app/windows/runner/Runner.rc +121 -0
  235. package/flutter_app/windows/runner/flutter_window.cpp +533 -0
  236. package/flutter_app/windows/runner/flutter_window.h +37 -0
  237. package/flutter_app/windows/runner/main.cpp +53 -0
  238. package/flutter_app/windows/runner/resource.h +16 -0
  239. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  240. package/flutter_app/windows/runner/runner.exe.manifest +14 -0
  241. package/flutter_app/windows/runner/utils.cpp +65 -0
  242. package/flutter_app/windows/runner/utils.h +19 -0
  243. package/flutter_app/windows/runner/win32_window.cpp +299 -0
  244. package/flutter_app/windows/runner/win32_window.h +102 -0
  245. package/lib/install_helpers.js +31 -0
  246. package/lib/manager.js +201 -6
  247. package/package.json +5 -3
  248. package/server/db/database.js +125 -1
  249. package/server/http/middleware.js +55 -2
  250. package/server/http/routes.js +1 -0
  251. package/server/http/static.js +41 -1
  252. package/server/index.js +3 -0
  253. package/server/public/.last_build_id +1 -1
  254. package/server/public/assets/AssetManifest.bin +1 -1
  255. package/server/public/assets/AssetManifest.bin.json +1 -1
  256. package/server/public/assets/NOTICES +281 -1
  257. package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
  258. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  259. package/server/public/canvaskit/wimp.wasm +0 -0
  260. package/server/public/flutter_bootstrap.js +2 -2
  261. package/server/public/main.dart.js +86707 -82621
  262. package/server/routes/auth.js +26 -5
  263. package/server/routes/integrations.js +108 -1
  264. package/server/routes/memory.js +11 -2
  265. package/server/routes/settings.js +84 -2
  266. package/server/routes/wearable.js +67 -0
  267. package/server/services/ai/engine.js +3 -2
  268. package/server/services/ai/models.js +30 -0
  269. package/server/services/ai/providers/githubCopilot.js +97 -0
  270. package/server/services/ai/providers/openai.js +2 -1
  271. package/server/services/ai/providers/openaiCodex.js +31 -0
  272. package/server/services/ai/settings.js +20 -0
  273. package/server/services/ai/toolSelector.js +54 -1
  274. package/server/services/ai/tools.js +88 -5
  275. package/server/services/desktop/screenRecorder.js +65 -9
  276. package/server/services/integrations/env.js +5 -0
  277. package/server/services/integrations/figma/provider.js +1 -0
  278. package/server/services/integrations/github/common.js +106 -0
  279. package/server/services/integrations/github/provider.js +499 -0
  280. package/server/services/integrations/github/repos.js +1124 -0
  281. package/server/services/integrations/google/provider.js +1 -0
  282. package/server/services/integrations/manager.js +88 -12
  283. package/server/services/integrations/microsoft/provider.js +1 -0
  284. package/server/services/integrations/oauth_provider.js +25 -8
  285. package/server/services/integrations/provider_config_store.js +85 -0
  286. package/server/services/integrations/registry.js +4 -2
  287. package/server/services/integrations/spotify/provider.js +1 -0
  288. package/server/services/integrations/trello/provider.js +842 -0
  289. package/server/services/manager.js +46 -1
  290. package/server/services/mcp/client.js +120 -23
  291. package/server/services/memory/manager.js +39 -2
  292. package/server/services/messaging/access_policy.js +10 -0
  293. package/server/services/messaging/manager.js +49 -0
  294. package/server/services/messaging/meshtastic.js +277 -0
  295. package/server/services/messaging/meshtastic_env.js +40 -0
  296. package/server/services/messaging/meshtastic_protocol.js +530 -0
  297. package/server/services/messaging/meshtastic_protocol.test.js +33 -0
  298. package/server/services/messaging/meshtastic_tcp_transport.js +25 -0
  299. package/server/services/tasks/runtime.js +1 -1
  300. package/server/services/voice/openaiClient.js +4 -1
  301. package/server/services/voice/openaiSpeech.js +6 -1
  302. package/server/services/voice/providers.js +144 -15
  303. package/server/services/voice/runtimeManager.js +156 -19
  304. package/server/services/voice/turnRunner.js +29 -9
  305. package/server/services/wearable/firmware_manifest.js +353 -0
  306. package/server/services/wearable/gateway.js +350 -0
  307. package/server/services/wearable/protocol.js +45 -0
  308. package/server/services/wearable/service.js +240 -0
  309. package/server/services/widgets/service.js +3 -0
  310. package/server/utils/local_secrets.js +56 -0
  311. package/server/utils/logger.js +37 -9
  312. package/server/services/integrations/home_assistant/provider.js +0 -350
@@ -0,0 +1,2396 @@
1
+ part of 'main.dart';
2
+
3
+ class _OptimizedScreenshotPayload {
4
+ const _OptimizedScreenshotPayload({
5
+ required this.bytes,
6
+ required this.mimeType,
7
+ required this.resizedOrReencoded,
8
+ });
9
+
10
+ final Uint8List bytes;
11
+ final String mimeType;
12
+ final bool resizedOrReencoded;
13
+ }
14
+
15
+ class DevicesPanel extends StatefulWidget {
16
+ const DevicesPanel({super.key, required this.controller});
17
+
18
+ final NeoAgentController controller;
19
+
20
+ @override
21
+ State<DevicesPanel> createState() => _DevicesPanelState();
22
+ }
23
+
24
+ class _DevicesPanelState extends State<DevicesPanel> {
25
+ late final TextEditingController _browserUrlController;
26
+ late final TextEditingController _androidLaunchController;
27
+ late final TextEditingController _desktopLaunchController;
28
+ late final TextEditingController _textEntryController;
29
+ Timer? _surfaceFrameTimer;
30
+ _DeviceSurface _surface = _DeviceSurface.browser;
31
+
32
+ @override
33
+ void initState() {
34
+ super.initState();
35
+ _browserUrlController = TextEditingController(text: _browserUrlPlaceholder);
36
+ _androidLaunchController = TextEditingController(
37
+ text: _androidLaunchPlaceholder,
38
+ );
39
+ _desktopLaunchController = TextEditingController();
40
+ _textEntryController = TextEditingController();
41
+ WidgetsBinding.instance.addPostFrameCallback((_) {
42
+ if (!mounted) {
43
+ return;
44
+ }
45
+ unawaited(_bootstrapSurface());
46
+ });
47
+ _surfaceFrameTimer = Timer.periodic(const Duration(seconds: 3), (_) {
48
+ unawaited(_refreshSurfaceFrame());
49
+ });
50
+ }
51
+
52
+ @override
53
+ void dispose() {
54
+ for (final controller in <TextEditingController>[
55
+ _browserUrlController,
56
+ _androidLaunchController,
57
+ _desktopLaunchController,
58
+ _textEntryController,
59
+ ]) {
60
+ controller.dispose();
61
+ }
62
+ _surfaceFrameTimer?.cancel();
63
+ super.dispose();
64
+ }
65
+
66
+ bool get _isBrowser => _surface == _DeviceSurface.browser;
67
+ bool get _isDesktop => _surface == _DeviceSurface.desktop;
68
+
69
+ bool get _androidOnline {
70
+ final status = widget.controller.androidRuntime;
71
+ final devices = _jsonMapList(status['devices'], fallbackToMapValues: true);
72
+ return devices.any((device) => device['status']?.toString() == 'device');
73
+ }
74
+
75
+ bool get _androidStarting =>
76
+ widget.controller.androidRuntime['starting'] == true;
77
+
78
+ List<Map<String, dynamic>> get _onlineDesktopDevices => widget
79
+ .controller
80
+ .desktopDevices
81
+ .where((device) => device['online'] == true)
82
+ .toList(growable: false);
83
+
84
+ bool get _desktopOnline => _onlineDesktopDevices.isNotEmpty;
85
+
86
+ bool get _desktopRequiresSelection =>
87
+ _isDesktop &&
88
+ _onlineDesktopDevices.length > 1 &&
89
+ (widget.controller.selectedDesktopDeviceId ?? '').isEmpty;
90
+
91
+ String? get _activeScreenshotPath {
92
+ if (_isBrowser) {
93
+ return widget.controller.browserScreenshotPath;
94
+ }
95
+ if (_isDesktop) {
96
+ return widget.controller.desktopScreenshotPath;
97
+ }
98
+ return widget.controller.androidScreenshotPath;
99
+ }
100
+
101
+ Future<void> _bootstrapSurface() async {
102
+ await widget.controller.refreshDevices();
103
+ await _ensurePreview();
104
+ }
105
+
106
+ Future<void> _ensurePreview() async {
107
+ final controller = widget.controller;
108
+ if (_isBrowser) {
109
+ if (controller.browserRuntime['launched'] != true) {
110
+ return;
111
+ }
112
+ if ((controller.browserScreenshotPath ?? '').isEmpty) {
113
+ final currentUrl =
114
+ controller.browserRuntime['pageInfo'] is Map<dynamic, dynamic>
115
+ ? (controller.browserRuntime['pageInfo'] as Map)['url']?.toString()
116
+ : null;
117
+ if (currentUrl != null && currentUrl.isNotEmpty) {
118
+ await controller.navigateBrowserRuntime(url: currentUrl);
119
+ } else {
120
+ await controller.screenshotBrowserRuntime();
121
+ }
122
+ }
123
+ return;
124
+ }
125
+
126
+ if (_isDesktop) {
127
+ if (_desktopRequiresSelection || !_desktopOnline) {
128
+ return;
129
+ }
130
+ if ((controller.desktopScreenshotPath ?? '').isEmpty) {
131
+ await controller.screenshotDesktopRuntime();
132
+ }
133
+ return;
134
+ }
135
+
136
+ if (_androidOnline && (controller.androidScreenshotPath ?? '').isEmpty) {
137
+ await controller.screenshotAndroidRuntime();
138
+ }
139
+ }
140
+
141
+ Future<void> _refreshSurfaceFrame() async {
142
+ if (!mounted ||
143
+ widget.controller.selectedSection != AppSection.devices ||
144
+ widget.controller.isRunningDeviceAction ||
145
+ widget.controller.isRefreshingDevices) {
146
+ return;
147
+ }
148
+ if (_isBrowser) {
149
+ await widget.controller.refreshBrowserFrameRuntime();
150
+ return;
151
+ }
152
+ if (_isDesktop) {
153
+ await widget.controller.refreshDesktopFrameRuntime();
154
+ return;
155
+ }
156
+ if (_androidStarting) {
157
+ await widget.controller.refreshDevices();
158
+ if (_androidOnline &&
159
+ (widget.controller.androidScreenshotPath ?? '').isEmpty) {
160
+ await widget.controller.screenshotAndroidRuntime();
161
+ }
162
+ return;
163
+ }
164
+ await widget.controller.refreshAndroidFrameRuntime();
165
+ }
166
+
167
+ Future<void> _switchSurface(int delta) async {
168
+ final surfaces = _DeviceSurface.values;
169
+ final currentIndex = surfaces.indexOf(_surface);
170
+ final nextIndex = (currentIndex + delta) % surfaces.length;
171
+ setState(
172
+ () =>
173
+ _surface = surfaces[nextIndex < 0 ? surfaces.length - 1 : nextIndex],
174
+ );
175
+ await _ensurePreview();
176
+ }
177
+
178
+ Future<void> _openPrimary() async {
179
+ final controller = widget.controller;
180
+ if (_isBrowser) {
181
+ await controller.navigateBrowserRuntime(
182
+ url: _browserUrlController.text.trim(),
183
+ );
184
+ return;
185
+ }
186
+
187
+ if (_isDesktop) {
188
+ if (_desktopRequiresSelection) {
189
+ final selectedId = widget.controller.selectedDesktopDeviceId;
190
+ if ((selectedId ?? '').isEmpty && _onlineDesktopDevices.length == 1) {
191
+ await controller.selectDesktopDeviceRuntime(
192
+ _onlineDesktopDevices.first['deviceId']?.toString() ?? '',
193
+ );
194
+ } else {
195
+ await controller.openDesktopSelectionRuntime();
196
+ }
197
+ return;
198
+ }
199
+ if (!_desktopOnline) {
200
+ return;
201
+ }
202
+ final selectedId = widget.controller.selectedDesktopDeviceId;
203
+ if ((selectedId ?? '').isEmpty && _onlineDesktopDevices.length == 1) {
204
+ await controller.selectDesktopDeviceRuntime(
205
+ _onlineDesktopDevices.first['deviceId']?.toString() ?? '',
206
+ );
207
+ }
208
+ final raw = _desktopLaunchController.text.trim();
209
+ if (raw.isNotEmpty) {
210
+ await controller.launchDesktopAppRuntime(raw);
211
+ return;
212
+ }
213
+ await controller.observeDesktopRuntime();
214
+ return;
215
+ }
216
+
217
+ if (!_androidOnline) {
218
+ await controller.startAndroidRuntime();
219
+ await widget.controller.refreshDevices();
220
+ if (_androidOnline) {
221
+ await controller.screenshotAndroidRuntime();
222
+ }
223
+ return;
224
+ }
225
+
226
+ final raw = _androidLaunchController.text.trim();
227
+ if (raw.isEmpty) {
228
+ return;
229
+ }
230
+ if (raw.startsWith('http://') || raw.startsWith('https://')) {
231
+ await controller.openAndroidIntentRuntime(
232
+ action: 'android.intent.action.VIEW',
233
+ dataUri: raw,
234
+ );
235
+ return;
236
+ }
237
+ await controller.openAndroidAppRuntime(packageName: raw);
238
+ }
239
+
240
+ Future<void> _sleepPrimary() async {
241
+ final controller = widget.controller;
242
+ if (_isBrowser) {
243
+ if (controller.browserRuntime['launched'] != true) {
244
+ return;
245
+ }
246
+ await controller.closeBrowserRuntime();
247
+ return;
248
+ }
249
+ if (_isDesktop) {
250
+ final selectedId = widget.controller.selectedDesktopDeviceId;
251
+ if ((selectedId ?? '').isNotEmpty) {
252
+ await controller.pauseDesktopDeviceRuntime(selectedId!);
253
+ }
254
+ return;
255
+ }
256
+ if (!_androidOnline) {
257
+ return;
258
+ }
259
+ await controller.stopAndroidRuntime();
260
+ }
261
+
262
+ Future<void> _sendText() async {
263
+ final text = _textEntryController.text;
264
+ if (text.trim().isEmpty) {
265
+ return;
266
+ }
267
+ if (_isBrowser) {
268
+ await widget.controller.typeBrowserTextRuntime(text, pressEnter: true);
269
+ } else if (_isDesktop) {
270
+ await widget.controller.typeDesktopRuntime(text, pressEnter: true);
271
+ } else {
272
+ await widget.controller.typeAndroidRuntime(<String, dynamic>{
273
+ 'text': text,
274
+ 'pressEnter': true,
275
+ });
276
+ }
277
+ }
278
+
279
+ Future<void> _handleTap(Offset point) async {
280
+ if (_isBrowser) {
281
+ await widget.controller.clickBrowserPointRuntime(
282
+ x: point.dx.round(),
283
+ y: point.dy.round(),
284
+ );
285
+ return;
286
+ }
287
+ if (_isDesktop) {
288
+ if (_desktopRequiresSelection) {
289
+ return;
290
+ }
291
+ await widget.controller.clickDesktopRuntime(
292
+ x: point.dx.round(),
293
+ y: point.dy.round(),
294
+ );
295
+ return;
296
+ }
297
+ if (!_androidOnline) {
298
+ await widget.controller.startAndroidRuntime();
299
+ return;
300
+ }
301
+ await widget.controller.tapAndroidRuntime(<String, dynamic>{
302
+ 'x': point.dx.round(),
303
+ 'y': point.dy.round(),
304
+ });
305
+ }
306
+
307
+ Future<void> _handleSwipe(Offset start, Offset end) async {
308
+ if (_isBrowser) {
309
+ await widget.controller.scrollBrowserRuntime(
310
+ deltaY: (start.dy - end.dy).round(),
311
+ );
312
+ return;
313
+ }
314
+ if (_isDesktop) {
315
+ if (_desktopRequiresSelection) {
316
+ return;
317
+ }
318
+ await widget.controller.dragDesktopRuntime(
319
+ x1: start.dx.round(),
320
+ y1: start.dy.round(),
321
+ x2: end.dx.round(),
322
+ y2: end.dy.round(),
323
+ );
324
+ return;
325
+ }
326
+ if (!_androidOnline) {
327
+ return;
328
+ }
329
+ await widget.controller.swipeAndroidRuntime(<String, dynamic>{
330
+ 'x1': start.dx.round(),
331
+ 'y1': start.dy.round(),
332
+ 'x2': end.dx.round(),
333
+ 'y2': end.dy.round(),
334
+ 'durationMs': 280,
335
+ });
336
+ }
337
+
338
+ Future<void> _runQuickAction(String action) async {
339
+ final controller = widget.controller;
340
+ switch (action) {
341
+ case 'browser_refresh':
342
+ await controller.navigateBrowserRuntime(
343
+ url: controller.browserRuntime['pageInfo'] is Map<dynamic, dynamic>
344
+ ? ((controller.browserRuntime['pageInfo'] as Map)['url']
345
+ ?.toString() ??
346
+ _browserUrlController.text.trim())
347
+ : _browserUrlController.text.trim(),
348
+ );
349
+ break;
350
+ case 'browser_enter':
351
+ await controller.pressBrowserKeyRuntime('Enter');
352
+ break;
353
+ case 'desktop_enter':
354
+ await controller.pressDesktopKeyRuntime('Return');
355
+ break;
356
+ case 'desktop_escape':
357
+ await controller.pressDesktopKeyRuntime('Escape');
358
+ break;
359
+ case 'android_back':
360
+ await controller.pressAndroidKeyRuntime('back');
361
+ break;
362
+ case 'android_home':
363
+ await controller.pressAndroidKeyRuntime('home');
364
+ break;
365
+ case 'android_recent':
366
+ await controller.pressAndroidKeyRuntime('app_switch');
367
+ break;
368
+ case 'surface_refresh':
369
+ await _ensurePreview();
370
+ break;
371
+ }
372
+ }
373
+
374
+ @override
375
+ Widget build(BuildContext context) {
376
+ final controller = widget.controller;
377
+ final browserStatus = controller.browserRuntime;
378
+ final prefersExtension = controller.browserBackend == 'extension';
379
+ final extensionConnected = controller.browserExtensionConnected;
380
+ final usingExtension = prefersExtension && extensionConnected;
381
+ final browserFallbackLabel = controller.cloudBrowserBackend == 'vm'
382
+ ? 'cloud VM'
383
+ : 'local host';
384
+ final browserPageInfo = browserStatus['pageInfo'] is Map<dynamic, dynamic>
385
+ ? Map<String, dynamic>.from(browserStatus['pageInfo'] as Map)
386
+ : const <String, dynamic>{};
387
+ final selectedDesktopDevice = controller.desktopDevices
388
+ .where(
389
+ (device) => device['deviceId'] == controller.selectedDesktopDeviceId,
390
+ )
391
+ .cast<Map<String, dynamic>?>()
392
+ .firstWhere((device) => device != null, orElse: () => null);
393
+ final desktopDeviceOnline = selectedDesktopDevice?['online'] == true;
394
+ return ListView(
395
+ padding: _pagePadding(context),
396
+ children: <Widget>[
397
+ _PageTitle(
398
+ title: 'Remote Device',
399
+ subtitle:
400
+ 'Tap, swipe, and type directly on the live surface. Use the arrows below to switch between browser, phone, and desktop.',
401
+ trailing: OutlinedButton.icon(
402
+ onPressed:
403
+ controller.isRefreshingDevices ||
404
+ controller.isRunningDeviceAction
405
+ ? null
406
+ : _bootstrapSurface,
407
+ icon: Icon(Icons.sync),
408
+ label: Text('Sync Surface'),
409
+ ),
410
+ ),
411
+ if (controller.errorMessage case final message?)
412
+ Padding(
413
+ padding: const EdgeInsets.only(bottom: 16),
414
+ child: _InlineError(message: message),
415
+ ),
416
+ Center(
417
+ child: ConstrainedBox(
418
+ constraints: const BoxConstraints(maxWidth: 980),
419
+ child: Card(
420
+ color: _bgCard,
421
+ shape: RoundedRectangleBorder(
422
+ borderRadius: BorderRadius.circular(28),
423
+ side: BorderSide(color: _borderLight),
424
+ ),
425
+ child: Padding(
426
+ padding: const EdgeInsets.all(20),
427
+ child: Column(
428
+ children: <Widget>[
429
+ _DeviceSurfaceHeader(
430
+ surface: _surface,
431
+ browserStatus: browserStatus,
432
+ browserPageInfo: browserPageInfo,
433
+ androidRuntime: controller.androidRuntime,
434
+ androidOnline: _androidOnline,
435
+ desktopRuntime: controller.desktopRuntime,
436
+ desktopDevices: controller.desktopDevices,
437
+ selectedDesktopDeviceId:
438
+ controller.selectedDesktopDeviceId,
439
+ browserExtensionPreferred: prefersExtension,
440
+ browserExtensionActive: usingExtension,
441
+ browserFallbackLabel: browserFallbackLabel,
442
+ ),
443
+ if (_isDesktop) ...<Widget>[
444
+ const SizedBox(height: 14),
445
+ DropdownButtonFormField<String>(
446
+ isExpanded: true,
447
+ initialValue: selectedDesktopDevice?['deviceId']
448
+ ?.toString(),
449
+ decoration: const InputDecoration(
450
+ labelText: 'Desktop device',
451
+ prefixIcon: Icon(Icons.computer_outlined),
452
+ ),
453
+ hint: const Text('Select a companion desktop'),
454
+ items: controller.desktopDevices.map((device) {
455
+ final deviceId = device['deviceId']?.toString() ?? '';
456
+ final label =
457
+ device['label']?.toString().trim().isNotEmpty ==
458
+ true
459
+ ? device['label'].toString()
460
+ : (device['hostname']?.toString() ?? deviceId);
461
+ final os =
462
+ device['platform']?.toString() ?? 'desktop';
463
+ final state = device['online'] == true
464
+ ? (device['paused'] == true ? 'paused' : 'online')
465
+ : 'offline';
466
+ return DropdownMenuItem<String>(
467
+ value: deviceId,
468
+ child: Text(
469
+ '$label · $os · $state',
470
+ maxLines: 1,
471
+ overflow: TextOverflow.ellipsis,
472
+ softWrap: false,
473
+ ),
474
+ );
475
+ }).toList(),
476
+ onChanged: controller.isRunningDeviceAction
477
+ ? null
478
+ : (value) {
479
+ if (value == null || value.isEmpty) {
480
+ return;
481
+ }
482
+ unawaited(
483
+ controller.selectDesktopDeviceRuntime(value),
484
+ );
485
+ },
486
+ ),
487
+ const SizedBox(height: 10),
488
+ Wrap(
489
+ spacing: 10,
490
+ runSpacing: 10,
491
+ children: <Widget>[
492
+ _DotStatus(
493
+ label: desktopDeviceOnline
494
+ ? 'Companion live'
495
+ : controller.desktopCompanionConnected
496
+ ? 'Companion live'
497
+ : controller.desktopCompanionConnecting
498
+ ? 'Connecting'
499
+ : 'Companion idle',
500
+ color: desktopDeviceOnline
501
+ ? _success
502
+ : controller.desktopCompanionConnected
503
+ ? _success
504
+ : controller.desktopCompanionConnecting
505
+ ? _accent
506
+ : _warning,
507
+ ),
508
+ if (selectedDesktopDevice != null)
509
+ _DotStatus(
510
+ label: selectedDesktopDevice['paused'] == true
511
+ ? 'Paused'
512
+ : (selectedDesktopDevice['online'] == true
513
+ ? 'Ready'
514
+ : 'Offline'),
515
+ color: selectedDesktopDevice['paused'] == true
516
+ ? _warning
517
+ : (selectedDesktopDevice['online'] == true
518
+ ? _success
519
+ : _textMuted),
520
+ ),
521
+ ],
522
+ ),
523
+ ],
524
+ const SizedBox(height: 16),
525
+ _DeviceLaunchBar(
526
+ surface: _surface,
527
+ controller: _isBrowser
528
+ ? _browserUrlController
529
+ : (_isDesktop
530
+ ? _desktopLaunchController
531
+ : _androidLaunchController),
532
+ active: _isBrowser
533
+ ? browserStatus['launched'] == true
534
+ : (_isDesktop
535
+ ? _desktopOnline
536
+ : _androidOnline || _androidStarting),
537
+ starting: !_isBrowser && !_isDesktop && _androidStarting,
538
+ busy: controller.isRunningDeviceAction,
539
+ onSubmit: _openPrimary,
540
+ onSleep: _sleepPrimary,
541
+ ),
542
+ const SizedBox(height: 18),
543
+ _InteractiveSurfacePreview(
544
+ surface: _surface,
545
+ controller: controller,
546
+ screenshotPath: _activeScreenshotPath,
547
+ busy: controller.isRunningDeviceAction,
548
+ wakingUp: !_isBrowser && !_isDesktop && _androidStarting,
549
+ enabled: _isBrowser || _isDesktop || _androidOnline,
550
+ connectRequired: _desktopRequiresSelection,
551
+ onTapPoint: _handleTap,
552
+ onSwipe: _handleSwipe,
553
+ onWakeRequested: _openPrimary,
554
+ ),
555
+ if (!_isBrowser && !_isDesktop) ...<Widget>[
556
+ const SizedBox(height: 12),
557
+ _AndroidNavDock(
558
+ busy: controller.isRunningDeviceAction,
559
+ androidOnline: _androidOnline,
560
+ onAction: _runQuickAction,
561
+ ),
562
+ if (kIsWeb) ...<Widget>[
563
+ const SizedBox(height: 14),
564
+ AndroidApkDropZone(
565
+ enabled: _androidOnline,
566
+ busy: controller.isRunningDeviceAction,
567
+ onInstall: ({required filename, required bytes}) {
568
+ return controller.installAndroidApkRuntime(
569
+ filename: filename,
570
+ bytes: bytes,
571
+ );
572
+ },
573
+ ),
574
+ ],
575
+ ],
576
+ const SizedBox(height: 18),
577
+ _DeviceTypeDock(
578
+ controller: _textEntryController,
579
+ busy: controller.isRunningDeviceAction,
580
+ surface: _surface,
581
+ onSubmit: _sendText,
582
+ ),
583
+ if (_isBrowser || _isDesktop) ...<Widget>[
584
+ const SizedBox(height: 14),
585
+ _DeviceQuickActions(
586
+ surface: _surface,
587
+ androidOnline: _androidOnline,
588
+ busy: controller.isRunningDeviceAction,
589
+ onAction: _runQuickAction,
590
+ ),
591
+ ],
592
+ const SizedBox(height: 14),
593
+ _SurfaceSwitcher(
594
+ surface: _surface,
595
+ onPrevious: () => _switchSurface(-1),
596
+ onNext: () => _switchSurface(1),
597
+ ),
598
+ ],
599
+ ),
600
+ ),
601
+ ),
602
+ ),
603
+ ),
604
+ ],
605
+ );
606
+ }
607
+ }
608
+
609
+ enum _DeviceSurface { browser, android, desktop }
610
+
611
+ extension _DeviceSurfaceX on _DeviceSurface {
612
+ String get label => switch (this) {
613
+ _DeviceSurface.browser => 'Browser',
614
+ _DeviceSurface.android => 'Phone',
615
+ _DeviceSurface.desktop => 'Desktop',
616
+ };
617
+
618
+ String get helper => switch (this) {
619
+ _DeviceSurface.browser => 'Tap to click. Drag to scroll.',
620
+ _DeviceSurface.android => 'Tap to touch. Drag to swipe.',
621
+ _DeviceSurface.desktop =>
622
+ 'Tap to click. Drag to drag windows or selections.',
623
+ };
624
+
625
+ IconData get icon => switch (this) {
626
+ _DeviceSurface.browser => Icons.language_outlined,
627
+ _DeviceSurface.android => Icons.smartphone_outlined,
628
+ _DeviceSurface.desktop => Icons.computer_outlined,
629
+ };
630
+ }
631
+
632
+ class _DeviceSurfaceHeader extends StatelessWidget {
633
+ const _DeviceSurfaceHeader({
634
+ required this.surface,
635
+ required this.browserStatus,
636
+ required this.browserPageInfo,
637
+ required this.androidRuntime,
638
+ required this.androidOnline,
639
+ required this.desktopRuntime,
640
+ required this.desktopDevices,
641
+ required this.selectedDesktopDeviceId,
642
+ required this.browserExtensionPreferred,
643
+ required this.browserExtensionActive,
644
+ required this.browserFallbackLabel,
645
+ });
646
+
647
+ final _DeviceSurface surface;
648
+ final Map<String, dynamic> browserStatus;
649
+ final Map<String, dynamic> browserPageInfo;
650
+ final Map<String, dynamic> androidRuntime;
651
+ final bool androidOnline;
652
+ final Map<String, dynamic> desktopRuntime;
653
+ final List<Map<String, dynamic>> desktopDevices;
654
+ final String? selectedDesktopDeviceId;
655
+ final bool browserExtensionPreferred;
656
+ final bool browserExtensionActive;
657
+ final String browserFallbackLabel;
658
+
659
+ @override
660
+ Widget build(BuildContext context) {
661
+ return LayoutBuilder(
662
+ builder: (context, constraints) {
663
+ final compact = constraints.maxWidth < 720;
664
+ final androidStarting = androidRuntime['starting'] == true;
665
+ final androidVersion = _androidRuntimeVersionLabel(androidRuntime);
666
+ final selectedDesktop = desktopDevices
667
+ .where((device) => device['deviceId'] == selectedDesktopDeviceId)
668
+ .cast<Map<String, dynamic>?>()
669
+ .firstWhere((device) => device != null, orElse: () => null);
670
+ final desktopOnlineCount = desktopDevices
671
+ .where((device) => device['online'] == true)
672
+ .length;
673
+ final title = switch (surface) {
674
+ _DeviceSurface.browser =>
675
+ (browserPageInfo['title']?.toString().trim().isNotEmpty ?? false)
676
+ ? browserPageInfo['title'].toString()
677
+ : 'Live Browser',
678
+ _DeviceSurface.android => 'Android Phone',
679
+ _DeviceSurface.desktop =>
680
+ selectedDesktop?['label']?.toString().trim().isNotEmpty == true
681
+ ? selectedDesktop!['label'].toString()
682
+ : 'Desktop Companion',
683
+ };
684
+ final subtitle = switch (surface) {
685
+ _DeviceSurface.browser =>
686
+ browserExtensionPreferred && !browserExtensionActive
687
+ ? 'No extension device is active. Using the $browserFallbackLabel browser fallback.'
688
+ : (browserPageInfo['url']?.toString() ??
689
+ 'Ready for navigation'),
690
+ _DeviceSurface.android =>
691
+ androidOnline
692
+ ? androidVersion == null
693
+ ? 'Tap and swipe directly on the preview.'
694
+ : '$androidVersion · Tap and swipe directly on the preview.'
695
+ : androidStarting
696
+ ? (androidRuntime['startupPhase']
697
+ ?.toString()
698
+ .trim()
699
+ .isNotEmpty ??
700
+ false)
701
+ ? androidRuntime['startupPhase'].toString()
702
+ : 'Starting the phone. This can take a little while.'
703
+ : (androidRuntime['lastLogLine']
704
+ ?.toString()
705
+ .trim()
706
+ .isNotEmpty ??
707
+ false)
708
+ ? androidRuntime['lastLogLine'].toString()
709
+ : androidVersion != null
710
+ ? '$androidVersion selected. Phone is offline.'
711
+ : 'Phone is offline. Open or start it from below.',
712
+ _DeviceSurface.desktop =>
713
+ selectedDesktop == null
714
+ ? desktopOnlineCount > 1
715
+ ? 'Multiple desktop companions are online. Pick the machine you want to control.'
716
+ : desktopOnlineCount == 1
717
+ ? 'One desktop companion is online. Open the surface to fetch the latest frame.'
718
+ : 'No desktop companion is online. Enable Companion Mode on a signed-in desktop app.'
719
+ : '${selectedDesktop['platform'] ?? 'desktop'} · ${selectedDesktop['hostname'] ?? 'unknown host'}',
720
+ };
721
+ final statusLabel = surface == _DeviceSurface.browser
722
+ ? browserExtensionPreferred && !browserExtensionActive
723
+ ? 'Fallback'
724
+ : browserExtensionActive
725
+ ? 'Extension'
726
+ : (browserStatus['launched'] == true ? 'Live' : 'Sleeping')
727
+ : surface == _DeviceSurface.desktop
728
+ ? selectedDesktop == null
729
+ ? (desktopOnlineCount > 0 ? 'Select Device' : 'Offline')
730
+ : (selectedDesktop['paused'] == true
731
+ ? 'Paused'
732
+ : (selectedDesktop['online'] == true
733
+ ? 'Live'
734
+ : 'Offline'))
735
+ : (androidOnline
736
+ ? 'Live'
737
+ : androidStarting
738
+ ? 'Starting'
739
+ : 'Offline');
740
+ final statusColor = surface == _DeviceSurface.browser
741
+ ? (browserStatus['launched'] == true ? _success : _warning)
742
+ : surface == _DeviceSurface.desktop
743
+ ? (selectedDesktop?['paused'] == true
744
+ ? _warning
745
+ : (selectedDesktop?['online'] == true ? _success : _warning))
746
+ : (androidOnline
747
+ ? _success
748
+ : (androidStarting ? _accent : _warning));
749
+
750
+ final textColumn = Expanded(
751
+ child: Column(
752
+ crossAxisAlignment: CrossAxisAlignment.start,
753
+ children: <Widget>[
754
+ Text(
755
+ title,
756
+ maxLines: 1,
757
+ overflow: TextOverflow.ellipsis,
758
+ style: TextStyle(fontSize: 21, fontWeight: FontWeight.w800),
759
+ ),
760
+ const SizedBox(height: 4),
761
+ Text(
762
+ subtitle,
763
+ maxLines: 2,
764
+ overflow: TextOverflow.ellipsis,
765
+ style: TextStyle(color: _textSecondary, height: 1.4),
766
+ ),
767
+ ],
768
+ ),
769
+ );
770
+
771
+ final statusChip = Flexible(
772
+ child: Align(
773
+ alignment: compact ? Alignment.centerLeft : Alignment.centerRight,
774
+ child: _DotStatus(label: statusLabel, color: statusColor),
775
+ ),
776
+ );
777
+
778
+ if (compact) {
779
+ return Column(
780
+ crossAxisAlignment: CrossAxisAlignment.start,
781
+ children: <Widget>[
782
+ Row(
783
+ crossAxisAlignment: CrossAxisAlignment.start,
784
+ children: <Widget>[
785
+ Container(
786
+ width: 48,
787
+ height: 48,
788
+ decoration: BoxDecoration(
789
+ color: _accentMuted,
790
+ borderRadius: BorderRadius.circular(14),
791
+ ),
792
+ child: Icon(surface.icon, color: _textPrimary),
793
+ ),
794
+ const SizedBox(width: 14),
795
+ textColumn,
796
+ ],
797
+ ),
798
+ const SizedBox(height: 12),
799
+ statusChip,
800
+ ],
801
+ );
802
+ }
803
+
804
+ return Row(
805
+ children: <Widget>[
806
+ Container(
807
+ width: 48,
808
+ height: 48,
809
+ decoration: BoxDecoration(
810
+ color: _accentMuted,
811
+ borderRadius: BorderRadius.circular(14),
812
+ ),
813
+ child: Icon(surface.icon, color: _textPrimary),
814
+ ),
815
+ const SizedBox(width: 14),
816
+ textColumn,
817
+ const SizedBox(width: 12),
818
+ statusChip,
819
+ ],
820
+ );
821
+ },
822
+ );
823
+ }
824
+ }
825
+
826
+ class _DeviceLaunchBar extends StatelessWidget {
827
+ const _DeviceLaunchBar({
828
+ required this.surface,
829
+ required this.controller,
830
+ required this.active,
831
+ required this.starting,
832
+ required this.busy,
833
+ required this.onSubmit,
834
+ required this.onSleep,
835
+ });
836
+
837
+ final _DeviceSurface surface;
838
+ final TextEditingController controller;
839
+ final bool active;
840
+ final bool starting;
841
+ final bool busy;
842
+ final Future<void> Function() onSubmit;
843
+ final Future<void> Function() onSleep;
844
+
845
+ @override
846
+ Widget build(BuildContext context) {
847
+ final hint = switch (surface) {
848
+ _DeviceSurface.browser => _browserUrlPlaceholder,
849
+ _DeviceSurface.android => _packageOrUrlHint,
850
+ _DeviceSurface.desktop =>
851
+ 'Launch an app on the selected desktop (optional)',
852
+ };
853
+ final buttonLabel = switch (surface) {
854
+ _DeviceSurface.browser => 'Open',
855
+ _DeviceSurface.android => starting ? 'Starting...' : 'Launch',
856
+ _DeviceSurface.desktop => 'Refresh',
857
+ };
858
+ final sleepLabel = switch (surface) {
859
+ _DeviceSurface.browser => 'Sleep Browser',
860
+ _DeviceSurface.android => 'Sleep Phone',
861
+ _DeviceSurface.desktop => 'Pause Desktop',
862
+ };
863
+ final narrow = MediaQuery.sizeOf(context).width < 720;
864
+
865
+ final input = TextField(
866
+ controller: controller,
867
+ onSubmitted: (_) => onSubmit(),
868
+ decoration: InputDecoration(
869
+ hintText: hint,
870
+ prefixIcon: Icon(
871
+ surface == _DeviceSurface.browser
872
+ ? Icons.travel_explore
873
+ : surface == _DeviceSurface.desktop
874
+ ? Icons.apps_outlined
875
+ : Icons.open_in_new,
876
+ ),
877
+ ),
878
+ );
879
+
880
+ final button = FilledButton.icon(
881
+ onPressed: busy || starting ? null : onSubmit,
882
+ icon: Icon(
883
+ surface == _DeviceSurface.browser
884
+ ? Icons.arrow_forward
885
+ : surface == _DeviceSurface.desktop
886
+ ? Icons.desktop_windows_outlined
887
+ : Icons.play_arrow,
888
+ ),
889
+ label: Text(buttonLabel),
890
+ );
891
+ final sleepButton = OutlinedButton.icon(
892
+ onPressed: busy || !active ? null : onSleep,
893
+ icon: Icon(Icons.bedtime_outlined),
894
+ label: Text(sleepLabel),
895
+ );
896
+
897
+ if (narrow) {
898
+ return Column(
899
+ children: <Widget>[
900
+ input,
901
+ const SizedBox(height: 10),
902
+ Row(
903
+ children: <Widget>[
904
+ Expanded(child: button),
905
+ const SizedBox(width: 10),
906
+ Expanded(child: sleepButton),
907
+ ],
908
+ ),
909
+ ],
910
+ );
911
+ }
912
+
913
+ return Row(
914
+ children: <Widget>[
915
+ Expanded(child: input),
916
+ const SizedBox(width: 10),
917
+ sleepButton,
918
+ const SizedBox(width: 10),
919
+ button,
920
+ ],
921
+ );
922
+ }
923
+ }
924
+
925
+ class _DeviceTypeDock extends StatelessWidget {
926
+ const _DeviceTypeDock({
927
+ required this.controller,
928
+ required this.busy,
929
+ required this.surface,
930
+ required this.onSubmit,
931
+ });
932
+
933
+ final TextEditingController controller;
934
+ final bool busy;
935
+ final _DeviceSurface surface;
936
+ final Future<void> Function() onSubmit;
937
+
938
+ @override
939
+ Widget build(BuildContext context) {
940
+ final hint = switch (surface) {
941
+ _DeviceSurface.browser => 'Type into the currently focused field',
942
+ _DeviceSurface.android => 'Type into the current phone field',
943
+ _DeviceSurface.desktop => 'Type into the focused desktop field',
944
+ };
945
+ final narrow = MediaQuery.sizeOf(context).width < 720;
946
+
947
+ final input = TextField(
948
+ controller: controller,
949
+ onSubmitted: (_) => onSubmit(),
950
+ decoration: InputDecoration(
951
+ hintText: hint,
952
+ prefixIcon: Icon(Icons.keyboard_outlined),
953
+ ),
954
+ );
955
+
956
+ final button = FilledButton.icon(
957
+ onPressed: busy ? null : onSubmit,
958
+ icon: Icon(Icons.send_rounded),
959
+ label: Text('Send'),
960
+ );
961
+
962
+ if (narrow) {
963
+ return Column(
964
+ children: <Widget>[
965
+ input,
966
+ const SizedBox(height: 10),
967
+ SizedBox(width: double.infinity, child: button),
968
+ ],
969
+ );
970
+ }
971
+
972
+ return Row(
973
+ children: <Widget>[
974
+ Expanded(child: input),
975
+ const SizedBox(width: 10),
976
+ button,
977
+ ],
978
+ );
979
+ }
980
+ }
981
+
982
+ class _DeviceQuickActions extends StatelessWidget {
983
+ const _DeviceQuickActions({
984
+ required this.surface,
985
+ required this.androidOnline,
986
+ required this.busy,
987
+ required this.onAction,
988
+ });
989
+
990
+ final _DeviceSurface surface;
991
+ final bool androidOnline;
992
+ final bool busy;
993
+ final Future<void> Function(String action) onAction;
994
+
995
+ @override
996
+ Widget build(BuildContext context) {
997
+ final actions = switch (surface) {
998
+ _DeviceSurface.browser => const <MapEntry<String, IconData>>[
999
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1000
+ MapEntry<String, IconData>('browser_refresh', Icons.replay_rounded),
1001
+ MapEntry<String, IconData>(
1002
+ 'browser_enter',
1003
+ Icons.keyboard_return_rounded,
1004
+ ),
1005
+ ],
1006
+ _DeviceSurface.desktop => const <MapEntry<String, IconData>>[
1007
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1008
+ MapEntry<String, IconData>(
1009
+ 'desktop_enter',
1010
+ Icons.keyboard_return_rounded,
1011
+ ),
1012
+ MapEntry<String, IconData>('desktop_escape', Icons.close_fullscreen),
1013
+ ],
1014
+ _DeviceSurface.android => const <MapEntry<String, IconData>>[
1015
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1016
+ ],
1017
+ };
1018
+
1019
+ return Wrap(
1020
+ spacing: 10,
1021
+ runSpacing: 10,
1022
+ children: actions.map((entry) {
1023
+ final disabled =
1024
+ busy ||
1025
+ (surface != _DeviceSurface.browser &&
1026
+ entry.key.startsWith('browser_')) ||
1027
+ (surface != _DeviceSurface.desktop &&
1028
+ entry.key.startsWith('desktop_')) ||
1029
+ (!androidOnline && entry.key.startsWith('android_'));
1030
+ return InkWell(
1031
+ onTap: disabled ? null : () => onAction(entry.key),
1032
+ borderRadius: BorderRadius.circular(14),
1033
+ child: Ink(
1034
+ padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 10),
1035
+ decoration: BoxDecoration(
1036
+ color: disabled ? _bgSecondary : _bgTertiary,
1037
+ borderRadius: BorderRadius.circular(14),
1038
+ border: Border.all(color: _borderLight),
1039
+ ),
1040
+ child: Icon(
1041
+ entry.value,
1042
+ color: disabled ? _textMuted : _textPrimary,
1043
+ ),
1044
+ ),
1045
+ );
1046
+ }).toList(),
1047
+ );
1048
+ }
1049
+ }
1050
+
1051
+ class _AndroidNavDock extends StatelessWidget {
1052
+ const _AndroidNavDock({
1053
+ required this.busy,
1054
+ required this.androidOnline,
1055
+ required this.onAction,
1056
+ });
1057
+
1058
+ final bool busy;
1059
+ final bool androidOnline;
1060
+ final Future<void> Function(String action) onAction;
1061
+
1062
+ @override
1063
+ Widget build(BuildContext context) {
1064
+ const actions = <MapEntry<String, IconData>>[
1065
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1066
+ MapEntry<String, IconData>('android_back', Icons.arrow_back_rounded),
1067
+ MapEntry<String, IconData>('android_home', Icons.home_rounded),
1068
+ MapEntry<String, IconData>('android_recent', Icons.crop_square_rounded),
1069
+ ];
1070
+
1071
+ return Container(
1072
+ width: double.infinity,
1073
+ padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 10),
1074
+ decoration: BoxDecoration(
1075
+ color: _bgSecondary,
1076
+ borderRadius: BorderRadius.circular(18),
1077
+ border: Border.all(color: _borderLight),
1078
+ ),
1079
+ child: Row(
1080
+ mainAxisAlignment: MainAxisAlignment.spaceEvenly,
1081
+ children: actions.map((entry) {
1082
+ final disabled =
1083
+ busy || (!androidOnline && entry.key.startsWith('android_'));
1084
+ return IconButton.filledTonal(
1085
+ onPressed: disabled ? null : () => onAction(entry.key),
1086
+ icon: Icon(entry.value),
1087
+ );
1088
+ }).toList(),
1089
+ ),
1090
+ );
1091
+ }
1092
+ }
1093
+
1094
+ class _SurfaceSwitcher extends StatelessWidget {
1095
+ const _SurfaceSwitcher({
1096
+ required this.surface,
1097
+ required this.onPrevious,
1098
+ required this.onNext,
1099
+ });
1100
+
1101
+ final _DeviceSurface surface;
1102
+ final VoidCallback onPrevious;
1103
+ final VoidCallback onNext;
1104
+
1105
+ @override
1106
+ Widget build(BuildContext context) {
1107
+ return LayoutBuilder(
1108
+ builder: (context, constraints) {
1109
+ final compact = constraints.maxWidth < 520;
1110
+ final labelColumn = Column(
1111
+ mainAxisSize: MainAxisSize.min,
1112
+ children: <Widget>[
1113
+ Text(
1114
+ surface.label,
1115
+ textAlign: TextAlign.center,
1116
+ style: TextStyle(fontSize: 16, fontWeight: FontWeight.w800),
1117
+ ),
1118
+ const SizedBox(height: 4),
1119
+ Text(
1120
+ surface.helper,
1121
+ textAlign: TextAlign.center,
1122
+ maxLines: compact ? 3 : 2,
1123
+ overflow: TextOverflow.ellipsis,
1124
+ style: TextStyle(color: _textSecondary),
1125
+ ),
1126
+ ],
1127
+ );
1128
+
1129
+ if (compact) {
1130
+ return Column(
1131
+ mainAxisSize: MainAxisSize.min,
1132
+ children: <Widget>[
1133
+ Row(
1134
+ mainAxisAlignment: MainAxisAlignment.center,
1135
+ children: <Widget>[
1136
+ IconButton.filledTonal(
1137
+ onPressed: onPrevious,
1138
+ icon: Icon(Icons.arrow_back_ios_new_rounded),
1139
+ ),
1140
+ const SizedBox(width: 14),
1141
+ Flexible(child: labelColumn),
1142
+ const SizedBox(width: 14),
1143
+ IconButton.filledTonal(
1144
+ onPressed: onNext,
1145
+ icon: Icon(Icons.arrow_forward_ios_rounded),
1146
+ ),
1147
+ ],
1148
+ ),
1149
+ ],
1150
+ );
1151
+ }
1152
+
1153
+ return Row(
1154
+ mainAxisAlignment: MainAxisAlignment.center,
1155
+ children: <Widget>[
1156
+ IconButton.filledTonal(
1157
+ onPressed: onPrevious,
1158
+ icon: Icon(Icons.arrow_back_ios_new_rounded),
1159
+ ),
1160
+ const SizedBox(width: 14),
1161
+ labelColumn,
1162
+ const SizedBox(width: 14),
1163
+ IconButton.filledTonal(
1164
+ onPressed: onNext,
1165
+ icon: Icon(Icons.arrow_forward_ios_rounded),
1166
+ ),
1167
+ ],
1168
+ );
1169
+ },
1170
+ );
1171
+ }
1172
+ }
1173
+
1174
+ class _InteractiveSurfacePreview extends StatefulWidget {
1175
+ const _InteractiveSurfacePreview({
1176
+ required this.surface,
1177
+ required this.controller,
1178
+ required this.screenshotPath,
1179
+ required this.busy,
1180
+ required this.wakingUp,
1181
+ required this.enabled,
1182
+ required this.connectRequired,
1183
+ required this.onTapPoint,
1184
+ required this.onSwipe,
1185
+ required this.onWakeRequested,
1186
+ });
1187
+
1188
+ final _DeviceSurface surface;
1189
+ final NeoAgentController controller;
1190
+ final String? screenshotPath;
1191
+ final bool busy;
1192
+ final bool wakingUp;
1193
+ final bool enabled;
1194
+ final bool connectRequired;
1195
+ final Future<void> Function(Offset point) onTapPoint;
1196
+ final Future<void> Function(Offset start, Offset end) onSwipe;
1197
+ final Future<void> Function() onWakeRequested;
1198
+
1199
+ @override
1200
+ State<_InteractiveSurfacePreview> createState() =>
1201
+ _InteractiveSurfacePreviewState();
1202
+ }
1203
+
1204
+ class _InteractiveSurfacePreviewState
1205
+ extends State<_InteractiveSurfacePreview> {
1206
+ ImageStream? _imageStream;
1207
+ ImageStreamListener? _imageListener;
1208
+ Size? _pixelSize;
1209
+ Uint8List? _imageBytes;
1210
+ Object? _imageError;
1211
+ Offset? _dragStart;
1212
+ Offset? _dragEnd;
1213
+
1214
+ @override
1215
+ void initState() {
1216
+ super.initState();
1217
+ unawaited(_loadImage());
1218
+ }
1219
+
1220
+ @override
1221
+ void didUpdateWidget(covariant _InteractiveSurfacePreview oldWidget) {
1222
+ super.didUpdateWidget(oldWidget);
1223
+ if (oldWidget.screenshotPath != widget.screenshotPath) {
1224
+ unawaited(_loadImage());
1225
+ }
1226
+ }
1227
+
1228
+ @override
1229
+ void dispose() {
1230
+ _detachImageListener();
1231
+ super.dispose();
1232
+ }
1233
+
1234
+ void _detachImageListener() {
1235
+ if (_imageStream != null && _imageListener != null) {
1236
+ _imageStream!.removeListener(_imageListener!);
1237
+ }
1238
+ _imageStream = null;
1239
+ _imageListener = null;
1240
+ }
1241
+
1242
+ Future<void> _loadImage() async {
1243
+ _detachImageListener();
1244
+ final path = widget.screenshotPath;
1245
+ if (path == null || path.isEmpty) {
1246
+ if (!mounted) {
1247
+ return;
1248
+ }
1249
+ setState(() {
1250
+ _pixelSize = null;
1251
+ _imageBytes = null;
1252
+ _imageError = null;
1253
+ });
1254
+ return;
1255
+ }
1256
+ try {
1257
+ final bytes = await widget.controller.fetchRuntimeAssetBytes(path);
1258
+ if (!mounted || widget.screenshotPath != path) {
1259
+ return;
1260
+ }
1261
+ setState(() {
1262
+ _imageBytes = bytes;
1263
+ _imageError = null;
1264
+ _pixelSize = null;
1265
+ });
1266
+ final provider = MemoryImage(bytes);
1267
+ final stream = provider.resolve(const ImageConfiguration());
1268
+ final listener = ImageStreamListener((image, _) {
1269
+ if (!mounted || widget.screenshotPath != path) {
1270
+ return;
1271
+ }
1272
+ setState(() {
1273
+ _pixelSize = Size(
1274
+ image.image.width.toDouble(),
1275
+ image.image.height.toDouble(),
1276
+ );
1277
+ });
1278
+ });
1279
+ _imageStream = stream;
1280
+ _imageListener = listener;
1281
+ stream.addListener(listener);
1282
+ } catch (error) {
1283
+ if (!mounted || widget.screenshotPath != path) {
1284
+ return;
1285
+ }
1286
+ setState(() {
1287
+ _imageBytes = null;
1288
+ _pixelSize = null;
1289
+ _imageError = error;
1290
+ });
1291
+ }
1292
+ }
1293
+
1294
+ Offset? _mapToPixels(Offset localPosition, Size boxSize) {
1295
+ final pixelSize = _pixelSize;
1296
+ if (pixelSize == null) {
1297
+ return null;
1298
+ }
1299
+ final boxAspect = boxSize.width / boxSize.height;
1300
+ final imageAspect = pixelSize.width / pixelSize.height;
1301
+ late final double renderWidth;
1302
+ late final double renderHeight;
1303
+ late final double offsetX;
1304
+ late final double offsetY;
1305
+
1306
+ if (boxAspect > imageAspect) {
1307
+ renderHeight = boxSize.height;
1308
+ renderWidth = renderHeight * imageAspect;
1309
+ offsetX = (boxSize.width - renderWidth) / 2;
1310
+ offsetY = 0;
1311
+ } else {
1312
+ renderWidth = boxSize.width;
1313
+ renderHeight = renderWidth / imageAspect;
1314
+ offsetX = 0;
1315
+ offsetY = (boxSize.height - renderHeight) / 2;
1316
+ }
1317
+
1318
+ if (localPosition.dx < offsetX ||
1319
+ localPosition.dx > offsetX + renderWidth ||
1320
+ localPosition.dy < offsetY ||
1321
+ localPosition.dy > offsetY + renderHeight) {
1322
+ return null;
1323
+ }
1324
+
1325
+ return Offset(
1326
+ ((localPosition.dx - offsetX) / renderWidth) * pixelSize.width,
1327
+ ((localPosition.dy - offsetY) / renderHeight) * pixelSize.height,
1328
+ );
1329
+ }
1330
+
1331
+ @override
1332
+ Widget build(BuildContext context) {
1333
+ final path = widget.screenshotPath;
1334
+ final hasImage = path != null && path.isNotEmpty;
1335
+ final aspectRatio = switch (widget.surface) {
1336
+ _DeviceSurface.browser => 16 / 10,
1337
+ _DeviceSurface.android => 10 / 16,
1338
+ _DeviceSurface.desktop => 16 / 10,
1339
+ };
1340
+
1341
+ return Container(
1342
+ decoration: BoxDecoration(
1343
+ gradient: LinearGradient(
1344
+ begin: Alignment.topLeft,
1345
+ end: Alignment.bottomRight,
1346
+ colors: <Color>[_bgTertiary, _bgSecondary],
1347
+ ),
1348
+ borderRadius: BorderRadius.circular(24),
1349
+ border: Border.all(color: _borderLight),
1350
+ ),
1351
+ padding: const EdgeInsets.all(14),
1352
+ child: Column(
1353
+ children: <Widget>[
1354
+ ConstrainedBox(
1355
+ constraints: BoxConstraints(
1356
+ maxHeight: widget.surface == _DeviceSurface.android ? 640 : 560,
1357
+ ),
1358
+ child: AspectRatio(
1359
+ aspectRatio: aspectRatio,
1360
+ child: ClipRRect(
1361
+ borderRadius: BorderRadius.circular(20),
1362
+ child: LayoutBuilder(
1363
+ builder: (context, constraints) {
1364
+ final boxSize = Size(
1365
+ constraints.maxWidth,
1366
+ constraints.maxHeight,
1367
+ );
1368
+ if (!hasImage) {
1369
+ return _EmptySurfaceState(
1370
+ surface: widget.surface,
1371
+ enabled: widget.enabled,
1372
+ busy: widget.busy,
1373
+ isLoadingPreview: widget.wakingUp,
1374
+ connectRequired: widget.connectRequired,
1375
+ errorMessage: _imageError?.toString(),
1376
+ onPressed: widget.onWakeRequested,
1377
+ );
1378
+ }
1379
+ final imageBytes = _imageBytes;
1380
+ if (imageBytes == null) {
1381
+ return _EmptySurfaceState(
1382
+ surface: widget.surface,
1383
+ enabled: widget.enabled,
1384
+ busy: widget.busy,
1385
+ isLoadingPreview: _imageError == null,
1386
+ connectRequired: widget.connectRequired,
1387
+ errorMessage: _imageError?.toString(),
1388
+ onPressed: widget.onWakeRequested,
1389
+ );
1390
+ }
1391
+ return GestureDetector(
1392
+ onTapUp: widget.busy
1393
+ ? null
1394
+ : (details) async {
1395
+ final point = _mapToPixels(
1396
+ details.localPosition,
1397
+ boxSize,
1398
+ );
1399
+ if (point != null) {
1400
+ await widget.onTapPoint(point);
1401
+ }
1402
+ },
1403
+ onPanStart: widget.busy
1404
+ ? null
1405
+ : (details) {
1406
+ _dragStart = details.localPosition;
1407
+ _dragEnd = details.localPosition;
1408
+ },
1409
+ onPanUpdate: widget.busy
1410
+ ? null
1411
+ : (details) {
1412
+ _dragEnd = details.localPosition;
1413
+ },
1414
+ onPanEnd: widget.busy
1415
+ ? null
1416
+ : (_) async {
1417
+ final start = _dragStart;
1418
+ final end = _dragEnd;
1419
+ _dragStart = null;
1420
+ _dragEnd = null;
1421
+ if (start == null || end == null) {
1422
+ return;
1423
+ }
1424
+ if ((start - end).distance < 12) {
1425
+ return;
1426
+ }
1427
+ final mappedStart = _mapToPixels(start, boxSize);
1428
+ final mappedEnd = _mapToPixels(end, boxSize);
1429
+ if (mappedStart != null && mappedEnd != null) {
1430
+ await widget.onSwipe(mappedStart, mappedEnd);
1431
+ }
1432
+ },
1433
+ child: Stack(
1434
+ fit: StackFit.expand,
1435
+ children: <Widget>[
1436
+ Container(color: _bgSecondary),
1437
+ Image.memory(
1438
+ imageBytes,
1439
+ fit: BoxFit.contain,
1440
+ gaplessPlayback: true,
1441
+ ),
1442
+ Positioned(
1443
+ left: 12,
1444
+ right: 12,
1445
+ bottom: 12,
1446
+ child: Container(
1447
+ padding: const EdgeInsets.symmetric(
1448
+ horizontal: 12,
1449
+ vertical: 10,
1450
+ ),
1451
+ decoration: BoxDecoration(
1452
+ color: const Color(0xB205080D),
1453
+ borderRadius: BorderRadius.circular(14),
1454
+ border: Border.all(color: _borderLight),
1455
+ ),
1456
+ child: Text(
1457
+ widget.surface.helper,
1458
+ textAlign: TextAlign.center,
1459
+ style: TextStyle(color: _textPrimary),
1460
+ ),
1461
+ ),
1462
+ ),
1463
+ if (widget.busy)
1464
+ const Center(child: CircularProgressIndicator()),
1465
+ ],
1466
+ ),
1467
+ );
1468
+ },
1469
+ ),
1470
+ ),
1471
+ ),
1472
+ ),
1473
+ ],
1474
+ ),
1475
+ );
1476
+ }
1477
+ }
1478
+
1479
+ class _EmptySurfaceState extends StatelessWidget {
1480
+ const _EmptySurfaceState({
1481
+ required this.surface,
1482
+ required this.enabled,
1483
+ required this.busy,
1484
+ required this.isLoadingPreview,
1485
+ required this.connectRequired,
1486
+ this.errorMessage,
1487
+ required this.onPressed,
1488
+ });
1489
+
1490
+ final _DeviceSurface surface;
1491
+ final bool enabled;
1492
+ final bool busy;
1493
+ final bool isLoadingPreview;
1494
+ final bool connectRequired;
1495
+ final String? errorMessage;
1496
+ final Future<void> Function() onPressed;
1497
+
1498
+ @override
1499
+ Widget build(BuildContext context) {
1500
+ final label = switch (surface) {
1501
+ _DeviceSurface.browser =>
1502
+ connectRequired
1503
+ ? 'Open Settings'
1504
+ : (busy ? 'Opening Browser...' : 'Wake Browser'),
1505
+ _DeviceSurface.android =>
1506
+ busy
1507
+ ? 'Starting Phone...'
1508
+ : (enabled ? 'Refresh Phone' : 'Start Phone'),
1509
+ _DeviceSurface.desktop =>
1510
+ connectRequired
1511
+ ? 'Select Desktop'
1512
+ : (busy ? 'Refreshing Desktop...' : 'Refresh Desktop'),
1513
+ };
1514
+ final message = switch ((surface, busy, isLoadingPreview)) {
1515
+ (_DeviceSurface.browser, true, _) =>
1516
+ 'Opening the browser and downloading the first preview...',
1517
+ (_DeviceSurface.browser, false, true) =>
1518
+ 'Downloading the latest browser preview...',
1519
+ (_DeviceSurface.desktop, true, _) =>
1520
+ 'Refreshing the selected desktop companion...',
1521
+ (_DeviceSurface.desktop, false, true) =>
1522
+ 'Downloading the latest desktop preview...',
1523
+ (_DeviceSurface.android, true, _) =>
1524
+ 'Waking the phone and downloading the first preview. This can take a little while.',
1525
+ (_DeviceSurface.android, false, true) =>
1526
+ 'Downloading the latest phone preview...',
1527
+ _ =>
1528
+ surface == _DeviceSurface.browser
1529
+ ? connectRequired
1530
+ ? 'Chrome extension is not connected. Use Settings to download, load, and pair the extension on the remote machine.'
1531
+ : 'Browser is sleeping. Press Open to start it.'
1532
+ : surface == _DeviceSurface.desktop
1533
+ ? connectRequired
1534
+ ? 'Multiple desktop companions are online. Select a machine before sending clicks or keystrokes.'
1535
+ : (errorMessage != null && errorMessage!.trim().isNotEmpty)
1536
+ ? errorMessage!
1537
+ : 'No desktop frame is loaded yet. Refresh the selected machine to capture a frame.'
1538
+ : (errorMessage != null && errorMessage!.trim().isNotEmpty)
1539
+ ? errorMessage!
1540
+ : 'Phone is offline. Press Start Phone to boot it.',
1541
+ };
1542
+ return Container(
1543
+ color: _bgSecondary,
1544
+ alignment: Alignment.center,
1545
+ child: Column(
1546
+ mainAxisSize: MainAxisSize.min,
1547
+ children: <Widget>[
1548
+ if (busy || isLoadingPreview) ...<Widget>[
1549
+ const SizedBox(
1550
+ width: 38,
1551
+ height: 38,
1552
+ child: CircularProgressIndicator(strokeWidth: 3),
1553
+ ),
1554
+ const SizedBox(height: 14),
1555
+ ] else
1556
+ Icon(surface.icon, size: 46, color: _textSecondary),
1557
+ if (!(busy || isLoadingPreview)) const SizedBox(height: 12),
1558
+ Text(
1559
+ message,
1560
+ style: TextStyle(color: _textSecondary),
1561
+ textAlign: TextAlign.center,
1562
+ ),
1563
+ if (errorMessage != null &&
1564
+ surface == _DeviceSurface.browser &&
1565
+ errorMessage!.trim().isNotEmpty) ...<Widget>[
1566
+ const SizedBox(height: 10),
1567
+ ConstrainedBox(
1568
+ constraints: const BoxConstraints(maxWidth: 320),
1569
+ child: Text(
1570
+ errorMessage!,
1571
+ textAlign: TextAlign.center,
1572
+ style: TextStyle(color: _textMuted, fontSize: 12),
1573
+ ),
1574
+ ),
1575
+ ],
1576
+ const SizedBox(height: 16),
1577
+ FilledButton.icon(
1578
+ onPressed: busy ? null : onPressed,
1579
+ icon: Icon(Icons.play_arrow_rounded),
1580
+ label: Text(label),
1581
+ ),
1582
+ ],
1583
+ ),
1584
+ );
1585
+ }
1586
+ }
1587
+
1588
+ // ignore: unused_element
1589
+ class _RuntimeControlCard extends StatelessWidget {
1590
+ const _RuntimeControlCard({
1591
+ required this.title,
1592
+ required this.subtitle,
1593
+ required this.status,
1594
+ required this.child,
1595
+ });
1596
+
1597
+ final String title;
1598
+ final String subtitle;
1599
+ final Widget status;
1600
+ final Widget child;
1601
+
1602
+ @override
1603
+ Widget build(BuildContext context) {
1604
+ return Card(
1605
+ child: Padding(
1606
+ padding: const EdgeInsets.all(20),
1607
+ child: Column(
1608
+ crossAxisAlignment: CrossAxisAlignment.start,
1609
+ children: <Widget>[
1610
+ Row(
1611
+ crossAxisAlignment: CrossAxisAlignment.start,
1612
+ children: <Widget>[
1613
+ Expanded(
1614
+ child: Column(
1615
+ crossAxisAlignment: CrossAxisAlignment.start,
1616
+ children: <Widget>[
1617
+ Text(
1618
+ title,
1619
+ style: TextStyle(
1620
+ fontSize: 20,
1621
+ fontWeight: FontWeight.w800,
1622
+ ),
1623
+ ),
1624
+ const SizedBox(height: 6),
1625
+ Text(
1626
+ subtitle,
1627
+ style: TextStyle(color: _textSecondary, height: 1.5),
1628
+ ),
1629
+ ],
1630
+ ),
1631
+ ),
1632
+ const SizedBox(width: 12),
1633
+ status,
1634
+ ],
1635
+ ),
1636
+ const SizedBox(height: 18),
1637
+ child,
1638
+ ],
1639
+ ),
1640
+ ),
1641
+ );
1642
+ }
1643
+ }
1644
+
1645
+ // ignore: unused_element
1646
+ class _BrowserControls extends StatelessWidget {
1647
+ const _BrowserControls({
1648
+ required this.controller,
1649
+ required this.browserStatus,
1650
+ required this.browserPageInfo,
1651
+ required this.urlController,
1652
+ required this.waitForController,
1653
+ required this.clickSelectorController,
1654
+ required this.clickTextController,
1655
+ required this.fillSelectorController,
1656
+ required this.fillValueController,
1657
+ });
1658
+
1659
+ final NeoAgentController controller;
1660
+ final Map<String, dynamic> browserStatus;
1661
+ final Map<String, dynamic> browserPageInfo;
1662
+ final TextEditingController urlController;
1663
+ final TextEditingController waitForController;
1664
+ final TextEditingController clickSelectorController;
1665
+ final TextEditingController clickTextController;
1666
+ final TextEditingController fillSelectorController;
1667
+ final TextEditingController fillValueController;
1668
+
1669
+ @override
1670
+ Widget build(BuildContext context) {
1671
+ final launched = browserStatus['launched'] == true;
1672
+ return Column(
1673
+ crossAxisAlignment: CrossAxisAlignment.start,
1674
+ children: <Widget>[
1675
+ Wrap(
1676
+ spacing: 10,
1677
+ runSpacing: 10,
1678
+ children: <Widget>[
1679
+ _MetaPill(
1680
+ label: launched ? 'Launched' : 'Idle',
1681
+ icon: Icons.language_outlined,
1682
+ ),
1683
+ _MetaPill(
1684
+ label: 'Pages ${browserStatus['pages'] ?? 0}',
1685
+ icon: Icons.filter_none_outlined,
1686
+ ),
1687
+ _MetaPill(
1688
+ label: browserStatus['headless'] == false
1689
+ ? 'Visible window'
1690
+ : 'Headless',
1691
+ icon: Icons.visibility_outlined,
1692
+ ),
1693
+ ],
1694
+ ),
1695
+ if ((browserPageInfo['url']?.toString().isNotEmpty ?? false) ||
1696
+ (browserPageInfo['title']?.toString().isNotEmpty ??
1697
+ false)) ...<Widget>[
1698
+ const SizedBox(height: 14),
1699
+ SelectableText(
1700
+ '${browserPageInfo['title'] ?? 'Untitled'}\n${browserPageInfo['url'] ?? ''}',
1701
+ style: TextStyle(color: _textSecondary, height: 1.5),
1702
+ ),
1703
+ ],
1704
+ const SizedBox(height: 18),
1705
+ _DeviceFieldRow(
1706
+ children: <Widget>[
1707
+ _DeviceField(
1708
+ label: 'URL',
1709
+ child: TextField(controller: urlController),
1710
+ ),
1711
+ _DeviceField(
1712
+ label: 'Wait For Selector',
1713
+ child: TextField(controller: waitForController),
1714
+ ),
1715
+ ],
1716
+ ),
1717
+ const SizedBox(height: 10),
1718
+ Wrap(
1719
+ spacing: 10,
1720
+ runSpacing: 10,
1721
+ children: <Widget>[
1722
+ FilledButton.icon(
1723
+ onPressed: controller.isRunningDeviceAction
1724
+ ? null
1725
+ : controller.launchBrowserRuntime,
1726
+ icon: Icon(Icons.rocket_launch_outlined),
1727
+ label: Text('Launch'),
1728
+ ),
1729
+ FilledButton.icon(
1730
+ onPressed: controller.isRunningDeviceAction
1731
+ ? null
1732
+ : () => controller.navigateBrowserRuntime(
1733
+ url: urlController.text.trim(),
1734
+ waitFor: waitForController.text.trim(),
1735
+ ),
1736
+ icon: Icon(Icons.open_in_browser_outlined),
1737
+ label: Text('Navigate'),
1738
+ ),
1739
+ OutlinedButton.icon(
1740
+ onPressed: controller.isRunningDeviceAction
1741
+ ? null
1742
+ : controller.screenshotBrowserRuntime,
1743
+ icon: Icon(Icons.photo_camera_back_outlined),
1744
+ label: Text('Screenshot'),
1745
+ ),
1746
+ OutlinedButton.icon(
1747
+ onPressed: controller.isRunningDeviceAction
1748
+ ? null
1749
+ : controller.closeBrowserRuntime,
1750
+ icon: Icon(Icons.close),
1751
+ label: Text('Close'),
1752
+ ),
1753
+ ],
1754
+ ),
1755
+ const SizedBox(height: 18),
1756
+ _DeviceFieldRow(
1757
+ children: <Widget>[
1758
+ _DeviceField(
1759
+ label: 'Click Selector',
1760
+ child: TextField(controller: clickSelectorController),
1761
+ ),
1762
+ _DeviceField(
1763
+ label: 'Click Text',
1764
+ child: TextField(controller: clickTextController),
1765
+ ),
1766
+ ],
1767
+ ),
1768
+ const SizedBox(height: 10),
1769
+ Wrap(
1770
+ spacing: 10,
1771
+ runSpacing: 10,
1772
+ children: <Widget>[
1773
+ OutlinedButton(
1774
+ onPressed: controller.isRunningDeviceAction
1775
+ ? null
1776
+ : () => controller.clickBrowserRuntime(
1777
+ selector: clickSelectorController.text.trim(),
1778
+ ),
1779
+ child: Text('Click Selector'),
1780
+ ),
1781
+ OutlinedButton(
1782
+ onPressed: controller.isRunningDeviceAction
1783
+ ? null
1784
+ : () => controller.clickBrowserRuntime(
1785
+ text: clickTextController.text.trim(),
1786
+ ),
1787
+ child: Text('Click Text'),
1788
+ ),
1789
+ ],
1790
+ ),
1791
+ const SizedBox(height: 18),
1792
+ _DeviceFieldRow(
1793
+ children: <Widget>[
1794
+ _DeviceField(
1795
+ label: 'Type Selector',
1796
+ child: TextField(controller: fillSelectorController),
1797
+ ),
1798
+ _DeviceField(
1799
+ label: 'Value',
1800
+ child: TextField(controller: fillValueController),
1801
+ ),
1802
+ ],
1803
+ ),
1804
+ const SizedBox(height: 10),
1805
+ OutlinedButton.icon(
1806
+ onPressed: controller.isRunningDeviceAction
1807
+ ? null
1808
+ : () => controller.fillBrowserRuntime(
1809
+ selector: fillSelectorController.text.trim(),
1810
+ value: fillValueController.text,
1811
+ ),
1812
+ icon: Icon(Icons.keyboard_outlined),
1813
+ label: Text('Type Into Field'),
1814
+ ),
1815
+ const SizedBox(height: 18),
1816
+ _RuntimePreview(
1817
+ title: 'Latest Browser Screenshot',
1818
+ screenshotPath: controller.browserScreenshotPath,
1819
+ controller: controller,
1820
+ ),
1821
+ if (controller.browserLastResult?.trim().isNotEmpty ??
1822
+ false) ...<Widget>[
1823
+ const SizedBox(height: 14),
1824
+ _ResultBlock(
1825
+ label: 'Last browser result',
1826
+ value: controller.browserLastResult!,
1827
+ ),
1828
+ ],
1829
+ ],
1830
+ );
1831
+ }
1832
+ }
1833
+
1834
+ // ignore: unused_element
1835
+ class _AndroidControls extends StatelessWidget {
1836
+ const _AndroidControls({
1837
+ required this.controller,
1838
+ required this.androidStatus,
1839
+ required this.androidDevices,
1840
+ required this.packageController,
1841
+ required this.activityController,
1842
+ required this.intentActionController,
1843
+ required this.intentDataController,
1844
+ required this.tapTextController,
1845
+ required this.tapDescriptionController,
1846
+ required this.tapResourceIdController,
1847
+ required this.tapXController,
1848
+ required this.tapYController,
1849
+ required this.typeTextController,
1850
+ required this.typeFieldTextController,
1851
+ required this.typeFieldDescriptionController,
1852
+ required this.waitTextController,
1853
+ required this.keyController,
1854
+ required this.swipeX1Controller,
1855
+ required this.swipeY1Controller,
1856
+ required this.swipeX2Controller,
1857
+ required this.swipeY2Controller,
1858
+ required this.toInt,
1859
+ });
1860
+
1861
+ final NeoAgentController controller;
1862
+ final Map<String, dynamic> androidStatus;
1863
+ final List<Map<String, dynamic>> androidDevices;
1864
+ final TextEditingController packageController;
1865
+ final TextEditingController activityController;
1866
+ final TextEditingController intentActionController;
1867
+ final TextEditingController intentDataController;
1868
+ final TextEditingController tapTextController;
1869
+ final TextEditingController tapDescriptionController;
1870
+ final TextEditingController tapResourceIdController;
1871
+ final TextEditingController tapXController;
1872
+ final TextEditingController tapYController;
1873
+ final TextEditingController typeTextController;
1874
+ final TextEditingController typeFieldTextController;
1875
+ final TextEditingController typeFieldDescriptionController;
1876
+ final TextEditingController waitTextController;
1877
+ final TextEditingController keyController;
1878
+ final TextEditingController swipeX1Controller;
1879
+ final TextEditingController swipeY1Controller;
1880
+ final TextEditingController swipeX2Controller;
1881
+ final TextEditingController swipeY2Controller;
1882
+ final int? Function(String text) toInt;
1883
+
1884
+ @override
1885
+ Widget build(BuildContext context) {
1886
+ return Column(
1887
+ crossAxisAlignment: CrossAxisAlignment.start,
1888
+ children: <Widget>[
1889
+ Wrap(
1890
+ spacing: 10,
1891
+ runSpacing: 10,
1892
+ children: <Widget>[
1893
+ _MetaPill(
1894
+ label: androidStatus['bootstrapped'] == true
1895
+ ? 'SDK Ready'
1896
+ : 'Bootstrap Needed',
1897
+ icon: Icons.adb_outlined,
1898
+ ),
1899
+ _MetaPill(
1900
+ label: androidStatus['serial']?.toString().isNotEmpty == true
1901
+ ? androidStatus['serial'].toString()
1902
+ : 'No active serial',
1903
+ icon: Icons.phone_android_outlined,
1904
+ ),
1905
+ _MetaPill(
1906
+ label: '${androidDevices.length} device(s)',
1907
+ icon: Icons.devices_other_outlined,
1908
+ ),
1909
+ ],
1910
+ ),
1911
+ const SizedBox(height: 18),
1912
+ Wrap(
1913
+ spacing: 10,
1914
+ runSpacing: 10,
1915
+ children: <Widget>[
1916
+ FilledButton.icon(
1917
+ onPressed: controller.isRunningDeviceAction
1918
+ ? null
1919
+ : controller.startAndroidRuntime,
1920
+ icon: Icon(Icons.play_arrow_outlined),
1921
+ label: Text('Start Emulator'),
1922
+ ),
1923
+ OutlinedButton.icon(
1924
+ onPressed: controller.isRunningDeviceAction
1925
+ ? null
1926
+ : controller.stopAndroidRuntime,
1927
+ icon: Icon(Icons.stop_circle_outlined),
1928
+ label: Text('Stop'),
1929
+ ),
1930
+ OutlinedButton.icon(
1931
+ onPressed: controller.isRunningDeviceAction
1932
+ ? null
1933
+ : controller.screenshotAndroidRuntime,
1934
+ icon: Icon(Icons.photo_camera_outlined),
1935
+ label: Text('Screenshot'),
1936
+ ),
1937
+ OutlinedButton.icon(
1938
+ onPressed: controller.isRunningDeviceAction
1939
+ ? null
1940
+ : controller.dumpAndroidUiRuntime,
1941
+ icon: Icon(Icons.data_object_outlined),
1942
+ label: Text('Dump UI'),
1943
+ ),
1944
+ OutlinedButton.icon(
1945
+ onPressed: controller.isRunningDeviceAction
1946
+ ? null
1947
+ : controller.refreshAndroidApps,
1948
+ icon: Icon(Icons.apps_outlined),
1949
+ label: Text('Load Apps'),
1950
+ ),
1951
+ ],
1952
+ ),
1953
+ const SizedBox(height: 18),
1954
+ _DeviceFieldRow(
1955
+ children: <Widget>[
1956
+ _DeviceField(
1957
+ label: 'Package',
1958
+ child: TextField(controller: packageController),
1959
+ ),
1960
+ _DeviceField(
1961
+ label: 'Activity',
1962
+ child: TextField(controller: activityController),
1963
+ ),
1964
+ ],
1965
+ ),
1966
+ const SizedBox(height: 10),
1967
+ Wrap(
1968
+ spacing: 10,
1969
+ runSpacing: 10,
1970
+ children: <Widget>[
1971
+ OutlinedButton.icon(
1972
+ onPressed: controller.isRunningDeviceAction
1973
+ ? null
1974
+ : () => controller.openAndroidAppRuntime(
1975
+ packageName: packageController.text.trim(),
1976
+ activity: activityController.text.trim(),
1977
+ ),
1978
+ icon: Icon(Icons.apps),
1979
+ label: Text('Open App'),
1980
+ ),
1981
+ if (controller.androidInstalledApps.isNotEmpty)
1982
+ SizedBox(width: 1, height: 1, child: Container()),
1983
+ ],
1984
+ ),
1985
+ if (controller.androidInstalledApps.isNotEmpty) ...<Widget>[
1986
+ const SizedBox(height: 8),
1987
+ Wrap(
1988
+ spacing: 8,
1989
+ runSpacing: 8,
1990
+ children: controller.androidInstalledApps.take(10).map((appId) {
1991
+ return ActionChip(
1992
+ label: Text(appId),
1993
+ onPressed: () => packageController.text = appId,
1994
+ );
1995
+ }).toList(),
1996
+ ),
1997
+ ],
1998
+ const SizedBox(height: 18),
1999
+ _DeviceFieldRow(
2000
+ children: <Widget>[
2001
+ _DeviceField(
2002
+ label: 'Intent Action',
2003
+ child: TextField(controller: intentActionController),
2004
+ ),
2005
+ _DeviceField(
2006
+ label: 'Intent Data',
2007
+ child: TextField(controller: intentDataController),
2008
+ ),
2009
+ ],
2010
+ ),
2011
+ const SizedBox(height: 10),
2012
+ OutlinedButton.icon(
2013
+ onPressed: controller.isRunningDeviceAction
2014
+ ? null
2015
+ : () => controller.openAndroidIntentRuntime(
2016
+ action: intentActionController.text.trim(),
2017
+ dataUri: intentDataController.text.trim(),
2018
+ packageName: packageController.text.trim(),
2019
+ ),
2020
+ icon: Icon(Icons.route_outlined),
2021
+ label: Text('Open Intent'),
2022
+ ),
2023
+ const SizedBox(height: 18),
2024
+ _DeviceFieldRow(
2025
+ children: <Widget>[
2026
+ _DeviceField(
2027
+ label: 'Wait For Text',
2028
+ child: TextField(controller: waitTextController),
2029
+ ),
2030
+ _DeviceField(
2031
+ label: 'Key',
2032
+ child: TextField(controller: keyController),
2033
+ ),
2034
+ ],
2035
+ ),
2036
+ const SizedBox(height: 10),
2037
+ Wrap(
2038
+ spacing: 10,
2039
+ runSpacing: 10,
2040
+ children: <Widget>[
2041
+ OutlinedButton(
2042
+ onPressed: controller.isRunningDeviceAction
2043
+ ? null
2044
+ : () => controller.waitForAndroidRuntime(<String, dynamic>{
2045
+ 'text': waitTextController.text.trim(),
2046
+ 'timeoutMs': 20000,
2047
+ 'intervalMs': 1200,
2048
+ }),
2049
+ child: Text('Wait For UI'),
2050
+ ),
2051
+ OutlinedButton(
2052
+ onPressed: controller.isRunningDeviceAction
2053
+ ? null
2054
+ : () => controller.pressAndroidKeyRuntime(
2055
+ keyController.text.trim(),
2056
+ ),
2057
+ child: Text('Press Key'),
2058
+ ),
2059
+ ],
2060
+ ),
2061
+ const SizedBox(height: 18),
2062
+ _DeviceFieldRow(
2063
+ children: <Widget>[
2064
+ _DeviceField(
2065
+ label: 'Tap Text',
2066
+ child: TextField(controller: tapTextController),
2067
+ ),
2068
+ _DeviceField(
2069
+ label: 'Tap Description',
2070
+ child: TextField(controller: tapDescriptionController),
2071
+ ),
2072
+ ],
2073
+ ),
2074
+ const SizedBox(height: 10),
2075
+ _DeviceFieldRow(
2076
+ children: <Widget>[
2077
+ _DeviceField(
2078
+ label: 'Tap Resource Id',
2079
+ child: TextField(controller: tapResourceIdController),
2080
+ ),
2081
+ _DeviceField(
2082
+ label: 'Tap X / Y',
2083
+ child: Row(
2084
+ children: <Widget>[
2085
+ Expanded(child: TextField(controller: tapXController)),
2086
+ const SizedBox(width: 8),
2087
+ Expanded(child: TextField(controller: tapYController)),
2088
+ ],
2089
+ ),
2090
+ ),
2091
+ ],
2092
+ ),
2093
+ const SizedBox(height: 10),
2094
+ OutlinedButton.icon(
2095
+ onPressed: controller.isRunningDeviceAction
2096
+ ? null
2097
+ : () => controller.tapAndroidRuntime(<String, dynamic>{
2098
+ if (tapTextController.text.trim().isNotEmpty)
2099
+ 'text': tapTextController.text.trim(),
2100
+ if (tapDescriptionController.text.trim().isNotEmpty)
2101
+ 'description': tapDescriptionController.text.trim(),
2102
+ if (tapResourceIdController.text.trim().isNotEmpty)
2103
+ 'resourceId': tapResourceIdController.text.trim(),
2104
+ if (toInt(tapXController.text) != null)
2105
+ 'x': toInt(tapXController.text),
2106
+ if (toInt(tapYController.text) != null)
2107
+ 'y': toInt(tapYController.text),
2108
+ }),
2109
+ icon: Icon(Icons.touch_app_outlined),
2110
+ label: Text('Tap'),
2111
+ ),
2112
+ const SizedBox(height: 18),
2113
+ _DeviceFieldRow(
2114
+ children: <Widget>[
2115
+ _DeviceField(
2116
+ label: 'Type Text',
2117
+ child: TextField(controller: typeTextController),
2118
+ ),
2119
+ _DeviceField(
2120
+ label: 'Focus Field Text / Description',
2121
+ child: Row(
2122
+ children: <Widget>[
2123
+ Expanded(
2124
+ child: TextField(controller: typeFieldTextController),
2125
+ ),
2126
+ const SizedBox(width: 8),
2127
+ Expanded(
2128
+ child: TextField(
2129
+ controller: typeFieldDescriptionController,
2130
+ ),
2131
+ ),
2132
+ ],
2133
+ ),
2134
+ ),
2135
+ ],
2136
+ ),
2137
+ const SizedBox(height: 10),
2138
+ OutlinedButton.icon(
2139
+ onPressed: controller.isRunningDeviceAction
2140
+ ? null
2141
+ : () => controller.typeAndroidRuntime(<String, dynamic>{
2142
+ 'text': typeTextController.text,
2143
+ if (typeFieldTextController.text.trim().isNotEmpty)
2144
+ 'textSelector': typeFieldTextController.text.trim(),
2145
+ if (typeFieldDescriptionController.text.trim().isNotEmpty)
2146
+ 'description': typeFieldDescriptionController.text.trim(),
2147
+ 'pressEnter': true,
2148
+ }),
2149
+ icon: Icon(Icons.keyboard_outlined),
2150
+ label: Text('Type'),
2151
+ ),
2152
+ const SizedBox(height: 18),
2153
+ _DeviceFieldRow(
2154
+ children: <Widget>[
2155
+ _DeviceField(
2156
+ label: 'Swipe X1 / Y1',
2157
+ child: Row(
2158
+ children: <Widget>[
2159
+ Expanded(child: TextField(controller: swipeX1Controller)),
2160
+ const SizedBox(width: 8),
2161
+ Expanded(child: TextField(controller: swipeY1Controller)),
2162
+ ],
2163
+ ),
2164
+ ),
2165
+ _DeviceField(
2166
+ label: 'Swipe X2 / Y2',
2167
+ child: Row(
2168
+ children: <Widget>[
2169
+ Expanded(child: TextField(controller: swipeX2Controller)),
2170
+ const SizedBox(width: 8),
2171
+ Expanded(child: TextField(controller: swipeY2Controller)),
2172
+ ],
2173
+ ),
2174
+ ),
2175
+ ],
2176
+ ),
2177
+ const SizedBox(height: 10),
2178
+ OutlinedButton.icon(
2179
+ onPressed: controller.isRunningDeviceAction
2180
+ ? null
2181
+ : () => controller.swipeAndroidRuntime(<String, dynamic>{
2182
+ 'x1': toInt(swipeX1Controller.text),
2183
+ 'y1': toInt(swipeY1Controller.text),
2184
+ 'x2': toInt(swipeX2Controller.text),
2185
+ 'y2': toInt(swipeY2Controller.text),
2186
+ }),
2187
+ icon: Icon(Icons.swipe_outlined),
2188
+ label: Text('Swipe'),
2189
+ ),
2190
+ const SizedBox(height: 18),
2191
+ _RuntimePreview(
2192
+ title: 'Latest Android Screenshot',
2193
+ screenshotPath: controller.androidScreenshotPath,
2194
+ controller: controller,
2195
+ ),
2196
+ if (controller.androidUiPreview.isNotEmpty) ...<Widget>[
2197
+ const SizedBox(height: 14),
2198
+ Text(
2199
+ 'Latest UI dump preview',
2200
+ style: TextStyle(fontWeight: FontWeight.w700),
2201
+ ),
2202
+ const SizedBox(height: 10),
2203
+ ...controller.androidUiPreview.take(6).map((node) {
2204
+ final title = node['text']?.toString().trim().isNotEmpty == true
2205
+ ? node['text'].toString()
2206
+ : node['description']?.toString().trim().isNotEmpty == true
2207
+ ? node['description'].toString()
2208
+ : node['resourceId']?.toString().trim().isNotEmpty == true
2209
+ ? node['resourceId'].toString()
2210
+ : node['className']?.toString() ?? 'node';
2211
+ return Container(
2212
+ margin: const EdgeInsets.only(bottom: 8),
2213
+ padding: const EdgeInsets.all(12),
2214
+ decoration: BoxDecoration(
2215
+ color: _bgSecondary,
2216
+ borderRadius: BorderRadius.circular(10),
2217
+ border: Border.all(color: _border),
2218
+ ),
2219
+ child: Text(
2220
+ '$title\n${node['packageName'] ?? ''}',
2221
+ style: TextStyle(color: _textSecondary, height: 1.5),
2222
+ ),
2223
+ );
2224
+ }),
2225
+ ],
2226
+ if (controller.androidLastResult?.trim().isNotEmpty ??
2227
+ false) ...<Widget>[
2228
+ const SizedBox(height: 14),
2229
+ _ResultBlock(
2230
+ label: 'Last Android result',
2231
+ value: controller.androidLastResult!,
2232
+ ),
2233
+ ],
2234
+ ],
2235
+ );
2236
+ }
2237
+ }
2238
+
2239
+ class _DeviceFieldRow extends StatelessWidget {
2240
+ const _DeviceFieldRow({required this.children});
2241
+
2242
+ final List<Widget> children;
2243
+
2244
+ @override
2245
+ Widget build(BuildContext context) {
2246
+ if (children.isEmpty) {
2247
+ return const SizedBox.shrink();
2248
+ }
2249
+ final stacked = MediaQuery.sizeOf(context).width < 860;
2250
+ if (stacked) {
2251
+ return Column(
2252
+ children: children
2253
+ .map(
2254
+ (child) => Padding(
2255
+ padding: const EdgeInsets.only(bottom: 10),
2256
+ child: child,
2257
+ ),
2258
+ )
2259
+ .toList(),
2260
+ );
2261
+ }
2262
+ return Row(
2263
+ crossAxisAlignment: CrossAxisAlignment.start,
2264
+ children: <Widget>[
2265
+ for (var index = 0; index < children.length; index++) ...<Widget>[
2266
+ if (index > 0) const SizedBox(width: 12),
2267
+ Expanded(child: children[index]),
2268
+ ],
2269
+ ],
2270
+ );
2271
+ }
2272
+ }
2273
+
2274
+ class _DeviceField extends StatelessWidget {
2275
+ const _DeviceField({required this.label, required this.child});
2276
+
2277
+ final String label;
2278
+ final Widget child;
2279
+
2280
+ @override
2281
+ Widget build(BuildContext context) {
2282
+ return Column(
2283
+ crossAxisAlignment: CrossAxisAlignment.start,
2284
+ children: <Widget>[
2285
+ Text(
2286
+ label,
2287
+ style: TextStyle(
2288
+ color: _textSecondary,
2289
+ fontSize: 12,
2290
+ fontWeight: FontWeight.w700,
2291
+ ),
2292
+ ),
2293
+ const SizedBox(height: 8),
2294
+ child,
2295
+ ],
2296
+ );
2297
+ }
2298
+ }
2299
+
2300
+ class _RuntimePreview extends StatelessWidget {
2301
+ const _RuntimePreview({
2302
+ required this.title,
2303
+ required this.screenshotPath,
2304
+ required this.controller,
2305
+ });
2306
+
2307
+ final String title;
2308
+ final String? screenshotPath;
2309
+ final NeoAgentController controller;
2310
+
2311
+ @override
2312
+ Widget build(BuildContext context) {
2313
+ if (screenshotPath == null || screenshotPath!.isEmpty) {
2314
+ return _EmptyCard(
2315
+ title: title,
2316
+ subtitle:
2317
+ 'Run a screenshot-capable action to preview the live runtime.',
2318
+ );
2319
+ }
2320
+
2321
+ final uri = controller.resolveRuntimeAsset(screenshotPath!);
2322
+ return Card(
2323
+ color: _bgSecondary,
2324
+ child: Padding(
2325
+ padding: const EdgeInsets.all(14),
2326
+ child: Column(
2327
+ crossAxisAlignment: CrossAxisAlignment.start,
2328
+ children: <Widget>[
2329
+ Text(title, style: TextStyle(fontWeight: FontWeight.w700)),
2330
+ const SizedBox(height: 8),
2331
+ Text(
2332
+ screenshotPath!,
2333
+ style: TextStyle(color: _textSecondary, fontSize: 12),
2334
+ ),
2335
+ const SizedBox(height: 12),
2336
+ ClipRRect(
2337
+ borderRadius: BorderRadius.circular(12),
2338
+ child: Image.network(
2339
+ uri.toString(),
2340
+ headers: controller.authenticatedImageHeaders,
2341
+ fit: BoxFit.cover,
2342
+ errorBuilder: (context, _, __) {
2343
+ return Container(
2344
+ height: 220,
2345
+ color: _bgCard,
2346
+ alignment: Alignment.center,
2347
+ child: Text(
2348
+ 'Could not load preview image',
2349
+ style: TextStyle(color: _textSecondary),
2350
+ ),
2351
+ );
2352
+ },
2353
+ ),
2354
+ ),
2355
+ ],
2356
+ ),
2357
+ ),
2358
+ );
2359
+ }
2360
+ }
2361
+
2362
+ class _ResultBlock extends StatelessWidget {
2363
+ const _ResultBlock({required this.label, required this.value});
2364
+
2365
+ final String label;
2366
+ final String value;
2367
+
2368
+ @override
2369
+ Widget build(BuildContext context) {
2370
+ return Container(
2371
+ width: double.infinity,
2372
+ padding: const EdgeInsets.all(14),
2373
+ decoration: BoxDecoration(
2374
+ color: _bgSecondary,
2375
+ borderRadius: BorderRadius.circular(12),
2376
+ border: Border.all(color: _border),
2377
+ ),
2378
+ child: Column(
2379
+ crossAxisAlignment: CrossAxisAlignment.start,
2380
+ children: <Widget>[
2381
+ Text(label, style: TextStyle(fontWeight: FontWeight.w700)),
2382
+ const SizedBox(height: 10),
2383
+ SelectableText(
2384
+ value,
2385
+ style: TextStyle(
2386
+ fontFamily: GoogleFonts.jetBrainsMono().fontFamily,
2387
+ fontSize: 12,
2388
+ color: _textSecondary,
2389
+ height: 1.5,
2390
+ ),
2391
+ ),
2392
+ ],
2393
+ ),
2394
+ );
2395
+ }
2396
+ }