neoagent 2.3.1-beta.0 → 2.3.1-beta.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (406) hide show
  1. package/.env.example +63 -5
  2. package/README.md +16 -5
  3. package/com.neoagent.plist +1 -1
  4. package/docs/automation.md +82 -41
  5. package/docs/capabilities.md +85 -89
  6. package/docs/configuration.md +89 -87
  7. package/docs/getting-started.md +46 -27
  8. package/docs/hardware.md +1 -1
  9. package/docs/index.md +24 -30
  10. package/docs/integrations.md +46 -60
  11. package/docs/migration.md +41 -146
  12. package/docs/operations.md +23 -20
  13. package/docs/skills.md +20 -20
  14. package/docs/why-neoagent.md +15 -13
  15. package/extensions/chrome-browser/background.mjs +10 -2
  16. package/flutter_app/.metadata +42 -0
  17. package/flutter_app/DESIGN_SYSTEM.md +166 -0
  18. package/flutter_app/README.md +21 -0
  19. package/flutter_app/analysis_options.yaml +32 -0
  20. package/flutter_app/android/app/build.gradle.kts +111 -0
  21. package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
  22. package/flutter_app/android/app/src/main/AndroidManifest.xml +185 -0
  23. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +888 -0
  24. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/auto/NeoAgentCarAppService.kt +114 -0
  25. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
  26. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
  27. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
  28. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
  29. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
  30. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
  31. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
  32. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
  33. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
  34. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
  35. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnection.kt +118 -0
  36. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnectionService.kt +86 -0
  37. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
  38. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
  39. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
  40. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
  41. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
  42. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
  43. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
  44. package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
  45. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
  46. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
  47. package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
  48. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
  49. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
  50. package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
  51. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  52. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  53. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  54. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  55. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  56. package/flutter_app/android/app/src/main/res/values/arrays.xml +8 -0
  57. package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
  58. package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
  59. package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
  60. package/flutter_app/android/app/src/main/res/xml/automotive_app_desc.xml +4 -0
  61. package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
  62. package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
  63. package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
  64. package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
  65. package/flutter_app/android/build.gradle.kts +24 -0
  66. package/flutter_app/android/ci-release.keystore +0 -0
  67. package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
  68. package/flutter_app/android/gradle.properties +3 -0
  69. package/flutter_app/android/key.properties +4 -0
  70. package/flutter_app/android/settings.gradle.kts +26 -0
  71. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  72. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  73. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  74. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  75. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  76. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  77. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  78. package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
  79. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  80. package/flutter_app/lib/features/location/location_service.dart +117 -0
  81. package/flutter_app/lib/features/notifications/notification_interceptor.dart +73 -0
  82. package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +571 -0
  83. package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +275 -0
  84. package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +366 -0
  85. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +57 -0
  86. package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +218 -0
  87. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +46 -0
  88. package/flutter_app/lib/main.dart +107 -0
  89. package/flutter_app/lib/main_account_settings.dart +1340 -0
  90. package/flutter_app/lib/main_admin.dart +900 -0
  91. package/flutter_app/lib/main_app_shell.dart +2162 -0
  92. package/flutter_app/lib/main_chat.dart +3902 -0
  93. package/flutter_app/lib/main_controller.dart +7229 -0
  94. package/flutter_app/lib/main_devices.dart +1738 -0
  95. package/flutter_app/lib/main_integrations.dart +917 -0
  96. package/flutter_app/lib/main_launcher.dart +959 -0
  97. package/flutter_app/lib/main_launcher_entry.dart +5 -0
  98. package/flutter_app/lib/main_models.dart +3807 -0
  99. package/flutter_app/lib/main_navigation.dart +206 -0
  100. package/flutter_app/lib/main_operations.dart +5249 -0
  101. package/flutter_app/lib/main_recordings.dart +920 -0
  102. package/flutter_app/lib/main_runtime.dart +869 -0
  103. package/flutter_app/lib/main_settings.dart +2288 -0
  104. package/flutter_app/lib/main_shared.dart +3393 -0
  105. package/flutter_app/lib/main_spacing.dart +33 -0
  106. package/flutter_app/lib/main_theme.dart +276 -0
  107. package/flutter_app/lib/main_unified.dart +388 -0
  108. package/flutter_app/lib/main_voice_assistant.dart +957 -0
  109. package/flutter_app/lib/src/analytics_service.dart +294 -0
  110. package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
  111. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
  112. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
  113. package/flutter_app/lib/src/android_app_installer.dart +22 -0
  114. package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
  115. package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
  116. package/flutter_app/lib/src/android_auto_bridge.dart +59 -0
  117. package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
  118. package/flutter_app/lib/src/app_launch_bridge.dart +58 -0
  119. package/flutter_app/lib/src/app_release_updater.dart +511 -0
  120. package/flutter_app/lib/src/backend_client.dart +1870 -0
  121. package/flutter_app/lib/src/desktop_companion.dart +2 -0
  122. package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
  123. package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
  124. package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
  125. package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
  126. package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
  127. package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
  128. package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
  129. package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
  130. package/flutter_app/lib/src/health_bridge.dart +136 -0
  131. package/flutter_app/lib/src/live_voice_capture.dart +85 -0
  132. package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
  133. package/flutter_app/lib/src/network/app_http_client.dart +53 -0
  134. package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
  135. package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
  136. package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
  137. package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
  138. package/flutter_app/lib/src/oauth_launcher.dart +33 -0
  139. package/flutter_app/lib/src/oauth_launcher_io.dart +95 -0
  140. package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
  141. package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
  142. package/flutter_app/lib/src/recording_bridge.dart +232 -0
  143. package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
  144. package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
  145. package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
  146. package/flutter_app/lib/src/recording_payloads.dart +86 -0
  147. package/flutter_app/lib/src/theme/palette.dart +81 -0
  148. package/flutter_app/lib/src/web_app_update_monitor.dart +17 -0
  149. package/flutter_app/lib/src/web_app_update_monitor_stub.dart +24 -0
  150. package/flutter_app/lib/src/web_app_update_monitor_web.dart +123 -0
  151. package/flutter_app/lib/src/widget_bridge.dart +49 -0
  152. package/flutter_app/linux/CMakeLists.txt +128 -0
  153. package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
  154. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +47 -0
  155. package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
  156. package/flutter_app/linux/flutter/generated_plugins.cmake +32 -0
  157. package/flutter_app/linux/runner/CMakeLists.txt +26 -0
  158. package/flutter_app/linux/runner/main.cc +6 -0
  159. package/flutter_app/linux/runner/my_application.cc +144 -0
  160. package/flutter_app/linux/runner/my_application.h +18 -0
  161. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  162. package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
  163. package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
  164. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +48 -0
  165. package/flutter_app/macos/Podfile +42 -0
  166. package/flutter_app/macos/Podfile.lock +87 -0
  167. package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
  168. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
  169. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  170. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  171. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  172. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  173. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  174. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  175. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  176. package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
  177. package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
  178. package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
  179. package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
  180. package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
  181. package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
  182. package/flutter_app/macos/Runner/Info.plist +36 -0
  183. package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
  184. package/flutter_app/macos/Runner/Release.entitlements +12 -0
  185. package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
  186. package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  187. package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
  188. package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
  189. package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  190. package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
  191. package/flutter_app/patch_strings.py +12 -0
  192. package/flutter_app/pubspec.lock +1256 -0
  193. package/flutter_app/pubspec.yaml +59 -0
  194. package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
  195. package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
  196. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
  197. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
  198. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
  199. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
  200. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
  201. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
  202. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
  203. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
  204. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
  205. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
  206. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
  207. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
  208. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
  209. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
  210. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
  211. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
  212. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
  213. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
  214. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
  215. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
  216. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
  217. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
  218. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
  219. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
  220. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
  221. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
  222. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
  223. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
  224. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
  225. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
  226. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
  227. package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
  228. package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
  229. package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
  230. package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
  231. package/flutter_app/tool/generate_desktop_branding.py +219 -0
  232. package/flutter_app/web/favicon.png +0 -0
  233. package/flutter_app/web/favicon.svg +12 -0
  234. package/flutter_app/web/icons/Icon-192.png +0 -0
  235. package/flutter_app/web/icons/Icon-512.png +0 -0
  236. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  237. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  238. package/flutter_app/web/index.html +39 -0
  239. package/flutter_app/web/manifest.json +35 -0
  240. package/flutter_app/windows/CMakeLists.txt +108 -0
  241. package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
  242. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +50 -0
  243. package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
  244. package/flutter_app/windows/flutter/generated_plugins.cmake +36 -0
  245. package/flutter_app/windows/runner/CMakeLists.txt +41 -0
  246. package/flutter_app/windows/runner/Runner.rc +121 -0
  247. package/flutter_app/windows/runner/flutter_window.cpp +533 -0
  248. package/flutter_app/windows/runner/flutter_window.h +37 -0
  249. package/flutter_app/windows/runner/main.cpp +53 -0
  250. package/flutter_app/windows/runner/resource.h +16 -0
  251. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  252. package/flutter_app/windows/runner/runner.exe.manifest +14 -0
  253. package/flutter_app/windows/runner/utils.cpp +65 -0
  254. package/flutter_app/windows/runner/utils.h +19 -0
  255. package/flutter_app/windows/runner/win32_window.cpp +299 -0
  256. package/flutter_app/windows/runner/win32_window.h +102 -0
  257. package/lib/install_helpers.js +31 -0
  258. package/lib/manager.js +465 -43
  259. package/package.json +11 -3
  260. package/runtime/paths.js +137 -0
  261. package/server/config/analytics.js +30 -0
  262. package/server/db/database.js +363 -21
  263. package/server/guest-agent.android.package.json +13 -0
  264. package/server/guest-agent.browser.package.json +14 -0
  265. package/server/guest_agent.js +98 -52
  266. package/server/http/middleware.js +55 -2
  267. package/server/http/routes.js +6 -1
  268. package/server/http/static.js +41 -1
  269. package/server/index.js +16 -16
  270. package/server/public/.last_build_id +1 -1
  271. package/server/public/assets/AssetManifest.bin +1 -1
  272. package/server/public/assets/AssetManifest.bin.json +1 -1
  273. package/server/public/assets/AssetManifest.json +1 -1
  274. package/server/public/assets/NOTICES +679 -129
  275. package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
  276. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  277. package/server/public/assets/packages/mixpanel_flutter/assets/mixpanel.js +3 -0
  278. package/server/public/canvaskit/wimp.wasm +0 -0
  279. package/server/public/flutter_bootstrap.js +2 -2
  280. package/server/public/main.dart.js +95866 -87333
  281. package/server/routes/account.js +26 -1
  282. package/server/routes/agents.js +6 -2
  283. package/server/routes/android.js +22 -6
  284. package/server/routes/auth.js +25 -5
  285. package/server/routes/browser.js +47 -13
  286. package/server/routes/integrations.js +119 -9
  287. package/server/routes/memory.js +179 -8
  288. package/server/routes/runtime.js +14 -0
  289. package/server/routes/screenHistory.js +46 -0
  290. package/server/routes/settings.js +102 -24
  291. package/server/routes/skills.js +12 -0
  292. package/server/routes/social_video.js +66 -0
  293. package/server/routes/tasks.js +22 -2
  294. package/server/routes/triggers.js +90 -0
  295. package/server/routes/wearable.js +67 -0
  296. package/server/routes/widgets.js +4 -4
  297. package/server/services/ai/capabilityHealth.js +23 -51
  298. package/server/services/ai/compaction.js +1 -1
  299. package/server/services/ai/deliverables/artifact_helpers.js +190 -0
  300. package/server/services/ai/deliverables/contracts.js +113 -0
  301. package/server/services/ai/deliverables/index.js +20 -0
  302. package/server/services/ai/deliverables/selector.js +94 -0
  303. package/server/services/ai/deliverables/validator.js +63 -0
  304. package/server/services/ai/deliverables/workflows.js +207 -0
  305. package/server/services/ai/engine.js +372 -27
  306. package/server/services/ai/hooks.js +127 -0
  307. package/server/services/ai/integrated_tools/index.js +98 -0
  308. package/server/services/ai/integrated_tools/remotion.js +373 -0
  309. package/server/services/ai/integrated_tools/shared.js +204 -0
  310. package/server/services/ai/integrated_tools/slidev.js +207 -0
  311. package/server/services/ai/loopPolicy.js +146 -0
  312. package/server/services/ai/models.js +36 -0
  313. package/server/services/ai/providers/githubCopilot.js +97 -0
  314. package/server/services/ai/providers/openai.js +2 -1
  315. package/server/services/ai/providers/openaiCodex.js +384 -0
  316. package/server/services/ai/recordingInsights.js +11 -13
  317. package/server/services/ai/runEvents.js +100 -0
  318. package/server/services/ai/settings.js +20 -0
  319. package/server/services/ai/systemPrompt.js +16 -2
  320. package/server/services/ai/taskAnalysis.js +2 -0
  321. package/server/services/ai/toolResult.js +20 -0
  322. package/server/services/ai/toolRunner.js +46 -6
  323. package/server/services/ai/toolSelector.js +54 -1
  324. package/server/services/ai/tools.js +360 -127
  325. package/server/services/android/android_bootstrap_worker.js +63 -0
  326. package/server/services/android/controller.js +442 -1760
  327. package/server/services/bootstrap_helpers.js +0 -34
  328. package/server/services/browser/controller.js +326 -52
  329. package/server/services/cli/executor.js +40 -1
  330. package/server/services/desktop/screenRecorder.js +182 -0
  331. package/server/services/integrations/env.js +10 -0
  332. package/server/services/integrations/figma/provider.js +1 -0
  333. package/server/services/integrations/github/common.js +106 -0
  334. package/server/services/integrations/github/provider.js +499 -0
  335. package/server/services/integrations/github/repos.js +1124 -0
  336. package/server/services/integrations/google/provider.js +1 -0
  337. package/server/services/integrations/manager.js +88 -12
  338. package/server/services/integrations/microsoft/provider.js +1 -0
  339. package/server/services/integrations/oauth_provider.js +25 -8
  340. package/server/services/integrations/provider_config_store.js +85 -0
  341. package/server/services/integrations/registry.js +8 -2
  342. package/server/services/integrations/spotify/provider.js +488 -0
  343. package/server/services/integrations/trello/provider.js +842 -0
  344. package/server/services/integrations/weather/provider.js +559 -0
  345. package/server/services/integrations/whatsapp/provider.js +6 -2
  346. package/server/services/manager.js +109 -154
  347. package/server/services/mcp/client.js +120 -23
  348. package/server/services/memory/llm_transfer.js +218 -0
  349. package/server/services/memory/manager.js +299 -35
  350. package/server/services/messaging/access_policy.js +10 -0
  351. package/server/services/messaging/automation.js +1 -1
  352. package/server/services/messaging/manager.js +93 -8
  353. package/server/services/messaging/meshtastic.js +277 -0
  354. package/server/services/messaging/meshtastic_env.js +40 -0
  355. package/server/services/messaging/meshtastic_protocol.js +530 -0
  356. package/server/services/messaging/meshtastic_tcp_transport.js +25 -0
  357. package/server/services/runtime/backends/local-vm.js +221 -158
  358. package/server/services/runtime/docker-vm-manager.js +392 -0
  359. package/server/services/runtime/guest_bootstrap.js +654 -0
  360. package/server/services/runtime/manager.js +85 -29
  361. package/server/services/runtime/settings.js +22 -31
  362. package/server/services/runtime/validation.js +16 -31
  363. package/server/services/skills/base_catalog.js +33 -0
  364. package/server/services/social_video/adapters/base.js +26 -0
  365. package/server/services/social_video/adapters/index.js +27 -0
  366. package/server/services/social_video/adapters/instagram.js +17 -0
  367. package/server/services/social_video/adapters/tiktok.js +17 -0
  368. package/server/services/social_video/adapters/x.js +17 -0
  369. package/server/services/social_video/adapters/youtube.js +17 -0
  370. package/server/services/social_video/captions.js +187 -0
  371. package/server/services/social_video/frame.js +42 -0
  372. package/server/services/social_video/index.js +7 -0
  373. package/server/services/social_video/metadata.js +63 -0
  374. package/server/services/social_video/result.js +63 -0
  375. package/server/services/social_video/service.js +720 -0
  376. package/server/services/social_video/url.js +83 -0
  377. package/server/services/tasks/adapters/index.js +2 -0
  378. package/server/services/tasks/adapters/manual.js +12 -0
  379. package/server/services/tasks/adapters/schedule.js +33 -5
  380. package/server/services/tasks/adapters/weather_event.js +84 -0
  381. package/server/services/tasks/integration_runtime.js +86 -1
  382. package/server/services/tasks/runtime.js +3 -3
  383. package/server/services/tasks/task_repository.js +3 -6
  384. package/server/services/voice/agentBridge.js +20 -4
  385. package/server/services/voice/message.js +3 -0
  386. package/server/services/voice/openaiClient.js +4 -1
  387. package/server/services/voice/openaiSpeech.js +6 -1
  388. package/server/services/voice/providers.js +144 -15
  389. package/server/services/voice/runtime.js +8 -8
  390. package/server/services/voice/runtimeManager.js +292 -20
  391. package/server/services/voice/turnRunner.js +29 -9
  392. package/server/services/wearable/firmware_manifest.js +353 -0
  393. package/server/services/wearable/gateway.js +350 -0
  394. package/server/services/wearable/protocol.js +45 -0
  395. package/server/services/wearable/service.js +240 -0
  396. package/server/services/websocket.js +3 -1
  397. package/server/services/widgets/focus_widget.js +137 -0
  398. package/server/services/widgets/service.js +182 -6
  399. package/server/services/workspace/manager.js +301 -0
  400. package/server/utils/deployment.js +16 -9
  401. package/server/utils/local_secrets.js +56 -0
  402. package/server/utils/logger.js +37 -9
  403. package/server/services/integrations/home_assistant/provider.js +0 -350
  404. package/server/services/messaging/access_policy.test.js +0 -228
  405. package/server/services/runtime/backends/host.js +0 -37
  406. package/server/services/runtime/qemu.js +0 -391
@@ -0,0 +1,1738 @@
1
+ part of 'main.dart';
2
+
3
+ class _OptimizedScreenshotPayload {
4
+ const _OptimizedScreenshotPayload({
5
+ required this.bytes,
6
+ required this.mimeType,
7
+ required this.resizedOrReencoded,
8
+ });
9
+
10
+ final Uint8List bytes;
11
+ final String mimeType;
12
+ final bool resizedOrReencoded;
13
+ }
14
+
15
+ class DevicesPanel extends StatefulWidget {
16
+ const DevicesPanel({super.key, required this.controller});
17
+
18
+ final NeoAgentController controller;
19
+
20
+ @override
21
+ State<DevicesPanel> createState() => _DevicesPanelState();
22
+ }
23
+
24
+ class _DevicesPanelState extends State<DevicesPanel> {
25
+ late final TextEditingController _browserUrlController;
26
+ late final TextEditingController _androidLaunchController;
27
+ late final TextEditingController _desktopLaunchController;
28
+ late final TextEditingController _textEntryController;
29
+ Timer? _surfaceFrameTimer;
30
+ _DeviceSurface _surface = _DeviceSurface.browser;
31
+ _DeviceSurface? _runningSurface;
32
+
33
+ @override
34
+ void initState() {
35
+ super.initState();
36
+ _browserUrlController = TextEditingController(text: _browserUrlPlaceholder);
37
+ _androidLaunchController = TextEditingController(
38
+ text: _androidLaunchPlaceholder,
39
+ );
40
+ _desktopLaunchController = TextEditingController();
41
+ _textEntryController = TextEditingController();
42
+ WidgetsBinding.instance.addPostFrameCallback((_) {
43
+ if (!mounted) {
44
+ return;
45
+ }
46
+ unawaited(_bootstrapSurface());
47
+ });
48
+ _surfaceFrameTimer = Timer.periodic(const Duration(seconds: 3), (_) {
49
+ unawaited(_refreshSurfaceFrame());
50
+ });
51
+ }
52
+
53
+ @override
54
+ void dispose() {
55
+ for (final controller in <TextEditingController>[
56
+ _browserUrlController,
57
+ _androidLaunchController,
58
+ _desktopLaunchController,
59
+ _textEntryController,
60
+ ]) {
61
+ controller.dispose();
62
+ }
63
+ _surfaceFrameTimer?.cancel();
64
+ super.dispose();
65
+ }
66
+
67
+ bool get _isBrowser => _surface == _DeviceSurface.browser;
68
+ bool get _isDesktop => _surface == _DeviceSurface.desktop;
69
+
70
+ bool get _isCurrentSurfaceBusy =>
71
+ widget.controller.isRunningDeviceAction &&
72
+ (_runningSurface == null || _runningSurface == _surface);
73
+
74
+ Future<T> _runOnSurface<T>(Future<T> Function() action) async {
75
+ final surface = _surface;
76
+ if (mounted) setState(() => _runningSurface = surface);
77
+ try {
78
+ return await action();
79
+ } finally {
80
+ if (mounted) setState(() => _runningSurface = null);
81
+ }
82
+ }
83
+
84
+ bool get _androidOnline {
85
+ final status = widget.controller.androidRuntime;
86
+ final devices = _jsonMapList(status['devices'], fallbackToMapValues: true);
87
+ return devices.any((device) => device['status']?.toString() == 'device');
88
+ }
89
+
90
+ bool get _androidStarting =>
91
+ widget.controller.androidRuntime['starting'] == true;
92
+
93
+ List<Map<String, dynamic>> get _onlineDesktopDevices => widget
94
+ .controller
95
+ .desktopDevices
96
+ .where((device) => device['online'] == true)
97
+ .toList(growable: false);
98
+
99
+ bool get _desktopOnline => _onlineDesktopDevices.isNotEmpty;
100
+
101
+ bool get _desktopRequiresSelection =>
102
+ _isDesktop &&
103
+ _onlineDesktopDevices.length > 1 &&
104
+ (widget.controller.selectedDesktopDeviceId ?? '').isEmpty;
105
+
106
+ String? get _activeScreenshotPath {
107
+ if (_isBrowser) {
108
+ return widget.controller.browserScreenshotPath;
109
+ }
110
+ if (_isDesktop) {
111
+ return widget.controller.desktopScreenshotPath;
112
+ }
113
+ return widget.controller.androidScreenshotPath;
114
+ }
115
+
116
+ Future<void> _bootstrapSurface() async {
117
+ await widget.controller.refreshDevices();
118
+ await _ensurePreview();
119
+ }
120
+
121
+ Future<void> _ensurePreview() async {
122
+ final controller = widget.controller;
123
+ if (_isBrowser) {
124
+ if (controller.browserRuntime['launched'] != true) {
125
+ return;
126
+ }
127
+ if ((controller.browserScreenshotPath ?? '').isEmpty) {
128
+ final currentUrl =
129
+ controller.browserRuntime['pageInfo'] is Map<dynamic, dynamic>
130
+ ? (controller.browserRuntime['pageInfo'] as Map)['url']?.toString()
131
+ : null;
132
+ if (currentUrl != null && currentUrl.isNotEmpty) {
133
+ await controller.navigateBrowserRuntime(url: currentUrl);
134
+ } else {
135
+ await controller.screenshotBrowserRuntime();
136
+ }
137
+ }
138
+ return;
139
+ }
140
+
141
+ if (_isDesktop) {
142
+ if (_desktopRequiresSelection || !_desktopOnline) {
143
+ return;
144
+ }
145
+ if ((controller.desktopScreenshotPath ?? '').isEmpty) {
146
+ await controller.screenshotDesktopRuntime();
147
+ }
148
+ return;
149
+ }
150
+
151
+ if (_androidOnline && (controller.androidScreenshotPath ?? '').isEmpty) {
152
+ await controller.screenshotAndroidRuntime();
153
+ }
154
+ }
155
+
156
+ Future<void> _refreshSurfaceFrame() async {
157
+ if (!mounted ||
158
+ widget.controller.selectedSection != AppSection.devices ||
159
+ widget.controller.isRunningDeviceAction ||
160
+ widget.controller.isRefreshingDevices) {
161
+ return;
162
+ }
163
+ if (_isBrowser) {
164
+ await widget.controller.refreshBrowserFrameRuntime();
165
+ return;
166
+ }
167
+ if (_isDesktop) {
168
+ await widget.controller.refreshDesktopFrameRuntime();
169
+ return;
170
+ }
171
+ if (_androidStarting) {
172
+ await widget.controller.refreshDevices();
173
+ if (_androidOnline &&
174
+ (widget.controller.androidScreenshotPath ?? '').isEmpty) {
175
+ await widget.controller.screenshotAndroidRuntime();
176
+ }
177
+ return;
178
+ }
179
+ await widget.controller.refreshAndroidFrameRuntime();
180
+ }
181
+
182
+ Future<void> _selectSurface(_DeviceSurface surface) async {
183
+ setState(() => _surface = surface);
184
+ await _ensurePreview();
185
+ }
186
+
187
+ Future<void> _openPrimary() => _runOnSurface(_openPrimaryInner);
188
+
189
+ Future<void> _openPrimaryInner() async {
190
+ final controller = widget.controller;
191
+ if (_isBrowser) {
192
+ await controller.navigateBrowserRuntime(
193
+ url: _browserUrlController.text.trim(),
194
+ );
195
+ return;
196
+ }
197
+
198
+ if (_isDesktop) {
199
+ if (_desktopRequiresSelection) {
200
+ final selectedId = widget.controller.selectedDesktopDeviceId;
201
+ if ((selectedId ?? '').isEmpty && _onlineDesktopDevices.length == 1) {
202
+ await controller.selectDesktopDeviceRuntime(
203
+ _onlineDesktopDevices.first['deviceId']?.toString() ?? '',
204
+ );
205
+ } else {
206
+ await controller.openDesktopSelectionRuntime();
207
+ }
208
+ return;
209
+ }
210
+ if (!_desktopOnline) {
211
+ return;
212
+ }
213
+ final selectedId = widget.controller.selectedDesktopDeviceId;
214
+ if ((selectedId ?? '').isEmpty && _onlineDesktopDevices.length == 1) {
215
+ await controller.selectDesktopDeviceRuntime(
216
+ _onlineDesktopDevices.first['deviceId']?.toString() ?? '',
217
+ );
218
+ }
219
+ final raw = _desktopLaunchController.text.trim();
220
+ if (raw.isNotEmpty) {
221
+ await controller.launchDesktopAppRuntime(raw);
222
+ return;
223
+ }
224
+ await controller.observeDesktopRuntime();
225
+ return;
226
+ }
227
+
228
+ if (!_androidOnline) {
229
+ await controller.startAndroidRuntime();
230
+ await widget.controller.refreshDevices();
231
+ if (_androidOnline) {
232
+ await controller.screenshotAndroidRuntime();
233
+ }
234
+ return;
235
+ }
236
+
237
+ final raw = _androidLaunchController.text.trim();
238
+ if (raw.isEmpty) {
239
+ return;
240
+ }
241
+ if (raw.startsWith('http://') || raw.startsWith('https://')) {
242
+ await controller.openAndroidIntentRuntime(
243
+ action: 'android.intent.action.VIEW',
244
+ dataUri: raw,
245
+ );
246
+ return;
247
+ }
248
+ await controller.openAndroidAppRuntime(packageName: raw);
249
+ }
250
+
251
+ Future<void> _sleepPrimary() => _runOnSurface(_sleepPrimaryInner);
252
+
253
+ Future<void> _sleepPrimaryInner() async {
254
+ final controller = widget.controller;
255
+ if (_isBrowser) {
256
+ if (controller.browserRuntime['launched'] != true) {
257
+ return;
258
+ }
259
+ await controller.closeBrowserRuntime();
260
+ return;
261
+ }
262
+ if (_isDesktop) {
263
+ final selectedId = widget.controller.selectedDesktopDeviceId;
264
+ if ((selectedId ?? '').isNotEmpty) {
265
+ await controller.pauseDesktopDeviceRuntime(selectedId!);
266
+ }
267
+ return;
268
+ }
269
+ if (!_androidOnline) {
270
+ return;
271
+ }
272
+ await controller.stopAndroidRuntime();
273
+ }
274
+
275
+ Future<void> _sendText() => _runOnSurface(() async {
276
+ final text = _textEntryController.text;
277
+ if (text.trim().isEmpty) {
278
+ return;
279
+ }
280
+ if (_isBrowser) {
281
+ await widget.controller.typeBrowserTextRuntime(text, pressEnter: true);
282
+ } else if (_isDesktop) {
283
+ await widget.controller.typeDesktopRuntime(text, pressEnter: true);
284
+ } else {
285
+ await widget.controller.typeAndroidRuntime(<String, dynamic>{
286
+ 'text': text,
287
+ 'pressEnter': true,
288
+ });
289
+ }
290
+ });
291
+
292
+ Future<void> _handleTap(Offset point) => _runOnSurface(() async {
293
+ if (_isBrowser) {
294
+ await widget.controller.clickBrowserPointRuntime(
295
+ x: point.dx.round(),
296
+ y: point.dy.round(),
297
+ );
298
+ return;
299
+ }
300
+ if (_isDesktop) {
301
+ if (_desktopRequiresSelection) {
302
+ return;
303
+ }
304
+ await widget.controller.clickDesktopRuntime(
305
+ x: point.dx.round(),
306
+ y: point.dy.round(),
307
+ );
308
+ return;
309
+ }
310
+ if (!_androidOnline) {
311
+ await widget.controller.startAndroidRuntime();
312
+ return;
313
+ }
314
+ await widget.controller.tapAndroidRuntime(<String, dynamic>{
315
+ 'x': point.dx.round(),
316
+ 'y': point.dy.round(),
317
+ });
318
+ });
319
+
320
+ Future<void> _handleSwipe(Offset start, Offset end) => _runOnSurface(() async {
321
+ if (_isBrowser) {
322
+ await widget.controller.scrollBrowserRuntime(
323
+ deltaY: (start.dy - end.dy).round(),
324
+ );
325
+ return;
326
+ }
327
+ if (_isDesktop) {
328
+ if (_desktopRequiresSelection) {
329
+ return;
330
+ }
331
+ await widget.controller.dragDesktopRuntime(
332
+ x1: start.dx.round(),
333
+ y1: start.dy.round(),
334
+ x2: end.dx.round(),
335
+ y2: end.dy.round(),
336
+ );
337
+ return;
338
+ }
339
+ if (!_androidOnline) {
340
+ return;
341
+ }
342
+ await widget.controller.swipeAndroidRuntime(<String, dynamic>{
343
+ 'x1': start.dx.round(),
344
+ 'y1': start.dy.round(),
345
+ 'x2': end.dx.round(),
346
+ 'y2': end.dy.round(),
347
+ 'durationMs': 280,
348
+ });
349
+ });
350
+
351
+ Future<void> _runQuickAction(String action) => _runOnSurface(() async {
352
+ final controller = widget.controller;
353
+ switch (action) {
354
+ case 'browser_refresh':
355
+ await controller.navigateBrowserRuntime(
356
+ url: controller.browserRuntime['pageInfo'] is Map<dynamic, dynamic>
357
+ ? ((controller.browserRuntime['pageInfo'] as Map)['url']
358
+ ?.toString() ??
359
+ _browserUrlController.text.trim())
360
+ : _browserUrlController.text.trim(),
361
+ );
362
+ break;
363
+ case 'browser_enter':
364
+ await controller.pressBrowserKeyRuntime('Enter');
365
+ break;
366
+ case 'desktop_enter':
367
+ await controller.pressDesktopKeyRuntime('Return');
368
+ break;
369
+ case 'desktop_escape':
370
+ await controller.pressDesktopKeyRuntime('Escape');
371
+ break;
372
+ case 'android_back':
373
+ await controller.pressAndroidKeyRuntime('back');
374
+ break;
375
+ case 'android_home':
376
+ await controller.pressAndroidKeyRuntime('home');
377
+ break;
378
+ case 'android_recent':
379
+ await controller.pressAndroidKeyRuntime('app_switch');
380
+ break;
381
+ case 'surface_refresh':
382
+ await _ensurePreview();
383
+ break;
384
+ }
385
+ });
386
+
387
+ @override
388
+ Widget build(BuildContext context) {
389
+ final controller = widget.controller;
390
+ final browserStatus = controller.browserRuntime;
391
+ final prefersExtension = controller.browserBackend == 'extension';
392
+ final extensionConnected = controller.browserExtensionConnected;
393
+ final usingExtension = prefersExtension && extensionConnected;
394
+ final browserFallbackLabel = 'cloud browser runtime';
395
+ final browserPageInfo = browserStatus['pageInfo'] is Map<dynamic, dynamic>
396
+ ? Map<String, dynamic>.from(browserStatus['pageInfo'] as Map)
397
+ : const <String, dynamic>{};
398
+ final selectedDesktopDevice = controller.desktopDevices
399
+ .where(
400
+ (device) => device['deviceId'] == controller.selectedDesktopDeviceId,
401
+ )
402
+ .cast<Map<String, dynamic>?>()
403
+ .firstWhere((device) => device != null, orElse: () => null);
404
+ final desktopDeviceOnline = selectedDesktopDevice?['online'] == true;
405
+ return ListView(
406
+ padding: _pagePadding(context),
407
+ children: <Widget>[
408
+ _PageTitle(
409
+ title: 'Remote Device',
410
+ subtitle:
411
+ 'Tap, swipe, and type directly on the live surface. Use the arrows below to switch between browser, phone, and desktop.',
412
+ trailing: OutlinedButton.icon(
413
+ onPressed:
414
+ controller.isRefreshingDevices ||
415
+ controller.isRunningDeviceAction
416
+ ? null
417
+ : _bootstrapSurface,
418
+ icon: Icon(Icons.sync),
419
+ label: Text('Sync Surface'),
420
+ ),
421
+ ),
422
+ if (controller.errorMessage case final message?)
423
+ Padding(
424
+ padding: const EdgeInsets.only(bottom: 16),
425
+ child: _InlineError(message: message),
426
+ ),
427
+ Center(
428
+ child: ConstrainedBox(
429
+ constraints: const BoxConstraints(maxWidth: 980),
430
+ child: Card(
431
+ color: _bgCard,
432
+ shape: RoundedRectangleBorder(
433
+ borderRadius: BorderRadius.circular(28),
434
+ side: BorderSide(color: _borderLight),
435
+ ),
436
+ child: Padding(
437
+ padding: const EdgeInsets.all(20),
438
+ child: Column(
439
+ children: <Widget>[
440
+ _DeviceSurfaceHeader(
441
+ surface: _surface,
442
+ browserStatus: browserStatus,
443
+ browserPageInfo: browserPageInfo,
444
+ androidRuntime: controller.androidRuntime,
445
+ androidOnline: _androidOnline,
446
+ desktopRuntime: controller.desktopRuntime,
447
+ desktopDevices: controller.desktopDevices,
448
+ selectedDesktopDeviceId:
449
+ controller.selectedDesktopDeviceId,
450
+ browserExtensionPreferred: prefersExtension,
451
+ browserExtensionActive: usingExtension,
452
+ browserFallbackLabel: browserFallbackLabel,
453
+ ),
454
+ if (_isDesktop) ...<Widget>[
455
+ const SizedBox(height: 14),
456
+ DropdownButtonFormField<String>(
457
+ isExpanded: true,
458
+ initialValue: selectedDesktopDevice?['deviceId']
459
+ ?.toString(),
460
+ decoration: const InputDecoration(
461
+ labelText: 'Desktop device',
462
+ prefixIcon: Icon(Icons.computer_outlined),
463
+ ),
464
+ hint: const Text('Select a companion desktop'),
465
+ items: controller.desktopDevices.map((device) {
466
+ final deviceId = device['deviceId']?.toString() ?? '';
467
+ final label =
468
+ device['label']?.toString().trim().isNotEmpty ==
469
+ true
470
+ ? device['label'].toString()
471
+ : (device['hostname']?.toString() ?? deviceId);
472
+ final os =
473
+ device['platform']?.toString() ?? 'desktop';
474
+ final state = device['online'] == true
475
+ ? (device['paused'] == true ? 'paused' : 'online')
476
+ : 'offline';
477
+ return DropdownMenuItem<String>(
478
+ value: deviceId,
479
+ child: Text(
480
+ '$label · $os · $state',
481
+ maxLines: 1,
482
+ overflow: TextOverflow.ellipsis,
483
+ softWrap: false,
484
+ ),
485
+ );
486
+ }).toList(),
487
+ onChanged: _isCurrentSurfaceBusy
488
+ ? null
489
+ : (value) {
490
+ if (value == null || value.isEmpty) {
491
+ return;
492
+ }
493
+ unawaited(
494
+ controller.selectDesktopDeviceRuntime(value),
495
+ );
496
+ },
497
+ ),
498
+ const SizedBox(height: 10),
499
+ Wrap(
500
+ spacing: 10,
501
+ runSpacing: 10,
502
+ children: <Widget>[
503
+ _DotStatus(
504
+ label: desktopDeviceOnline
505
+ ? 'Companion live'
506
+ : controller.desktopCompanionConnected
507
+ ? 'Companion live'
508
+ : controller.desktopCompanionConnecting
509
+ ? 'Connecting'
510
+ : 'Companion idle',
511
+ color: desktopDeviceOnline
512
+ ? _success
513
+ : controller.desktopCompanionConnected
514
+ ? _success
515
+ : controller.desktopCompanionConnecting
516
+ ? _accent
517
+ : _warning,
518
+ ),
519
+ if (selectedDesktopDevice != null)
520
+ _DotStatus(
521
+ label: selectedDesktopDevice['paused'] == true
522
+ ? 'Paused'
523
+ : (selectedDesktopDevice['online'] == true
524
+ ? 'Ready'
525
+ : 'Offline'),
526
+ color: selectedDesktopDevice['paused'] == true
527
+ ? _warning
528
+ : (selectedDesktopDevice['online'] == true
529
+ ? _success
530
+ : _textMuted),
531
+ ),
532
+ ],
533
+ ),
534
+ ],
535
+ const SizedBox(height: 16),
536
+ _DeviceLaunchBar(
537
+ surface: _surface,
538
+ controller: _isBrowser
539
+ ? _browserUrlController
540
+ : (_isDesktop
541
+ ? _desktopLaunchController
542
+ : _androidLaunchController),
543
+ active: _isBrowser
544
+ ? browserStatus['launched'] == true
545
+ : (_isDesktop
546
+ ? _desktopOnline
547
+ : _androidOnline || _androidStarting),
548
+ starting: !_isBrowser && !_isDesktop && _androidStarting,
549
+ busy: _isCurrentSurfaceBusy,
550
+ onSubmit: _openPrimary,
551
+ onSleep: _sleepPrimary,
552
+ ),
553
+ const SizedBox(height: 18),
554
+ _InteractiveSurfacePreview(
555
+ surface: _surface,
556
+ controller: controller,
557
+ screenshotPath: _activeScreenshotPath,
558
+ busy: _isCurrentSurfaceBusy,
559
+ wakingUp: !_isBrowser && !_isDesktop && _androidStarting,
560
+ enabled: _isBrowser || _isDesktop || _androidOnline,
561
+ connectRequired: _desktopRequiresSelection,
562
+ onTapPoint: _handleTap,
563
+ onSwipe: _handleSwipe,
564
+ onWakeRequested: _openPrimary,
565
+ ),
566
+ if (!_isBrowser && !_isDesktop) ...<Widget>[
567
+ const SizedBox(height: 12),
568
+ _AndroidNavDock(
569
+ busy: _isCurrentSurfaceBusy,
570
+ androidOnline: _androidOnline,
571
+ onAction: _runQuickAction,
572
+ ),
573
+ if (kIsWeb) ...<Widget>[
574
+ const SizedBox(height: 14),
575
+ AndroidApkDropZone(
576
+ enabled: _androidOnline,
577
+ busy: _isCurrentSurfaceBusy,
578
+ onInstall: ({required filename, required bytes}) {
579
+ return controller.installAndroidApkRuntime(
580
+ filename: filename,
581
+ bytes: bytes,
582
+ );
583
+ },
584
+ ),
585
+ ],
586
+ ],
587
+ const SizedBox(height: 18),
588
+ _DeviceTypeDock(
589
+ controller: _textEntryController,
590
+ busy: _isCurrentSurfaceBusy,
591
+ surface: _surface,
592
+ onSubmit: _sendText,
593
+ ),
594
+ if (_isBrowser || _isDesktop) ...<Widget>[
595
+ const SizedBox(height: 14),
596
+ _DeviceQuickActions(
597
+ surface: _surface,
598
+ androidOnline: _androidOnline,
599
+ busy: _isCurrentSurfaceBusy,
600
+ onAction: _runQuickAction,
601
+ ),
602
+ ],
603
+ const SizedBox(height: 14),
604
+ _SurfaceSwitcher(
605
+ surface: _surface,
606
+ onSelect: _selectSurface,
607
+ ),
608
+ ],
609
+ ),
610
+ ),
611
+ ),
612
+ ),
613
+ ),
614
+ ],
615
+ );
616
+ }
617
+ }
618
+
619
+ enum _DeviceSurface { browser, android, desktop }
620
+
621
+ extension _DeviceSurfaceX on _DeviceSurface {
622
+ String get label => switch (this) {
623
+ _DeviceSurface.browser => 'Browser',
624
+ _DeviceSurface.android => 'Phone',
625
+ _DeviceSurface.desktop => 'Desktop',
626
+ };
627
+
628
+ String get helper => switch (this) {
629
+ _DeviceSurface.browser => 'Tap to click. Drag to scroll.',
630
+ _DeviceSurface.android => 'Tap to touch. Drag to swipe.',
631
+ _DeviceSurface.desktop =>
632
+ 'Tap to click. Drag to drag windows or selections.',
633
+ };
634
+
635
+ IconData get icon => switch (this) {
636
+ _DeviceSurface.browser => Icons.language_outlined,
637
+ _DeviceSurface.android => Icons.smartphone_outlined,
638
+ _DeviceSurface.desktop => Icons.computer_outlined,
639
+ };
640
+ }
641
+
642
+ class _DeviceSurfaceHeader extends StatelessWidget {
643
+ const _DeviceSurfaceHeader({
644
+ required this.surface,
645
+ required this.browserStatus,
646
+ required this.browserPageInfo,
647
+ required this.androidRuntime,
648
+ required this.androidOnline,
649
+ required this.desktopRuntime,
650
+ required this.desktopDevices,
651
+ required this.selectedDesktopDeviceId,
652
+ required this.browserExtensionPreferred,
653
+ required this.browserExtensionActive,
654
+ required this.browserFallbackLabel,
655
+ });
656
+
657
+ final _DeviceSurface surface;
658
+ final Map<String, dynamic> browserStatus;
659
+ final Map<String, dynamic> browserPageInfo;
660
+ final Map<String, dynamic> androidRuntime;
661
+ final bool androidOnline;
662
+ final Map<String, dynamic> desktopRuntime;
663
+ final List<Map<String, dynamic>> desktopDevices;
664
+ final String? selectedDesktopDeviceId;
665
+ final bool browserExtensionPreferred;
666
+ final bool browserExtensionActive;
667
+ final String browserFallbackLabel;
668
+
669
+ @override
670
+ Widget build(BuildContext context) {
671
+ return LayoutBuilder(
672
+ builder: (context, constraints) {
673
+ final compact = constraints.maxWidth < 720;
674
+ final androidStarting = androidRuntime['starting'] == true;
675
+ final androidVersion = _androidRuntimeVersionLabel(androidRuntime);
676
+ final selectedDesktop = desktopDevices
677
+ .where((device) => device['deviceId'] == selectedDesktopDeviceId)
678
+ .cast<Map<String, dynamic>?>()
679
+ .firstWhere((device) => device != null, orElse: () => null);
680
+ final desktopOnlineCount = desktopDevices
681
+ .where((device) => device['online'] == true)
682
+ .length;
683
+ final title = switch (surface) {
684
+ _DeviceSurface.browser =>
685
+ (browserPageInfo['title']?.toString().trim().isNotEmpty ?? false)
686
+ ? browserPageInfo['title'].toString()
687
+ : 'Live Browser',
688
+ _DeviceSurface.android => 'Android Phone',
689
+ _DeviceSurface.desktop =>
690
+ selectedDesktop?['label']?.toString().trim().isNotEmpty == true
691
+ ? selectedDesktop!['label'].toString()
692
+ : 'Desktop Companion',
693
+ };
694
+ final subtitle = switch (surface) {
695
+ _DeviceSurface.browser =>
696
+ browserExtensionPreferred && !browserExtensionActive
697
+ ? 'No extension device is active. Using the $browserFallbackLabel.'
698
+ : (browserPageInfo['url']?.toString() ??
699
+ 'Ready for navigation'),
700
+ _DeviceSurface.android =>
701
+ androidOnline
702
+ ? androidVersion == null
703
+ ? 'Tap and swipe directly on the preview.'
704
+ : '$androidVersion · Tap and swipe directly on the preview.'
705
+ : androidStarting
706
+ ? (androidRuntime['startupPhase']
707
+ ?.toString()
708
+ .trim()
709
+ .isNotEmpty ??
710
+ false)
711
+ ? androidRuntime['startupPhase'].toString()
712
+ : 'Starting the phone. This can take a little while.'
713
+ : (androidRuntime['lastLogLine']
714
+ ?.toString()
715
+ .trim()
716
+ .isNotEmpty ??
717
+ false)
718
+ ? androidRuntime['lastLogLine'].toString()
719
+ : androidVersion != null
720
+ ? '$androidVersion selected. Phone is offline.'
721
+ : 'Phone is offline. Open or start it from below.',
722
+ _DeviceSurface.desktop =>
723
+ selectedDesktop == null
724
+ ? desktopOnlineCount > 1
725
+ ? 'Multiple desktop companions are online. Pick the machine you want to control.'
726
+ : desktopOnlineCount == 1
727
+ ? 'One desktop companion is online. Open the surface to fetch the latest frame.'
728
+ : 'No desktop companion is online. Enable Companion Mode on a signed-in desktop app.'
729
+ : '${selectedDesktop['platform'] ?? 'desktop'} · ${selectedDesktop['hostname'] ?? 'unknown host'}',
730
+ };
731
+ final statusLabel = surface == _DeviceSurface.browser
732
+ ? browserExtensionPreferred && !browserExtensionActive
733
+ ? 'Fallback'
734
+ : browserExtensionActive
735
+ ? 'Extension'
736
+ : (browserStatus['launched'] == true ? 'Live' : 'Sleeping')
737
+ : surface == _DeviceSurface.desktop
738
+ ? selectedDesktop == null
739
+ ? (desktopOnlineCount > 0 ? 'Select Device' : 'Offline')
740
+ : (selectedDesktop['paused'] == true
741
+ ? 'Paused'
742
+ : (selectedDesktop['online'] == true
743
+ ? 'Live'
744
+ : 'Offline'))
745
+ : (androidOnline
746
+ ? 'Live'
747
+ : androidStarting
748
+ ? 'Starting'
749
+ : 'Offline');
750
+ final statusColor = surface == _DeviceSurface.browser
751
+ ? (browserStatus['launched'] == true ? _success : _warning)
752
+ : surface == _DeviceSurface.desktop
753
+ ? (selectedDesktop?['paused'] == true
754
+ ? _warning
755
+ : (selectedDesktop?['online'] == true ? _success : _warning))
756
+ : (androidOnline
757
+ ? _success
758
+ : (androidStarting ? _accent : _warning));
759
+
760
+ final textColumn = Expanded(
761
+ child: Column(
762
+ crossAxisAlignment: CrossAxisAlignment.start,
763
+ children: <Widget>[
764
+ Text(
765
+ title,
766
+ maxLines: 1,
767
+ overflow: TextOverflow.ellipsis,
768
+ style: TextStyle(fontSize: 21, fontWeight: FontWeight.w800),
769
+ ),
770
+ const SizedBox(height: 4),
771
+ Text(
772
+ subtitle,
773
+ maxLines: 2,
774
+ overflow: TextOverflow.ellipsis,
775
+ style: TextStyle(color: _textSecondary, height: 1.4),
776
+ ),
777
+ ],
778
+ ),
779
+ );
780
+
781
+ final statusChip = Flexible(
782
+ child: Align(
783
+ alignment: compact ? Alignment.centerLeft : Alignment.centerRight,
784
+ child: _DotStatus(label: statusLabel, color: statusColor),
785
+ ),
786
+ );
787
+
788
+ if (compact) {
789
+ return Column(
790
+ crossAxisAlignment: CrossAxisAlignment.start,
791
+ children: <Widget>[
792
+ Row(
793
+ crossAxisAlignment: CrossAxisAlignment.start,
794
+ children: <Widget>[
795
+ Container(
796
+ width: 48,
797
+ height: 48,
798
+ decoration: BoxDecoration(
799
+ color: _accentMuted,
800
+ borderRadius: BorderRadius.circular(14),
801
+ ),
802
+ child: Icon(surface.icon, color: _textPrimary),
803
+ ),
804
+ const SizedBox(width: 14),
805
+ textColumn,
806
+ ],
807
+ ),
808
+ const SizedBox(height: 12),
809
+ statusChip,
810
+ ],
811
+ );
812
+ }
813
+
814
+ return Row(
815
+ children: <Widget>[
816
+ Container(
817
+ width: 48,
818
+ height: 48,
819
+ decoration: BoxDecoration(
820
+ color: _accentMuted,
821
+ borderRadius: BorderRadius.circular(14),
822
+ ),
823
+ child: Icon(surface.icon, color: _textPrimary),
824
+ ),
825
+ const SizedBox(width: 14),
826
+ textColumn,
827
+ const SizedBox(width: 12),
828
+ statusChip,
829
+ ],
830
+ );
831
+ },
832
+ );
833
+ }
834
+ }
835
+
836
+ class _DeviceLaunchBar extends StatelessWidget {
837
+ const _DeviceLaunchBar({
838
+ required this.surface,
839
+ required this.controller,
840
+ required this.active,
841
+ required this.starting,
842
+ required this.busy,
843
+ required this.onSubmit,
844
+ required this.onSleep,
845
+ });
846
+
847
+ final _DeviceSurface surface;
848
+ final TextEditingController controller;
849
+ final bool active;
850
+ final bool starting;
851
+ final bool busy;
852
+ final Future<void> Function() onSubmit;
853
+ final Future<void> Function() onSleep;
854
+
855
+ @override
856
+ Widget build(BuildContext context) {
857
+ final hint = switch (surface) {
858
+ _DeviceSurface.browser => _browserUrlPlaceholder,
859
+ _DeviceSurface.android => _packageOrUrlHint,
860
+ _DeviceSurface.desktop =>
861
+ 'Launch an app on the selected desktop (optional)',
862
+ };
863
+ final buttonLabel = switch (surface) {
864
+ _DeviceSurface.browser => 'Open',
865
+ _DeviceSurface.android => starting ? 'Starting...' : 'Launch',
866
+ _DeviceSurface.desktop => 'Refresh',
867
+ };
868
+ final sleepLabel = switch (surface) {
869
+ _DeviceSurface.browser => 'Sleep Browser',
870
+ _DeviceSurface.android => 'Sleep Phone',
871
+ _DeviceSurface.desktop => 'Pause Desktop',
872
+ };
873
+ final narrow = MediaQuery.sizeOf(context).width < 720;
874
+
875
+ final input = TextField(
876
+ controller: controller,
877
+ onSubmitted: (_) => onSubmit(),
878
+ decoration: InputDecoration(
879
+ hintText: hint,
880
+ prefixIcon: Icon(
881
+ surface == _DeviceSurface.browser
882
+ ? Icons.travel_explore
883
+ : surface == _DeviceSurface.desktop
884
+ ? Icons.apps_outlined
885
+ : Icons.open_in_new,
886
+ ),
887
+ ),
888
+ );
889
+
890
+ final button = FilledButton.icon(
891
+ onPressed: busy || starting ? null : onSubmit,
892
+ icon: Icon(
893
+ surface == _DeviceSurface.browser
894
+ ? Icons.arrow_forward
895
+ : surface == _DeviceSurface.desktop
896
+ ? Icons.desktop_windows_outlined
897
+ : Icons.play_arrow,
898
+ ),
899
+ label: Text(buttonLabel),
900
+ );
901
+ final sleepButton = OutlinedButton.icon(
902
+ onPressed: busy || !active ? null : onSleep,
903
+ icon: Icon(Icons.bedtime_outlined),
904
+ label: Text(sleepLabel),
905
+ );
906
+
907
+ if (narrow) {
908
+ return Column(
909
+ children: <Widget>[
910
+ input,
911
+ const SizedBox(height: 10),
912
+ Row(
913
+ children: <Widget>[
914
+ Expanded(child: button),
915
+ const SizedBox(width: 10),
916
+ Expanded(child: sleepButton),
917
+ ],
918
+ ),
919
+ ],
920
+ );
921
+ }
922
+
923
+ return Row(
924
+ children: <Widget>[
925
+ Expanded(child: input),
926
+ const SizedBox(width: 10),
927
+ sleepButton,
928
+ const SizedBox(width: 10),
929
+ button,
930
+ ],
931
+ );
932
+ }
933
+ }
934
+
935
+ class _DeviceTypeDock extends StatelessWidget {
936
+ const _DeviceTypeDock({
937
+ required this.controller,
938
+ required this.busy,
939
+ required this.surface,
940
+ required this.onSubmit,
941
+ });
942
+
943
+ final TextEditingController controller;
944
+ final bool busy;
945
+ final _DeviceSurface surface;
946
+ final Future<void> Function() onSubmit;
947
+
948
+ @override
949
+ Widget build(BuildContext context) {
950
+ final hint = switch (surface) {
951
+ _DeviceSurface.browser => 'Type into the currently focused field',
952
+ _DeviceSurface.android => 'Type into the current phone field',
953
+ _DeviceSurface.desktop => 'Type into the focused desktop field',
954
+ };
955
+ final narrow = MediaQuery.sizeOf(context).width < 720;
956
+
957
+ final input = TextField(
958
+ controller: controller,
959
+ onSubmitted: (_) => onSubmit(),
960
+ decoration: InputDecoration(
961
+ hintText: hint,
962
+ prefixIcon: Icon(Icons.keyboard_outlined),
963
+ ),
964
+ );
965
+
966
+ final button = FilledButton.icon(
967
+ onPressed: busy ? null : onSubmit,
968
+ icon: Icon(Icons.send_rounded),
969
+ label: Text('Send'),
970
+ );
971
+
972
+ if (narrow) {
973
+ return Column(
974
+ children: <Widget>[
975
+ input,
976
+ const SizedBox(height: 10),
977
+ SizedBox(width: double.infinity, child: button),
978
+ ],
979
+ );
980
+ }
981
+
982
+ return Row(
983
+ children: <Widget>[
984
+ Expanded(child: input),
985
+ const SizedBox(width: 10),
986
+ button,
987
+ ],
988
+ );
989
+ }
990
+ }
991
+
992
+ class _DeviceQuickActions extends StatelessWidget {
993
+ const _DeviceQuickActions({
994
+ required this.surface,
995
+ required this.androidOnline,
996
+ required this.busy,
997
+ required this.onAction,
998
+ });
999
+
1000
+ final _DeviceSurface surface;
1001
+ final bool androidOnline;
1002
+ final bool busy;
1003
+ final Future<void> Function(String action) onAction;
1004
+
1005
+ @override
1006
+ Widget build(BuildContext context) {
1007
+ final actions = switch (surface) {
1008
+ _DeviceSurface.browser => const <MapEntry<String, IconData>>[
1009
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1010
+ MapEntry<String, IconData>('browser_refresh', Icons.replay_rounded),
1011
+ MapEntry<String, IconData>(
1012
+ 'browser_enter',
1013
+ Icons.keyboard_return_rounded,
1014
+ ),
1015
+ ],
1016
+ _DeviceSurface.desktop => const <MapEntry<String, IconData>>[
1017
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1018
+ MapEntry<String, IconData>(
1019
+ 'desktop_enter',
1020
+ Icons.keyboard_return_rounded,
1021
+ ),
1022
+ MapEntry<String, IconData>('desktop_escape', Icons.close_fullscreen),
1023
+ ],
1024
+ _DeviceSurface.android => const <MapEntry<String, IconData>>[
1025
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1026
+ ],
1027
+ };
1028
+
1029
+ return Wrap(
1030
+ spacing: 10,
1031
+ runSpacing: 10,
1032
+ children: actions.map((entry) {
1033
+ final disabled =
1034
+ busy ||
1035
+ (surface != _DeviceSurface.browser &&
1036
+ entry.key.startsWith('browser_')) ||
1037
+ (surface != _DeviceSurface.desktop &&
1038
+ entry.key.startsWith('desktop_')) ||
1039
+ (!androidOnline && entry.key.startsWith('android_'));
1040
+ return InkWell(
1041
+ onTap: disabled ? null : () => onAction(entry.key),
1042
+ borderRadius: BorderRadius.circular(14),
1043
+ child: Ink(
1044
+ padding: const EdgeInsets.symmetric(horizontal: 14, vertical: 10),
1045
+ decoration: BoxDecoration(
1046
+ color: disabled ? _bgSecondary : _bgTertiary,
1047
+ borderRadius: BorderRadius.circular(14),
1048
+ border: Border.all(color: _borderLight),
1049
+ ),
1050
+ child: Icon(
1051
+ entry.value,
1052
+ color: disabled ? _textMuted : _textPrimary,
1053
+ ),
1054
+ ),
1055
+ );
1056
+ }).toList(),
1057
+ );
1058
+ }
1059
+ }
1060
+
1061
+ class _AndroidNavDock extends StatelessWidget {
1062
+ const _AndroidNavDock({
1063
+ required this.busy,
1064
+ required this.androidOnline,
1065
+ required this.onAction,
1066
+ });
1067
+
1068
+ final bool busy;
1069
+ final bool androidOnline;
1070
+ final Future<void> Function(String action) onAction;
1071
+
1072
+ @override
1073
+ Widget build(BuildContext context) {
1074
+ const actions = <MapEntry<String, IconData>>[
1075
+ MapEntry<String, IconData>('surface_refresh', Icons.refresh_rounded),
1076
+ MapEntry<String, IconData>('android_back', Icons.arrow_back_rounded),
1077
+ MapEntry<String, IconData>('android_home', Icons.home_rounded),
1078
+ MapEntry<String, IconData>('android_recent', Icons.crop_square_rounded),
1079
+ ];
1080
+
1081
+ return Container(
1082
+ width: double.infinity,
1083
+ padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 10),
1084
+ decoration: BoxDecoration(
1085
+ color: _bgSecondary,
1086
+ borderRadius: BorderRadius.circular(18),
1087
+ border: Border.all(color: _borderLight),
1088
+ ),
1089
+ child: Row(
1090
+ mainAxisAlignment: MainAxisAlignment.spaceEvenly,
1091
+ children: actions.map((entry) {
1092
+ final disabled =
1093
+ busy || (!androidOnline && entry.key.startsWith('android_'));
1094
+ return IconButton.filledTonal(
1095
+ tooltip: entry.key.replaceAll('_', ' '),
1096
+ onPressed: disabled ? null : () => onAction(entry.key),
1097
+ icon: Icon(entry.value),
1098
+ );
1099
+ }).toList(),
1100
+ ),
1101
+ );
1102
+ }
1103
+ }
1104
+
1105
+ class _SurfaceSwitcher extends StatelessWidget {
1106
+ const _SurfaceSwitcher({
1107
+ required this.surface,
1108
+ required this.onSelect,
1109
+ });
1110
+
1111
+ final _DeviceSurface surface;
1112
+ final Future<void> Function(_DeviceSurface) onSelect;
1113
+
1114
+ @override
1115
+ Widget build(BuildContext context) {
1116
+ return Column(
1117
+ mainAxisSize: MainAxisSize.min,
1118
+ children: <Widget>[
1119
+ Wrap(
1120
+ spacing: 8,
1121
+ runSpacing: 8,
1122
+ alignment: WrapAlignment.center,
1123
+ children: _DeviceSurface.values.map((s) {
1124
+ final selected = s == surface;
1125
+ return ChoiceChip(
1126
+ avatar: Icon(
1127
+ s.icon,
1128
+ size: 16,
1129
+ color: selected ? _textPrimary : _textSecondary,
1130
+ ),
1131
+ label: Text(s.label),
1132
+ selected: selected,
1133
+ onSelected: (_) => onSelect(s),
1134
+ selectedColor: _accentMuted,
1135
+ backgroundColor: _bgCard,
1136
+ side: BorderSide(
1137
+ color: selected
1138
+ ? _accent.withValues(alpha: 0.42)
1139
+ : _borderLight,
1140
+ ),
1141
+ labelStyle: TextStyle(
1142
+ color: selected ? _textPrimary : _textSecondary,
1143
+ fontWeight:
1144
+ selected ? FontWeight.w700 : FontWeight.w500,
1145
+ ),
1146
+ );
1147
+ }).toList(),
1148
+ ),
1149
+ const SizedBox(height: 8),
1150
+ Text(
1151
+ surface.helper,
1152
+ textAlign: TextAlign.center,
1153
+ style: TextStyle(color: _textSecondary),
1154
+ ),
1155
+ ],
1156
+ );
1157
+ }
1158
+ }
1159
+
1160
+ class _InteractiveSurfacePreview extends StatefulWidget {
1161
+ const _InteractiveSurfacePreview({
1162
+ required this.surface,
1163
+ required this.controller,
1164
+ required this.screenshotPath,
1165
+ required this.busy,
1166
+ required this.wakingUp,
1167
+ required this.enabled,
1168
+ required this.connectRequired,
1169
+ required this.onTapPoint,
1170
+ required this.onSwipe,
1171
+ required this.onWakeRequested,
1172
+ });
1173
+
1174
+ final _DeviceSurface surface;
1175
+ final NeoAgentController controller;
1176
+ final String? screenshotPath;
1177
+ final bool busy;
1178
+ final bool wakingUp;
1179
+ final bool enabled;
1180
+ final bool connectRequired;
1181
+ final Future<void> Function(Offset point) onTapPoint;
1182
+ final Future<void> Function(Offset start, Offset end) onSwipe;
1183
+ final Future<void> Function() onWakeRequested;
1184
+
1185
+ @override
1186
+ State<_InteractiveSurfacePreview> createState() =>
1187
+ _InteractiveSurfacePreviewState();
1188
+ }
1189
+
1190
+ class _InteractiveSurfacePreviewState
1191
+ extends State<_InteractiveSurfacePreview> {
1192
+ ImageStream? _imageStream;
1193
+ ImageStreamListener? _imageListener;
1194
+ Size? _pixelSize;
1195
+ Uint8List? _imageBytes;
1196
+ Object? _imageError;
1197
+ Offset? _dragStart;
1198
+ Offset? _dragEnd;
1199
+
1200
+ @override
1201
+ void initState() {
1202
+ super.initState();
1203
+ unawaited(_loadImage());
1204
+ }
1205
+
1206
+ @override
1207
+ void didUpdateWidget(covariant _InteractiveSurfacePreview oldWidget) {
1208
+ super.didUpdateWidget(oldWidget);
1209
+ if (oldWidget.screenshotPath != widget.screenshotPath) {
1210
+ unawaited(_loadImage());
1211
+ }
1212
+ }
1213
+
1214
+ @override
1215
+ void dispose() {
1216
+ _detachImageListener();
1217
+ super.dispose();
1218
+ }
1219
+
1220
+ void _detachImageListener() {
1221
+ if (_imageStream != null && _imageListener != null) {
1222
+ _imageStream!.removeListener(_imageListener!);
1223
+ }
1224
+ _imageStream = null;
1225
+ _imageListener = null;
1226
+ }
1227
+
1228
+ Future<void> _loadImage() async {
1229
+ _detachImageListener();
1230
+ final path = widget.screenshotPath;
1231
+ if (path == null || path.isEmpty) {
1232
+ if (!mounted) {
1233
+ return;
1234
+ }
1235
+ setState(() {
1236
+ _pixelSize = null;
1237
+ _imageBytes = null;
1238
+ _imageError = null;
1239
+ });
1240
+ return;
1241
+ }
1242
+ try {
1243
+ final bytes = await widget.controller.fetchRuntimeAssetBytes(path);
1244
+ if (!mounted || widget.screenshotPath != path) {
1245
+ return;
1246
+ }
1247
+ setState(() {
1248
+ _imageBytes = bytes;
1249
+ _imageError = null;
1250
+ _pixelSize = null;
1251
+ });
1252
+ final provider = MemoryImage(bytes);
1253
+ final stream = provider.resolve(const ImageConfiguration());
1254
+ final listener = ImageStreamListener((image, _) {
1255
+ if (!mounted || widget.screenshotPath != path) {
1256
+ return;
1257
+ }
1258
+ setState(() {
1259
+ _pixelSize = Size(
1260
+ image.image.width.toDouble(),
1261
+ image.image.height.toDouble(),
1262
+ );
1263
+ });
1264
+ });
1265
+ _imageStream = stream;
1266
+ _imageListener = listener;
1267
+ stream.addListener(listener);
1268
+ } catch (error) {
1269
+ if (!mounted || widget.screenshotPath != path) {
1270
+ return;
1271
+ }
1272
+ setState(() {
1273
+ _imageBytes = null;
1274
+ _pixelSize = null;
1275
+ _imageError = error;
1276
+ });
1277
+ }
1278
+ }
1279
+
1280
+ Offset? _mapToPixels(Offset localPosition, Size boxSize) {
1281
+ final pixelSize = _pixelSize;
1282
+ if (pixelSize == null) {
1283
+ return null;
1284
+ }
1285
+ final boxAspect = boxSize.width / boxSize.height;
1286
+ final imageAspect = pixelSize.width / pixelSize.height;
1287
+ late final double renderWidth;
1288
+ late final double renderHeight;
1289
+ late final double offsetX;
1290
+ late final double offsetY;
1291
+
1292
+ if (boxAspect > imageAspect) {
1293
+ renderHeight = boxSize.height;
1294
+ renderWidth = renderHeight * imageAspect;
1295
+ offsetX = (boxSize.width - renderWidth) / 2;
1296
+ offsetY = 0;
1297
+ } else {
1298
+ renderWidth = boxSize.width;
1299
+ renderHeight = renderWidth / imageAspect;
1300
+ offsetX = 0;
1301
+ offsetY = (boxSize.height - renderHeight) / 2;
1302
+ }
1303
+
1304
+ if (localPosition.dx < offsetX ||
1305
+ localPosition.dx > offsetX + renderWidth ||
1306
+ localPosition.dy < offsetY ||
1307
+ localPosition.dy > offsetY + renderHeight) {
1308
+ return null;
1309
+ }
1310
+
1311
+ return Offset(
1312
+ ((localPosition.dx - offsetX) / renderWidth) * pixelSize.width,
1313
+ ((localPosition.dy - offsetY) / renderHeight) * pixelSize.height,
1314
+ );
1315
+ }
1316
+
1317
+ @override
1318
+ Widget build(BuildContext context) {
1319
+ final path = widget.screenshotPath;
1320
+ final hasImage = path != null && path.isNotEmpty;
1321
+ final aspectRatio = switch (widget.surface) {
1322
+ _DeviceSurface.browser => 16 / 10,
1323
+ _DeviceSurface.android => 10 / 16,
1324
+ _DeviceSurface.desktop => 16 / 10,
1325
+ };
1326
+
1327
+ return Container(
1328
+ decoration: BoxDecoration(
1329
+ gradient: LinearGradient(
1330
+ begin: Alignment.topLeft,
1331
+ end: Alignment.bottomRight,
1332
+ colors: <Color>[_bgTertiary, _bgSecondary],
1333
+ ),
1334
+ borderRadius: BorderRadius.circular(24),
1335
+ border: Border.all(color: _borderLight),
1336
+ ),
1337
+ padding: const EdgeInsets.all(14),
1338
+ child: Column(
1339
+ children: <Widget>[
1340
+ ConstrainedBox(
1341
+ constraints: BoxConstraints(
1342
+ maxHeight: widget.surface == _DeviceSurface.android ? 640 : 560,
1343
+ ),
1344
+ child: AspectRatio(
1345
+ aspectRatio: aspectRatio,
1346
+ child: ClipRRect(
1347
+ borderRadius: BorderRadius.circular(20),
1348
+ child: LayoutBuilder(
1349
+ builder: (context, constraints) {
1350
+ final boxSize = Size(
1351
+ constraints.maxWidth,
1352
+ constraints.maxHeight,
1353
+ );
1354
+ if (!hasImage) {
1355
+ return _EmptySurfaceState(
1356
+ surface: widget.surface,
1357
+ enabled: widget.enabled,
1358
+ busy: widget.busy,
1359
+ isLoadingPreview: widget.wakingUp,
1360
+ connectRequired: widget.connectRequired,
1361
+ errorMessage: _imageError?.toString(),
1362
+ onPressed: widget.onWakeRequested,
1363
+ );
1364
+ }
1365
+ final imageBytes = _imageBytes;
1366
+ if (imageBytes == null) {
1367
+ return _EmptySurfaceState(
1368
+ surface: widget.surface,
1369
+ enabled: widget.enabled,
1370
+ busy: widget.busy,
1371
+ isLoadingPreview: _imageError == null,
1372
+ connectRequired: widget.connectRequired,
1373
+ errorMessage: _imageError?.toString(),
1374
+ onPressed: widget.onWakeRequested,
1375
+ );
1376
+ }
1377
+ return Semantics(
1378
+ button: true,
1379
+ label: 'Device surface preview — tap to interact, swipe to scroll',
1380
+ child: GestureDetector(
1381
+ onTapUp: widget.busy
1382
+ ? null
1383
+ : (details) async {
1384
+ final point = _mapToPixels(
1385
+ details.localPosition,
1386
+ boxSize,
1387
+ );
1388
+ if (point != null) {
1389
+ await widget.onTapPoint(point);
1390
+ }
1391
+ },
1392
+ onPanStart: widget.busy
1393
+ ? null
1394
+ : (details) {
1395
+ _dragStart = details.localPosition;
1396
+ _dragEnd = details.localPosition;
1397
+ },
1398
+ onPanUpdate: widget.busy
1399
+ ? null
1400
+ : (details) {
1401
+ _dragEnd = details.localPosition;
1402
+ },
1403
+ onPanEnd: widget.busy
1404
+ ? null
1405
+ : (_) async {
1406
+ final start = _dragStart;
1407
+ final end = _dragEnd;
1408
+ _dragStart = null;
1409
+ _dragEnd = null;
1410
+ if (start == null || end == null) {
1411
+ return;
1412
+ }
1413
+ if ((start - end).distance < 12) {
1414
+ return;
1415
+ }
1416
+ final mappedStart = _mapToPixels(start, boxSize);
1417
+ final mappedEnd = _mapToPixels(end, boxSize);
1418
+ if (mappedStart != null && mappedEnd != null) {
1419
+ await widget.onSwipe(mappedStart, mappedEnd);
1420
+ }
1421
+ },
1422
+ child: Stack(
1423
+ fit: StackFit.expand,
1424
+ children: <Widget>[
1425
+ Container(color: _bgSecondary),
1426
+ Image.memory(
1427
+ imageBytes,
1428
+ fit: BoxFit.contain,
1429
+ gaplessPlayback: true,
1430
+ ),
1431
+ Positioned(
1432
+ left: 12,
1433
+ right: 12,
1434
+ bottom: 12,
1435
+ child: Container(
1436
+ padding: const EdgeInsets.symmetric(
1437
+ horizontal: 12,
1438
+ vertical: 10,
1439
+ ),
1440
+ decoration: BoxDecoration(
1441
+ color: const Color(0xB205080D),
1442
+ borderRadius: BorderRadius.circular(14),
1443
+ border: Border.all(color: _borderLight),
1444
+ ),
1445
+ child: Text(
1446
+ widget.surface.helper,
1447
+ textAlign: TextAlign.center,
1448
+ style: TextStyle(color: _textPrimary),
1449
+ ),
1450
+ ),
1451
+ ),
1452
+ if (widget.busy)
1453
+ const Center(child: CircularProgressIndicator()),
1454
+ ],
1455
+ ),
1456
+ ),
1457
+ );
1458
+ },
1459
+ ),
1460
+ ),
1461
+ ),
1462
+ ),
1463
+ ],
1464
+ ),
1465
+ );
1466
+ }
1467
+ }
1468
+
1469
+ class _EmptySurfaceState extends StatelessWidget {
1470
+ const _EmptySurfaceState({
1471
+ required this.surface,
1472
+ required this.enabled,
1473
+ required this.busy,
1474
+ required this.isLoadingPreview,
1475
+ required this.connectRequired,
1476
+ this.errorMessage,
1477
+ required this.onPressed,
1478
+ });
1479
+
1480
+ final _DeviceSurface surface;
1481
+ final bool enabled;
1482
+ final bool busy;
1483
+ final bool isLoadingPreview;
1484
+ final bool connectRequired;
1485
+ final String? errorMessage;
1486
+ final Future<void> Function() onPressed;
1487
+
1488
+ @override
1489
+ Widget build(BuildContext context) {
1490
+ final label = switch (surface) {
1491
+ _DeviceSurface.browser =>
1492
+ connectRequired
1493
+ ? 'Open Settings'
1494
+ : (busy ? 'Opening Browser...' : 'Wake Browser'),
1495
+ _DeviceSurface.android =>
1496
+ busy
1497
+ ? 'Starting Phone...'
1498
+ : (enabled ? 'Refresh Phone' : 'Start Phone'),
1499
+ _DeviceSurface.desktop =>
1500
+ connectRequired
1501
+ ? 'Select Desktop'
1502
+ : (busy ? 'Refreshing Desktop...' : 'Refresh Desktop'),
1503
+ };
1504
+ final message = switch ((surface, busy, isLoadingPreview)) {
1505
+ (_DeviceSurface.browser, true, _) =>
1506
+ 'Opening the browser and downloading the first preview...',
1507
+ (_DeviceSurface.browser, false, true) =>
1508
+ 'Downloading the latest browser preview...',
1509
+ (_DeviceSurface.desktop, true, _) =>
1510
+ 'Refreshing the selected desktop companion...',
1511
+ (_DeviceSurface.desktop, false, true) =>
1512
+ 'Downloading the latest desktop preview...',
1513
+ (_DeviceSurface.android, true, _) =>
1514
+ 'Waking the phone and downloading the first preview. This can take a little while.',
1515
+ (_DeviceSurface.android, false, true) =>
1516
+ 'Downloading the latest phone preview...',
1517
+ _ =>
1518
+ surface == _DeviceSurface.browser
1519
+ ? connectRequired
1520
+ ? 'Chrome extension is not connected. Use Settings to download, load, and pair the extension on the remote machine.'
1521
+ : 'Browser is sleeping. Press Open to start it.'
1522
+ : surface == _DeviceSurface.desktop
1523
+ ? connectRequired
1524
+ ? 'Multiple desktop companions are online. Select a machine before sending clicks or keystrokes.'
1525
+ : (errorMessage != null && errorMessage!.trim().isNotEmpty)
1526
+ ? errorMessage!
1527
+ : 'No desktop frame is loaded yet. Refresh the selected machine to capture a frame.'
1528
+ : (errorMessage != null && errorMessage!.trim().isNotEmpty)
1529
+ ? errorMessage!
1530
+ : 'Phone is offline. Press Start Phone to boot it.',
1531
+ };
1532
+ return Container(
1533
+ color: _bgSecondary,
1534
+ alignment: Alignment.center,
1535
+ child: Column(
1536
+ mainAxisSize: MainAxisSize.min,
1537
+ children: <Widget>[
1538
+ if (busy || isLoadingPreview) ...<Widget>[
1539
+ const SizedBox(
1540
+ width: 38,
1541
+ height: 38,
1542
+ child: CircularProgressIndicator(strokeWidth: 3),
1543
+ ),
1544
+ const SizedBox(height: 14),
1545
+ ] else
1546
+ Icon(surface.icon, size: 46, color: _textSecondary),
1547
+ if (!(busy || isLoadingPreview)) const SizedBox(height: 12),
1548
+ Text(
1549
+ message,
1550
+ style: TextStyle(color: _textSecondary),
1551
+ textAlign: TextAlign.center,
1552
+ ),
1553
+ if (errorMessage != null &&
1554
+ surface == _DeviceSurface.browser &&
1555
+ errorMessage!.trim().isNotEmpty) ...<Widget>[
1556
+ const SizedBox(height: 10),
1557
+ ConstrainedBox(
1558
+ constraints: const BoxConstraints(maxWidth: 320),
1559
+ child: Text(
1560
+ errorMessage!,
1561
+ textAlign: TextAlign.center,
1562
+ style: TextStyle(color: _textMuted, fontSize: 12),
1563
+ ),
1564
+ ),
1565
+ ],
1566
+ const SizedBox(height: 16),
1567
+ FilledButton.icon(
1568
+ onPressed: busy ? null : onPressed,
1569
+ icon: Icon(Icons.play_arrow_rounded),
1570
+ label: Text(label),
1571
+ ),
1572
+ ],
1573
+ ),
1574
+ );
1575
+ }
1576
+ }
1577
+
1578
+
1579
+
1580
+
1581
+ class _DeviceFieldRow extends StatelessWidget {
1582
+ const _DeviceFieldRow({required this.children});
1583
+
1584
+ final List<Widget> children;
1585
+
1586
+ @override
1587
+ Widget build(BuildContext context) {
1588
+ if (children.isEmpty) {
1589
+ return const SizedBox.shrink();
1590
+ }
1591
+ final stacked = MediaQuery.sizeOf(context).width < 860;
1592
+ if (stacked) {
1593
+ return Column(
1594
+ children: children
1595
+ .map(
1596
+ (child) => Padding(
1597
+ padding: const EdgeInsets.only(bottom: 10),
1598
+ child: child,
1599
+ ),
1600
+ )
1601
+ .toList(),
1602
+ );
1603
+ }
1604
+ return Row(
1605
+ crossAxisAlignment: CrossAxisAlignment.start,
1606
+ children: <Widget>[
1607
+ for (var index = 0; index < children.length; index++) ...<Widget>[
1608
+ if (index > 0) const SizedBox(width: 12),
1609
+ Expanded(child: children[index]),
1610
+ ],
1611
+ ],
1612
+ );
1613
+ }
1614
+ }
1615
+
1616
+ class _DeviceField extends StatelessWidget {
1617
+ const _DeviceField({required this.label, required this.child});
1618
+
1619
+ final String label;
1620
+ final Widget child;
1621
+
1622
+ @override
1623
+ Widget build(BuildContext context) {
1624
+ return Column(
1625
+ crossAxisAlignment: CrossAxisAlignment.start,
1626
+ children: <Widget>[
1627
+ Text(
1628
+ label,
1629
+ style: TextStyle(
1630
+ color: _textSecondary,
1631
+ fontSize: 12,
1632
+ fontWeight: FontWeight.w700,
1633
+ ),
1634
+ ),
1635
+ const SizedBox(height: 8),
1636
+ child,
1637
+ ],
1638
+ );
1639
+ }
1640
+ }
1641
+
1642
+ class _RuntimePreview extends StatelessWidget {
1643
+ const _RuntimePreview({
1644
+ required this.title,
1645
+ required this.screenshotPath,
1646
+ required this.controller,
1647
+ });
1648
+
1649
+ final String title;
1650
+ final String? screenshotPath;
1651
+ final NeoAgentController controller;
1652
+
1653
+ @override
1654
+ Widget build(BuildContext context) {
1655
+ if (screenshotPath == null || screenshotPath!.isEmpty) {
1656
+ return _EmptyCard(
1657
+ title: title,
1658
+ subtitle:
1659
+ 'Run a screenshot-capable action to preview the live runtime.',
1660
+ );
1661
+ }
1662
+
1663
+ final uri = controller.resolveRuntimeAsset(screenshotPath!);
1664
+ return Card(
1665
+ color: _bgSecondary,
1666
+ child: Padding(
1667
+ padding: const EdgeInsets.all(14),
1668
+ child: Column(
1669
+ crossAxisAlignment: CrossAxisAlignment.start,
1670
+ children: <Widget>[
1671
+ Text(title, style: TextStyle(fontWeight: FontWeight.w700)),
1672
+ const SizedBox(height: 8),
1673
+ Text(
1674
+ screenshotPath!,
1675
+ style: TextStyle(color: _textSecondary, fontSize: 12),
1676
+ ),
1677
+ const SizedBox(height: 12),
1678
+ ClipRRect(
1679
+ borderRadius: BorderRadius.circular(12),
1680
+ child: Image.network(
1681
+ uri.toString(),
1682
+ headers: controller.authenticatedImageHeaders,
1683
+ fit: BoxFit.cover,
1684
+ errorBuilder: (context, _, __) {
1685
+ return Container(
1686
+ height: 220,
1687
+ color: _bgCard,
1688
+ alignment: Alignment.center,
1689
+ child: Text(
1690
+ 'Could not load preview image',
1691
+ style: TextStyle(color: _textSecondary),
1692
+ ),
1693
+ );
1694
+ },
1695
+ ),
1696
+ ),
1697
+ ],
1698
+ ),
1699
+ ),
1700
+ );
1701
+ }
1702
+ }
1703
+
1704
+ class _ResultBlock extends StatelessWidget {
1705
+ const _ResultBlock({required this.label, required this.value});
1706
+
1707
+ final String label;
1708
+ final String value;
1709
+
1710
+ @override
1711
+ Widget build(BuildContext context) {
1712
+ return Container(
1713
+ width: double.infinity,
1714
+ padding: const EdgeInsets.all(14),
1715
+ decoration: BoxDecoration(
1716
+ color: _bgSecondary,
1717
+ borderRadius: BorderRadius.circular(12),
1718
+ border: Border.all(color: _border),
1719
+ ),
1720
+ child: Column(
1721
+ crossAxisAlignment: CrossAxisAlignment.start,
1722
+ children: <Widget>[
1723
+ Text(label, style: TextStyle(fontWeight: FontWeight.w700)),
1724
+ const SizedBox(height: 10),
1725
+ SelectableText(
1726
+ value,
1727
+ style: TextStyle(
1728
+ fontFamily: GoogleFonts.jetBrainsMono().fontFamily,
1729
+ fontSize: 12,
1730
+ color: _textSecondary,
1731
+ height: 1.5,
1732
+ ),
1733
+ ),
1734
+ ],
1735
+ ),
1736
+ );
1737
+ }
1738
+ }