neoagent 2.3.1-beta.0 → 2.3.1-beta.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (406) hide show
  1. package/.env.example +63 -5
  2. package/README.md +16 -5
  3. package/com.neoagent.plist +1 -1
  4. package/docs/automation.md +82 -41
  5. package/docs/capabilities.md +85 -89
  6. package/docs/configuration.md +89 -87
  7. package/docs/getting-started.md +46 -27
  8. package/docs/hardware.md +1 -1
  9. package/docs/index.md +24 -30
  10. package/docs/integrations.md +46 -60
  11. package/docs/migration.md +41 -146
  12. package/docs/operations.md +23 -20
  13. package/docs/skills.md +20 -20
  14. package/docs/why-neoagent.md +15 -13
  15. package/extensions/chrome-browser/background.mjs +10 -2
  16. package/flutter_app/.metadata +42 -0
  17. package/flutter_app/DESIGN_SYSTEM.md +166 -0
  18. package/flutter_app/README.md +21 -0
  19. package/flutter_app/analysis_options.yaml +32 -0
  20. package/flutter_app/android/app/build.gradle.kts +111 -0
  21. package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
  22. package/flutter_app/android/app/src/main/AndroidManifest.xml +185 -0
  23. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +888 -0
  24. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/auto/NeoAgentCarAppService.kt +114 -0
  25. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
  26. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
  27. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
  28. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
  29. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
  30. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
  31. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
  32. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
  33. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
  34. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
  35. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnection.kt +118 -0
  36. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnectionService.kt +86 -0
  37. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
  38. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
  39. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
  40. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
  41. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
  42. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
  43. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
  44. package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
  45. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
  46. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
  47. package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
  48. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
  49. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
  50. package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
  51. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  52. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  53. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  54. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  55. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  56. package/flutter_app/android/app/src/main/res/values/arrays.xml +8 -0
  57. package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
  58. package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
  59. package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
  60. package/flutter_app/android/app/src/main/res/xml/automotive_app_desc.xml +4 -0
  61. package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
  62. package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
  63. package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
  64. package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
  65. package/flutter_app/android/build.gradle.kts +24 -0
  66. package/flutter_app/android/ci-release.keystore +0 -0
  67. package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
  68. package/flutter_app/android/gradle.properties +3 -0
  69. package/flutter_app/android/key.properties +4 -0
  70. package/flutter_app/android/settings.gradle.kts +26 -0
  71. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  72. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  73. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  74. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  75. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  76. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  77. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  78. package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
  79. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  80. package/flutter_app/lib/features/location/location_service.dart +117 -0
  81. package/flutter_app/lib/features/notifications/notification_interceptor.dart +73 -0
  82. package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +571 -0
  83. package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +275 -0
  84. package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +366 -0
  85. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +57 -0
  86. package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +218 -0
  87. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +46 -0
  88. package/flutter_app/lib/main.dart +107 -0
  89. package/flutter_app/lib/main_account_settings.dart +1340 -0
  90. package/flutter_app/lib/main_admin.dart +900 -0
  91. package/flutter_app/lib/main_app_shell.dart +2162 -0
  92. package/flutter_app/lib/main_chat.dart +3902 -0
  93. package/flutter_app/lib/main_controller.dart +7229 -0
  94. package/flutter_app/lib/main_devices.dart +1738 -0
  95. package/flutter_app/lib/main_integrations.dart +917 -0
  96. package/flutter_app/lib/main_launcher.dart +959 -0
  97. package/flutter_app/lib/main_launcher_entry.dart +5 -0
  98. package/flutter_app/lib/main_models.dart +3807 -0
  99. package/flutter_app/lib/main_navigation.dart +206 -0
  100. package/flutter_app/lib/main_operations.dart +5249 -0
  101. package/flutter_app/lib/main_recordings.dart +920 -0
  102. package/flutter_app/lib/main_runtime.dart +869 -0
  103. package/flutter_app/lib/main_settings.dart +2288 -0
  104. package/flutter_app/lib/main_shared.dart +3393 -0
  105. package/flutter_app/lib/main_spacing.dart +33 -0
  106. package/flutter_app/lib/main_theme.dart +276 -0
  107. package/flutter_app/lib/main_unified.dart +388 -0
  108. package/flutter_app/lib/main_voice_assistant.dart +957 -0
  109. package/flutter_app/lib/src/analytics_service.dart +294 -0
  110. package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
  111. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
  112. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
  113. package/flutter_app/lib/src/android_app_installer.dart +22 -0
  114. package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
  115. package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
  116. package/flutter_app/lib/src/android_auto_bridge.dart +59 -0
  117. package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
  118. package/flutter_app/lib/src/app_launch_bridge.dart +58 -0
  119. package/flutter_app/lib/src/app_release_updater.dart +511 -0
  120. package/flutter_app/lib/src/backend_client.dart +1870 -0
  121. package/flutter_app/lib/src/desktop_companion.dart +2 -0
  122. package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
  123. package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
  124. package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
  125. package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
  126. package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
  127. package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
  128. package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
  129. package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
  130. package/flutter_app/lib/src/health_bridge.dart +136 -0
  131. package/flutter_app/lib/src/live_voice_capture.dart +85 -0
  132. package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
  133. package/flutter_app/lib/src/network/app_http_client.dart +53 -0
  134. package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
  135. package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
  136. package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
  137. package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
  138. package/flutter_app/lib/src/oauth_launcher.dart +33 -0
  139. package/flutter_app/lib/src/oauth_launcher_io.dart +95 -0
  140. package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
  141. package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
  142. package/flutter_app/lib/src/recording_bridge.dart +232 -0
  143. package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
  144. package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
  145. package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
  146. package/flutter_app/lib/src/recording_payloads.dart +86 -0
  147. package/flutter_app/lib/src/theme/palette.dart +81 -0
  148. package/flutter_app/lib/src/web_app_update_monitor.dart +17 -0
  149. package/flutter_app/lib/src/web_app_update_monitor_stub.dart +24 -0
  150. package/flutter_app/lib/src/web_app_update_monitor_web.dart +123 -0
  151. package/flutter_app/lib/src/widget_bridge.dart +49 -0
  152. package/flutter_app/linux/CMakeLists.txt +128 -0
  153. package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
  154. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +47 -0
  155. package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
  156. package/flutter_app/linux/flutter/generated_plugins.cmake +32 -0
  157. package/flutter_app/linux/runner/CMakeLists.txt +26 -0
  158. package/flutter_app/linux/runner/main.cc +6 -0
  159. package/flutter_app/linux/runner/my_application.cc +144 -0
  160. package/flutter_app/linux/runner/my_application.h +18 -0
  161. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  162. package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
  163. package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
  164. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +48 -0
  165. package/flutter_app/macos/Podfile +42 -0
  166. package/flutter_app/macos/Podfile.lock +87 -0
  167. package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
  168. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
  169. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  170. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  171. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  172. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  173. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  174. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  175. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  176. package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
  177. package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
  178. package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
  179. package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
  180. package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
  181. package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
  182. package/flutter_app/macos/Runner/Info.plist +36 -0
  183. package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
  184. package/flutter_app/macos/Runner/Release.entitlements +12 -0
  185. package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
  186. package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  187. package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
  188. package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
  189. package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  190. package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
  191. package/flutter_app/patch_strings.py +12 -0
  192. package/flutter_app/pubspec.lock +1256 -0
  193. package/flutter_app/pubspec.yaml +59 -0
  194. package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
  195. package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
  196. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
  197. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
  198. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
  199. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
  200. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
  201. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
  202. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
  203. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
  204. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
  205. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
  206. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
  207. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
  208. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
  209. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
  210. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
  211. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
  212. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
  213. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
  214. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
  215. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
  216. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
  217. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
  218. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
  219. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
  220. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
  221. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
  222. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
  223. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
  224. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
  225. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
  226. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
  227. package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
  228. package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
  229. package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
  230. package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
  231. package/flutter_app/tool/generate_desktop_branding.py +219 -0
  232. package/flutter_app/web/favicon.png +0 -0
  233. package/flutter_app/web/favicon.svg +12 -0
  234. package/flutter_app/web/icons/Icon-192.png +0 -0
  235. package/flutter_app/web/icons/Icon-512.png +0 -0
  236. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  237. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  238. package/flutter_app/web/index.html +39 -0
  239. package/flutter_app/web/manifest.json +35 -0
  240. package/flutter_app/windows/CMakeLists.txt +108 -0
  241. package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
  242. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +50 -0
  243. package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
  244. package/flutter_app/windows/flutter/generated_plugins.cmake +36 -0
  245. package/flutter_app/windows/runner/CMakeLists.txt +41 -0
  246. package/flutter_app/windows/runner/Runner.rc +121 -0
  247. package/flutter_app/windows/runner/flutter_window.cpp +533 -0
  248. package/flutter_app/windows/runner/flutter_window.h +37 -0
  249. package/flutter_app/windows/runner/main.cpp +53 -0
  250. package/flutter_app/windows/runner/resource.h +16 -0
  251. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  252. package/flutter_app/windows/runner/runner.exe.manifest +14 -0
  253. package/flutter_app/windows/runner/utils.cpp +65 -0
  254. package/flutter_app/windows/runner/utils.h +19 -0
  255. package/flutter_app/windows/runner/win32_window.cpp +299 -0
  256. package/flutter_app/windows/runner/win32_window.h +102 -0
  257. package/lib/install_helpers.js +31 -0
  258. package/lib/manager.js +465 -43
  259. package/package.json +11 -3
  260. package/runtime/paths.js +137 -0
  261. package/server/config/analytics.js +30 -0
  262. package/server/db/database.js +363 -21
  263. package/server/guest-agent.android.package.json +13 -0
  264. package/server/guest-agent.browser.package.json +14 -0
  265. package/server/guest_agent.js +98 -52
  266. package/server/http/middleware.js +55 -2
  267. package/server/http/routes.js +6 -1
  268. package/server/http/static.js +41 -1
  269. package/server/index.js +16 -16
  270. package/server/public/.last_build_id +1 -1
  271. package/server/public/assets/AssetManifest.bin +1 -1
  272. package/server/public/assets/AssetManifest.bin.json +1 -1
  273. package/server/public/assets/AssetManifest.json +1 -1
  274. package/server/public/assets/NOTICES +679 -129
  275. package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
  276. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  277. package/server/public/assets/packages/mixpanel_flutter/assets/mixpanel.js +3 -0
  278. package/server/public/canvaskit/wimp.wasm +0 -0
  279. package/server/public/flutter_bootstrap.js +2 -2
  280. package/server/public/main.dart.js +95866 -87333
  281. package/server/routes/account.js +26 -1
  282. package/server/routes/agents.js +6 -2
  283. package/server/routes/android.js +22 -6
  284. package/server/routes/auth.js +25 -5
  285. package/server/routes/browser.js +47 -13
  286. package/server/routes/integrations.js +119 -9
  287. package/server/routes/memory.js +179 -8
  288. package/server/routes/runtime.js +14 -0
  289. package/server/routes/screenHistory.js +46 -0
  290. package/server/routes/settings.js +102 -24
  291. package/server/routes/skills.js +12 -0
  292. package/server/routes/social_video.js +66 -0
  293. package/server/routes/tasks.js +22 -2
  294. package/server/routes/triggers.js +90 -0
  295. package/server/routes/wearable.js +67 -0
  296. package/server/routes/widgets.js +4 -4
  297. package/server/services/ai/capabilityHealth.js +23 -51
  298. package/server/services/ai/compaction.js +1 -1
  299. package/server/services/ai/deliverables/artifact_helpers.js +190 -0
  300. package/server/services/ai/deliverables/contracts.js +113 -0
  301. package/server/services/ai/deliverables/index.js +20 -0
  302. package/server/services/ai/deliverables/selector.js +94 -0
  303. package/server/services/ai/deliverables/validator.js +63 -0
  304. package/server/services/ai/deliverables/workflows.js +207 -0
  305. package/server/services/ai/engine.js +372 -27
  306. package/server/services/ai/hooks.js +127 -0
  307. package/server/services/ai/integrated_tools/index.js +98 -0
  308. package/server/services/ai/integrated_tools/remotion.js +373 -0
  309. package/server/services/ai/integrated_tools/shared.js +204 -0
  310. package/server/services/ai/integrated_tools/slidev.js +207 -0
  311. package/server/services/ai/loopPolicy.js +146 -0
  312. package/server/services/ai/models.js +36 -0
  313. package/server/services/ai/providers/githubCopilot.js +97 -0
  314. package/server/services/ai/providers/openai.js +2 -1
  315. package/server/services/ai/providers/openaiCodex.js +384 -0
  316. package/server/services/ai/recordingInsights.js +11 -13
  317. package/server/services/ai/runEvents.js +100 -0
  318. package/server/services/ai/settings.js +20 -0
  319. package/server/services/ai/systemPrompt.js +16 -2
  320. package/server/services/ai/taskAnalysis.js +2 -0
  321. package/server/services/ai/toolResult.js +20 -0
  322. package/server/services/ai/toolRunner.js +46 -6
  323. package/server/services/ai/toolSelector.js +54 -1
  324. package/server/services/ai/tools.js +360 -127
  325. package/server/services/android/android_bootstrap_worker.js +63 -0
  326. package/server/services/android/controller.js +442 -1760
  327. package/server/services/bootstrap_helpers.js +0 -34
  328. package/server/services/browser/controller.js +326 -52
  329. package/server/services/cli/executor.js +40 -1
  330. package/server/services/desktop/screenRecorder.js +182 -0
  331. package/server/services/integrations/env.js +10 -0
  332. package/server/services/integrations/figma/provider.js +1 -0
  333. package/server/services/integrations/github/common.js +106 -0
  334. package/server/services/integrations/github/provider.js +499 -0
  335. package/server/services/integrations/github/repos.js +1124 -0
  336. package/server/services/integrations/google/provider.js +1 -0
  337. package/server/services/integrations/manager.js +88 -12
  338. package/server/services/integrations/microsoft/provider.js +1 -0
  339. package/server/services/integrations/oauth_provider.js +25 -8
  340. package/server/services/integrations/provider_config_store.js +85 -0
  341. package/server/services/integrations/registry.js +8 -2
  342. package/server/services/integrations/spotify/provider.js +488 -0
  343. package/server/services/integrations/trello/provider.js +842 -0
  344. package/server/services/integrations/weather/provider.js +559 -0
  345. package/server/services/integrations/whatsapp/provider.js +6 -2
  346. package/server/services/manager.js +109 -154
  347. package/server/services/mcp/client.js +120 -23
  348. package/server/services/memory/llm_transfer.js +218 -0
  349. package/server/services/memory/manager.js +299 -35
  350. package/server/services/messaging/access_policy.js +10 -0
  351. package/server/services/messaging/automation.js +1 -1
  352. package/server/services/messaging/manager.js +93 -8
  353. package/server/services/messaging/meshtastic.js +277 -0
  354. package/server/services/messaging/meshtastic_env.js +40 -0
  355. package/server/services/messaging/meshtastic_protocol.js +530 -0
  356. package/server/services/messaging/meshtastic_tcp_transport.js +25 -0
  357. package/server/services/runtime/backends/local-vm.js +221 -158
  358. package/server/services/runtime/docker-vm-manager.js +392 -0
  359. package/server/services/runtime/guest_bootstrap.js +654 -0
  360. package/server/services/runtime/manager.js +85 -29
  361. package/server/services/runtime/settings.js +22 -31
  362. package/server/services/runtime/validation.js +16 -31
  363. package/server/services/skills/base_catalog.js +33 -0
  364. package/server/services/social_video/adapters/base.js +26 -0
  365. package/server/services/social_video/adapters/index.js +27 -0
  366. package/server/services/social_video/adapters/instagram.js +17 -0
  367. package/server/services/social_video/adapters/tiktok.js +17 -0
  368. package/server/services/social_video/adapters/x.js +17 -0
  369. package/server/services/social_video/adapters/youtube.js +17 -0
  370. package/server/services/social_video/captions.js +187 -0
  371. package/server/services/social_video/frame.js +42 -0
  372. package/server/services/social_video/index.js +7 -0
  373. package/server/services/social_video/metadata.js +63 -0
  374. package/server/services/social_video/result.js +63 -0
  375. package/server/services/social_video/service.js +720 -0
  376. package/server/services/social_video/url.js +83 -0
  377. package/server/services/tasks/adapters/index.js +2 -0
  378. package/server/services/tasks/adapters/manual.js +12 -0
  379. package/server/services/tasks/adapters/schedule.js +33 -5
  380. package/server/services/tasks/adapters/weather_event.js +84 -0
  381. package/server/services/tasks/integration_runtime.js +86 -1
  382. package/server/services/tasks/runtime.js +3 -3
  383. package/server/services/tasks/task_repository.js +3 -6
  384. package/server/services/voice/agentBridge.js +20 -4
  385. package/server/services/voice/message.js +3 -0
  386. package/server/services/voice/openaiClient.js +4 -1
  387. package/server/services/voice/openaiSpeech.js +6 -1
  388. package/server/services/voice/providers.js +144 -15
  389. package/server/services/voice/runtime.js +8 -8
  390. package/server/services/voice/runtimeManager.js +292 -20
  391. package/server/services/voice/turnRunner.js +29 -9
  392. package/server/services/wearable/firmware_manifest.js +353 -0
  393. package/server/services/wearable/gateway.js +350 -0
  394. package/server/services/wearable/protocol.js +45 -0
  395. package/server/services/wearable/service.js +240 -0
  396. package/server/services/websocket.js +3 -1
  397. package/server/services/widgets/focus_widget.js +137 -0
  398. package/server/services/widgets/service.js +182 -6
  399. package/server/services/workspace/manager.js +301 -0
  400. package/server/utils/deployment.js +16 -9
  401. package/server/utils/local_secrets.js +56 -0
  402. package/server/utils/logger.js +37 -9
  403. package/server/services/integrations/home_assistant/provider.js +0 -350
  404. package/server/services/messaging/access_policy.test.js +0 -228
  405. package/server/services/runtime/backends/host.js +0 -37
  406. package/server/services/runtime/qemu.js +0 -391
@@ -0,0 +1,533 @@
1
+ #include "flutter_window.h"
2
+
3
+ #include <flutter/encodable_value.h>
4
+
5
+ #include <gdiplus.h>
6
+ #include <shellapi.h>
7
+ #include <windows.h>
8
+
9
+ #include <cctype>
10
+ #include <cstring>
11
+ #include <cwchar>
12
+ #include <cmath>
13
+ #include <memory>
14
+ #include <optional>
15
+ #include <sstream>
16
+ #include <string>
17
+ #include <vector>
18
+
19
+ #include "flutter/generated_plugin_registrant.h"
20
+
21
+ namespace {
22
+
23
+ using flutter::EncodableList;
24
+ using flutter::EncodableMap;
25
+ using flutter::EncodableValue;
26
+
27
+ std::wstring Utf8ToWide(const std::string& value) {
28
+ if (value.empty()) {
29
+ return std::wstring();
30
+ }
31
+ const int length =
32
+ MultiByteToWideChar(CP_UTF8, 0, value.c_str(), -1, nullptr, 0);
33
+ if (length <= 0) {
34
+ return std::wstring();
35
+ }
36
+ std::wstring wide(static_cast<size_t>(length), L'\0');
37
+ MultiByteToWideChar(
38
+ CP_UTF8, 0, value.c_str(), -1, wide.data(), length);
39
+ wide.resize(wcslen(wide.c_str()));
40
+ return wide;
41
+ }
42
+
43
+ std::string WideToUtf8(const std::wstring& value) {
44
+ if (value.empty()) {
45
+ return std::string();
46
+ }
47
+ const int length =
48
+ WideCharToMultiByte(CP_UTF8, 0, value.c_str(), -1, nullptr, 0, nullptr, nullptr);
49
+ if (length <= 0) {
50
+ return std::string();
51
+ }
52
+ std::string utf8(static_cast<size_t>(length), '\0');
53
+ WideCharToMultiByte(
54
+ CP_UTF8, 0, value.c_str(), -1, utf8.data(), length, nullptr, nullptr);
55
+ utf8.resize(strlen(utf8.c_str()));
56
+ return utf8;
57
+ }
58
+
59
+ const EncodableValue* MapValue(const EncodableMap& map, const char* key) {
60
+ auto it = map.find(EncodableValue(key));
61
+ if (it == map.end()) {
62
+ return nullptr;
63
+ }
64
+ return &it->second;
65
+ }
66
+
67
+ bool GetInt(const EncodableMap& map, const char* key, int* out) {
68
+ const EncodableValue* value = MapValue(map, key);
69
+ if (value == nullptr) {
70
+ return false;
71
+ }
72
+ if (std::holds_alternative<int>(*value)) {
73
+ *out = std::get<int>(*value);
74
+ return true;
75
+ }
76
+ if (std::holds_alternative<int64_t>(*value)) {
77
+ *out = static_cast<int>(std::get<int64_t>(*value));
78
+ return true;
79
+ }
80
+ if (std::holds_alternative<double>(*value)) {
81
+ *out = static_cast<int>(std::lround(std::get<double>(*value)));
82
+ return true;
83
+ }
84
+ return false;
85
+ }
86
+
87
+ std::string GetString(const EncodableMap& map, const char* key,
88
+ const std::string& fallback = std::string()) {
89
+ const EncodableValue* value = MapValue(map, key);
90
+ if (value == nullptr) {
91
+ return fallback;
92
+ }
93
+ if (std::holds_alternative<std::string>(*value)) {
94
+ return std::get<std::string>(*value);
95
+ }
96
+ return fallback;
97
+ }
98
+
99
+ bool GetBool(const EncodableMap& map, const char* key, bool fallback = false) {
100
+ const EncodableValue* value = MapValue(map, key);
101
+ if (value == nullptr) {
102
+ return fallback;
103
+ }
104
+ if (std::holds_alternative<bool>(*value)) {
105
+ return std::get<bool>(*value);
106
+ }
107
+ return fallback;
108
+ }
109
+
110
+ struct DisplayInfo {
111
+ std::wstring id;
112
+ RECT rect;
113
+ bool primary;
114
+ std::wstring label;
115
+ };
116
+
117
+ BOOL CALLBACK CollectDisplays(HMONITOR monitor, HDC, LPRECT, LPARAM data) {
118
+ auto* displays = reinterpret_cast<std::vector<DisplayInfo>*>(data);
119
+ MONITORINFOEXW info;
120
+ info.cbSize = sizeof(MONITORINFOEXW);
121
+ if (!GetMonitorInfoW(monitor, &info)) {
122
+ return TRUE;
123
+ }
124
+
125
+ std::wstringstream id;
126
+ id << info.szDevice;
127
+ displays->push_back(DisplayInfo{
128
+ id.str(),
129
+ info.rcMonitor,
130
+ (info.dwFlags & MONITORINFOF_PRIMARY) != 0,
131
+ info.szDevice,
132
+ });
133
+ return TRUE;
134
+ }
135
+
136
+ std::vector<DisplayInfo> EnumerateDisplays() {
137
+ std::vector<DisplayInfo> displays;
138
+ EnumDisplayMonitors(nullptr, nullptr, CollectDisplays,
139
+ reinterpret_cast<LPARAM>(&displays));
140
+ return displays;
141
+ }
142
+
143
+ DisplayInfo ResolveDisplay(const std::string& requested_id) {
144
+ const auto displays = EnumerateDisplays();
145
+ if (!requested_id.empty()) {
146
+ const std::wstring requested_wide = Utf8ToWide(requested_id);
147
+ for (const auto& display : displays) {
148
+ if (display.id == requested_wide) {
149
+ return display;
150
+ }
151
+ }
152
+ }
153
+ for (const auto& display : displays) {
154
+ if (display.primary) {
155
+ return display;
156
+ }
157
+ }
158
+ return displays.empty()
159
+ ? DisplayInfo{L"primary", RECT{0, 0, GetSystemMetrics(SM_CXSCREEN),
160
+ GetSystemMetrics(SM_CYSCREEN)},
161
+ true, L"Primary Display"}
162
+ : displays.front();
163
+ }
164
+
165
+ CLSID PngEncoderClsid() {
166
+ UINT count = 0;
167
+ UINT size = 0;
168
+ Gdiplus::GetImageEncodersSize(&count, &size);
169
+ std::vector<BYTE> buffer(size);
170
+ auto* codecs = reinterpret_cast<Gdiplus::ImageCodecInfo*>(buffer.data());
171
+ Gdiplus::GetImageEncoders(count, size, codecs);
172
+ for (UINT index = 0; index < count; ++index) {
173
+ if (wcscmp(codecs[index].MimeType, L"image/png") == 0) {
174
+ return codecs[index].Clsid;
175
+ }
176
+ }
177
+ return CLSID{};
178
+ }
179
+
180
+ std::vector<uint8_t> CaptureDisplayPng(const DisplayInfo& display) {
181
+ const int width = display.rect.right - display.rect.left;
182
+ const int height = display.rect.bottom - display.rect.top;
183
+ HDC screen_dc = GetDC(nullptr);
184
+ HDC memory_dc = CreateCompatibleDC(screen_dc);
185
+ HBITMAP bitmap = CreateCompatibleBitmap(screen_dc, width, height);
186
+ HGDIOBJ old_object = SelectObject(memory_dc, bitmap);
187
+
188
+ BitBlt(memory_dc, 0, 0, width, height, screen_dc, display.rect.left,
189
+ display.rect.top, SRCCOPY | CAPTUREBLT);
190
+
191
+ Gdiplus::Bitmap image(bitmap, nullptr);
192
+ IStream* stream = nullptr;
193
+ CreateStreamOnHGlobal(nullptr, TRUE, &stream);
194
+ const CLSID encoder = PngEncoderClsid();
195
+ image.Save(stream, &encoder, nullptr);
196
+
197
+ STATSTG stats;
198
+ stream->Stat(&stats, STATFLAG_NONAME);
199
+ std::vector<uint8_t> bytes(static_cast<size_t>(stats.cbSize.QuadPart));
200
+ LARGE_INTEGER seek_start{};
201
+ ULARGE_INTEGER new_position{};
202
+ stream->Seek(seek_start, STREAM_SEEK_SET, &new_position);
203
+ ULONG read = 0;
204
+ stream->Read(bytes.data(), static_cast<ULONG>(bytes.size()), &read);
205
+
206
+ stream->Release();
207
+ SelectObject(memory_dc, old_object);
208
+ DeleteObject(bitmap);
209
+ DeleteDC(memory_dc);
210
+ ReleaseDC(nullptr, screen_dc);
211
+ return bytes;
212
+ }
213
+
214
+ EncodableList DisplaysToEncodable() {
215
+ EncodableList list;
216
+ for (const auto& display : EnumerateDisplays()) {
217
+ const int width = display.rect.right - display.rect.left;
218
+ const int height = display.rect.bottom - display.rect.top;
219
+ EncodableMap item;
220
+ item[EncodableValue("id")] = EncodableValue(WideToUtf8(display.id));
221
+ item[EncodableValue("label")] = EncodableValue(WideToUtf8(display.label));
222
+ item[EncodableValue("width")] = EncodableValue(width);
223
+ item[EncodableValue("height")] = EncodableValue(height);
224
+ item[EncodableValue("scaleFactor")] = EncodableValue(1.0);
225
+ item[EncodableValue("primary")] = EncodableValue(display.primary);
226
+ list.emplace_back(item);
227
+ }
228
+ return list;
229
+ }
230
+
231
+ std::string ForegroundWindowTitle() {
232
+ HWND window = GetForegroundWindow();
233
+ if (window == nullptr) {
234
+ return std::string();
235
+ }
236
+ wchar_t buffer[512];
237
+ const int length = GetWindowTextW(window, buffer, 512);
238
+ if (length <= 0) {
239
+ return std::string();
240
+ }
241
+ return WideToUtf8(std::wstring(buffer, buffer + length));
242
+ }
243
+
244
+ void SendMouseButton(DWORD flag) {
245
+ INPUT input{};
246
+ input.type = INPUT_MOUSE;
247
+ input.mi.dwFlags = flag;
248
+ SendInput(1, &input, sizeof(INPUT));
249
+ }
250
+
251
+ WORD VirtualKeyForString(const std::string& key) {
252
+ const std::string lowered = [&]() {
253
+ std::string value = key;
254
+ for (auto& ch : value) {
255
+ ch = static_cast<char>(tolower(ch));
256
+ }
257
+ return value;
258
+ }();
259
+ if (lowered == "enter" || lowered == "return") return VK_RETURN;
260
+ if (lowered == "tab") return VK_TAB;
261
+ if (lowered == "space") return VK_SPACE;
262
+ if (lowered == "escape" || lowered == "esc") return VK_ESCAPE;
263
+ if (lowered == "backspace" || lowered == "delete") return VK_BACK;
264
+ if (lowered == "left") return VK_LEFT;
265
+ if (lowered == "right") return VK_RIGHT;
266
+ if (lowered == "up") return VK_UP;
267
+ if (lowered == "down") return VK_DOWN;
268
+ return 0;
269
+ }
270
+
271
+ void SendVirtualKey(WORD key_code) {
272
+ INPUT inputs[2]{};
273
+ inputs[0].type = INPUT_KEYBOARD;
274
+ inputs[0].ki.wVk = key_code;
275
+ inputs[1].type = INPUT_KEYBOARD;
276
+ inputs[1].ki.wVk = key_code;
277
+ inputs[1].ki.dwFlags = KEYEVENTF_KEYUP;
278
+ SendInput(2, inputs, sizeof(INPUT));
279
+ }
280
+
281
+ void SendUnicodeText(const std::wstring& text) {
282
+ for (const wchar_t ch : text) {
283
+ INPUT inputs[2]{};
284
+ inputs[0].type = INPUT_KEYBOARD;
285
+ inputs[0].ki.dwFlags = KEYEVENTF_UNICODE;
286
+ inputs[0].ki.wScan = ch;
287
+ inputs[1].type = INPUT_KEYBOARD;
288
+ inputs[1].ki.dwFlags = KEYEVENTF_UNICODE | KEYEVENTF_KEYUP;
289
+ inputs[1].ki.wScan = ch;
290
+ SendInput(2, inputs, sizeof(INPUT));
291
+ }
292
+ }
293
+
294
+ } // namespace
295
+
296
+ FlutterWindow::FlutterWindow(const flutter::DartProject& project)
297
+ : project_(project) {}
298
+
299
+ FlutterWindow::~FlutterWindow() {}
300
+
301
+ bool FlutterWindow::OnCreate() {
302
+ if (!Win32Window::OnCreate()) {
303
+ return false;
304
+ }
305
+
306
+ Gdiplus::GdiplusStartupInput gdiplus_startup_input;
307
+ ULONG_PTR gdiplus_token = 0;
308
+ Gdiplus::GdiplusStartup(&gdiplus_token, &gdiplus_startup_input, nullptr);
309
+
310
+ RECT frame = GetClientArea();
311
+
312
+ flutter_controller_ = std::make_unique<flutter::FlutterViewController>(
313
+ frame.right - frame.left, frame.bottom - frame.top, project_);
314
+ if (!flutter_controller_->engine() || !flutter_controller_->view()) {
315
+ return false;
316
+ }
317
+ RegisterPlugins(flutter_controller_->engine());
318
+ SetChildContent(flutter_controller_->view()->GetNativeWindow());
319
+
320
+ desktop_channel_ =
321
+ std::make_unique<flutter::MethodChannel<EncodableValue>>(
322
+ flutter_controller_->engine()->messenger(),
323
+ "neoagent/desktop_companion_native",
324
+ &flutter::StandardMethodCodec::GetInstance());
325
+
326
+ desktop_channel_->SetMethodCallHandler(
327
+ [](const flutter::MethodCall<EncodableValue>& call,
328
+ std::unique_ptr<flutter::MethodResult<EncodableValue>> result) {
329
+ const auto* arguments = std::get_if<EncodableMap>(call.arguments());
330
+
331
+ if (call.method_name() == "getStatus") {
332
+ EncodableMap permissions;
333
+ permissions[EncodableValue("screenCapture")] = EncodableValue("available");
334
+ permissions[EncodableValue("inputControl")] = EncodableValue("available");
335
+ permissions[EncodableValue("accessibility")] = EncodableValue("available");
336
+
337
+ EncodableMap status;
338
+ status[EncodableValue("permissions")] = EncodableValue(permissions);
339
+ status[EncodableValue("displays")] = EncodableValue(DisplaysToEncodable());
340
+ status[EncodableValue("activeDisplayId")] =
341
+ EncodableValue(WideToUtf8(ResolveDisplay("").id));
342
+ const std::string window_title = ForegroundWindowTitle();
343
+ if (!window_title.empty()) {
344
+ status[EncodableValue("frontmostWindowTitle")] =
345
+ EncodableValue(window_title);
346
+ }
347
+ result->Success(EncodableValue(status));
348
+ return;
349
+ }
350
+
351
+ if (call.method_name() == "listDisplays") {
352
+ result->Success(EncodableValue(DisplaysToEncodable()));
353
+ return;
354
+ }
355
+
356
+ if (call.method_name() == "captureFrame") {
357
+ const std::string display_id =
358
+ arguments == nullptr ? std::string() : GetString(*arguments, "displayId");
359
+ const DisplayInfo display = ResolveDisplay(display_id);
360
+ const auto bytes = CaptureDisplayPng(display);
361
+
362
+ EncodableMap payload;
363
+ payload[EncodableValue("bytes")] =
364
+ EncodableValue(std::vector<uint8_t>(bytes.begin(), bytes.end()));
365
+ payload[EncodableValue("mimeType")] = EncodableValue("image/png");
366
+ payload[EncodableValue("width")] =
367
+ EncodableValue(display.rect.right - display.rect.left);
368
+ payload[EncodableValue("height")] =
369
+ EncodableValue(display.rect.bottom - display.rect.top);
370
+ payload[EncodableValue("displayId")] =
371
+ EncodableValue(WideToUtf8(display.id));
372
+ payload[EncodableValue("displays")] =
373
+ EncodableValue(DisplaysToEncodable());
374
+ const std::string window_title = ForegroundWindowTitle();
375
+ if (!window_title.empty()) {
376
+ payload[EncodableValue("frontmostWindowTitle")] =
377
+ EncodableValue(window_title);
378
+ }
379
+ result->Success(EncodableValue(payload));
380
+ return;
381
+ }
382
+
383
+ if (call.method_name() == "click") {
384
+ if (arguments == nullptr) {
385
+ result->Error("invalid_arguments", "Missing click payload.");
386
+ return;
387
+ }
388
+ int x = 0;
389
+ int y = 0;
390
+ GetInt(*arguments, "x", &x);
391
+ GetInt(*arguments, "y", &y);
392
+ const std::string button = GetString(*arguments, "button", "left");
393
+ SetCursorPos(x, y);
394
+ if (button == "right") {
395
+ SendMouseButton(MOUSEEVENTF_RIGHTDOWN);
396
+ SendMouseButton(MOUSEEVENTF_RIGHTUP);
397
+ } else if (button == "middle") {
398
+ SendMouseButton(MOUSEEVENTF_MIDDLEDOWN);
399
+ SendMouseButton(MOUSEEVENTF_MIDDLEUP);
400
+ } else {
401
+ SendMouseButton(MOUSEEVENTF_LEFTDOWN);
402
+ SendMouseButton(MOUSEEVENTF_LEFTUP);
403
+ }
404
+ result->Success(EncodableValue());
405
+ return;
406
+ }
407
+
408
+ if (call.method_name() == "drag") {
409
+ if (arguments == nullptr) {
410
+ result->Error("invalid_arguments", "Missing drag payload.");
411
+ return;
412
+ }
413
+ int x1 = 0, y1 = 0, x2 = 0, y2 = 0, duration_ms = 280;
414
+ GetInt(*arguments, "x1", &x1);
415
+ GetInt(*arguments, "y1", &y1);
416
+ GetInt(*arguments, "x2", &x2);
417
+ GetInt(*arguments, "y2", &y2);
418
+ GetInt(*arguments, "durationMs", &duration_ms);
419
+ const int steps = std::max(4, std::min(90, duration_ms / 16));
420
+ SetCursorPos(x1, y1);
421
+ SendMouseButton(MOUSEEVENTF_LEFTDOWN);
422
+ for (int step = 1; step <= steps; ++step) {
423
+ const double t = static_cast<double>(step) / steps;
424
+ const int nx = static_cast<int>(std::lround(x1 + ((x2 - x1) * t)));
425
+ const int ny = static_cast<int>(std::lround(y1 + ((y2 - y1) * t)));
426
+ SetCursorPos(nx, ny);
427
+ Sleep(std::max(1, duration_ms / std::max(1, steps)));
428
+ }
429
+ SendMouseButton(MOUSEEVENTF_LEFTUP);
430
+ result->Success(EncodableValue());
431
+ return;
432
+ }
433
+
434
+ if (call.method_name() == "scroll") {
435
+ if (arguments == nullptr) {
436
+ result->Error("invalid_arguments", "Missing scroll payload.");
437
+ return;
438
+ }
439
+ int delta_x = 0;
440
+ int delta_y = 0;
441
+ GetInt(*arguments, "deltaX", &delta_x);
442
+ GetInt(*arguments, "deltaY", &delta_y);
443
+ if (delta_y != 0) {
444
+ INPUT input{};
445
+ input.type = INPUT_MOUSE;
446
+ input.mi.dwFlags = MOUSEEVENTF_WHEEL;
447
+ input.mi.mouseData = static_cast<DWORD>(delta_y);
448
+ SendInput(1, &input, sizeof(INPUT));
449
+ }
450
+ if (delta_x != 0) {
451
+ INPUT input{};
452
+ input.type = INPUT_MOUSE;
453
+ input.mi.dwFlags = MOUSEEVENTF_HWHEEL;
454
+ input.mi.mouseData = static_cast<DWORD>(delta_x);
455
+ SendInput(1, &input, sizeof(INPUT));
456
+ }
457
+ result->Success(EncodableValue());
458
+ return;
459
+ }
460
+
461
+ if (call.method_name() == "typeText") {
462
+ if (arguments == nullptr) {
463
+ result->Error("invalid_arguments", "Missing text payload.");
464
+ return;
465
+ }
466
+ const std::wstring text = Utf8ToWide(GetString(*arguments, "text", ""));
467
+ const bool press_enter = GetBool(*arguments, "pressEnter", false);
468
+ if (!text.empty()) {
469
+ SendUnicodeText(text);
470
+ }
471
+ if (press_enter) {
472
+ SendVirtualKey(VK_RETURN);
473
+ }
474
+ result->Success(EncodableValue());
475
+ return;
476
+ }
477
+
478
+ if (call.method_name() == "pressKey") {
479
+ if (arguments == nullptr) {
480
+ result->Error("invalid_arguments", "Missing key payload.");
481
+ return;
482
+ }
483
+ const std::string key = GetString(*arguments, "key", "");
484
+ const WORD virtual_key = VirtualKeyForString(key);
485
+ if (virtual_key == 0) {
486
+ result->Error("unsupported_key", "Key is not supported.");
487
+ return;
488
+ }
489
+ SendVirtualKey(virtual_key);
490
+ result->Success(EncodableValue());
491
+ return;
492
+ }
493
+
494
+ result->NotImplemented();
495
+ });
496
+
497
+ flutter_controller_->engine()->SetNextFrameCallback([&]() { this->Show(); });
498
+ flutter_controller_->ForceRedraw();
499
+
500
+ return true;
501
+ }
502
+
503
+ void FlutterWindow::OnDestroy() {
504
+ if (flutter_controller_) {
505
+ flutter_controller_ = nullptr;
506
+ }
507
+ desktop_channel_.reset();
508
+ Win32Window::OnDestroy();
509
+ }
510
+
511
+ LRESULT FlutterWindow::MessageHandler(HWND hwnd, UINT const message,
512
+ WPARAM const wparam,
513
+ LPARAM const lparam) noexcept {
514
+ if (flutter_controller_) {
515
+ std::optional<LRESULT> result =
516
+ flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam,
517
+ lparam);
518
+ if (result) {
519
+ return *result;
520
+ }
521
+ }
522
+
523
+ switch (message) {
524
+ case WM_FONTCHANGE:
525
+ if (flutter_controller_ != nullptr &&
526
+ flutter_controller_->engine() != nullptr) {
527
+ flutter_controller_->engine()->ReloadSystemFonts();
528
+ }
529
+ break;
530
+ }
531
+
532
+ return Win32Window::MessageHandler(hwnd, message, wparam, lparam);
533
+ }
@@ -0,0 +1,37 @@
1
+ #ifndef RUNNER_FLUTTER_WINDOW_H_
2
+ #define RUNNER_FLUTTER_WINDOW_H_
3
+
4
+ #include <flutter/dart_project.h>
5
+ #include <flutter/flutter_view_controller.h>
6
+ #include <flutter/method_channel.h>
7
+ #include <flutter/standard_method_codec.h>
8
+
9
+ #include <memory>
10
+
11
+ #include "win32_window.h"
12
+
13
+ // A window that does nothing but host a Flutter view.
14
+ class FlutterWindow : public Win32Window {
15
+ public:
16
+ // Creates a new FlutterWindow hosting a Flutter view running |project|.
17
+ explicit FlutterWindow(const flutter::DartProject& project);
18
+ virtual ~FlutterWindow();
19
+
20
+ protected:
21
+ // Win32Window:
22
+ bool OnCreate() override;
23
+ void OnDestroy() override;
24
+ LRESULT MessageHandler(HWND window, UINT const message, WPARAM const wparam,
25
+ LPARAM const lparam) noexcept override;
26
+
27
+ private:
28
+ // The project to run.
29
+ flutter::DartProject project_;
30
+
31
+ // The Flutter instance hosted by this window.
32
+ std::unique_ptr<flutter::FlutterViewController> flutter_controller_;
33
+ std::unique_ptr<flutter::MethodChannel<flutter::EncodableValue>>
34
+ desktop_channel_;
35
+ };
36
+
37
+ #endif // RUNNER_FLUTTER_WINDOW_H_
@@ -0,0 +1,53 @@
1
+ #include <flutter/dart_project.h>
2
+ #include <flutter/flutter_view_controller.h>
3
+ #include <windows.h>
4
+
5
+ #include "flutter_window.h"
6
+ #include "utils.h"
7
+
8
+ int APIENTRY wWinMain(_In_ HINSTANCE instance, _In_opt_ HINSTANCE prev,
9
+ _In_ wchar_t *command_line, _In_ int show_command) {
10
+ // Attach to console when present (e.g., 'flutter run') or create a
11
+ // new console when running with a debugger.
12
+ if (!::AttachConsole(ATTACH_PARENT_PROCESS) && ::IsDebuggerPresent()) {
13
+ CreateAndAttachConsole();
14
+ }
15
+
16
+ // Initialize COM, so that it is available for use in the library and/or
17
+ // plugins.
18
+ ::CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
19
+
20
+ flutter::DartProject project(L"data");
21
+
22
+ std::vector<std::string> command_line_arguments =
23
+ GetCommandLineArguments();
24
+
25
+ project.set_dart_entrypoint_arguments(std::move(command_line_arguments));
26
+
27
+ FlutterWindow window(project);
28
+ Win32Window::Point origin(10, 10);
29
+ Win32Window::Size size(1280, 720);
30
+ if (!window.Create(L"NeoAgent", origin, size)) {
31
+ ::CoUninitialize();
32
+ return EXIT_FAILURE;
33
+ }
34
+ window.SetQuitOnClose(false);
35
+
36
+ ::MSG msg;
37
+ while (true) {
38
+ const BOOL result = ::GetMessage(&msg, nullptr, 0, 0);
39
+ if (result == -1) {
40
+ ::OutputDebugStringW(L"GetMessage failed in Windows runner message loop.\n");
41
+ ::CoUninitialize();
42
+ return EXIT_FAILURE;
43
+ }
44
+ if (result == 0) {
45
+ break;
46
+ }
47
+ ::TranslateMessage(&msg);
48
+ ::DispatchMessage(&msg);
49
+ }
50
+
51
+ ::CoUninitialize();
52
+ return EXIT_SUCCESS;
53
+ }
@@ -0,0 +1,16 @@
1
+ //{{NO_DEPENDENCIES}}
2
+ // Microsoft Visual C++ generated include file.
3
+ // Used by Runner.rc
4
+ //
5
+ #define IDI_APP_ICON 101
6
+
7
+ // Next default values for new objects
8
+ //
9
+ #ifdef APSTUDIO_INVOKED
10
+ #ifndef APSTUDIO_READONLY_SYMBOLS
11
+ #define _APS_NEXT_RESOURCE_VALUE 102
12
+ #define _APS_NEXT_COMMAND_VALUE 40001
13
+ #define _APS_NEXT_CONTROL_VALUE 1001
14
+ #define _APS_NEXT_SYMED_VALUE 101
15
+ #endif
16
+ #endif
@@ -0,0 +1,14 @@
1
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
2
+ <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
3
+ <application xmlns="urn:schemas-microsoft-com:asm.v3">
4
+ <windowsSettings>
5
+ <dpiAwareness xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">PerMonitorV2</dpiAwareness>
6
+ </windowsSettings>
7
+ </application>
8
+ <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
9
+ <application>
10
+ <!-- Windows 10 and Windows 11 -->
11
+ <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
12
+ </application>
13
+ </compatibility>
14
+ </assembly>