neoagent 2.3.1-beta.0 → 2.3.1-beta.100

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (406) hide show
  1. package/.env.example +63 -5
  2. package/README.md +16 -5
  3. package/com.neoagent.plist +1 -1
  4. package/docs/automation.md +82 -41
  5. package/docs/capabilities.md +85 -89
  6. package/docs/configuration.md +89 -87
  7. package/docs/getting-started.md +46 -27
  8. package/docs/hardware.md +1 -1
  9. package/docs/index.md +24 -30
  10. package/docs/integrations.md +46 -60
  11. package/docs/migration.md +41 -146
  12. package/docs/operations.md +23 -20
  13. package/docs/skills.md +20 -20
  14. package/docs/why-neoagent.md +15 -13
  15. package/extensions/chrome-browser/background.mjs +10 -2
  16. package/flutter_app/.metadata +42 -0
  17. package/flutter_app/DESIGN_SYSTEM.md +166 -0
  18. package/flutter_app/README.md +21 -0
  19. package/flutter_app/analysis_options.yaml +32 -0
  20. package/flutter_app/android/app/build.gradle.kts +111 -0
  21. package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
  22. package/flutter_app/android/app/src/main/AndroidManifest.xml +185 -0
  23. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +888 -0
  24. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/auto/NeoAgentCarAppService.kt +114 -0
  25. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
  26. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
  27. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
  28. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
  29. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
  30. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
  31. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
  32. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
  33. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
  34. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
  35. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnection.kt +118 -0
  36. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnectionService.kt +86 -0
  37. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
  38. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
  39. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
  40. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
  41. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
  42. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
  43. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
  44. package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
  45. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
  46. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
  47. package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
  48. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
  49. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
  50. package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
  51. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  52. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  53. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  54. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  55. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  56. package/flutter_app/android/app/src/main/res/values/arrays.xml +8 -0
  57. package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
  58. package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
  59. package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
  60. package/flutter_app/android/app/src/main/res/xml/automotive_app_desc.xml +4 -0
  61. package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
  62. package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
  63. package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
  64. package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
  65. package/flutter_app/android/build.gradle.kts +24 -0
  66. package/flutter_app/android/ci-release.keystore +0 -0
  67. package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
  68. package/flutter_app/android/gradle.properties +3 -0
  69. package/flutter_app/android/key.properties +4 -0
  70. package/flutter_app/android/settings.gradle.kts +26 -0
  71. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  72. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  73. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  74. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  75. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  76. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  77. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  78. package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
  79. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  80. package/flutter_app/lib/features/location/location_service.dart +117 -0
  81. package/flutter_app/lib/features/notifications/notification_interceptor.dart +73 -0
  82. package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +571 -0
  83. package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +275 -0
  84. package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +366 -0
  85. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +57 -0
  86. package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +218 -0
  87. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +46 -0
  88. package/flutter_app/lib/main.dart +107 -0
  89. package/flutter_app/lib/main_account_settings.dart +1340 -0
  90. package/flutter_app/lib/main_admin.dart +900 -0
  91. package/flutter_app/lib/main_app_shell.dart +2162 -0
  92. package/flutter_app/lib/main_chat.dart +3902 -0
  93. package/flutter_app/lib/main_controller.dart +7229 -0
  94. package/flutter_app/lib/main_devices.dart +1738 -0
  95. package/flutter_app/lib/main_integrations.dart +917 -0
  96. package/flutter_app/lib/main_launcher.dart +959 -0
  97. package/flutter_app/lib/main_launcher_entry.dart +5 -0
  98. package/flutter_app/lib/main_models.dart +3807 -0
  99. package/flutter_app/lib/main_navigation.dart +206 -0
  100. package/flutter_app/lib/main_operations.dart +5249 -0
  101. package/flutter_app/lib/main_recordings.dart +920 -0
  102. package/flutter_app/lib/main_runtime.dart +869 -0
  103. package/flutter_app/lib/main_settings.dart +2288 -0
  104. package/flutter_app/lib/main_shared.dart +3393 -0
  105. package/flutter_app/lib/main_spacing.dart +33 -0
  106. package/flutter_app/lib/main_theme.dart +276 -0
  107. package/flutter_app/lib/main_unified.dart +388 -0
  108. package/flutter_app/lib/main_voice_assistant.dart +957 -0
  109. package/flutter_app/lib/src/analytics_service.dart +294 -0
  110. package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
  111. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
  112. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
  113. package/flutter_app/lib/src/android_app_installer.dart +22 -0
  114. package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
  115. package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
  116. package/flutter_app/lib/src/android_auto_bridge.dart +59 -0
  117. package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
  118. package/flutter_app/lib/src/app_launch_bridge.dart +58 -0
  119. package/flutter_app/lib/src/app_release_updater.dart +511 -0
  120. package/flutter_app/lib/src/backend_client.dart +1870 -0
  121. package/flutter_app/lib/src/desktop_companion.dart +2 -0
  122. package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
  123. package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
  124. package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
  125. package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
  126. package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
  127. package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
  128. package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
  129. package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
  130. package/flutter_app/lib/src/health_bridge.dart +136 -0
  131. package/flutter_app/lib/src/live_voice_capture.dart +85 -0
  132. package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
  133. package/flutter_app/lib/src/network/app_http_client.dart +53 -0
  134. package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
  135. package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
  136. package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
  137. package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
  138. package/flutter_app/lib/src/oauth_launcher.dart +33 -0
  139. package/flutter_app/lib/src/oauth_launcher_io.dart +95 -0
  140. package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
  141. package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
  142. package/flutter_app/lib/src/recording_bridge.dart +232 -0
  143. package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
  144. package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
  145. package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
  146. package/flutter_app/lib/src/recording_payloads.dart +86 -0
  147. package/flutter_app/lib/src/theme/palette.dart +81 -0
  148. package/flutter_app/lib/src/web_app_update_monitor.dart +17 -0
  149. package/flutter_app/lib/src/web_app_update_monitor_stub.dart +24 -0
  150. package/flutter_app/lib/src/web_app_update_monitor_web.dart +123 -0
  151. package/flutter_app/lib/src/widget_bridge.dart +49 -0
  152. package/flutter_app/linux/CMakeLists.txt +128 -0
  153. package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
  154. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +47 -0
  155. package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
  156. package/flutter_app/linux/flutter/generated_plugins.cmake +32 -0
  157. package/flutter_app/linux/runner/CMakeLists.txt +26 -0
  158. package/flutter_app/linux/runner/main.cc +6 -0
  159. package/flutter_app/linux/runner/my_application.cc +144 -0
  160. package/flutter_app/linux/runner/my_application.h +18 -0
  161. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  162. package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
  163. package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
  164. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +48 -0
  165. package/flutter_app/macos/Podfile +42 -0
  166. package/flutter_app/macos/Podfile.lock +87 -0
  167. package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
  168. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
  169. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  170. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  171. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  172. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  173. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  174. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  175. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  176. package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
  177. package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
  178. package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
  179. package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
  180. package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
  181. package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
  182. package/flutter_app/macos/Runner/Info.plist +36 -0
  183. package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
  184. package/flutter_app/macos/Runner/Release.entitlements +12 -0
  185. package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
  186. package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  187. package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
  188. package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
  189. package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  190. package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
  191. package/flutter_app/patch_strings.py +12 -0
  192. package/flutter_app/pubspec.lock +1256 -0
  193. package/flutter_app/pubspec.yaml +59 -0
  194. package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
  195. package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
  196. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
  197. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
  198. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
  199. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
  200. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
  201. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
  202. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
  203. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
  204. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
  205. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
  206. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
  207. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
  208. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
  209. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
  210. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
  211. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
  212. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
  213. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
  214. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
  215. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
  216. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
  217. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
  218. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
  219. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
  220. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
  221. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
  222. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
  223. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
  224. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
  225. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
  226. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
  227. package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
  228. package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
  229. package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
  230. package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
  231. package/flutter_app/tool/generate_desktop_branding.py +219 -0
  232. package/flutter_app/web/favicon.png +0 -0
  233. package/flutter_app/web/favicon.svg +12 -0
  234. package/flutter_app/web/icons/Icon-192.png +0 -0
  235. package/flutter_app/web/icons/Icon-512.png +0 -0
  236. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  237. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  238. package/flutter_app/web/index.html +39 -0
  239. package/flutter_app/web/manifest.json +35 -0
  240. package/flutter_app/windows/CMakeLists.txt +108 -0
  241. package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
  242. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +50 -0
  243. package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
  244. package/flutter_app/windows/flutter/generated_plugins.cmake +36 -0
  245. package/flutter_app/windows/runner/CMakeLists.txt +41 -0
  246. package/flutter_app/windows/runner/Runner.rc +121 -0
  247. package/flutter_app/windows/runner/flutter_window.cpp +533 -0
  248. package/flutter_app/windows/runner/flutter_window.h +37 -0
  249. package/flutter_app/windows/runner/main.cpp +53 -0
  250. package/flutter_app/windows/runner/resource.h +16 -0
  251. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  252. package/flutter_app/windows/runner/runner.exe.manifest +14 -0
  253. package/flutter_app/windows/runner/utils.cpp +65 -0
  254. package/flutter_app/windows/runner/utils.h +19 -0
  255. package/flutter_app/windows/runner/win32_window.cpp +299 -0
  256. package/flutter_app/windows/runner/win32_window.h +102 -0
  257. package/lib/install_helpers.js +31 -0
  258. package/lib/manager.js +465 -43
  259. package/package.json +11 -3
  260. package/runtime/paths.js +137 -0
  261. package/server/config/analytics.js +30 -0
  262. package/server/db/database.js +363 -21
  263. package/server/guest-agent.android.package.json +13 -0
  264. package/server/guest-agent.browser.package.json +14 -0
  265. package/server/guest_agent.js +98 -52
  266. package/server/http/middleware.js +55 -2
  267. package/server/http/routes.js +6 -1
  268. package/server/http/static.js +41 -1
  269. package/server/index.js +16 -16
  270. package/server/public/.last_build_id +1 -1
  271. package/server/public/assets/AssetManifest.bin +1 -1
  272. package/server/public/assets/AssetManifest.bin.json +1 -1
  273. package/server/public/assets/AssetManifest.json +1 -1
  274. package/server/public/assets/NOTICES +679 -129
  275. package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
  276. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  277. package/server/public/assets/packages/mixpanel_flutter/assets/mixpanel.js +3 -0
  278. package/server/public/canvaskit/wimp.wasm +0 -0
  279. package/server/public/flutter_bootstrap.js +2 -2
  280. package/server/public/main.dart.js +95866 -87333
  281. package/server/routes/account.js +26 -1
  282. package/server/routes/agents.js +6 -2
  283. package/server/routes/android.js +22 -6
  284. package/server/routes/auth.js +25 -5
  285. package/server/routes/browser.js +47 -13
  286. package/server/routes/integrations.js +119 -9
  287. package/server/routes/memory.js +179 -8
  288. package/server/routes/runtime.js +14 -0
  289. package/server/routes/screenHistory.js +46 -0
  290. package/server/routes/settings.js +102 -24
  291. package/server/routes/skills.js +12 -0
  292. package/server/routes/social_video.js +66 -0
  293. package/server/routes/tasks.js +22 -2
  294. package/server/routes/triggers.js +90 -0
  295. package/server/routes/wearable.js +67 -0
  296. package/server/routes/widgets.js +4 -4
  297. package/server/services/ai/capabilityHealth.js +23 -51
  298. package/server/services/ai/compaction.js +1 -1
  299. package/server/services/ai/deliverables/artifact_helpers.js +190 -0
  300. package/server/services/ai/deliverables/contracts.js +113 -0
  301. package/server/services/ai/deliverables/index.js +20 -0
  302. package/server/services/ai/deliverables/selector.js +94 -0
  303. package/server/services/ai/deliverables/validator.js +63 -0
  304. package/server/services/ai/deliverables/workflows.js +207 -0
  305. package/server/services/ai/engine.js +372 -27
  306. package/server/services/ai/hooks.js +127 -0
  307. package/server/services/ai/integrated_tools/index.js +98 -0
  308. package/server/services/ai/integrated_tools/remotion.js +373 -0
  309. package/server/services/ai/integrated_tools/shared.js +204 -0
  310. package/server/services/ai/integrated_tools/slidev.js +207 -0
  311. package/server/services/ai/loopPolicy.js +146 -0
  312. package/server/services/ai/models.js +36 -0
  313. package/server/services/ai/providers/githubCopilot.js +97 -0
  314. package/server/services/ai/providers/openai.js +2 -1
  315. package/server/services/ai/providers/openaiCodex.js +384 -0
  316. package/server/services/ai/recordingInsights.js +11 -13
  317. package/server/services/ai/runEvents.js +100 -0
  318. package/server/services/ai/settings.js +20 -0
  319. package/server/services/ai/systemPrompt.js +16 -2
  320. package/server/services/ai/taskAnalysis.js +2 -0
  321. package/server/services/ai/toolResult.js +20 -0
  322. package/server/services/ai/toolRunner.js +46 -6
  323. package/server/services/ai/toolSelector.js +54 -1
  324. package/server/services/ai/tools.js +360 -127
  325. package/server/services/android/android_bootstrap_worker.js +63 -0
  326. package/server/services/android/controller.js +442 -1760
  327. package/server/services/bootstrap_helpers.js +0 -34
  328. package/server/services/browser/controller.js +326 -52
  329. package/server/services/cli/executor.js +40 -1
  330. package/server/services/desktop/screenRecorder.js +182 -0
  331. package/server/services/integrations/env.js +10 -0
  332. package/server/services/integrations/figma/provider.js +1 -0
  333. package/server/services/integrations/github/common.js +106 -0
  334. package/server/services/integrations/github/provider.js +499 -0
  335. package/server/services/integrations/github/repos.js +1124 -0
  336. package/server/services/integrations/google/provider.js +1 -0
  337. package/server/services/integrations/manager.js +88 -12
  338. package/server/services/integrations/microsoft/provider.js +1 -0
  339. package/server/services/integrations/oauth_provider.js +25 -8
  340. package/server/services/integrations/provider_config_store.js +85 -0
  341. package/server/services/integrations/registry.js +8 -2
  342. package/server/services/integrations/spotify/provider.js +488 -0
  343. package/server/services/integrations/trello/provider.js +842 -0
  344. package/server/services/integrations/weather/provider.js +559 -0
  345. package/server/services/integrations/whatsapp/provider.js +6 -2
  346. package/server/services/manager.js +109 -154
  347. package/server/services/mcp/client.js +120 -23
  348. package/server/services/memory/llm_transfer.js +218 -0
  349. package/server/services/memory/manager.js +299 -35
  350. package/server/services/messaging/access_policy.js +10 -0
  351. package/server/services/messaging/automation.js +1 -1
  352. package/server/services/messaging/manager.js +93 -8
  353. package/server/services/messaging/meshtastic.js +277 -0
  354. package/server/services/messaging/meshtastic_env.js +40 -0
  355. package/server/services/messaging/meshtastic_protocol.js +530 -0
  356. package/server/services/messaging/meshtastic_tcp_transport.js +25 -0
  357. package/server/services/runtime/backends/local-vm.js +221 -158
  358. package/server/services/runtime/docker-vm-manager.js +392 -0
  359. package/server/services/runtime/guest_bootstrap.js +654 -0
  360. package/server/services/runtime/manager.js +85 -29
  361. package/server/services/runtime/settings.js +22 -31
  362. package/server/services/runtime/validation.js +16 -31
  363. package/server/services/skills/base_catalog.js +33 -0
  364. package/server/services/social_video/adapters/base.js +26 -0
  365. package/server/services/social_video/adapters/index.js +27 -0
  366. package/server/services/social_video/adapters/instagram.js +17 -0
  367. package/server/services/social_video/adapters/tiktok.js +17 -0
  368. package/server/services/social_video/adapters/x.js +17 -0
  369. package/server/services/social_video/adapters/youtube.js +17 -0
  370. package/server/services/social_video/captions.js +187 -0
  371. package/server/services/social_video/frame.js +42 -0
  372. package/server/services/social_video/index.js +7 -0
  373. package/server/services/social_video/metadata.js +63 -0
  374. package/server/services/social_video/result.js +63 -0
  375. package/server/services/social_video/service.js +720 -0
  376. package/server/services/social_video/url.js +83 -0
  377. package/server/services/tasks/adapters/index.js +2 -0
  378. package/server/services/tasks/adapters/manual.js +12 -0
  379. package/server/services/tasks/adapters/schedule.js +33 -5
  380. package/server/services/tasks/adapters/weather_event.js +84 -0
  381. package/server/services/tasks/integration_runtime.js +86 -1
  382. package/server/services/tasks/runtime.js +3 -3
  383. package/server/services/tasks/task_repository.js +3 -6
  384. package/server/services/voice/agentBridge.js +20 -4
  385. package/server/services/voice/message.js +3 -0
  386. package/server/services/voice/openaiClient.js +4 -1
  387. package/server/services/voice/openaiSpeech.js +6 -1
  388. package/server/services/voice/providers.js +144 -15
  389. package/server/services/voice/runtime.js +8 -8
  390. package/server/services/voice/runtimeManager.js +292 -20
  391. package/server/services/voice/turnRunner.js +29 -9
  392. package/server/services/wearable/firmware_manifest.js +353 -0
  393. package/server/services/wearable/gateway.js +350 -0
  394. package/server/services/wearable/protocol.js +45 -0
  395. package/server/services/wearable/service.js +240 -0
  396. package/server/services/websocket.js +3 -1
  397. package/server/services/widgets/focus_widget.js +137 -0
  398. package/server/services/widgets/service.js +182 -6
  399. package/server/services/workspace/manager.js +301 -0
  400. package/server/utils/deployment.js +16 -9
  401. package/server/utils/local_secrets.js +56 -0
  402. package/server/utils/logger.js +37 -9
  403. package/server/services/integrations/home_assistant/provider.js +0 -350
  404. package/server/services/messaging/access_policy.test.js +0 -228
  405. package/server/services/runtime/backends/host.js +0 -37
  406. package/server/services/runtime/qemu.js +0 -391
@@ -1,7 +1,3 @@
1
- 'use strict';
2
-
3
- const HEADLESS_BROWSER_SETTING_KEY = 'headless_browser';
4
-
5
1
  function getErrorMessage(error) {
6
2
  if (error instanceof Error && error.message) {
7
3
  return error.message;
@@ -10,34 +6,6 @@ function getErrorMessage(error) {
10
6
  return String(error);
11
7
  }
12
8
 
13
- function isEnabledSetting(value) {
14
- return value !== 'false' && value !== false && value !== '0';
15
- }
16
-
17
- function restoreBrowserHeadlessPreference(browserController, database) {
18
- const userCount =
19
- database.prepare('SELECT COUNT(*) AS count FROM users').get()?.count || 0;
20
- const headlessSetting =
21
- userCount === 1
22
- ? database
23
- .prepare(
24
- 'SELECT value FROM user_settings WHERE user_id = (SELECT id FROM users LIMIT 1) AND key = ?',
25
- )
26
- .get(HEADLESS_BROWSER_SETTING_KEY)
27
- : null;
28
-
29
- if (!headlessSetting) {
30
- return { restored: false, userCount };
31
- }
32
-
33
- browserController.headless = isEnabledSetting(headlessSetting.value);
34
- return {
35
- restored: true,
36
- userCount,
37
- headless: browserController.headless,
38
- };
39
- }
40
-
41
9
  function runBackgroundTask(errorPrefix, task, logger = console.error) {
42
10
  return Promise.resolve()
43
11
  .then(task)
@@ -48,7 +16,5 @@ function runBackgroundTask(errorPrefix, task, logger = console.error) {
48
16
 
49
17
  module.exports = {
50
18
  getErrorMessage,
51
- isEnabledSetting,
52
- restoreBrowserHeadlessPreference,
53
19
  runBackgroundTask,
54
20
  };
@@ -1,16 +1,20 @@
1
1
  const path = require('path');
2
2
  const fs = require('fs');
3
+ const { spawn } = require('child_process');
3
4
  const { DATA_DIR } = require('../../../runtime/paths');
4
5
 
5
6
  const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
6
7
  if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
8
+ const BROWSER_PROFILE_ROOT = path.join(DATA_DIR, 'browser-profiles');
9
+ if (!fs.existsSync(BROWSER_PROFILE_ROOT)) fs.mkdirSync(BROWSER_PROFILE_ROOT, { recursive: true });
10
+ const BROWSER_READY_MARKER = '/var/lib/neoagent/browser-runtime-ready';
7
11
 
8
12
  const USER_AGENTS = [
9
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
10
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
13
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
14
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
11
15
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
12
16
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
13
- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
17
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
14
18
  ];
15
19
 
16
20
  const VIEWPORTS = [
@@ -22,12 +26,31 @@ const VIEWPORTS = [
22
26
 
23
27
  function resolveBrowserExecutablePath() {
24
28
  const explicitPath =
29
+ process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH ||
25
30
  process.env.PUPPETEER_EXECUTABLE_PATH ||
26
31
  process.env.CHROME_BIN ||
27
32
  process.env.CHROMIUM_BIN;
28
33
 
29
34
  if (explicitPath && fs.existsSync(explicitPath)) return explicitPath;
30
35
 
36
+ const bundledCandidates = [
37
+ () => require('playwright-chromium').chromium.executablePath(),
38
+ ];
39
+ for (const resolveBundled of bundledCandidates) {
40
+ try {
41
+ const bundledPath = resolveBundled();
42
+ if (bundledPath && fs.existsSync(bundledPath)) {
43
+ if (process.platform === 'linux') {
44
+ const wrappedPath = path.join(path.dirname(bundledPath), 'chrome-wrapper');
45
+ if (fs.existsSync(wrappedPath)) {
46
+ return wrappedPath;
47
+ }
48
+ }
49
+ return bundledPath;
50
+ }
51
+ } catch {}
52
+ }
53
+
31
54
  const platformCandidates = process.platform === 'darwin'
32
55
  ? [
33
56
  '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
@@ -53,6 +76,38 @@ function resolveBrowserExecutablePath() {
53
76
  return platformCandidates.find((candidate) => fs.existsSync(candidate)) || null;
54
77
  }
55
78
 
79
+ function installPlaywrightBrowserBinary(browserName) {
80
+ const packageRoot = path.dirname(require.resolve('playwright-chromium/package.json'));
81
+ const cliPath = path.join(packageRoot, 'cli.js');
82
+ return new Promise((resolve, reject) => {
83
+ const args = [cliPath, 'install', '--no-shell', browserName];
84
+ const child = spawn(process.execPath, args, {
85
+ stdio: ['ignore', 'pipe', 'pipe'],
86
+ });
87
+ let stdout = '';
88
+ let stderr = '';
89
+
90
+ child.stdout.on('data', (data) => {
91
+ stdout += data.toString();
92
+ });
93
+ child.stderr.on('data', (data) => {
94
+ stderr += data.toString();
95
+ });
96
+ child.on('error', (error) => {
97
+ const detail = String(error?.message || `playwright install ${browserName} failed`).trim();
98
+ reject(new Error(detail));
99
+ });
100
+ child.on('close', (code) => {
101
+ if (code === 0) {
102
+ resolve();
103
+ return;
104
+ }
105
+ const detail = String(stderr || stdout || `playwright install ${browserName} exited with code ${code ?? 'unknown'}`).trim();
106
+ reject(new Error(detail));
107
+ });
108
+ });
109
+ }
110
+
56
111
  function rand(min, max) {
57
112
  return Math.floor(Math.random() * (max - min + 1)) + min;
58
113
  }
@@ -61,6 +116,22 @@ function sleep(ms) {
61
116
  return new Promise(r => setTimeout(r, ms));
62
117
  }
63
118
 
119
+ async function waitForFile(filePath, options = {}) {
120
+ const timeoutMs = Math.max(0, Number(options.timeoutMs || 0));
121
+ const intervalMs = Math.max(100, Number(options.intervalMs || 500));
122
+ if (!filePath || timeoutMs <= 0 || fs.existsSync(filePath)) {
123
+ return fs.existsSync(filePath);
124
+ }
125
+ const startedAt = Date.now();
126
+ while (Date.now() - startedAt < timeoutMs) {
127
+ await sleep(intervalMs);
128
+ if (fs.existsSync(filePath)) {
129
+ return true;
130
+ }
131
+ }
132
+ return fs.existsSync(filePath);
133
+ }
134
+
64
135
  function buildIsolatedEvaluationExpression(script) {
65
136
  const source = String(script || 'undefined');
66
137
  // Evaluate each snippet inside a fresh function scope so repeated calls do not
@@ -68,26 +139,59 @@ function buildIsolatedEvaluationExpression(script) {
68
139
  return `(() => eval(${JSON.stringify(source)}))()`;
69
140
  }
70
141
 
142
+ function normalizeWaitUntil(waitUntil) {
143
+ const value = String(waitUntil || '').trim().toLowerCase();
144
+ if (value === 'networkidle0' || value === 'networkidle2') {
145
+ return 'networkidle';
146
+ }
147
+ if (value === 'load' || value === 'domcontentloaded' || value === 'networkidle' || value === 'commit') {
148
+ return value;
149
+ }
150
+ return 'domcontentloaded';
151
+ }
152
+
153
+ function clearChromiumSingletonLocks(profileDir) {
154
+ const lockEntries = [
155
+ 'SingletonLock',
156
+ 'SingletonSocket',
157
+ 'SingletonCookie',
158
+ 'SingletonStartupLock',
159
+ 'DevToolsActivePort',
160
+ ];
161
+ for (const entry of lockEntries) {
162
+ const targetPath = path.join(profileDir, entry);
163
+ try {
164
+ fs.rmSync(targetPath, { force: true, recursive: true });
165
+ } catch {}
166
+ }
167
+ }
168
+
71
169
  class BrowserController {
72
170
  constructor(options = {}) {
73
171
  this.io = options.io || null;
74
172
  this.userId = options.userId != null ? String(options.userId) : null;
75
173
  this.artifactStore = options.artifactStore || null;
76
174
  this.runtimeBackend = options.runtimeBackend || 'host';
175
+ this.engine = 'chromium';
77
176
  this.browser = null;
177
+ this.context = null;
78
178
  this.page = null;
179
+ this.displayProcess = null;
180
+ this.displayValue = process.env.DISPLAY || null;
79
181
  this.launching = false;
80
- this.headless = true;
182
+ this.launchPromise = null;
183
+ this.browserBinaryInstallPromise = null;
184
+ this.headless = false;
81
185
  this._viewport = VIEWPORTS[0];
82
186
  this._userAgent = USER_AGENTS[0];
187
+ this.profileDir = path.join(BROWSER_PROFILE_ROOT, this.userId || 'default');
188
+ if (!fs.existsSync(this.profileDir)) fs.mkdirSync(this.profileDir, { recursive: true });
83
189
  }
84
190
 
85
191
  async setHeadless(val) {
86
- const wasHeadless = this.headless;
87
- this.headless = val !== false && val !== 'false';
88
- if (wasHeadless !== this.headless) {
89
- await this.close().catch(() => { });
90
- }
192
+ void val;
193
+ // Browser sessions inside the VM always run headed.
194
+ this.headless = false;
91
195
  }
92
196
 
93
197
  async closeBrowser() {
@@ -98,14 +202,20 @@ class BrowserController {
98
202
  const ua = this._userAgent;
99
203
  const vp = this._viewport;
100
204
 
101
- await page.setUserAgent(ua);
102
- await page.setViewport(vp);
103
- await page.setExtraHTTPHeaders({
104
- 'Accept-Language': 'en-US,en;q=0.9',
105
- });
205
+ if (typeof page.setUserAgent === 'function') {
206
+ await page.setUserAgent(ua);
207
+ }
208
+ if (typeof page.setViewport === 'function') {
209
+ await page.setViewport(vp);
210
+ }
211
+ if (typeof page.setExtraHTTPHeaders === 'function') {
212
+ await page.setExtraHTTPHeaders({
213
+ 'Accept-Language': 'en-US,en;q=0.9',
214
+ });
215
+ }
106
216
 
107
217
  // Inject fingerprint overrides before any page script runs
108
- await page.evaluateOnNewDocument(`
218
+ const script = `
109
219
  (() => {
110
220
  // Remove webdriver flag
111
221
  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
@@ -148,47 +258,138 @@ class BrowserController {
148
258
  return arr;
149
259
  }
150
260
  });
261
+
262
+ // WebGL Spoofing
263
+ const getParameterProxyHandler = {
264
+ apply: function(target, ctx, args) {
265
+ const param = args[0];
266
+ // UNMASKED_VENDOR_WEBGL
267
+ if (param === 37445) return 'Google Inc. (Apple)';
268
+ // UNMASKED_RENDERER_WEBGL
269
+ if (param === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)';
270
+ return Reflect.apply(target, ctx, args);
271
+ }
272
+ };
273
+ const getParam = WebGLRenderingContext.prototype.getParameter;
274
+ WebGLRenderingContext.prototype.getParameter = new Proxy(getParam, getParameterProxyHandler);
275
+ if (typeof WebGL2RenderingContext !== 'undefined') {
276
+ const getParam2 = WebGL2RenderingContext.prototype.getParameter;
277
+ WebGL2RenderingContext.prototype.getParameter = new Proxy(getParam2, getParameterProxyHandler);
278
+ }
279
+
280
+ // Canvas Spoofing (slight noise)
281
+ const originalFillText = CanvasRenderingContext2D.prototype.fillText;
282
+ CanvasRenderingContext2D.prototype.fillText = function(...args) {
283
+ if (!this._spoofing_applied) {
284
+ this._spoofing_applied = true;
285
+ const r = Math.random() * 0.0001;
286
+ const g = Math.random() * 0.0001;
287
+ const b = Math.random() * 0.0001;
288
+ this.fillStyle = \`rgba(\${Math.floor(r * 255)}, \${Math.floor(g * 255)}, \${Math.floor(b * 255)}, 0.01)\`;
289
+ originalFillText.call(this, "spoof", 0, 0);
290
+ }
291
+ return originalFillText.apply(this, args);
292
+ };
293
+
294
+ // Media Devices Spoofing
295
+ if (navigator.mediaDevices && navigator.mediaDevices.enumerateDevices) {
296
+ const originalEnumerateDevices = navigator.mediaDevices.enumerateDevices.bind(navigator.mediaDevices);
297
+ navigator.mediaDevices.enumerateDevices = async () => {
298
+ return [
299
+ { kind: 'audioinput', deviceId: 'default', groupId: 'a', label: 'MacBook Pro Microphone' },
300
+ { kind: 'audiooutput', deviceId: 'default', groupId: 'b', label: 'MacBook Pro Speakers' },
301
+ { kind: 'videoinput', deviceId: 'default', groupId: 'c', label: 'FaceTime HD Camera' }
302
+ ];
303
+ };
304
+ }
151
305
  })();
152
- `);
306
+ `;
307
+ if (typeof page.evaluateOnNewDocument === 'function') {
308
+ await page.evaluateOnNewDocument(script);
309
+ } else if (typeof page.addInitScript === 'function') {
310
+ await page.addInitScript(script);
311
+ }
153
312
  }
154
313
 
155
314
  async ensureBrowser() {
156
315
  if (this.browser && this.browser.isConnected()) return;
157
- if (this.launching) {
158
- await sleep(2000);
316
+ if (this.launchPromise) {
317
+ await this.launchPromise;
159
318
  return;
160
319
  }
161
320
 
162
321
  this.launching = true;
163
- try {
164
- const puppeteer = require('puppeteer-extra');
165
- const StealthPlugin = require('puppeteer-extra-plugin-stealth');
166
- puppeteer.use(StealthPlugin());
322
+ this.launchPromise = (async () => {
323
+ const runtimeReady = await waitForFile(BROWSER_READY_MARKER, {
324
+ timeoutMs: 10 * 60 * 1000,
325
+ intervalMs: 1000,
326
+ });
327
+ if (!runtimeReady) {
328
+ throw new Error('Browser runtime provisioning is still in progress inside the VM. Retry shortly.');
329
+ }
330
+ await this.ensureVirtualDisplay();
167
331
 
168
332
  this._userAgent = USER_AGENTS[rand(0, USER_AGENTS.length - 1)];
169
333
  this._viewport = VIEWPORTS[rand(0, VIEWPORTS.length - 1)];
170
334
 
171
- this.browser = await puppeteer.launch({
172
- headless: this.headless ? 'new' : false,
173
- executablePath: resolveBrowserExecutablePath() || undefined,
174
- args: [
175
- '--no-sandbox',
176
- '--disable-setuid-sandbox',
177
- '--disable-dev-shm-usage',
178
- '--disable-blink-features=AutomationControlled',
179
- '--disable-infobars',
180
- '--no-first-run',
181
- '--no-default-browser-check',
182
- '--lang=en-US,en',
183
- `--window-size=${this._viewport.width},${this._viewport.height}`,
184
- ],
185
- defaultViewport: this._viewport,
186
- ignoreDefaultArgs: ['--enable-automation'],
187
- });
335
+ let executablePath = resolveBrowserExecutablePath();
336
+ if (!executablePath) {
337
+ if (!this.browserBinaryInstallPromise) {
338
+ this.browserBinaryInstallPromise = installPlaywrightBrowserBinary(this.engine);
339
+ }
340
+ try {
341
+ await this.browserBinaryInstallPromise;
342
+ } finally {
343
+ this.browserBinaryInstallPromise = null;
344
+ }
345
+ executablePath = resolveBrowserExecutablePath();
346
+ }
347
+
348
+ if (!executablePath) {
349
+ throw new Error(`No ${this.engine} executable found for the VM browser runtime.`);
350
+ }
351
+
352
+ const launchEnv = {
353
+ ...process.env,
354
+ ...(this.displayValue ? { DISPLAY: this.displayValue } : {}),
355
+ };
188
356
 
189
- this.page = await this.browser.newPage();
357
+ const launchArgs = [
358
+ '--no-sandbox',
359
+ '--disable-setuid-sandbox',
360
+ '--disable-dev-shm-usage',
361
+ '--disable-crash-reporter',
362
+ '--disable-background-networking',
363
+ '--disable-component-update',
364
+ '--disable-blink-features=AutomationControlled',
365
+ '--disable-infobars',
366
+ '--no-first-run',
367
+ '--no-default-browser-check',
368
+ '--disable-gpu',
369
+ '--lang=en-US,en',
370
+ `--window-size=${this._viewport.width},${this._viewport.height}`,
371
+ ];
372
+
373
+ const playwright = require('playwright-chromium');
374
+ clearChromiumSingletonLocks(this.profileDir);
375
+ this.context = await playwright.chromium.launchPersistentContext(this.profileDir, {
376
+ headless: false,
377
+ executablePath,
378
+ env: launchEnv,
379
+ args: launchArgs,
380
+ viewport: this._viewport,
381
+ ignoreHTTPSErrors: false,
382
+ timeout: 120000,
383
+ });
384
+ this.browser = typeof this.context.browser === 'function' ? this.context.browser() : null;
385
+ this.page = this.context.pages()[0] || await this.context.newPage();
190
386
  await this._applyStealthToPage(this.page);
387
+ })();
388
+
389
+ try {
390
+ await this.launchPromise;
191
391
  } finally {
392
+ this.launchPromise = null;
192
393
  this.launching = false;
193
394
  }
194
395
  }
@@ -196,7 +397,11 @@ class BrowserController {
196
397
  async ensurePage() {
197
398
  await this.ensureBrowser();
198
399
  if (!this.page || this.page.isClosed()) {
199
- this.page = await this.browser.newPage();
400
+ if (this.context && typeof this.context.newPage === 'function') {
401
+ this.page = await this.context.newPage();
402
+ } else {
403
+ this.page = await this.browser.newPage();
404
+ }
200
405
  await this._applyStealthToPage(this.page);
201
406
  }
202
407
  return this.page;
@@ -253,7 +458,7 @@ class BrowserController {
253
458
 
254
459
  try {
255
460
  const response = await page.goto(url, {
256
- waitUntil: 'networkidle2',
461
+ waitUntil: normalizeWaitUntil(options.waitUntil),
257
462
  timeout: 30000
258
463
  });
259
464
 
@@ -285,7 +490,9 @@ class BrowserController {
285
490
  url: currentUrl,
286
491
  status: response?.status() || 0,
287
492
  bodyText,
288
- screenshotPath: screenshot?.screenshotPath || null
493
+ screenshotPath: screenshot?.screenshotPath || null,
494
+ artifactId: screenshot?.artifactId || null,
495
+ fullPath: screenshot?.fullPath || null
289
496
  };
290
497
  } catch (err) {
291
498
  let screenshot = null;
@@ -293,7 +500,9 @@ class BrowserController {
293
500
  return {
294
501
  error: err.message,
295
502
  url,
296
- screenshotPath: screenshot?.screenshotPath || null
503
+ screenshotPath: screenshot?.screenshotPath || null,
504
+ artifactId: screenshot?.artifactId || null,
505
+ fullPath: screenshot?.fullPath || null
297
506
  };
298
507
  }
299
508
  }
@@ -335,7 +544,9 @@ class BrowserController {
335
544
  success: true,
336
545
  url: page.url(),
337
546
  title: await page.title(),
338
- screenshotPath: screenshotResult?.screenshotPath || null
547
+ screenshotPath: screenshotResult?.screenshotPath || null,
548
+ artifactId: screenshotResult?.artifactId || null,
549
+ fullPath: screenshotResult?.fullPath || null
339
550
  };
340
551
  } catch (err) {
341
552
  return { error: err.message };
@@ -364,7 +575,9 @@ class BrowserController {
364
575
  y: py,
365
576
  url: page.url(),
366
577
  title: await page.title(),
367
- screenshotPath: screenshotResult?.screenshotPath || null
578
+ screenshotPath: screenshotResult?.screenshotPath || null,
579
+ artifactId: screenshotResult?.artifactId || null,
580
+ fullPath: screenshotResult?.fullPath || null
368
581
  };
369
582
  } catch (err) {
370
583
  return { error: err.message };
@@ -388,7 +601,9 @@ class BrowserController {
388
601
  success: true,
389
602
  url: page.url(),
390
603
  title: await page.title(),
391
- screenshotPath: screenshotResult?.screenshotPath || null
604
+ screenshotPath: screenshotResult?.screenshotPath || null,
605
+ artifactId: screenshotResult?.artifactId || null,
606
+ fullPath: screenshotResult?.fullPath || null
392
607
  };
393
608
  } catch (err) {
394
609
  return { error: err.message };
@@ -419,7 +634,9 @@ class BrowserController {
419
634
  return {
420
635
  success: true,
421
636
  typed: text,
422
- screenshotPath: screenshotResult?.screenshotPath || null
637
+ screenshotPath: screenshotResult?.screenshotPath || null,
638
+ artifactId: screenshotResult?.artifactId || null,
639
+ fullPath: screenshotResult?.fullPath || null
423
640
  };
424
641
  } catch (err) {
425
642
  return { error: err.message };
@@ -445,7 +662,9 @@ class BrowserController {
445
662
  return {
446
663
  success: true,
447
664
  typed: String(text || ''),
448
- screenshotPath: screenshotResult?.screenshotPath || null
665
+ screenshotPath: screenshotResult?.screenshotPath || null,
666
+ artifactId: screenshotResult?.artifactId || null,
667
+ fullPath: screenshotResult?.fullPath || null
449
668
  };
450
669
  } catch (err) {
451
670
  return { error: err.message };
@@ -469,7 +688,9 @@ class BrowserController {
469
688
  return {
470
689
  success: true,
471
690
  key: normalized,
472
- screenshotPath: screenshotResult?.screenshotPath || null
691
+ screenshotPath: screenshotResult?.screenshotPath || null,
692
+ artifactId: screenshotResult?.artifactId || null,
693
+ fullPath: screenshotResult?.fullPath || null
473
694
  };
474
695
  } catch (err) {
475
696
  return { error: err.message };
@@ -520,15 +741,20 @@ class BrowserController {
520
741
  }
521
742
 
522
743
  async launch(options = {}) {
744
+ void options;
523
745
  await this.ensureBrowser();
524
746
  return { success: true };
525
747
  }
526
748
 
527
749
  isLaunched() {
528
- return !!(this.browser && this.browser.isConnected());
750
+ if (this.context) return true;
751
+ return !!(this.browser && typeof this.browser.isConnected === 'function' && this.browser.isConnected());
529
752
  }
530
753
 
531
754
  getPageCount() {
755
+ if (this.context && typeof this.context.pages === 'function') {
756
+ try { return this.context.pages().length; } catch { return 0; }
757
+ }
532
758
  if (!this.browser) return 0;
533
759
  try { return this.browser.pages ? 1 : 0; } catch { return 0; }
534
760
  }
@@ -553,16 +779,64 @@ class BrowserController {
553
779
  };
554
780
  }
555
781
 
782
+ async getCookies() {
783
+ await this.ensureBrowser();
784
+ if (!this.context || typeof this.context.cookies !== 'function') {
785
+ return { cookies: [] };
786
+ }
787
+ const cookies = await this.context.cookies().catch(() => []);
788
+ return {
789
+ cookies: Array.isArray(cookies) ? cookies : [],
790
+ };
791
+ }
792
+
556
793
  async close() {
557
794
  if (this.page && !this.page.isClosed()) {
558
795
  await this.page.close().catch(() => { });
559
796
  }
797
+ if (this.context) {
798
+ await this.context.close().catch(() => { });
799
+ this.context = null;
800
+ this.browser = null;
801
+ this.page = null;
802
+ return;
803
+ }
560
804
  if (this.browser) {
561
805
  await this.browser.close().catch(() => { });
562
806
  this.browser = null;
563
807
  this.page = null;
564
808
  }
565
809
  }
810
+
811
+ async ensureVirtualDisplay() {
812
+ if (process.platform !== 'linux') {
813
+ return;
814
+ }
815
+ if (this.displayProcess && !this.displayProcess.killed) {
816
+ return;
817
+ }
818
+ if (this.displayValue && String(this.displayValue).trim()) {
819
+ return;
820
+ }
821
+
822
+ const display = ':99';
823
+ const child = spawn('Xvfb', [display, '-screen', '0', '1440x900x24', '-ac', '-nolisten', 'tcp'], {
824
+ stdio: ['ignore', 'ignore', 'pipe'],
825
+ });
826
+
827
+ let launchError = '';
828
+ child.stderr.on('data', (chunk) => {
829
+ launchError += chunk.toString();
830
+ });
831
+
832
+ await sleep(1000);
833
+ if (child.exitCode != null) {
834
+ throw new Error(`Failed to start Xvfb: ${String(launchError || `exit code ${child.exitCode}`).trim()}`);
835
+ }
836
+
837
+ this.displayProcess = child;
838
+ this.displayValue = display;
839
+ }
566
840
  }
567
841
 
568
- module.exports = { BrowserController, resolveBrowserExecutablePath, buildIsolatedEvaluationExpression };
842
+ module.exports = { BrowserController, resolveBrowserExecutablePath, buildIsolatedEvaluationExpression, normalizeWaitUntil };
@@ -7,6 +7,38 @@ const FORCE_KILL_GRACE_MS = 5000;
7
7
  const MAX_STDOUT_CHARS = 50000;
8
8
  const MAX_STDERR_CHARS = 10000;
9
9
 
10
+ function resolveDefaultShell() {
11
+ const candidates = [
12
+ process.env.SHELL,
13
+ '/bin/zsh',
14
+ '/bin/bash',
15
+ '/bin/sh',
16
+ ].filter(Boolean);
17
+
18
+ for (const candidate of candidates) {
19
+ try {
20
+ execFileSync(candidate, ['-lc', 'printf ok'], {
21
+ timeout: 3000,
22
+ encoding: 'utf8',
23
+ stdio: ['ignore', 'pipe', 'pipe'],
24
+ });
25
+ return candidate;
26
+ } catch {}
27
+ }
28
+
29
+ try {
30
+ execFileSync('/bin/sh', ['-lc', 'printf ok'], {
31
+ timeout: 3000,
32
+ encoding: 'utf8',
33
+ stdio: ['ignore', 'pipe', 'pipe'],
34
+ });
35
+ return '/bin/sh';
36
+ } catch (error) {
37
+ console.warn('[CLI] No usable shell found for executor:', error?.message || error);
38
+ return null;
39
+ }
40
+ }
41
+
10
42
  function clampTimeout(value, fallback) {
11
43
  const parsed = Number(value);
12
44
  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
@@ -46,7 +78,10 @@ function wrapCommandForShell(command, shellPath) {
46
78
  class CLIExecutor {
47
79
  constructor() {
48
80
  this.activeProcesses = new Map();
49
- this.defaultShell = process.env.SHELL || '/bin/zsh';
81
+ this.defaultShell = resolveDefaultShell();
82
+ if (!this.defaultShell) {
83
+ throw new Error('No usable shell found for CLI execution.');
84
+ }
50
85
  }
51
86
 
52
87
  _getLoginPath() {
@@ -258,6 +293,10 @@ class CLIExecutor {
258
293
  return false;
259
294
  }
260
295
 
296
+ isManaged(pid) {
297
+ return this.activeProcesses.has(pid);
298
+ }
299
+
261
300
  killAll(reason = 'aborted') {
262
301
  for (const [pid, proc] of this.activeProcesses) {
263
302
  proc.__neoagentKilled = true;