neoagent 2.3.1-beta.11 → 2.3.1-beta.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +53 -5
- package/README.md +16 -7
- package/com.neoagent.plist +1 -1
- package/docs/automation.md +82 -41
- package/docs/capabilities.md +85 -89
- package/docs/configuration.md +87 -94
- package/docs/getting-started.md +46 -27
- package/docs/hardware.md +1 -1
- package/docs/index.md +24 -30
- package/docs/integrations.md +46 -64
- package/docs/migration.md +41 -146
- package/docs/operations.md +23 -20
- package/docs/skills.md +20 -20
- package/docs/why-neoagent.md +15 -13
- package/extensions/chrome-browser/background.mjs +10 -2
- package/flutter_app/.metadata +42 -0
- package/flutter_app/DESIGN_SYSTEM.md +166 -0
- package/flutter_app/README.md +21 -0
- package/flutter_app/analysis_options.yaml +32 -0
- package/flutter_app/android/app/build.gradle.kts +111 -0
- package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
- package/flutter_app/android/app/src/main/AndroidManifest.xml +185 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +888 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/auto/NeoAgentCarAppService.kt +114 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnection.kt +118 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/telecom/NeoAgentConnectionService.kt +86 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
- package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
- package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
- package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
- package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
- package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
- package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
- package/flutter_app/android/app/src/main/res/values/arrays.xml +8 -0
- package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
- package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
- package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
- package/flutter_app/android/app/src/main/res/xml/automotive_app_desc.xml +4 -0
- package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
- package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
- package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
- package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
- package/flutter_app/android/build.gradle.kts +24 -0
- package/flutter_app/android/ci-release.keystore +0 -0
- package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
- package/flutter_app/android/gradle.properties +3 -0
- package/flutter_app/android/key.properties +4 -0
- package/flutter_app/android/settings.gradle.kts +26 -0
- package/flutter_app/assets/branding/app_icon_1024.png +0 -0
- package/flutter_app/assets/branding/app_icon_128.png +0 -0
- package/flutter_app/assets/branding/app_icon_192.png +0 -0
- package/flutter_app/assets/branding/app_icon_256.png +0 -0
- package/flutter_app/assets/branding/app_icon_32.png +0 -0
- package/flutter_app/assets/branding/app_icon_512.png +0 -0
- package/flutter_app/assets/branding/app_icon_64.png +0 -0
- package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
- package/flutter_app/assets/branding/tray_icon_template.png +0 -0
- package/flutter_app/lib/features/location/location_service.dart +117 -0
- package/flutter_app/lib/features/notifications/notification_interceptor.dart +73 -0
- package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +571 -0
- package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +275 -0
- package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +366 -0
- package/flutter_app/lib/features/onboarding/onboarding_shell.dart +57 -0
- package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +218 -0
- package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +46 -0
- package/flutter_app/lib/main.dart +107 -0
- package/flutter_app/lib/main_account_settings.dart +1340 -0
- package/flutter_app/lib/main_admin.dart +900 -0
- package/flutter_app/lib/main_app_shell.dart +2162 -0
- package/flutter_app/lib/main_chat.dart +3902 -0
- package/flutter_app/lib/main_controller.dart +7229 -0
- package/flutter_app/lib/main_devices.dart +1738 -0
- package/flutter_app/lib/main_integrations.dart +917 -0
- package/flutter_app/lib/main_launcher.dart +959 -0
- package/flutter_app/lib/main_launcher_entry.dart +5 -0
- package/flutter_app/lib/main_models.dart +3807 -0
- package/flutter_app/lib/main_navigation.dart +206 -0
- package/flutter_app/lib/main_operations.dart +5249 -0
- package/flutter_app/lib/main_recordings.dart +920 -0
- package/flutter_app/lib/main_runtime.dart +869 -0
- package/flutter_app/lib/main_settings.dart +2288 -0
- package/flutter_app/lib/main_shared.dart +3393 -0
- package/flutter_app/lib/main_spacing.dart +33 -0
- package/flutter_app/lib/main_theme.dart +276 -0
- package/flutter_app/lib/main_unified.dart +388 -0
- package/flutter_app/lib/main_voice_assistant.dart +957 -0
- package/flutter_app/lib/src/analytics_service.dart +294 -0
- package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
- package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
- package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
- package/flutter_app/lib/src/android_app_installer.dart +22 -0
- package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
- package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
- package/flutter_app/lib/src/android_auto_bridge.dart +59 -0
- package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
- package/flutter_app/lib/src/app_launch_bridge.dart +58 -0
- package/flutter_app/lib/src/app_release_updater.dart +511 -0
- package/flutter_app/lib/src/backend_client.dart +1870 -0
- package/flutter_app/lib/src/desktop_companion.dart +2 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
- package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
- package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
- package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
- package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
- package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
- package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
- package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
- package/flutter_app/lib/src/health_bridge.dart +136 -0
- package/flutter_app/lib/src/live_voice_capture.dart +85 -0
- package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
- package/flutter_app/lib/src/network/app_http_client.dart +53 -0
- package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
- package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
- package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
- package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
- package/flutter_app/lib/src/oauth_launcher.dart +33 -0
- package/flutter_app/lib/src/oauth_launcher_io.dart +95 -0
- package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
- package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
- package/flutter_app/lib/src/recording_bridge.dart +232 -0
- package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
- package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
- package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
- package/flutter_app/lib/src/recording_payloads.dart +86 -0
- package/flutter_app/lib/src/theme/palette.dart +81 -0
- package/flutter_app/lib/src/web_app_update_monitor.dart +17 -0
- package/flutter_app/lib/src/web_app_update_monitor_stub.dart +24 -0
- package/flutter_app/lib/src/web_app_update_monitor_web.dart +123 -0
- package/flutter_app/lib/src/widget_bridge.dart +49 -0
- package/flutter_app/linux/CMakeLists.txt +128 -0
- package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
- package/flutter_app/linux/flutter/generated_plugin_registrant.cc +47 -0
- package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
- package/flutter_app/linux/flutter/generated_plugins.cmake +32 -0
- package/flutter_app/linux/runner/CMakeLists.txt +26 -0
- package/flutter_app/linux/runner/main.cc +6 -0
- package/flutter_app/linux/runner/my_application.cc +144 -0
- package/flutter_app/linux/runner/my_application.h +18 -0
- package/flutter_app/linux/runner/resources/app_icon.png +0 -0
- package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
- package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
- package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +48 -0
- package/flutter_app/macos/Podfile +42 -0
- package/flutter_app/macos/Podfile.lock +87 -0
- package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
- package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
- package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
- package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
- package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
- package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
- package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
- package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
- package/flutter_app/macos/Runner/Info.plist +36 -0
- package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
- package/flutter_app/macos/Runner/Release.entitlements +12 -0
- package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
- package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
- package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
- package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
- package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
- package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
- package/flutter_app/patch_strings.py +12 -0
- package/flutter_app/pubspec.lock +1256 -0
- package/flutter_app/pubspec.yaml +59 -0
- package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
- package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
- package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
- package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
- package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
- package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
- package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
- package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
- package/flutter_app/tool/generate_desktop_branding.py +219 -0
- package/flutter_app/web/favicon.png +0 -0
- package/flutter_app/web/favicon.svg +12 -0
- package/flutter_app/web/icons/Icon-192.png +0 -0
- package/flutter_app/web/icons/Icon-512.png +0 -0
- package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
- package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
- package/flutter_app/web/index.html +39 -0
- package/flutter_app/web/manifest.json +35 -0
- package/flutter_app/windows/CMakeLists.txt +108 -0
- package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.cc +50 -0
- package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
- package/flutter_app/windows/flutter/generated_plugins.cmake +36 -0
- package/flutter_app/windows/runner/CMakeLists.txt +41 -0
- package/flutter_app/windows/runner/Runner.rc +121 -0
- package/flutter_app/windows/runner/flutter_window.cpp +533 -0
- package/flutter_app/windows/runner/flutter_window.h +37 -0
- package/flutter_app/windows/runner/main.cpp +53 -0
- package/flutter_app/windows/runner/resource.h +16 -0
- package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
- package/flutter_app/windows/runner/runner.exe.manifest +14 -0
- package/flutter_app/windows/runner/utils.cpp +65 -0
- package/flutter_app/windows/runner/utils.h +19 -0
- package/flutter_app/windows/runner/win32_window.cpp +299 -0
- package/flutter_app/windows/runner/win32_window.h +102 -0
- package/lib/install_helpers.js +31 -0
- package/lib/manager.js +579 -122
- package/package.json +10 -3
- package/runtime/paths.js +137 -0
- package/server/config/analytics.js +30 -0
- package/server/db/database.js +296 -22
- package/server/guest-agent.android.package.json +13 -0
- package/server/guest-agent.browser.package.json +14 -0
- package/server/guest_agent.js +98 -52
- package/server/http/middleware.js +5 -2
- package/server/http/routes.js +3 -0
- package/server/http/static.js +41 -1
- package/server/index.js +15 -16
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/AssetManifest.bin +1 -1
- package/server/public/assets/AssetManifest.bin.json +1 -1
- package/server/public/assets/AssetManifest.json +1 -1
- package/server/public/assets/NOTICES +644 -155
- package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/assets/packages/mixpanel_flutter/assets/mixpanel.js +3 -0
- package/server/public/canvaskit/wimp.wasm +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/main.dart.js +96067 -88139
- package/server/routes/account.js +26 -1
- package/server/routes/agents.js +6 -2
- package/server/routes/android.js +22 -6
- package/server/routes/auth.js +25 -5
- package/server/routes/browser.js +47 -13
- package/server/routes/integrations.js +22 -1
- package/server/routes/memory.js +168 -6
- package/server/routes/runtime.js +14 -0
- package/server/routes/settings.js +102 -24
- package/server/routes/skills.js +12 -0
- package/server/routes/social_video.js +66 -0
- package/server/routes/tasks.js +22 -2
- package/server/routes/triggers.js +44 -35
- package/server/routes/wearable.js +67 -0
- package/server/routes/widgets.js +4 -4
- package/server/services/ai/capabilityHealth.js +23 -51
- package/server/services/ai/compaction.js +1 -1
- package/server/services/ai/deliverables/artifact_helpers.js +190 -0
- package/server/services/ai/deliverables/contracts.js +113 -0
- package/server/services/ai/deliverables/index.js +20 -0
- package/server/services/ai/deliverables/selector.js +94 -0
- package/server/services/ai/deliverables/validator.js +63 -0
- package/server/services/ai/deliverables/workflows.js +207 -0
- package/server/services/ai/engine.js +363 -27
- package/server/services/ai/hooks.js +127 -0
- package/server/services/ai/integrated_tools/index.js +98 -0
- package/server/services/ai/integrated_tools/remotion.js +373 -0
- package/server/services/ai/integrated_tools/shared.js +204 -0
- package/server/services/ai/integrated_tools/slidev.js +207 -0
- package/server/services/ai/loopPolicy.js +146 -0
- package/server/services/ai/models.js +31 -4
- package/server/services/ai/providers/claudeCode.js +49 -0
- package/server/services/ai/providers/openai.js +2 -1
- package/server/services/ai/providers/openaiCodex.js +422 -15
- package/server/services/ai/recordingInsights.js +11 -13
- package/server/services/ai/runEvents.js +100 -0
- package/server/services/ai/settings.js +12 -2
- package/server/services/ai/systemPrompt.js +15 -1
- package/server/services/ai/taskAnalysis.js +2 -0
- package/server/services/ai/toolResult.js +20 -0
- package/server/services/ai/toolRunner.js +46 -6
- package/server/services/ai/toolSelector.js +54 -1
- package/server/services/ai/tools.js +210 -116
- package/server/services/android/android_bootstrap_worker.js +63 -0
- package/server/services/android/controller.js +442 -1760
- package/server/services/bootstrap_helpers.js +0 -34
- package/server/services/browser/controller.js +279 -49
- package/server/services/cli/executor.js +40 -1
- package/server/services/desktop/screenRecorder.js +65 -9
- package/server/services/integrations/figma/provider.js +1 -0
- package/server/services/integrations/github/provider.js +3 -3
- package/server/services/integrations/google/provider.js +1 -0
- package/server/services/integrations/manager.js +25 -5
- package/server/services/integrations/microsoft/provider.js +1 -0
- package/server/services/integrations/oauth_provider.js +15 -5
- package/server/services/integrations/provider_config_store.js +22 -13
- package/server/services/integrations/registry.js +2 -4
- package/server/services/integrations/spotify/provider.js +1 -0
- package/server/services/integrations/trello/provider.js +842 -0
- package/server/services/manager.js +88 -155
- package/server/services/mcp/client.js +120 -23
- package/server/services/memory/llm_transfer.js +218 -0
- package/server/services/memory/manager.js +260 -33
- package/server/services/messaging/access_policy.js +10 -0
- package/server/services/messaging/automation.js +1 -1
- package/server/services/messaging/manager.js +64 -1
- package/server/services/messaging/meshtastic.js +277 -0
- package/server/services/messaging/meshtastic_env.js +40 -0
- package/server/services/messaging/meshtastic_protocol.js +530 -0
- package/server/services/messaging/meshtastic_tcp_transport.js +25 -0
- package/server/services/runtime/backends/local-vm.js +221 -158
- package/server/services/runtime/docker-vm-manager.js +392 -0
- package/server/services/runtime/guest_bootstrap.js +654 -0
- package/server/services/runtime/manager.js +85 -29
- package/server/services/runtime/settings.js +22 -31
- package/server/services/runtime/validation.js +16 -31
- package/server/services/social_video/adapters/base.js +26 -0
- package/server/services/social_video/adapters/index.js +27 -0
- package/server/services/social_video/adapters/instagram.js +17 -0
- package/server/services/social_video/adapters/tiktok.js +17 -0
- package/server/services/social_video/adapters/x.js +17 -0
- package/server/services/social_video/adapters/youtube.js +17 -0
- package/server/services/social_video/captions.js +187 -0
- package/server/services/social_video/frame.js +42 -0
- package/server/services/social_video/index.js +7 -0
- package/server/services/social_video/metadata.js +63 -0
- package/server/services/social_video/result.js +63 -0
- package/server/services/social_video/service.js +720 -0
- package/server/services/social_video/url.js +83 -0
- package/server/services/tasks/integration_runtime.js +1 -1
- package/server/services/tasks/runtime.js +1 -1
- package/server/services/tasks/task_repository.js +3 -6
- package/server/services/voice/openaiSpeech.js +6 -1
- package/server/services/voice/providers.js +142 -14
- package/server/services/voice/runtime.js +8 -8
- package/server/services/voice/runtimeManager.js +156 -19
- package/server/services/voice/turnRunner.js +29 -9
- package/server/services/wearable/firmware_manifest.js +353 -0
- package/server/services/wearable/gateway.js +350 -0
- package/server/services/wearable/protocol.js +45 -0
- package/server/services/wearable/service.js +240 -0
- package/server/services/websocket.js +3 -1
- package/server/services/widgets/focus_widget.js +137 -0
- package/server/services/widgets/service.js +133 -2
- package/server/services/workspace/manager.js +301 -0
- package/server/utils/deployment.js +16 -9
- package/server/services/integrations/home_assistant/provider.js +0 -630
- package/server/services/messaging/access_policy.test.js +0 -228
- package/server/services/runtime/backends/host.js +0 -37
- package/server/services/runtime/qemu.js +0 -391
|
@@ -112,6 +112,26 @@ function compactToolResult(toolName, toolArgs = {}, toolResult, options = {}) {
|
|
|
112
112
|
});
|
|
113
113
|
break;
|
|
114
114
|
|
|
115
|
+
case 'social_video_extract':
|
|
116
|
+
envelope = trimObject({
|
|
117
|
+
tool: toolName,
|
|
118
|
+
platform: toolResult?.platform,
|
|
119
|
+
sourceUrl: toolResult?.sourceUrl,
|
|
120
|
+
resolvedUrl: toolResult?.resolvedUrl,
|
|
121
|
+
title: clampText(toolResult?.title || '', Math.floor(softLimit * 0.25)),
|
|
122
|
+
description: clampText(toolResult?.description || '', Math.floor(softLimit * 0.25)),
|
|
123
|
+
transcriptSource: toolResult?.transcriptSource,
|
|
124
|
+
transcriptPreview: lineExcerpt(toolResult?.transcript || '', 6, Math.floor(softLimit * 0.35)),
|
|
125
|
+
frameImage: trimObject({
|
|
126
|
+
url: toolResult?.frameImage?.url,
|
|
127
|
+
source: toolResult?.frameImage?.source,
|
|
128
|
+
}),
|
|
129
|
+
setupReady: toolResult?.setup?.ready,
|
|
130
|
+
warningCount: Array.isArray(toolResult?.warnings) ? toolResult.warnings.length : 0,
|
|
131
|
+
errorCount: Array.isArray(toolResult?.errors) ? toolResult.errors.length : 0,
|
|
132
|
+
});
|
|
133
|
+
break;
|
|
134
|
+
|
|
115
135
|
case 'android_dump_ui':
|
|
116
136
|
case 'android_observe':
|
|
117
137
|
envelope = trimObject({
|
|
@@ -13,6 +13,15 @@ function shellEscape(value) {
|
|
|
13
13
|
return `'${text.replace(/'/g, `'\\''`)}'`;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
+
// Shell metacharacters that must not appear in a skill command template.
|
|
17
|
+
const SHELL_METACHAR_RE = /[;&|`$\n\r(){}\\<>]/;
|
|
18
|
+
|
|
19
|
+
function isValidCommandTemplate(template) {
|
|
20
|
+
// Strip all {placeholder} tokens, then reject any remaining shell metacharacters.
|
|
21
|
+
const bare = String(template).replace(/\{[^{}]*\}/g, '');
|
|
22
|
+
return !SHELL_METACHAR_RE.test(bare);
|
|
23
|
+
}
|
|
24
|
+
|
|
16
25
|
function clampText(value, maxChars) {
|
|
17
26
|
const text = String(value || '').trim().replace(/\s+/g, ' ');
|
|
18
27
|
if (!text) return '';
|
|
@@ -20,10 +29,19 @@ function clampText(value, maxChars) {
|
|
|
20
29
|
return `${text.slice(0, Math.max(0, maxChars - 1)).trimEnd()}...`;
|
|
21
30
|
}
|
|
22
31
|
|
|
32
|
+
function isValidUserId(userId) {
|
|
33
|
+
if (typeof userId === 'number') {
|
|
34
|
+
return Number.isInteger(userId) && userId > 0;
|
|
35
|
+
}
|
|
36
|
+
if (typeof userId === 'string') {
|
|
37
|
+
return userId.trim() !== '';
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
|
|
23
42
|
class SkillRunner {
|
|
24
43
|
constructor(options = {}) {
|
|
25
44
|
this.skills = new Map();
|
|
26
|
-
this.executor = options.executor || null;
|
|
27
45
|
this.runtimeManager = options.runtimeManager || null;
|
|
28
46
|
}
|
|
29
47
|
|
|
@@ -170,16 +188,38 @@ class SkillRunner {
|
|
|
170
188
|
}
|
|
171
189
|
|
|
172
190
|
if (skill.metadata.command) {
|
|
173
|
-
|
|
191
|
+
if (!isValidCommandTemplate(skill.metadata.command)) {
|
|
192
|
+
return { error: `Skill '${toolName}' has an invalid command template` };
|
|
193
|
+
}
|
|
174
194
|
let command = skill.metadata.command;
|
|
175
195
|
for (const [key, value] of Object.entries(args)) {
|
|
176
196
|
command = command.replaceAll(`{${key}}`, shellEscape(value));
|
|
177
197
|
}
|
|
178
|
-
if (
|
|
179
|
-
return
|
|
198
|
+
if (!isValidUserId(context.userId)) {
|
|
199
|
+
return {
|
|
200
|
+
error: 'Missing or invalid userId',
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
if (!this.runtimeManager) {
|
|
204
|
+
return {
|
|
205
|
+
error: 'VM runtime is required',
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
try {
|
|
209
|
+
return await this.runtimeManager.executeCommand(context.userId, command);
|
|
210
|
+
} catch (err) {
|
|
211
|
+
const commandName = skill?.name || toolName || 'unknown';
|
|
212
|
+
console.error('[SkillRunner] Skill command execution failed:', {
|
|
213
|
+
userId: context.userId,
|
|
214
|
+
commandName,
|
|
215
|
+
command: String(command).slice(0, 200),
|
|
216
|
+
error: err?.message || String(err),
|
|
217
|
+
});
|
|
218
|
+
return {
|
|
219
|
+
error: 'Skill command execution failed',
|
|
220
|
+
details: err?.message || String(err),
|
|
221
|
+
};
|
|
180
222
|
}
|
|
181
|
-
const executor = this.executor || new CLIExecutor();
|
|
182
|
-
return await executor.execute(command, { cwd: skill.dir });
|
|
183
223
|
}
|
|
184
224
|
|
|
185
225
|
return {
|
|
@@ -13,6 +13,42 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
const MCP_ALWAYS_INCLUDE_THRESHOLD = 20;
|
|
16
|
+
const MAX_TOOLS = 128; // Strict provider limit (e.g. Github Copilot / OpenAI)
|
|
17
|
+
|
|
18
|
+
function ensureRequiredTools(selectedTools = [], builtInTools = [], options = {}) {
|
|
19
|
+
const requiredNames = [];
|
|
20
|
+
if (options.widgetId) requiredNames.push('save_widget_snapshot');
|
|
21
|
+
if (!requiredNames.length) return selectedTools;
|
|
22
|
+
|
|
23
|
+
const selected = Array.isArray(selectedTools) ? [...selectedTools] : [];
|
|
24
|
+
|
|
25
|
+
for (const toolName of requiredNames) {
|
|
26
|
+
if (selected.some((tool) => tool?.name === toolName)) continue;
|
|
27
|
+
const required = builtInTools.find((tool) => tool?.name === toolName);
|
|
28
|
+
if (!required) continue;
|
|
29
|
+
|
|
30
|
+
if (selected.length < MAX_TOOLS) {
|
|
31
|
+
selected.push(required);
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Keep within provider tool cap: replace the last non-required tool.
|
|
36
|
+
let replaced = false;
|
|
37
|
+
for (let index = selected.length - 1; index >= 0; index -= 1) {
|
|
38
|
+
const currentName = selected[index]?.name;
|
|
39
|
+
if (!requiredNames.includes(currentName)) {
|
|
40
|
+
selected[index] = required;
|
|
41
|
+
replaced = true;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (!replaced && selected.length > 0) {
|
|
46
|
+
selected[selected.length - 1] = required;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
return selected;
|
|
51
|
+
}
|
|
16
52
|
|
|
17
53
|
function selectMcpTools(task, mcpTools = []) {
|
|
18
54
|
if (!mcpTools.length) return [];
|
|
@@ -33,7 +69,24 @@ function selectMcpTools(task, mcpTools = []) {
|
|
|
33
69
|
}
|
|
34
70
|
|
|
35
71
|
function selectToolsForTask(task, builtInTools = [], mcpTools = [], _options = {}) {
|
|
36
|
-
|
|
72
|
+
const selectedMcp = selectMcpTools(task, mcpTools);
|
|
73
|
+
const options = _options || {};
|
|
74
|
+
let selected;
|
|
75
|
+
|
|
76
|
+
if (builtInTools.length + selectedMcp.length <= MAX_TOOLS) {
|
|
77
|
+
selected = [...builtInTools, ...selectedMcp];
|
|
78
|
+
return ensureRequiredTools(selected, builtInTools, options);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// If we exceed the limit, prioritize base tools and take as many MCP tools as fit
|
|
82
|
+
const remainingSpace = MAX_TOOLS - builtInTools.length;
|
|
83
|
+
if (remainingSpace > 0) {
|
|
84
|
+
selected = [...builtInTools, ...selectedMcp.slice(0, remainingSpace)];
|
|
85
|
+
return ensureRequiredTools(selected, builtInTools, options);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
selected = builtInTools.slice(0, MAX_TOOLS);
|
|
89
|
+
return ensureRequiredTools(selected, builtInTools, options);
|
|
37
90
|
}
|
|
38
91
|
|
|
39
92
|
module.exports = { selectToolsForTask, selectMcpTools };
|
|
@@ -9,6 +9,10 @@ const {
|
|
|
9
9
|
normalizeOutgoingMessageForPlatform,
|
|
10
10
|
} = require('../messaging/formatting_guides');
|
|
11
11
|
const { INTERIM_KINDS, normalizeInterimKind } = require('./interim');
|
|
12
|
+
const {
|
|
13
|
+
executeIntegratedTool,
|
|
14
|
+
getIntegratedToolDefinitions,
|
|
15
|
+
} = require('./integrated_tools');
|
|
12
16
|
|
|
13
17
|
function compactText(text, maxChars = 120) {
|
|
14
18
|
const str = String(text || '').replace(/\s+/g, ' ').trim();
|
|
@@ -192,6 +196,54 @@ function isProactiveTrigger(triggerSource) {
|
|
|
192
196
|
return triggerSource === 'schedule' || triggerSource === 'tasks';
|
|
193
197
|
}
|
|
194
198
|
|
|
199
|
+
function normalizeSendMessagePurpose(value) {
|
|
200
|
+
const normalized = String(value || '').trim().toLowerCase();
|
|
201
|
+
if (normalized === 'final_result' || normalized === 'blocker' || normalized === 'no_response') {
|
|
202
|
+
return normalized;
|
|
203
|
+
}
|
|
204
|
+
return '';
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function validateProactiveSendMessageArgs({ purpose, normalizedMessage }) {
|
|
208
|
+
const normalizedPurpose = normalizeSendMessagePurpose(purpose);
|
|
209
|
+
if (!normalizedPurpose) {
|
|
210
|
+
return {
|
|
211
|
+
ok: false,
|
|
212
|
+
error: 'Background send_message requires purpose=final_result, blocker, or no_response.',
|
|
213
|
+
reason: 'Background send_message requires purpose=final_result, blocker, or no_response.',
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if (normalizedPurpose === 'no_response') {
|
|
218
|
+
if (normalizedMessage !== '[NO RESPONSE]') {
|
|
219
|
+
return {
|
|
220
|
+
ok: false,
|
|
221
|
+
error: 'purpose=no_response requires content "[NO RESPONSE]".',
|
|
222
|
+
reason: 'purpose=no_response requires content "[NO RESPONSE]".',
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
return {
|
|
226
|
+
ok: false,
|
|
227
|
+
skipped: true,
|
|
228
|
+
suppressed: true,
|
|
229
|
+
reason: 'no_response',
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
if (normalizedMessage === '[NO RESPONSE]') {
|
|
234
|
+
return {
|
|
235
|
+
ok: false,
|
|
236
|
+
error: `purpose=${normalizedPurpose} cannot use content "[NO RESPONSE]".`,
|
|
237
|
+
reason: `purpose=${normalizedPurpose} cannot use content "[NO RESPONSE]".`,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
return {
|
|
242
|
+
ok: true,
|
|
243
|
+
purpose: normalizedPurpose,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
195
247
|
function getRunState(engine, runId) {
|
|
196
248
|
if (!engine || !runId) return null;
|
|
197
249
|
return engine.activeRuns.get(runId) || null;
|
|
@@ -742,14 +794,15 @@ function getAvailableTools(app, options = {}) {
|
|
|
742
794
|
},
|
|
743
795
|
{
|
|
744
796
|
name: 'send_message',
|
|
745
|
-
description: `Send a message on a connected messaging platform. Supports WhatsApp (text/media), Telnyx Voice (phone calls — TTS), Discord, Telegram, Slack, Google Chat, Microsoft Teams, Matrix, Signal, iMessage/BlueBubbles, IRC, Feishu, LINE, Mattermost, Nextcloud Talk, Nostr, Synology Chat, Tlon, Twitch, Zalo, WeChat, WebChat, and configurable webhook bridges. ${buildSendMessageFormattingReference()} For WhatsApp: use media_path to attach files. Use content "[NO RESPONSE]" only when the user explicitly asked for silence or no reply.`,
|
|
797
|
+
description: `Send a message on a connected messaging platform. Supports WhatsApp (text/media), Telnyx Voice (phone calls — TTS), Discord, Telegram, Slack, Google Chat, Microsoft Teams, Matrix, Signal, iMessage/BlueBubbles, IRC, Feishu, LINE, Mattermost, Nextcloud Talk, Nostr, Synology Chat, Tlon, Twitch, Zalo, WeChat, WebChat, and configurable webhook bridges. ${buildSendMessageFormattingReference()} For WhatsApp: use media_path to attach files. Use content "[NO RESPONSE]" only when the user explicitly asked for silence or no reply. For background task or schedule runs, set purpose to final_result, blocker, or no_response.`,
|
|
746
798
|
parameters: {
|
|
747
799
|
type: 'object',
|
|
748
800
|
properties: {
|
|
749
801
|
platform: { type: 'string', description: 'Platform name, for example whatsapp, telnyx, discord, telegram, slack, google_chat, teams, matrix, signal, imessage, bluebubbles, irc, line, mattermost, or webchat' },
|
|
750
802
|
to: { type: 'string', description: 'Recipient/chat ID for the connected platform, such as a WhatsApp chat ID, Telnyx call_control_id, Slack channel ID, Matrix room ID, Discord channel snowflake / "dm_<userId>", Telegram "dm_<userId>" / raw group chat ID, IRC channel, or webhook target' },
|
|
751
803
|
content: { type: 'string', description: 'Message text. Write one compact natural chat reply; the runtime adapts final formatting for the destination platform.' },
|
|
752
|
-
media_path: { type: 'string', description: 'WhatsApp only: absolute path to a local file to attach. Leave empty for text-only or Telnyx.' }
|
|
804
|
+
media_path: { type: 'string', description: 'WhatsApp only: absolute path to a local file to attach. Leave empty for text-only or Telnyx.' },
|
|
805
|
+
purpose: { type: 'string', enum: ['final_result', 'blocker', 'no_response'], description: 'For background task or schedule runs, required intent for this outbound message. Use final_result for a concrete useful outcome, blocker for a real issue the user should know about, or no_response to intentionally send nothing.' }
|
|
753
806
|
},
|
|
754
807
|
required: ['platform', 'to', 'content']
|
|
755
808
|
}
|
|
@@ -1118,6 +1171,7 @@ function getAvailableTools(app, options = {}) {
|
|
|
1118
1171
|
required: ['prompt']
|
|
1119
1172
|
}
|
|
1120
1173
|
},
|
|
1174
|
+
...getIntegratedToolDefinitions(),
|
|
1121
1175
|
{
|
|
1122
1176
|
name: 'generate_table',
|
|
1123
1177
|
description: 'Format data into a markdown table. The resulting markdown will be returned to you. You MUST include it in your next message to the user so they can see it.',
|
|
@@ -1213,9 +1267,46 @@ function getAvailableTools(app, options = {}) {
|
|
|
1213
1267
|
},
|
|
1214
1268
|
required: ['query']
|
|
1215
1269
|
}
|
|
1270
|
+
},
|
|
1271
|
+
{
|
|
1272
|
+
name: 'social_video_extract',
|
|
1273
|
+
description: 'Extract title, description, transcript, and one representative frame image from a public social video URL (YouTube, TikTok, Instagram, or X) without social API keys.',
|
|
1274
|
+
parameters: {
|
|
1275
|
+
type: 'object',
|
|
1276
|
+
properties: {
|
|
1277
|
+
url: { type: 'string', description: 'Public social video URL.' },
|
|
1278
|
+
include_frame: { type: 'boolean', description: 'Whether to return one representative frame image artifact (default true).' },
|
|
1279
|
+
force_stt: { type: 'boolean', description: 'Force speech-to-text fallback even if captions are present.' }
|
|
1280
|
+
},
|
|
1281
|
+
required: ['url']
|
|
1282
|
+
}
|
|
1216
1283
|
}
|
|
1217
1284
|
];
|
|
1218
1285
|
|
|
1286
|
+
// task_complete — always available. Lets the AI explicitly signal that
|
|
1287
|
+
// the task is fully done and provide the final response. This replaces
|
|
1288
|
+
// the opaque directAnswerEligible heuristic as the primary loop-exit
|
|
1289
|
+
// mechanism and gives the AI real agency over when it's finished.
|
|
1290
|
+
tools.push({
|
|
1291
|
+
name: 'task_complete',
|
|
1292
|
+
description: 'Signal that the task is fully complete and provide the final response. Call this exactly once when all steps are done and you have a complete answer ready. Do NOT call it if you still have work to do.',
|
|
1293
|
+
parameters: {
|
|
1294
|
+
type: 'object',
|
|
1295
|
+
properties: {
|
|
1296
|
+
message: {
|
|
1297
|
+
type: 'string',
|
|
1298
|
+
description: 'Your complete final response to the user. Write it as if it were your reply — do not summarize or reference prior steps.'
|
|
1299
|
+
},
|
|
1300
|
+
confidence: {
|
|
1301
|
+
type: 'string',
|
|
1302
|
+
enum: ['high', 'medium', 'low'],
|
|
1303
|
+
description: 'How confident are you the task is fully and correctly complete? Use "low" if you had to make assumptions.'
|
|
1304
|
+
}
|
|
1305
|
+
},
|
|
1306
|
+
required: ['message']
|
|
1307
|
+
}
|
|
1308
|
+
});
|
|
1309
|
+
|
|
1219
1310
|
const allowInterimUpdates = (
|
|
1220
1311
|
(options.triggerSource === 'web' || options.triggerSource === 'messaging' || options.triggerSource === 'voice_live')
|
|
1221
1312
|
&& options.triggerType !== 'subagent'
|
|
@@ -1315,23 +1406,16 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1315
1406
|
if (manager && typeof manager.getBrowserProviderForUser === 'function') {
|
|
1316
1407
|
return manager.getBrowserProviderForUser(userId);
|
|
1317
1408
|
}
|
|
1318
|
-
|
|
1319
|
-
if (typeof scoped === 'function') {
|
|
1320
|
-
return scoped(userId);
|
|
1321
|
-
}
|
|
1322
|
-
return app?.locals?.browserController || engine.browserController;
|
|
1409
|
+
throw new Error('Browser provider is unavailable. VM runtime is required.');
|
|
1323
1410
|
};
|
|
1324
1411
|
const ac = () => {
|
|
1325
1412
|
const manager = runtime();
|
|
1326
1413
|
if (manager && typeof manager.getAndroidProviderForUser === 'function') {
|
|
1327
1414
|
return manager.getAndroidProviderForUser(userId);
|
|
1328
1415
|
}
|
|
1329
|
-
|
|
1330
|
-
if (typeof scoped === 'function') {
|
|
1331
|
-
return scoped(userId);
|
|
1332
|
-
}
|
|
1333
|
-
return app?.locals?.androidController || engine.androidController;
|
|
1416
|
+
throw new Error('Android provider is unavailable. VM runtime is required.');
|
|
1334
1417
|
};
|
|
1418
|
+
const wc = () => app?.locals?.workspaceManager || engine.workspaceManager || null;
|
|
1335
1419
|
const dc = () => {
|
|
1336
1420
|
const scoped = app?.locals?.getDesktopProviderForUser;
|
|
1337
1421
|
if (typeof scoped === 'function') {
|
|
@@ -1345,7 +1429,9 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1345
1429
|
const sk = () => app?.locals?.skillRunner || engine.skillRunner;
|
|
1346
1430
|
const taskRuntime = () => app?.locals?.taskRuntime || engine.taskRuntime;
|
|
1347
1431
|
const rec = () => app?.locals?.recordingManager || null;
|
|
1432
|
+
const socialVideo = () => app?.locals?.socialVideoService || null;
|
|
1348
1433
|
const widgets = () => app?.locals?.widgetService || null;
|
|
1434
|
+
const artifactStore = app?.locals?.artifactStore || null;
|
|
1349
1435
|
|
|
1350
1436
|
const integrationManager = integrations();
|
|
1351
1437
|
if (integrationManager) {
|
|
@@ -1370,35 +1456,37 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1370
1456
|
}
|
|
1371
1457
|
}
|
|
1372
1458
|
|
|
1459
|
+
const integratedToolResult = await executeIntegratedTool(toolName, args, {
|
|
1460
|
+
userId,
|
|
1461
|
+
agentId,
|
|
1462
|
+
cliExecutor: runtime() && typeof runtime().getCommandExecutorForUser === 'function'
|
|
1463
|
+
? await runtime().getCommandExecutorForUser(userId)
|
|
1464
|
+
: null,
|
|
1465
|
+
workspaceManager: wc(),
|
|
1466
|
+
artifactStore,
|
|
1467
|
+
});
|
|
1468
|
+
if (integratedToolResult !== null) {
|
|
1469
|
+
return integratedToolResult;
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1373
1472
|
switch (toolName) {
|
|
1473
|
+
// task_complete is handled at the engine loop level before executeTool
|
|
1474
|
+
// is called. If it somehow reaches here, return a no-op success so the
|
|
1475
|
+
// loop-level handler can still read the args from the tool call object.
|
|
1476
|
+
case 'task_complete':
|
|
1477
|
+
return { success: true, handled_by: 'engine_loop' };
|
|
1478
|
+
|
|
1374
1479
|
case 'execute_command': {
|
|
1375
1480
|
const runtimeManager = runtime();
|
|
1376
|
-
if (runtimeManager
|
|
1377
|
-
|
|
1378
|
-
return await runtimeManager.executeCommand(userId, args.command, {
|
|
1379
|
-
cwd: args.cwd,
|
|
1380
|
-
timeout: args.timeout || (args.pty ? 20 * 60 * 1000 : 15 * 60 * 1000),
|
|
1381
|
-
stdinInput: args.stdin_input,
|
|
1382
|
-
pty: args.pty === true,
|
|
1383
|
-
inputs: args.inputs || [],
|
|
1384
|
-
onSpawn,
|
|
1385
|
-
});
|
|
1481
|
+
if (!runtimeManager || typeof runtimeManager.executeCommand !== 'function') {
|
|
1482
|
+
return { error: 'Command execution is unavailable. VM runtime is required.' };
|
|
1386
1483
|
}
|
|
1387
|
-
|
|
1388
|
-
const executor = app?.locals?.cliExecutor || engine.cliExecutor || new CLIExecutor();
|
|
1389
|
-
const onSpawn = (pid) => engine.attachProcessToRun(runId, pid);
|
|
1390
|
-
if (args.pty) {
|
|
1391
|
-
return await executor.executeInteractive(args.command, args.inputs || [], {
|
|
1392
|
-
cwd: args.cwd,
|
|
1393
|
-
timeout: args.timeout || 20 * 60 * 1000,
|
|
1394
|
-
onSpawn
|
|
1395
|
-
});
|
|
1396
|
-
}
|
|
1397
|
-
return await executor.execute(args.command, {
|
|
1484
|
+
return await runtimeManager.executeCommand(userId, args.command, {
|
|
1398
1485
|
cwd: args.cwd,
|
|
1399
|
-
timeout: args.timeout || 15 * 60 * 1000,
|
|
1486
|
+
timeout: args.timeout || (args.pty ? 20 * 60 * 1000 : 15 * 60 * 1000),
|
|
1400
1487
|
stdinInput: args.stdin_input,
|
|
1401
|
-
|
|
1488
|
+
pty: args.pty === true,
|
|
1489
|
+
inputs: args.inputs || [],
|
|
1402
1490
|
});
|
|
1403
1491
|
}
|
|
1404
1492
|
|
|
@@ -1906,6 +1994,22 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1906
1994
|
};
|
|
1907
1995
|
}
|
|
1908
1996
|
|
|
1997
|
+
case 'social_video_extract': {
|
|
1998
|
+
const service = socialVideo();
|
|
1999
|
+
if (!service || typeof service.extractFromUrl !== 'function') {
|
|
2000
|
+
return { error: 'Social video extraction service is unavailable.' };
|
|
2001
|
+
}
|
|
2002
|
+
const sourceUrl = String(args.url || '').trim();
|
|
2003
|
+
if (!sourceUrl) {
|
|
2004
|
+
return { error: 'url is required' };
|
|
2005
|
+
}
|
|
2006
|
+
return await service.extractFromUrl(userId, sourceUrl, {
|
|
2007
|
+
includeFrame: args.include_frame !== false,
|
|
2008
|
+
forceStt: args.force_stt === true,
|
|
2009
|
+
agentId,
|
|
2010
|
+
});
|
|
2011
|
+
}
|
|
2012
|
+
|
|
1909
2013
|
case 'memory_write': {
|
|
1910
2014
|
const { MemoryManager } = require('../memory/manager');
|
|
1911
2015
|
const mm = new MemoryManager();
|
|
@@ -1986,6 +2090,25 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1986
2090
|
stripNoResponseMarker: false
|
|
1987
2091
|
});
|
|
1988
2092
|
const suppressReply = normalizedMessage === '[NO RESPONSE]';
|
|
2093
|
+
if (isProactiveTrigger(triggerSource)) {
|
|
2094
|
+
const proactiveValidation = validateProactiveSendMessageArgs({
|
|
2095
|
+
purpose: args.purpose,
|
|
2096
|
+
normalizedMessage,
|
|
2097
|
+
});
|
|
2098
|
+
if (!proactiveValidation.ok) {
|
|
2099
|
+
if (proactiveValidation.error) {
|
|
2100
|
+
return {
|
|
2101
|
+
error: proactiveValidation.error,
|
|
2102
|
+
};
|
|
2103
|
+
}
|
|
2104
|
+
return {
|
|
2105
|
+
sent: false,
|
|
2106
|
+
suppressed: proactiveValidation.suppressed === true,
|
|
2107
|
+
skipped: proactiveValidation.skipped === true,
|
|
2108
|
+
reason: proactiveValidation.reason,
|
|
2109
|
+
};
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
1989
2112
|
if (!suppressReply && hasAlreadySentProactiveMessage({
|
|
1990
2113
|
triggerSource,
|
|
1991
2114
|
runState,
|
|
@@ -2017,21 +2140,14 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2017
2140
|
|
|
2018
2141
|
case 'read_file': {
|
|
2019
2142
|
try {
|
|
2020
|
-
const
|
|
2021
|
-
if (
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
content: sliced.length > 20000 ? sliced.slice(0, 20000) + '\n...[truncated]' : sliced,
|
|
2029
|
-
totalLines: lines.length,
|
|
2030
|
-
rangeShown: [start + 1, Math.min(end, lines.length)]
|
|
2031
|
-
};
|
|
2032
|
-
}
|
|
2033
|
-
const content = fs.readFileSync(args.path, encoding);
|
|
2034
|
-
return { content: content.length > 20000 ? content.slice(0, 20000) + '\n...[truncated]' : content };
|
|
2143
|
+
const workspace = wc();
|
|
2144
|
+
if (!workspace) return { error: 'Workspace service is unavailable.' };
|
|
2145
|
+
return workspace.readFile(userId, {
|
|
2146
|
+
path: args.path,
|
|
2147
|
+
encoding: args.encoding || 'utf-8',
|
|
2148
|
+
start_line: args.start_line,
|
|
2149
|
+
end_line: args.end_line,
|
|
2150
|
+
});
|
|
2035
2151
|
} catch (err) {
|
|
2036
2152
|
return { error: err.message };
|
|
2037
2153
|
}
|
|
@@ -2039,14 +2155,13 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2039
2155
|
|
|
2040
2156
|
case 'write_file': {
|
|
2041
2157
|
try {
|
|
2042
|
-
const
|
|
2043
|
-
if (!
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
}
|
|
2049
|
-
return { success: true, path: args.path };
|
|
2158
|
+
const workspace = wc();
|
|
2159
|
+
if (!workspace) return { error: 'Workspace service is unavailable.' };
|
|
2160
|
+
return workspace.writeFile(userId, {
|
|
2161
|
+
path: args.path,
|
|
2162
|
+
content: args.content,
|
|
2163
|
+
mode: args.mode,
|
|
2164
|
+
});
|
|
2050
2165
|
} catch (err) {
|
|
2051
2166
|
return { error: err.message };
|
|
2052
2167
|
}
|
|
@@ -2054,23 +2169,12 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2054
2169
|
|
|
2055
2170
|
case 'edit_file': {
|
|
2056
2171
|
try {
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
if (content.includes(edit.oldText)) {
|
|
2064
|
-
content = content.replace(edit.oldText, edit.newText);
|
|
2065
|
-
modified = true;
|
|
2066
|
-
report.push({ success: true, edit: edit.oldText.slice(0, 50) + '...' });
|
|
2067
|
-
} else {
|
|
2068
|
-
report.push({ success: false, error: 'Target text not found', edit: edit.oldText.slice(0, 50) + '...' });
|
|
2069
|
-
}
|
|
2070
|
-
}
|
|
2071
|
-
|
|
2072
|
-
if (modified) fs.writeFileSync(args.path, content);
|
|
2073
|
-
return { success: modified, report, path: args.path };
|
|
2172
|
+
const workspace = wc();
|
|
2173
|
+
if (!workspace) return { error: 'Workspace service is unavailable.' };
|
|
2174
|
+
return workspace.editFile(userId, {
|
|
2175
|
+
path: args.path,
|
|
2176
|
+
edits: args.edits,
|
|
2177
|
+
});
|
|
2074
2178
|
} catch (err) {
|
|
2075
2179
|
return { error: err.message };
|
|
2076
2180
|
}
|
|
@@ -2078,28 +2182,13 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2078
2182
|
|
|
2079
2183
|
case 'list_directory': {
|
|
2080
2184
|
try {
|
|
2081
|
-
const
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
const item = {
|
|
2089
|
-
name: e.name,
|
|
2090
|
-
type: e.isDirectory() ? 'directory' : 'file',
|
|
2091
|
-
path: fullPath,
|
|
2092
|
-
size: stats.size,
|
|
2093
|
-
mtime: stats.mtime.toISOString()
|
|
2094
|
-
};
|
|
2095
|
-
result.push(item);
|
|
2096
|
-
if (e.isDirectory() && currentDepth < maxDepth && !e.name.startsWith('.') && e.name !== 'node_modules') {
|
|
2097
|
-
result.push(...recurse(fullPath, currentDepth + 1));
|
|
2098
|
-
}
|
|
2099
|
-
}
|
|
2100
|
-
return result;
|
|
2101
|
-
};
|
|
2102
|
-
return { entries: recurse(args.path) };
|
|
2185
|
+
const workspace = wc();
|
|
2186
|
+
if (!workspace) return { error: 'Workspace service is unavailable.' };
|
|
2187
|
+
return workspace.listDirectory(userId, {
|
|
2188
|
+
path: args.path,
|
|
2189
|
+
depth: args.depth,
|
|
2190
|
+
recursive: args.recursive,
|
|
2191
|
+
});
|
|
2103
2192
|
} catch (err) {
|
|
2104
2193
|
return { error: err.message };
|
|
2105
2194
|
}
|
|
@@ -2107,23 +2196,13 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2107
2196
|
|
|
2108
2197
|
case 'search_files': {
|
|
2109
2198
|
try {
|
|
2110
|
-
const
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
const lines = (result.stdout || '').split('\n').filter(Boolean);
|
|
2118
|
-
const matches = lines.map(line => {
|
|
2119
|
-
const parts = line.split(':');
|
|
2120
|
-
return {
|
|
2121
|
-
file: parts[0],
|
|
2122
|
-
line: parseInt(parts[1]),
|
|
2123
|
-
content: parts.slice(2).join(':').trim()
|
|
2124
|
-
};
|
|
2199
|
+
const workspace = wc();
|
|
2200
|
+
if (!workspace) return { error: 'Workspace service is unavailable.' };
|
|
2201
|
+
return workspace.searchFiles(userId, {
|
|
2202
|
+
path: args.path,
|
|
2203
|
+
query: args.query,
|
|
2204
|
+
include: args.include,
|
|
2125
2205
|
});
|
|
2126
|
-
return { matches, count: matches.length };
|
|
2127
2206
|
} catch (err) {
|
|
2128
2207
|
return { error: err.message };
|
|
2129
2208
|
}
|
|
@@ -2488,7 +2567,17 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2488
2567
|
if (!widgetService) return { error: 'Widget service not available' };
|
|
2489
2568
|
if (!widgetId) return { error: 'save_widget_snapshot is only available during widget refresh runs.' };
|
|
2490
2569
|
try {
|
|
2491
|
-
const
|
|
2570
|
+
const snapshotPayload = (
|
|
2571
|
+
args
|
|
2572
|
+
&& typeof args === 'object'
|
|
2573
|
+
&& !Array.isArray(args)
|
|
2574
|
+
&& args.snapshot
|
|
2575
|
+
&& typeof args.snapshot === 'object'
|
|
2576
|
+
&& !Array.isArray(args.snapshot)
|
|
2577
|
+
)
|
|
2578
|
+
? args.snapshot
|
|
2579
|
+
: args;
|
|
2580
|
+
const snapshot = widgetService.saveSnapshot(userId, widgetId, snapshotPayload, {
|
|
2492
2581
|
sourceRunId: runId,
|
|
2493
2582
|
status: 'ready',
|
|
2494
2583
|
});
|
|
@@ -2676,7 +2765,12 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2676
2765
|
const { detectPromptInjection } = require('../../utils/security');
|
|
2677
2766
|
const mcpManager = mcp();
|
|
2678
2767
|
if (mcpManager) {
|
|
2679
|
-
|
|
2768
|
+
let mcpResult = null;
|
|
2769
|
+
try {
|
|
2770
|
+
mcpResult = await mcpManager.callToolByName(toolName, args, userId, { agentId });
|
|
2771
|
+
} catch (mcpErr) {
|
|
2772
|
+
return { error: mcpErr.message, tool: toolName, source: 'mcp' };
|
|
2773
|
+
}
|
|
2680
2774
|
if (mcpResult !== null) {
|
|
2681
2775
|
const resultText = typeof mcpResult === 'string' ? mcpResult : JSON.stringify(mcpResult);
|
|
2682
2776
|
if (detectPromptInjection(resultText)) {
|
|
@@ -2701,4 +2795,4 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2701
2795
|
}
|
|
2702
2796
|
}
|
|
2703
2797
|
|
|
2704
|
-
module.exports = { getAvailableTools, executeTool };
|
|
2798
|
+
module.exports = { getAvailableTools, executeTool, validateProactiveSendMessageArgs };
|