neoagent 2.3.1-beta.2 → 2.3.1-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. package/.env.example +39 -0
  2. package/README.md +2 -0
  3. package/docs/capabilities.md +2 -2
  4. package/docs/configuration.md +13 -5
  5. package/docs/integrations.md +4 -1
  6. package/flutter_app/.metadata +42 -0
  7. package/flutter_app/README.md +21 -0
  8. package/flutter_app/analysis_options.yaml +32 -0
  9. package/flutter_app/android/app/build.gradle.kts +109 -0
  10. package/flutter_app/android/app/src/debug/AndroidManifest.xml +7 -0
  11. package/flutter_app/android/app/src/main/AndroidManifest.xml +147 -0
  12. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +747 -0
  13. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthConnectGateway.kt +280 -0
  14. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncNotifications.kt +113 -0
  15. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncPayload.kt +57 -0
  16. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncScheduler.kt +78 -0
  17. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/HealthSyncWorker.kt +253 -0
  18. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/health/PermissionsRationaleActivity.kt +46 -0
  19. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingBootReceiver.kt +21 -0
  20. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingForegroundService.kt +586 -0
  21. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingStateStore.kt +78 -0
  22. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/recording/RecordingUploadClient.kt +104 -0
  23. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiHomeWidgetProvider.kt +457 -0
  24. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/AiWidgetStore.kt +194 -0
  25. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/VoiceLaunchWidgetProvider.kt +67 -0
  26. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetConfigActivity.kt +228 -0
  27. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncScheduler.kt +72 -0
  28. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetSyncWorker.kt +186 -0
  29. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/widgets/WidgetTaskRunWorker.kt +210 -0
  30. package/flutter_app/android/app/src/main/res/drawable/launch_background.xml +12 -0
  31. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_bg.xml +11 -0
  32. package/flutter_app/android/app/src/main/res/drawable/neoagent_ai_widget_task_bg.xml +8 -0
  33. package/flutter_app/android/app/src/main/res/drawable-v21/launch_background.xml +12 -0
  34. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget.xml +138 -0
  35. package/flutter_app/android/app/src/main/res/layout/neoagent_ai_widget_task_row.xml +52 -0
  36. package/flutter_app/android/app/src/main/res/layout/neoagent_voice_widget.xml +49 -0
  37. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  38. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  39. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  40. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  41. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  42. package/flutter_app/android/app/src/main/res/values/strings.xml +12 -0
  43. package/flutter_app/android/app/src/main/res/values/styles.xml +18 -0
  44. package/flutter_app/android/app/src/main/res/values-night/styles.xml +18 -0
  45. package/flutter_app/android/app/src/main/res/xml/file_paths.xml +6 -0
  46. package/flutter_app/android/app/src/main/res/xml/neoagent_ai_widget_info.xml +12 -0
  47. package/flutter_app/android/app/src/main/res/xml/neoagent_voice_widget_info.xml +12 -0
  48. package/flutter_app/android/app/src/profile/AndroidManifest.xml +7 -0
  49. package/flutter_app/android/build.gradle.kts +24 -0
  50. package/flutter_app/android/ci-release.keystore +0 -0
  51. package/flutter_app/android/gradle/wrapper/gradle-wrapper.properties +5 -0
  52. package/flutter_app/android/gradle.properties +3 -0
  53. package/flutter_app/android/key.properties +4 -0
  54. package/flutter_app/android/settings.gradle.kts +26 -0
  55. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  56. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  57. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  58. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  59. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  60. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  61. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  62. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  63. package/flutter_app/lib/features/location/location_service.dart +119 -0
  64. package/flutter_app/lib/features/notifications/notification_interceptor.dart +97 -0
  65. package/flutter_app/lib/main.dart +23057 -0
  66. package/flutter_app/lib/main_app_shell.dart +1682 -0
  67. package/flutter_app/lib/main_integrations.dart +931 -0
  68. package/flutter_app/lib/main_launcher.dart +959 -0
  69. package/flutter_app/lib/main_launcher_entry.dart +5 -0
  70. package/flutter_app/lib/main_models.dart +3473 -0
  71. package/flutter_app/lib/main_shared.dart +2861 -0
  72. package/flutter_app/lib/main_theme.dart +204 -0
  73. package/flutter_app/lib/main_voice_assistant.dart +831 -0
  74. package/flutter_app/lib/src/android_apk_drop_zone.dart +32 -0
  75. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +16 -0
  76. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +348 -0
  77. package/flutter_app/lib/src/android_app_installer.dart +22 -0
  78. package/flutter_app/lib/src/android_app_installer_io.dart +122 -0
  79. package/flutter_app/lib/src/android_app_installer_stub.dart +21 -0
  80. package/flutter_app/lib/src/android_launcher_bridge.dart +239 -0
  81. package/flutter_app/lib/src/app_launch_bridge.dart +29 -0
  82. package/flutter_app/lib/src/app_release_updater.dart +511 -0
  83. package/flutter_app/lib/src/backend_client.dart +1833 -0
  84. package/flutter_app/lib/src/desktop_companion.dart +2 -0
  85. package/flutter_app/lib/src/desktop_companion_actions.dart +586 -0
  86. package/flutter_app/lib/src/desktop_companion_io.dart +538 -0
  87. package/flutter_app/lib/src/desktop_companion_stub.dart +59 -0
  88. package/flutter_app/lib/src/desktop_native_bridge.dart +91 -0
  89. package/flutter_app/lib/src/desktop_screen_capture.dart +21 -0
  90. package/flutter_app/lib/src/desktop_screen_capture_io.dart +142 -0
  91. package/flutter_app/lib/src/desktop_screen_capture_stub.dart +12 -0
  92. package/flutter_app/lib/src/diagnostics_logger.dart +119 -0
  93. package/flutter_app/lib/src/health_bridge.dart +136 -0
  94. package/flutter_app/lib/src/live_voice_capture.dart +85 -0
  95. package/flutter_app/lib/src/messaging_access_summary.dart +46 -0
  96. package/flutter_app/lib/src/network/app_http_client.dart +53 -0
  97. package/flutter_app/lib/src/network/app_http_client_factory.dart +6 -0
  98. package/flutter_app/lib/src/network/app_http_client_io.dart +138 -0
  99. package/flutter_app/lib/src/network/app_http_client_stub.dart +3 -0
  100. package/flutter_app/lib/src/network/app_http_client_web.dart +94 -0
  101. package/flutter_app/lib/src/oauth_launcher.dart +33 -0
  102. package/flutter_app/lib/src/oauth_launcher_io.dart +77 -0
  103. package/flutter_app/lib/src/oauth_launcher_stub.dart +33 -0
  104. package/flutter_app/lib/src/oauth_launcher_web.dart +107 -0
  105. package/flutter_app/lib/src/recording_bridge.dart +232 -0
  106. package/flutter_app/lib/src/recording_bridge_io.dart +1019 -0
  107. package/flutter_app/lib/src/recording_bridge_stub.dart +120 -0
  108. package/flutter_app/lib/src/recording_bridge_web.dart +689 -0
  109. package/flutter_app/lib/src/recording_payloads.dart +86 -0
  110. package/flutter_app/lib/src/theme/palette.dart +81 -0
  111. package/flutter_app/lib/src/widget_bridge.dart +49 -0
  112. package/flutter_app/linux/CMakeLists.txt +128 -0
  113. package/flutter_app/linux/flutter/CMakeLists.txt +88 -0
  114. package/flutter_app/linux/flutter/generated_plugin_registrant.cc +43 -0
  115. package/flutter_app/linux/flutter/generated_plugin_registrant.h +15 -0
  116. package/flutter_app/linux/flutter/generated_plugins.cmake +31 -0
  117. package/flutter_app/linux/runner/CMakeLists.txt +26 -0
  118. package/flutter_app/linux/runner/main.cc +6 -0
  119. package/flutter_app/linux/runner/my_application.cc +144 -0
  120. package/flutter_app/linux/runner/my_application.h +18 -0
  121. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  122. package/flutter_app/macos/Flutter/Flutter-Debug.xcconfig +2 -0
  123. package/flutter_app/macos/Flutter/Flutter-Release.xcconfig +2 -0
  124. package/flutter_app/macos/Flutter/GeneratedPluginRegistrant.swift +40 -0
  125. package/flutter_app/macos/Podfile +42 -0
  126. package/flutter_app/macos/Podfile.lock +87 -0
  127. package/flutter_app/macos/Runner/AppDelegate.swift +576 -0
  128. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json +68 -0
  129. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  130. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  131. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  132. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  133. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  134. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  135. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  136. package/flutter_app/macos/Runner/Base.lproj/MainMenu.xib +342 -0
  137. package/flutter_app/macos/Runner/Configs/AppInfo.xcconfig +14 -0
  138. package/flutter_app/macos/Runner/Configs/Debug.xcconfig +2 -0
  139. package/flutter_app/macos/Runner/Configs/Release.xcconfig +2 -0
  140. package/flutter_app/macos/Runner/Configs/Warnings.xcconfig +13 -0
  141. package/flutter_app/macos/Runner/DebugProfile.entitlements +16 -0
  142. package/flutter_app/macos/Runner/Info.plist +36 -0
  143. package/flutter_app/macos/Runner/MainFlutterWindow.swift +19 -0
  144. package/flutter_app/macos/Runner/Release.entitlements +12 -0
  145. package/flutter_app/macos/Runner.xcodeproj/project.pbxproj +801 -0
  146. package/flutter_app/macos/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  147. package/flutter_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +99 -0
  148. package/flutter_app/macos/Runner.xcworkspace/contents.xcworkspacedata +10 -0
  149. package/flutter_app/macos/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -0
  150. package/flutter_app/macos/RunnerTests/RunnerTests.swift +12 -0
  151. package/flutter_app/patch_strings.py +12 -0
  152. package/flutter_app/pubspec.lock +1088 -0
  153. package/flutter_app/pubspec.yaml +53 -0
  154. package/flutter_app/test/messaging_access_summary_test.dart +22 -0
  155. package/flutter_app/test/recording_payloads_test.dart +53 -0
  156. package/flutter_app/third_party/desktop_audio_capture/LICENSE +21 -0
  157. package/flutter_app/third_party/desktop_audio_capture/README.md +262 -0
  158. package/flutter_app/third_party/desktop_audio_capture/lib/audio_capture.dart +65 -0
  159. package/flutter_app/third_party/desktop_audio_capture/lib/config/mic_audio_config.dart +153 -0
  160. package/flutter_app/third_party/desktop_audio_capture/lib/config/system_adudio_config.dart +110 -0
  161. package/flutter_app/third_party/desktop_audio_capture/lib/mic/mic_audio_capture.dart +461 -0
  162. package/flutter_app/third_party/desktop_audio_capture/lib/model/audio_status.dart +91 -0
  163. package/flutter_app/third_party/desktop_audio_capture/lib/model/decibel_data.dart +106 -0
  164. package/flutter_app/third_party/desktop_audio_capture/lib/model/input_device_type.dart +219 -0
  165. package/flutter_app/third_party/desktop_audio_capture/lib/system/system_audio_capture.dart +336 -0
  166. package/flutter_app/third_party/desktop_audio_capture/linux/CMakeLists.txt +101 -0
  167. package/flutter_app/third_party/desktop_audio_capture/linux/audio_capture_plugin.cc +692 -0
  168. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/audio_capture_plugin.h +35 -0
  169. package/flutter_app/third_party/desktop_audio_capture/linux/include/audio_capture/mic_capture_plugin.h +36 -0
  170. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/audio_capture_plugin.h +32 -0
  171. package/flutter_app/third_party/desktop_audio_capture/linux/include/desktop_audio_capture/mic_capture_plugin.h +32 -0
  172. package/flutter_app/third_party/desktop_audio_capture/linux/mic_capture_plugin.cc +878 -0
  173. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/AudioCapturePlugin.swift +27 -0
  174. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/MicCapturePlugin.swift +1172 -0
  175. package/flutter_app/third_party/desktop_audio_capture/macos/Classes/SystemCapturePlugin.swift +655 -0
  176. package/flutter_app/third_party/desktop_audio_capture/macos/Resources/PrivacyInfo.xcprivacy +12 -0
  177. package/flutter_app/third_party/desktop_audio_capture/macos/desktop_audio_capture.podspec +30 -0
  178. package/flutter_app/third_party/desktop_audio_capture/pubspec.yaml +87 -0
  179. package/flutter_app/third_party/desktop_audio_capture/windows/CMakeLists.txt +105 -0
  180. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.cpp +80 -0
  181. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin.h +31 -0
  182. package/flutter_app/third_party/desktop_audio_capture/windows/audio_capture_plugin_c_api.cpp +12 -0
  183. package/flutter_app/third_party/desktop_audio_capture/windows/include/audio_capture/audio_capture_plugin_c_api.h +23 -0
  184. package/flutter_app/third_party/desktop_audio_capture/windows/include/desktop_audio_capture/audio_capture_plugin.h +25 -0
  185. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.cpp +1117 -0
  186. package/flutter_app/third_party/desktop_audio_capture/windows/mic_capture_plugin.h +115 -0
  187. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.cpp +777 -0
  188. package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h +87 -0
  189. package/flutter_app/third_party/flutter_secure_storage_linux/linux/CMakeLists.txt +30 -0
  190. package/flutter_app/third_party/flutter_secure_storage_linux/linux/flutter_secure_storage_linux_plugin.cc +215 -0
  191. package/flutter_app/third_party/flutter_secure_storage_linux/linux/include/flutter_secure_storage_linux/flutter_secure_storage_linux_plugin.h +27 -0
  192. package/flutter_app/third_party/flutter_secure_storage_linux/pubspec.yaml +20 -0
  193. package/flutter_app/tool/generate_desktop_branding.py +219 -0
  194. package/flutter_app/web/favicon.png +0 -0
  195. package/flutter_app/web/favicon.svg +12 -0
  196. package/flutter_app/web/icons/Icon-192.png +0 -0
  197. package/flutter_app/web/icons/Icon-512.png +0 -0
  198. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  199. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  200. package/flutter_app/web/index.html +39 -0
  201. package/flutter_app/web/manifest.json +35 -0
  202. package/flutter_app/windows/CMakeLists.txt +108 -0
  203. package/flutter_app/windows/flutter/CMakeLists.txt +109 -0
  204. package/flutter_app/windows/flutter/generated_plugin_registrant.cc +47 -0
  205. package/flutter_app/windows/flutter/generated_plugin_registrant.h +15 -0
  206. package/flutter_app/windows/flutter/generated_plugins.cmake +35 -0
  207. package/flutter_app/windows/runner/CMakeLists.txt +41 -0
  208. package/flutter_app/windows/runner/Runner.rc +121 -0
  209. package/flutter_app/windows/runner/flutter_window.cpp +533 -0
  210. package/flutter_app/windows/runner/flutter_window.h +37 -0
  211. package/flutter_app/windows/runner/main.cpp +53 -0
  212. package/flutter_app/windows/runner/resource.h +16 -0
  213. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  214. package/flutter_app/windows/runner/runner.exe.manifest +14 -0
  215. package/flutter_app/windows/runner/utils.cpp +65 -0
  216. package/flutter_app/windows/runner/utils.h +19 -0
  217. package/flutter_app/windows/runner/win32_window.cpp +299 -0
  218. package/flutter_app/windows/runner/win32_window.h +102 -0
  219. package/lib/manager.js +231 -7
  220. package/package.json +3 -1
  221. package/server/db/database.js +68 -0
  222. package/server/http/middleware.js +50 -0
  223. package/server/http/routes.js +3 -1
  224. package/server/index.js +1 -0
  225. package/server/public/.last_build_id +1 -1
  226. package/server/public/assets/NOTICES +61 -0
  227. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  228. package/server/public/flutter_bootstrap.js +1 -1
  229. package/server/public/main.dart.js +65262 -64422
  230. package/server/routes/integrations.js +86 -0
  231. package/server/routes/memory.js +11 -2
  232. package/server/routes/screenHistory.js +46 -0
  233. package/server/routes/triggers.js +81 -0
  234. package/server/services/ai/models.js +30 -0
  235. package/server/services/ai/providers/githubCopilot.js +97 -0
  236. package/server/services/ai/providers/openai.js +2 -1
  237. package/server/services/ai/providers/openaiCodex.js +31 -0
  238. package/server/services/ai/settings.js +20 -0
  239. package/server/services/ai/systemPrompt.js +1 -1
  240. package/server/services/ai/tools.js +35 -6
  241. package/server/services/browser/controller.js +47 -3
  242. package/server/services/desktop/screenRecorder.js +172 -0
  243. package/server/services/integrations/env.js +5 -0
  244. package/server/services/integrations/github/common.js +106 -0
  245. package/server/services/integrations/github/provider.js +499 -0
  246. package/server/services/integrations/github/repos.js +1124 -0
  247. package/server/services/integrations/home_assistant/provider.js +306 -26
  248. package/server/services/integrations/manager.js +63 -7
  249. package/server/services/integrations/oauth_provider.js +13 -6
  250. package/server/services/integrations/provider_config_store.js +76 -0
  251. package/server/services/integrations/registry.js +4 -0
  252. package/server/services/integrations/trello/provider.js +744 -0
  253. package/server/services/integrations/whatsapp/provider.js +6 -2
  254. package/server/services/manager.js +22 -0
  255. package/server/services/memory/manager.js +39 -2
  256. package/server/services/skills/base_catalog.js +33 -0
  257. package/server/services/tasks/adapters/index.js +1 -0
  258. package/server/services/tasks/adapters/manual.js +12 -0
  259. package/server/services/tasks/runtime.js +1 -1
  260. package/server/services/voice/openaiClient.js +4 -1
  261. package/server/services/voice/providers.js +2 -1
  262. package/server/services/widgets/service.js +49 -4
  263. package/server/utils/local_secrets.js +56 -0
  264. package/server/utils/logger.js +37 -9
@@ -361,4 +361,90 @@ router.get('/:provider/tools/status', (req, res) => {
361
361
  }
362
362
  });
363
363
 
364
+ router.get('/:provider/config', (req, res) => {
365
+ try {
366
+ const manager = getIntegrationManager(req);
367
+ if (!manager) {
368
+ throw new Error('Official integration manager is not available on app.locals.integrationManager.');
369
+ }
370
+ const provider = manager.getProvider(req.params.provider);
371
+ if (!provider) {
372
+ throw new Error(`Unknown integration provider: ${req.params.provider}`);
373
+ }
374
+ if (typeof provider.getUserConfig !== 'function') {
375
+ return res.status(404).json({
376
+ error: `${provider.label} does not support per-user configuration.`,
377
+ });
378
+ }
379
+ const config = provider.getUserConfig({
380
+ userId: req.session.userId,
381
+ agentId: resolveAgentId(req.session.userId, getAgentIdFromRequest(req)),
382
+ });
383
+ res.json({
384
+ provider: provider.key,
385
+ config,
386
+ });
387
+ } catch (err) {
388
+ res.status(400).json({ error: sanitizeError(err) });
389
+ }
390
+ });
391
+
392
+ router.put('/:provider/config', async (req, res) => {
393
+ try {
394
+ const manager = getIntegrationManager(req);
395
+ if (!manager) {
396
+ throw new Error('Official integration manager is not available on app.locals.integrationManager.');
397
+ }
398
+ const provider = manager.getProvider(req.params.provider);
399
+ if (!provider) {
400
+ throw new Error(`Unknown integration provider: ${req.params.provider}`);
401
+ }
402
+ if (typeof provider.saveUserConfig !== 'function') {
403
+ return res.status(404).json({
404
+ error: `${provider.label} does not support per-user configuration.`,
405
+ });
406
+ }
407
+ const config = await provider.saveUserConfig({
408
+ userId: req.session.userId,
409
+ agentId: resolveAgentId(req.session.userId, getAgentIdFromRequest(req)),
410
+ config: req.body?.config || req.body || {},
411
+ });
412
+ res.json({
413
+ provider: provider.key,
414
+ config,
415
+ saved: true,
416
+ });
417
+ } catch (err) {
418
+ res.status(400).json({ error: sanitizeError(err) });
419
+ }
420
+ });
421
+
422
+ router.delete('/:provider/config', async (req, res) => {
423
+ try {
424
+ const manager = getIntegrationManager(req);
425
+ if (!manager) {
426
+ throw new Error('Official integration manager is not available on app.locals.integrationManager.');
427
+ }
428
+ const provider = manager.getProvider(req.params.provider);
429
+ if (!provider) {
430
+ throw new Error(`Unknown integration provider: ${req.params.provider}`);
431
+ }
432
+ if (typeof provider.clearUserConfig !== 'function') {
433
+ return res.status(404).json({
434
+ error: `${provider.label} does not support per-user configuration.`,
435
+ });
436
+ }
437
+ await provider.clearUserConfig({
438
+ userId: req.session.userId,
439
+ agentId: resolveAgentId(req.session.userId, getAgentIdFromRequest(req)),
440
+ });
441
+ res.json({
442
+ provider: provider.key,
443
+ cleared: true,
444
+ });
445
+ } catch (err) {
446
+ res.status(400).json({ error: sanitizeError(err) });
447
+ }
448
+ });
449
+
364
450
  module.exports = router;
@@ -1,11 +1,20 @@
1
1
  const express = require('express');
2
2
  const router = express.Router();
3
+ const rateLimit = require('express-rate-limit');
3
4
  const { requireAuth } = require('../middleware/auth');
4
5
  const { sanitizeError } = require('../utils/security');
5
6
  const { getAgentIdFromRequest, resolveAgentId } = require('../services/agents/manager');
6
7
 
7
8
  router.use(requireAuth);
8
9
 
10
+ const apiKeyMutationLimiter = rateLimit({
11
+ windowMs: 15 * 60 * 1000,
12
+ max: 60,
13
+ message: { error: 'Too many API key update attempts, try again later' },
14
+ standardHeaders: true,
15
+ legacyHeaders: false,
16
+ });
17
+
9
18
  function normalizeMemoryIds(value) {
10
19
  return [...new Set(
11
20
  (Array.isArray(value) ? value : [])
@@ -232,12 +241,12 @@ router.get('/api-keys', (req, res) => {
232
241
  res.json(masked);
233
242
  });
234
243
 
235
- router.put('/api-keys/:service', (req, res) => {
244
+ router.put('/api-keys/:service', apiKeyMutationLimiter, (req, res) => {
236
245
  req.app.locals.memoryManager.setApiKey(req.params.service, req.body.key, req.session.userId);
237
246
  res.json({ success: true });
238
247
  });
239
248
 
240
- router.delete('/api-keys/:service', (req, res) => {
249
+ router.delete('/api-keys/:service', apiKeyMutationLimiter, (req, res) => {
241
250
  req.app.locals.memoryManager.deleteApiKey(req.params.service, req.session.userId);
242
251
  res.json({ success: true });
243
252
  });
@@ -0,0 +1,46 @@
1
+ 'use strict';
2
+
3
+ const express = require('express');
4
+ const db = require('../db/database');
5
+ const { getErrorMessage } = require('../services/bootstrap_helpers');
6
+
7
+ const router = express.Router();
8
+
9
+ router.get('/search', (req, res) => {
10
+ const { q, limit = 50, offset = 0 } = req.query;
11
+
12
+ if (!req.user || !req.user.id) {
13
+ return res.status(401).json({ error: 'Unauthorized' });
14
+ }
15
+
16
+ try {
17
+ let results = [];
18
+ if (q) {
19
+ // Full text search
20
+ results = db.prepare(`
21
+ SELECT s.id, s.timestamp, s.app_name, s.text_content
22
+ FROM screen_history_fts fts
23
+ JOIN screen_history s ON fts.rowid = s.id
24
+ WHERE screen_history_fts MATCH ? AND s.user_id = ?
25
+ ORDER BY s.timestamp DESC
26
+ LIMIT ? OFFSET ?
27
+ `).all(q, req.user.id, Number(limit), Number(offset));
28
+ } else {
29
+ // Recent history
30
+ results = db.prepare(`
31
+ SELECT id, timestamp, app_name, text_content
32
+ FROM screen_history
33
+ WHERE user_id = ?
34
+ ORDER BY timestamp DESC
35
+ LIMIT ? OFFSET ?
36
+ `).all(req.user.id, Number(limit), Number(offset));
37
+ }
38
+
39
+ res.json({ results });
40
+ } catch (err) {
41
+ console.error('[ScreenHistory] Search error:', getErrorMessage(err));
42
+ res.status(500).json({ error: 'Failed to search screen history' });
43
+ }
44
+ });
45
+
46
+ module.exports = router;
@@ -0,0 +1,81 @@
1
+ 'use strict';
2
+
3
+ const express = require('express');
4
+ const db = require('../db/database');
5
+ const { getErrorMessage } = require('../services/bootstrap_helpers');
6
+ const { requireAuth } = require('../middleware/auth');
7
+
8
+ const router = express.Router();
9
+
10
+ router.use(requireAuth);
11
+
12
+ router.post('/geofence', async (req, res) => {
13
+ const { label, latitude, longitude, radius_meters, action } = req.body;
14
+
15
+ try {
16
+ const userRow = db.prepare('SELECT id FROM users WHERE id = ?').get(req.user.id);
17
+ if (!userRow) return res.status(401).json({ error: 'Unauthorized' });
18
+
19
+ console.log(`[Triggers] Geofence entered: ${label} by user ${req.user.id}`);
20
+
21
+ // If an agentEngine is running, we can inject a prompt to process this context
22
+ const agentEngine = req.app.locals.agentEngine;
23
+ if (agentEngine) {
24
+ // Find active agent or use default
25
+ const defaultAgentId = db.prepare('SELECT id FROM agents WHERE user_id = ? ORDER BY is_default DESC LIMIT 1').get(req.user.id)?.id;
26
+
27
+ if (defaultAgentId) {
28
+ // Fire and forget a trigger message to the agent
29
+ agentEngine.handleBackgroundTrigger(req.user.id, defaultAgentId, {
30
+ source: 'geofence',
31
+ label,
32
+ action: action || 'User entered a geofenced area. Check if there are any active reminders or tasks related to this location.'
33
+ }).catch(err => console.error('[Triggers] Agent evaluation failed:', err));
34
+ }
35
+ }
36
+
37
+ res.json({ success: true, message: 'Geofence trigger processed' });
38
+ } catch (err) {
39
+ console.error('[Triggers] Geofence error:', getErrorMessage(err));
40
+ res.status(500).json({ error: 'Failed to process geofence trigger' });
41
+ }
42
+ });
43
+
44
+ router.post('/notification', async (req, res) => {
45
+ const { app_package, title, body, action_taken } = req.body;
46
+
47
+ try {
48
+ const userRow = db.prepare('SELECT id FROM users WHERE id = ?').get(req.user.id);
49
+ if (!userRow) return res.status(401).json({ error: 'Unauthorized' });
50
+
51
+ console.log(`[Triggers] Notification received: ${app_package} - ${title}`);
52
+
53
+ db.prepare(`
54
+ INSERT INTO notification_history (user_id, app_package, title, body, action_taken)
55
+ VALUES (?, ?, ?, ?, ?)
56
+ `).run(req.user.id, app_package || 'unknown', title || '', body || '', action_taken || 'none');
57
+
58
+ // Notify agent engine to proactively evaluate the notification
59
+ const agentEngine = req.app.locals.agentEngine;
60
+ if (agentEngine) {
61
+ const defaultAgentId = db.prepare('SELECT id FROM agents WHERE user_id = ? ORDER BY is_default DESC LIMIT 1').get(req.user.id)?.id;
62
+
63
+ if (defaultAgentId) {
64
+ agentEngine.handleBackgroundTrigger(req.user.id, defaultAgentId, {
65
+ source: 'notification',
66
+ app_package,
67
+ title,
68
+ body,
69
+ instruction: 'Evaluate this notification. If it is an important reminder, calendar event, or urgent message, inform the user or take appropriate action.'
70
+ }).catch(err => console.error('[Triggers] Agent evaluation failed:', err));
71
+ }
72
+ }
73
+
74
+ res.json({ success: true, message: 'Notification trigger processed and stored' });
75
+ } catch (err) {
76
+ console.error('[Triggers] Notification error:', getErrorMessage(err));
77
+ res.status(500).json({ error: 'Failed to process notification trigger' });
78
+ }
79
+ });
80
+
81
+ module.exports = router;
@@ -3,6 +3,8 @@ const { GoogleProvider } = require('./providers/google');
3
3
  const { GrokProvider } = require('./providers/grok');
4
4
  const { OllamaProvider } = require('./providers/ollama');
5
5
  const { OpenAIProvider } = require('./providers/openai');
6
+ const { GithubCopilotProvider } = require('./providers/githubCopilot');
7
+ const { OpenAICodexProvider } = require('./providers/openaiCodex');
6
8
  const {
7
9
  AI_PROVIDER_DEFINITIONS,
8
10
  getProviderConfigs,
@@ -16,6 +18,30 @@ const STATIC_MODELS = [
16
18
  provider: 'grok',
17
19
  purpose: 'general'
18
20
  },
21
+ {
22
+ id: 'gpt-5.3',
23
+ label: 'GPT-5.3 (Copilot Default)',
24
+ provider: 'github-copilot',
25
+ purpose: 'general'
26
+ },
27
+ {
28
+ id: 'gpt-4.1',
29
+ label: 'GPT-4.1 (Copilot Fast)',
30
+ provider: 'github-copilot',
31
+ purpose: 'coding'
32
+ },
33
+ {
34
+ id: 'gpt-5.3-codex',
35
+ label: 'GPT-5.3 (Codex Default)',
36
+ provider: 'openai-codex',
37
+ purpose: 'general'
38
+ },
39
+ {
40
+ id: 'gpt-4.1-codex',
41
+ label: 'GPT-4.1 (Codex Fast)',
42
+ provider: 'openai-codex',
43
+ purpose: 'coding'
44
+ },
19
45
  {
20
46
  id: 'gpt-5-nano',
21
47
  label: 'GPT-5 Nano (Fast / Subagents)',
@@ -302,6 +328,10 @@ function createProviderInstance(providerStr, userId = null, configOverrides = {}
302
328
  return new AnthropicProvider({ apiKey: runtime.apiKey, baseUrl: runtime.baseUrl, ...providerOverrides });
303
329
  } else if (providerStr === 'ollama') {
304
330
  return new OllamaProvider({ baseUrl: runtime.baseUrl, ...providerOverrides });
331
+ } else if (providerStr === 'github-copilot') {
332
+ return new GithubCopilotProvider({ apiKey: runtime.apiKey, ...providerOverrides });
333
+ } else if (providerStr === 'openai-codex') {
334
+ return new OpenAICodexProvider({ apiKey: runtime.apiKey, ...providerOverrides });
305
335
  }
306
336
  throw new Error(`Unknown provider: ${providerStr}`);
307
337
  }
@@ -0,0 +1,97 @@
1
+ const { OpenAIProvider } = require('./openai');
2
+
3
+ class GithubCopilotProvider extends OpenAIProvider {
4
+ constructor(config = {}) {
5
+ // GitHub Copilot base URL defaults to the individual endpoint
6
+ const defaultBaseUrl = 'https://api.individual.githubcopilot.com';
7
+ const baseUrl = config.baseUrl || process.env.GITHUB_COPILOT_BASE_URL || defaultBaseUrl;
8
+
9
+ super({
10
+ ...config,
11
+ apiKey: config.apiKey || process.env.GITHUB_COPILOT_ACCESS_TOKEN,
12
+ baseUrl,
13
+ // Pass special headers required by GitHub Copilot
14
+ defaultHeaders: {
15
+ 'Editor-Version': 'vscode/1.90.0',
16
+ 'Editor-Plugin-Version': 'copilot-chat/0.15.0',
17
+ 'User-Agent': 'GithubCopilot/1.155.0',
18
+ 'X-Github-Api-Version': '2023-07-07',
19
+ 'Copilot-Integration-Id': 'vscode-chat'
20
+ }
21
+ });
22
+ this.name = 'github-copilot';
23
+ this.githubToken = config.apiKey || process.env.GITHUB_COPILOT_ACCESS_TOKEN;
24
+ this.copilotToken = null;
25
+ this.tokenExpiresAt = 0;
26
+ this._refreshPromise = null;
27
+ }
28
+
29
+ async _refreshCopilotToken() {
30
+ if (this._refreshPromise) return this._refreshPromise;
31
+
32
+ const now = Math.floor(Date.now() / 1000);
33
+ // Refresh token if missing or expiring in less than 5 minutes
34
+ if (this.copilotToken && this.tokenExpiresAt >= now + 300) {
35
+ return;
36
+ }
37
+
38
+ this._refreshPromise = (async () => {
39
+ try {
40
+ if (!this.githubToken) {
41
+ throw new Error('GitHub Copilot access token is missing. Please run `neoagent login github-copilot`.');
42
+ }
43
+
44
+ const res = await fetch('https://api.github.com/copilot_internal/v2/token', {
45
+ headers: {
46
+ 'Authorization': `token ${this.githubToken}`,
47
+ 'Accept': 'application/json',
48
+ 'User-Agent': 'NeoAgent/1.0.0'
49
+ }
50
+ });
51
+
52
+ if (!res.ok) {
53
+ const errorText = await res.text().catch(() => 'Unknown error');
54
+ throw new Error(`Failed to refresh GitHub Copilot token: HTTP ${res.status} - ${errorText}`);
55
+ }
56
+
57
+ const data = await res.json();
58
+ if (!data || typeof data.token !== 'string' || !data.token) {
59
+ throw new Error('Invalid token response from GitHub Copilot.');
60
+ }
61
+
62
+ this.copilotToken = data.token;
63
+ this.tokenExpiresAt = typeof data.expires_at === 'number'
64
+ ? data.expires_at
65
+ : Math.floor(new Date(data.expires_at).getTime() / 1000);
66
+
67
+ if (isNaN(this.tokenExpiresAt)) {
68
+ this.tokenExpiresAt = Math.floor(Date.now() / 1000) + 1800;
69
+ }
70
+
71
+ // Update the client's API key
72
+ this.client.apiKey = this.copilotToken;
73
+ } finally {
74
+ this._refreshPromise = null;
75
+ }
76
+ })();
77
+
78
+ return this._refreshPromise;
79
+ }
80
+
81
+ async chat(messages, tools = [], options = {}) {
82
+ await this._refreshCopilotToken();
83
+ return super.chat(messages, tools, options);
84
+ }
85
+
86
+ async *stream(messages, tools = [], options = {}) {
87
+ await this._refreshCopilotToken();
88
+ yield* super.stream(messages, tools, options);
89
+ }
90
+
91
+ async analyzeImage(options = {}) {
92
+ await this._refreshCopilotToken();
93
+ return super.analyzeImage(options);
94
+ }
95
+ }
96
+
97
+ module.exports = { GithubCopilotProvider };
@@ -29,7 +29,8 @@ class OpenAIProvider extends BaseProvider {
29
29
  };
30
30
  this.client = new OpenAI({
31
31
  apiKey: config.apiKey || process.env.OPENAI_API_KEY,
32
- baseURL: config.baseUrl || process.env.OPENAI_BASE_URL || undefined
32
+ baseURL: config.baseUrl || process.env.OPENAI_BASE_URL || undefined,
33
+ defaultHeaders: config.defaultHeaders || undefined
33
34
  });
34
35
  }
35
36
 
@@ -0,0 +1,31 @@
1
+ const { OpenAIProvider } = require('./openai');
2
+
3
+ class OpenAICodexProvider extends OpenAIProvider {
4
+ constructor(config = {}) {
5
+ const officialBaseUrl = 'https://api.openai.com/v1';
6
+ const baseUrl = config.baseUrl || process.env.OPENAI_CODEX_BASE_URL || 'https://chatgpt.com/backend-api/codex';
7
+
8
+ if (!baseUrl.includes('api.openai.com') && !baseUrl.includes('chatgpt.com')) {
9
+ console.warn(`[OpenAICodex] Using non-official base URL: ${baseUrl}`);
10
+ } else if (baseUrl.includes('chatgpt.com')) {
11
+ console.info(`[OpenAICodex] Using ChatGPT subscription endpoint: ${baseUrl}`);
12
+ }
13
+
14
+ super({
15
+ ...config,
16
+ apiKey: config.apiKey || process.env.OPENAI_CODEX_ACCESS_TOKEN,
17
+ baseUrl,
18
+ defaultHeaders: {
19
+ 'Editor-Version': process.env.OPENAI_CODEX_EDITOR_VERSION || 'vscode/1.99.0',
20
+ 'Editor-Plugin-Version': process.env.OPENAI_CODEX_EDITOR_PLUGIN_VERSION || 'neoagent/1.0.0',
21
+ 'User-Agent': process.env.OPENAI_CODEX_USER_AGENT || 'NeoAgent/1.0.0'
22
+ }
23
+ });
24
+ this.name = 'openai-codex';
25
+ }
26
+
27
+ // OpenAI Codex (subscription-based) uses the OAuth token directly as the API key.
28
+ // The base URL routes it through the ChatGPT backend-api.
29
+ }
30
+
31
+ module.exports = { OpenAICodexProvider };
@@ -59,6 +59,26 @@ const AI_PROVIDER_DEFINITIONS = Object.freeze({
59
59
  defaultEnabled: false,
60
60
  defaultBaseUrl: 'https://api.minimax.io/anthropic'
61
61
  },
62
+ 'github-copilot': {
63
+ id: 'github-copilot',
64
+ label: 'GitHub Copilot',
65
+ description: 'Use your GitHub Copilot subscription as an AI provider.',
66
+ envKey: 'GITHUB_COPILOT_ACCESS_TOKEN',
67
+ supportsApiKey: true,
68
+ supportsBaseUrl: true,
69
+ defaultEnabled: false,
70
+ defaultBaseUrl: 'https://api.githubcopilot.com'
71
+ },
72
+ 'openai-codex': {
73
+ id: 'openai-codex',
74
+ label: 'OpenAI Codex',
75
+ description: 'Use your ChatGPT/Codex subscription as an AI provider.',
76
+ envKey: 'OPENAI_CODEX_ACCESS_TOKEN',
77
+ supportsApiKey: true,
78
+ supportsBaseUrl: true,
79
+ defaultEnabled: false,
80
+ defaultBaseUrl: 'https://api.openai.com/v1'
81
+ },
62
82
  ollama: {
63
83
  id: 'ollama',
64
84
  label: 'Ollama',
@@ -138,7 +138,7 @@ When drafting on behalf of the user, match their likely voice from available con
138
138
  If the user approves a previously shown draft, send that draft rather than silently rewriting it.
139
139
 
140
140
  TASKS
141
- Use one-time schedule triggers for single reminders or delayed actions, recurring schedule triggers for repeating automation, and official integration triggers when the task should react to connected Gmail, Outlook, Slack, Teams, or WhatsApp Personal events. When calling task tools, prefer one unified trigger section: trigger={ type, config }. Make task prompts self-contained: who/what to check, exact action to take, when to notify, and which channel to use if known.
141
+ Use manual triggers for run-on-demand tasks, one-time schedule triggers for single reminders or delayed actions, recurring schedule triggers for repeating automation, and official integration triggers when the task should react to connected Gmail, Outlook, Slack, Teams, or WhatsApp Personal events. When calling task tools, prefer one unified trigger section: trigger={ type, config }. Make task prompts self-contained: who/what to check, exact action to take, when to notify, and which channel to use if known.
142
142
  Do not create vague tasks like "check this" when the future run would not know what "this" means. Resolve references into names, links, file paths, IDs, dates, and success criteria before saving the task.
143
143
  For notification tasks, distinguish between notifying the user in their current messaging channel, emailing the user, and contacting someone else. Default reminders should notify the user through the active messaging channel unless the user explicitly asks for email, phone, or a third party.
144
144
  When creating or updating a task, include whether it should notify every time, only on change, only on errors, or only when a condition is met. If unspecified, choose the least noisy useful behavior and say what you chose.
@@ -973,8 +973,8 @@ function getAvailableTools(app, options = {}) {
973
973
  type: 'object',
974
974
  properties: {
975
975
  name: { type: 'string', description: 'Short descriptive name for the task.' },
976
- trigger: { type: 'object', description: 'Unified trigger object. Prefer { type: "schedule" | integration_trigger_type, config: {...} }.' },
977
- trigger_type: { type: 'string', description: 'Trigger type such as schedule, gmail_message_received, outlook_email_received, slack_message_received, teams_message_received, weather_event, or whatsapp_personal_message_received.' },
976
+ trigger: { type: 'object', description: 'Unified trigger object. Prefer { type: "manual" | "schedule" | integration_trigger_type, config: {...} }.' },
977
+ trigger_type: { type: 'string', description: 'Trigger type such as manual, schedule, gmail_message_received, outlook_email_received, slack_message_received, teams_message_received, weather_event, or whatsapp_personal_message_received.' },
978
978
  trigger_config: { type: 'object', description: 'Trigger-specific configuration object. For schedule triggers prefer { mode: "recurring", cronExpression: "m h dom mon dow" } or { mode: "one_time", runAt: ISO datetime }. 5-field cron only (seconds unsupported).' },
979
979
  prompt: { type: 'string', description: 'The instructions the agent will run when the trigger fires.' },
980
980
  enabled: { type: 'boolean', description: 'Whether to activate immediately.' },
@@ -1010,7 +1010,7 @@ function getAvailableTools(app, options = {}) {
1010
1010
  task_id: { type: 'number', description: 'The numeric ID of the task to update.' },
1011
1011
  name: { type: 'string', description: 'New name for the task.' },
1012
1012
  trigger: { type: 'object', description: 'Unified trigger object. Use { type, config } to update trigger in one section.' },
1013
- trigger_type: { type: 'string', description: 'Updated trigger type.' },
1013
+ trigger_type: { type: 'string', description: 'Updated trigger type, e.g. manual, schedule, or integration trigger type.' },
1014
1014
  trigger_config: { type: 'object', description: 'Updated trigger-specific configuration. For schedule triggers use mode+cronExpression (recurring) or mode+runAt (one_time).' },
1015
1015
  prompt: { type: 'string', description: 'Updated task prompt.' },
1016
1016
  enabled: { type: 'boolean', description: 'Enable or disable the task.' },
@@ -1152,6 +1152,17 @@ function getAvailableTools(app, options = {}) {
1152
1152
  required: ['image_path']
1153
1153
  }
1154
1154
  },
1155
+ {
1156
+ name: 'ocr_extract',
1157
+ description: 'Extract raw text from an image locally using Tesseract OCR. This is faster and completely offline compared to analyze_image.',
1158
+ parameters: {
1159
+ type: 'object',
1160
+ properties: {
1161
+ image_path: { type: 'string', description: 'Absolute path to the image file' }
1162
+ },
1163
+ required: ['image_path']
1164
+ }
1165
+ },
1155
1166
  {
1156
1167
  name: 'read_health_data',
1157
1168
  description: 'Read the user\'s synced mobile health data. Omit metric_type for a summary of all available metrics. With metric_type, returns an aggregate summary (total, avg, min, max over all stored data) plus the most recent individual records. Always report the summary figures — avoid listing every raw record.',
@@ -2311,13 +2322,17 @@ async function executeTool(toolName, args, context, engine) {
2311
2322
  if (!resolvedTrigger.hasType || !resolvedTrigger.triggerType) {
2312
2323
  return { error: 'Task trigger type is required (use trigger.type or trigger_type).' };
2313
2324
  }
2314
- if (!resolvedTrigger.hasConfig || resolvedTrigger.triggerConfig === undefined) {
2325
+ const normalizedTriggerType = String(resolvedTrigger.triggerType || '').trim();
2326
+ const triggerConfig = (!resolvedTrigger.hasConfig || resolvedTrigger.triggerConfig === undefined)
2327
+ ? (normalizedTriggerType === 'manual' ? {} : undefined)
2328
+ : resolvedTrigger.triggerConfig;
2329
+ if (triggerConfig === undefined) {
2315
2330
  return { error: 'Task trigger config is required (use trigger.config or trigger_config).' };
2316
2331
  }
2317
2332
  const task = await s.createTask(userId, {
2318
2333
  name: args.name,
2319
- triggerType: resolvedTrigger.triggerType,
2320
- triggerConfig: resolvedTrigger.triggerConfig,
2334
+ triggerType: normalizedTriggerType,
2335
+ triggerConfig,
2321
2336
  prompt: args.prompt,
2322
2337
  enabled: args.enabled !== false,
2323
2338
  model: args.model || null,
@@ -2580,6 +2595,20 @@ async function executeTool(toolName, args, context, engine) {
2580
2595
  }
2581
2596
  }
2582
2597
 
2598
+ case 'ocr_extract': {
2599
+ try {
2600
+ const fs = require('fs');
2601
+ if (!fs.existsSync(args.image_path)) {
2602
+ return { error: 'File not found: ' + args.image_path };
2603
+ }
2604
+ const Tesseract = require('tesseract.js');
2605
+ const result = await Tesseract.recognize(args.image_path, 'eng');
2606
+ return { text: result.data.text, confidence: result.data.confidence };
2607
+ } catch (err) {
2608
+ return { error: err.message };
2609
+ }
2610
+ }
2611
+
2583
2612
  case 'spawn_subagent': {
2584
2613
  try {
2585
2614
  const task = args.context ? `${args.task}\n\nContext: ${args.context}` : args.task;
@@ -6,11 +6,11 @@ const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
6
6
  if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
7
7
 
8
8
  const USER_AGENTS = [
9
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
10
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
9
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
10
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
11
11
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
12
12
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
13
- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
13
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
14
14
  ];
15
15
 
16
16
  const VIEWPORTS = [
@@ -148,6 +148,50 @@ class BrowserController {
148
148
  return arr;
149
149
  }
150
150
  });
151
+
152
+ // WebGL Spoofing
153
+ const getParameterProxyHandler = {
154
+ apply: function(target, ctx, args) {
155
+ const param = args[0];
156
+ // UNMASKED_VENDOR_WEBGL
157
+ if (param === 37445) return 'Google Inc. (Apple)';
158
+ // UNMASKED_RENDERER_WEBGL
159
+ if (param === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)';
160
+ return Reflect.apply(target, ctx, args);
161
+ }
162
+ };
163
+ const getParam = WebGLRenderingContext.prototype.getParameter;
164
+ WebGLRenderingContext.prototype.getParameter = new Proxy(getParam, getParameterProxyHandler);
165
+ if (typeof WebGL2RenderingContext !== 'undefined') {
166
+ const getParam2 = WebGL2RenderingContext.prototype.getParameter;
167
+ WebGL2RenderingContext.prototype.getParameter = new Proxy(getParam2, getParameterProxyHandler);
168
+ }
169
+
170
+ // Canvas Spoofing (slight noise)
171
+ const originalFillText = CanvasRenderingContext2D.prototype.fillText;
172
+ CanvasRenderingContext2D.prototype.fillText = function(...args) {
173
+ if (!this._spoofing_applied) {
174
+ this._spoofing_applied = true;
175
+ const r = Math.random() * 0.0001;
176
+ const g = Math.random() * 0.0001;
177
+ const b = Math.random() * 0.0001;
178
+ this.fillStyle = \`rgba(\${Math.floor(r * 255)}, \${Math.floor(g * 255)}, \${Math.floor(b * 255)}, 0.01)\`;
179
+ originalFillText.call(this, "spoof", 0, 0);
180
+ }
181
+ return originalFillText.apply(this, args);
182
+ };
183
+
184
+ // Media Devices Spoofing
185
+ if (navigator.mediaDevices && navigator.mediaDevices.enumerateDevices) {
186
+ const originalEnumerateDevices = navigator.mediaDevices.enumerateDevices.bind(navigator.mediaDevices);
187
+ navigator.mediaDevices.enumerateDevices = async () => {
188
+ return [
189
+ { kind: 'audioinput', deviceId: 'default', groupId: 'a', label: 'MacBook Pro Microphone' },
190
+ { kind: 'audiooutput', deviceId: 'default', groupId: 'b', label: 'MacBook Pro Speakers' },
191
+ { kind: 'videoinput', deviceId: 'default', groupId: 'c', label: 'FaceTime HD Camera' }
192
+ ];
193
+ };
194
+ }
151
195
  })();
152
196
  `);
153
197
  }