neoagent 2.4.1-beta.8 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/.env.example +33 -3
  2. package/LICENSE +111 -56
  3. package/README.md +8 -3
  4. package/docs/configuration.md +8 -0
  5. package/docs/getting-started.md +9 -3
  6. package/docs/index.md +4 -0
  7. package/extensions/chrome-browser/background.mjs +45 -7
  8. package/extensions/chrome-browser/icons/icon128.png +0 -0
  9. package/extensions/chrome-browser/icons/icon16.png +0 -0
  10. package/extensions/chrome-browser/icons/icon48.png +0 -0
  11. package/extensions/chrome-browser/icons/logo.svg +39 -8
  12. package/extensions/chrome-browser/manifest.json +3 -2
  13. package/extensions/chrome-browser/popup.html +5 -1
  14. package/extensions/chrome-browser/popup.js +15 -2
  15. package/flutter_app/android/app/src/main/AndroidManifest.xml +2 -1
  16. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  17. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  18. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  19. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  20. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  21. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  22. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  23. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  24. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  25. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  26. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  27. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  28. package/flutter_app/assets/branding/app_icon_light_1024.png +0 -0
  29. package/flutter_app/assets/branding/app_icon_light_128.png +0 -0
  30. package/flutter_app/assets/branding/app_icon_light_192.png +0 -0
  31. package/flutter_app/assets/branding/app_icon_light_256.png +0 -0
  32. package/flutter_app/assets/branding/app_icon_light_32.png +0 -0
  33. package/flutter_app/assets/branding/app_icon_light_512.png +0 -0
  34. package/flutter_app/assets/branding/app_icon_light_64.png +0 -0
  35. package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
  36. package/flutter_app/assets/branding/tray_icon_light_template.png +0 -0
  37. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  38. package/flutter_app/lib/features/location/location_service.dart +3 -0
  39. package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +391 -382
  40. package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +743 -0
  41. package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +18 -16
  42. package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +19 -18
  43. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +8 -1
  44. package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +16 -13
  45. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +17 -13
  46. package/flutter_app/lib/main.dart +3 -0
  47. package/flutter_app/lib/main_account_settings.dart +10 -34
  48. package/flutter_app/lib/main_admin.dart +14 -1
  49. package/flutter_app/lib/main_app_shell.dart +377 -340
  50. package/flutter_app/lib/main_chat.dart +707 -227
  51. package/flutter_app/lib/main_controller.dart +338 -46
  52. package/flutter_app/lib/main_devices.dart +851 -122
  53. package/flutter_app/lib/main_integrations.dart +255 -6
  54. package/flutter_app/lib/main_launcher.dart +1 -1
  55. package/flutter_app/lib/main_model_picker.dart +685 -0
  56. package/flutter_app/lib/main_models.dart +212 -0
  57. package/flutter_app/lib/main_navigation.dart +1 -9
  58. package/flutter_app/lib/main_operations.dart +1417 -614
  59. package/flutter_app/lib/main_settings.dart +764 -887
  60. package/flutter_app/lib/main_shared.dart +971 -443
  61. package/flutter_app/lib/main_spacing.dart +4 -4
  62. package/flutter_app/lib/main_theme.dart +22 -14
  63. package/flutter_app/lib/main_unified.dart +5 -72
  64. package/flutter_app/lib/src/android_apk_drop_zone.dart +24 -0
  65. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +13 -0
  66. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +219 -0
  67. package/flutter_app/lib/src/backend_client.dart +79 -0
  68. package/flutter_app/lib/src/desktop_companion_actions.dart +56 -7
  69. package/flutter_app/lib/src/desktop_companion_io.dart +77 -1
  70. package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
  71. package/flutter_app/lib/src/security/password_strength.dart +84 -0
  72. package/flutter_app/lib/src/stream_renderer.dart +205 -35
  73. package/flutter_app/lib/src/theme/palette.dart +76 -34
  74. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  75. package/flutter_app/macos/Runner/AppDelegate.swift +44 -0
  76. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  77. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  78. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  79. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  80. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  81. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  82. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  83. package/flutter_app/pubspec.lock +2 -2
  84. package/flutter_app/pubspec.yaml +7 -1
  85. package/flutter_app/tool/branding_source/neoagent-icon-1024.png +0 -0
  86. package/flutter_app/tool/branding_source/neoagent-icon-128.png +0 -0
  87. package/flutter_app/tool/branding_source/neoagent-icon-16.png +0 -0
  88. package/flutter_app/tool/branding_source/neoagent-icon-180.png +0 -0
  89. package/flutter_app/tool/branding_source/neoagent-icon-192.png +0 -0
  90. package/flutter_app/tool/branding_source/neoagent-icon-256.png +0 -0
  91. package/flutter_app/tool/branding_source/neoagent-icon-32.png +0 -0
  92. package/flutter_app/tool/branding_source/neoagent-icon-48.png +0 -0
  93. package/flutter_app/tool/branding_source/neoagent-icon-512.png +0 -0
  94. package/flutter_app/tool/branding_source/neoagent-icon-64.png +0 -0
  95. package/flutter_app/tool/branding_source/neoagent-icon.svg +43 -0
  96. package/flutter_app/tool/generate_desktop_branding.py +154 -152
  97. package/flutter_app/web/favicon.png +0 -0
  98. package/flutter_app/web/favicon.svg +40 -9
  99. package/flutter_app/web/favicon_light.svg +43 -0
  100. package/flutter_app/web/icons/Icon-192-light.png +0 -0
  101. package/flutter_app/web/icons/Icon-192.png +0 -0
  102. package/flutter_app/web/icons/Icon-512-light.png +0 -0
  103. package/flutter_app/web/icons/Icon-512.png +0 -0
  104. package/flutter_app/web/icons/Icon-maskable-192-light.png +0 -0
  105. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  106. package/flutter_app/web/icons/Icon-maskable-512-light.png +0 -0
  107. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  108. package/flutter_app/windows/runner/main.cpp +7 -1
  109. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  110. package/lib/manager.js +445 -83
  111. package/package.json +17 -3
  112. package/runtime/paths.js +3 -1
  113. package/server/admin/access.js +198 -0
  114. package/server/admin/admin.css +268 -0
  115. package/server/admin/admin.js +348 -0
  116. package/server/admin/analytics.js +128 -0
  117. package/server/admin/index.html +1015 -0
  118. package/server/admin/login.html +290 -0
  119. package/server/admin/logo.svg +43 -0
  120. package/server/admin/sql.js +134 -0
  121. package/server/admin/users.js +147 -0
  122. package/server/config/origins.js +3 -1
  123. package/server/db/database.js +92 -0
  124. package/server/db/ftsQuery.js +27 -0
  125. package/server/http/routes.js +1 -0
  126. package/server/http/static.js +23 -6
  127. package/server/index.js +1 -1
  128. package/server/middleware/adminAuth.js +48 -0
  129. package/server/middleware/auth.js +1 -40
  130. package/server/public/.last_build_id +1 -1
  131. package/server/public/app_icon.png +0 -0
  132. package/server/public/assets/AssetManifest.bin +1 -1
  133. package/server/public/assets/AssetManifest.bin.json +1 -1
  134. package/server/public/assets/assets/branding/app_icon_1024.png +0 -0
  135. package/server/public/assets/assets/branding/app_icon_256.png +0 -0
  136. package/server/public/assets/assets/branding/app_icon_512.png +0 -0
  137. package/server/public/assets/assets/branding/app_icon_light_1024.png +0 -0
  138. package/server/public/assets/assets/branding/app_icon_light_256.png +0 -0
  139. package/server/public/assets/assets/branding/app_icon_light_512.png +0 -0
  140. package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
  141. package/server/public/assets/assets/branding/tray_icon_light_template.png +0 -0
  142. package/server/public/assets/assets/branding/tray_icon_template.png +0 -0
  143. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  144. package/server/public/assets/web/icons/Icon-192.png +0 -0
  145. package/server/public/favicon.png +0 -0
  146. package/server/public/favicon.svg +40 -9
  147. package/server/public/favicon_light.svg +43 -0
  148. package/server/public/flutter_bootstrap.js +2 -2
  149. package/server/public/icons/Icon-192-light.png +0 -0
  150. package/server/public/icons/Icon-192.png +0 -0
  151. package/server/public/icons/Icon-512-light.png +0 -0
  152. package/server/public/icons/Icon-512.png +0 -0
  153. package/server/public/icons/Icon-maskable-192-light.png +0 -0
  154. package/server/public/icons/Icon-maskable-192.png +0 -0
  155. package/server/public/icons/Icon-maskable-512-light.png +0 -0
  156. package/server/public/icons/Icon-maskable-512.png +0 -0
  157. package/server/public/main.dart.js +88944 -85720
  158. package/server/routes/admin.js +632 -0
  159. package/server/routes/agent_profiles.js +3 -0
  160. package/server/routes/agents.js +1 -1
  161. package/server/routes/auth.js +3 -1
  162. package/server/routes/browser.js +14 -0
  163. package/server/routes/browser_extension.js +21 -4
  164. package/server/routes/desktop.js +10 -0
  165. package/server/routes/mcp.js +29 -13
  166. package/server/routes/memory.js +23 -1
  167. package/server/routes/messaging.js +2 -0
  168. package/server/routes/screenHistory.js +14 -8
  169. package/server/routes/settings.js +14 -2
  170. package/server/routes/stream.js +12 -3
  171. package/server/routes/triggers.js +4 -4
  172. package/server/routes/voice_assistant.js +36 -1
  173. package/server/routes/workspace.js +86 -0
  174. package/server/services/account/admin_two_factor.js +132 -0
  175. package/server/services/account/password_policy.js +6 -1
  176. package/server/services/ai/completion.js +44 -0
  177. package/server/services/ai/engine.js +321 -500
  178. package/server/services/ai/imageAnalysis.js +9 -5
  179. package/server/services/ai/logFormat.js +46 -0
  180. package/server/services/ai/loopPolicy.js +11 -0
  181. package/server/services/ai/messagingFallback.js +228 -0
  182. package/server/services/ai/models.js +194 -55
  183. package/server/services/ai/providerRetry.js +169 -0
  184. package/server/services/ai/providers/anthropic.js +15 -4
  185. package/server/services/ai/providers/google.js +38 -11
  186. package/server/services/ai/providers/grok.js +19 -68
  187. package/server/services/ai/providers/grokOauth.js +141 -0
  188. package/server/services/ai/providers/nvidia.js +154 -0
  189. package/server/services/ai/providers/ollama.js +76 -39
  190. package/server/services/ai/providers/openai.js +20 -31
  191. package/server/services/ai/providers/openaiCodex.js +6 -2
  192. package/server/services/ai/providers/openaiCompatible.js +70 -0
  193. package/server/services/ai/providers/openrouter.js +162 -0
  194. package/server/services/ai/settings.js +30 -0
  195. package/server/services/ai/systemPrompt.js +51 -29
  196. package/server/services/ai/taskAnalysis.js +60 -1
  197. package/server/services/ai/toolEvidence.js +207 -0
  198. package/server/services/ai/toolSelector.js +8 -1
  199. package/server/services/ai/tools.js +80 -15
  200. package/server/services/android/controller.js +36 -1
  201. package/server/services/browser/controller.js +18 -0
  202. package/server/services/browser/extension/manifest.js +33 -0
  203. package/server/services/browser/extension/provider.js +12 -6
  204. package/server/services/browser/extension/registry.js +229 -18
  205. package/server/services/desktop/protocol.js +1 -0
  206. package/server/services/desktop/provider.js +4 -0
  207. package/server/services/desktop/registry.js +104 -1
  208. package/server/services/desktop/screenRecorder.js +208 -98
  209. package/server/services/desktop/screen_recorder_support.js +46 -0
  210. package/server/services/integrations/google/provider.js +13 -0
  211. package/server/services/integrations/home_assistant/constants.js +88 -0
  212. package/server/services/integrations/home_assistant/network.js +207 -0
  213. package/server/services/integrations/home_assistant/provider.js +249 -0
  214. package/server/services/integrations/home_assistant/snapshot.js +98 -0
  215. package/server/services/integrations/home_assistant/tools.js +101 -0
  216. package/server/services/integrations/manager.js +5 -0
  217. package/server/services/integrations/registry.js +2 -0
  218. package/server/services/integrations/trello/provider.js +8 -2
  219. package/server/services/manager.js +51 -47
  220. package/server/services/mcp/client.js +208 -268
  221. package/server/services/mcp/client_support.js +172 -0
  222. package/server/services/mcp/recovery.js +116 -0
  223. package/server/services/mcp/tool_operations.js +123 -0
  224. package/server/services/memory/ingestion.js +185 -370
  225. package/server/services/memory/ingestion_coverage.js +129 -0
  226. package/server/services/memory/ingestion_documents.js +123 -0
  227. package/server/services/memory/ingestion_support.js +171 -0
  228. package/server/services/memory/intelligence.js +181 -0
  229. package/server/services/memory/manager.js +476 -32
  230. package/server/services/messaging/automation.js +71 -134
  231. package/server/services/messaging/formatting_guides.js +26 -1
  232. package/server/services/messaging/inbound_queue.js +111 -0
  233. package/server/services/messaging/manager.js +25 -8
  234. package/server/services/messaging/typing_keepalive.js +110 -0
  235. package/server/services/runtime/manager.js +7 -3
  236. package/server/services/runtime/settings.js +17 -0
  237. package/server/services/streaming/android-stream.js +296 -32
  238. package/server/services/streaming/stream-hub.js +1 -1
  239. package/server/services/tasks/integration_runtime.js +4 -1
  240. package/server/services/tasks/runtime.js +415 -56
  241. package/server/services/tasks/task_repository.js +48 -6
  242. package/server/services/websocket.js +8 -2
  243. package/server/services/widgets/service.js +17 -1
  244. package/server/services/workspace/manager.js +116 -0
  245. package/server/utils/logger.js +44 -6
  246. package/server/utils/security.js +3 -0
  247. package/server/utils/version.js +29 -19
  248. package/server/services/memory/openhuman_uplift.test.js +0 -98
@@ -9,7 +9,7 @@ const {
9
9
  } = require('../provider_config_store');
10
10
  const { getConnectionAccessMode } = require('../access');
11
11
  const { fetchJson } = require('../oauth_provider');
12
- const { encryptValue } = require('../secrets');
12
+ const { encryptValue, decryptValue } = require('../secrets');
13
13
 
14
14
  const TRELLO_APP = {
15
15
  id: 'trello',
@@ -732,7 +732,13 @@ function createTrelloProvider() {
732
732
  return 'Trello: native Trello access is connected in this run with one Trello account for board, list, card, comment, and search tools.';
733
733
  },
734
734
  async executeTool(toolName, args, connection) {
735
- return executeTrelloTool(toolName, args, connection);
735
+ let credentials = {};
736
+ try {
737
+ credentials = JSON.parse(decryptValue(connection.credentials_json || '{}') || '{}');
738
+ } catch {
739
+ credentials = {};
740
+ }
741
+ return executeTrelloTool(toolName, args, { connection, credentials });
736
742
  },
737
743
  getUserConfig({ userId, agentId }) {
738
744
  const normalizedUserId = Number(userId);
@@ -24,7 +24,10 @@ const { WorkspaceManager } = require('./workspace/manager');
24
24
  const { BrowserExtensionRegistry } = require('./browser/extension/registry');
25
25
  const { DesktopCompanionRegistry } = require('./desktop/registry');
26
26
  const { DesktopProvider } = require('./desktop/provider');
27
- const { ScreenRecorder } = require('./desktop/screenRecorder');
27
+ const {
28
+ ScreenRecorder,
29
+ hasOpenConnectionForUser,
30
+ } = require('./desktop/screenRecorder');
28
31
  const { WearableService } = require('./wearable/service');
29
32
  const { getRuntimeValidation } = require('./runtime/validation');
30
33
  const {
@@ -112,10 +115,11 @@ function createMemoryIngestionService(app, { memoryManager, integrationManager }
112
115
  new MemoryIngestionService({
113
116
  memoryManager,
114
117
  integrationManager,
118
+ intervalMs: process.env.NEOAGENT_MEMORY_INGESTION_INTERVAL_MS || undefined,
115
119
  }),
116
120
  );
117
- memoryIngestionService.start();
118
- logServiceReady('Memory ingestion service started');
121
+ const status = memoryIngestionService.start();
122
+ logServiceReady(`Memory ingestion service started (${status.intervalMs}ms interval)`);
119
123
  return memoryIngestionService;
120
124
  }
121
125
 
@@ -368,38 +372,26 @@ function createWearableService(app) {
368
372
  }
369
373
 
370
374
  function createScreenRecorder(app) {
371
- const hasActiveRemoteCaptureSession = () => {
372
- const desktopRegistry = app.locals.desktopCompanionRegistry;
373
- if (desktopRegistry?.connectionsByUser instanceof Map) {
374
- for (const userMap of desktopRegistry.connectionsByUser.values()) {
375
- if (!(userMap instanceof Map)) continue;
376
- for (const connection of userMap.values()) {
377
- if (typeof connection?.isOpen === 'function' && connection.isOpen()) {
378
- return true;
379
- }
380
- }
381
- }
382
- }
383
-
384
- const extensionRegistry = app.locals.browserExtensionRegistry;
385
- if (extensionRegistry?.connectionsByUser instanceof Map) {
386
- for (const connection of extensionRegistry.connectionsByUser.values()) {
387
- if (typeof connection?.isOpen === 'function' && connection.isOpen()) {
388
- return true;
389
- }
390
- }
391
- }
392
-
393
- return false;
375
+ const hasActiveCaptureSessionForUser = (userId) => {
376
+ return (
377
+ hasOpenConnectionForUser(app.locals.desktopCompanionRegistry, userId)
378
+ || hasOpenConnectionForUser(app.locals.browserExtensionRegistry, userId)
379
+ );
394
380
  };
395
381
 
396
382
  const screenRecorder = registerLocal(
397
383
  app,
398
384
  'screenRecorder',
399
- new ScreenRecorder({ hasActiveRemoteCaptureSession }),
385
+ new ScreenRecorder({ hasActiveCaptureSessionForUser }),
400
386
  );
401
- screenRecorder.start();
402
- logServiceReady('Screen recorder started');
387
+ const status = screenRecorder.start();
388
+ if (status.state === 'running') {
389
+ logServiceReady(
390
+ `Screen recorder started for user ${status.ownerUserId} (${status.intervalMs}ms interval)`,
391
+ );
392
+ } else {
393
+ logServiceReady(`Screen recorder ${status.state}: ${status.reason}`);
394
+ }
403
395
  return screenRecorder;
404
396
  }
405
397
 
@@ -535,12 +527,16 @@ async function stopServices(app) {
535
527
  console.log('[Services] Stopping services');
536
528
 
537
529
  if (app.locals.taskRuntime) {
538
- try {
539
- app.locals.taskRuntime.stop();
540
- logServiceReady('Task runtime stopped');
541
- } catch (err) {
542
- console.error('[Tasks] Stop error:', getErrorMessage(err));
543
- }
530
+ tasks.push(
531
+ Promise.resolve()
532
+ .then(() => app.locals.taskRuntime.stop())
533
+ .then((status) => {
534
+ logServiceReady(`Task runtime shutdown complete (${status.state})`);
535
+ })
536
+ .catch((err) => {
537
+ console.error('[Tasks] Stop error:', getErrorMessage(err));
538
+ }),
539
+ );
544
540
  }
545
541
 
546
542
  if (app.locals.streamHub) {
@@ -553,12 +549,16 @@ async function stopServices(app) {
553
549
  }
554
550
 
555
551
  if (app.locals.memoryIngestionService) {
556
- try {
557
- app.locals.memoryIngestionService.stop();
558
- logServiceReady('Memory ingestion service stopped');
559
- } catch (err) {
560
- console.error('[MemoryIngestion] Stop error:', getErrorMessage(err));
561
- }
552
+ tasks.push(
553
+ Promise.resolve()
554
+ .then(() => app.locals.memoryIngestionService.stop())
555
+ .then((status) => {
556
+ logServiceReady(`Memory ingestion shutdown complete (${status.state})`);
557
+ })
558
+ .catch((err) => {
559
+ console.error('[MemoryIngestion] Stop error:', getErrorMessage(err));
560
+ }),
561
+ );
562
562
  }
563
563
 
564
564
  if (app.locals.mcpClient) {
@@ -631,12 +631,16 @@ async function stopServices(app) {
631
631
  }
632
632
 
633
633
  if (app.locals.screenRecorder) {
634
- try {
635
- app.locals.screenRecorder.stop();
636
- logServiceReady('Screen recorder stopped');
637
- } catch (err) {
638
- console.error('[ScreenRecorder] Stop error:', getErrorMessage(err));
639
- }
634
+ tasks.push(
635
+ Promise.resolve()
636
+ .then(() => app.locals.screenRecorder.stop())
637
+ .then((status) => {
638
+ logServiceReady(`Screen recorder shutdown complete (${status.state})`);
639
+ })
640
+ .catch((err) => {
641
+ console.error('[ScreenRecorder] Stop error:', getErrorMessage(err));
642
+ }),
643
+ );
640
644
  }
641
645
 
642
646
  if (app.locals.browserExtensionRegistry) {