neoagent 2.4.1-beta.9 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/.env.example +33 -3
  2. package/LICENSE +111 -56
  3. package/README.md +8 -3
  4. package/docs/configuration.md +8 -0
  5. package/docs/getting-started.md +9 -3
  6. package/docs/index.md +4 -0
  7. package/extensions/chrome-browser/background.mjs +36 -4
  8. package/extensions/chrome-browser/icons/icon128.png +0 -0
  9. package/extensions/chrome-browser/icons/icon16.png +0 -0
  10. package/extensions/chrome-browser/icons/icon48.png +0 -0
  11. package/extensions/chrome-browser/icons/logo.svg +39 -8
  12. package/extensions/chrome-browser/manifest.json +3 -3
  13. package/extensions/chrome-browser/popup.html +5 -1
  14. package/extensions/chrome-browser/popup.js +13 -0
  15. package/flutter_app/android/app/src/main/AndroidManifest.xml +2 -1
  16. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  17. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  18. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  19. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  20. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  21. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  22. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  23. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  24. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  25. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  26. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  27. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  28. package/flutter_app/assets/branding/app_icon_light_1024.png +0 -0
  29. package/flutter_app/assets/branding/app_icon_light_128.png +0 -0
  30. package/flutter_app/assets/branding/app_icon_light_192.png +0 -0
  31. package/flutter_app/assets/branding/app_icon_light_256.png +0 -0
  32. package/flutter_app/assets/branding/app_icon_light_32.png +0 -0
  33. package/flutter_app/assets/branding/app_icon_light_512.png +0 -0
  34. package/flutter_app/assets/branding/app_icon_light_64.png +0 -0
  35. package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
  36. package/flutter_app/assets/branding/tray_icon_light_template.png +0 -0
  37. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  38. package/flutter_app/lib/features/location/location_service.dart +3 -0
  39. package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +391 -382
  40. package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +743 -0
  41. package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +18 -16
  42. package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +19 -18
  43. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +8 -1
  44. package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +16 -13
  45. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +17 -13
  46. package/flutter_app/lib/main.dart +2 -0
  47. package/flutter_app/lib/main_account_settings.dart +10 -34
  48. package/flutter_app/lib/main_admin.dart +14 -1
  49. package/flutter_app/lib/main_app_shell.dart +377 -340
  50. package/flutter_app/lib/main_chat.dart +707 -227
  51. package/flutter_app/lib/main_controller.dart +197 -42
  52. package/flutter_app/lib/main_devices.dart +436 -4
  53. package/flutter_app/lib/main_integrations.dart +255 -6
  54. package/flutter_app/lib/main_launcher.dart +1 -1
  55. package/flutter_app/lib/main_model_picker.dart +685 -0
  56. package/flutter_app/lib/main_models.dart +212 -0
  57. package/flutter_app/lib/main_navigation.dart +1 -9
  58. package/flutter_app/lib/main_operations.dart +1417 -614
  59. package/flutter_app/lib/main_settings.dart +673 -871
  60. package/flutter_app/lib/main_shared.dart +971 -443
  61. package/flutter_app/lib/main_spacing.dart +4 -4
  62. package/flutter_app/lib/main_theme.dart +22 -14
  63. package/flutter_app/lib/main_unified.dart +5 -72
  64. package/flutter_app/lib/src/android_apk_drop_zone.dart +24 -0
  65. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +13 -0
  66. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +219 -0
  67. package/flutter_app/lib/src/backend_client.dart +46 -0
  68. package/flutter_app/lib/src/desktop_companion_actions.dart +30 -7
  69. package/flutter_app/lib/src/desktop_companion_io.dart +71 -1
  70. package/flutter_app/lib/src/security/password_strength.dart +84 -0
  71. package/flutter_app/lib/src/stream_renderer.dart +42 -3
  72. package/flutter_app/lib/src/theme/palette.dart +76 -34
  73. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  74. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  75. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  76. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  77. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  78. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  79. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  80. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  81. package/flutter_app/pubspec.lock +2 -2
  82. package/flutter_app/pubspec.yaml +7 -1
  83. package/flutter_app/tool/branding_source/neoagent-icon-1024.png +0 -0
  84. package/flutter_app/tool/branding_source/neoagent-icon-128.png +0 -0
  85. package/flutter_app/tool/branding_source/neoagent-icon-16.png +0 -0
  86. package/flutter_app/tool/branding_source/neoagent-icon-180.png +0 -0
  87. package/flutter_app/tool/branding_source/neoagent-icon-192.png +0 -0
  88. package/flutter_app/tool/branding_source/neoagent-icon-256.png +0 -0
  89. package/flutter_app/tool/branding_source/neoagent-icon-32.png +0 -0
  90. package/flutter_app/tool/branding_source/neoagent-icon-48.png +0 -0
  91. package/flutter_app/tool/branding_source/neoagent-icon-512.png +0 -0
  92. package/flutter_app/tool/branding_source/neoagent-icon-64.png +0 -0
  93. package/flutter_app/tool/branding_source/neoagent-icon.svg +43 -0
  94. package/flutter_app/tool/generate_desktop_branding.py +154 -152
  95. package/flutter_app/web/favicon.png +0 -0
  96. package/flutter_app/web/favicon.svg +40 -9
  97. package/flutter_app/web/favicon_light.svg +43 -0
  98. package/flutter_app/web/icons/Icon-192-light.png +0 -0
  99. package/flutter_app/web/icons/Icon-192.png +0 -0
  100. package/flutter_app/web/icons/Icon-512-light.png +0 -0
  101. package/flutter_app/web/icons/Icon-512.png +0 -0
  102. package/flutter_app/web/icons/Icon-maskable-192-light.png +0 -0
  103. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  104. package/flutter_app/web/icons/Icon-maskable-512-light.png +0 -0
  105. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  106. package/flutter_app/windows/runner/main.cpp +7 -1
  107. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  108. package/lib/manager.js +445 -83
  109. package/package.json +17 -3
  110. package/runtime/paths.js +3 -1
  111. package/server/admin/access.js +198 -0
  112. package/server/admin/admin.css +268 -0
  113. package/server/admin/admin.js +348 -0
  114. package/server/admin/analytics.js +128 -0
  115. package/server/admin/index.html +1015 -0
  116. package/server/admin/login.html +290 -0
  117. package/server/admin/logo.svg +43 -0
  118. package/server/admin/sql.js +134 -0
  119. package/server/admin/users.js +147 -0
  120. package/server/config/origins.js +3 -1
  121. package/server/db/database.js +92 -0
  122. package/server/db/ftsQuery.js +27 -0
  123. package/server/http/routes.js +1 -0
  124. package/server/http/static.js +23 -6
  125. package/server/index.js +1 -1
  126. package/server/middleware/adminAuth.js +48 -0
  127. package/server/middleware/auth.js +1 -40
  128. package/server/public/.last_build_id +1 -1
  129. package/server/public/app_icon.png +0 -0
  130. package/server/public/assets/AssetManifest.bin +1 -1
  131. package/server/public/assets/AssetManifest.bin.json +1 -1
  132. package/server/public/assets/assets/branding/app_icon_1024.png +0 -0
  133. package/server/public/assets/assets/branding/app_icon_256.png +0 -0
  134. package/server/public/assets/assets/branding/app_icon_512.png +0 -0
  135. package/server/public/assets/assets/branding/app_icon_light_1024.png +0 -0
  136. package/server/public/assets/assets/branding/app_icon_light_256.png +0 -0
  137. package/server/public/assets/assets/branding/app_icon_light_512.png +0 -0
  138. package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
  139. package/server/public/assets/assets/branding/tray_icon_light_template.png +0 -0
  140. package/server/public/assets/assets/branding/tray_icon_template.png +0 -0
  141. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  142. package/server/public/assets/web/icons/Icon-192.png +0 -0
  143. package/server/public/favicon.png +0 -0
  144. package/server/public/favicon.svg +40 -9
  145. package/server/public/favicon_light.svg +43 -0
  146. package/server/public/flutter_bootstrap.js +2 -2
  147. package/server/public/icons/Icon-192-light.png +0 -0
  148. package/server/public/icons/Icon-192.png +0 -0
  149. package/server/public/icons/Icon-512-light.png +0 -0
  150. package/server/public/icons/Icon-512.png +0 -0
  151. package/server/public/icons/Icon-maskable-192-light.png +0 -0
  152. package/server/public/icons/Icon-maskable-192.png +0 -0
  153. package/server/public/icons/Icon-maskable-512-light.png +0 -0
  154. package/server/public/icons/Icon-maskable-512.png +0 -0
  155. package/server/public/main.dart.js +88537 -85936
  156. package/server/routes/admin.js +632 -0
  157. package/server/routes/agent_profiles.js +3 -0
  158. package/server/routes/agents.js +1 -1
  159. package/server/routes/auth.js +3 -1
  160. package/server/routes/mcp.js +29 -13
  161. package/server/routes/memory.js +23 -1
  162. package/server/routes/messaging.js +2 -0
  163. package/server/routes/screenHistory.js +14 -8
  164. package/server/routes/settings.js +10 -2
  165. package/server/routes/stream.js +1 -1
  166. package/server/routes/triggers.js +4 -4
  167. package/server/routes/voice_assistant.js +36 -1
  168. package/server/routes/workspace.js +86 -0
  169. package/server/services/account/admin_two_factor.js +132 -0
  170. package/server/services/account/password_policy.js +6 -1
  171. package/server/services/ai/completion.js +44 -0
  172. package/server/services/ai/engine.js +321 -500
  173. package/server/services/ai/imageAnalysis.js +9 -5
  174. package/server/services/ai/logFormat.js +46 -0
  175. package/server/services/ai/loopPolicy.js +11 -0
  176. package/server/services/ai/messagingFallback.js +228 -0
  177. package/server/services/ai/models.js +194 -55
  178. package/server/services/ai/providerRetry.js +169 -0
  179. package/server/services/ai/providers/anthropic.js +15 -4
  180. package/server/services/ai/providers/google.js +38 -11
  181. package/server/services/ai/providers/grok.js +19 -68
  182. package/server/services/ai/providers/grokOauth.js +141 -0
  183. package/server/services/ai/providers/nvidia.js +154 -0
  184. package/server/services/ai/providers/ollama.js +76 -39
  185. package/server/services/ai/providers/openai.js +20 -31
  186. package/server/services/ai/providers/openaiCodex.js +6 -2
  187. package/server/services/ai/providers/openaiCompatible.js +70 -0
  188. package/server/services/ai/providers/openrouter.js +162 -0
  189. package/server/services/ai/settings.js +30 -0
  190. package/server/services/ai/systemPrompt.js +51 -29
  191. package/server/services/ai/taskAnalysis.js +60 -1
  192. package/server/services/ai/toolEvidence.js +207 -0
  193. package/server/services/ai/toolSelector.js +8 -1
  194. package/server/services/ai/tools.js +78 -13
  195. package/server/services/browser/extension/registry.js +94 -9
  196. package/server/services/desktop/registry.js +104 -1
  197. package/server/services/desktop/screenRecorder.js +208 -98
  198. package/server/services/desktop/screen_recorder_support.js +46 -0
  199. package/server/services/integrations/google/provider.js +13 -0
  200. package/server/services/integrations/home_assistant/constants.js +88 -0
  201. package/server/services/integrations/home_assistant/network.js +207 -0
  202. package/server/services/integrations/home_assistant/provider.js +249 -0
  203. package/server/services/integrations/home_assistant/snapshot.js +98 -0
  204. package/server/services/integrations/home_assistant/tools.js +101 -0
  205. package/server/services/integrations/manager.js +5 -0
  206. package/server/services/integrations/registry.js +2 -0
  207. package/server/services/integrations/trello/provider.js +8 -2
  208. package/server/services/manager.js +51 -53
  209. package/server/services/mcp/client.js +208 -268
  210. package/server/services/mcp/client_support.js +172 -0
  211. package/server/services/mcp/recovery.js +116 -0
  212. package/server/services/mcp/tool_operations.js +123 -0
  213. package/server/services/memory/ingestion.js +185 -370
  214. package/server/services/memory/ingestion_coverage.js +129 -0
  215. package/server/services/memory/ingestion_documents.js +123 -0
  216. package/server/services/memory/ingestion_support.js +171 -0
  217. package/server/services/memory/intelligence.js +181 -0
  218. package/server/services/memory/manager.js +476 -32
  219. package/server/services/messaging/automation.js +71 -134
  220. package/server/services/messaging/formatting_guides.js +26 -1
  221. package/server/services/messaging/inbound_queue.js +111 -0
  222. package/server/services/messaging/manager.js +25 -8
  223. package/server/services/messaging/typing_keepalive.js +110 -0
  224. package/server/services/streaming/android-stream.js +293 -40
  225. package/server/services/tasks/integration_runtime.js +4 -1
  226. package/server/services/tasks/runtime.js +415 -56
  227. package/server/services/tasks/task_repository.js +48 -6
  228. package/server/services/websocket.js +8 -2
  229. package/server/services/widgets/service.js +17 -1
  230. package/server/services/workspace/manager.js +116 -0
  231. package/server/utils/logger.js +44 -6
  232. package/server/utils/security.js +3 -0
  233. package/server/utils/version.js +29 -19
  234. package/server/services/memory/openhuman_uplift.test.js +0 -98
@@ -9,7 +9,7 @@ const {
9
9
  } = require('../provider_config_store');
10
10
  const { getConnectionAccessMode } = require('../access');
11
11
  const { fetchJson } = require('../oauth_provider');
12
- const { encryptValue } = require('../secrets');
12
+ const { encryptValue, decryptValue } = require('../secrets');
13
13
 
14
14
  const TRELLO_APP = {
15
15
  id: 'trello',
@@ -732,7 +732,13 @@ function createTrelloProvider() {
732
732
  return 'Trello: native Trello access is connected in this run with one Trello account for board, list, card, comment, and search tools.';
733
733
  },
734
734
  async executeTool(toolName, args, connection) {
735
- return executeTrelloTool(toolName, args, connection);
735
+ let credentials = {};
736
+ try {
737
+ credentials = JSON.parse(decryptValue(connection.credentials_json || '{}') || '{}');
738
+ } catch {
739
+ credentials = {};
740
+ }
741
+ return executeTrelloTool(toolName, args, { connection, credentials });
736
742
  },
737
743
  getUserConfig({ userId, agentId }) {
738
744
  const normalizedUserId = Number(userId);
@@ -24,7 +24,10 @@ const { WorkspaceManager } = require('./workspace/manager');
24
24
  const { BrowserExtensionRegistry } = require('./browser/extension/registry');
25
25
  const { DesktopCompanionRegistry } = require('./desktop/registry');
26
26
  const { DesktopProvider } = require('./desktop/provider');
27
- const { ScreenRecorder } = require('./desktop/screenRecorder');
27
+ const {
28
+ ScreenRecorder,
29
+ hasOpenConnectionForUser,
30
+ } = require('./desktop/screenRecorder');
28
31
  const { WearableService } = require('./wearable/service');
29
32
  const { getRuntimeValidation } = require('./runtime/validation');
30
33
  const {
@@ -112,10 +115,11 @@ function createMemoryIngestionService(app, { memoryManager, integrationManager }
112
115
  new MemoryIngestionService({
113
116
  memoryManager,
114
117
  integrationManager,
118
+ intervalMs: process.env.NEOAGENT_MEMORY_INGESTION_INTERVAL_MS || undefined,
115
119
  }),
116
120
  );
117
- memoryIngestionService.start();
118
- logServiceReady('Memory ingestion service started');
121
+ const status = memoryIngestionService.start();
122
+ logServiceReady(`Memory ingestion service started (${status.intervalMs}ms interval)`);
119
123
  return memoryIngestionService;
120
124
  }
121
125
 
@@ -368,44 +372,26 @@ function createWearableService(app) {
368
372
  }
369
373
 
370
374
  function createScreenRecorder(app) {
371
- const hasActiveRemoteCaptureSession = () => {
372
- const desktopRegistry = app.locals.desktopCompanionRegistry;
373
- if (desktopRegistry?.connectionsByUser instanceof Map) {
374
- for (const userMap of desktopRegistry.connectionsByUser.values()) {
375
- if (!(userMap instanceof Map)) continue;
376
- for (const connection of userMap.values()) {
377
- if (typeof connection?.isOpen === 'function' && connection.isOpen()) {
378
- return true;
379
- }
380
- }
381
- }
382
- }
383
-
384
- const extensionRegistry = app.locals.browserExtensionRegistry;
385
- if (extensionRegistry?.connectionsByUser instanceof Map) {
386
- for (const value of extensionRegistry.connectionsByUser.values()) {
387
- if (value instanceof Map) {
388
- for (const connection of value.values()) {
389
- if (typeof connection?.isOpen === 'function' && connection.isOpen()) {
390
- return true;
391
- }
392
- }
393
- } else if (typeof value?.isOpen === 'function' && value.isOpen()) {
394
- return true;
395
- }
396
- }
397
- }
398
-
399
- return false;
375
+ const hasActiveCaptureSessionForUser = (userId) => {
376
+ return (
377
+ hasOpenConnectionForUser(app.locals.desktopCompanionRegistry, userId)
378
+ || hasOpenConnectionForUser(app.locals.browserExtensionRegistry, userId)
379
+ );
400
380
  };
401
381
 
402
382
  const screenRecorder = registerLocal(
403
383
  app,
404
384
  'screenRecorder',
405
- new ScreenRecorder({ hasActiveRemoteCaptureSession }),
385
+ new ScreenRecorder({ hasActiveCaptureSessionForUser }),
406
386
  );
407
- screenRecorder.start();
408
- logServiceReady('Screen recorder started');
387
+ const status = screenRecorder.start();
388
+ if (status.state === 'running') {
389
+ logServiceReady(
390
+ `Screen recorder started for user ${status.ownerUserId} (${status.intervalMs}ms interval)`,
391
+ );
392
+ } else {
393
+ logServiceReady(`Screen recorder ${status.state}: ${status.reason}`);
394
+ }
409
395
  return screenRecorder;
410
396
  }
411
397
 
@@ -541,12 +527,16 @@ async function stopServices(app) {
541
527
  console.log('[Services] Stopping services');
542
528
 
543
529
  if (app.locals.taskRuntime) {
544
- try {
545
- app.locals.taskRuntime.stop();
546
- logServiceReady('Task runtime stopped');
547
- } catch (err) {
548
- console.error('[Tasks] Stop error:', getErrorMessage(err));
549
- }
530
+ tasks.push(
531
+ Promise.resolve()
532
+ .then(() => app.locals.taskRuntime.stop())
533
+ .then((status) => {
534
+ logServiceReady(`Task runtime shutdown complete (${status.state})`);
535
+ })
536
+ .catch((err) => {
537
+ console.error('[Tasks] Stop error:', getErrorMessage(err));
538
+ }),
539
+ );
550
540
  }
551
541
 
552
542
  if (app.locals.streamHub) {
@@ -559,12 +549,16 @@ async function stopServices(app) {
559
549
  }
560
550
 
561
551
  if (app.locals.memoryIngestionService) {
562
- try {
563
- app.locals.memoryIngestionService.stop();
564
- logServiceReady('Memory ingestion service stopped');
565
- } catch (err) {
566
- console.error('[MemoryIngestion] Stop error:', getErrorMessage(err));
567
- }
552
+ tasks.push(
553
+ Promise.resolve()
554
+ .then(() => app.locals.memoryIngestionService.stop())
555
+ .then((status) => {
556
+ logServiceReady(`Memory ingestion shutdown complete (${status.state})`);
557
+ })
558
+ .catch((err) => {
559
+ console.error('[MemoryIngestion] Stop error:', getErrorMessage(err));
560
+ }),
561
+ );
568
562
  }
569
563
 
570
564
  if (app.locals.mcpClient) {
@@ -637,12 +631,16 @@ async function stopServices(app) {
637
631
  }
638
632
 
639
633
  if (app.locals.screenRecorder) {
640
- try {
641
- app.locals.screenRecorder.stop();
642
- logServiceReady('Screen recorder stopped');
643
- } catch (err) {
644
- console.error('[ScreenRecorder] Stop error:', getErrorMessage(err));
645
- }
634
+ tasks.push(
635
+ Promise.resolve()
636
+ .then(() => app.locals.screenRecorder.stop())
637
+ .then((status) => {
638
+ logServiceReady(`Screen recorder shutdown complete (${status.state})`);
639
+ })
640
+ .catch((err) => {
641
+ console.error('[ScreenRecorder] Stop error:', getErrorMessage(err));
642
+ }),
643
+ );
646
644
  }
647
645
 
648
646
  if (app.locals.browserExtensionRegistry) {