neoagent 3.2.1-beta.1 → 3.2.1-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/extensions/chrome-browser/background.mjs +318 -88
  2. package/extensions/chrome-browser/http.mjs +136 -0
  3. package/extensions/chrome-browser/protocol.mjs +654 -90
  4. package/flutter_app/lib/main_chat.dart +118 -739
  5. package/flutter_app/lib/main_controller.dart +157 -24
  6. package/flutter_app/lib/main_integrations.dart +607 -8
  7. package/flutter_app/lib/main_models.dart +7 -2
  8. package/flutter_app/lib/main_operations.dart +334 -370
  9. package/flutter_app/lib/main_security.dart +266 -112
  10. package/flutter_app/lib/main_settings.dart +342 -3
  11. package/flutter_app/lib/main_shared.dart +14 -11
  12. package/flutter_app/lib/src/backend_client.dart +97 -0
  13. package/flutter_app/lib/src/desktop_companion_actions.dart +185 -31
  14. package/flutter_app/lib/src/desktop_companion_io.dart +319 -86
  15. package/flutter_app/windows/runner/flutter_window.cpp +143 -32
  16. package/landing/index.html +3 -1
  17. package/lib/manager.js +106 -89
  18. package/lib/schema_migrations.js +115 -13
  19. package/package.json +30 -15
  20. package/runtime/paths.js +49 -5
  21. package/server/db/database.js +2 -2
  22. package/server/guest-agent.cli.package.json +13 -0
  23. package/server/guest_agent.js +85 -40
  24. package/server/http/middleware.js +24 -0
  25. package/server/http/routes.js +12 -6
  26. package/server/public/.last_build_id +1 -1
  27. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  28. package/server/public/flutter_bootstrap.js +2 -2
  29. package/server/public/main.dart.js +81930 -80509
  30. package/server/routes/admin.js +1 -1
  31. package/server/routes/android.js +30 -34
  32. package/server/routes/behavior.js +80 -0
  33. package/server/routes/browser.js +23 -15
  34. package/server/routes/desktop.js +18 -1
  35. package/server/routes/integrations.js +107 -1
  36. package/server/routes/memory.js +1 -0
  37. package/server/routes/settings.js +20 -5
  38. package/server/routes/social_reach.js +12 -3
  39. package/server/routes/social_video.js +4 -0
  40. package/server/services/agents/manager.js +1 -1
  41. package/server/services/ai/capabilityHealth.js +62 -96
  42. package/server/services/ai/compaction.js +7 -2
  43. package/server/services/ai/history.js +45 -6
  44. package/server/services/ai/integrated_tools/http_request.js +8 -0
  45. package/server/services/ai/loop/agent_engine_core.js +452 -176
  46. package/server/services/ai/loop/blank_recovery.js +5 -4
  47. package/server/services/ai/loop/callbacks.js +1 -0
  48. package/server/services/ai/loop/completion_judge.js +291 -8
  49. package/server/services/ai/loop/conversation_loop.js +515 -342
  50. package/server/services/ai/loop/messaging_delivery.js +176 -57
  51. package/server/services/ai/loop/model_call_guard.js +91 -0
  52. package/server/services/ai/loop/model_io.js +20 -45
  53. package/server/services/ai/loop/progress_classification.js +2 -0
  54. package/server/services/ai/loop/tool_dispatch.js +19 -8
  55. package/server/services/ai/loopPolicy.js +48 -21
  56. package/server/services/ai/messagingFallback.js +17 -17
  57. package/server/services/ai/model_discovery.js +227 -0
  58. package/server/services/ai/model_failure_cache.js +108 -0
  59. package/server/services/ai/model_identity.js +71 -0
  60. package/server/services/ai/models.js +68 -163
  61. package/server/services/ai/providerRetry.js +17 -59
  62. package/server/services/ai/provider_selector.js +166 -0
  63. package/server/services/ai/providers/anthropic.js +2 -2
  64. package/server/services/ai/providers/claudeCode.js +21 -33
  65. package/server/services/ai/providers/githubCopilot.js +41 -20
  66. package/server/services/ai/providers/google.js +135 -97
  67. package/server/services/ai/providers/grok.js +4 -3
  68. package/server/services/ai/providers/grokOauth.js +19 -27
  69. package/server/services/ai/providers/nvidia.js +10 -5
  70. package/server/services/ai/providers/ollama.js +111 -84
  71. package/server/services/ai/providers/ollama_stream.js +142 -0
  72. package/server/services/ai/providers/openai.js +39 -5
  73. package/server/services/ai/providers/openaiCodex.js +11 -4
  74. package/server/services/ai/providers/openrouter.js +29 -7
  75. package/server/services/ai/providers/provider_error.js +36 -0
  76. package/server/services/ai/settings.js +26 -2
  77. package/server/services/ai/systemPrompt.js +32 -123
  78. package/server/services/ai/taskAnalysis.js +104 -11
  79. package/server/services/ai/toolEvidence.js +256 -29
  80. package/server/services/ai/tools.js +248 -117
  81. package/server/services/android/controller.js +770 -237
  82. package/server/services/android/process.js +140 -0
  83. package/server/services/android/sdk_download.js +143 -0
  84. package/server/services/android/uia.js +6 -5
  85. package/server/services/artifacts/store.js +24 -0
  86. package/server/services/behavior/config.js +251 -0
  87. package/server/services/behavior/defaults.js +68 -0
  88. package/server/services/behavior/delivery.js +176 -0
  89. package/server/services/behavior/index.js +43 -0
  90. package/server/services/behavior/model_client.js +35 -0
  91. package/server/services/behavior/modules/agent_identity.js +37 -0
  92. package/server/services/behavior/modules/channel_style.js +28 -0
  93. package/server/services/behavior/modules/index.js +29 -0
  94. package/server/services/behavior/modules/norms.js +101 -0
  95. package/server/services/behavior/modules/persona.js +48 -0
  96. package/server/services/behavior/modules/persona_prompt.js +33 -0
  97. package/server/services/behavior/modules/social_memory.js +86 -0
  98. package/server/services/behavior/modules/social_observability.js +94 -0
  99. package/server/services/behavior/modules/social_signals.js +41 -0
  100. package/server/services/behavior/modules/theory_of_mind.js +110 -0
  101. package/server/services/behavior/modules/turn_taking.js +237 -0
  102. package/server/services/behavior/pipeline.js +285 -0
  103. package/server/services/behavior/registry.js +78 -0
  104. package/server/services/behavior/signals.js +107 -0
  105. package/server/services/behavior/state.js +99 -0
  106. package/server/services/behavior/system_prompt.js +75 -0
  107. package/server/services/browser/controller.js +843 -385
  108. package/server/services/browser/extension/gateway.js +40 -16
  109. package/server/services/browser/extension/protocol.js +15 -1
  110. package/server/services/browser/extension/provider.js +71 -47
  111. package/server/services/browser/extension/registry.js +155 -34
  112. package/server/services/cli/executor.js +62 -9
  113. package/server/services/credentials/bitwarden_cli.js +322 -0
  114. package/server/services/credentials/broker.js +594 -0
  115. package/server/services/desktop/gateway.js +41 -4
  116. package/server/services/desktop/protocol.js +3 -0
  117. package/server/services/desktop/provider.js +39 -42
  118. package/server/services/desktop/registry.js +137 -52
  119. package/server/services/integrations/bitwarden/constants.js +14 -0
  120. package/server/services/integrations/bitwarden/provider.js +197 -0
  121. package/server/services/integrations/bitwarden/snapshot.js +65 -0
  122. package/server/services/integrations/figma/provider.js +78 -12
  123. package/server/services/integrations/github/common.js +11 -6
  124. package/server/services/integrations/github/provider.js +52 -53
  125. package/server/services/integrations/google/provider.js +55 -19
  126. package/server/services/integrations/home_assistant/network.js +17 -20
  127. package/server/services/integrations/home_assistant/provider.js +7 -5
  128. package/server/services/integrations/home_assistant/tools.js +17 -5
  129. package/server/services/integrations/http.js +51 -0
  130. package/server/services/integrations/manager.js +159 -53
  131. package/server/services/integrations/microsoft/provider.js +80 -13
  132. package/server/services/integrations/neoarchive/provider.js +55 -29
  133. package/server/services/integrations/neorecall/client.js +17 -10
  134. package/server/services/integrations/neorecall/provider.js +20 -11
  135. package/server/services/integrations/notion/provider.js +16 -13
  136. package/server/services/integrations/oauth_provider.js +115 -51
  137. package/server/services/integrations/registry.js +2 -0
  138. package/server/services/integrations/slack/provider.js +98 -9
  139. package/server/services/integrations/spotify/provider.js +67 -71
  140. package/server/services/integrations/trello/provider.js +21 -7
  141. package/server/services/integrations/weather/provider.js +18 -12
  142. package/server/services/integrations/whatsapp/provider.js +76 -16
  143. package/server/services/manager.js +110 -1
  144. package/server/services/memory/embedding_index.js +20 -8
  145. package/server/services/memory/embeddings.js +151 -90
  146. package/server/services/memory/ingestion.js +50 -9
  147. package/server/services/memory/ingestion_documents.js +13 -3
  148. package/server/services/memory/manager.js +66 -52
  149. package/server/services/messaging/access_policy.js +10 -6
  150. package/server/services/messaging/automation.js +240 -34
  151. package/server/services/messaging/discord.js +11 -1
  152. package/server/services/messaging/formatting_guides.js +5 -4
  153. package/server/services/messaging/http_platforms.js +37 -16
  154. package/server/services/messaging/inbound_queue.js +143 -28
  155. package/server/services/messaging/inbound_store.js +257 -0
  156. package/server/services/messaging/manager.js +344 -51
  157. package/server/services/messaging/telegram.js +10 -1
  158. package/server/services/messaging/typing_keepalive.js +5 -2
  159. package/server/services/messaging/whatsapp.js +33 -14
  160. package/server/services/network/http.js +210 -0
  161. package/server/services/network/safe_request.js +307 -0
  162. package/server/services/runtime/backends/local-vm.js +227 -67
  163. package/server/services/runtime/docker-vm-manager.js +9 -0
  164. package/server/services/runtime/guest_bootstrap.js +30 -4
  165. package/server/services/runtime/guest_image.js +43 -12
  166. package/server/services/runtime/manager.js +77 -23
  167. package/server/services/runtime/validation.js +7 -6
  168. package/server/services/security/tool_categories.js +6 -0
  169. package/server/services/social_reach/channels/github.js +10 -4
  170. package/server/services/social_reach/channels/reddit.js +4 -4
  171. package/server/services/social_reach/channels/rss.js +2 -2
  172. package/server/services/social_reach/channels/social_video.js +13 -8
  173. package/server/services/social_reach/channels/v2ex.js +21 -8
  174. package/server/services/social_reach/channels/x.js +2 -2
  175. package/server/services/social_reach/channels/xueqiu.js +5 -5
  176. package/server/services/social_reach/service.js +9 -6
  177. package/server/services/social_reach/utils.js +65 -14
  178. package/server/services/social_video/captions.js +2 -2
  179. package/server/services/social_video/service.js +343 -68
  180. package/server/services/tasks/integration_runtime.js +18 -8
  181. package/server/services/tasks/runtime.js +39 -4
  182. package/server/services/voice/agentBridge.js +17 -4
  183. package/server/services/voice/bufferedLiveRelayAdapter.js +5 -0
  184. package/server/services/voice/liveSession.js +31 -0
  185. package/server/services/voice/message.js +1 -1
  186. package/server/services/voice/openaiSpeech.js +33 -8
  187. package/server/services/voice/providers.js +233 -151
  188. package/server/services/voice/runtime.js +2 -2
  189. package/server/services/voice/runtimeManager.js +118 -20
  190. package/server/services/voice/turnRunner.js +6 -0
  191. package/server/services/wearable/firmware_manifest.js +51 -13
  192. package/server/services/wearable/service.js +1 -0
  193. package/server/utils/abort.js +96 -0
  194. package/server/utils/cloud-security.js +110 -3
  195. package/server/utils/files.js +31 -0
  196. package/server/utils/image_payload.js +95 -0
  197. package/server/utils/logger.js +19 -0
  198. package/server/utils/retry.js +107 -0
@@ -132,6 +132,7 @@ class NeoAgentController extends ChangeNotifier {
132
132
  const <LinkedAuthProviderItem>[];
133
133
  QrLoginChallenge? qrLoginChallenge;
134
134
  Map<String, dynamic> settings = const <String, dynamic>{};
135
+ Map<String, dynamic> behaviorConfig = const <String, dynamic>{};
135
136
  Map<String, dynamic>? versionInfo;
136
137
  Map<String, dynamic>? backendHealthStatus;
137
138
  HealthBridgeStatus? deviceHealthStatus;
@@ -1420,6 +1421,7 @@ class NeoAgentController extends ChangeNotifier {
1420
1421
  usageAndLimits = null;
1421
1422
  linkedAuthProviders = const <LinkedAuthProviderItem>[];
1422
1423
  settings = const <String, dynamic>{};
1424
+ behaviorConfig = const <String, dynamic>{};
1423
1425
  chatMessages = const <ChatEntry>[];
1424
1426
  _resetChatHistoryPagination();
1425
1427
  agentProfiles = const <AgentProfile>[];
@@ -1958,6 +1960,11 @@ class NeoAgentController extends ChangeNotifier {
1958
1960
  _backendClient.fetchSettings(backendUrl, agentId: agentId),
1959
1961
  const <String, dynamic>{},
1960
1962
  );
1963
+ final behaviorFuture = _softRefreshLoad<Map<String, dynamic>>(
1964
+ 'behavior_config',
1965
+ _backendClient.fetchBehaviorConfig(backendUrl, agentId: agentId),
1966
+ const <String, dynamic>{},
1967
+ );
1961
1968
  final runsFuture = _softRefreshLoad<Map<String, dynamic>>(
1962
1969
  'runs',
1963
1970
  _backendClient.fetchRuns(backendUrl, agentId: agentId),
@@ -2094,6 +2101,7 @@ class NeoAgentController extends ChangeNotifier {
2094
2101
  final modelsResponse = await modelsFuture;
2095
2102
  final providersResponse = await providersFuture;
2096
2103
  final settingsResponse = await settingsFuture;
2104
+ final behaviorResponse = await behaviorFuture;
2097
2105
  final runsResponse = await runsFuture;
2098
2106
  final timelineResponse = await timelineFuture;
2099
2107
  final versionResponse = await versionFuture;
@@ -2137,6 +2145,9 @@ class NeoAgentController extends ChangeNotifier {
2137
2145
  );
2138
2146
 
2139
2147
  settings = Map<String, dynamic>.from(settingsResponse);
2148
+ behaviorConfig = behaviorResponse['config'] is Map
2149
+ ? Map<String, dynamic>.from(behaviorResponse['config'] as Map)
2150
+ : const <String, dynamic>{};
2140
2151
  recentRuns = _decodeModelList(
2141
2152
  'runs',
2142
2153
  runsResponse['runs'],
@@ -4221,6 +4232,27 @@ class NeoAgentController extends ChangeNotifier {
4221
4232
  }
4222
4233
  }
4223
4234
 
4235
+ Future<void> saveBehaviorConfig(Map<String, dynamic> config) async {
4236
+ isSavingSettings = true;
4237
+ errorMessage = null;
4238
+ notifyListeners();
4239
+ try {
4240
+ final response = await _backendClient.saveBehaviorConfig(
4241
+ backendUrl,
4242
+ config,
4243
+ agentId: _scopedAgentId,
4244
+ );
4245
+ behaviorConfig = response['config'] is Map
4246
+ ? Map<String, dynamic>.from(response['config'] as Map)
4247
+ : Map<String, dynamic>.from(config);
4248
+ } catch (error) {
4249
+ errorMessage = _friendlyErrorMessage(error);
4250
+ } finally {
4251
+ isSavingSettings = false;
4252
+ notifyListeners();
4253
+ }
4254
+ }
4255
+
4224
4256
  void _applyAccountResponse(Map<String, dynamic> response) {
4225
4257
  if (response['user'] is Map) {
4226
4258
  user = Map<String, dynamic>.from(response['user'] as Map);
@@ -4890,6 +4922,88 @@ class NeoAgentController extends ChangeNotifier {
4890
4922
  }
4891
4923
  }
4892
4924
 
4925
+ Future<Map<String, dynamic>> unlockBitwarden(
4926
+ String masterPassword, {
4927
+ required int idleTimeoutMinutes,
4928
+ }) async {
4929
+ try {
4930
+ errorMessage = null;
4931
+ return await _backendClient.unlockBitwarden(
4932
+ backendUrl,
4933
+ masterPassword: masterPassword,
4934
+ idleTimeoutMinutes: idleTimeoutMinutes,
4935
+ agentId: _scopedAgentId,
4936
+ );
4937
+ } catch (error) {
4938
+ errorMessage = _friendlyErrorMessage(error);
4939
+ rethrow;
4940
+ } finally {
4941
+ notifyListeners();
4942
+ }
4943
+ }
4944
+
4945
+ Future<void> lockBitwarden() async {
4946
+ try {
4947
+ errorMessage = null;
4948
+ await _backendClient.lockBitwarden(backendUrl, agentId: _scopedAgentId);
4949
+ } catch (error) {
4950
+ errorMessage = _friendlyErrorMessage(error);
4951
+ rethrow;
4952
+ } finally {
4953
+ notifyListeners();
4954
+ }
4955
+ }
4956
+
4957
+ Future<List<Map<String, dynamic>>> fetchBitwardenItems() async {
4958
+ final response = await _backendClient.fetchBitwardenItems(
4959
+ backendUrl,
4960
+ agentId: _scopedAgentId,
4961
+ );
4962
+ return _jsonMapList(
4963
+ response['items'],
4964
+ ).map((row) => Map<String, dynamic>.from(row)).toList();
4965
+ }
4966
+
4967
+ Future<List<Map<String, dynamic>>> fetchCredentialBindings() async {
4968
+ final response = await _backendClient.fetchCredentialBindings(
4969
+ backendUrl,
4970
+ agentId: _scopedAgentId,
4971
+ );
4972
+ return _jsonMapList(
4973
+ response['bindings'],
4974
+ ).map((row) => Map<String, dynamic>.from(row)).toList();
4975
+ }
4976
+
4977
+ Future<void> createCredentialBinding(Map<String, dynamic> binding) async {
4978
+ try {
4979
+ errorMessage = null;
4980
+ await _backendClient.createCredentialBinding(
4981
+ backendUrl,
4982
+ binding: binding,
4983
+ agentId: _scopedAgentId,
4984
+ );
4985
+ await refreshSkills();
4986
+ } catch (error) {
4987
+ errorMessage = _friendlyErrorMessage(error);
4988
+ rethrow;
4989
+ }
4990
+ }
4991
+
4992
+ Future<void> deleteCredentialBinding(String bindingId) async {
4993
+ try {
4994
+ errorMessage = null;
4995
+ await _backendClient.deleteCredentialBinding(
4996
+ backendUrl,
4997
+ bindingId,
4998
+ agentId: _scopedAgentId,
4999
+ );
5000
+ await refreshSkills();
5001
+ } catch (error) {
5002
+ errorMessage = _friendlyErrorMessage(error);
5003
+ rethrow;
5004
+ }
5005
+ }
5006
+
4893
5007
  Future<void> disconnectOfficialIntegration(
4894
5008
  String providerId, {
4895
5009
  required int connectionId,
@@ -5256,10 +5370,20 @@ class NeoAgentController extends ChangeNotifier {
5256
5370
  }
5257
5371
 
5258
5372
  Future<void> updateAssistantBehaviorNotes(String content) async {
5259
- await _backendClient.saveSettings(backendUrl, <String, dynamic>{
5260
- 'assistant_behavior_notes': content,
5261
- }, agentId: _scopedAgentId);
5262
- await refreshMemory();
5373
+ isSavingSettings = true;
5374
+ errorMessage = null;
5375
+ notifyListeners();
5376
+ try {
5377
+ await _backendClient.saveSettings(backendUrl, <String, dynamic>{
5378
+ 'assistant_behavior_notes': content,
5379
+ }, agentId: _scopedAgentId);
5380
+ await refreshMemory();
5381
+ } catch (error) {
5382
+ errorMessage = _friendlyErrorMessage(error);
5383
+ } finally {
5384
+ isSavingSettings = false;
5385
+ notifyListeners();
5386
+ }
5263
5387
  }
5264
5388
 
5265
5389
  Future<void> updateCoreMemory(String key, String value) async {
@@ -5950,11 +6074,13 @@ class NeoAgentController extends ChangeNotifier {
5950
6074
  List<String> get enabledModelIds {
5951
6075
  final raw = settings['enabled_models'];
5952
6076
  if (raw is List) {
5953
- final knownIds = supportedModels.map((model) => model.id).toSet();
5954
- final filtered = raw
5955
- .map((item) => item.toString())
5956
- .where((id) => knownIds.contains(id))
5957
- .toList();
6077
+ final filtered = <String>[];
6078
+ for (final item in raw) {
6079
+ final model = _modelForValue(item.toString(), supportedModels);
6080
+ if (model != null && !filtered.contains(model.id)) {
6081
+ filtered.add(model.id);
6082
+ }
6083
+ }
5958
6084
  if (filtered.isNotEmpty) {
5959
6085
  return filtered;
5960
6086
  }
@@ -5965,18 +6091,30 @@ class NeoAgentController extends ChangeNotifier {
5965
6091
  .toList();
5966
6092
  }
5967
6093
 
5968
- String get defaultChatModel =>
5969
- settings['default_chat_model']?.toString() ?? 'auto';
6094
+ String get defaultChatModel => _ensureModelValue(
6095
+ settings['default_chat_model']?.toString() ?? 'auto',
6096
+ supportedModels,
6097
+ allowAuto: true,
6098
+ );
5970
6099
 
5971
- String get defaultSubagentModel =>
5972
- settings['default_subagent_model']?.toString() ?? 'auto';
6100
+ String get defaultSubagentModel => _ensureModelValue(
6101
+ settings['default_subagent_model']?.toString() ?? 'auto',
6102
+ supportedModels,
6103
+ allowAuto: true,
6104
+ );
5973
6105
 
5974
- String get defaultSpeechModel =>
5975
- settings['default_speech_model']?.toString() ?? 'auto';
6106
+ String get defaultSpeechModel => _ensureModelValue(
6107
+ settings['default_speech_model']?.toString() ?? 'auto',
6108
+ supportedModels,
6109
+ allowAuto: true,
6110
+ );
5976
6111
 
5977
- String get fallbackModel =>
5978
- settings['fallback_model_id']?.toString() ??
5979
- _firstAvailableModelId(supportedModels);
6112
+ String get fallbackModel => _ensureModelValue(
6113
+ settings['fallback_model_id']?.toString() ??
6114
+ _firstAvailableModelId(supportedModels),
6115
+ supportedModels,
6116
+ allowAuto: false,
6117
+ );
5980
6118
 
5981
6119
  String get voiceSttProvider =>
5982
6120
  _settingString('voice_stt_provider', 'openai', lowercase: true);
@@ -6117,12 +6255,7 @@ class NeoAgentController extends ChangeNotifier {
6117
6255
  }
6118
6256
 
6119
6257
  ModelMeta? _modelById(String id) {
6120
- for (final model in supportedModels) {
6121
- if (model.id == id) {
6122
- return model;
6123
- }
6124
- }
6125
- return null;
6258
+ return _modelForValue(id, supportedModels);
6126
6259
  }
6127
6260
 
6128
6261
  void _ensureUpdatePolling() {