neoagent 2.4.1-beta.8 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. package/.env.example +33 -3
  2. package/LICENSE +111 -56
  3. package/README.md +8 -3
  4. package/docs/configuration.md +8 -0
  5. package/docs/getting-started.md +9 -3
  6. package/docs/index.md +4 -0
  7. package/extensions/chrome-browser/background.mjs +45 -7
  8. package/extensions/chrome-browser/icons/icon128.png +0 -0
  9. package/extensions/chrome-browser/icons/icon16.png +0 -0
  10. package/extensions/chrome-browser/icons/icon48.png +0 -0
  11. package/extensions/chrome-browser/icons/logo.svg +39 -8
  12. package/extensions/chrome-browser/manifest.json +3 -2
  13. package/extensions/chrome-browser/popup.html +5 -1
  14. package/extensions/chrome-browser/popup.js +15 -2
  15. package/flutter_app/android/app/src/main/AndroidManifest.xml +2 -1
  16. package/flutter_app/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  17. package/flutter_app/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  18. package/flutter_app/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  19. package/flutter_app/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  20. package/flutter_app/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  21. package/flutter_app/assets/branding/app_icon_1024.png +0 -0
  22. package/flutter_app/assets/branding/app_icon_128.png +0 -0
  23. package/flutter_app/assets/branding/app_icon_192.png +0 -0
  24. package/flutter_app/assets/branding/app_icon_256.png +0 -0
  25. package/flutter_app/assets/branding/app_icon_32.png +0 -0
  26. package/flutter_app/assets/branding/app_icon_512.png +0 -0
  27. package/flutter_app/assets/branding/app_icon_64.png +0 -0
  28. package/flutter_app/assets/branding/app_icon_light_1024.png +0 -0
  29. package/flutter_app/assets/branding/app_icon_light_128.png +0 -0
  30. package/flutter_app/assets/branding/app_icon_light_192.png +0 -0
  31. package/flutter_app/assets/branding/app_icon_light_256.png +0 -0
  32. package/flutter_app/assets/branding/app_icon_light_32.png +0 -0
  33. package/flutter_app/assets/branding/app_icon_light_512.png +0 -0
  34. package/flutter_app/assets/branding/app_icon_light_64.png +0 -0
  35. package/flutter_app/assets/branding/onboarding_intro.mp4 +0 -0
  36. package/flutter_app/assets/branding/tray_icon_light_template.png +0 -0
  37. package/flutter_app/assets/branding/tray_icon_template.png +0 -0
  38. package/flutter_app/lib/features/location/location_service.dart +3 -0
  39. package/flutter_app/lib/features/onboarding/onboarding_chrome.dart +391 -382
  40. package/flutter_app/lib/features/onboarding/onboarding_companion_step.dart +743 -0
  41. package/flutter_app/lib/features/onboarding/onboarding_messaging_step.dart +18 -16
  42. package/flutter_app/lib/features/onboarding/onboarding_model_step.dart +19 -18
  43. package/flutter_app/lib/features/onboarding/onboarding_shell.dart +8 -1
  44. package/flutter_app/lib/features/onboarding/onboarding_video_step.dart +16 -13
  45. package/flutter_app/lib/features/onboarding/onboarding_welcome_step.dart +17 -13
  46. package/flutter_app/lib/main.dart +3 -0
  47. package/flutter_app/lib/main_account_settings.dart +10 -34
  48. package/flutter_app/lib/main_admin.dart +14 -1
  49. package/flutter_app/lib/main_app_shell.dart +377 -340
  50. package/flutter_app/lib/main_chat.dart +707 -227
  51. package/flutter_app/lib/main_controller.dart +338 -46
  52. package/flutter_app/lib/main_devices.dart +851 -122
  53. package/flutter_app/lib/main_integrations.dart +255 -6
  54. package/flutter_app/lib/main_launcher.dart +1 -1
  55. package/flutter_app/lib/main_model_picker.dart +685 -0
  56. package/flutter_app/lib/main_models.dart +212 -0
  57. package/flutter_app/lib/main_navigation.dart +1 -9
  58. package/flutter_app/lib/main_operations.dart +1417 -614
  59. package/flutter_app/lib/main_settings.dart +764 -887
  60. package/flutter_app/lib/main_shared.dart +971 -443
  61. package/flutter_app/lib/main_spacing.dart +4 -4
  62. package/flutter_app/lib/main_theme.dart +22 -14
  63. package/flutter_app/lib/main_unified.dart +5 -72
  64. package/flutter_app/lib/src/android_apk_drop_zone.dart +24 -0
  65. package/flutter_app/lib/src/android_apk_drop_zone_stub.dart +13 -0
  66. package/flutter_app/lib/src/android_apk_drop_zone_web.dart +219 -0
  67. package/flutter_app/lib/src/backend_client.dart +79 -0
  68. package/flutter_app/lib/src/desktop_companion_actions.dart +56 -7
  69. package/flutter_app/lib/src/desktop_companion_io.dart +77 -1
  70. package/flutter_app/lib/src/desktop_native_bridge.dart +13 -0
  71. package/flutter_app/lib/src/security/password_strength.dart +84 -0
  72. package/flutter_app/lib/src/stream_renderer.dart +205 -35
  73. package/flutter_app/lib/src/theme/palette.dart +76 -34
  74. package/flutter_app/linux/runner/resources/app_icon.png +0 -0
  75. package/flutter_app/macos/Runner/AppDelegate.swift +44 -0
  76. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png +0 -0
  77. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png +0 -0
  78. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png +0 -0
  79. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png +0 -0
  80. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png +0 -0
  81. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png +0 -0
  82. package/flutter_app/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png +0 -0
  83. package/flutter_app/pubspec.lock +2 -2
  84. package/flutter_app/pubspec.yaml +7 -1
  85. package/flutter_app/tool/branding_source/neoagent-icon-1024.png +0 -0
  86. package/flutter_app/tool/branding_source/neoagent-icon-128.png +0 -0
  87. package/flutter_app/tool/branding_source/neoagent-icon-16.png +0 -0
  88. package/flutter_app/tool/branding_source/neoagent-icon-180.png +0 -0
  89. package/flutter_app/tool/branding_source/neoagent-icon-192.png +0 -0
  90. package/flutter_app/tool/branding_source/neoagent-icon-256.png +0 -0
  91. package/flutter_app/tool/branding_source/neoagent-icon-32.png +0 -0
  92. package/flutter_app/tool/branding_source/neoagent-icon-48.png +0 -0
  93. package/flutter_app/tool/branding_source/neoagent-icon-512.png +0 -0
  94. package/flutter_app/tool/branding_source/neoagent-icon-64.png +0 -0
  95. package/flutter_app/tool/branding_source/neoagent-icon.svg +43 -0
  96. package/flutter_app/tool/generate_desktop_branding.py +154 -152
  97. package/flutter_app/web/favicon.png +0 -0
  98. package/flutter_app/web/favicon.svg +40 -9
  99. package/flutter_app/web/favicon_light.svg +43 -0
  100. package/flutter_app/web/icons/Icon-192-light.png +0 -0
  101. package/flutter_app/web/icons/Icon-192.png +0 -0
  102. package/flutter_app/web/icons/Icon-512-light.png +0 -0
  103. package/flutter_app/web/icons/Icon-512.png +0 -0
  104. package/flutter_app/web/icons/Icon-maskable-192-light.png +0 -0
  105. package/flutter_app/web/icons/Icon-maskable-192.png +0 -0
  106. package/flutter_app/web/icons/Icon-maskable-512-light.png +0 -0
  107. package/flutter_app/web/icons/Icon-maskable-512.png +0 -0
  108. package/flutter_app/windows/runner/main.cpp +7 -1
  109. package/flutter_app/windows/runner/resources/app_icon.ico +0 -0
  110. package/lib/manager.js +445 -83
  111. package/package.json +17 -3
  112. package/runtime/paths.js +3 -1
  113. package/server/admin/access.js +198 -0
  114. package/server/admin/admin.css +268 -0
  115. package/server/admin/admin.js +348 -0
  116. package/server/admin/analytics.js +128 -0
  117. package/server/admin/index.html +1015 -0
  118. package/server/admin/login.html +290 -0
  119. package/server/admin/logo.svg +43 -0
  120. package/server/admin/sql.js +134 -0
  121. package/server/admin/users.js +147 -0
  122. package/server/config/origins.js +3 -1
  123. package/server/db/database.js +92 -0
  124. package/server/db/ftsQuery.js +27 -0
  125. package/server/http/routes.js +1 -0
  126. package/server/http/static.js +23 -6
  127. package/server/index.js +1 -1
  128. package/server/middleware/adminAuth.js +48 -0
  129. package/server/middleware/auth.js +1 -40
  130. package/server/public/.last_build_id +1 -1
  131. package/server/public/app_icon.png +0 -0
  132. package/server/public/assets/AssetManifest.bin +1 -1
  133. package/server/public/assets/AssetManifest.bin.json +1 -1
  134. package/server/public/assets/assets/branding/app_icon_1024.png +0 -0
  135. package/server/public/assets/assets/branding/app_icon_256.png +0 -0
  136. package/server/public/assets/assets/branding/app_icon_512.png +0 -0
  137. package/server/public/assets/assets/branding/app_icon_light_1024.png +0 -0
  138. package/server/public/assets/assets/branding/app_icon_light_256.png +0 -0
  139. package/server/public/assets/assets/branding/app_icon_light_512.png +0 -0
  140. package/server/public/assets/assets/branding/onboarding_intro.mp4 +0 -0
  141. package/server/public/assets/assets/branding/tray_icon_light_template.png +0 -0
  142. package/server/public/assets/assets/branding/tray_icon_template.png +0 -0
  143. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  144. package/server/public/assets/web/icons/Icon-192.png +0 -0
  145. package/server/public/favicon.png +0 -0
  146. package/server/public/favicon.svg +40 -9
  147. package/server/public/favicon_light.svg +43 -0
  148. package/server/public/flutter_bootstrap.js +2 -2
  149. package/server/public/icons/Icon-192-light.png +0 -0
  150. package/server/public/icons/Icon-192.png +0 -0
  151. package/server/public/icons/Icon-512-light.png +0 -0
  152. package/server/public/icons/Icon-512.png +0 -0
  153. package/server/public/icons/Icon-maskable-192-light.png +0 -0
  154. package/server/public/icons/Icon-maskable-192.png +0 -0
  155. package/server/public/icons/Icon-maskable-512-light.png +0 -0
  156. package/server/public/icons/Icon-maskable-512.png +0 -0
  157. package/server/public/main.dart.js +88944 -85720
  158. package/server/routes/admin.js +632 -0
  159. package/server/routes/agent_profiles.js +3 -0
  160. package/server/routes/agents.js +1 -1
  161. package/server/routes/auth.js +3 -1
  162. package/server/routes/browser.js +14 -0
  163. package/server/routes/browser_extension.js +21 -4
  164. package/server/routes/desktop.js +10 -0
  165. package/server/routes/mcp.js +29 -13
  166. package/server/routes/memory.js +23 -1
  167. package/server/routes/messaging.js +2 -0
  168. package/server/routes/screenHistory.js +14 -8
  169. package/server/routes/settings.js +14 -2
  170. package/server/routes/stream.js +12 -3
  171. package/server/routes/triggers.js +4 -4
  172. package/server/routes/voice_assistant.js +36 -1
  173. package/server/routes/workspace.js +86 -0
  174. package/server/services/account/admin_two_factor.js +132 -0
  175. package/server/services/account/password_policy.js +6 -1
  176. package/server/services/ai/completion.js +44 -0
  177. package/server/services/ai/engine.js +321 -500
  178. package/server/services/ai/imageAnalysis.js +9 -5
  179. package/server/services/ai/logFormat.js +46 -0
  180. package/server/services/ai/loopPolicy.js +11 -0
  181. package/server/services/ai/messagingFallback.js +228 -0
  182. package/server/services/ai/models.js +194 -55
  183. package/server/services/ai/providerRetry.js +169 -0
  184. package/server/services/ai/providers/anthropic.js +15 -4
  185. package/server/services/ai/providers/google.js +38 -11
  186. package/server/services/ai/providers/grok.js +19 -68
  187. package/server/services/ai/providers/grokOauth.js +141 -0
  188. package/server/services/ai/providers/nvidia.js +154 -0
  189. package/server/services/ai/providers/ollama.js +76 -39
  190. package/server/services/ai/providers/openai.js +20 -31
  191. package/server/services/ai/providers/openaiCodex.js +6 -2
  192. package/server/services/ai/providers/openaiCompatible.js +70 -0
  193. package/server/services/ai/providers/openrouter.js +162 -0
  194. package/server/services/ai/settings.js +30 -0
  195. package/server/services/ai/systemPrompt.js +51 -29
  196. package/server/services/ai/taskAnalysis.js +60 -1
  197. package/server/services/ai/toolEvidence.js +207 -0
  198. package/server/services/ai/toolSelector.js +8 -1
  199. package/server/services/ai/tools.js +80 -15
  200. package/server/services/android/controller.js +36 -1
  201. package/server/services/browser/controller.js +18 -0
  202. package/server/services/browser/extension/manifest.js +33 -0
  203. package/server/services/browser/extension/provider.js +12 -6
  204. package/server/services/browser/extension/registry.js +229 -18
  205. package/server/services/desktop/protocol.js +1 -0
  206. package/server/services/desktop/provider.js +4 -0
  207. package/server/services/desktop/registry.js +104 -1
  208. package/server/services/desktop/screenRecorder.js +208 -98
  209. package/server/services/desktop/screen_recorder_support.js +46 -0
  210. package/server/services/integrations/google/provider.js +13 -0
  211. package/server/services/integrations/home_assistant/constants.js +88 -0
  212. package/server/services/integrations/home_assistant/network.js +207 -0
  213. package/server/services/integrations/home_assistant/provider.js +249 -0
  214. package/server/services/integrations/home_assistant/snapshot.js +98 -0
  215. package/server/services/integrations/home_assistant/tools.js +101 -0
  216. package/server/services/integrations/manager.js +5 -0
  217. package/server/services/integrations/registry.js +2 -0
  218. package/server/services/integrations/trello/provider.js +8 -2
  219. package/server/services/manager.js +51 -47
  220. package/server/services/mcp/client.js +208 -268
  221. package/server/services/mcp/client_support.js +172 -0
  222. package/server/services/mcp/recovery.js +116 -0
  223. package/server/services/mcp/tool_operations.js +123 -0
  224. package/server/services/memory/ingestion.js +185 -370
  225. package/server/services/memory/ingestion_coverage.js +129 -0
  226. package/server/services/memory/ingestion_documents.js +123 -0
  227. package/server/services/memory/ingestion_support.js +171 -0
  228. package/server/services/memory/intelligence.js +181 -0
  229. package/server/services/memory/manager.js +476 -32
  230. package/server/services/messaging/automation.js +71 -134
  231. package/server/services/messaging/formatting_guides.js +26 -1
  232. package/server/services/messaging/inbound_queue.js +111 -0
  233. package/server/services/messaging/manager.js +25 -8
  234. package/server/services/messaging/typing_keepalive.js +110 -0
  235. package/server/services/runtime/manager.js +7 -3
  236. package/server/services/runtime/settings.js +17 -0
  237. package/server/services/streaming/android-stream.js +296 -32
  238. package/server/services/streaming/stream-hub.js +1 -1
  239. package/server/services/tasks/integration_runtime.js +4 -1
  240. package/server/services/tasks/runtime.js +415 -56
  241. package/server/services/tasks/task_repository.js +48 -6
  242. package/server/services/websocket.js +8 -2
  243. package/server/services/widgets/service.js +17 -1
  244. package/server/services/workspace/manager.js +116 -0
  245. package/server/utils/logger.js +44 -6
  246. package/server/utils/security.js +3 -0
  247. package/server/utils/version.js +29 -19
  248. package/server/services/memory/openhuman_uplift.test.js +0 -98
@@ -23,11 +23,11 @@ abstract class AppBreakpoints {
23
23
  abstract class AppRadius {
24
24
  static const double tag = 8.0;
25
25
  static const double card = 14.0;
26
- static const double input = 18.0;
27
- static const double panel = 28.0;
26
+ static const double input = 14.0;
27
+ static const double panel = 22.0;
28
28
  static const double pill = 999.0;
29
29
  }
30
30
 
31
31
  /// Deep dark color used for QR code modules rendered on a white background.
32
- /// Matches the dark theme's primary background hue for brand consistency.
33
- const Color _qrDarkColor = Color(0xFF04111D);
32
+ /// Matches the dark theme's deep-olive background hue for brand consistency.
33
+ const Color _qrDarkColor = Color(0xFF0E1511);
@@ -1,5 +1,7 @@
1
1
  part of 'main.dart';
2
2
 
3
+ // ignore_for_file: unused_element
4
+
3
5
  const NeoAgentPalette _darkPalette = darkPalette;
4
6
  const NeoAgentPalette _lightPalette = lightPalette;
5
7
 
@@ -16,8 +18,8 @@ Color get _bgCard => _palette.bgCard;
16
18
  Color get _textPrimary => _palette.textPrimary;
17
19
  Color get _textSecondary => _palette.textSecondary;
18
20
  Color get _textMuted => _palette.textMuted;
19
- const Color _brandAccent = Color(0xFF8F6D3E);
20
- const Color _brandAccentAlt = Color(0xFF2F7D6E);
21
+ const Color _brandAccent = Color(0xFFB07D2B);
22
+ const Color _brandAccentAlt = Color(0xFF5E6B4C);
21
23
  Color get _accent => _palette.accent;
22
24
  Color get _accentHover => _palette.accentHover;
23
25
  Color get _accentAlt => _palette.accentAlt;
@@ -102,10 +104,12 @@ TextStyle _heroTitleStyle([double size = 24]) => TextStyle(
102
104
  color: _textPrimary,
103
105
  );
104
106
 
105
- TextStyle _sectionEyebrowStyle() => TextStyle(
106
- fontSize: 12,
107
- fontWeight: FontWeight.w700,
108
- letterSpacing: 1.0,
107
+ // Eyebrow labels ("CONTROL SURFACE") use the mono typeface for telemetry
108
+ // flavor, matching the design system's `--mono` eyebrow treatment (Geist Mono).
109
+ TextStyle _sectionEyebrowStyle() => GoogleFonts.geistMono(
110
+ fontSize: 11,
111
+ fontWeight: FontWeight.w600,
112
+ letterSpacing: 1.6,
109
113
  color: _accentHover,
110
114
  );
111
115
 
@@ -128,7 +132,9 @@ ThemeData _buildNeoAgentTheme(NeoAgentPalette palette, Brightness brightness) {
128
132
  onSurface: palette.textPrimary,
129
133
  error: palette.danger,
130
134
  ),
131
- textTheme: base.textTheme.apply(
135
+ // Geist is the Control Surface design system's base typeface.
136
+ // Telemetry / mono runs stay on Geist Mono at the call sites.
137
+ textTheme: GoogleFonts.geistTextTheme(base.textTheme).apply(
132
138
  bodyColor: palette.textPrimary,
133
139
  displayColor: palette.textPrimary,
134
140
  ),
@@ -143,7 +149,7 @@ ThemeData _buildNeoAgentTheme(NeoAgentPalette palette, Brightness brightness) {
143
149
  elevation: brightness == Brightness.dark ? 8 : 3,
144
150
  margin: EdgeInsets.zero,
145
151
  shape: RoundedRectangleBorder(
146
- borderRadius: BorderRadius.circular(28),
152
+ borderRadius: BorderRadius.circular(AppRadius.panel),
147
153
  side: BorderSide(
148
154
  color: Colors.white.withValues(
149
155
  alpha: brightness == Brightness.dark ? 0.08 : 0.22,
@@ -157,15 +163,15 @@ ThemeData _buildNeoAgentTheme(NeoAgentPalette palette, Brightness brightness) {
157
163
  alpha: brightness == Brightness.dark ? 0.82 : 0.84,
158
164
  ),
159
165
  border: OutlineInputBorder(
160
- borderRadius: BorderRadius.circular(18),
166
+ borderRadius: BorderRadius.circular(AppRadius.input),
161
167
  borderSide: BorderSide(color: palette.borderLight),
162
168
  ),
163
169
  enabledBorder: OutlineInputBorder(
164
- borderRadius: BorderRadius.circular(18),
170
+ borderRadius: BorderRadius.circular(AppRadius.input),
165
171
  borderSide: BorderSide(color: palette.borderLight),
166
172
  ),
167
173
  focusedBorder: OutlineInputBorder(
168
- borderRadius: BorderRadius.circular(18),
174
+ borderRadius: BorderRadius.circular(AppRadius.input),
169
175
  borderSide: BorderSide(color: palette.accentHover, width: 1.4),
170
176
  ),
171
177
  contentPadding: const EdgeInsets.symmetric(horizontal: 16, vertical: 16),
@@ -191,7 +197,7 @@ ThemeData _buildNeoAgentTheme(NeoAgentPalette palette, Brightness brightness) {
191
197
  elevation: 0,
192
198
  padding: const EdgeInsets.symmetric(horizontal: 18, vertical: 16),
193
199
  shape: RoundedRectangleBorder(
194
- borderRadius: BorderRadius.circular(18),
200
+ borderRadius: BorderRadius.circular(AppRadius.input),
195
201
  side: BorderSide(
196
202
  color: Colors.white.withValues(
197
203
  alpha: brightness == Brightness.dark ? 0.14 : 0.26,
@@ -217,7 +223,9 @@ ThemeData _buildNeoAgentTheme(NeoAgentPalette palette, Brightness brightness) {
217
223
  alpha: brightness == Brightness.dark ? 0.2 : 0.5,
218
224
  ),
219
225
  padding: const EdgeInsets.symmetric(horizontal: 18, vertical: 16),
220
- shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(18)),
226
+ shape: RoundedRectangleBorder(
227
+ borderRadius: BorderRadius.circular(AppRadius.input),
228
+ ),
221
229
  textStyle: const TextStyle(fontSize: 15, fontWeight: FontWeight.w700),
222
230
  ),
223
231
  ),
@@ -264,7 +272,7 @@ ThemeData _buildNeoAgentTheme(NeoAgentPalette palette, Brightness brightness) {
264
272
  backgroundColor: palette.bgCard.withValues(alpha: 0.96),
265
273
  surfaceTintColor: Colors.transparent,
266
274
  shape: RoundedRectangleBorder(
267
- borderRadius: BorderRadius.circular(24),
275
+ borderRadius: BorderRadius.circular(AppRadius.panel),
268
276
  side: BorderSide(color: palette.borderLight),
269
277
  ),
270
278
  ),
@@ -2,7 +2,7 @@ part of 'main.dart';
2
2
 
3
3
  enum _ToolsPageTab { integrations, mcp, skills }
4
4
 
5
- enum _RunsPageTab { runs, logs }
5
+ enum _RunsPageTab { runs }
6
6
 
7
7
  enum _SettingsWorkspaceSection { app, account, security }
8
8
 
@@ -126,49 +126,7 @@ class RunsAndLogsPanel extends StatefulWidget {
126
126
  State<RunsAndLogsPanel> createState() => _RunsAndLogsPanelState();
127
127
  }
128
128
 
129
- class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
130
- with SingleTickerProviderStateMixin {
131
- late final TabController _tabController;
132
-
133
- @override
134
- void initState() {
135
- super.initState();
136
- _tabController = TabController(
137
- length: _RunsPageTab.values.length,
138
- vsync: this,
139
- initialIndex: _tabForSection(widget.controller.selectedSection).index,
140
- );
141
- }
142
-
143
- @override
144
- void didUpdateWidget(covariant RunsAndLogsPanel oldWidget) {
145
- super.didUpdateWidget(oldWidget);
146
- final selectedSection = widget.controller.selectedSection;
147
- if (selectedSection != oldWidget.controller.selectedSection &&
148
- (selectedSection == AppSection.runs ||
149
- selectedSection == AppSection.logs)) {
150
- final targetIndex = _tabForSection(selectedSection).index;
151
- if (_tabController.index != targetIndex) {
152
- _tabController.index = targetIndex;
153
- }
154
- }
155
- }
156
-
157
- @override
158
- void dispose() {
159
- _tabController.dispose();
160
- super.dispose();
161
- }
162
-
163
- _RunsPageTab _tabForSection(AppSection section) {
164
- switch (section) {
165
- case AppSection.logs:
166
- return _RunsPageTab.logs;
167
- default:
168
- return _RunsPageTab.runs;
169
- }
170
- }
171
-
129
+ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel> {
172
130
  @override
173
131
  Widget build(BuildContext context) {
174
132
  final controller = widget.controller;
@@ -177,37 +135,12 @@ class _RunsAndLogsPanelState extends State<RunsAndLogsPanel>
177
135
  child: Column(
178
136
  children: <Widget>[
179
137
  const _PageTitle(
180
- title: 'Runs & Logs',
181
- subtitle:
182
- 'Inspect execution history, failures, tool traces, and diagnostics from one workspace.',
183
- ),
184
- const SizedBox(height: 12),
185
- Container(
186
- decoration: BoxDecoration(
187
- color: _bgSecondary,
188
- borderRadius: BorderRadius.circular(14),
189
- border: Border.all(color: _border),
190
- ),
191
- child: TabBar(
192
- controller: _tabController,
193
- dividerColor: _border,
194
- indicatorSize: TabBarIndicatorSize.tab,
195
- labelStyle: const TextStyle(fontWeight: FontWeight.w700),
196
- tabs: <Widget>[
197
- Tab(text: 'Runs (${controller.recentRuns.length})'),
198
- Tab(text: 'Logs (${controller.logs.length})'),
199
- ],
200
- ),
138
+ title: 'Runs',
139
+ subtitle: 'Inspect execution history, failures, and tool traces.',
201
140
  ),
202
141
  const SizedBox(height: 12),
203
142
  Expanded(
204
- child: TabBarView(
205
- controller: _tabController,
206
- children: <Widget>[
207
- RunsPanel(controller: controller, embedded: true),
208
- LogsPanel(controller: controller, embedded: true),
209
- ],
210
- ),
143
+ child: RunsPanel(controller: controller, embedded: true),
211
144
  ),
212
145
  ],
213
146
  ),
@@ -30,3 +30,27 @@ class AndroidApkDropZone extends StatelessWidget {
30
30
  );
31
31
  }
32
32
  }
33
+
34
+ /// Compact tile variant — fits inside an actions row.
35
+ class AndroidApkTile extends StatelessWidget {
36
+ const AndroidApkTile({
37
+ super.key,
38
+ required this.enabled,
39
+ required this.busy,
40
+ required this.onInstall,
41
+ });
42
+
43
+ final bool enabled;
44
+ final bool busy;
45
+ final AndroidApkInstallCallback onInstall;
46
+
47
+ @override
48
+ Widget build(BuildContext context) {
49
+ return buildAndroidApkTile(
50
+ context,
51
+ enabled: enabled,
52
+ busy: busy,
53
+ onInstall: onInstall,
54
+ );
55
+ }
56
+ }
@@ -14,3 +14,16 @@ Widget buildAndroidApkDropZone(
14
14
  }) {
15
15
  return const SizedBox.shrink();
16
16
  }
17
+
18
+ Widget buildAndroidApkTile(
19
+ BuildContext context, {
20
+ required bool enabled,
21
+ required bool busy,
22
+ required Future<void> Function({
23
+ required String filename,
24
+ required Uint8List bytes,
25
+ })
26
+ onInstall,
27
+ }) {
28
+ return const SizedBox.shrink();
29
+ }
@@ -346,3 +346,222 @@ bool _isSupportedInstallFile(String filename) {
346
346
  final normalized = filename.toLowerCase();
347
347
  return _supportedAndroidInstallExtensions.any(normalized.endsWith);
348
348
  }
349
+
350
+ // ─────────────────────────────────────────────────────────────────────────────
351
+ // Compact tile variant — used inside the Android actions box.
352
+ // Same file-picker / drag-and-drop logic but rendered as a small square tile
353
+ // that fits neatly alongside other action tiles.
354
+ // ─────────────────────────────────────────────────────────────────────────────
355
+
356
+ int _androidApkTileViewId = 0;
357
+
358
+ Widget buildAndroidApkTile(
359
+ BuildContext context, {
360
+ required bool enabled,
361
+ required bool busy,
362
+ required Future<void> Function({
363
+ required String filename,
364
+ required Uint8List bytes,
365
+ })
366
+ onInstall,
367
+ }) {
368
+ return _AndroidApkTileWeb(enabled: enabled, busy: busy, onInstall: onInstall);
369
+ }
370
+
371
+ class _AndroidApkTileWeb extends StatefulWidget {
372
+ const _AndroidApkTileWeb({
373
+ required this.enabled,
374
+ required this.busy,
375
+ required this.onInstall,
376
+ });
377
+
378
+ final bool enabled;
379
+ final bool busy;
380
+ final Future<void> Function({required String filename, required Uint8List bytes}) onInstall;
381
+
382
+ @override
383
+ State<_AndroidApkTileWeb> createState() => _AndroidApkTileWebState();
384
+ }
385
+
386
+ class _AndroidApkTileWebState extends State<_AndroidApkTileWeb> {
387
+ late final String _viewType;
388
+ late final html.DivElement _dropElement;
389
+ late final html.FileUploadInputElement _fileInput;
390
+ final List<StreamSubscription<dynamic>> _subs = [];
391
+ bool _dragActive = false;
392
+
393
+ @override
394
+ void initState() {
395
+ super.initState();
396
+ _viewType = 'neoagent-android-apk-tile-${_androidApkTileViewId++}';
397
+ _dropElement = html.DivElement()
398
+ ..setAttribute('role', 'button')
399
+ ..setAttribute('aria-label', 'Install APK — click or drop a .apk file')
400
+ ..tabIndex = 0
401
+ ..style.width = '100%'
402
+ ..style.height = '100%'
403
+ ..style.display = 'block'
404
+ ..style.background = 'rgba(0,0,0,0.001)'
405
+ ..style.cursor = 'pointer';
406
+ _fileInput = html.FileUploadInputElement()
407
+ ..accept = '.apk,.apks'
408
+ ..multiple = false
409
+ ..style.display = 'none';
410
+ _dropElement.append(_fileInput);
411
+
412
+ _subs.addAll([
413
+ _dropElement.onClick.listen((_) => _openPicker()),
414
+ _dropElement.onKeyDown.listen((e) {
415
+ if (e.key == 'Enter' || e.key == ' ') {
416
+ e.preventDefault();
417
+ _openPicker();
418
+ }
419
+ }),
420
+ _dropElement.onDragEnter.listen((e) {
421
+ e.preventDefault();
422
+ if (!_dragActive && mounted) setState(() => _dragActive = true);
423
+ }),
424
+ _dropElement.onDragOver.listen((e) {
425
+ e.preventDefault();
426
+ e.dataTransfer.dropEffect = 'copy';
427
+ if (!_dragActive && mounted) setState(() => _dragActive = true);
428
+ }),
429
+ _dropElement.onDragLeave.listen((e) {
430
+ e.preventDefault();
431
+ if (_dragActive && mounted) setState(() => _dragActive = false);
432
+ }),
433
+ _dropElement.onDrop.listen((e) {
434
+ e.preventDefault();
435
+ if (_dragActive && mounted) setState(() => _dragActive = false);
436
+ final files = e.dataTransfer.files;
437
+ if (files != null && files.isNotEmpty) unawaited(_handleFile(files.first));
438
+ }),
439
+ _fileInput.onChange.listen((_) {
440
+ final files = _fileInput.files;
441
+ if (files != null && files.isNotEmpty) unawaited(_handleFile(files.first));
442
+ }),
443
+ ]);
444
+
445
+ ui_web.platformViewRegistry.registerViewFactory(_viewType, (int _) => _dropElement);
446
+ }
447
+
448
+ @override
449
+ void dispose() {
450
+ for (final s in _subs) {
451
+ s.cancel();
452
+ }
453
+ _dropElement.remove();
454
+ super.dispose();
455
+ }
456
+
457
+ void _openPicker() {
458
+ if (!widget.enabled || widget.busy) return;
459
+ _fileInput.value = '';
460
+ _fileInput.click();
461
+ }
462
+
463
+ Future<void> _handleFile(html.File file) async {
464
+ if (!widget.enabled || widget.busy) return;
465
+ if (!_isSupportedInstallFile(file.name)) {
466
+ _showError('Only .apk or .apks files can be installed.');
467
+ return;
468
+ }
469
+ try {
470
+ final bytes = await _readFileBytes(file);
471
+ if (mounted) await widget.onInstall(filename: file.name, bytes: bytes);
472
+ } catch (e) {
473
+ _showError(e.toString().replaceFirst('Exception: ', ''));
474
+ }
475
+ }
476
+
477
+ Future<Uint8List> _readFileBytes(html.File file) {
478
+ final completer = Completer<Uint8List>();
479
+ final reader = html.FileReader();
480
+ reader.onLoad.listen((_) {
481
+ final result = reader.result;
482
+ if (result is ByteBuffer) { completer.complete(Uint8List.view(result)); return; }
483
+ if (result is Uint8List) { completer.complete(result); return; }
484
+ if (!completer.isCompleted) completer.completeError(StateError('Could not read the APK.'));
485
+ });
486
+ reader.onError.listen((_) {
487
+ if (!completer.isCompleted) completer.completeError(reader.error ?? StateError('Read error'));
488
+ });
489
+ reader.readAsArrayBuffer(file);
490
+ return completer.future;
491
+ }
492
+
493
+ void _showError(String msg) {
494
+ if (!mounted) return;
495
+ ScaffoldMessenger.maybeOf(context)?.showSnackBar(SnackBar(content: Text(msg)));
496
+ }
497
+
498
+ @override
499
+ Widget build(BuildContext context) {
500
+ final cs = Theme.of(context).colorScheme;
501
+ final isDark = Theme.of(context).brightness == Brightness.dark;
502
+ final borderColor = _dragActive ? cs.primary : cs.outlineVariant;
503
+ final bgColor = _dragActive
504
+ ? cs.primary.withValues(alpha: isDark ? 0.16 : 0.10)
505
+ : widget.enabled
506
+ ? cs.surfaceContainerHighest.withValues(alpha: isDark ? 0.50 : 0.70)
507
+ : cs.surfaceContainerHighest.withValues(alpha: isDark ? 0.28 : 0.44);
508
+ final iconColor = _dragActive
509
+ ? cs.primary
510
+ : widget.enabled
511
+ ? cs.onSurface
512
+ : cs.onSurface.withValues(alpha: 0.38);
513
+ final labelColor = _dragActive ? cs.primary : cs.onSurfaceVariant;
514
+
515
+ return SizedBox(
516
+ width: 80,
517
+ height: 72,
518
+ child: Stack(
519
+ children: [
520
+ // Visual tile
521
+ Positioned.fill(
522
+ child: AnimatedContainer(
523
+ duration: const Duration(milliseconds: 150),
524
+ decoration: BoxDecoration(
525
+ color: bgColor,
526
+ borderRadius: BorderRadius.circular(12),
527
+ border: Border.all(color: borderColor),
528
+ ),
529
+ child: Column(
530
+ mainAxisAlignment: MainAxisAlignment.center,
531
+ children: [
532
+ Icon(
533
+ widget.busy
534
+ ? Icons.hourglass_top_rounded
535
+ : Icons.install_mobile_outlined,
536
+ size: 20,
537
+ color: iconColor,
538
+ ),
539
+ const SizedBox(height: 5),
540
+ Text(
541
+ widget.busy ? 'Installing…' : 'Install APK',
542
+ style: TextStyle(
543
+ fontSize: 10,
544
+ height: 1.2,
545
+ fontWeight: FontWeight.w600,
546
+ color: labelColor,
547
+ ),
548
+ textAlign: TextAlign.center,
549
+ maxLines: 2,
550
+ overflow: TextOverflow.ellipsis,
551
+ ),
552
+ ],
553
+ ),
554
+ ),
555
+ ),
556
+ // Transparent HTML element captures all drag / click events
557
+ Positioned.fill(
558
+ child: ClipRRect(
559
+ borderRadius: BorderRadius.circular(12),
560
+ child: HtmlElementView(viewType: _viewType),
561
+ ),
562
+ ),
563
+ ],
564
+ ),
565
+ );
566
+ }
567
+ }
@@ -498,6 +498,15 @@ class BackendClient {
498
498
  return getMap(baseUrl, '/api/browser-extension/status');
499
499
  }
500
500
 
501
+ Future<Map<String, dynamic>> selectBrowserExtensionToken(
502
+ String baseUrl, {
503
+ required String tokenId,
504
+ }) async {
505
+ return postMap(baseUrl, '/api/browser-extension/select-token', <String, dynamic>{
506
+ 'tokenId': tokenId,
507
+ });
508
+ }
509
+
501
510
  Future<Map<String, dynamic>> launchBrowser(
502
511
  String baseUrl, {
503
512
  Map<String, dynamic>? payload,
@@ -546,6 +555,17 @@ class BackendClient {
546
555
  });
547
556
  }
548
557
 
558
+ Future<Map<String, dynamic>> hoverBrowserPoint(
559
+ String baseUrl, {
560
+ required int x,
561
+ required int y,
562
+ }) async {
563
+ return postMap(baseUrl, '/api/browser/mouse-move', <String, dynamic>{
564
+ 'x': x,
565
+ 'y': y,
566
+ });
567
+ }
568
+
549
569
  Future<Map<String, dynamic>> fillBrowser(
550
570
  String baseUrl, {
551
571
  required String selector,
@@ -675,6 +695,41 @@ class BackendClient {
675
695
  return getMap(baseUrl, '/api/desktop/devices');
676
696
  }
677
697
 
698
+ Future<Map<String, dynamic>> fetchWorkspaceDirectory(
699
+ String baseUrl, {
700
+ String path = '.',
701
+ }) async {
702
+ return getMap(
703
+ baseUrl,
704
+ '/api/workspace/files?path=${Uri.encodeQueryComponent(path)}',
705
+ );
706
+ }
707
+
708
+ Future<Map<String, dynamic>> fetchWorkspaceFile(
709
+ String baseUrl, {
710
+ required String path,
711
+ }) async {
712
+ return getMap(
713
+ baseUrl,
714
+ '/api/workspace/files/content?path=${Uri.encodeQueryComponent(path)}',
715
+ );
716
+ }
717
+
718
+ Future<Map<String, dynamic>> saveWorkspaceFile(
719
+ String baseUrl, {
720
+ required String path,
721
+ required String content,
722
+ }) async {
723
+ return putMap(baseUrl, '/api/workspace/files/content', <String, dynamic>{
724
+ 'path': path,
725
+ 'content': content,
726
+ });
727
+ }
728
+
729
+ String workspaceDownloadPath(String path) {
730
+ return '/api/workspace/files/download?path=${Uri.encodeQueryComponent(path)}';
731
+ }
732
+
678
733
  Future<Map<String, dynamic>> selectDesktopDevice(
679
734
  String baseUrl, {
680
735
  required String deviceId,
@@ -719,6 +774,19 @@ class BackendClient {
719
774
  });
720
775
  }
721
776
 
777
+ Future<Map<String, dynamic>> hoverDesktop(
778
+ String baseUrl, {
779
+ String? deviceId,
780
+ required int x,
781
+ required int y,
782
+ }) async {
783
+ return postMap(baseUrl, '/api/desktop/mouse-move', <String, dynamic>{
784
+ if (deviceId != null && deviceId.isNotEmpty) 'deviceId': deviceId,
785
+ 'x': x,
786
+ 'y': y,
787
+ });
788
+ }
789
+
722
790
  Future<Map<String, dynamic>> dragDesktop(
723
791
  String baseUrl, {
724
792
  String? deviceId,
@@ -1611,6 +1679,17 @@ class BackendClient {
1611
1679
  await deleteMap(baseUrl, '/api/recordings/$sessionId');
1612
1680
  }
1613
1681
 
1682
+ Future<Map<String, dynamic>> transcribeAudio(
1683
+ String baseUrl, {
1684
+ required String audioBase64,
1685
+ String mimeType = 'audio/pcm;rate=16000;channels=1',
1686
+ }) {
1687
+ return postMap(baseUrl, '/api/voice-assistant/transcribe', <String, dynamic>{
1688
+ 'audioBase64': audioBase64,
1689
+ 'mimeType': mimeType,
1690
+ });
1691
+ }
1692
+
1614
1693
  Future<Map<String, dynamic>> runVoiceAssistantTurn(
1615
1694
  String baseUrl, {
1616
1695
  required String sessionId,
@@ -9,6 +9,20 @@ import 'package:package_info_plus/package_info_plus.dart';
9
9
  import 'desktop_native_bridge.dart';
10
10
  import 'desktop_screen_capture.dart';
11
11
 
12
+ // ─── Isolate helpers for JPEG compression ────────────────────────────────────
13
+ // `compressToJpeg` offloads the CPU-intensive pure-Dart PNG→JPEG conversion
14
+ // to a background isolate via `compute()` so the main isolate's event loop
15
+ // stays free to process incoming WebSocket commands (click, drag, etc.)
16
+ // immediately, rather than queuing behind a 300–600 ms compression job.
17
+
18
+ typedef _JpegArgs = ({Uint8List bytes, int quality});
19
+
20
+ Uint8List _compressJpegInIsolate(_JpegArgs args) {
21
+ final decoded = img.decodeImage(args.bytes);
22
+ if (decoded == null) return args.bytes;
23
+ return Uint8List.fromList(img.encodeJpg(decoded, quality: args.quality));
24
+ }
25
+
12
26
  class DesktopCompanionSnapshot {
13
27
  const DesktopCompanionSnapshot({
14
28
  required this.screenshotBase64,
@@ -86,9 +100,15 @@ class DesktopCompanionActions {
86
100
  if (bytes is! Uint8List || bytes.isEmpty) {
87
101
  return null;
88
102
  }
89
- final decoded = img.decodeImage(bytes);
90
- final width = (frame['width'] as num?)?.round() ?? decoded?.width ?? 0;
91
- final height = (frame['height'] as num?)?.round() ?? decoded?.height ?? 0;
103
+ // Prefer dimensions reported by the native bridge; only fall back to a
104
+ // pure-Dart image decode (which is slow) when the bridge omits them.
105
+ final nativeWidth = (frame['width'] as num?)?.round();
106
+ final nativeHeight = (frame['height'] as num?)?.round();
107
+ final decoded = (nativeWidth == null || nativeHeight == null)
108
+ ? img.decodeImage(bytes)
109
+ : null;
110
+ final width = nativeWidth ?? decoded?.width ?? 0;
111
+ final height = nativeHeight ?? decoded?.height ?? 0;
92
112
  final displays = _normalizeDisplays(
93
113
  frame['displays'],
94
114
  fallbackDisplayId:
@@ -182,11 +202,14 @@ class DesktopCompanionActions {
182
202
  int quality,
183
203
  ) async {
184
204
  final raw = _decodeScreenshotBytes(snapshot.screenshotBase64);
205
+ // Already JPEG — return immediately without any heavy work on this isolate.
185
206
  if (_looksLikeJpeg(raw)) return raw;
186
- final decoded = img.decodeImage(raw);
187
- if (decoded == null) return raw;
188
- final normalizedQuality = quality.clamp(30, 95);
189
- return Uint8List.fromList(img.encodeJpg(decoded, quality: normalizedQuality));
207
+ // Run the pure-Dart PNG decode + JPEG encode in a background isolate so the
208
+ // main isolate's event loop stays responsive for incoming commands.
209
+ return compute(
210
+ _compressJpegInIsolate,
211
+ (bytes: raw, quality: quality.clamp(30, 95)),
212
+ );
190
213
  }
191
214
 
192
215
  Future<Map<String, Object?>> observe({
@@ -235,6 +258,32 @@ class DesktopCompanionActions {
235
258
  return <String, Object?>{'success': true, 'x': x, 'y': y, 'button': button};
236
259
  }
237
260
 
261
+ Future<Map<String, Object?>> mouseMove({
262
+ required int x,
263
+ required int y,
264
+ String? displayId,
265
+ }) async {
266
+ await _assertInputSupported('mouseMove');
267
+ if (_usesNativeDesktopBridge) {
268
+ await _nativeBridge.mouseMove(
269
+ x: x,
270
+ y: y,
271
+ displayId: displayId,
272
+ );
273
+ } else if (defaultTargetPlatform == TargetPlatform.linux) {
274
+ await _run(
275
+ _ShellCommand('xdotool', <String>[
276
+ 'mousemove',
277
+ '$x',
278
+ '$y',
279
+ ]),
280
+ );
281
+ } else {
282
+ throw Exception('mouseMove is not supported on this platform.');
283
+ }
284
+ return <String, Object?>{'success': true, 'x': x, 'y': y};
285
+ }
286
+
238
287
  Future<Map<String, Object?>> drag({
239
288
  required int x1,
240
289
  required int y1,