neoagent 3.2.1-beta.1 → 3.2.1-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/extensions/chrome-browser/background.mjs +318 -88
  2. package/extensions/chrome-browser/http.mjs +136 -0
  3. package/extensions/chrome-browser/protocol.mjs +654 -90
  4. package/flutter_app/lib/main_chat.dart +118 -739
  5. package/flutter_app/lib/main_controller.dart +111 -20
  6. package/flutter_app/lib/main_integrations.dart +607 -8
  7. package/flutter_app/lib/main_models.dart +3 -0
  8. package/flutter_app/lib/main_operations.dart +334 -321
  9. package/flutter_app/lib/main_security.dart +266 -112
  10. package/flutter_app/lib/main_settings.dart +4 -3
  11. package/flutter_app/lib/main_shared.dart +14 -11
  12. package/flutter_app/lib/src/backend_client.dart +78 -0
  13. package/flutter_app/lib/src/desktop_companion_actions.dart +185 -31
  14. package/flutter_app/lib/src/desktop_companion_io.dart +319 -86
  15. package/flutter_app/windows/runner/flutter_window.cpp +143 -32
  16. package/landing/index.html +3 -1
  17. package/lib/manager.js +106 -89
  18. package/lib/schema_migrations.js +115 -13
  19. package/package.json +30 -15
  20. package/runtime/paths.js +49 -5
  21. package/server/db/database.js +2 -2
  22. package/server/guest-agent.cli.package.json +13 -0
  23. package/server/guest_agent.js +85 -40
  24. package/server/http/middleware.js +24 -0
  25. package/server/http/routes.js +11 -6
  26. package/server/public/.last_build_id +1 -1
  27. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  28. package/server/public/flutter_bootstrap.js +2 -2
  29. package/server/public/main.dart.js +73083 -72209
  30. package/server/routes/admin.js +1 -1
  31. package/server/routes/android.js +30 -34
  32. package/server/routes/browser.js +23 -15
  33. package/server/routes/desktop.js +18 -1
  34. package/server/routes/integrations.js +107 -1
  35. package/server/routes/memory.js +1 -0
  36. package/server/routes/settings.js +16 -5
  37. package/server/routes/social_reach.js +12 -3
  38. package/server/routes/social_video.js +4 -0
  39. package/server/services/agents/manager.js +1 -1
  40. package/server/services/ai/capabilityHealth.js +62 -96
  41. package/server/services/ai/compaction.js +7 -2
  42. package/server/services/ai/history.js +45 -6
  43. package/server/services/ai/integrated_tools/http_request.js +8 -0
  44. package/server/services/ai/loop/agent_engine_core.js +390 -162
  45. package/server/services/ai/loop/blank_recovery.js +5 -4
  46. package/server/services/ai/loop/callbacks.js +1 -0
  47. package/server/services/ai/loop/completion_judge.js +121 -5
  48. package/server/services/ai/loop/conversation_loop.js +510 -332
  49. package/server/services/ai/loop/messaging_delivery.js +129 -57
  50. package/server/services/ai/loop/model_call_guard.js +91 -0
  51. package/server/services/ai/loop/model_io.js +20 -45
  52. package/server/services/ai/loop/progress_classification.js +2 -0
  53. package/server/services/ai/loop/tool_dispatch.js +19 -8
  54. package/server/services/ai/loopPolicy.js +48 -21
  55. package/server/services/ai/messagingFallback.js +17 -17
  56. package/server/services/ai/model_discovery.js +227 -0
  57. package/server/services/ai/model_failure_cache.js +108 -0
  58. package/server/services/ai/model_identity.js +71 -0
  59. package/server/services/ai/models.js +68 -163
  60. package/server/services/ai/providerRetry.js +17 -59
  61. package/server/services/ai/provider_selector.js +166 -0
  62. package/server/services/ai/providers/anthropic.js +2 -2
  63. package/server/services/ai/providers/claudeCode.js +21 -33
  64. package/server/services/ai/providers/githubCopilot.js +41 -20
  65. package/server/services/ai/providers/google.js +135 -97
  66. package/server/services/ai/providers/grok.js +4 -3
  67. package/server/services/ai/providers/grokOauth.js +19 -27
  68. package/server/services/ai/providers/nvidia.js +10 -5
  69. package/server/services/ai/providers/ollama.js +111 -84
  70. package/server/services/ai/providers/ollama_stream.js +142 -0
  71. package/server/services/ai/providers/openai.js +39 -5
  72. package/server/services/ai/providers/openaiCodex.js +11 -4
  73. package/server/services/ai/providers/openrouter.js +29 -7
  74. package/server/services/ai/providers/provider_error.js +36 -0
  75. package/server/services/ai/settings.js +26 -2
  76. package/server/services/ai/systemPrompt.js +19 -12
  77. package/server/services/ai/taskAnalysis.js +58 -10
  78. package/server/services/ai/terminal_reply.js +18 -0
  79. package/server/services/ai/toolEvidence.js +350 -29
  80. package/server/services/ai/tools.js +190 -111
  81. package/server/services/android/controller.js +770 -237
  82. package/server/services/android/process.js +140 -0
  83. package/server/services/android/sdk_download.js +143 -0
  84. package/server/services/android/uia.js +6 -5
  85. package/server/services/artifacts/store.js +24 -0
  86. package/server/services/browser/controller.js +843 -385
  87. package/server/services/browser/extension/gateway.js +40 -16
  88. package/server/services/browser/extension/protocol.js +15 -1
  89. package/server/services/browser/extension/provider.js +71 -47
  90. package/server/services/browser/extension/registry.js +155 -34
  91. package/server/services/cli/executor.js +62 -9
  92. package/server/services/credentials/bitwarden_cli.js +322 -0
  93. package/server/services/credentials/broker.js +594 -0
  94. package/server/services/desktop/gateway.js +41 -4
  95. package/server/services/desktop/protocol.js +3 -0
  96. package/server/services/desktop/provider.js +39 -42
  97. package/server/services/desktop/registry.js +137 -52
  98. package/server/services/integrations/bitwarden/constants.js +14 -0
  99. package/server/services/integrations/bitwarden/provider.js +197 -0
  100. package/server/services/integrations/bitwarden/snapshot.js +65 -0
  101. package/server/services/integrations/figma/provider.js +78 -12
  102. package/server/services/integrations/github/common.js +11 -6
  103. package/server/services/integrations/github/provider.js +52 -53
  104. package/server/services/integrations/google/provider.js +55 -19
  105. package/server/services/integrations/home_assistant/network.js +17 -20
  106. package/server/services/integrations/home_assistant/provider.js +7 -5
  107. package/server/services/integrations/home_assistant/tools.js +17 -5
  108. package/server/services/integrations/http.js +51 -0
  109. package/server/services/integrations/manager.js +159 -53
  110. package/server/services/integrations/microsoft/provider.js +80 -13
  111. package/server/services/integrations/neoarchive/provider.js +55 -29
  112. package/server/services/integrations/neorecall/client.js +17 -10
  113. package/server/services/integrations/neorecall/provider.js +20 -11
  114. package/server/services/integrations/notion/provider.js +16 -13
  115. package/server/services/integrations/oauth_provider.js +115 -51
  116. package/server/services/integrations/registry.js +2 -0
  117. package/server/services/integrations/slack/provider.js +98 -9
  118. package/server/services/integrations/spotify/provider.js +67 -71
  119. package/server/services/integrations/trello/provider.js +21 -7
  120. package/server/services/integrations/weather/provider.js +18 -12
  121. package/server/services/integrations/whatsapp/provider.js +76 -16
  122. package/server/services/manager.js +110 -1
  123. package/server/services/memory/embedding_index.js +20 -8
  124. package/server/services/memory/embeddings.js +151 -90
  125. package/server/services/memory/ingestion.js +50 -9
  126. package/server/services/memory/ingestion_documents.js +13 -3
  127. package/server/services/memory/manager.js +52 -19
  128. package/server/services/messaging/automation.js +85 -10
  129. package/server/services/messaging/formatting_guides.js +7 -4
  130. package/server/services/messaging/http_platforms.js +33 -13
  131. package/server/services/messaging/inbound_queue.js +78 -24
  132. package/server/services/messaging/inbound_store.js +224 -0
  133. package/server/services/messaging/manager.js +326 -51
  134. package/server/services/messaging/typing_keepalive.js +5 -2
  135. package/server/services/messaging/whatsapp.js +22 -14
  136. package/server/services/network/http.js +210 -0
  137. package/server/services/network/safe_request.js +307 -0
  138. package/server/services/runtime/backends/local-vm.js +227 -67
  139. package/server/services/runtime/docker-vm-manager.js +9 -0
  140. package/server/services/runtime/guest_bootstrap.js +30 -4
  141. package/server/services/runtime/guest_image.js +43 -12
  142. package/server/services/runtime/manager.js +77 -23
  143. package/server/services/runtime/validation.js +7 -6
  144. package/server/services/security/tool_categories.js +6 -0
  145. package/server/services/social_reach/channels/github.js +10 -4
  146. package/server/services/social_reach/channels/reddit.js +4 -4
  147. package/server/services/social_reach/channels/rss.js +2 -2
  148. package/server/services/social_reach/channels/social_video.js +12 -7
  149. package/server/services/social_reach/channels/v2ex.js +21 -8
  150. package/server/services/social_reach/channels/x.js +2 -2
  151. package/server/services/social_reach/channels/xueqiu.js +5 -5
  152. package/server/services/social_reach/service.js +9 -6
  153. package/server/services/social_reach/utils.js +65 -14
  154. package/server/services/social_video/service.js +160 -50
  155. package/server/services/tasks/integration_runtime.js +18 -8
  156. package/server/services/tasks/runtime.js +39 -4
  157. package/server/services/voice/agentBridge.js +17 -4
  158. package/server/services/voice/bufferedLiveRelayAdapter.js +5 -0
  159. package/server/services/voice/liveSession.js +31 -0
  160. package/server/services/voice/message.js +1 -1
  161. package/server/services/voice/openaiSpeech.js +33 -8
  162. package/server/services/voice/providers.js +233 -151
  163. package/server/services/voice/runtime.js +2 -2
  164. package/server/services/voice/runtimeManager.js +118 -20
  165. package/server/services/voice/turnRunner.js +6 -0
  166. package/server/services/wearable/firmware_manifest.js +51 -13
  167. package/server/services/wearable/service.js +1 -0
  168. package/server/utils/abort.js +96 -0
  169. package/server/utils/cloud-security.js +110 -3
  170. package/server/utils/files.js +31 -0
  171. package/server/utils/image_payload.js +95 -0
  172. package/server/utils/retry.js +107 -0
@@ -1,6 +1,10 @@
1
+ 'use strict';
2
+
1
3
  const fs = require('fs');
2
4
  const path = require('path');
3
5
  const { DATA_DIR } = require('../../../../runtime/paths');
6
+ const { fetchResponseText, waitForAbortableResult } = require('../../network/http');
7
+ const { createAbortError } = require('../../../utils/abort');
4
8
 
5
9
  const APK_UPLOAD_ROOT = path.resolve(
6
10
  process.env.NEOAGENT_ANDROID_APK_BASE_DIR
@@ -13,6 +17,25 @@ function sleep(ms) {
13
17
  return new Promise((resolve) => setTimeout(resolve, ms));
14
18
  }
15
19
 
20
+ function abortError(signal, fallback = 'Operation aborted.') {
21
+ return createAbortError(signal, fallback);
22
+ }
23
+
24
+ function delayWithSignal(ms, signal) {
25
+ if (signal?.aborted) return Promise.reject(abortError(signal));
26
+ return new Promise((resolve, reject) => {
27
+ const timer = setTimeout(() => {
28
+ signal?.removeEventListener('abort', onAbort);
29
+ resolve();
30
+ }, ms);
31
+ const onAbort = () => {
32
+ clearTimeout(timer);
33
+ reject(abortError(signal));
34
+ };
35
+ signal?.addEventListener('abort', onAbort, { once: true });
36
+ });
37
+ }
38
+
16
39
  function assertPathInside(baseDir, candidatePath, label) {
17
40
  const resolvedBase = path.resolve(baseDir);
18
41
  const resolvedCandidate = path.resolve(candidatePath);
@@ -54,12 +77,16 @@ class RuntimeHttpClient {
54
77
  let lastError = null;
55
78
 
56
79
  while (Date.now() - startedAt < timeoutMs) {
80
+ if (options.signal?.aborted) throw abortError(options.signal);
57
81
  if (!checkLiveness()) {
58
82
  throw new Error('Guest runtime process exited unexpectedly during bootstrap.');
59
83
  }
60
84
  const elapsed = Math.round((Date.now() - startedAt) / 1000);
61
85
  try {
62
- const health = await this.request('GET', '/health', undefined, { timeoutMs: 2000 });
86
+ const health = await this.request('GET', '/health', undefined, {
87
+ timeoutMs: 2000,
88
+ signal: options.signal,
89
+ });
63
90
  if (health?.status === 'ok') {
64
91
  console.log(`[Runtime] Guest agent ready after ${elapsed}s`);
65
92
  return health;
@@ -71,7 +98,7 @@ class RuntimeHttpClient {
71
98
  console.log(`[Runtime] Waiting for guest agent health... (${elapsed}s elapsed, last error: ${error.message})`);
72
99
  }
73
100
  }
74
- await new Promise((resolve) => setTimeout(resolve, intervalMs));
101
+ await delayWithSignal(intervalMs, options.signal);
75
102
  }
76
103
 
77
104
  if (lastError) {
@@ -81,49 +108,59 @@ class RuntimeHttpClient {
81
108
  }
82
109
 
83
110
  async request(method, pathname, body, options = {}) {
84
- const retryCount = Math.max(0, Number(options.retryCount ?? 6));
111
+ const normalizedMethod = String(method || 'GET').toUpperCase();
112
+ const safeToRetry = ['GET', 'HEAD', 'OPTIONS'].includes(normalizedMethod);
113
+ const retryCount = Math.max(0, Number(options.retryCount ?? (safeToRetry ? 6 : 0)));
85
114
  const retryDelayMs = Math.max(100, Number(options.retryDelayMs ?? 1000));
86
115
  let lastError = null;
87
116
 
88
117
  for (let attempt = 0; attempt <= retryCount; attempt += 1) {
89
- const controller = options.timeoutMs ? new AbortController() : null;
90
- const timer = controller ? setTimeout(() => controller.abort(new Error(`Request timed out after ${options.timeoutMs} ms.`)), options.timeoutMs) : null;
118
+ if (options.signal?.aborted) throw abortError(options.signal);
91
119
  try {
92
- const response = await fetch(`${this.baseUrl}${pathname}`, {
93
- method,
120
+ const { response, text } = await fetchResponseText(`${this.baseUrl}${pathname}`, {
121
+ method: normalizedMethod,
94
122
  headers: {
95
123
  'content-type': 'application/json',
96
124
  ...(this.token ? { authorization: `Bearer ${this.token}` } : {}),
97
125
  },
98
126
  body: body === undefined ? undefined : JSON.stringify(body),
99
- signal: controller?.signal,
127
+ signal: options.signal,
128
+ timeoutMs: Number(options.timeoutMs || 30000),
129
+ maxResponseBytes: Number(options.maxResponseBytes || 32 * 1024 * 1024),
130
+ serviceName: 'Guest runtime',
131
+ timeoutCode: 'RUNTIME_HTTP_TIMEOUT',
132
+ tooLargeCode: 'RUNTIME_HTTP_RESPONSE_TOO_LARGE',
100
133
  });
101
134
 
102
135
  const contentType = response.headers.get('content-type') || '';
103
- const payload = contentType.includes('application/json')
104
- ? await response.json().catch(() => ({}))
105
- : { text: await response.text().catch(() => '') };
136
+ let payload = { text };
137
+ if (contentType.includes('application/json')) {
138
+ try {
139
+ payload = JSON.parse(text);
140
+ } catch {
141
+ payload = {};
142
+ }
143
+ }
106
144
 
107
145
  if (!response.ok) {
108
146
  const errorMessage = payload?.error || payload?.text || `Runtime request failed: ${response.status}`;
109
- throw new Error(errorMessage);
147
+ const error = new Error(errorMessage);
148
+ error.status = response.status;
149
+ throw error;
110
150
  }
111
151
  if (typeof this.onActivity === 'function') {
112
152
  this.onActivity();
113
153
  }
114
154
  return payload;
115
155
  } catch (error) {
156
+ if (options.signal?.aborted) throw abortError(options.signal);
116
157
  lastError = error;
117
158
  const message = String(error?.message || error);
118
159
  const retryable = /fetch failed|ECONNREFUSED|ECONNRESET|socket hang up|timed out/i.test(message);
119
160
  if (!retryable || attempt === retryCount) {
120
161
  throw error;
121
162
  }
122
- await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
123
- } finally {
124
- if (timer) {
125
- clearTimeout(timer);
126
- }
163
+ await delayWithSignal(retryDelayMs, options.signal);
127
164
  }
128
165
  }
129
166
 
@@ -131,13 +168,14 @@ class RuntimeHttpClient {
131
168
  }
132
169
 
133
170
  async requestStream(method, pathname, stream, options = {}) {
134
- const retryCount = Math.max(0, Number(options.retryCount ?? 6));
171
+ const retryCount = 0;
135
172
  const retryDelayMs = Math.max(100, Number(options.retryDelayMs ?? 1000));
136
173
  let lastError = null;
137
174
 
138
175
  for (let attempt = 0; attempt <= retryCount; attempt += 1) {
176
+ if (options.signal?.aborted) throw abortError(options.signal);
139
177
  try {
140
- const response = await fetch(`${this.baseUrl}${pathname}`, {
178
+ const { response, text } = await fetchResponseText(`${this.baseUrl}${pathname}`, {
141
179
  method,
142
180
  headers: {
143
181
  ...(options.contentType ? { 'content-type': options.contentType } : {}),
@@ -147,6 +185,12 @@ class RuntimeHttpClient {
147
185
  },
148
186
  body: stream,
149
187
  duplex: 'half',
188
+ signal: options.signal,
189
+ timeoutMs: Number(options.timeoutMs || 60000),
190
+ maxResponseBytes: Number(options.maxResponseBytes || 2 * 1024 * 1024),
191
+ serviceName: 'Guest runtime upload',
192
+ timeoutCode: 'RUNTIME_HTTP_TIMEOUT',
193
+ tooLargeCode: 'RUNTIME_HTTP_RESPONSE_TOO_LARGE',
150
194
  });
151
195
 
152
196
  if (response.ok && typeof this.onActivity === 'function') {
@@ -154,23 +198,31 @@ class RuntimeHttpClient {
154
198
  }
155
199
 
156
200
  const contentType = response.headers.get('content-type') || '';
157
- const payload = contentType.includes('application/json')
158
- ? await response.json().catch(() => ({}))
159
- : { text: await response.text().catch(() => '') };
201
+ let payload = { text };
202
+ if (contentType.includes('application/json')) {
203
+ try {
204
+ payload = JSON.parse(text);
205
+ } catch {
206
+ payload = {};
207
+ }
208
+ }
160
209
 
161
210
  if (!response.ok) {
162
211
  const errorMessage = payload?.error || payload?.text || `Runtime request failed: ${response.status}`;
163
- throw new Error(errorMessage);
212
+ const error = new Error(errorMessage);
213
+ error.status = response.status;
214
+ throw error;
164
215
  }
165
216
  return payload;
166
217
  } catch (error) {
218
+ if (options.signal?.aborted) throw abortError(options.signal);
167
219
  lastError = error;
168
220
  const message = String(error?.message || error);
169
221
  const retryable = /fetch failed|ECONNREFUSED|ECONNRESET|socket hang up|timed out/i.test(message);
170
222
  if (!retryable || attempt === retryCount) {
171
223
  throw error;
172
224
  }
173
- await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
225
+ await delayWithSignal(retryDelayMs, options.signal);
174
226
  }
175
227
  }
176
228
 
@@ -186,7 +238,17 @@ class VmBrowserProvider {
186
238
  this.headless = true;
187
239
  }
188
240
 
189
- async #materialize(result) {
241
+ async #request(method, pathname, body, options = {}) {
242
+ const payload = body && typeof body === 'object' ? { ...body } : body;
243
+ const signal = options.signal || payload?.signal || null;
244
+ if (payload && typeof payload === 'object') delete payload.signal;
245
+ return this.client.request(method, pathname, payload, {
246
+ timeoutMs: Number(options.timeoutMs || 120_000),
247
+ signal,
248
+ });
249
+ }
250
+
251
+ async #materialize(result, options = {}) {
190
252
  if (!result || !this.artifactStore || this.userId == null) {
191
253
  return result;
192
254
  }
@@ -205,16 +267,18 @@ class VmBrowserProvider {
205
267
  let file = null;
206
268
  const maxAttempts = 20;
207
269
  for (let attempt = 0; attempt < maxAttempts && !file?.content; attempt += 1) {
270
+ if (options.signal?.aborted) throw abortError(options.signal);
208
271
  for (const candidate of readablePathCandidates) {
209
272
  try {
210
- file = await this.client.request('POST', '/files/read', {
273
+ file = await this.#request('POST', '/files/read', {
211
274
  path: candidate,
212
275
  encoding: 'base64',
213
- });
276
+ }, { signal: options.signal, timeoutMs: 10_000 });
214
277
  if (file?.content) {
215
278
  break;
216
279
  }
217
280
  } catch (error) {
281
+ if (options.signal?.aborted) throw abortError(options.signal);
218
282
  if (attempt === maxAttempts - 1) {
219
283
  console.warn('[Runtime:browser_vm] screenshot materialization read failed', {
220
284
  userId: this.userId,
@@ -225,7 +289,7 @@ class VmBrowserProvider {
225
289
  }
226
290
  }
227
291
  if (!file?.content) {
228
- await sleep(250);
292
+ await delayWithSignal(250, options.signal);
229
293
  }
230
294
  }
231
295
  if (!file?.content) {
@@ -243,6 +307,7 @@ class VmBrowserProvider {
243
307
  }
244
308
  return result;
245
309
  }
310
+ if (options.signal?.aborted) throw abortError(options.signal);
246
311
 
247
312
  const allocation = this.artifactStore.allocateFile(this.userId, {
248
313
  kind: 'browser-screenshot',
@@ -261,43 +326,96 @@ class VmBrowserProvider {
261
326
  };
262
327
  }
263
328
 
264
- async navigate(url, options = {}) { return this.#materialize(await this.client.request('POST', '/browser/navigate', { url, ...options })); }
265
- async click(selector, text, screenshot = true) { return this.#materialize(await this.client.request('POST', '/browser/click', { selector, text, screenshot })); }
266
- async clickPoint(x, y, screenshot = true) { return this.#materialize(await this.client.request('POST', '/browser/click-point', { x, y, screenshot })); }
267
- async type(selector, text, options = {}) { return this.#materialize(await this.client.request('POST', '/browser/fill', { selector, value: text, ...options })); }
268
- async typeText(text, options = {}) { return this.#materialize(await this.client.request('POST', '/browser/type-text', { text, ...options })); }
269
- async pressKey(key, screenshot = true) { return this.#materialize(await this.client.request('POST', '/browser/press-key', { key, screenshot })); }
270
- async scroll(deltaX, deltaY, screenshot = true) { return this.#materialize(await this.client.request('POST', '/browser/scroll', { deltaX, deltaY, screenshot })); }
271
- async extract(selector, attribute, all = false) { return this.client.request('POST', '/browser/extract', { selector, attribute, all }); }
272
- async evaluate(script) { return this.client.request('POST', '/browser/execute', { code: script }); }
273
- async screenshot(options = {}) { return this.#materialize(await this.client.request('POST', '/browser/screenshot', options)); }
329
+ async navigate(url, options = {}) {
330
+ return this.#materialize(
331
+ await this.#request('POST', '/browser/navigate', { url, ...options }, options),
332
+ options,
333
+ );
334
+ }
335
+ async click(selector, text, screenshot = true, options = {}) {
336
+ return this.#materialize(
337
+ await this.#request('POST', '/browser/click', { selector, text, screenshot }, options),
338
+ options,
339
+ );
340
+ }
341
+ async clickPoint(x, y, screenshot = true, options = {}) {
342
+ return this.#materialize(
343
+ await this.#request('POST', '/browser/click-point', { x, y, screenshot }, options),
344
+ options,
345
+ );
346
+ }
347
+ async type(selector, text, options = {}) {
348
+ return this.#materialize(
349
+ await this.#request('POST', '/browser/fill', { selector, value: text, ...options }, options),
350
+ options,
351
+ );
352
+ }
353
+ async typeText(text, options = {}) {
354
+ return this.#materialize(
355
+ await this.#request('POST', '/browser/type-text', { text, ...options }, options),
356
+ options,
357
+ );
358
+ }
359
+ async pressKey(key, screenshot = true, options = {}) {
360
+ return this.#materialize(
361
+ await this.#request('POST', '/browser/press-key', { key, screenshot }, options),
362
+ options,
363
+ );
364
+ }
365
+ async scroll(deltaX, deltaY, screenshot = true, options = {}) {
366
+ return this.#materialize(
367
+ await this.#request('POST', '/browser/scroll', { deltaX, deltaY, screenshot }, options),
368
+ options,
369
+ );
370
+ }
371
+ async extract(selector, attribute, all = false, options = {}) {
372
+ return this.#request('POST', '/browser/extract', { selector, attribute, all }, options);
373
+ }
374
+ async evaluate(script, options = {}) {
375
+ return this.#request('POST', '/browser/execute', { code: script }, options);
376
+ }
377
+ async screenshot(options = {}) {
378
+ return this.#materialize(
379
+ await this.#request('POST', '/browser/screenshot', options, options),
380
+ options,
381
+ );
382
+ }
274
383
  async screenshotJpeg(quality = 80, options = {}) {
275
- const result = await this.client.request('POST', '/browser/screenshot-jpeg', { ...options, quality });
384
+ const result = await this.#request('POST', '/browser/screenshot-jpeg', { ...options, quality }, options);
276
385
  const content = String(result?.contentBase64 || '');
277
386
  if (!content) throw new Error('VM browser screenshot-jpeg returned no data.');
278
387
  return Buffer.from(content, 'base64');
279
388
  }
280
- async launch(options = {}) { return this.client.request('POST', '/browser/launch', options); }
281
- async closeBrowser() { return this.client.request('POST', '/browser/close'); }
282
- async fill(selector, value) { return this.type(selector, value); }
283
- async extractContent(options = {}) { return this.client.request('POST', '/browser/extract', options); }
284
- async executeJS(code) { return this.evaluate(code); }
285
- async getPageInfo() {
286
- const status = await this.client.request('GET', '/browser/status');
389
+ async launch(options = {}) { return this.#request('POST', '/browser/launch', options, options); }
390
+ async closeBrowser(options = {}) { return this.#request('POST', '/browser/close', undefined, options); }
391
+ async fill(selector, value, options = {}) { return this.type(selector, value, options); }
392
+ async fillCredential(input, options = {}) {
393
+ return this.#request('POST', '/browser/credential-fill', input, options);
394
+ }
395
+ async submitProtectedCredential(protectedFillId, options = {}) {
396
+ return this.#request('POST', '/browser/credential-submit', { protectedFillId }, options);
397
+ }
398
+ async cancelProtectedCredential(protectedFillId, options = {}) {
399
+ return this.#request('POST', '/browser/credential-cancel', { protectedFillId }, options);
400
+ }
401
+ async extractContent(options = {}) { return this.#request('POST', '/browser/extract', options, options); }
402
+ async executeJS(code, options = {}) { return this.evaluate(code, options); }
403
+ async getPageInfo(options = {}) {
404
+ const status = await this.client.request('GET', '/browser/status', undefined, options);
287
405
  this.headless = status?.headless !== false;
288
406
  return status?.pageInfo || null;
289
407
  }
290
- async isLaunched() {
291
- const status = await this.client.request('GET', '/browser/status');
408
+ async isLaunched(options = {}) {
409
+ const status = await this.client.request('GET', '/browser/status', undefined, options);
292
410
  this.headless = status?.headless !== false;
293
411
  return status?.launched === true;
294
412
  }
295
- async getPageCount() {
296
- const status = await this.client.request('GET', '/browser/status');
413
+ async getPageCount(options = {}) {
414
+ const status = await this.client.request('GET', '/browser/status', undefined, options);
297
415
  return Number(status?.pages || 0);
298
416
  }
299
- async getCookies() {
300
- return this.client.request('GET', '/browser/cookies');
417
+ async getCookies(options = {}) {
418
+ return this.client.request('GET', '/browser/cookies', undefined, options);
301
419
  }
302
420
  async setHeadless(value) {
303
421
  this.headless = true;
@@ -308,11 +426,17 @@ class VmBrowserProvider {
308
426
  class LocalVmExecutionBackend {
309
427
  constructor(options = {}) {
310
428
  this.vmManager = options.vmManager;
311
- this.runtimeProfile = options.runtimeProfile === 'android' ? 'android' : 'browser_cli';
429
+ const runtimeProfile = String(options.runtimeProfile || '').trim();
430
+ this.runtimeProfile = ['android', 'browser', 'cli', 'browser_cli'].includes(runtimeProfile)
431
+ ? runtimeProfile
432
+ : 'browser_cli';
312
433
  this.token = options.token || process.env.NEOAGENT_VM_GUEST_TOKEN || '';
313
434
  this.artifactStore = options.artifactStore || null;
314
435
  this.lastActivity = new Map();
315
436
  this.reaperInterval = null;
437
+ this.reaperInFlight = false;
438
+ this.shuttingDown = false;
439
+ this.shutdownPromise = null;
316
440
 
317
441
  if (IDLE_TIMEOUT_MS > 0) {
318
442
  this.#startIdleReaper();
@@ -329,26 +453,48 @@ class LocalVmExecutionBackend {
329
453
  #startIdleReaper() {
330
454
  if (this.reaperInterval) return;
331
455
  this.reaperInterval = setInterval(async () => {
456
+ if (this.reaperInFlight || this.shuttingDown) return;
457
+ this.reaperInFlight = true;
332
458
  const now = Date.now();
333
- for (const [userId, lastUsed] of this.lastActivity.entries()) {
334
- if (now - lastUsed > IDLE_TIMEOUT_MS) {
335
- console.log(`[Runtime:${this.runtimeProfile}] User ${userId} runtime idle for ${Math.round((now - lastUsed) / 1000)}s, shutting down VM.`);
336
- this.lastActivity.delete(userId);
337
- try {
338
- await this.vmManager?.killVm?.(userId);
339
- } catch (err) {
340
- console.error(`[Runtime:${this.runtimeProfile}] Failed to shut down idle VM for user ${userId}:`, err.message);
459
+ try {
460
+ for (const [userId, lastUsed] of this.lastActivity.entries()) {
461
+ if (now - lastUsed > IDLE_TIMEOUT_MS) {
462
+ console.log(`[Runtime:${this.runtimeProfile}] User ${userId} runtime idle for ${Math.round((now - lastUsed) / 1000)}s, shutting down VM.`);
463
+ this.lastActivity.delete(userId);
464
+ try {
465
+ await this.vmManager?.killVm?.(userId);
466
+ } catch (err) {
467
+ console.error(`[Runtime:${this.runtimeProfile}] Failed to shut down idle VM for user ${userId}:`, err.message);
468
+ }
341
469
  }
342
470
  }
471
+ } finally {
472
+ this.reaperInFlight = false;
343
473
  }
344
474
  }, Math.min(IDLE_TIMEOUT_MS, 60 * 1000));
475
+ this.reaperInterval.unref?.();
345
476
  }
346
477
 
347
- async #clientForUser(userId) {
478
+ async #clientForUser(userId, options = {}) {
479
+ if (options.signal?.aborted) throw abortError(options.signal);
480
+ if (this.shuttingDown) {
481
+ const error = new Error('Local VM runtime is shutting down.');
482
+ error.code = 'RUNTIME_SHUTTING_DOWN';
483
+ throw error;
484
+ }
348
485
  if (!this.vmManager) {
349
486
  throw new Error('Local VM manager is not available.');
350
487
  }
351
- const session = await this.vmManager.ensureVm(userId);
488
+ const session = await waitForAbortableResult(
489
+ Promise.resolve(this.vmManager.ensureVm(userId, { signal: options.signal })),
490
+ options.signal,
491
+ 'VM startup aborted.',
492
+ );
493
+ if (this.shuttingDown) {
494
+ const error = new Error('Local VM runtime is shutting down.');
495
+ error.code = 'RUNTIME_SHUTTING_DOWN';
496
+ throw error;
497
+ }
352
498
  this.#touch(userId);
353
499
  const client = new RuntimeHttpClient(session.baseUrl, session.guestToken || this.token, {
354
500
  onActivity: () => this.#touch(userId),
@@ -356,6 +502,7 @@ class LocalVmExecutionBackend {
356
502
  try {
357
503
  await client.waitForHealth({
358
504
  timeoutMs: Number(process.env.NEOAGENT_VM_BOOT_TIMEOUT_MS || 20 * 60 * 1000),
505
+ signal: options.signal,
359
506
  checkLiveness: () => {
360
507
  const key = String(userId || '').trim();
361
508
  const session = this.vmManager.instances.get(key);
@@ -363,6 +510,7 @@ class LocalVmExecutionBackend {
363
510
  },
364
511
  });
365
512
  } catch (error) {
513
+ if (options.signal?.aborted) throw abortError(options.signal);
366
514
  const runtimeError = typeof session.getLastError === 'function' ? session.getLastError() : '';
367
515
  const detail = runtimeError ? ` ${runtimeError}` : '';
368
516
  throw new Error(`${error.message}${detail}`.trim());
@@ -371,7 +519,11 @@ class LocalVmExecutionBackend {
371
519
  }
372
520
 
373
521
  async executeCommand(userId, command, options = {}) {
374
- const client = await this.#clientForUser(userId);
522
+ const client = await this.#clientForUser(userId, options);
523
+ const requestedCommandTimeout = Number(options.timeout || 0);
524
+ const transportTimeout = requestedCommandTimeout > 0
525
+ ? Math.min(30 * 60 * 1000, requestedCommandTimeout + 30_000)
526
+ : 16 * 60 * 1000;
375
527
  return client.request('POST', '/exec', {
376
528
  command,
377
529
  cwd: options.cwd,
@@ -379,6 +531,10 @@ class LocalVmExecutionBackend {
379
531
  stdin_input: options.stdinInput,
380
532
  pty: options.pty === true,
381
533
  inputs: options.inputs || [],
534
+ }, {
535
+ timeoutMs: transportTimeout,
536
+ retryCount: 0,
537
+ signal: options.signal,
382
538
  });
383
539
  }
384
540
 
@@ -390,8 +546,8 @@ class LocalVmExecutionBackend {
390
546
  });
391
547
  }
392
548
 
393
- async getBrowserProviderForUser(userId) {
394
- return new VmBrowserProvider(await this.#clientForUser(userId), {
549
+ async getBrowserProviderForUser(userId, options = {}) {
550
+ return new VmBrowserProvider(await this.#clientForUser(userId, options), {
395
551
  userId,
396
552
  artifactStore: this.artifactStore,
397
553
  });
@@ -431,11 +587,15 @@ class LocalVmExecutionBackend {
431
587
  }
432
588
 
433
589
  async shutdown() {
590
+ if (this.shutdownPromise) return this.shutdownPromise;
591
+ this.shuttingDown = true;
434
592
  if (this.reaperInterval) {
435
593
  clearInterval(this.reaperInterval);
436
594
  this.reaperInterval = null;
437
595
  }
438
- await this.vmManager?.shutdown?.();
596
+ this.shutdownPromise = Promise.resolve(this.vmManager?.shutdown?.());
597
+ await this.shutdownPromise;
598
+ return { state: 'stopped' };
439
599
  }
440
600
  }
441
601
 
@@ -306,6 +306,15 @@ class DockerVMManager {
306
306
  return Boolean(session && isContainerRunning(session.containerId));
307
307
  }
308
308
 
309
+ // A side-effect-free snapshot for request-path capability reporting. Unlike
310
+ // hasVm(), this deliberately avoids a synchronous `docker inspect`, which can
311
+ // block the Node event loop. Tool execution still calls ensureVm() and performs
312
+ // the authoritative liveness check before using a tracked session.
313
+ hasTrackedVm(userId) {
314
+ const key = String(userId || '').trim();
315
+ return Boolean(key && this.instances.has(key));
316
+ }
317
+
309
318
  // Used by validation.js — cached to avoid docker calls on every status poll.
310
319
  getReadiness() {
311
320
  const now = Date.now();
@@ -7,6 +7,22 @@ const VM_ROOT = path.join(DATA_DIR, 'runtime-vms');
7
7
  const GUEST_BOOTSTRAP_ROOT = path.join(VM_ROOT, 'guest-bootstrap');
8
8
  const REPO_ROOT = path.resolve(__dirname, '../../..');
9
9
  const GUEST_PAYLOAD_PROFILES = Object.freeze({
10
+ browser: [
11
+ { source: 'server/guest-agent.browser.package.json', target: 'package.json' },
12
+ { source: 'runtime/env.js', target: 'runtime/env.js' },
13
+ { source: 'runtime/paths.js', target: 'runtime/paths.js' },
14
+ { source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
15
+ { source: 'server/services/browser', target: 'server/services/browser' },
16
+ { source: 'server/services/android/process.js', target: 'server/services/android/process.js' },
17
+ { source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
18
+ ],
19
+ cli: [
20
+ { source: 'server/guest-agent.cli.package.json', target: 'package.json' },
21
+ { source: 'runtime/env.js', target: 'runtime/env.js' },
22
+ { source: 'runtime/paths.js', target: 'runtime/paths.js' },
23
+ { source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
24
+ { source: 'server/services/cli', target: 'server/services/cli' },
25
+ ],
10
26
  browser_cli: [
11
27
  { source: 'server/guest-agent.browser.package.json', target: 'package.json' },
12
28
  { source: 'runtime/env.js', target: 'runtime/env.js' },
@@ -14,6 +30,8 @@ const GUEST_PAYLOAD_PROFILES = Object.freeze({
14
30
  { source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
15
31
  { source: 'server/services/cli', target: 'server/services/cli' },
16
32
  { source: 'server/services/browser', target: 'server/services/browser' },
33
+ { source: 'server/services/android/process.js', target: 'server/services/android/process.js' },
34
+ { source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
17
35
  ],
18
36
  android: [
19
37
  { source: 'server/guest-agent.android.package.json', target: 'package.json' },
@@ -22,6 +40,9 @@ const GUEST_PAYLOAD_PROFILES = Object.freeze({
22
40
  { source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
23
41
  { source: 'server/services/cli', target: 'server/services/cli' },
24
42
  { source: 'server/services/android', target: 'server/services/android' },
43
+ { source: 'server/utils/abort.js', target: 'server/utils/abort.js' },
44
+ { source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
45
+ { source: 'server/utils/image_payload.js', target: 'server/utils/image_payload.js' },
25
46
  ],
26
47
  });
27
48
 
@@ -32,7 +53,10 @@ function encodeGuestToken(value) {
32
53
  }
33
54
 
34
55
  function normalizeRuntimeProfile(runtimeProfile) {
35
- return runtimeProfile === 'android' ? 'android' : 'browser_cli';
56
+ const normalized = String(runtimeProfile || '').trim();
57
+ return ['android', 'browser', 'cli', 'browser_cli'].includes(normalized)
58
+ ? normalized
59
+ : 'browser_cli';
36
60
  }
37
61
 
38
62
  // Copy the guest runtime source files for a profile into `stagingRoot`, producing
@@ -84,10 +108,12 @@ function createCloudInitScript({
84
108
  runtimeProfile = 'browser_cli',
85
109
  }) {
86
110
  const normalizedProfile = normalizeRuntimeProfile(runtimeProfile);
87
- const includeBrowser = normalizedProfile === 'browser_cli';
111
+ const includeBrowser = ['browser', 'browser_cli'].includes(normalizedProfile);
88
112
  const guestUtilityPackages = includeBrowser
89
113
  ? 'curl ca-certificates gnupg git rsync unzip xvfb dbus-x11'
90
- : 'curl ca-certificates gnupg git rsync unzip dbus-x11 adb';
114
+ : normalizedProfile === 'android'
115
+ ? 'curl ca-certificates gnupg git rsync unzip dbus-x11 adb'
116
+ : 'curl ca-certificates gnupg git rsync unzip';
91
117
  const guestTokenB64 = encodeGuestToken(guestToken);
92
118
  const envFile = '/etc/neoagent/neoagent.env';
93
119
  const appDir = '/opt/neoagent';
@@ -208,7 +234,7 @@ function createCloudInitUserData({
208
234
  runtimeProfile = 'browser_cli',
209
235
  }) {
210
236
  const normalizedProfile = normalizeRuntimeProfile(runtimeProfile);
211
- const includeBrowser = normalizedProfile === 'browser_cli';
237
+ const includeBrowser = ['browser', 'browser_cli'].includes(normalizedProfile);
212
238
  const guestAgentInnerCommand = includeBrowser
213
239
  ? 'set -a; . /etc/neoagent/neoagent.env; set +a; cd /opt/neoagent && env DISPLAY=:99 PLAYWRIGHT_BROWSERS_PATH=/opt/neoagent/.playwright-browsers /usr/bin/env node server/guest_agent.js 2>&1 | tee -a /var/log/neoagent-guest-agent.log >/dev/console'
214
240
  : 'set -a; . /etc/neoagent/neoagent.env; set +a; cd /opt/neoagent && /usr/bin/env node server/guest_agent.js 2>&1 | tee -a /var/log/neoagent-guest-agent.log >/dev/console';