neoagent 3.2.1-beta.1 → 3.2.1-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/chrome-browser/background.mjs +318 -88
- package/extensions/chrome-browser/http.mjs +136 -0
- package/extensions/chrome-browser/protocol.mjs +654 -90
- package/flutter_app/lib/main_chat.dart +118 -739
- package/flutter_app/lib/main_controller.dart +111 -20
- package/flutter_app/lib/main_integrations.dart +607 -8
- package/flutter_app/lib/main_models.dart +3 -0
- package/flutter_app/lib/main_operations.dart +334 -321
- package/flutter_app/lib/main_security.dart +266 -112
- package/flutter_app/lib/main_settings.dart +4 -3
- package/flutter_app/lib/main_shared.dart +14 -11
- package/flutter_app/lib/src/backend_client.dart +78 -0
- package/flutter_app/lib/src/desktop_companion_actions.dart +185 -31
- package/flutter_app/lib/src/desktop_companion_io.dart +319 -86
- package/flutter_app/windows/runner/flutter_window.cpp +143 -32
- package/landing/index.html +3 -1
- package/lib/manager.js +106 -89
- package/lib/schema_migrations.js +115 -13
- package/package.json +30 -15
- package/runtime/paths.js +49 -5
- package/server/db/database.js +2 -2
- package/server/guest-agent.cli.package.json +13 -0
- package/server/guest_agent.js +85 -40
- package/server/http/middleware.js +24 -0
- package/server/http/routes.js +11 -6
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/main.dart.js +73083 -72209
- package/server/routes/admin.js +1 -1
- package/server/routes/android.js +30 -34
- package/server/routes/browser.js +23 -15
- package/server/routes/desktop.js +18 -1
- package/server/routes/integrations.js +107 -1
- package/server/routes/memory.js +1 -0
- package/server/routes/settings.js +16 -5
- package/server/routes/social_reach.js +12 -3
- package/server/routes/social_video.js +4 -0
- package/server/services/agents/manager.js +1 -1
- package/server/services/ai/capabilityHealth.js +62 -96
- package/server/services/ai/compaction.js +7 -2
- package/server/services/ai/history.js +45 -6
- package/server/services/ai/integrated_tools/http_request.js +8 -0
- package/server/services/ai/loop/agent_engine_core.js +390 -162
- package/server/services/ai/loop/blank_recovery.js +5 -4
- package/server/services/ai/loop/callbacks.js +1 -0
- package/server/services/ai/loop/completion_judge.js +121 -5
- package/server/services/ai/loop/conversation_loop.js +510 -332
- package/server/services/ai/loop/messaging_delivery.js +129 -57
- package/server/services/ai/loop/model_call_guard.js +91 -0
- package/server/services/ai/loop/model_io.js +20 -45
- package/server/services/ai/loop/progress_classification.js +2 -0
- package/server/services/ai/loop/tool_dispatch.js +19 -8
- package/server/services/ai/loopPolicy.js +48 -21
- package/server/services/ai/messagingFallback.js +17 -17
- package/server/services/ai/model_discovery.js +227 -0
- package/server/services/ai/model_failure_cache.js +108 -0
- package/server/services/ai/model_identity.js +71 -0
- package/server/services/ai/models.js +68 -163
- package/server/services/ai/providerRetry.js +17 -59
- package/server/services/ai/provider_selector.js +166 -0
- package/server/services/ai/providers/anthropic.js +2 -2
- package/server/services/ai/providers/claudeCode.js +21 -33
- package/server/services/ai/providers/githubCopilot.js +41 -20
- package/server/services/ai/providers/google.js +135 -97
- package/server/services/ai/providers/grok.js +4 -3
- package/server/services/ai/providers/grokOauth.js +19 -27
- package/server/services/ai/providers/nvidia.js +10 -5
- package/server/services/ai/providers/ollama.js +111 -84
- package/server/services/ai/providers/ollama_stream.js +142 -0
- package/server/services/ai/providers/openai.js +39 -5
- package/server/services/ai/providers/openaiCodex.js +11 -4
- package/server/services/ai/providers/openrouter.js +29 -7
- package/server/services/ai/providers/provider_error.js +36 -0
- package/server/services/ai/settings.js +26 -2
- package/server/services/ai/systemPrompt.js +19 -12
- package/server/services/ai/taskAnalysis.js +58 -10
- package/server/services/ai/terminal_reply.js +18 -0
- package/server/services/ai/toolEvidence.js +350 -29
- package/server/services/ai/tools.js +190 -111
- package/server/services/android/controller.js +770 -237
- package/server/services/android/process.js +140 -0
- package/server/services/android/sdk_download.js +143 -0
- package/server/services/android/uia.js +6 -5
- package/server/services/artifacts/store.js +24 -0
- package/server/services/browser/controller.js +843 -385
- package/server/services/browser/extension/gateway.js +40 -16
- package/server/services/browser/extension/protocol.js +15 -1
- package/server/services/browser/extension/provider.js +71 -47
- package/server/services/browser/extension/registry.js +155 -34
- package/server/services/cli/executor.js +62 -9
- package/server/services/credentials/bitwarden_cli.js +322 -0
- package/server/services/credentials/broker.js +594 -0
- package/server/services/desktop/gateway.js +41 -4
- package/server/services/desktop/protocol.js +3 -0
- package/server/services/desktop/provider.js +39 -42
- package/server/services/desktop/registry.js +137 -52
- package/server/services/integrations/bitwarden/constants.js +14 -0
- package/server/services/integrations/bitwarden/provider.js +197 -0
- package/server/services/integrations/bitwarden/snapshot.js +65 -0
- package/server/services/integrations/figma/provider.js +78 -12
- package/server/services/integrations/github/common.js +11 -6
- package/server/services/integrations/github/provider.js +52 -53
- package/server/services/integrations/google/provider.js +55 -19
- package/server/services/integrations/home_assistant/network.js +17 -20
- package/server/services/integrations/home_assistant/provider.js +7 -5
- package/server/services/integrations/home_assistant/tools.js +17 -5
- package/server/services/integrations/http.js +51 -0
- package/server/services/integrations/manager.js +159 -53
- package/server/services/integrations/microsoft/provider.js +80 -13
- package/server/services/integrations/neoarchive/provider.js +55 -29
- package/server/services/integrations/neorecall/client.js +17 -10
- package/server/services/integrations/neorecall/provider.js +20 -11
- package/server/services/integrations/notion/provider.js +16 -13
- package/server/services/integrations/oauth_provider.js +115 -51
- package/server/services/integrations/registry.js +2 -0
- package/server/services/integrations/slack/provider.js +98 -9
- package/server/services/integrations/spotify/provider.js +67 -71
- package/server/services/integrations/trello/provider.js +21 -7
- package/server/services/integrations/weather/provider.js +18 -12
- package/server/services/integrations/whatsapp/provider.js +76 -16
- package/server/services/manager.js +110 -1
- package/server/services/memory/embedding_index.js +20 -8
- package/server/services/memory/embeddings.js +151 -90
- package/server/services/memory/ingestion.js +50 -9
- package/server/services/memory/ingestion_documents.js +13 -3
- package/server/services/memory/manager.js +52 -19
- package/server/services/messaging/automation.js +85 -10
- package/server/services/messaging/formatting_guides.js +7 -4
- package/server/services/messaging/http_platforms.js +33 -13
- package/server/services/messaging/inbound_queue.js +78 -24
- package/server/services/messaging/inbound_store.js +224 -0
- package/server/services/messaging/manager.js +326 -51
- package/server/services/messaging/typing_keepalive.js +5 -2
- package/server/services/messaging/whatsapp.js +22 -14
- package/server/services/network/http.js +210 -0
- package/server/services/network/safe_request.js +307 -0
- package/server/services/runtime/backends/local-vm.js +227 -67
- package/server/services/runtime/docker-vm-manager.js +9 -0
- package/server/services/runtime/guest_bootstrap.js +30 -4
- package/server/services/runtime/guest_image.js +43 -12
- package/server/services/runtime/manager.js +77 -23
- package/server/services/runtime/validation.js +7 -6
- package/server/services/security/tool_categories.js +6 -0
- package/server/services/social_reach/channels/github.js +10 -4
- package/server/services/social_reach/channels/reddit.js +4 -4
- package/server/services/social_reach/channels/rss.js +2 -2
- package/server/services/social_reach/channels/social_video.js +12 -7
- package/server/services/social_reach/channels/v2ex.js +21 -8
- package/server/services/social_reach/channels/x.js +2 -2
- package/server/services/social_reach/channels/xueqiu.js +5 -5
- package/server/services/social_reach/service.js +9 -6
- package/server/services/social_reach/utils.js +65 -14
- package/server/services/social_video/service.js +160 -50
- package/server/services/tasks/integration_runtime.js +18 -8
- package/server/services/tasks/runtime.js +39 -4
- package/server/services/voice/agentBridge.js +17 -4
- package/server/services/voice/bufferedLiveRelayAdapter.js +5 -0
- package/server/services/voice/liveSession.js +31 -0
- package/server/services/voice/message.js +1 -1
- package/server/services/voice/openaiSpeech.js +33 -8
- package/server/services/voice/providers.js +233 -151
- package/server/services/voice/runtime.js +2 -2
- package/server/services/voice/runtimeManager.js +118 -20
- package/server/services/voice/turnRunner.js +6 -0
- package/server/services/wearable/firmware_manifest.js +51 -13
- package/server/services/wearable/service.js +1 -0
- package/server/utils/abort.js +96 -0
- package/server/utils/cloud-security.js +110 -3
- package/server/utils/files.js +31 -0
- package/server/utils/image_payload.js +95 -0
- package/server/utils/retry.js +107 -0
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
1
3
|
const fs = require('fs');
|
|
2
4
|
const path = require('path');
|
|
3
5
|
const { DATA_DIR } = require('../../../../runtime/paths');
|
|
6
|
+
const { fetchResponseText, waitForAbortableResult } = require('../../network/http');
|
|
7
|
+
const { createAbortError } = require('../../../utils/abort');
|
|
4
8
|
|
|
5
9
|
const APK_UPLOAD_ROOT = path.resolve(
|
|
6
10
|
process.env.NEOAGENT_ANDROID_APK_BASE_DIR
|
|
@@ -13,6 +17,25 @@ function sleep(ms) {
|
|
|
13
17
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
14
18
|
}
|
|
15
19
|
|
|
20
|
+
function abortError(signal, fallback = 'Operation aborted.') {
|
|
21
|
+
return createAbortError(signal, fallback);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function delayWithSignal(ms, signal) {
|
|
25
|
+
if (signal?.aborted) return Promise.reject(abortError(signal));
|
|
26
|
+
return new Promise((resolve, reject) => {
|
|
27
|
+
const timer = setTimeout(() => {
|
|
28
|
+
signal?.removeEventListener('abort', onAbort);
|
|
29
|
+
resolve();
|
|
30
|
+
}, ms);
|
|
31
|
+
const onAbort = () => {
|
|
32
|
+
clearTimeout(timer);
|
|
33
|
+
reject(abortError(signal));
|
|
34
|
+
};
|
|
35
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
|
|
16
39
|
function assertPathInside(baseDir, candidatePath, label) {
|
|
17
40
|
const resolvedBase = path.resolve(baseDir);
|
|
18
41
|
const resolvedCandidate = path.resolve(candidatePath);
|
|
@@ -54,12 +77,16 @@ class RuntimeHttpClient {
|
|
|
54
77
|
let lastError = null;
|
|
55
78
|
|
|
56
79
|
while (Date.now() - startedAt < timeoutMs) {
|
|
80
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
57
81
|
if (!checkLiveness()) {
|
|
58
82
|
throw new Error('Guest runtime process exited unexpectedly during bootstrap.');
|
|
59
83
|
}
|
|
60
84
|
const elapsed = Math.round((Date.now() - startedAt) / 1000);
|
|
61
85
|
try {
|
|
62
|
-
const health = await this.request('GET', '/health', undefined, {
|
|
86
|
+
const health = await this.request('GET', '/health', undefined, {
|
|
87
|
+
timeoutMs: 2000,
|
|
88
|
+
signal: options.signal,
|
|
89
|
+
});
|
|
63
90
|
if (health?.status === 'ok') {
|
|
64
91
|
console.log(`[Runtime] Guest agent ready after ${elapsed}s`);
|
|
65
92
|
return health;
|
|
@@ -71,7 +98,7 @@ class RuntimeHttpClient {
|
|
|
71
98
|
console.log(`[Runtime] Waiting for guest agent health... (${elapsed}s elapsed, last error: ${error.message})`);
|
|
72
99
|
}
|
|
73
100
|
}
|
|
74
|
-
await
|
|
101
|
+
await delayWithSignal(intervalMs, options.signal);
|
|
75
102
|
}
|
|
76
103
|
|
|
77
104
|
if (lastError) {
|
|
@@ -81,49 +108,59 @@ class RuntimeHttpClient {
|
|
|
81
108
|
}
|
|
82
109
|
|
|
83
110
|
async request(method, pathname, body, options = {}) {
|
|
84
|
-
const
|
|
111
|
+
const normalizedMethod = String(method || 'GET').toUpperCase();
|
|
112
|
+
const safeToRetry = ['GET', 'HEAD', 'OPTIONS'].includes(normalizedMethod);
|
|
113
|
+
const retryCount = Math.max(0, Number(options.retryCount ?? (safeToRetry ? 6 : 0)));
|
|
85
114
|
const retryDelayMs = Math.max(100, Number(options.retryDelayMs ?? 1000));
|
|
86
115
|
let lastError = null;
|
|
87
116
|
|
|
88
117
|
for (let attempt = 0; attempt <= retryCount; attempt += 1) {
|
|
89
|
-
|
|
90
|
-
const timer = controller ? setTimeout(() => controller.abort(new Error(`Request timed out after ${options.timeoutMs} ms.`)), options.timeoutMs) : null;
|
|
118
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
91
119
|
try {
|
|
92
|
-
const response = await
|
|
93
|
-
method,
|
|
120
|
+
const { response, text } = await fetchResponseText(`${this.baseUrl}${pathname}`, {
|
|
121
|
+
method: normalizedMethod,
|
|
94
122
|
headers: {
|
|
95
123
|
'content-type': 'application/json',
|
|
96
124
|
...(this.token ? { authorization: `Bearer ${this.token}` } : {}),
|
|
97
125
|
},
|
|
98
126
|
body: body === undefined ? undefined : JSON.stringify(body),
|
|
99
|
-
signal:
|
|
127
|
+
signal: options.signal,
|
|
128
|
+
timeoutMs: Number(options.timeoutMs || 30000),
|
|
129
|
+
maxResponseBytes: Number(options.maxResponseBytes || 32 * 1024 * 1024),
|
|
130
|
+
serviceName: 'Guest runtime',
|
|
131
|
+
timeoutCode: 'RUNTIME_HTTP_TIMEOUT',
|
|
132
|
+
tooLargeCode: 'RUNTIME_HTTP_RESPONSE_TOO_LARGE',
|
|
100
133
|
});
|
|
101
134
|
|
|
102
135
|
const contentType = response.headers.get('content-type') || '';
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
136
|
+
let payload = { text };
|
|
137
|
+
if (contentType.includes('application/json')) {
|
|
138
|
+
try {
|
|
139
|
+
payload = JSON.parse(text);
|
|
140
|
+
} catch {
|
|
141
|
+
payload = {};
|
|
142
|
+
}
|
|
143
|
+
}
|
|
106
144
|
|
|
107
145
|
if (!response.ok) {
|
|
108
146
|
const errorMessage = payload?.error || payload?.text || `Runtime request failed: ${response.status}`;
|
|
109
|
-
|
|
147
|
+
const error = new Error(errorMessage);
|
|
148
|
+
error.status = response.status;
|
|
149
|
+
throw error;
|
|
110
150
|
}
|
|
111
151
|
if (typeof this.onActivity === 'function') {
|
|
112
152
|
this.onActivity();
|
|
113
153
|
}
|
|
114
154
|
return payload;
|
|
115
155
|
} catch (error) {
|
|
156
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
116
157
|
lastError = error;
|
|
117
158
|
const message = String(error?.message || error);
|
|
118
159
|
const retryable = /fetch failed|ECONNREFUSED|ECONNRESET|socket hang up|timed out/i.test(message);
|
|
119
160
|
if (!retryable || attempt === retryCount) {
|
|
120
161
|
throw error;
|
|
121
162
|
}
|
|
122
|
-
await
|
|
123
|
-
} finally {
|
|
124
|
-
if (timer) {
|
|
125
|
-
clearTimeout(timer);
|
|
126
|
-
}
|
|
163
|
+
await delayWithSignal(retryDelayMs, options.signal);
|
|
127
164
|
}
|
|
128
165
|
}
|
|
129
166
|
|
|
@@ -131,13 +168,14 @@ class RuntimeHttpClient {
|
|
|
131
168
|
}
|
|
132
169
|
|
|
133
170
|
async requestStream(method, pathname, stream, options = {}) {
|
|
134
|
-
const retryCount =
|
|
171
|
+
const retryCount = 0;
|
|
135
172
|
const retryDelayMs = Math.max(100, Number(options.retryDelayMs ?? 1000));
|
|
136
173
|
let lastError = null;
|
|
137
174
|
|
|
138
175
|
for (let attempt = 0; attempt <= retryCount; attempt += 1) {
|
|
176
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
139
177
|
try {
|
|
140
|
-
const response = await
|
|
178
|
+
const { response, text } = await fetchResponseText(`${this.baseUrl}${pathname}`, {
|
|
141
179
|
method,
|
|
142
180
|
headers: {
|
|
143
181
|
...(options.contentType ? { 'content-type': options.contentType } : {}),
|
|
@@ -147,6 +185,12 @@ class RuntimeHttpClient {
|
|
|
147
185
|
},
|
|
148
186
|
body: stream,
|
|
149
187
|
duplex: 'half',
|
|
188
|
+
signal: options.signal,
|
|
189
|
+
timeoutMs: Number(options.timeoutMs || 60000),
|
|
190
|
+
maxResponseBytes: Number(options.maxResponseBytes || 2 * 1024 * 1024),
|
|
191
|
+
serviceName: 'Guest runtime upload',
|
|
192
|
+
timeoutCode: 'RUNTIME_HTTP_TIMEOUT',
|
|
193
|
+
tooLargeCode: 'RUNTIME_HTTP_RESPONSE_TOO_LARGE',
|
|
150
194
|
});
|
|
151
195
|
|
|
152
196
|
if (response.ok && typeof this.onActivity === 'function') {
|
|
@@ -154,23 +198,31 @@ class RuntimeHttpClient {
|
|
|
154
198
|
}
|
|
155
199
|
|
|
156
200
|
const contentType = response.headers.get('content-type') || '';
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
201
|
+
let payload = { text };
|
|
202
|
+
if (contentType.includes('application/json')) {
|
|
203
|
+
try {
|
|
204
|
+
payload = JSON.parse(text);
|
|
205
|
+
} catch {
|
|
206
|
+
payload = {};
|
|
207
|
+
}
|
|
208
|
+
}
|
|
160
209
|
|
|
161
210
|
if (!response.ok) {
|
|
162
211
|
const errorMessage = payload?.error || payload?.text || `Runtime request failed: ${response.status}`;
|
|
163
|
-
|
|
212
|
+
const error = new Error(errorMessage);
|
|
213
|
+
error.status = response.status;
|
|
214
|
+
throw error;
|
|
164
215
|
}
|
|
165
216
|
return payload;
|
|
166
217
|
} catch (error) {
|
|
218
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
167
219
|
lastError = error;
|
|
168
220
|
const message = String(error?.message || error);
|
|
169
221
|
const retryable = /fetch failed|ECONNREFUSED|ECONNRESET|socket hang up|timed out/i.test(message);
|
|
170
222
|
if (!retryable || attempt === retryCount) {
|
|
171
223
|
throw error;
|
|
172
224
|
}
|
|
173
|
-
await
|
|
225
|
+
await delayWithSignal(retryDelayMs, options.signal);
|
|
174
226
|
}
|
|
175
227
|
}
|
|
176
228
|
|
|
@@ -186,7 +238,17 @@ class VmBrowserProvider {
|
|
|
186
238
|
this.headless = true;
|
|
187
239
|
}
|
|
188
240
|
|
|
189
|
-
async #
|
|
241
|
+
async #request(method, pathname, body, options = {}) {
|
|
242
|
+
const payload = body && typeof body === 'object' ? { ...body } : body;
|
|
243
|
+
const signal = options.signal || payload?.signal || null;
|
|
244
|
+
if (payload && typeof payload === 'object') delete payload.signal;
|
|
245
|
+
return this.client.request(method, pathname, payload, {
|
|
246
|
+
timeoutMs: Number(options.timeoutMs || 120_000),
|
|
247
|
+
signal,
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async #materialize(result, options = {}) {
|
|
190
252
|
if (!result || !this.artifactStore || this.userId == null) {
|
|
191
253
|
return result;
|
|
192
254
|
}
|
|
@@ -205,16 +267,18 @@ class VmBrowserProvider {
|
|
|
205
267
|
let file = null;
|
|
206
268
|
const maxAttempts = 20;
|
|
207
269
|
for (let attempt = 0; attempt < maxAttempts && !file?.content; attempt += 1) {
|
|
270
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
208
271
|
for (const candidate of readablePathCandidates) {
|
|
209
272
|
try {
|
|
210
|
-
file = await this
|
|
273
|
+
file = await this.#request('POST', '/files/read', {
|
|
211
274
|
path: candidate,
|
|
212
275
|
encoding: 'base64',
|
|
213
|
-
});
|
|
276
|
+
}, { signal: options.signal, timeoutMs: 10_000 });
|
|
214
277
|
if (file?.content) {
|
|
215
278
|
break;
|
|
216
279
|
}
|
|
217
280
|
} catch (error) {
|
|
281
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
218
282
|
if (attempt === maxAttempts - 1) {
|
|
219
283
|
console.warn('[Runtime:browser_vm] screenshot materialization read failed', {
|
|
220
284
|
userId: this.userId,
|
|
@@ -225,7 +289,7 @@ class VmBrowserProvider {
|
|
|
225
289
|
}
|
|
226
290
|
}
|
|
227
291
|
if (!file?.content) {
|
|
228
|
-
await
|
|
292
|
+
await delayWithSignal(250, options.signal);
|
|
229
293
|
}
|
|
230
294
|
}
|
|
231
295
|
if (!file?.content) {
|
|
@@ -243,6 +307,7 @@ class VmBrowserProvider {
|
|
|
243
307
|
}
|
|
244
308
|
return result;
|
|
245
309
|
}
|
|
310
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
246
311
|
|
|
247
312
|
const allocation = this.artifactStore.allocateFile(this.userId, {
|
|
248
313
|
kind: 'browser-screenshot',
|
|
@@ -261,43 +326,96 @@ class VmBrowserProvider {
|
|
|
261
326
|
};
|
|
262
327
|
}
|
|
263
328
|
|
|
264
|
-
async navigate(url, options = {}) {
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
async
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
329
|
+
async navigate(url, options = {}) {
|
|
330
|
+
return this.#materialize(
|
|
331
|
+
await this.#request('POST', '/browser/navigate', { url, ...options }, options),
|
|
332
|
+
options,
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
async click(selector, text, screenshot = true, options = {}) {
|
|
336
|
+
return this.#materialize(
|
|
337
|
+
await this.#request('POST', '/browser/click', { selector, text, screenshot }, options),
|
|
338
|
+
options,
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
async clickPoint(x, y, screenshot = true, options = {}) {
|
|
342
|
+
return this.#materialize(
|
|
343
|
+
await this.#request('POST', '/browser/click-point', { x, y, screenshot }, options),
|
|
344
|
+
options,
|
|
345
|
+
);
|
|
346
|
+
}
|
|
347
|
+
async type(selector, text, options = {}) {
|
|
348
|
+
return this.#materialize(
|
|
349
|
+
await this.#request('POST', '/browser/fill', { selector, value: text, ...options }, options),
|
|
350
|
+
options,
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
async typeText(text, options = {}) {
|
|
354
|
+
return this.#materialize(
|
|
355
|
+
await this.#request('POST', '/browser/type-text', { text, ...options }, options),
|
|
356
|
+
options,
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
async pressKey(key, screenshot = true, options = {}) {
|
|
360
|
+
return this.#materialize(
|
|
361
|
+
await this.#request('POST', '/browser/press-key', { key, screenshot }, options),
|
|
362
|
+
options,
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
async scroll(deltaX, deltaY, screenshot = true, options = {}) {
|
|
366
|
+
return this.#materialize(
|
|
367
|
+
await this.#request('POST', '/browser/scroll', { deltaX, deltaY, screenshot }, options),
|
|
368
|
+
options,
|
|
369
|
+
);
|
|
370
|
+
}
|
|
371
|
+
async extract(selector, attribute, all = false, options = {}) {
|
|
372
|
+
return this.#request('POST', '/browser/extract', { selector, attribute, all }, options);
|
|
373
|
+
}
|
|
374
|
+
async evaluate(script, options = {}) {
|
|
375
|
+
return this.#request('POST', '/browser/execute', { code: script }, options);
|
|
376
|
+
}
|
|
377
|
+
async screenshot(options = {}) {
|
|
378
|
+
return this.#materialize(
|
|
379
|
+
await this.#request('POST', '/browser/screenshot', options, options),
|
|
380
|
+
options,
|
|
381
|
+
);
|
|
382
|
+
}
|
|
274
383
|
async screenshotJpeg(quality = 80, options = {}) {
|
|
275
|
-
const result = await this
|
|
384
|
+
const result = await this.#request('POST', '/browser/screenshot-jpeg', { ...options, quality }, options);
|
|
276
385
|
const content = String(result?.contentBase64 || '');
|
|
277
386
|
if (!content) throw new Error('VM browser screenshot-jpeg returned no data.');
|
|
278
387
|
return Buffer.from(content, 'base64');
|
|
279
388
|
}
|
|
280
|
-
async launch(options = {}) { return this
|
|
281
|
-
async closeBrowser() { return this
|
|
282
|
-
async fill(selector, value) { return this.type(selector, value); }
|
|
283
|
-
async
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
389
|
+
async launch(options = {}) { return this.#request('POST', '/browser/launch', options, options); }
|
|
390
|
+
async closeBrowser(options = {}) { return this.#request('POST', '/browser/close', undefined, options); }
|
|
391
|
+
async fill(selector, value, options = {}) { return this.type(selector, value, options); }
|
|
392
|
+
async fillCredential(input, options = {}) {
|
|
393
|
+
return this.#request('POST', '/browser/credential-fill', input, options);
|
|
394
|
+
}
|
|
395
|
+
async submitProtectedCredential(protectedFillId, options = {}) {
|
|
396
|
+
return this.#request('POST', '/browser/credential-submit', { protectedFillId }, options);
|
|
397
|
+
}
|
|
398
|
+
async cancelProtectedCredential(protectedFillId, options = {}) {
|
|
399
|
+
return this.#request('POST', '/browser/credential-cancel', { protectedFillId }, options);
|
|
400
|
+
}
|
|
401
|
+
async extractContent(options = {}) { return this.#request('POST', '/browser/extract', options, options); }
|
|
402
|
+
async executeJS(code, options = {}) { return this.evaluate(code, options); }
|
|
403
|
+
async getPageInfo(options = {}) {
|
|
404
|
+
const status = await this.client.request('GET', '/browser/status', undefined, options);
|
|
287
405
|
this.headless = status?.headless !== false;
|
|
288
406
|
return status?.pageInfo || null;
|
|
289
407
|
}
|
|
290
|
-
async isLaunched() {
|
|
291
|
-
const status = await this.client.request('GET', '/browser/status');
|
|
408
|
+
async isLaunched(options = {}) {
|
|
409
|
+
const status = await this.client.request('GET', '/browser/status', undefined, options);
|
|
292
410
|
this.headless = status?.headless !== false;
|
|
293
411
|
return status?.launched === true;
|
|
294
412
|
}
|
|
295
|
-
async getPageCount() {
|
|
296
|
-
const status = await this.client.request('GET', '/browser/status');
|
|
413
|
+
async getPageCount(options = {}) {
|
|
414
|
+
const status = await this.client.request('GET', '/browser/status', undefined, options);
|
|
297
415
|
return Number(status?.pages || 0);
|
|
298
416
|
}
|
|
299
|
-
async getCookies() {
|
|
300
|
-
return this.client.request('GET', '/browser/cookies');
|
|
417
|
+
async getCookies(options = {}) {
|
|
418
|
+
return this.client.request('GET', '/browser/cookies', undefined, options);
|
|
301
419
|
}
|
|
302
420
|
async setHeadless(value) {
|
|
303
421
|
this.headless = true;
|
|
@@ -308,11 +426,17 @@ class VmBrowserProvider {
|
|
|
308
426
|
class LocalVmExecutionBackend {
|
|
309
427
|
constructor(options = {}) {
|
|
310
428
|
this.vmManager = options.vmManager;
|
|
311
|
-
|
|
429
|
+
const runtimeProfile = String(options.runtimeProfile || '').trim();
|
|
430
|
+
this.runtimeProfile = ['android', 'browser', 'cli', 'browser_cli'].includes(runtimeProfile)
|
|
431
|
+
? runtimeProfile
|
|
432
|
+
: 'browser_cli';
|
|
312
433
|
this.token = options.token || process.env.NEOAGENT_VM_GUEST_TOKEN || '';
|
|
313
434
|
this.artifactStore = options.artifactStore || null;
|
|
314
435
|
this.lastActivity = new Map();
|
|
315
436
|
this.reaperInterval = null;
|
|
437
|
+
this.reaperInFlight = false;
|
|
438
|
+
this.shuttingDown = false;
|
|
439
|
+
this.shutdownPromise = null;
|
|
316
440
|
|
|
317
441
|
if (IDLE_TIMEOUT_MS > 0) {
|
|
318
442
|
this.#startIdleReaper();
|
|
@@ -329,26 +453,48 @@ class LocalVmExecutionBackend {
|
|
|
329
453
|
#startIdleReaper() {
|
|
330
454
|
if (this.reaperInterval) return;
|
|
331
455
|
this.reaperInterval = setInterval(async () => {
|
|
456
|
+
if (this.reaperInFlight || this.shuttingDown) return;
|
|
457
|
+
this.reaperInFlight = true;
|
|
332
458
|
const now = Date.now();
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
459
|
+
try {
|
|
460
|
+
for (const [userId, lastUsed] of this.lastActivity.entries()) {
|
|
461
|
+
if (now - lastUsed > IDLE_TIMEOUT_MS) {
|
|
462
|
+
console.log(`[Runtime:${this.runtimeProfile}] User ${userId} runtime idle for ${Math.round((now - lastUsed) / 1000)}s, shutting down VM.`);
|
|
463
|
+
this.lastActivity.delete(userId);
|
|
464
|
+
try {
|
|
465
|
+
await this.vmManager?.killVm?.(userId);
|
|
466
|
+
} catch (err) {
|
|
467
|
+
console.error(`[Runtime:${this.runtimeProfile}] Failed to shut down idle VM for user ${userId}:`, err.message);
|
|
468
|
+
}
|
|
341
469
|
}
|
|
342
470
|
}
|
|
471
|
+
} finally {
|
|
472
|
+
this.reaperInFlight = false;
|
|
343
473
|
}
|
|
344
474
|
}, Math.min(IDLE_TIMEOUT_MS, 60 * 1000));
|
|
475
|
+
this.reaperInterval.unref?.();
|
|
345
476
|
}
|
|
346
477
|
|
|
347
|
-
async #clientForUser(userId) {
|
|
478
|
+
async #clientForUser(userId, options = {}) {
|
|
479
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
480
|
+
if (this.shuttingDown) {
|
|
481
|
+
const error = new Error('Local VM runtime is shutting down.');
|
|
482
|
+
error.code = 'RUNTIME_SHUTTING_DOWN';
|
|
483
|
+
throw error;
|
|
484
|
+
}
|
|
348
485
|
if (!this.vmManager) {
|
|
349
486
|
throw new Error('Local VM manager is not available.');
|
|
350
487
|
}
|
|
351
|
-
const session = await
|
|
488
|
+
const session = await waitForAbortableResult(
|
|
489
|
+
Promise.resolve(this.vmManager.ensureVm(userId, { signal: options.signal })),
|
|
490
|
+
options.signal,
|
|
491
|
+
'VM startup aborted.',
|
|
492
|
+
);
|
|
493
|
+
if (this.shuttingDown) {
|
|
494
|
+
const error = new Error('Local VM runtime is shutting down.');
|
|
495
|
+
error.code = 'RUNTIME_SHUTTING_DOWN';
|
|
496
|
+
throw error;
|
|
497
|
+
}
|
|
352
498
|
this.#touch(userId);
|
|
353
499
|
const client = new RuntimeHttpClient(session.baseUrl, session.guestToken || this.token, {
|
|
354
500
|
onActivity: () => this.#touch(userId),
|
|
@@ -356,6 +502,7 @@ class LocalVmExecutionBackend {
|
|
|
356
502
|
try {
|
|
357
503
|
await client.waitForHealth({
|
|
358
504
|
timeoutMs: Number(process.env.NEOAGENT_VM_BOOT_TIMEOUT_MS || 20 * 60 * 1000),
|
|
505
|
+
signal: options.signal,
|
|
359
506
|
checkLiveness: () => {
|
|
360
507
|
const key = String(userId || '').trim();
|
|
361
508
|
const session = this.vmManager.instances.get(key);
|
|
@@ -363,6 +510,7 @@ class LocalVmExecutionBackend {
|
|
|
363
510
|
},
|
|
364
511
|
});
|
|
365
512
|
} catch (error) {
|
|
513
|
+
if (options.signal?.aborted) throw abortError(options.signal);
|
|
366
514
|
const runtimeError = typeof session.getLastError === 'function' ? session.getLastError() : '';
|
|
367
515
|
const detail = runtimeError ? ` ${runtimeError}` : '';
|
|
368
516
|
throw new Error(`${error.message}${detail}`.trim());
|
|
@@ -371,7 +519,11 @@ class LocalVmExecutionBackend {
|
|
|
371
519
|
}
|
|
372
520
|
|
|
373
521
|
async executeCommand(userId, command, options = {}) {
|
|
374
|
-
const client = await this.#clientForUser(userId);
|
|
522
|
+
const client = await this.#clientForUser(userId, options);
|
|
523
|
+
const requestedCommandTimeout = Number(options.timeout || 0);
|
|
524
|
+
const transportTimeout = requestedCommandTimeout > 0
|
|
525
|
+
? Math.min(30 * 60 * 1000, requestedCommandTimeout + 30_000)
|
|
526
|
+
: 16 * 60 * 1000;
|
|
375
527
|
return client.request('POST', '/exec', {
|
|
376
528
|
command,
|
|
377
529
|
cwd: options.cwd,
|
|
@@ -379,6 +531,10 @@ class LocalVmExecutionBackend {
|
|
|
379
531
|
stdin_input: options.stdinInput,
|
|
380
532
|
pty: options.pty === true,
|
|
381
533
|
inputs: options.inputs || [],
|
|
534
|
+
}, {
|
|
535
|
+
timeoutMs: transportTimeout,
|
|
536
|
+
retryCount: 0,
|
|
537
|
+
signal: options.signal,
|
|
382
538
|
});
|
|
383
539
|
}
|
|
384
540
|
|
|
@@ -390,8 +546,8 @@ class LocalVmExecutionBackend {
|
|
|
390
546
|
});
|
|
391
547
|
}
|
|
392
548
|
|
|
393
|
-
async getBrowserProviderForUser(userId) {
|
|
394
|
-
return new VmBrowserProvider(await this.#clientForUser(userId), {
|
|
549
|
+
async getBrowserProviderForUser(userId, options = {}) {
|
|
550
|
+
return new VmBrowserProvider(await this.#clientForUser(userId, options), {
|
|
395
551
|
userId,
|
|
396
552
|
artifactStore: this.artifactStore,
|
|
397
553
|
});
|
|
@@ -431,11 +587,15 @@ class LocalVmExecutionBackend {
|
|
|
431
587
|
}
|
|
432
588
|
|
|
433
589
|
async shutdown() {
|
|
590
|
+
if (this.shutdownPromise) return this.shutdownPromise;
|
|
591
|
+
this.shuttingDown = true;
|
|
434
592
|
if (this.reaperInterval) {
|
|
435
593
|
clearInterval(this.reaperInterval);
|
|
436
594
|
this.reaperInterval = null;
|
|
437
595
|
}
|
|
438
|
-
|
|
596
|
+
this.shutdownPromise = Promise.resolve(this.vmManager?.shutdown?.());
|
|
597
|
+
await this.shutdownPromise;
|
|
598
|
+
return { state: 'stopped' };
|
|
439
599
|
}
|
|
440
600
|
}
|
|
441
601
|
|
|
@@ -306,6 +306,15 @@ class DockerVMManager {
|
|
|
306
306
|
return Boolean(session && isContainerRunning(session.containerId));
|
|
307
307
|
}
|
|
308
308
|
|
|
309
|
+
// A side-effect-free snapshot for request-path capability reporting. Unlike
|
|
310
|
+
// hasVm(), this deliberately avoids a synchronous `docker inspect`, which can
|
|
311
|
+
// block the Node event loop. Tool execution still calls ensureVm() and performs
|
|
312
|
+
// the authoritative liveness check before using a tracked session.
|
|
313
|
+
hasTrackedVm(userId) {
|
|
314
|
+
const key = String(userId || '').trim();
|
|
315
|
+
return Boolean(key && this.instances.has(key));
|
|
316
|
+
}
|
|
317
|
+
|
|
309
318
|
// Used by validation.js — cached to avoid docker calls on every status poll.
|
|
310
319
|
getReadiness() {
|
|
311
320
|
const now = Date.now();
|
|
@@ -7,6 +7,22 @@ const VM_ROOT = path.join(DATA_DIR, 'runtime-vms');
|
|
|
7
7
|
const GUEST_BOOTSTRAP_ROOT = path.join(VM_ROOT, 'guest-bootstrap');
|
|
8
8
|
const REPO_ROOT = path.resolve(__dirname, '../../..');
|
|
9
9
|
const GUEST_PAYLOAD_PROFILES = Object.freeze({
|
|
10
|
+
browser: [
|
|
11
|
+
{ source: 'server/guest-agent.browser.package.json', target: 'package.json' },
|
|
12
|
+
{ source: 'runtime/env.js', target: 'runtime/env.js' },
|
|
13
|
+
{ source: 'runtime/paths.js', target: 'runtime/paths.js' },
|
|
14
|
+
{ source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
|
|
15
|
+
{ source: 'server/services/browser', target: 'server/services/browser' },
|
|
16
|
+
{ source: 'server/services/android/process.js', target: 'server/services/android/process.js' },
|
|
17
|
+
{ source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
|
|
18
|
+
],
|
|
19
|
+
cli: [
|
|
20
|
+
{ source: 'server/guest-agent.cli.package.json', target: 'package.json' },
|
|
21
|
+
{ source: 'runtime/env.js', target: 'runtime/env.js' },
|
|
22
|
+
{ source: 'runtime/paths.js', target: 'runtime/paths.js' },
|
|
23
|
+
{ source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
|
|
24
|
+
{ source: 'server/services/cli', target: 'server/services/cli' },
|
|
25
|
+
],
|
|
10
26
|
browser_cli: [
|
|
11
27
|
{ source: 'server/guest-agent.browser.package.json', target: 'package.json' },
|
|
12
28
|
{ source: 'runtime/env.js', target: 'runtime/env.js' },
|
|
@@ -14,6 +30,8 @@ const GUEST_PAYLOAD_PROFILES = Object.freeze({
|
|
|
14
30
|
{ source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
|
|
15
31
|
{ source: 'server/services/cli', target: 'server/services/cli' },
|
|
16
32
|
{ source: 'server/services/browser', target: 'server/services/browser' },
|
|
33
|
+
{ source: 'server/services/android/process.js', target: 'server/services/android/process.js' },
|
|
34
|
+
{ source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
|
|
17
35
|
],
|
|
18
36
|
android: [
|
|
19
37
|
{ source: 'server/guest-agent.android.package.json', target: 'package.json' },
|
|
@@ -22,6 +40,9 @@ const GUEST_PAYLOAD_PROFILES = Object.freeze({
|
|
|
22
40
|
{ source: 'server/guest_agent.js', target: 'server/guest_agent.js' },
|
|
23
41
|
{ source: 'server/services/cli', target: 'server/services/cli' },
|
|
24
42
|
{ source: 'server/services/android', target: 'server/services/android' },
|
|
43
|
+
{ source: 'server/utils/abort.js', target: 'server/utils/abort.js' },
|
|
44
|
+
{ source: 'server/utils/cloud-security.js', target: 'server/utils/cloud-security.js' },
|
|
45
|
+
{ source: 'server/utils/image_payload.js', target: 'server/utils/image_payload.js' },
|
|
25
46
|
],
|
|
26
47
|
});
|
|
27
48
|
|
|
@@ -32,7 +53,10 @@ function encodeGuestToken(value) {
|
|
|
32
53
|
}
|
|
33
54
|
|
|
34
55
|
function normalizeRuntimeProfile(runtimeProfile) {
|
|
35
|
-
|
|
56
|
+
const normalized = String(runtimeProfile || '').trim();
|
|
57
|
+
return ['android', 'browser', 'cli', 'browser_cli'].includes(normalized)
|
|
58
|
+
? normalized
|
|
59
|
+
: 'browser_cli';
|
|
36
60
|
}
|
|
37
61
|
|
|
38
62
|
// Copy the guest runtime source files for a profile into `stagingRoot`, producing
|
|
@@ -84,10 +108,12 @@ function createCloudInitScript({
|
|
|
84
108
|
runtimeProfile = 'browser_cli',
|
|
85
109
|
}) {
|
|
86
110
|
const normalizedProfile = normalizeRuntimeProfile(runtimeProfile);
|
|
87
|
-
const includeBrowser =
|
|
111
|
+
const includeBrowser = ['browser', 'browser_cli'].includes(normalizedProfile);
|
|
88
112
|
const guestUtilityPackages = includeBrowser
|
|
89
113
|
? 'curl ca-certificates gnupg git rsync unzip xvfb dbus-x11'
|
|
90
|
-
:
|
|
114
|
+
: normalizedProfile === 'android'
|
|
115
|
+
? 'curl ca-certificates gnupg git rsync unzip dbus-x11 adb'
|
|
116
|
+
: 'curl ca-certificates gnupg git rsync unzip';
|
|
91
117
|
const guestTokenB64 = encodeGuestToken(guestToken);
|
|
92
118
|
const envFile = '/etc/neoagent/neoagent.env';
|
|
93
119
|
const appDir = '/opt/neoagent';
|
|
@@ -208,7 +234,7 @@ function createCloudInitUserData({
|
|
|
208
234
|
runtimeProfile = 'browser_cli',
|
|
209
235
|
}) {
|
|
210
236
|
const normalizedProfile = normalizeRuntimeProfile(runtimeProfile);
|
|
211
|
-
const includeBrowser =
|
|
237
|
+
const includeBrowser = ['browser', 'browser_cli'].includes(normalizedProfile);
|
|
212
238
|
const guestAgentInnerCommand = includeBrowser
|
|
213
239
|
? 'set -a; . /etc/neoagent/neoagent.env; set +a; cd /opt/neoagent && env DISPLAY=:99 PLAYWRIGHT_BROWSERS_PATH=/opt/neoagent/.playwright-browsers /usr/bin/env node server/guest_agent.js 2>&1 | tee -a /var/log/neoagent-guest-agent.log >/dev/console'
|
|
214
240
|
: 'set -a; . /etc/neoagent/neoagent.env; set +a; cd /opt/neoagent && /usr/bin/env node server/guest_agent.js 2>&1 | tee -a /var/log/neoagent-guest-agent.log >/dev/console';
|