neoagent 3.2.1-beta.2 → 3.2.1-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/chrome-browser/background.mjs +318 -88
- package/extensions/chrome-browser/http.mjs +136 -0
- package/extensions/chrome-browser/protocol.mjs +511 -89
- package/flutter_app/lib/main_chat.dart +118 -739
- package/flutter_app/lib/main_controller.dart +29 -20
- package/flutter_app/lib/main_models.dart +3 -0
- package/flutter_app/lib/main_operations.dart +334 -321
- package/flutter_app/lib/main_settings.dart +4 -3
- package/flutter_app/lib/main_shared.dart +14 -11
- package/flutter_app/lib/src/desktop_companion_actions.dart +185 -31
- package/flutter_app/lib/src/desktop_companion_io.dart +319 -86
- package/flutter_app/windows/runner/flutter_window.cpp +143 -32
- package/lib/manager.js +106 -89
- package/lib/schema_migrations.js +67 -13
- package/package.json +20 -13
- package/runtime/paths.js +49 -5
- package/server/guest_agent.js +52 -30
- package/server/http/middleware.js +24 -0
- package/server/http/routes.js +4 -6
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +30803 -30805
- package/server/routes/admin.js +1 -1
- package/server/routes/android.js +30 -34
- package/server/routes/browser.js +23 -15
- package/server/routes/desktop.js +18 -1
- package/server/routes/integrations.js +5 -1
- package/server/routes/memory.js +1 -0
- package/server/routes/settings.js +16 -5
- package/server/routes/social_reach.js +12 -3
- package/server/routes/social_video.js +4 -0
- package/server/services/ai/compaction.js +7 -2
- package/server/services/ai/history.js +44 -5
- package/server/services/ai/integrated_tools/http_request.js +8 -0
- package/server/services/ai/loop/agent_engine_core.js +347 -162
- package/server/services/ai/loop/callbacks.js +1 -0
- package/server/services/ai/loop/completion_judge.js +12 -0
- package/server/services/ai/loop/conversation_loop.js +348 -242
- package/server/services/ai/loop/messaging_delivery.js +129 -57
- package/server/services/ai/loop/model_call_guard.js +91 -0
- package/server/services/ai/loop/model_io.js +8 -41
- package/server/services/ai/loop/tool_dispatch.js +19 -8
- package/server/services/ai/loopPolicy.js +24 -19
- package/server/services/ai/model_discovery.js +227 -0
- package/server/services/ai/model_identity.js +71 -0
- package/server/services/ai/models.js +67 -162
- package/server/services/ai/providerRetry.js +17 -59
- package/server/services/ai/provider_selector.js +111 -0
- package/server/services/ai/providers/anthropic.js +2 -2
- package/server/services/ai/providers/claudeCode.js +15 -28
- package/server/services/ai/providers/githubCopilot.js +36 -16
- package/server/services/ai/providers/google.js +23 -5
- package/server/services/ai/providers/grok.js +4 -3
- package/server/services/ai/providers/grokOauth.js +13 -22
- package/server/services/ai/providers/nvidia.js +10 -5
- package/server/services/ai/providers/ollama.js +102 -82
- package/server/services/ai/providers/ollama_stream.js +142 -0
- package/server/services/ai/providers/openai.js +39 -5
- package/server/services/ai/providers/openaiCodex.js +11 -4
- package/server/services/ai/providers/openrouter.js +29 -7
- package/server/services/ai/providers/provider_error.js +36 -0
- package/server/services/ai/settings.js +26 -2
- package/server/services/ai/taskAnalysis.js +5 -1
- package/server/services/ai/terminal_reply.js +45 -0
- package/server/services/ai/toolEvidence.js +58 -29
- package/server/services/ai/tools.js +124 -111
- package/server/services/android/controller.js +770 -237
- package/server/services/android/process.js +140 -0
- package/server/services/android/sdk_download.js +143 -0
- package/server/services/android/uia.js +6 -5
- package/server/services/artifacts/store.js +24 -0
- package/server/services/browser/controller.js +736 -385
- package/server/services/browser/extension/gateway.js +40 -16
- package/server/services/browser/extension/protocol.js +12 -1
- package/server/services/browser/extension/provider.js +59 -47
- package/server/services/browser/extension/registry.js +155 -34
- package/server/services/cli/executor.js +62 -9
- package/server/services/desktop/gateway.js +41 -4
- package/server/services/desktop/protocol.js +3 -0
- package/server/services/desktop/provider.js +39 -42
- package/server/services/desktop/registry.js +137 -52
- package/server/services/integrations/figma/provider.js +78 -12
- package/server/services/integrations/github/common.js +11 -6
- package/server/services/integrations/github/provider.js +52 -53
- package/server/services/integrations/google/provider.js +55 -19
- package/server/services/integrations/home_assistant/network.js +17 -20
- package/server/services/integrations/home_assistant/provider.js +7 -5
- package/server/services/integrations/home_assistant/tools.js +17 -5
- package/server/services/integrations/http.js +51 -0
- package/server/services/integrations/manager.js +158 -53
- package/server/services/integrations/microsoft/provider.js +80 -13
- package/server/services/integrations/neoarchive/provider.js +55 -29
- package/server/services/integrations/neorecall/client.js +17 -10
- package/server/services/integrations/neorecall/provider.js +20 -11
- package/server/services/integrations/notion/provider.js +16 -13
- package/server/services/integrations/oauth_provider.js +115 -51
- package/server/services/integrations/slack/provider.js +98 -9
- package/server/services/integrations/spotify/provider.js +67 -71
- package/server/services/integrations/trello/provider.js +21 -7
- package/server/services/integrations/weather/provider.js +18 -12
- package/server/services/integrations/whatsapp/provider.js +76 -16
- package/server/services/manager.js +87 -1
- package/server/services/memory/embedding_index.js +20 -8
- package/server/services/memory/embeddings.js +151 -90
- package/server/services/memory/ingestion.js +50 -9
- package/server/services/memory/ingestion_documents.js +13 -3
- package/server/services/memory/manager.js +52 -19
- package/server/services/messaging/automation.js +84 -9
- package/server/services/messaging/http_platforms.js +33 -13
- package/server/services/messaging/inbound_queue.js +78 -24
- package/server/services/messaging/inbound_store.js +224 -0
- package/server/services/messaging/manager.js +326 -51
- package/server/services/messaging/typing_keepalive.js +5 -2
- package/server/services/network/http.js +210 -0
- package/server/services/network/safe_request.js +307 -0
- package/server/services/runtime/backends/local-vm.js +214 -66
- package/server/services/runtime/manager.js +17 -12
- package/server/services/social_reach/channels/github.js +10 -4
- package/server/services/social_reach/channels/reddit.js +4 -4
- package/server/services/social_reach/channels/rss.js +2 -2
- package/server/services/social_reach/channels/social_video.js +12 -7
- package/server/services/social_reach/channels/v2ex.js +21 -8
- package/server/services/social_reach/channels/x.js +2 -2
- package/server/services/social_reach/channels/xueqiu.js +5 -5
- package/server/services/social_reach/service.js +9 -6
- package/server/services/social_reach/utils.js +65 -14
- package/server/services/social_video/service.js +160 -50
- package/server/services/tasks/integration_runtime.js +18 -8
- package/server/services/tasks/runtime.js +39 -4
- package/server/services/voice/agentBridge.js +17 -4
- package/server/services/voice/bufferedLiveRelayAdapter.js +5 -0
- package/server/services/voice/liveSession.js +31 -0
- package/server/services/voice/openaiSpeech.js +33 -8
- package/server/services/voice/providers.js +233 -151
- package/server/services/voice/runtimeManager.js +118 -20
- package/server/services/voice/turnRunner.js +6 -0
- package/server/services/wearable/firmware_manifest.js +51 -13
- package/server/services/wearable/service.js +1 -0
- package/server/utils/abort.js +96 -0
- package/server/utils/cloud-security.js +110 -3
- package/server/utils/files.js +31 -0
- package/server/utils/image_payload.js +95 -0
- package/server/utils/retry.js +107 -0
|
@@ -3,6 +3,7 @@ const { BROWSER_EXTENSION_WS_PATH } = require('./protocol');
|
|
|
3
3
|
|
|
4
4
|
const DEFAULT_UPGRADE_RATE_LIMIT_WINDOW_MS = 60 * 1000;
|
|
5
5
|
const DEFAULT_UPGRADE_RATE_LIMIT_MAX = 30;
|
|
6
|
+
const MAX_EXTENSION_MESSAGE_BYTES = 32 * 1024 * 1024;
|
|
6
7
|
|
|
7
8
|
function rejectUpgrade(socket, statusCode, message) {
|
|
8
9
|
try {
|
|
@@ -22,8 +23,13 @@ function rejectUpgrade(socket, statusCode, message) {
|
|
|
22
23
|
}
|
|
23
24
|
|
|
24
25
|
function bindBrowserExtensionGateway(httpServer, app) {
|
|
25
|
-
const wss = new WebSocketServer({
|
|
26
|
+
const wss = new WebSocketServer({
|
|
27
|
+
noServer: true,
|
|
28
|
+
maxPayload: MAX_EXTENSION_MESSAGE_BYTES,
|
|
29
|
+
});
|
|
26
30
|
const attemptsByIp = new Map();
|
|
31
|
+
let closing = false;
|
|
32
|
+
let closePromise = null;
|
|
27
33
|
const windowMs = Number(process.env.NEOAGENT_BROWSER_EXTENSION_UPGRADE_WINDOW_MS || DEFAULT_UPGRADE_RATE_LIMIT_WINDOW_MS);
|
|
28
34
|
|
|
29
35
|
const cleanupTimer = setInterval(() => {
|
|
@@ -50,7 +56,7 @@ function bindBrowserExtensionGateway(httpServer, app) {
|
|
|
50
56
|
return entry.count > maxAttempts;
|
|
51
57
|
}
|
|
52
58
|
|
|
53
|
-
|
|
59
|
+
const handleUpgrade = (req, socket, head) => {
|
|
54
60
|
let url;
|
|
55
61
|
try {
|
|
56
62
|
url = new URL(req.url, 'http://localhost');
|
|
@@ -60,6 +66,10 @@ function bindBrowserExtensionGateway(httpServer, app) {
|
|
|
60
66
|
if (url.pathname !== BROWSER_EXTENSION_WS_PATH) {
|
|
61
67
|
return;
|
|
62
68
|
}
|
|
69
|
+
if (closing) {
|
|
70
|
+
rejectUpgrade(socket, 503, 'Service Unavailable');
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
63
73
|
|
|
64
74
|
const remoteAddress = req.socket?.remoteAddress || 'unknown';
|
|
65
75
|
if (isRateLimited(remoteAddress)) {
|
|
@@ -81,24 +91,38 @@ function bindBrowserExtensionGateway(httpServer, app) {
|
|
|
81
91
|
}
|
|
82
92
|
|
|
83
93
|
wss.handleUpgrade(req, socket, head, (ws) => {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
+
try {
|
|
95
|
+
registry.registerConnection(tokenRow, ws, {
|
|
96
|
+
remoteAddress,
|
|
97
|
+
userAgent: req.headers['user-agent'] || null,
|
|
98
|
+
});
|
|
99
|
+
ws.send(JSON.stringify({
|
|
100
|
+
type: 'hello',
|
|
101
|
+
ok: true,
|
|
102
|
+
userId: tokenRow.user_id,
|
|
103
|
+
tokenId: tokenRow.id,
|
|
104
|
+
}));
|
|
105
|
+
} catch (error) {
|
|
106
|
+
try { ws.close(1012, String(error?.message || 'Service unavailable').slice(0, 120)); } catch {}
|
|
107
|
+
}
|
|
94
108
|
});
|
|
95
|
-
}
|
|
109
|
+
};
|
|
110
|
+
httpServer.on('upgrade', handleUpgrade);
|
|
96
111
|
|
|
97
112
|
app.locals.browserExtensionGateway = {
|
|
98
|
-
close: () =>
|
|
113
|
+
close: () => {
|
|
114
|
+
if (closePromise) return closePromise;
|
|
115
|
+
closing = true;
|
|
99
116
|
clearInterval(cleanupTimer);
|
|
100
|
-
|
|
101
|
-
|
|
117
|
+
httpServer.removeListener('upgrade', handleUpgrade);
|
|
118
|
+
for (const client of wss.clients) {
|
|
119
|
+
try { client.terminate(); } catch {}
|
|
120
|
+
}
|
|
121
|
+
closePromise = new Promise((resolve) => {
|
|
122
|
+
wss.close(() => resolve());
|
|
123
|
+
});
|
|
124
|
+
return closePromise;
|
|
125
|
+
},
|
|
102
126
|
};
|
|
103
127
|
|
|
104
128
|
return wss;
|
|
@@ -1,6 +1,15 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
1
3
|
const BROWSER_EXTENSION_WS_PATH = '/api/browser-extension/ws';
|
|
2
4
|
const EXTENSION_PROTOCOL_VERSION = 1;
|
|
3
5
|
|
|
6
|
+
const EXTENSION_MESSAGE_TYPES = Object.freeze({
|
|
7
|
+
COMMAND: 'command',
|
|
8
|
+
RESULT: 'result',
|
|
9
|
+
URL_VALIDATION_REQUEST: 'urlValidationRequest',
|
|
10
|
+
URL_VALIDATION_RESULT: 'urlValidationResult',
|
|
11
|
+
});
|
|
12
|
+
|
|
4
13
|
const EXTENSION_COMMANDS = Object.freeze({
|
|
5
14
|
LAUNCH: 'launch',
|
|
6
15
|
NAVIGATE: 'navigate',
|
|
@@ -16,6 +25,7 @@ const EXTENSION_COMMANDS = Object.freeze({
|
|
|
16
25
|
CLOSE: 'close',
|
|
17
26
|
GET_PAGE_INFO: 'getPageInfo',
|
|
18
27
|
GET_COOKIES: 'getCookies',
|
|
28
|
+
CANCEL_COMMAND: 'cancelCommand',
|
|
19
29
|
});
|
|
20
30
|
|
|
21
31
|
class ExtensionBrowserUnavailableError extends Error {
|
|
@@ -28,7 +38,7 @@ class ExtensionBrowserUnavailableError extends Error {
|
|
|
28
38
|
|
|
29
39
|
function createCommandMessage(id, command, payload = {}) {
|
|
30
40
|
return {
|
|
31
|
-
type:
|
|
41
|
+
type: EXTENSION_MESSAGE_TYPES.COMMAND,
|
|
32
42
|
version: EXTENSION_PROTOCOL_VERSION,
|
|
33
43
|
id,
|
|
34
44
|
command,
|
|
@@ -46,6 +56,7 @@ function parseExtensionMessage(data) {
|
|
|
46
56
|
module.exports = {
|
|
47
57
|
BROWSER_EXTENSION_WS_PATH,
|
|
48
58
|
EXTENSION_PROTOCOL_VERSION,
|
|
59
|
+
EXTENSION_MESSAGE_TYPES,
|
|
49
60
|
EXTENSION_COMMANDS,
|
|
50
61
|
ExtensionBrowserUnavailableError,
|
|
51
62
|
createCommandMessage,
|
|
@@ -1,18 +1,15 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
1
3
|
const fs = require('fs');
|
|
2
4
|
const path = require('path');
|
|
3
5
|
const { DATA_DIR } = require('../../../../runtime/paths');
|
|
6
|
+
const { writeBufferAtomic } = require('../../../utils/files');
|
|
7
|
+
const { decodeBase64Image } = require('../../../utils/image_payload');
|
|
4
8
|
const { EXTENSION_COMMANDS, ExtensionBrowserUnavailableError } = require('./protocol');
|
|
5
9
|
|
|
6
10
|
const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
|
|
7
11
|
if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
|
|
8
12
|
|
|
9
|
-
function extractBase64Png(value) {
|
|
10
|
-
const text = String(value || '');
|
|
11
|
-
if (!text) return null;
|
|
12
|
-
const match = text.match(/^data:image\/png;base64,(.+)$/);
|
|
13
|
-
return match ? match[1] : text;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
13
|
class ExtensionBrowserProvider {
|
|
17
14
|
constructor(options = {}) {
|
|
18
15
|
this.registry = options.registry;
|
|
@@ -31,11 +28,18 @@ class ExtensionBrowserProvider {
|
|
|
31
28
|
|
|
32
29
|
async #dispatch(command, payload = {}, options = {}) {
|
|
33
30
|
this.#assertReady();
|
|
34
|
-
const
|
|
31
|
+
const safePayload = { ...(payload || {}) };
|
|
32
|
+
const signal = options.signal || safePayload.signal || null;
|
|
33
|
+
const timeoutMs = options.timeoutMs ?? safePayload.timeoutMs;
|
|
34
|
+
delete safePayload.signal;
|
|
35
|
+
delete safePayload.timeoutMs;
|
|
36
|
+
const result = await this.registry.dispatch(this.userId, command, safePayload, {
|
|
35
37
|
...options,
|
|
38
|
+
signal,
|
|
39
|
+
timeoutMs,
|
|
36
40
|
tokenId: options.tokenId || this.tokenId,
|
|
37
41
|
});
|
|
38
|
-
return this.#materialize(result);
|
|
42
|
+
return this.#materialize(result, { signal });
|
|
39
43
|
}
|
|
40
44
|
|
|
41
45
|
#disconnect() {
|
|
@@ -46,18 +50,17 @@ class ExtensionBrowserProvider {
|
|
|
46
50
|
}
|
|
47
51
|
}
|
|
48
52
|
|
|
49
|
-
#writeScreenshotArtifact(
|
|
50
|
-
const buffer = Buffer.from(base64, 'base64');
|
|
53
|
+
async #writeScreenshotArtifact(image, options = {}) {
|
|
51
54
|
if (this.artifactStore && this.userId != null) {
|
|
52
|
-
const artifact = this.artifactStore.
|
|
55
|
+
const artifact = await this.artifactStore.createBufferArtifact(this.userId, {
|
|
53
56
|
kind: 'browser-screenshot',
|
|
54
57
|
backend: 'extension',
|
|
55
|
-
extension:
|
|
56
|
-
contentType:
|
|
58
|
+
extension: image.extension,
|
|
59
|
+
contentType: image.contentType,
|
|
57
60
|
filenameBase: 'browser-extension-screenshot',
|
|
61
|
+
content: image.buffer,
|
|
62
|
+
signal: options.signal,
|
|
58
63
|
});
|
|
59
|
-
fs.writeFileSync(artifact.storagePath, buffer);
|
|
60
|
-
this.artifactStore.finalizeFile(artifact.artifactId, artifact.storagePath);
|
|
61
64
|
return {
|
|
62
65
|
screenshotPath: artifact.url,
|
|
63
66
|
artifactId: artifact.artifactId,
|
|
@@ -68,7 +71,7 @@ class ExtensionBrowserProvider {
|
|
|
68
71
|
|
|
69
72
|
const filename = `browser_extension_${Date.now()}_${Math.random().toString(16).slice(2)}.png`;
|
|
70
73
|
const fullPath = path.join(SCREENSHOTS_DIR, filename);
|
|
71
|
-
|
|
74
|
+
await writeBufferAtomic(fullPath, image.buffer, { signal: options.signal });
|
|
72
75
|
return {
|
|
73
76
|
screenshotPath: `/screenshots/${filename}`,
|
|
74
77
|
artifactId: null,
|
|
@@ -77,13 +80,12 @@ class ExtensionBrowserProvider {
|
|
|
77
80
|
};
|
|
78
81
|
}
|
|
79
82
|
|
|
80
|
-
#materialize(result) {
|
|
83
|
+
async #materialize(result, options = {}) {
|
|
81
84
|
if (!result || typeof result !== 'object') return result;
|
|
82
85
|
const raw = result.screenshotDataUrl || result.screenshotData || result.screenshotBase64;
|
|
83
86
|
if (!raw) return result;
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
const screenshot = this.#writeScreenshotArtifact(base64);
|
|
87
|
+
const image = decodeBase64Image(raw, { allowedTypes: ['image/png'] });
|
|
88
|
+
const screenshot = await this.#writeScreenshotArtifact(image, options);
|
|
87
89
|
const next = { ...result, ...screenshot };
|
|
88
90
|
delete next.screenshotDataUrl;
|
|
89
91
|
delete next.screenshotData;
|
|
@@ -92,79 +94,90 @@ class ExtensionBrowserProvider {
|
|
|
92
94
|
}
|
|
93
95
|
|
|
94
96
|
navigate(url, options = {}) {
|
|
95
|
-
return this.#dispatch(EXTENSION_COMMANDS.NAVIGATE, { url, ...options });
|
|
97
|
+
return this.#dispatch(EXTENSION_COMMANDS.NAVIGATE, { url, ...options }, options);
|
|
96
98
|
}
|
|
97
99
|
|
|
98
|
-
click(selector, text, screenshot = true) {
|
|
99
|
-
return this.#dispatch(EXTENSION_COMMANDS.CLICK, { selector, text, screenshot });
|
|
100
|
+
click(selector, text, screenshot = true, options = {}) {
|
|
101
|
+
return this.#dispatch(EXTENSION_COMMANDS.CLICK, { selector, text, screenshot }, options);
|
|
100
102
|
}
|
|
101
103
|
|
|
102
|
-
clickPoint(x, y, screenshot = true) {
|
|
103
|
-
return this.#dispatch(EXTENSION_COMMANDS.CLICK_POINT, { x, y, screenshot });
|
|
104
|
+
clickPoint(x, y, screenshot = true, options = {}) {
|
|
105
|
+
return this.#dispatch(EXTENSION_COMMANDS.CLICK_POINT, { x, y, screenshot }, options);
|
|
104
106
|
}
|
|
105
107
|
|
|
106
108
|
type(selector, text, options = {}) {
|
|
107
|
-
return this.#dispatch(EXTENSION_COMMANDS.TYPE, { selector, text, ...options });
|
|
109
|
+
return this.#dispatch(EXTENSION_COMMANDS.TYPE, { selector, text, ...options }, options);
|
|
108
110
|
}
|
|
109
111
|
|
|
110
112
|
typeText(text, options = {}) {
|
|
111
|
-
return this.#dispatch(EXTENSION_COMMANDS.TYPE_TEXT, { text, ...options });
|
|
113
|
+
return this.#dispatch(EXTENSION_COMMANDS.TYPE_TEXT, { text, ...options }, options);
|
|
112
114
|
}
|
|
113
115
|
|
|
114
|
-
pressKey(key, screenshot = true) {
|
|
115
|
-
return this.#dispatch(EXTENSION_COMMANDS.PRESS_KEY, { key, screenshot });
|
|
116
|
+
pressKey(key, screenshot = true, options = {}) {
|
|
117
|
+
return this.#dispatch(EXTENSION_COMMANDS.PRESS_KEY, { key, screenshot }, options);
|
|
116
118
|
}
|
|
117
119
|
|
|
118
|
-
scroll(deltaX = 0, deltaY = 0, screenshot = true) {
|
|
119
|
-
return this.#dispatch(EXTENSION_COMMANDS.SCROLL, { deltaX, deltaY, screenshot });
|
|
120
|
+
scroll(deltaX = 0, deltaY = 0, screenshot = true, options = {}) {
|
|
121
|
+
return this.#dispatch(EXTENSION_COMMANDS.SCROLL, { deltaX, deltaY, screenshot }, options);
|
|
120
122
|
}
|
|
121
123
|
|
|
122
|
-
extract(selector, attribute, all = false) {
|
|
123
|
-
return this.#dispatch(EXTENSION_COMMANDS.EXTRACT, { selector, attribute, all });
|
|
124
|
+
extract(selector, attribute, all = false, options = {}) {
|
|
125
|
+
return this.#dispatch(EXTENSION_COMMANDS.EXTRACT, { selector, attribute, all }, options);
|
|
124
126
|
}
|
|
125
127
|
|
|
126
|
-
evaluate(script) {
|
|
127
|
-
return this.#dispatch(EXTENSION_COMMANDS.EVALUATE, { script });
|
|
128
|
+
evaluate(script, options = {}) {
|
|
129
|
+
return this.#dispatch(EXTENSION_COMMANDS.EVALUATE, { script }, options);
|
|
128
130
|
}
|
|
129
131
|
|
|
130
132
|
screenshot(options = {}) {
|
|
131
|
-
return this.#dispatch(EXTENSION_COMMANDS.SCREENSHOT, options);
|
|
133
|
+
return this.#dispatch(EXTENSION_COMMANDS.SCREENSHOT, options, options);
|
|
132
134
|
}
|
|
133
135
|
|
|
134
136
|
launch(options = {}) {
|
|
135
|
-
return this.#dispatch(EXTENSION_COMMANDS.LAUNCH, options);
|
|
137
|
+
return this.#dispatch(EXTENSION_COMMANDS.LAUNCH, options, options);
|
|
136
138
|
}
|
|
137
139
|
|
|
138
|
-
async closeBrowser() {
|
|
140
|
+
async closeBrowser(options = {}) {
|
|
139
141
|
if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId, this.tokenId)) {
|
|
140
142
|
return { success: true, extensionConnected: false };
|
|
141
143
|
}
|
|
142
|
-
const result = await this.#dispatch(EXTENSION_COMMANDS.CLOSE, {});
|
|
144
|
+
const result = await this.#dispatch(EXTENSION_COMMANDS.CLOSE, {}, options);
|
|
143
145
|
this.#disconnect();
|
|
144
146
|
return { ...result, success: result?.success !== false, extensionConnected: false };
|
|
145
147
|
}
|
|
146
148
|
|
|
147
|
-
fill(selector, value) {
|
|
148
|
-
return this.type(selector, String(value));
|
|
149
|
+
fill(selector, value, options = {}) {
|
|
150
|
+
return this.type(selector, String(value), options);
|
|
149
151
|
}
|
|
150
152
|
|
|
151
153
|
extractContent(options = {}) {
|
|
152
|
-
return this.extract(options.selector, options.attribute, options.all);
|
|
154
|
+
return this.extract(options.selector, options.attribute, options.all, options);
|
|
153
155
|
}
|
|
154
156
|
|
|
155
|
-
executeJS(code) {
|
|
156
|
-
return this.evaluate(code);
|
|
157
|
+
executeJS(code, options = {}) {
|
|
158
|
+
return this.evaluate(code, options);
|
|
157
159
|
}
|
|
158
160
|
|
|
159
|
-
async getPageInfo() {
|
|
161
|
+
async getPageInfo(options = {}) {
|
|
160
162
|
if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId, this.tokenId)) {
|
|
161
163
|
return { url: null, title: null, extensionConnected: false };
|
|
162
164
|
}
|
|
163
165
|
return this.registry.dispatch(this.userId, EXTENSION_COMMANDS.GET_PAGE_INFO, {}, {
|
|
164
166
|
tokenId: this.tokenId,
|
|
167
|
+
signal: options.signal,
|
|
165
168
|
});
|
|
166
169
|
}
|
|
167
170
|
|
|
171
|
+
async getCookies(options = {}) {
|
|
172
|
+
const pageInfo = await this.getPageInfo(options);
|
|
173
|
+
let hostname = '';
|
|
174
|
+
try { hostname = new URL(String(pageInfo?.url || '')).hostname; } catch {}
|
|
175
|
+
if (!hostname) return { cookies: [], domains: [] };
|
|
176
|
+
return this.#dispatch(EXTENSION_COMMANDS.GET_COOKIES, {
|
|
177
|
+
domains: [hostname],
|
|
178
|
+
}, options);
|
|
179
|
+
}
|
|
180
|
+
|
|
168
181
|
isLaunched() {
|
|
169
182
|
return Boolean(this.registry && this.userId != null && this.registry.isConnected(this.userId, this.tokenId));
|
|
170
183
|
}
|
|
@@ -181,5 +194,4 @@ class ExtensionBrowserProvider {
|
|
|
181
194
|
|
|
182
195
|
module.exports = {
|
|
183
196
|
ExtensionBrowserProvider,
|
|
184
|
-
extractBase64Png,
|
|
185
197
|
};
|
|
@@ -1,6 +1,13 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
1
3
|
const crypto = require('crypto');
|
|
2
4
|
const db = require('../../../db/database');
|
|
5
|
+
const { createAbortError } = require('../../../utils/abort');
|
|
6
|
+
const { validateCloudUrlWithDns } = require('../../../utils/cloud-security');
|
|
3
7
|
const {
|
|
8
|
+
EXTENSION_COMMANDS,
|
|
9
|
+
EXTENSION_MESSAGE_TYPES,
|
|
10
|
+
EXTENSION_PROTOCOL_VERSION,
|
|
4
11
|
ExtensionBrowserUnavailableError,
|
|
5
12
|
createCommandMessage,
|
|
6
13
|
parseExtensionMessage,
|
|
@@ -11,6 +18,8 @@ const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
|
|
|
11
18
|
const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
|
|
12
19
|
const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
|
|
13
20
|
const DEFAULT_PRESENCE_TOUCH_INTERVAL_MS = 15 * 1000;
|
|
21
|
+
const DEFAULT_URL_VALIDATION_LIMIT = 128;
|
|
22
|
+
const MAX_URL_VALIDATION_CHARS = 8192;
|
|
14
23
|
|
|
15
24
|
function sha256(value) {
|
|
16
25
|
return crypto.createHash('sha256').update(String(value || '')).digest('hex');
|
|
@@ -48,7 +57,13 @@ class BrowserExtensionRegistry {
|
|
|
48
57
|
this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_INTERVAL_MS || DEFAULT_HEARTBEAT_INTERVAL_MS);
|
|
49
58
|
this.heartbeatTimeoutMs = Number(options.heartbeatTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_TIMEOUT_MS || DEFAULT_HEARTBEAT_TIMEOUT_MS);
|
|
50
59
|
this.presenceTouchIntervalMs = Number(options.presenceTouchIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_PRESENCE_TOUCH_INTERVAL_MS || DEFAULT_PRESENCE_TOUCH_INTERVAL_MS);
|
|
60
|
+
this.urlValidator = options.urlValidator || validateCloudUrlWithDns;
|
|
61
|
+
this.urlValidationLimit = Math.max(1, Math.min(
|
|
62
|
+
512,
|
|
63
|
+
Number(options.urlValidationLimit || DEFAULT_URL_VALIDATION_LIMIT),
|
|
64
|
+
));
|
|
51
65
|
this.connectionsByUser = new Map();
|
|
66
|
+
this.closed = false;
|
|
52
67
|
}
|
|
53
68
|
|
|
54
69
|
#getUserConnections(userId, create = false) {
|
|
@@ -204,6 +219,9 @@ class BrowserExtensionRegistry {
|
|
|
204
219
|
}
|
|
205
220
|
|
|
206
221
|
registerConnection(tokenRow, ws, meta = {}) {
|
|
222
|
+
if (this.closed) {
|
|
223
|
+
throw new ExtensionBrowserUnavailableError('Browser extension registry is shutting down.');
|
|
224
|
+
}
|
|
207
225
|
const userId = String(tokenRow.user_id);
|
|
208
226
|
const userMap = this.#getUserConnections(userId, true);
|
|
209
227
|
const existing = userMap.get(tokenRow.id);
|
|
@@ -223,11 +241,15 @@ class BrowserExtensionRegistry {
|
|
|
223
241
|
presenceTouchIntervalMs: this.presenceTouchIntervalMs,
|
|
224
242
|
});
|
|
225
243
|
userMap.set(tokenRow.id, connection);
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
244
|
+
try {
|
|
245
|
+
this.db.prepare(
|
|
246
|
+
`UPDATE browser_extension_tokens
|
|
247
|
+
SET last_connected_at = datetime('now'), last_seen_at = datetime('now')
|
|
248
|
+
WHERE id = ?`
|
|
249
|
+
).run(tokenRow.id);
|
|
250
|
+
} catch (error) {
|
|
251
|
+
console.warn('[BrowserExtension] Failed to record connection presence:', error?.message);
|
|
252
|
+
}
|
|
231
253
|
return connection;
|
|
232
254
|
}
|
|
233
255
|
|
|
@@ -322,9 +344,13 @@ class BrowserExtensionRegistry {
|
|
|
322
344
|
const userMap = this.#getUserConnections(userId);
|
|
323
345
|
const connection = userMap?.get(String(tokenId));
|
|
324
346
|
if (!connection?.isOpen()) return;
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
347
|
+
try {
|
|
348
|
+
this.db.prepare(
|
|
349
|
+
`UPDATE browser_extension_tokens SET last_seen_at = datetime('now') WHERE id = ?`
|
|
350
|
+
).run(tokenId);
|
|
351
|
+
} catch (error) {
|
|
352
|
+
console.warn('[BrowserExtension] Failed to update connection presence:', error?.message);
|
|
353
|
+
}
|
|
328
354
|
}
|
|
329
355
|
|
|
330
356
|
async dispatch(userId, command, payload = {}, options = {}) {
|
|
@@ -333,12 +359,14 @@ class BrowserExtensionRegistry {
|
|
|
333
359
|
throw new ExtensionBrowserUnavailableError();
|
|
334
360
|
}
|
|
335
361
|
const result = await connection.sendCommand(command, payload, options);
|
|
336
|
-
this.
|
|
337
|
-
`UPDATE browser_extension_tokens SET last_seen_at = datetime('now') WHERE id = ?`
|
|
338
|
-
).run(connection.tokenId);
|
|
362
|
+
this.touchPresence(userId, connection.tokenId);
|
|
339
363
|
return result;
|
|
340
364
|
}
|
|
341
365
|
|
|
366
|
+
validateBrowserUrl(url, options = {}) {
|
|
367
|
+
return this.urlValidator(url, options);
|
|
368
|
+
}
|
|
369
|
+
|
|
342
370
|
getStatus(userId) {
|
|
343
371
|
const userMap = this.#getUserConnections(userId);
|
|
344
372
|
let selectedTokenId = this.getSelectedTokenId(userId);
|
|
@@ -396,22 +424,22 @@ class BrowserExtensionRegistry {
|
|
|
396
424
|
).run(userId);
|
|
397
425
|
}
|
|
398
426
|
|
|
399
|
-
const
|
|
400
|
-
|
|
427
|
+
const userMap = this.#getUserConnections(userId);
|
|
428
|
+
const connections = targetTokenId
|
|
429
|
+
? [userMap?.get(targetTokenId)].filter(Boolean)
|
|
430
|
+
: Array.from(userMap?.values() || []);
|
|
431
|
+
for (const connection of connections) {
|
|
401
432
|
connection.close('extension token revoked');
|
|
402
433
|
}
|
|
403
434
|
return { success: true };
|
|
404
435
|
}
|
|
405
436
|
|
|
406
437
|
closeAll() {
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
} else {
|
|
413
|
-
connection.close('server shutdown');
|
|
414
|
-
}
|
|
438
|
+
this.closed = true;
|
|
439
|
+
const connections = Array.from(this.connectionsByUser.values())
|
|
440
|
+
.flatMap((entry) => entry instanceof Map ? Array.from(entry.values()) : [entry]);
|
|
441
|
+
for (const connection of connections) {
|
|
442
|
+
connection.close('server shutdown');
|
|
415
443
|
}
|
|
416
444
|
this.connectionsByUser.clear();
|
|
417
445
|
}
|
|
@@ -433,24 +461,34 @@ class ExtensionBrowserConnection {
|
|
|
433
461
|
this.lastPongAt = Date.now();
|
|
434
462
|
this.lastPresenceTouchAt = 0;
|
|
435
463
|
this.heartbeatTimer = null;
|
|
464
|
+
this.urlValidationCount = 0;
|
|
465
|
+
this.urlValidationController = new AbortController();
|
|
466
|
+
this.closed = false;
|
|
436
467
|
|
|
437
|
-
ws.on('message', (data) =>
|
|
468
|
+
ws.on('message', (data) => {
|
|
469
|
+
this.#handleMessage(data).catch(() => {});
|
|
470
|
+
});
|
|
438
471
|
ws.on('pong', () => {
|
|
439
472
|
this.lastPongAt = Date.now();
|
|
440
473
|
this.touchPresence();
|
|
441
474
|
});
|
|
442
475
|
ws.on('close', () => this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.')));
|
|
443
|
-
ws.on('error', (error) =>
|
|
476
|
+
ws.on('error', (error) => {
|
|
477
|
+
try { ws.terminate(); } catch {}
|
|
478
|
+
this.#closePending(error);
|
|
479
|
+
});
|
|
444
480
|
this.#startHeartbeat();
|
|
445
481
|
}
|
|
446
482
|
|
|
447
483
|
isOpen() {
|
|
448
|
-
return this.ws && this.ws.readyState === 1;
|
|
484
|
+
return !this.closed && this.ws && this.ws.readyState === 1;
|
|
449
485
|
}
|
|
450
486
|
|
|
451
487
|
close(reason) {
|
|
488
|
+
const wasOpen = this.isOpen();
|
|
489
|
+
this.closed = true;
|
|
452
490
|
try {
|
|
453
|
-
if (
|
|
491
|
+
if (wasOpen) {
|
|
454
492
|
this.ws.close(1000, String(reason || 'closing').slice(0, 120));
|
|
455
493
|
}
|
|
456
494
|
} catch {}
|
|
@@ -478,25 +516,57 @@ class ExtensionBrowserConnection {
|
|
|
478
516
|
return Promise.reject(new ExtensionBrowserUnavailableError());
|
|
479
517
|
}
|
|
480
518
|
const id = crypto.randomUUID();
|
|
481
|
-
const
|
|
519
|
+
const requestedTimeout = Number(options.timeoutMs || this.timeoutMs);
|
|
520
|
+
const timeoutMs = Number.isFinite(requestedTimeout)
|
|
521
|
+
? Math.min(10 * 60 * 1000, Math.max(100, requestedTimeout))
|
|
522
|
+
: DEFAULT_COMMAND_TIMEOUT_MS;
|
|
482
523
|
const message = createCommandMessage(id, command, payload);
|
|
483
524
|
return new Promise((resolve, reject) => {
|
|
484
|
-
const
|
|
525
|
+
const signal = options.signal || null;
|
|
526
|
+
const sendCancellation = () => {
|
|
527
|
+
if (!this.isOpen() || command === EXTENSION_COMMANDS.CANCEL_COMMAND) return;
|
|
528
|
+
try {
|
|
529
|
+
this.ws.send(JSON.stringify(createCommandMessage(
|
|
530
|
+
crypto.randomUUID(),
|
|
531
|
+
EXTENSION_COMMANDS.CANCEL_COMMAND,
|
|
532
|
+
{ commandId: id },
|
|
533
|
+
)));
|
|
534
|
+
} catch {}
|
|
535
|
+
};
|
|
536
|
+
const cleanup = () => {
|
|
537
|
+
clearTimeout(timer);
|
|
538
|
+
signal?.removeEventListener('abort', onAbort);
|
|
485
539
|
this.pending.delete(id);
|
|
486
|
-
|
|
540
|
+
};
|
|
541
|
+
const onAbort = () => {
|
|
542
|
+
cleanup();
|
|
543
|
+
sendCancellation();
|
|
544
|
+
reject(createAbortError(signal, `Browser extension command aborted: ${command}`));
|
|
545
|
+
};
|
|
546
|
+
const timer = setTimeout(() => {
|
|
547
|
+
cleanup();
|
|
548
|
+
sendCancellation();
|
|
549
|
+
const error = new Error(`Browser extension command timed out: ${command}`);
|
|
550
|
+
error.code = 'BROWSER_EXTENSION_COMMAND_TIMEOUT';
|
|
551
|
+
reject(error);
|
|
487
552
|
}, timeoutMs);
|
|
488
|
-
|
|
553
|
+
timer.unref?.();
|
|
554
|
+
if (signal?.aborted) {
|
|
555
|
+
onAbort();
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
558
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
559
|
+
this.pending.set(id, { resolve, reject, timer, command, signal, onAbort });
|
|
489
560
|
try {
|
|
490
561
|
this.ws.send(JSON.stringify(message));
|
|
491
562
|
} catch (error) {
|
|
492
|
-
|
|
493
|
-
this.pending.delete(id);
|
|
563
|
+
cleanup();
|
|
494
564
|
reject(error);
|
|
495
565
|
}
|
|
496
566
|
});
|
|
497
567
|
}
|
|
498
568
|
|
|
499
|
-
#handleMessage(data) {
|
|
569
|
+
async #handleMessage(data) {
|
|
500
570
|
this.lastPongAt = Date.now();
|
|
501
571
|
this.touchPresence();
|
|
502
572
|
let message;
|
|
@@ -505,21 +575,70 @@ class ExtensionBrowserConnection {
|
|
|
505
575
|
} catch {
|
|
506
576
|
return;
|
|
507
577
|
}
|
|
508
|
-
if (!message ||
|
|
578
|
+
if (!message || !message.id) {
|
|
509
579
|
return;
|
|
510
580
|
}
|
|
581
|
+
if (message.type === EXTENSION_MESSAGE_TYPES.URL_VALIDATION_REQUEST) {
|
|
582
|
+
await this.#handleUrlValidationRequest(message);
|
|
583
|
+
return;
|
|
584
|
+
}
|
|
585
|
+
if (message.type !== EXTENSION_MESSAGE_TYPES.RESULT) return;
|
|
511
586
|
const pending = this.pending.get(message.id);
|
|
512
587
|
if (!pending) return;
|
|
513
588
|
clearTimeout(pending.timer);
|
|
589
|
+
pending.signal?.removeEventListener('abort', pending.onAbort);
|
|
514
590
|
this.pending.delete(message.id);
|
|
515
591
|
if (message.ok === false) {
|
|
516
592
|
pending.reject(new Error(message.error || `Browser extension command failed: ${pending.command}`));
|
|
517
593
|
return;
|
|
518
594
|
}
|
|
519
|
-
pending.resolve(message.result
|
|
595
|
+
pending.resolve(message.result ?? {});
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
async #handleUrlValidationRequest(message) {
|
|
599
|
+
const id = typeof message.id === 'string' ? message.id : '';
|
|
600
|
+
if (!id || id.length > 200 || !this.isOpen()) return;
|
|
601
|
+
|
|
602
|
+
let allowed = false;
|
|
603
|
+
const url = typeof message.url === 'string' ? message.url : '';
|
|
604
|
+
const versionMatches = Number(message.version) === EXTENSION_PROTOCOL_VERSION;
|
|
605
|
+
const validationLimit = Number(this.registry.urlValidationLimit) || DEFAULT_URL_VALIDATION_LIMIT;
|
|
606
|
+
if (
|
|
607
|
+
versionMatches
|
|
608
|
+
&& url.length > 0
|
|
609
|
+
&& url.length <= MAX_URL_VALIDATION_CHARS
|
|
610
|
+
&& this.urlValidationCount < validationLimit
|
|
611
|
+
&& typeof this.registry.validateBrowserUrl === 'function'
|
|
612
|
+
) {
|
|
613
|
+
this.urlValidationCount += 1;
|
|
614
|
+
try {
|
|
615
|
+
const result = await this.registry.validateBrowserUrl(url, {
|
|
616
|
+
signal: this.urlValidationController.signal,
|
|
617
|
+
});
|
|
618
|
+
allowed = result?.allowed === true;
|
|
619
|
+
} catch {
|
|
620
|
+
allowed = false;
|
|
621
|
+
} finally {
|
|
622
|
+
this.urlValidationCount -= 1;
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
if (!this.isOpen()) return;
|
|
627
|
+
try {
|
|
628
|
+
this.ws.send(JSON.stringify({
|
|
629
|
+
type: EXTENSION_MESSAGE_TYPES.URL_VALIDATION_RESULT,
|
|
630
|
+
version: EXTENSION_PROTOCOL_VERSION,
|
|
631
|
+
id,
|
|
632
|
+
allowed,
|
|
633
|
+
}));
|
|
634
|
+
} catch {}
|
|
520
635
|
}
|
|
521
636
|
|
|
522
637
|
#closePending(error) {
|
|
638
|
+
this.closed = true;
|
|
639
|
+
if (!this.urlValidationController.signal.aborted) {
|
|
640
|
+
this.urlValidationController.abort(error);
|
|
641
|
+
}
|
|
523
642
|
this.registry.unregisterConnection(this);
|
|
524
643
|
if (this.heartbeatTimer) {
|
|
525
644
|
clearInterval(this.heartbeatTimer);
|
|
@@ -527,6 +646,7 @@ class ExtensionBrowserConnection {
|
|
|
527
646
|
}
|
|
528
647
|
for (const pending of this.pending.values()) {
|
|
529
648
|
clearTimeout(pending.timer);
|
|
649
|
+
pending.signal?.removeEventListener('abort', pending.onAbort);
|
|
530
650
|
pending.reject(error);
|
|
531
651
|
}
|
|
532
652
|
this.pending.clear();
|
|
@@ -559,6 +679,7 @@ class ExtensionBrowserConnection {
|
|
|
559
679
|
}
|
|
560
680
|
|
|
561
681
|
module.exports = {
|
|
682
|
+
ExtensionBrowserConnection,
|
|
562
683
|
BrowserExtensionRegistry,
|
|
563
684
|
sha256,
|
|
564
685
|
};
|