neoagent 2.1.18-beta.21 → 2.1.18-beta.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/docs/capabilities.md +3 -1
  2. package/docs/configuration.md +2 -0
  3. package/extensions/chrome-browser/background.mjs +209 -0
  4. package/extensions/chrome-browser/manifest.json +17 -0
  5. package/extensions/chrome-browser/popup.css +69 -0
  6. package/extensions/chrome-browser/popup.html +28 -0
  7. package/extensions/chrome-browser/popup.js +81 -0
  8. package/extensions/chrome-browser/protocol.mjs +309 -0
  9. package/package.json +4 -2
  10. package/server/config/origins.js +10 -0
  11. package/server/db/database.js +29 -0
  12. package/server/http/routes.js +1 -0
  13. package/server/index.js +2 -0
  14. package/server/public/flutter_bootstrap.js +1 -1
  15. package/server/public/main.dart.js +31247 -31180
  16. package/server/routes/browser_extension.js +133 -0
  17. package/server/services/ai/capabilityHealth.js +27 -0
  18. package/server/services/browser/extension/gateway.js +65 -0
  19. package/server/services/browser/extension/protocol.js +49 -0
  20. package/server/services/browser/extension/provider.js +178 -0
  21. package/server/services/browser/extension/registry.js +353 -0
  22. package/server/services/browser/extension/zip.js +133 -0
  23. package/server/services/manager.js +18 -0
  24. package/server/services/runtime/manager.js +7 -31
  25. package/server/services/runtime/settings.js +0 -4
package/server/routes/browser_extension.js ADDED
@@ -0,0 +1,133 @@
1
+ const express = require('express');
2
+ const path = require('path');
3
+ const { requireAuth } = require('../middleware/auth');
4
+ const { sanitizeError } = require('../utils/security');
5
+ const { createZipFromDirectory } = require('../services/browser/extension/zip');
6
+
7
+ const router = express.Router();
8
+ const EXTENSION_DIR = path.join(__dirname, '..', '..', 'extensions', 'chrome-browser');
9
+ const EXTENSION_MANIFEST = require('../../extensions/chrome-browser/manifest.json');
10
+
11
+ function getRegistry(req) {
12
+ const registry = req.app?.locals?.browserExtensionRegistry;
13
+ if (!registry) {
14
+ throw new Error('Browser extension registry is not available.');
15
+ }
16
+ return registry;
17
+ }
18
+
19
+ function baseUrlFor(req) {
20
+ const configured = process.env.NEOAGENT_PUBLIC_URL || process.env.PUBLIC_URL || '';
21
+ if (configured) return configured.replace(/\/+$/, '');
22
+ return `${req.protocol}://${req.get('host')}`;
23
+ }
24
+
25
+ function approvalUrlFor(req, pairingId) {
26
+ return `${baseUrlFor(req)}/api/browser-extension/pairing/${encodeURIComponent(pairingId)}/approve`;
27
+ }
28
+
29
+ router.post('/pairing/request', (req, res) => {
30
+ try {
31
+ const pairing = getRegistry(req).createPairingRequest({
32
+ extensionName: req.body?.extensionName,
33
+ userAgent: req.get('user-agent') || null,
34
+ });
35
+ res.json({
36
+ ...pairing,
37
+ approvalUrl: approvalUrlFor(req, pairing.pairingId),
38
+ serverUrl: baseUrlFor(req),
39
+ });
40
+ } catch (err) {
41
+ res.status(500).json({ error: sanitizeError(err) });
42
+ }
43
+ });
44
+
45
+ router.get('/latest', (req, res) => {
46
+ res.json({
47
+ name: EXTENSION_MANIFEST.name,
48
+ version: EXTENSION_MANIFEST.version,
49
+ minimumChromeVersion: EXTENSION_MANIFEST.minimum_chrome_version,
50
+ downloadUrl: `${baseUrlFor(req)}/api/browser-extension/download`,
51
+ });
52
+ });
53
+
54
+ router.get('/pairing/:pairingId/approve', requireAuth, (req, res) => {
55
+ try {
56
+ const row = getRegistry(req).getPairingRequest(req.params.pairingId);
57
+ if (!row) return res.status(404).send('Pairing request not found.');
58
+ const expired = Date.parse(row.expires_at) <= Date.now();
59
+ if (expired) return res.status(410).send('Pairing request expired. Start pairing again from the extension.');
60
+ if (row.status === 'claimed') return res.send('This browser extension is already paired.');
61
+ if (row.status === 'approved') return res.send('Extension approved. Return to the extension to finish pairing.');
62
+ if (row.status !== 'pending') {
63
+ return res.status(409).send('Pairing request is not pending.');
64
+ }
65
+ const action = `/api/browser-extension/pairing/${encodeURIComponent(req.params.pairingId)}/approve`;
66
+ res.type('html').send(`<!doctype html>
67
+ <html>
68
+ <head><meta charset="utf-8"><title>Pair NeoAgent Browser Extension</title></head>
69
+ <body style="font-family: system-ui, sans-serif; margin: 2rem; max-width: 42rem;">
70
+ <h1>Pair NeoAgent Browser Extension</h1>
71
+ <p>Approve this Chrome extension to control the connected browser for your NeoAgent account.</p>
72
+ <form method="post" action="${action}">
73
+ <button type="submit" style="font: inherit; padding: .7rem 1rem;">Approve Extension</button>
74
+ </form>
75
+ </body>
76
+ </html>`);
77
+ } catch (err) {
78
+ res.status(500).send(sanitizeError(err));
79
+ }
80
+ });
81
+
82
+ router.post('/pairing/:pairingId/approve', requireAuth, (req, res) => {
83
+ try {
84
+ const result = getRegistry(req).approvePairing(req.params.pairingId, req.session.userId);
85
+ if (String(req.get('accept') || '').includes('text/html')) {
86
+ return res.type('html').send('<!doctype html><p>Extension approved. Return to the extension to finish pairing.</p>');
87
+ }
88
+ res.json(result);
89
+ } catch (err) {
90
+ res.status(err.status || 500).json({ error: sanitizeError(err) });
91
+ }
92
+ });
93
+
94
+ router.post('/pairing/:pairingId/claim', (req, res) => {
95
+ try {
96
+ const result = getRegistry(req).claimPairing(req.params.pairingId, req.body?.pairingSecret, {
97
+ extensionName: req.body?.extensionName,
98
+ userAgent: req.get('user-agent') || null,
99
+ });
100
+ res.json(result);
101
+ } catch (err) {
102
+ res.status(err.status || 500).json({ error: sanitizeError(err) });
103
+ }
104
+ });
105
+
106
+ router.get('/status', requireAuth, (req, res) => {
107
+ try {
108
+ res.json(getRegistry(req).getStatus(req.session.userId));
109
+ } catch (err) {
110
+ res.status(500).json({ error: sanitizeError(err) });
111
+ }
112
+ });
113
+
114
+ router.post('/revoke', requireAuth, (req, res) => {
115
+ try {
116
+ res.json(getRegistry(req).revoke(req.session.userId, req.body?.tokenId || null));
117
+ } catch (err) {
118
+ res.status(500).json({ error: sanitizeError(err) });
119
+ }
120
+ });
121
+
122
+ router.get('/download', requireAuth, (req, res) => {
123
+ try {
124
+ const zip = createZipFromDirectory(EXTENSION_DIR);
125
+ res.setHeader('content-type', 'application/zip');
126
+ res.setHeader('content-disposition', 'attachment; filename="neoagent-chrome-browser-extension.zip"');
127
+ res.send(zip);
128
+ } catch (err) {
129
+ res.status(500).json({ error: sanitizeError(err) });
130
+ }
131
+ });
132
+
133
+ module.exports = router;
package/server/services/ai/capabilityHealth.js CHANGED
@@ -38,6 +38,33 @@ function summarizeCapabilityHealth(health) {
38
38
  async function getBrowserHealth(userId, app, engine) {
39
39
  const runtimeManager = app?.locals?.runtimeManager || engine?.runtimeManager || null;
40
40
  const executablePath = resolveBrowserExecutablePath();
41
+ const runtimeSettings = typeof runtimeManager?.getSettings === 'function'
42
+ ? runtimeManager.getSettings(userId)
43
+ : null;
44
+ if (runtimeSettings?.browser_backend === 'extension') {
45
+ const extensionStatus = app?.locals?.browserExtensionRegistry?.getStatus(userId);
46
+ const activeTokens = Array.isArray(extensionStatus?.tokens)
47
+ ? extensionStatus.tokens.filter((token) => token.status === 'active')
48
+ : [];
49
+ const connected = extensionStatus?.connected === true;
50
+ const configured = connected || activeTokens.length > 0;
51
+ return capabilityEntry({
52
+ connected,
53
+ configured,
54
+ healthy: connected,
55
+ degraded: configured && !connected,
56
+ summary: connected
57
+ ? 'Browser extension is connected.'
58
+ : configured
59
+ ? 'Browser extension is paired but not connected.'
60
+ : 'Browser extension backend is selected but no extension is paired.',
61
+ details: {
62
+ backend: 'extension',
63
+ activeTokenCount: activeTokens.length,
64
+ activeTokenId: extensionStatus?.activeTokenId || null,
65
+ },
66
+ });
67
+ }
41
68
  let controller = null;
42
69
  let resolutionError = null;
43
70
 
package/server/services/browser/extension/gateway.js ADDED
@@ -0,0 +1,65 @@
1
+ const { WebSocketServer } = require('ws');
2
+ const { BROWSER_EXTENSION_WS_PATH } = require('./protocol');
3
+
4
+ function rejectUpgrade(socket, statusCode, message) {
5
+ try {
6
+ socket.write(
7
+ `HTTP/1.1 ${statusCode} ${message}\r\n` +
8
+ 'Connection: close\r\n' +
9
+ '\r\n',
10
+ );
11
+ } catch {}
12
+ try { socket.destroy(); } catch {}
13
+ }
14
+
15
+ function bindBrowserExtensionGateway(httpServer, app) {
16
+ const wss = new WebSocketServer({ noServer: true });
17
+
18
+ httpServer.on('upgrade', (req, socket, head) => {
19
+ let url;
20
+ try {
21
+ url = new URL(req.url, 'http://localhost');
22
+ } catch {
23
+ return;
24
+ }
25
+ if (url.pathname !== BROWSER_EXTENSION_WS_PATH) {
26
+ return;
27
+ }
28
+
29
+ const registry = app?.locals?.browserExtensionRegistry;
30
+ if (!registry || typeof registry.validateToken !== 'function') {
31
+ rejectUpgrade(socket, 503, 'Service Unavailable');
32
+ return;
33
+ }
34
+
35
+ const token = url.searchParams.get('token') || req.headers['x-neoagent-extension-token'];
36
+ const tokenRow = registry.validateToken(token);
37
+ if (!tokenRow) {
38
+ rejectUpgrade(socket, 401, 'Unauthorized');
39
+ return;
40
+ }
41
+
42
+ wss.handleUpgrade(req, socket, head, (ws) => {
43
+ registry.registerConnection(tokenRow, ws, {
44
+ remoteAddress: req.socket?.remoteAddress || null,
45
+ userAgent: req.headers['user-agent'] || null,
46
+ });
47
+ ws.send(JSON.stringify({
48
+ type: 'hello',
49
+ ok: true,
50
+ userId: tokenRow.user_id,
51
+ tokenId: tokenRow.id,
52
+ }));
53
+ });
54
+ });
55
+
56
+ app.locals.browserExtensionGateway = {
57
+ close: () => new Promise((resolve) => wss.close(() => resolve())),
58
+ };
59
+
60
+ return wss;
61
+ }
62
+
63
+ module.exports = {
64
+ bindBrowserExtensionGateway,
65
+ };
package/server/services/browser/extension/protocol.js ADDED
@@ -0,0 +1,49 @@
1
+ const BROWSER_EXTENSION_WS_PATH = '/api/browser-extension/ws';
2
+
3
+ const EXTENSION_COMMANDS = Object.freeze({
4
+ LAUNCH: 'launch',
5
+ NAVIGATE: 'navigate',
6
+ CLICK: 'click',
7
+ CLICK_POINT: 'clickPoint',
8
+ TYPE: 'type',
9
+ TYPE_TEXT: 'typeText',
10
+ PRESS_KEY: 'pressKey',
11
+ SCROLL: 'scroll',
12
+ EXTRACT: 'extract',
13
+ EVALUATE: 'evaluate',
14
+ SCREENSHOT: 'screenshot',
15
+ CLOSE: 'close',
16
+ GET_PAGE_INFO: 'getPageInfo',
17
+ });
18
+
19
+ class ExtensionBrowserUnavailableError extends Error {
20
+ constructor(message = 'Extension browser not connected.') {
21
+ super(message);
22
+ this.name = 'ExtensionBrowserUnavailableError';
23
+ this.code = 'EXTENSION_BROWSER_NOT_CONNECTED';
24
+ }
25
+ }
26
+
27
+ function createCommandMessage(id, command, payload = {}) {
28
+ return {
29
+ type: 'command',
30
+ id,
31
+ command,
32
+ payload,
33
+ };
34
+ }
35
+
36
+ function parseExtensionMessage(data) {
37
+ const raw = Buffer.isBuffer(data) ? data.toString('utf8') : String(data || '');
38
+ if (!raw) return null;
39
+ const parsed = JSON.parse(raw);
40
+ return parsed && typeof parsed === 'object' ? parsed : null;
41
+ }
42
+
43
+ module.exports = {
44
+ BROWSER_EXTENSION_WS_PATH,
45
+ EXTENSION_COMMANDS,
46
+ ExtensionBrowserUnavailableError,
47
+ createCommandMessage,
48
+ parseExtensionMessage,
49
+ };
package/server/services/browser/extension/provider.js ADDED
@@ -0,0 +1,178 @@
1
+ const fs = require('fs');
2
+ const path = require('path');
3
+ const { DATA_DIR } = require('../../../../runtime/paths');
4
+ const { EXTENSION_COMMANDS, ExtensionBrowserUnavailableError } = require('./protocol');
5
+
6
+ const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
7
+ if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
8
+
9
+ function extractBase64Png(value) {
10
+ const text = String(value || '');
11
+ if (!text) return null;
12
+ const match = text.match(/^data:image\/png;base64,(.+)$/);
13
+ return match ? match[1] : text;
14
+ }
15
+
16
+ class ExtensionBrowserProvider {
17
+ constructor(options = {}) {
18
+ this.registry = options.registry;
19
+ this.userId = options.userId != null ? String(options.userId) : null;
20
+ this.artifactStore = options.artifactStore || null;
21
+ this.headless = false;
22
+ }
23
+
24
+ #assertReady() {
25
+ if (!this.registry || this.userId == null) {
26
+ throw new ExtensionBrowserUnavailableError();
27
+ }
28
+ }
29
+
30
+ async #dispatch(command, payload = {}, options = {}) {
31
+ this.#assertReady();
32
+ const result = await this.registry.dispatch(this.userId, command, payload, options);
33
+ return this.#materialize(result);
34
+ }
35
+
36
+ #disconnect() {
37
+ if (!this.registry || this.userId == null) return;
38
+ const connection = this.registry.getConnection(this.userId);
39
+ if (connection) {
40
+ connection.close('browser extension provider closed');
41
+ }
42
+ }
43
+
44
+ #writeScreenshotArtifact(base64) {
45
+ const buffer = Buffer.from(base64, 'base64');
46
+ if (this.artifactStore && this.userId != null) {
47
+ const artifact = this.artifactStore.allocateFile(this.userId, {
48
+ kind: 'browser-screenshot',
49
+ backend: 'extension',
50
+ extension: 'png',
51
+ contentType: 'image/png',
52
+ filenameBase: 'browser-extension-screenshot',
53
+ });
54
+ fs.writeFileSync(artifact.storagePath, buffer);
55
+ this.artifactStore.finalizeFile(artifact.artifactId, artifact.storagePath);
56
+ return {
57
+ screenshotPath: artifact.url,
58
+ artifactId: artifact.artifactId,
59
+ filename: path.basename(artifact.storagePath),
60
+ fullPath: artifact.storagePath,
61
+ };
62
+ }
63
+
64
+ const filename = `browser_extension_${Date.now()}_${Math.random().toString(16).slice(2)}.png`;
65
+ const fullPath = path.join(SCREENSHOTS_DIR, filename);
66
+ fs.writeFileSync(fullPath, buffer);
67
+ return {
68
+ screenshotPath: `/screenshots/${filename}`,
69
+ artifactId: null,
70
+ filename,
71
+ fullPath,
72
+ };
73
+ }
74
+
75
+ #materialize(result) {
76
+ if (!result || typeof result !== 'object') return result;
77
+ const raw = result.screenshotDataUrl || result.screenshotData || result.screenshotBase64;
78
+ if (!raw) return result;
79
+ const base64 = extractBase64Png(raw);
80
+ if (!base64) return result;
81
+ const screenshot = this.#writeScreenshotArtifact(base64);
82
+ const next = { ...result, ...screenshot };
83
+ delete next.screenshotDataUrl;
84
+ delete next.screenshotData;
85
+ delete next.screenshotBase64;
86
+ return next;
87
+ }
88
+
89
+ navigate(url, options = {}) {
90
+ return this.#dispatch(EXTENSION_COMMANDS.NAVIGATE, { url, ...options });
91
+ }
92
+
93
+ click(selector, text, screenshot = true) {
94
+ return this.#dispatch(EXTENSION_COMMANDS.CLICK, { selector, text, screenshot });
95
+ }
96
+
97
+ clickPoint(x, y, screenshot = true) {
98
+ return this.#dispatch(EXTENSION_COMMANDS.CLICK_POINT, { x, y, screenshot });
99
+ }
100
+
101
+ type(selector, text, options = {}) {
102
+ return this.#dispatch(EXTENSION_COMMANDS.TYPE, { selector, text, ...options });
103
+ }
104
+
105
+ typeText(text, options = {}) {
106
+ return this.#dispatch(EXTENSION_COMMANDS.TYPE_TEXT, { text, ...options });
107
+ }
108
+
109
+ pressKey(key, screenshot = true) {
110
+ return this.#dispatch(EXTENSION_COMMANDS.PRESS_KEY, { key, screenshot });
111
+ }
112
+
113
+ scroll(deltaX = 0, deltaY = 0, screenshot = true) {
114
+ return this.#dispatch(EXTENSION_COMMANDS.SCROLL, { deltaX, deltaY, screenshot });
115
+ }
116
+
117
+ extract(selector, attribute, all = false) {
118
+ return this.#dispatch(EXTENSION_COMMANDS.EXTRACT, { selector, attribute, all });
119
+ }
120
+
121
+ evaluate(script) {
122
+ return this.#dispatch(EXTENSION_COMMANDS.EVALUATE, { script });
123
+ }
124
+
125
+ screenshot(options = {}) {
126
+ return this.#dispatch(EXTENSION_COMMANDS.SCREENSHOT, options);
127
+ }
128
+
129
+ launch(options = {}) {
130
+ return this.#dispatch(EXTENSION_COMMANDS.LAUNCH, options);
131
+ }
132
+
133
+ async closeBrowser() {
134
+ if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId)) {
135
+ return { success: true, extensionConnected: false };
136
+ }
137
+ const result = await this.#dispatch(EXTENSION_COMMANDS.CLOSE, {});
138
+ this.#disconnect();
139
+ return { ...result, success: result?.success !== false, extensionConnected: false };
140
+ }
141
+
142
+ fill(selector, value) {
143
+ return this.type(selector, String(value));
144
+ }
145
+
146
+ extractContent(options = {}) {
147
+ return this.extract(options.selector, options.attribute, options.all);
148
+ }
149
+
150
+ executeJS(code) {
151
+ return this.evaluate(code);
152
+ }
153
+
154
+ async getPageInfo() {
155
+ if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId)) {
156
+ return { url: null, title: null, extensionConnected: false };
157
+ }
158
+ return this.registry.dispatch(this.userId, EXTENSION_COMMANDS.GET_PAGE_INFO, {});
159
+ }
160
+
161
+ isLaunched() {
162
+ return Boolean(this.registry && this.userId != null && this.registry.isConnected(this.userId));
163
+ }
164
+
165
+ getPageCount() {
166
+ return this.isLaunched() ? 1 : 0;
167
+ }
168
+
169
+ setHeadless() {
170
+ this.headless = false;
171
+ return Promise.resolve({ success: false, unsupported: true });
172
+ }
173
+ }
174
+
175
+ module.exports = {
176
+ ExtensionBrowserProvider,
177
+ extractBase64Png,
178
+ };