amalgm 0.1.91 → 0.1.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/browser/attach.js +238 -0
- package/runtime/scripts/amalgm-mcp/browser/auth-tools.js +26 -40
- package/runtime/scripts/amalgm-mcp/browser/backend.js +100 -0
- package/runtime/scripts/amalgm-mcp/browser/capture.js +167 -0
- package/runtime/scripts/amalgm-mcp/browser/cdp.js +177 -0
- package/runtime/scripts/amalgm-mcp/browser/engine.js +34 -349
- package/runtime/scripts/amalgm-mcp/browser/index.js +28 -0
- package/runtime/scripts/amalgm-mcp/browser/recorder-tools.js +4 -4
- package/runtime/scripts/amalgm-mcp/browser/recorder.js +60 -72
- package/runtime/scripts/amalgm-mcp/browser/sessions.js +64 -0
- package/runtime/scripts/amalgm-mcp/browser/tools.js +35 -31
- package/runtime/scripts/amalgm-mcp/server/core-tools.js +4 -14
- package/runtime/scripts/amalgm-mcp/server/mcp.js +24 -1
- package/runtime/scripts/amalgm-mcp/tests/browser-capture.test.js +87 -0
- package/runtime/scripts/amalgm-mcp/tests/core-tools.test.js +59 -29
- package/runtime/scripts/amalgm-mcp/tests/platform-context.test.js +46 -0
- package/runtime/scripts/amalgm-mcp/tests/system-catalog.test.js +4 -4
- package/runtime/scripts/amalgm-mcp/tests/toolbox-service.test.js +2 -2
- package/runtime/scripts/amalgm-mcp/toolbox/loadout-context.js +33 -10
- package/runtime/scripts/chat-core/tool-shape.js +27 -10
- package/runtime/scripts/platform-context.txt +21 -20
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Minimal raw-CDP utilities — the only websocket/devtools plumbing in the
|
|
5
|
+
* browser module. attach.js uses it to verify the visible surface; capture.js
|
|
6
|
+
* uses it for pixels. No dependencies on other browser modules.
|
|
7
|
+
*
|
|
8
|
+
* Every exchange carries a hard deadline: a probe may fail, it can never
|
|
9
|
+
* wedge a caller.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const CONNECT_TIMEOUT_MS = 10_000;
|
|
13
|
+
const CALL_TIMEOUT_MS = 15_000;
|
|
14
|
+
|
|
15
|
+
function deadline(promise, ms, label) {
|
|
16
|
+
let timer;
|
|
17
|
+
return Promise.race([
|
|
18
|
+
promise,
|
|
19
|
+
new Promise((_, reject) => {
|
|
20
|
+
timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
21
|
+
}),
|
|
22
|
+
]).finally(() => clearTimeout(timer));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Minimal CDP client over one websocket: call/response with deadlines,
|
|
27
|
+
* fire-and-forget send (frame acks), event listeners.
|
|
28
|
+
*/
|
|
29
|
+
async function connectCdp(wsUrl) {
|
|
30
|
+
const WebSocket = require('ws');
|
|
31
|
+
const ws = new WebSocket(wsUrl, { perMessageDeflate: false });
|
|
32
|
+
try {
|
|
33
|
+
await deadline(new Promise((resolve, reject) => {
|
|
34
|
+
ws.once('open', resolve);
|
|
35
|
+
ws.once('error', reject);
|
|
36
|
+
}), CONNECT_TIMEOUT_MS, 'CDP connect');
|
|
37
|
+
} catch (err) {
|
|
38
|
+
// Free the half-open socket — a timed-out connect must not leak a
|
|
39
|
+
// connection that keeps the process (or a test run) alive.
|
|
40
|
+
try { ws.terminate(); } catch {}
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
let nextId = 0;
|
|
45
|
+
const pending = new Map();
|
|
46
|
+
const listeners = new Map();
|
|
47
|
+
|
|
48
|
+
ws.on('message', (raw) => {
|
|
49
|
+
let msg;
|
|
50
|
+
try { msg = JSON.parse(raw); } catch { return; }
|
|
51
|
+
if (msg.id && pending.has(msg.id)) {
|
|
52
|
+
const entry = pending.get(msg.id);
|
|
53
|
+
pending.delete(msg.id);
|
|
54
|
+
if (msg.error) entry.reject(new Error(msg.error.message || 'CDP error'));
|
|
55
|
+
else entry.resolve(msg.result || {});
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
if (msg.method && listeners.has(msg.method)) listeners.get(msg.method)(msg.params || {});
|
|
59
|
+
});
|
|
60
|
+
ws.on('close', () => {
|
|
61
|
+
for (const entry of pending.values()) entry.reject(new Error('CDP connection closed'));
|
|
62
|
+
pending.clear();
|
|
63
|
+
});
|
|
64
|
+
ws.on('error', () => {});
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
call(method, params = {}, timeoutMs = CALL_TIMEOUT_MS) {
|
|
68
|
+
const id = ++nextId;
|
|
69
|
+
const result = new Promise((resolve, reject) => pending.set(id, { resolve, reject }));
|
|
70
|
+
try { ws.send(JSON.stringify({ id, method, params })); } catch (err) {
|
|
71
|
+
pending.delete(id);
|
|
72
|
+
return Promise.reject(err);
|
|
73
|
+
}
|
|
74
|
+
return deadline(result, timeoutMs, method).catch((err) => {
|
|
75
|
+
pending.delete(id);
|
|
76
|
+
throw err;
|
|
77
|
+
});
|
|
78
|
+
},
|
|
79
|
+
send(method, params = {}) {
|
|
80
|
+
try { ws.send(JSON.stringify({ id: ++nextId, method, params })); } catch {}
|
|
81
|
+
},
|
|
82
|
+
on(method, handler) { listeners.set(method, handler); },
|
|
83
|
+
close() { try { ws.close(); } catch {} },
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** GET /json/list on the debugger port — accepts a ws/http url or a bare port. */
|
|
88
|
+
function targetListUrl(endpoint) {
|
|
89
|
+
const raw = String(endpoint || '').trim();
|
|
90
|
+
if (/^\d+$/.test(raw)) return `http://127.0.0.1:${raw}/json/list`;
|
|
91
|
+
const url = new URL(raw.replace(/^ws/, 'http'));
|
|
92
|
+
return `http://${url.hostname}:${url.port || 80}/json/list`;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function listTargets(endpoint) {
|
|
96
|
+
const http = require('http');
|
|
97
|
+
return deadline(new Promise((resolve, reject) => {
|
|
98
|
+
const req = http.get(targetListUrl(endpoint), (res) => {
|
|
99
|
+
let raw = '';
|
|
100
|
+
res.on('data', (chunk) => { raw += chunk.toString(); });
|
|
101
|
+
res.on('end', () => {
|
|
102
|
+
try { resolve(JSON.parse(raw)); } catch (err) { reject(err); }
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
req.on('error', reject);
|
|
106
|
+
}), 5_000, 'CDP target list');
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function pageTargets(targets) {
|
|
110
|
+
return (targets || []).filter((t) => t.type === 'page'
|
|
111
|
+
&& t.webSocketDebuggerUrl
|
|
112
|
+
&& !String(t.url || '').startsWith('devtools://'));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Locate the session's <webview> element inside a candidate app window:
|
|
117
|
+
* marker in the src attribute, exact current URL (the UI keeps src synced to
|
|
118
|
+
* the visible URL), then "the only webview". Returns layout geometry only
|
|
119
|
+
* when the element is actually visible — a hidden splitview reports 0x0 and
|
|
120
|
+
* resolves to null.
|
|
121
|
+
*/
|
|
122
|
+
function rectScript(markerText, currentUrl) {
|
|
123
|
+
const urlLiteral = JSON.stringify(String(currentUrl || ''));
|
|
124
|
+
return `(() => {
|
|
125
|
+
const views = [...document.querySelectorAll('webview')];
|
|
126
|
+
const byMarker = views.find((w) => (w.getAttribute('src') || '').includes(${JSON.stringify(markerText)}));
|
|
127
|
+
const byUrl = ${urlLiteral} ? views.find((w) => (w.src || '') === ${urlLiteral}) : null;
|
|
128
|
+
const match = byMarker || byUrl || (views.length === 1 ? views[0] : null);
|
|
129
|
+
if (!match) return null;
|
|
130
|
+
const r = match.getBoundingClientRect();
|
|
131
|
+
if (!r.width || !r.height) return null;
|
|
132
|
+
return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, vw: window.innerWidth, vh: window.innerHeight, dpr: window.devicePixelRatio || 1 });
|
|
133
|
+
})()`;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async function embedderRect(cdp, markerText, currentUrl) {
|
|
137
|
+
const { result } = await cdp.call('Runtime.evaluate', {
|
|
138
|
+
expression: rectScript(markerText, currentUrl),
|
|
139
|
+
returnByValue: true,
|
|
140
|
+
});
|
|
141
|
+
const raw = result && result.value;
|
|
142
|
+
if (!raw || typeof raw !== 'string') return null;
|
|
143
|
+
try { return JSON.parse(raw); } catch { return null; }
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Find the app window hosting the session's webview and the webview's
|
|
148
|
+
* on-screen rectangle. Null when the surface is hidden (or no webview
|
|
149
|
+
* matches) — callers decide whether to reopen or fail.
|
|
150
|
+
*/
|
|
151
|
+
async function findSurfaceRect(endpoint, markerText, currentUrl) {
|
|
152
|
+
const targets = await listTargets(endpoint);
|
|
153
|
+
for (const page of pageTargets(targets)) {
|
|
154
|
+
const cdp = await connectCdp(page.webSocketDebuggerUrl).catch(() => null);
|
|
155
|
+
if (!cdp) continue;
|
|
156
|
+
try {
|
|
157
|
+
const rect = await embedderRect(cdp, markerText, currentUrl);
|
|
158
|
+
if (rect) return { pageWsUrl: page.webSocketDebuggerUrl, rect };
|
|
159
|
+
} catch {
|
|
160
|
+
// Try the next window.
|
|
161
|
+
} finally {
|
|
162
|
+
cdp.close();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
module.exports = {
|
|
169
|
+
connectCdp,
|
|
170
|
+
deadline,
|
|
171
|
+
embedderRect,
|
|
172
|
+
findSurfaceRect,
|
|
173
|
+
listTargets,
|
|
174
|
+
pageTargets,
|
|
175
|
+
rectScript,
|
|
176
|
+
targetListUrl,
|
|
177
|
+
};
|
|
@@ -1,316 +1,34 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Browser
|
|
5
|
-
* deterministic backend routing by entrypoint.
|
|
4
|
+
* Browser verbs — the behavior behind every browser action.
|
|
6
5
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* in-tab split view, and selects that webview target — so commands drive
|
|
14
|
-
* the surface the user is watching. An explicit AMALGM_BROWSER_CDP_URL
|
|
15
|
-
* attaches to any other CDP endpoint the same way.
|
|
16
|
-
* 2. "cli" — standalone agent-browser chromium, headless. The deterministic
|
|
17
|
-
* path for npm-only machines, servers, and the web entrypoint.
|
|
6
|
+
* Built on small single-concern modules:
|
|
7
|
+
* backend.js — which browser: visible desktop surface vs headless cli
|
|
8
|
+
* attach.js — the visible-surface handshake + command transport
|
|
9
|
+
* capture.js — pixels: screenshots, and the recorder's frame source
|
|
10
|
+
* sessions.js — session identity, 1:1 with durable profiles
|
|
11
|
+
* cli.js — the agent-browser process wrapper
|
|
18
12
|
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* Sessions map to durable profiles; the default session for a tool call is
|
|
23
|
-
* the caller's chat session, so one conversation keeps one browser.
|
|
13
|
+
* This module stays the single import for tool handlers and tests; the
|
|
14
|
+
* modules above are its implementation. Visibility is an environment fact,
|
|
15
|
+
* never an argument — no verb takes a mode.
|
|
24
16
|
*/
|
|
25
17
|
|
|
26
18
|
const cli = require('./cli');
|
|
19
|
+
const backend = require('./backend');
|
|
20
|
+
const attach = require('./attach');
|
|
21
|
+
const capture = require('./capture');
|
|
22
|
+
const sessions = require('./sessions');
|
|
27
23
|
|
|
28
|
-
const
|
|
29
|
-
const
|
|
30
|
-
|
|
31
|
-
const SURFACE_EVENT_RESOURCE = 'browser_surfaces';
|
|
32
|
-
const WEBVIEW_MARKER_PREFIX = 'amalgm-bsid-';
|
|
33
|
-
const WEBVIEW_WAIT_MS = 15_000;
|
|
34
|
-
const WEBVIEW_POLL_MS = 500;
|
|
35
|
-
|
|
36
|
-
// ---------------------------------------------------------------------------
|
|
37
|
-
// Backend routing
|
|
38
|
-
// ---------------------------------------------------------------------------
|
|
39
|
-
|
|
40
|
-
function pidIsRunning(pid) {
|
|
41
|
-
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
42
|
-
try {
|
|
43
|
-
process.kill(pid, 0);
|
|
44
|
-
return true;
|
|
45
|
-
} catch (err) {
|
|
46
|
-
return Boolean(err && err.code === 'EPERM');
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Bridge advertisement candidates, most specific first. Every runtime
|
|
52
|
-
* generation can find the file written by the desktop app:
|
|
53
|
-
* - Electron-spawned and LaunchAgent runtimes have AMALGM_RUNTIME_STATE_DIR;
|
|
54
|
-
* - runtimes with only AMALGM_DIR (+ label) derive the same path;
|
|
55
|
-
* - bare dev runtimes fall back to the unscoped roots.
|
|
56
|
-
*/
|
|
57
|
-
function bridgeFileCandidates() {
|
|
58
|
-
const path = require('path');
|
|
59
|
-
const os = require('os');
|
|
60
|
-
const dirs = [];
|
|
61
|
-
if (process.env.AMALGM_RUNTIME_STATE_DIR) dirs.push(process.env.AMALGM_RUNTIME_STATE_DIR);
|
|
62
|
-
const amalgmDir = process.env.AMALGM_DIR;
|
|
63
|
-
const label = process.env.AMALGM_RUNTIME_LABEL || process.env.AMALGM_BRANCH;
|
|
64
|
-
if (amalgmDir && label) dirs.push(path.join(amalgmDir, 'runtimes', label));
|
|
65
|
-
if (amalgmDir) dirs.push(amalgmDir);
|
|
66
|
-
dirs.push(path.join(os.homedir(), '.amalgm'));
|
|
67
|
-
return [...new Set(dirs)].map((dir) => path.join(dir, 'electron-browser-bridge.json'));
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
function readBridgeAdvertisement() {
|
|
71
|
-
const fs = require('fs');
|
|
72
|
-
for (const file of bridgeFileCandidates()) {
|
|
73
|
-
try {
|
|
74
|
-
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
75
|
-
if (typeof data?.cdpUrl !== 'string' || !data.cdpUrl) continue;
|
|
76
|
-
if (typeof data?.pid === 'number' && !pidIsRunning(data.pid)) continue;
|
|
77
|
-
return { cdpUrl: data.cdpUrl, cdpPort: data.cdpPort, label: data.label, file };
|
|
78
|
-
} catch {}
|
|
79
|
-
}
|
|
80
|
-
return null;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/** Reduce a CDP url/port advertisement to the value agent-browser's --cdp flag takes. */
|
|
84
|
-
function cdpFlagValue(input) {
|
|
85
|
-
const raw = String(input || '').trim();
|
|
86
|
-
if (!raw) return null;
|
|
87
|
-
if (/^\d+$/.test(raw)) return raw;
|
|
88
|
-
try {
|
|
89
|
-
const url = new URL(raw);
|
|
90
|
-
if (url.port) return url.port;
|
|
91
|
-
} catch {}
|
|
92
|
-
return raw;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* The CDP endpoint for attached mode, as a --cdp flag value. agent-browser's
|
|
97
|
-
* `connect` command launches a managed browser; raw --cdp attach is the mode
|
|
98
|
-
* that discovers Electron <webview> targets, so that is all we use.
|
|
99
|
-
*/
|
|
100
|
-
function cdpEndpoint() {
|
|
101
|
-
if (process.env.AMALGM_BROWSER_CDP_URL) return cdpFlagValue(process.env.AMALGM_BROWSER_CDP_URL);
|
|
102
|
-
const advertisement = readBridgeAdvertisement();
|
|
103
|
-
if (!advertisement) return null;
|
|
104
|
-
if (advertisement.cdpPort) return String(advertisement.cdpPort);
|
|
105
|
-
return cdpFlagValue(advertisement.cdpUrl);
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
function mode() {
|
|
109
|
-
const requested = (process.env.AMALGM_BROWSER_BACKEND || '').toLowerCase();
|
|
110
|
-
if (['agent-browser', 'external', 'cli', 'headless'].includes(requested)) return 'cli';
|
|
111
|
-
if (['attached', 'cdp', 'electron'].includes(requested)) return 'attached';
|
|
112
|
-
return cdpEndpoint() ? 'attached' : 'cli';
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/** Options every cli call needs for the current backend. */
|
|
116
|
-
function cliOptions(session, extra = {}) {
|
|
117
|
-
const options = { session, ...extra };
|
|
118
|
-
if (mode() === 'attached') options.cdp = cdpEndpoint();
|
|
119
|
-
return options;
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
function sessionFor(args = {}, context = {}) {
|
|
123
|
-
return cli.safeId(
|
|
124
|
-
args.session
|
|
125
|
-
|| args.profile
|
|
126
|
-
|| args.browserProfileId
|
|
127
|
-
|| args.browserSessionId
|
|
128
|
-
|| args.sessionId
|
|
129
|
-
|| args.tabId
|
|
130
|
-
|| context?.callerSessionId
|
|
131
|
-
|| cli.DEFAULT_SESSION,
|
|
132
|
-
);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
function rememberProfile(session) {
|
|
136
|
-
try {
|
|
137
|
-
require('./profiles').touchBrowserProfile({
|
|
138
|
-
id: session,
|
|
139
|
-
name: session,
|
|
140
|
-
source: 'agent-browser',
|
|
141
|
-
profilePath: cli.profileDir(session),
|
|
142
|
-
}, { source: 'agent-browser' });
|
|
143
|
-
} catch {}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
function sessionInfo(session, extra = {}) {
|
|
147
|
-
rememberProfile(session);
|
|
148
|
-
const info = {
|
|
149
|
-
session,
|
|
150
|
-
browserSessionId: session,
|
|
151
|
-
browserProfileId: session,
|
|
152
|
-
backend: mode(),
|
|
153
|
-
...extra,
|
|
154
|
-
};
|
|
155
|
-
knownSessions.set(session, { ...knownSessions.get(session), ...info, lastActiveAt: Date.now() });
|
|
156
|
-
return info;
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// ---------------------------------------------------------------------------
|
|
160
|
-
// Attached mode: surface open + webview target selection
|
|
161
|
-
// ---------------------------------------------------------------------------
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* Ask the UI to open the in-tab browser surface for this session. Published
|
|
165
|
-
* on the runtime state-event bus; ChatView subscribes to browser_surfaces
|
|
166
|
-
* and mounts the split-view webview with a marker URL the target scan below
|
|
167
|
-
* can recognize.
|
|
168
|
-
*/
|
|
169
|
-
function requestSurfaceOpen(session, context) {
|
|
170
|
-
try {
|
|
171
|
-
const { appendStateEvent } = require('../state/events');
|
|
172
|
-
appendStateEvent({
|
|
173
|
-
resource: SURFACE_EVENT_RESOURCE,
|
|
174
|
-
op: 'insert',
|
|
175
|
-
id: session,
|
|
176
|
-
value: {
|
|
177
|
-
browserSessionId: session,
|
|
178
|
-
callerSessionId: context?.callerSessionId || null,
|
|
179
|
-
marker: `${WEBVIEW_MARKER_PREFIX}${session}`,
|
|
180
|
-
requestedAt: new Date().toISOString(),
|
|
181
|
-
},
|
|
182
|
-
source: 'browser-engine',
|
|
183
|
-
});
|
|
184
|
-
return true;
|
|
185
|
-
} catch {
|
|
186
|
-
return false;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
async function listWebviewTabs(session) {
|
|
191
|
-
const tabs = await cli.runCli(['tab'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
192
|
-
const list = tabs?.data?.tabs || tabs?.tabs || [];
|
|
193
|
-
return list.filter((tab) => tab.type === 'webview');
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
function matchWebview(webviews, session) {
|
|
197
|
-
const marker = `${WEBVIEW_MARKER_PREFIX}${session}`;
|
|
198
|
-
const byMarker = webviews.find((tab) => String(tab.url || '').includes(marker)
|
|
199
|
-
|| String(tab.title || '').includes(marker));
|
|
200
|
-
if (byMarker) return byMarker;
|
|
201
|
-
const previous = knownSessions.get(session)?.webviewTabId;
|
|
202
|
-
if (previous) {
|
|
203
|
-
const byPrevious = webviews.find((tab) => tab.tabId === previous);
|
|
204
|
-
if (byPrevious) return byPrevious;
|
|
205
|
-
}
|
|
206
|
-
if (webviews.length === 1) return webviews[0];
|
|
207
|
-
return null;
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
/**
|
|
211
|
-
* In attached mode, connect the session to the advertised chromium on first
|
|
212
|
-
* use and select the in-tab webview target so commands drive the surface the
|
|
213
|
-
* user is looking at. Fails closed: if no webview target can be found we
|
|
214
|
-
* error with instructions instead of silently driving the outer app window.
|
|
215
|
-
*/
|
|
216
|
-
async function ensureReady(session, context) {
|
|
217
|
-
if (mode() !== 'attached' || attachedSessions.has(session)) return;
|
|
218
|
-
|
|
219
|
-
let webviews = await listWebviewTabs(session);
|
|
220
|
-
let selected = matchWebview(webviews, session);
|
|
221
|
-
|
|
222
|
-
if (!selected) {
|
|
223
|
-
requestSurfaceOpen(session, context);
|
|
224
|
-
const deadline = Date.now() + WEBVIEW_WAIT_MS;
|
|
225
|
-
while (!selected && Date.now() < deadline) {
|
|
226
|
-
await new Promise((resolve) => setTimeout(resolve, WEBVIEW_POLL_MS));
|
|
227
|
-
webviews = await listWebviewTabs(session);
|
|
228
|
-
selected = matchWebview(webviews, session);
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
if (!selected && webviews.length === 0 && process.env.AMALGM_BROWSER_CDP_URL) {
|
|
233
|
-
// Explicit external CDP endpoints (plain chromium) have page targets,
|
|
234
|
-
// not webviews — drive whatever the daemon selected by default.
|
|
235
|
-
attachedSessions.add(session);
|
|
236
|
-
return;
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
if (!selected) {
|
|
240
|
-
throw new Error(
|
|
241
|
-
'Browser surface did not open in the desktop app. '
|
|
242
|
-
+ 'Retry, or force a headless browser with AMALGM_BROWSER_BACKEND=cli.',
|
|
243
|
-
);
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
247
|
-
await focusWebviewHost(session, selected);
|
|
248
|
-
knownSessions.set(session, { ...knownSessions.get(session), webviewTabId: selected.tabId });
|
|
249
|
-
attachedSessions.add(session);
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
/**
|
|
253
|
-
* Chromium drops CDP Input events to an Electron <webview> guest until the
|
|
254
|
-
* guest widget has focus (verified live: clicks report success but never
|
|
255
|
-
* land). One focus pass after target selection fixes input for the session:
|
|
256
|
-
* focus the <webview> element from the embedder page, then self-focus the
|
|
257
|
-
* guest. Best-effort — failures leave snapshot/eval working regardless.
|
|
258
|
-
*/
|
|
259
|
-
async function focusWebviewHost(session, selected) {
|
|
260
|
-
const marker = `${WEBVIEW_MARKER_PREFIX}${session}`;
|
|
261
|
-
try {
|
|
262
|
-
const tabs = await cli.runCli(['tab'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
263
|
-
const list = tabs?.data?.tabs || tabs?.tabs || [];
|
|
264
|
-
const outer = list.find((tab) => tab.type === 'page');
|
|
265
|
-
if (outer) {
|
|
266
|
-
const script = `(() => {
|
|
267
|
-
const views = [...document.querySelectorAll('webview')];
|
|
268
|
-
const target = views.find((w) => (w.getAttribute('src') || '').includes(${JSON.stringify(marker)}))
|
|
269
|
-
|| views.find((w) => (w.src || '') === ${JSON.stringify(String(selected.url || ''))})
|
|
270
|
-
|| views[views.length - 1];
|
|
271
|
-
if (target) target.focus();
|
|
272
|
-
return views.length;
|
|
273
|
-
})()`;
|
|
274
|
-
await cli.runCli(['tab', outer.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
275
|
-
await cli.runCli(['eval', script], cliOptions(session, { timeoutMs: 15_000 }));
|
|
276
|
-
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
277
|
-
}
|
|
278
|
-
await cli.runCli(['eval', 'window.focus()'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
279
|
-
} catch {
|
|
280
|
-
// Snapshot/eval still work without focus; input may need a retry.
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
function looksLikeLostConnection(err) {
|
|
285
|
-
return /clos|connect|target|socket|detach/i.test(String(err?.message || err || ''));
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
async function run(session, commandArgs, options = {}) {
|
|
289
|
-
const { context, ...cliExtra } = options;
|
|
290
|
-
await ensureReady(session, context);
|
|
291
|
-
try {
|
|
292
|
-
return await cli.runCli(commandArgs, cliOptions(session, cliExtra));
|
|
293
|
-
} catch (err) {
|
|
294
|
-
if (mode() === 'attached' && attachedSessions.has(session) && looksLikeLostConnection(err)) {
|
|
295
|
-
// The app restarted or the webview remounted; re-attach once.
|
|
296
|
-
attachedSessions.delete(session);
|
|
297
|
-
await ensureReady(session, context);
|
|
298
|
-
return cli.runCli(commandArgs, cliOptions(session, cliExtra));
|
|
299
|
-
}
|
|
300
|
-
throw err;
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
async function readText(session, commandArgs, timeoutMs) {
|
|
305
|
-
await ensureReady(session);
|
|
306
|
-
return cli.readText(commandArgs, session, timeoutMs, mode() === 'attached' ? cdpEndpoint() : null);
|
|
307
|
-
}
|
|
24
|
+
const { sessionFor, sessionInfo } = sessions;
|
|
25
|
+
const { run, readText } = attach;
|
|
308
26
|
|
|
309
27
|
/**
|
|
310
28
|
* Target grammar shared by click/fill:
|
|
311
|
-
* "@e12"
|
|
29
|
+
* "@e12" — ref from snapshot (preferred)
|
|
312
30
|
* "text=Sign in" — find by visible text
|
|
313
|
-
* anything else
|
|
31
|
+
* anything else — CSS selector
|
|
314
32
|
*/
|
|
315
33
|
function findArgs(target, action, value) {
|
|
316
34
|
if (target.startsWith('text=')) {
|
|
@@ -318,15 +36,11 @@ function findArgs(target, action, value) {
|
|
|
318
36
|
if (value !== undefined) command.push(value);
|
|
319
37
|
return command;
|
|
320
38
|
}
|
|
321
|
-
const command = [action
|
|
39
|
+
const command = [action, target];
|
|
322
40
|
if (value !== undefined) command.push(value);
|
|
323
41
|
return command;
|
|
324
42
|
}
|
|
325
43
|
|
|
326
|
-
// ---------------------------------------------------------------------------
|
|
327
|
-
// Core verbs
|
|
328
|
-
// ---------------------------------------------------------------------------
|
|
329
|
-
|
|
330
44
|
async function state(args = {}, context) {
|
|
331
45
|
const session = sessionFor(args, context);
|
|
332
46
|
let url = '';
|
|
@@ -351,19 +65,8 @@ async function snapshot(args = {}, context) {
|
|
|
351
65
|
|
|
352
66
|
async function screenshot(args = {}, context) {
|
|
353
67
|
const session = sessionFor(args, context);
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
const crypto = require('crypto');
|
|
357
|
-
const file = path.join(cli.screenshotDir(), `${session}-${Date.now()}-${crypto.randomUUID()}.png`);
|
|
358
|
-
await run(session, ['screenshot', ...(args.fullPage ? ['--full'] : []), file], { context });
|
|
359
|
-
const buffer = fs.readFileSync(file);
|
|
360
|
-
return {
|
|
361
|
-
...sessionInfo(session),
|
|
362
|
-
base64: buffer.toString('base64'),
|
|
363
|
-
bytes: buffer.length,
|
|
364
|
-
mode: args.fullPage ? 'full page' : 'viewport',
|
|
365
|
-
path: file,
|
|
366
|
-
};
|
|
68
|
+
const shot = await capture.screenshot(session, args, context);
|
|
69
|
+
return { ...sessionInfo(session), ...shot };
|
|
367
70
|
}
|
|
368
71
|
|
|
369
72
|
async function click(args = {}, context) {
|
|
@@ -425,18 +128,6 @@ async function wait(args = {}, context) {
|
|
|
425
128
|
throw new Error('wait requires one of: selector, url, ms');
|
|
426
129
|
}
|
|
427
130
|
|
|
428
|
-
async function tabs(args = {}, context) {
|
|
429
|
-
const session = sessionFor(args, context);
|
|
430
|
-
if (args.action === 'select' && args.tab) {
|
|
431
|
-
return run(session, ['tab', args.tab], { context });
|
|
432
|
-
}
|
|
433
|
-
if (args.action === 'new') {
|
|
434
|
-
return run(session, ['tab', 'new', ...(args.url ? [args.url] : [])], { context });
|
|
435
|
-
}
|
|
436
|
-
const result = await run(session, ['tab'], { context });
|
|
437
|
-
return { ...sessionInfo(session), tabs: result?.data?.tabs || result?.tabs || [] };
|
|
438
|
-
}
|
|
439
|
-
|
|
440
131
|
/**
|
|
441
132
|
* Full agent-browser CLI passthrough — the escape hatch that keeps the tool
|
|
442
133
|
* surface small while staying turing complete for browsers. Network mocking,
|
|
@@ -557,42 +248,36 @@ async function close(args = {}, context) {
|
|
|
557
248
|
const session = sessionFor(args, context);
|
|
558
249
|
// Attached mode drives the user's visible surface — never close it from
|
|
559
250
|
// the tool; just forget the session bookkeeping.
|
|
560
|
-
if (mode() !== 'attached') {
|
|
251
|
+
if (backend.mode() !== 'attached') {
|
|
561
252
|
await cli.runCli(['close'], { session });
|
|
562
253
|
}
|
|
563
|
-
attachedSessions.delete(session);
|
|
564
|
-
knownSessions.delete(session);
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
/** Resolve the CDP websocket URL for a session — recorder needs it. */
|
|
568
|
-
async function cdpUrlForSession(session) {
|
|
569
|
-
await ensureReady(session);
|
|
570
|
-
const result = await cli.runCli(['get', 'cdp-url'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
571
|
-
return result?.data?.cdpUrl || result?.cdpUrl || null;
|
|
254
|
+
sessions.attachedSessions.delete(session);
|
|
255
|
+
sessions.knownSessions.delete(session);
|
|
572
256
|
}
|
|
573
257
|
|
|
574
258
|
module.exports = {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
259
|
+
// Routing façade (backend.js) — kept here so engine stays the one import.
|
|
260
|
+
bridgeFileCandidates: backend.bridgeFileCandidates,
|
|
261
|
+
cdpEndpoint: backend.cdpEndpoint,
|
|
262
|
+
mode: backend.mode,
|
|
263
|
+
readBridgeAdvertisement: backend.readBridgeAdvertisement,
|
|
264
|
+
// Session façade (sessions.js).
|
|
265
|
+
knownSessions: sessions.knownSessions,
|
|
266
|
+
sessionFor,
|
|
267
|
+
sessionInfo,
|
|
268
|
+
// Verbs.
|
|
578
269
|
click,
|
|
579
270
|
close,
|
|
580
271
|
cua,
|
|
581
272
|
evaluate,
|
|
582
273
|
fill,
|
|
583
|
-
knownSessions,
|
|
584
274
|
loadState,
|
|
585
|
-
mode,
|
|
586
275
|
open,
|
|
587
276
|
passthrough,
|
|
588
277
|
press,
|
|
589
|
-
readBridgeAdvertisement,
|
|
590
278
|
saveState,
|
|
591
279
|
screenshot,
|
|
592
|
-
sessionFor,
|
|
593
|
-
sessionInfo,
|
|
594
280
|
snapshot,
|
|
595
281
|
state,
|
|
596
|
-
tabs,
|
|
597
282
|
wait,
|
|
598
283
|
};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The browser toolbox entry — one entry, four capability sections.
|
|
5
|
+
*
|
|
6
|
+
* Action names are short verbs; the toolbox layer prefixes them with the
|
|
7
|
+
* entry id, so agents see toolbox__browser_open, toolbox__browser_record_start,
|
|
8
|
+
* and so on. (Never toolbox__browser_browser_open — the name stutter that
|
|
9
|
+
* made agents guess wrong is dead.)
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
function withCapability(tools, capability) {
|
|
13
|
+
return tools.map((tool) => ({ ...tool, capability }));
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const DESCRIPTION = 'Web browsing with one persistent browser per chat session: '
|
|
17
|
+
+ 'open pages, read them as @ref snapshots, click/fill/press, run JavaScript, '
|
|
18
|
+
+ 'screenshot, record WebM videos, and reuse saved logins. Visible to the user '
|
|
19
|
+
+ 'in the desktop app split view; headless everywhere else — same actions either way.';
|
|
20
|
+
|
|
21
|
+
const ACTIONS = [
|
|
22
|
+
...withCapability(require('./tools'), 'core'),
|
|
23
|
+
...withCapability(require('./cua-tools'), 'computer-use'),
|
|
24
|
+
...withCapability(require('./recorder-tools'), 'recording'),
|
|
25
|
+
...withCapability(require('./auth-tools'), 'auth'),
|
|
26
|
+
];
|
|
27
|
+
|
|
28
|
+
module.exports = { ACTIONS, DESCRIPTION };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Recorder
|
|
5
|
-
* use (QA bots especially). Captures the browser session — including the
|
|
6
|
-
* desktop app's visible in-tab webview
|
|
7
|
-
* Amalgm project the work belongs to.
|
|
4
|
+
* Recorder actions — session recording as a first-class primitive any agent
|
|
5
|
+
* can use (QA bots especially). Captures the browser session — including the
|
|
6
|
+
* desktop app's visible in-tab webview, cropped so the file shows only the
|
|
7
|
+
* page — to a local WebM stored with the Amalgm project the work belongs to.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
const { textResult, errorResult } = require('../lib/tool-result');
|