amalgm 0.1.91 → 0.1.92
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/browser/attach.js +238 -0
- package/runtime/scripts/amalgm-mcp/browser/auth-tools.js +26 -40
- package/runtime/scripts/amalgm-mcp/browser/backend.js +100 -0
- package/runtime/scripts/amalgm-mcp/browser/capture.js +167 -0
- package/runtime/scripts/amalgm-mcp/browser/cdp.js +177 -0
- package/runtime/scripts/amalgm-mcp/browser/engine.js +34 -349
- package/runtime/scripts/amalgm-mcp/browser/index.js +28 -0
- package/runtime/scripts/amalgm-mcp/browser/recorder-tools.js +4 -4
- package/runtime/scripts/amalgm-mcp/browser/recorder.js +60 -72
- package/runtime/scripts/amalgm-mcp/browser/sessions.js +64 -0
- package/runtime/scripts/amalgm-mcp/browser/tools.js +35 -31
- package/runtime/scripts/amalgm-mcp/server/core-tools.js +4 -14
- package/runtime/scripts/amalgm-mcp/server/mcp.js +24 -1
- package/runtime/scripts/amalgm-mcp/tests/browser-capture.test.js +87 -0
- package/runtime/scripts/amalgm-mcp/tests/core-tools.test.js +59 -29
- package/runtime/scripts/amalgm-mcp/tests/platform-context.test.js +46 -0
- package/runtime/scripts/amalgm-mcp/tests/system-catalog.test.js +4 -4
- package/runtime/scripts/amalgm-mcp/tests/toolbox-service.test.js +2 -2
- package/runtime/scripts/amalgm-mcp/toolbox/loadout-context.js +33 -10
- package/runtime/scripts/chat-core/tool-shape.js +27 -10
- package/runtime/scripts/platform-context.txt +21 -20
package/package.json
CHANGED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Attached mode — drive the browser surface the user is watching.
|
|
5
|
+
*
|
|
6
|
+
* The desktop app advertises its chromium over CDP; the UI mounts the in-tab
|
|
7
|
+
* split-view <webview> with a marker URL. This module owns the handshake:
|
|
8
|
+
*
|
|
9
|
+
* 1. Ask the UI to open the surface (a browser_surfaces state event — the
|
|
10
|
+
* renderer subscribes and mounts the webview; an event, not an RPC).
|
|
11
|
+
* 2. Select the session's webview target by marker (fallbacks: previously
|
|
12
|
+
* selected target, then the only webview). Fail closed if none appears —
|
|
13
|
+
* never silently drive the outer app window, never silently go headless.
|
|
14
|
+
* 3. One focus pass — chromium drops CDP Input.* events to a <webview>
|
|
15
|
+
* guest until the guest widget has focus (reads work unfocused; clicks
|
|
16
|
+
* would "succeed" without landing).
|
|
17
|
+
*
|
|
18
|
+
* run() is the command transport every verb uses: it ensures the handshake,
|
|
19
|
+
* then shells out to agent-browser, re-attaching once if the connection was
|
|
20
|
+
* lost (app restart, webview remount).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const cli = require('./cli');
|
|
24
|
+
const backend = require('./backend');
|
|
25
|
+
const cdp = require('./cdp');
|
|
26
|
+
const { attachedSessions, knownSessions } = require('./sessions');
|
|
27
|
+
|
|
28
|
+
const SURFACE_EVENT_RESOURCE = 'browser_surfaces';
|
|
29
|
+
const WEBVIEW_MARKER_PREFIX = 'amalgm-bsid-';
|
|
30
|
+
const WEBVIEW_WAIT_MS = 15_000;
|
|
31
|
+
const WEBVIEW_POLL_MS = 500;
|
|
32
|
+
|
|
33
|
+
function marker(session) {
|
|
34
|
+
return `${WEBVIEW_MARKER_PREFIX}${session}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Options every cli call needs for the current backend. */
|
|
38
|
+
function cliOptions(session, extra = {}) {
|
|
39
|
+
const options = { session, ...extra };
|
|
40
|
+
if (backend.mode() === 'attached') options.cdp = backend.cdpEndpoint();
|
|
41
|
+
return options;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function requestSurfaceOpen(session, context) {
|
|
45
|
+
try {
|
|
46
|
+
const { appendStateEvent } = require('../state/events');
|
|
47
|
+
appendStateEvent({
|
|
48
|
+
resource: SURFACE_EVENT_RESOURCE,
|
|
49
|
+
op: 'insert',
|
|
50
|
+
id: session,
|
|
51
|
+
value: {
|
|
52
|
+
browserSessionId: session,
|
|
53
|
+
callerSessionId: context?.callerSessionId || null,
|
|
54
|
+
marker: marker(session),
|
|
55
|
+
requestedAt: new Date().toISOString(),
|
|
56
|
+
},
|
|
57
|
+
source: 'browser-engine',
|
|
58
|
+
});
|
|
59
|
+
return true;
|
|
60
|
+
} catch {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function listWebviewTabs(session) {
|
|
66
|
+
const tabs = await cli.runCli(['tab'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
67
|
+
const list = tabs?.data?.tabs || tabs?.tabs || [];
|
|
68
|
+
return list.filter((tab) => tab.type === 'webview');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The session's webview rectangle in the app window, or null when the
|
|
73
|
+
* splitview is hidden (0x0) or gone. Raw CDP, deadline-bounded.
|
|
74
|
+
*/
|
|
75
|
+
async function surfaceRect(session, hintUrl) {
|
|
76
|
+
const endpoint = backend.cdpEndpoint();
|
|
77
|
+
if (!endpoint) return null;
|
|
78
|
+
try {
|
|
79
|
+
return await cdp.findSurfaceRect(endpoint, marker(session), hintUrl || '');
|
|
80
|
+
} catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Attached mode's product promise is a surface the user can see. A webview
|
|
87
|
+
* target can exist while the splitview panel is hidden — input silently
|
|
88
|
+
* drops and captures have nothing to show. When that happens, ask the UI to
|
|
89
|
+
* reopen it (repeat browser_surfaces events open or focus the existing
|
|
90
|
+
* surface) and poll until it has real on-screen geometry.
|
|
91
|
+
*/
|
|
92
|
+
async function ensureSurfaceVisible(session, hintUrl, context, waitMs = WEBVIEW_WAIT_MS) {
|
|
93
|
+
let found = await surfaceRect(session, hintUrl);
|
|
94
|
+
if (found) return found;
|
|
95
|
+
requestSurfaceOpen(session, context);
|
|
96
|
+
const deadlineAt = Date.now() + waitMs;
|
|
97
|
+
while (Date.now() < deadlineAt) {
|
|
98
|
+
await new Promise((resolve) => setTimeout(resolve, WEBVIEW_POLL_MS));
|
|
99
|
+
found = await surfaceRect(session, hintUrl);
|
|
100
|
+
if (found) return found;
|
|
101
|
+
}
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function matchWebview(webviews, session) {
|
|
106
|
+
const markerText = marker(session);
|
|
107
|
+
const byMarker = webviews.find((tab) => String(tab.url || '').includes(markerText)
|
|
108
|
+
|| String(tab.title || '').includes(markerText));
|
|
109
|
+
if (byMarker) return byMarker;
|
|
110
|
+
const previous = knownSessions.get(session)?.webviewTabId;
|
|
111
|
+
if (previous) {
|
|
112
|
+
const byPrevious = webviews.find((tab) => tab.tabId === previous);
|
|
113
|
+
if (byPrevious) return byPrevious;
|
|
114
|
+
}
|
|
115
|
+
if (webviews.length === 1) return webviews[0];
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Connect the session to the advertised chromium on first use and select the
|
|
121
|
+
* in-tab webview target so commands drive the surface the user is looking at.
|
|
122
|
+
*/
|
|
123
|
+
async function ensureReady(session, context) {
|
|
124
|
+
if (backend.mode() !== 'attached' || attachedSessions.has(session)) return;
|
|
125
|
+
|
|
126
|
+
let webviews = await listWebviewTabs(session);
|
|
127
|
+
let selected = matchWebview(webviews, session);
|
|
128
|
+
|
|
129
|
+
if (!selected) {
|
|
130
|
+
requestSurfaceOpen(session, context);
|
|
131
|
+
const deadline = Date.now() + WEBVIEW_WAIT_MS;
|
|
132
|
+
while (!selected && Date.now() < deadline) {
|
|
133
|
+
await new Promise((resolve) => setTimeout(resolve, WEBVIEW_POLL_MS));
|
|
134
|
+
webviews = await listWebviewTabs(session);
|
|
135
|
+
selected = matchWebview(webviews, session);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (!selected && webviews.length === 0 && process.env.AMALGM_BROWSER_CDP_URL) {
|
|
140
|
+
// Explicit external CDP endpoints (plain chromium) have page targets,
|
|
141
|
+
// not webviews — drive whatever the daemon selected by default.
|
|
142
|
+
attachedSessions.add(session);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (!selected) {
|
|
147
|
+
throw new Error(
|
|
148
|
+
'Browser surface did not open in the desktop app. '
|
|
149
|
+
+ 'Retry, or force a headless browser with AMALGM_BROWSER_BACKEND=cli.',
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
154
|
+
|
|
155
|
+
// The webview target existing is not enough — the splitview panel must be
|
|
156
|
+
// on screen, or input drops and captures have nothing to render.
|
|
157
|
+
const visible = await ensureSurfaceVisible(session, selected.url, context);
|
|
158
|
+
if (!visible) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
'The browser surface is hidden in the desktop app and did not reopen. '
|
|
161
|
+
+ 'Bring the Amalgm window forward, or force a headless browser with AMALGM_BROWSER_BACKEND=cli.',
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
await focusWebviewHost(session, selected);
|
|
166
|
+
knownSessions.set(session, { ...knownSessions.get(session), webviewTabId: selected.tabId });
|
|
167
|
+
attachedSessions.add(session);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Chromium drops CDP Input events to an Electron <webview> guest until the
|
|
172
|
+
* guest widget has focus (verified live: clicks report success but never
|
|
173
|
+
* land). One focus pass after target selection fixes input for the session:
|
|
174
|
+
* focus the <webview> element from the embedder page, then self-focus the
|
|
175
|
+
* guest. Best-effort — failures leave snapshot/eval working regardless.
|
|
176
|
+
*/
|
|
177
|
+
async function focusWebviewHost(session, selected) {
|
|
178
|
+
const markerText = marker(session);
|
|
179
|
+
try {
|
|
180
|
+
const tabs = await cli.runCli(['tab'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
181
|
+
const list = tabs?.data?.tabs || tabs?.tabs || [];
|
|
182
|
+
const outer = list.find((tab) => tab.type === 'page');
|
|
183
|
+
if (outer) {
|
|
184
|
+
const script = `(() => {
|
|
185
|
+
const views = [...document.querySelectorAll('webview')];
|
|
186
|
+
const target = views.find((w) => (w.getAttribute('src') || '').includes(${JSON.stringify(markerText)}))
|
|
187
|
+
|| views.find((w) => (w.src || '') === ${JSON.stringify(String(selected.url || ''))})
|
|
188
|
+
|| views[views.length - 1];
|
|
189
|
+
if (target) target.focus();
|
|
190
|
+
return views.length;
|
|
191
|
+
})()`;
|
|
192
|
+
await cli.runCli(['tab', outer.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
193
|
+
await cli.runCli(['eval', script], cliOptions(session, { timeoutMs: 15_000 }));
|
|
194
|
+
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
195
|
+
}
|
|
196
|
+
await cli.runCli(['eval', 'window.focus()'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
197
|
+
} catch {
|
|
198
|
+
// Snapshot/eval still work without focus; input may need a retry.
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function looksLikeLostConnection(err) {
|
|
203
|
+
return /clos|connect|target|socket|detach/i.test(String(err?.message || err || ''));
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
async function run(session, commandArgs, options = {}) {
|
|
207
|
+
const { context, ...cliExtra } = options;
|
|
208
|
+
await ensureReady(session, context);
|
|
209
|
+
try {
|
|
210
|
+
return await cli.runCli(commandArgs, cliOptions(session, cliExtra));
|
|
211
|
+
} catch (err) {
|
|
212
|
+
if (backend.mode() === 'attached' && attachedSessions.has(session) && looksLikeLostConnection(err)) {
|
|
213
|
+
// The app restarted or the webview remounted; re-attach once.
|
|
214
|
+
attachedSessions.delete(session);
|
|
215
|
+
await ensureReady(session, context);
|
|
216
|
+
return cli.runCli(commandArgs, cliOptions(session, cliExtra));
|
|
217
|
+
}
|
|
218
|
+
throw err;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async function readText(session, commandArgs, timeoutMs) {
|
|
223
|
+
await ensureReady(session);
|
|
224
|
+
return cli.readText(commandArgs, session, timeoutMs, backend.mode() === 'attached' ? backend.cdpEndpoint() : null);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
module.exports = {
|
|
228
|
+
SURFACE_EVENT_RESOURCE,
|
|
229
|
+
WEBVIEW_MARKER_PREFIX,
|
|
230
|
+
cliOptions,
|
|
231
|
+
ensureReady,
|
|
232
|
+
ensureSurfaceVisible,
|
|
233
|
+
marker,
|
|
234
|
+
matchWebview,
|
|
235
|
+
readText,
|
|
236
|
+
requestSurfaceOpen,
|
|
237
|
+
run,
|
|
238
|
+
};
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Browser auth
|
|
5
|
-
* temporary login links. Split from the core
|
|
6
|
-
*
|
|
4
|
+
* Browser auth actions — durable profiles, encrypted auth bundles, and
|
|
5
|
+
* temporary login links. Split from the core surface: these manage login
|
|
6
|
+
* state, they don't drive pages.
|
|
7
|
+
*
|
|
8
|
+
* The human-handoff flow: auth_link_create mints a short-lived link → the
|
|
9
|
+
* user signs in on a visible surface → auth_save captures the session as an
|
|
10
|
+
* encrypted bundle → auth_load replays it into a profile later. Raw cookies
|
|
11
|
+
* and tokens never appear in tool results, and bundles never leave this
|
|
12
|
+
* computer.
|
|
7
13
|
*/
|
|
8
14
|
|
|
9
15
|
const { textResult, errorResult } = require('../lib/tool-result');
|
|
@@ -36,44 +42,24 @@ function authDomainsFrom(inputDomains, fallbackUrl) {
|
|
|
36
42
|
|
|
37
43
|
module.exports = [
|
|
38
44
|
{
|
|
39
|
-
name: '
|
|
40
|
-
description: 'List
|
|
45
|
+
name: 'auth_list',
|
|
46
|
+
description: 'List saved browser auth state on this computer: durable profiles, encrypted auth bundles, and pending login links. Raw cookies and tokens are never returned.',
|
|
41
47
|
inputSchema: { type: 'object', properties: {} },
|
|
42
48
|
async handler() {
|
|
43
49
|
try {
|
|
44
|
-
return jsonText({
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
},
|
|
50
|
-
{
|
|
51
|
-
name: 'browser_auth_bundles_list',
|
|
52
|
-
description: 'List encrypted browser auth bundles available on this computer. Raw cookies/storage are never returned.',
|
|
53
|
-
inputSchema: { type: 'object', properties: {} },
|
|
54
|
-
async handler() {
|
|
55
|
-
try {
|
|
56
|
-
return jsonText({ bundles: listBrowserAuthBundles() });
|
|
57
|
-
} catch (err) {
|
|
58
|
-
return errorResult(`List browser auth bundles failed: ${err.message}`);
|
|
59
|
-
}
|
|
60
|
-
},
|
|
61
|
-
},
|
|
62
|
-
{
|
|
63
|
-
name: 'browser_login_sessions_list',
|
|
64
|
-
description: 'List temporary browser login links on this computer. Raw tokens and cookies are never returned.',
|
|
65
|
-
inputSchema: { type: 'object', properties: {} },
|
|
66
|
-
async handler() {
|
|
67
|
-
try {
|
|
68
|
-
return jsonText({ sessions: listBrowserLoginSessions() });
|
|
50
|
+
return jsonText({
|
|
51
|
+
profiles: listBrowserProfiles(),
|
|
52
|
+
bundles: listBrowserAuthBundles(),
|
|
53
|
+
loginSessions: listBrowserLoginSessions(),
|
|
54
|
+
});
|
|
69
55
|
} catch (err) {
|
|
70
|
-
return errorResult(`
|
|
56
|
+
return errorResult(`auth_list failed: ${err.message}`);
|
|
71
57
|
}
|
|
72
58
|
},
|
|
73
59
|
},
|
|
74
60
|
{
|
|
75
|
-
name: '
|
|
76
|
-
description: 'Create a short-lived
|
|
61
|
+
name: 'auth_link_create',
|
|
62
|
+
description: 'Create a short-lived login link for the user when a site needs them to sign in. The user logs in on a visible browser surface; follow with auth_save to keep the session for later runs.',
|
|
77
63
|
inputSchema: {
|
|
78
64
|
type: 'object',
|
|
79
65
|
properties: {
|
|
@@ -108,13 +94,13 @@ module.exports = [
|
|
|
108
94
|
message: `Open this temporary login link: ${result.authLink}`,
|
|
109
95
|
});
|
|
110
96
|
} catch (err) {
|
|
111
|
-
return errorResult(`
|
|
97
|
+
return errorResult(`auth_link_create failed: ${err.message}`);
|
|
112
98
|
}
|
|
113
99
|
},
|
|
114
100
|
},
|
|
115
101
|
{
|
|
116
|
-
name: '
|
|
117
|
-
description: 'Save the current browser
|
|
102
|
+
name: 'auth_save',
|
|
103
|
+
description: 'Save the current browser session cookies/storage into an encrypted auth bundle for later agent use.',
|
|
118
104
|
inputSchema: {
|
|
119
105
|
type: 'object',
|
|
120
106
|
properties: {
|
|
@@ -140,13 +126,13 @@ module.exports = [
|
|
|
140
126
|
}, { source: 'browser-auth-save' });
|
|
141
127
|
return jsonText({ bundle });
|
|
142
128
|
} catch (err) {
|
|
143
|
-
return errorResult(`
|
|
129
|
+
return errorResult(`auth_save failed: ${err.message}`);
|
|
144
130
|
}
|
|
145
131
|
},
|
|
146
132
|
},
|
|
147
133
|
{
|
|
148
|
-
name: '
|
|
149
|
-
description: 'Load an encrypted
|
|
134
|
+
name: 'auth_load',
|
|
135
|
+
description: 'Load an encrypted auth bundle into a durable browser profile so an agent can start past login.',
|
|
150
136
|
inputSchema: {
|
|
151
137
|
type: 'object',
|
|
152
138
|
properties: {
|
|
@@ -172,7 +158,7 @@ module.exports = [
|
|
|
172
158
|
}, { source: 'browser-auth-load' });
|
|
173
159
|
return jsonText({ bundle: updated, loaded });
|
|
174
160
|
} catch (err) {
|
|
175
|
-
return errorResult(`
|
|
161
|
+
return errorResult(`auth_load failed: ${err.message}`);
|
|
176
162
|
}
|
|
177
163
|
},
|
|
178
164
|
},
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Backend routing — which browser this runtime drives.
|
|
5
|
+
*
|
|
6
|
+
* Exactly two backends:
|
|
7
|
+
* - "attached": the visible chromium the Amalgm desktop app advertises over
|
|
8
|
+
* CDP (or an explicit AMALGM_BROWSER_CDP_URL endpoint). The user watches
|
|
9
|
+
* this browser in the app's split view.
|
|
10
|
+
* - "cli": a standalone agent-browser chromium, headless. The deterministic
|
|
11
|
+
* path for npm-only machines, servers, Fly workers, and the web client.
|
|
12
|
+
*
|
|
13
|
+
* The rule: a fresh advertisement (or explicit CDP env) ⇒ attached;
|
|
14
|
+
* otherwise ⇒ cli. "Fresh" means the advertising pid is alive. Nothing else
|
|
15
|
+
* may influence the choice — visibility must be deterministic per entrypoint.
|
|
16
|
+
* AMALGM_BROWSER_BACKEND forces a backend for debugging.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
function pidIsRunning(pid) {
|
|
20
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
21
|
+
try {
|
|
22
|
+
process.kill(pid, 0);
|
|
23
|
+
return true;
|
|
24
|
+
} catch (err) {
|
|
25
|
+
return Boolean(err && err.code === 'EPERM');
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Bridge advertisement candidates, most specific first. Every runtime
|
|
31
|
+
* generation can find the file written by the desktop app:
|
|
32
|
+
* - Electron-spawned and LaunchAgent runtimes have AMALGM_RUNTIME_STATE_DIR;
|
|
33
|
+
* - runtimes with only AMALGM_DIR (+ label) derive the same path;
|
|
34
|
+
* - bare dev runtimes fall back to the unscoped roots.
|
|
35
|
+
*/
|
|
36
|
+
function bridgeFileCandidates() {
|
|
37
|
+
const path = require('path');
|
|
38
|
+
const os = require('os');
|
|
39
|
+
const dirs = [];
|
|
40
|
+
if (process.env.AMALGM_RUNTIME_STATE_DIR) dirs.push(process.env.AMALGM_RUNTIME_STATE_DIR);
|
|
41
|
+
const amalgmDir = process.env.AMALGM_DIR;
|
|
42
|
+
const label = process.env.AMALGM_RUNTIME_LABEL || process.env.AMALGM_BRANCH;
|
|
43
|
+
if (amalgmDir && label) dirs.push(path.join(amalgmDir, 'runtimes', label));
|
|
44
|
+
if (amalgmDir) dirs.push(amalgmDir);
|
|
45
|
+
dirs.push(path.join(os.homedir(), '.amalgm'));
|
|
46
|
+
return [...new Set(dirs)].map((dir) => path.join(dir, 'electron-browser-bridge.json'));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function readBridgeAdvertisement() {
|
|
50
|
+
const fs = require('fs');
|
|
51
|
+
for (const file of bridgeFileCandidates()) {
|
|
52
|
+
try {
|
|
53
|
+
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
54
|
+
if (typeof data?.cdpUrl !== 'string' || !data.cdpUrl) continue;
|
|
55
|
+
if (typeof data?.pid === 'number' && !pidIsRunning(data.pid)) continue;
|
|
56
|
+
return { cdpUrl: data.cdpUrl, cdpPort: data.cdpPort, label: data.label, file };
|
|
57
|
+
} catch {}
|
|
58
|
+
}
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Reduce a CDP url/port advertisement to the value agent-browser's --cdp flag takes. */
|
|
63
|
+
function cdpFlagValue(input) {
|
|
64
|
+
const raw = String(input || '').trim();
|
|
65
|
+
if (!raw) return null;
|
|
66
|
+
if (/^\d+$/.test(raw)) return raw;
|
|
67
|
+
try {
|
|
68
|
+
const url = new URL(raw);
|
|
69
|
+
if (url.port) return url.port;
|
|
70
|
+
} catch {}
|
|
71
|
+
return raw;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The CDP endpoint for attached mode, as a --cdp flag value. agent-browser's
|
|
76
|
+
* `connect` command launches a managed browser; raw --cdp attach is the mode
|
|
77
|
+
* that discovers Electron <webview> targets, so that is all we use.
|
|
78
|
+
*/
|
|
79
|
+
function cdpEndpoint() {
|
|
80
|
+
if (process.env.AMALGM_BROWSER_CDP_URL) return cdpFlagValue(process.env.AMALGM_BROWSER_CDP_URL);
|
|
81
|
+
const advertisement = readBridgeAdvertisement();
|
|
82
|
+
if (!advertisement) return null;
|
|
83
|
+
if (advertisement.cdpPort) return String(advertisement.cdpPort);
|
|
84
|
+
return cdpFlagValue(advertisement.cdpUrl);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function mode() {
|
|
88
|
+
const requested = (process.env.AMALGM_BROWSER_BACKEND || '').toLowerCase();
|
|
89
|
+
if (['agent-browser', 'external', 'cli', 'headless'].includes(requested)) return 'cli';
|
|
90
|
+
if (['attached', 'cdp', 'electron'].includes(requested)) return 'attached';
|
|
91
|
+
return cdpEndpoint() ? 'attached' : 'cli';
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
module.exports = {
|
|
95
|
+
bridgeFileCandidates,
|
|
96
|
+
cdpEndpoint,
|
|
97
|
+
cdpFlagValue,
|
|
98
|
+
mode,
|
|
99
|
+
readBridgeAdvertisement,
|
|
100
|
+
};
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Raster capture — screenshots and recording frames, over raw CDP.
|
|
5
|
+
*
|
|
6
|
+
* Pixels never go through the agent-browser daemon. Two reasons, both
|
|
7
|
+
* verified live against the desktop app:
|
|
8
|
+
*
|
|
9
|
+
* - Electron <webview> guest targets do not implement raster capture:
|
|
10
|
+
* Page.captureScreenshot never resolves and Page.startScreencast emits
|
|
11
|
+
* zero frames. (Playwright's screenshot waits on them forever.)
|
|
12
|
+
* - A capture stuck inside the per-session daemon starves every later
|
|
13
|
+
* command of that session — one hung screenshot used to wedge the whole
|
|
14
|
+
* browser until close.
|
|
15
|
+
*
|
|
16
|
+
* So captures resolve their own CDP target:
|
|
17
|
+
* - cli (headless) sessions capture their page target, which rasters fine.
|
|
18
|
+
* - attached sessions capture the app's top-level window — which also
|
|
19
|
+
* rasters fine — clipped (screenshots) or cropped (recordings) to the
|
|
20
|
+
* session's <webview> rectangle. The image is exactly the page the user
|
|
21
|
+
* sees; the surrounding app UI never reaches a tool result or a file.
|
|
22
|
+
*
|
|
23
|
+
* Every exchange carries a hard deadline (cdp.js): a capture may fail, it
|
|
24
|
+
* can never wedge a session.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const backend = require('./backend');
|
|
28
|
+
const cli = require('./cli');
|
|
29
|
+
const attach = require('./attach');
|
|
30
|
+
const cdp = require('./cdp');
|
|
31
|
+
|
|
32
|
+
const SCREENSHOT_TIMEOUT_MS = 20_000;
|
|
33
|
+
const FULL_PAGE_MAX_PX = 16_384;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Page.captureScreenshot clip (CSS px) for a webview rectangle. The capture
|
|
37
|
+
* renders at the device scale factor, so scale divides it back out — the
|
|
38
|
+
* image is CSS-pixel sized and its coordinates map 1:1 onto cua_click/
|
|
39
|
+
* cua_move input coordinates (the contract vision loops rely on).
|
|
40
|
+
*/
|
|
41
|
+
function clipFor(rect) {
|
|
42
|
+
return {
|
|
43
|
+
x: Math.max(0, Math.round(rect.x)),
|
|
44
|
+
y: Math.max(0, Math.round(rect.y)),
|
|
45
|
+
width: Math.max(1, Math.round(rect.width)),
|
|
46
|
+
height: Math.max(1, Math.round(rect.height)),
|
|
47
|
+
scale: 1 / (rect.dpr || 1),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* ffmpeg crop for a webview rectangle, expressed in viewport fractions so it
|
|
53
|
+
* holds for whatever resolution the screencast emits (frames arrive in
|
|
54
|
+
* device pixels, the rect is measured in CSS pixels — fractions cancel the
|
|
55
|
+
* scale). Crop happens before any frame reaches disk, so the recording file
|
|
56
|
+
* only ever contains the page, not the app around it.
|
|
57
|
+
*/
|
|
58
|
+
function cropFilterFor(rect) {
|
|
59
|
+
const fraction = (value, total) => Math.min(Math.max(value / total, 0), 1).toFixed(6);
|
|
60
|
+
const w = fraction(rect.width, rect.vw);
|
|
61
|
+
const h = fraction(rect.height, rect.vh);
|
|
62
|
+
const x = fraction(rect.x, rect.vw);
|
|
63
|
+
const y = fraction(rect.y, rect.vh);
|
|
64
|
+
return `crop=floor(iw*${w}/2)*2:floor(ih*${h}/2)*2:floor(iw*${x}):floor(ih*${y})`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The cli-mode daemon's CDP url (spawns the daemon on first use). */
|
|
68
|
+
async function daemonCdpUrl(session) {
|
|
69
|
+
const result = await cli.runCli(['get', 'cdp-url'], { session, timeoutMs: 15_000 });
|
|
70
|
+
return result?.data?.cdpUrl || result?.cdpUrl || null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Where to capture from for this session:
|
|
75
|
+
* { wsUrl, clip, crop } — clip bounds screenshots, crop bounds recordings;
|
|
76
|
+
* both null when the target's own surface is capturable as-is.
|
|
77
|
+
*/
|
|
78
|
+
async function plan(session, context) {
|
|
79
|
+
if (backend.mode() !== 'attached') {
|
|
80
|
+
const raw = await daemonCdpUrl(session);
|
|
81
|
+
if (!raw) throw new Error('Could not resolve a CDP endpoint for this browser session.');
|
|
82
|
+
let wsUrl = raw;
|
|
83
|
+
if (/\/devtools\/browser\//.test(raw)) {
|
|
84
|
+
const page = cdp.pageTargets(await cdp.listTargets(raw))[0];
|
|
85
|
+
if (!page) throw new Error('No capturable page target on the browser endpoint.');
|
|
86
|
+
wsUrl = page.webSocketDebuggerUrl;
|
|
87
|
+
}
|
|
88
|
+
return { wsUrl, clip: null, crop: null };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
await attach.ensureReady(session, context);
|
|
92
|
+
const endpoint = backend.cdpEndpoint();
|
|
93
|
+
const targets = await cdp.listTargets(endpoint);
|
|
94
|
+
|
|
95
|
+
if (!targets.some((t) => t.type === 'webview')) {
|
|
96
|
+
// Explicit external CDP endpoint (plain chromium): its page targets
|
|
97
|
+
// raster fine — capture the driven page directly.
|
|
98
|
+
const page = cdp.pageTargets(targets)[0];
|
|
99
|
+
if (!page) throw new Error('No capturable page target on the CDP endpoint.');
|
|
100
|
+
return { wsUrl: page.webSocketDebuggerUrl, clip: null, crop: null };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Desktop app: capture the window hosting this session's webview, bounded
|
|
104
|
+
// to the webview's on-screen rectangle. If the splitview is hidden, ask
|
|
105
|
+
// the UI to reopen it and wait briefly — captures self-heal instead of
|
|
106
|
+
// failing on a closed panel.
|
|
107
|
+
let currentUrl = '';
|
|
108
|
+
try { currentUrl = await attach.readText(session, ['get', 'url']); } catch {}
|
|
109
|
+
const surface = await attach.ensureSurfaceVisible(session, currentUrl, context, 5_000);
|
|
110
|
+
if (!surface) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
'The browser surface is not visible in the desktop app — bring the Amalgm window forward and retry.',
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
return {
|
|
116
|
+
wsUrl: surface.pageWsUrl,
|
|
117
|
+
clip: clipFor(surface.rect),
|
|
118
|
+
crop: cropFilterFor(surface.rect),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function screenshot(session, args = {}, context) {
|
|
123
|
+
const fullPage = Boolean(args.fullPage);
|
|
124
|
+
const source = await plan(session, context);
|
|
125
|
+
const client = await cdp.connectCdp(source.wsUrl);
|
|
126
|
+
try {
|
|
127
|
+
const params = { format: 'png' };
|
|
128
|
+
if (source.clip) {
|
|
129
|
+
params.clip = source.clip;
|
|
130
|
+
} else if (fullPage) {
|
|
131
|
+
const metrics = await client.call('Page.getLayoutMetrics');
|
|
132
|
+
const size = metrics.cssContentSize || metrics.contentSize;
|
|
133
|
+
if (size) {
|
|
134
|
+
params.clip = {
|
|
135
|
+
x: 0,
|
|
136
|
+
y: 0,
|
|
137
|
+
width: Math.min(Math.ceil(size.width), FULL_PAGE_MAX_PX),
|
|
138
|
+
height: Math.min(Math.ceil(size.height), FULL_PAGE_MAX_PX),
|
|
139
|
+
scale: 1,
|
|
140
|
+
};
|
|
141
|
+
params.captureBeyondViewport = true;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const { data } = await client.call('Page.captureScreenshot', params, SCREENSHOT_TIMEOUT_MS);
|
|
145
|
+
if (!data) throw new Error('Empty screenshot response');
|
|
146
|
+
return {
|
|
147
|
+
base64: data,
|
|
148
|
+
bytes: Buffer.byteLength(data, 'base64'),
|
|
149
|
+
mode: source.clip ? 'visible surface' : (fullPage ? 'full page' : 'viewport'),
|
|
150
|
+
};
|
|
151
|
+
} finally {
|
|
152
|
+
client.close();
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
module.exports = {
|
|
157
|
+
clipFor,
|
|
158
|
+
connectCdp: cdp.connectCdp,
|
|
159
|
+
cropFilterFor,
|
|
160
|
+
deadline: cdp.deadline,
|
|
161
|
+
listTargets: cdp.listTargets,
|
|
162
|
+
pageTargets: cdp.pageTargets,
|
|
163
|
+
plan,
|
|
164
|
+
rectScript: cdp.rectScript,
|
|
165
|
+
screenshot,
|
|
166
|
+
targetListUrl: cdp.targetListUrl,
|
|
167
|
+
};
|