amalgm 0.1.93 → 0.1.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/browser/attach.js +158 -21
- package/runtime/scripts/amalgm-mcp/browser/auth-tools.js +21 -2
- package/runtime/scripts/amalgm-mcp/browser/capture.js +60 -102
- package/runtime/scripts/amalgm-mcp/browser/cdp.js +18 -11
- package/runtime/scripts/amalgm-mcp/browser/engine.js +38 -3
- package/runtime/scripts/amalgm-mcp/browser/recorder.js +27 -13
- package/runtime/scripts/amalgm-mcp/browser/tools.js +29 -6
- package/runtime/scripts/amalgm-mcp/fs/rest.js +136 -0
- package/runtime/scripts/amalgm-mcp/server/routes/files.js +4 -0
- package/runtime/scripts/amalgm-mcp/tests/browser-capture.test.js +47 -34
- package/runtime/scripts/amalgm-mcp/tests/browser-transport.test.js +155 -0
- package/runtime/scripts/amalgm-mcp/tests/core-tools.test.js +2 -1
- package/runtime/scripts/platform-context.txt +7 -6
package/package.json
CHANGED
|
@@ -17,7 +17,12 @@
|
|
|
17
17
|
*
|
|
18
18
|
* run() is the command transport every verb uses: it ensures the handshake,
|
|
19
19
|
* then shells out to agent-browser, re-attaching once if the connection was
|
|
20
|
-
* lost (app restart, webview remount).
|
|
20
|
+
* lost (app restart, webview remount). In attached mode every call re-pins
|
|
21
|
+
* the session's webview target inside the same batch — the daemon's tab
|
|
22
|
+
* selection is process state that silently falls back to "the active tab"
|
|
23
|
+
* when the selected target dies, and that fallback is how one session ends
|
|
24
|
+
* up reading another session's page. A dead pin fails the call by name and
|
|
25
|
+
* triggers one re-attach instead.
|
|
21
26
|
*/
|
|
22
27
|
|
|
23
28
|
const cli = require('./cli');
|
|
@@ -29,6 +34,9 @@ const SURFACE_EVENT_RESOURCE = 'browser_surfaces';
|
|
|
29
34
|
const WEBVIEW_MARKER_PREFIX = 'amalgm-bsid-';
|
|
30
35
|
const WEBVIEW_WAIT_MS = 15_000;
|
|
31
36
|
const WEBVIEW_POLL_MS = 500;
|
|
37
|
+
const SURFACE_HIDDEN_MESSAGE = 'The browser surface is hidden in the desktop app and did not reopen. '
|
|
38
|
+
+ 'Retry; if it keeps failing, close this browser session and open again '
|
|
39
|
+
+ '(a fresh surface will mount), or force a headless browser with AMALGM_BROWSER_BACKEND=cli.';
|
|
32
40
|
|
|
33
41
|
function marker(session) {
|
|
34
42
|
return `${WEBVIEW_MARKER_PREFIX}${session}`;
|
|
@@ -172,13 +180,12 @@ async function ensureReady(session, context) {
|
|
|
172
180
|
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
173
181
|
|
|
174
182
|
// The webview target existing is not enough — the splitview panel must be
|
|
175
|
-
// on screen, or input drops and captures have nothing to render.
|
|
183
|
+
// on screen, or input drops and captures have nothing to render. (The app
|
|
184
|
+
// window itself may be hidden or minimized; that no longer matters — the
|
|
185
|
+
// app keeps an attached surface compositing wherever the window is.)
|
|
176
186
|
const visible = await ensureSurfaceVisible(session, selected.url, context);
|
|
177
187
|
if (!visible) {
|
|
178
|
-
throw new Error(
|
|
179
|
-
'The browser surface is hidden in the desktop app and did not reopen. '
|
|
180
|
-
+ 'Bring the Amalgm window forward, or force a headless browser with AMALGM_BROWSER_BACKEND=cli.',
|
|
181
|
-
);
|
|
188
|
+
throw new Error(SURFACE_HIDDEN_MESSAGE);
|
|
182
189
|
}
|
|
183
190
|
|
|
184
191
|
await focusWebviewHost(session, selected);
|
|
@@ -222,22 +229,83 @@ function looksLikeLostConnection(err) {
|
|
|
222
229
|
return /clos|connect|target|socket|detach/i.test(String(err?.message || err || ''));
|
|
223
230
|
}
|
|
224
231
|
|
|
225
|
-
|
|
232
|
+
/**
|
|
233
|
+
* A failed batch can surface two ways: a success envelope whose entries
|
|
234
|
+
* carry per-step failures, or an envelope-level rejection whose message is
|
|
235
|
+
* the serialized entries. Either way, when the failing step is the tab pin,
|
|
236
|
+
* reclassify it as a lost surface target so the retry path re-attaches —
|
|
237
|
+
* the raw "Tab tN not found" would otherwise read as a page error.
|
|
238
|
+
*/
|
|
239
|
+
function pinFailure(err, pin) {
|
|
240
|
+
if (!pin) return err;
|
|
241
|
+
try {
|
|
242
|
+
const entries = JSON.parse(String(err?.message || ''));
|
|
243
|
+
const failedPin = Array.isArray(entries) && entries.find((entry) => entry
|
|
244
|
+
&& Array.isArray(entry.command) && entry.command[0] === 'tab' && entry.error);
|
|
245
|
+
if (failedPin) {
|
|
246
|
+
return new Error(`surface target lost (${failedPin.command.join(' ')}: ${failedPin.error})`);
|
|
247
|
+
}
|
|
248
|
+
} catch {
|
|
249
|
+
// Not a serialized batch — leave the error as-is.
|
|
250
|
+
}
|
|
251
|
+
return err;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/**
|
|
255
|
+
* Execute commands against the session's browser, re-attaching once on a
|
|
256
|
+
* lost connection. In attached mode the session's webview target id is
|
|
257
|
+
* re-asserted as a `tab` step inside the same batch, so every command runs
|
|
258
|
+
* against the verified surface — never a daemon-side fallback. A dead pin
|
|
259
|
+
* throws "surface target lost", which the retry path treats as a remount.
|
|
260
|
+
*/
|
|
261
|
+
async function execute(session, commands, options = {}, single) {
|
|
226
262
|
const { context, ...cliExtra } = options;
|
|
227
263
|
await ensureReady(session, context);
|
|
264
|
+
|
|
265
|
+
const attempt = async () => {
|
|
266
|
+
const pin = backend.mode() === 'attached' ? knownSessions.get(session)?.webviewTabId : null;
|
|
267
|
+
const batchCommands = pin ? [['tab', pin], ...commands] : commands;
|
|
268
|
+
if (batchCommands.length === 1 && single) {
|
|
269
|
+
return cli.runCli(batchCommands[0], cliOptions(session, cliExtra));
|
|
270
|
+
}
|
|
271
|
+
let result;
|
|
272
|
+
try {
|
|
273
|
+
result = await cli.runCli(['batch', '--bail'], cliOptions(session, {
|
|
274
|
+
...cliExtra,
|
|
275
|
+
stdin: JSON.stringify(batchCommands.map((command) => command.map(String))),
|
|
276
|
+
}));
|
|
277
|
+
} catch (err) {
|
|
278
|
+
throw pinFailure(err, pin);
|
|
279
|
+
}
|
|
280
|
+
let entries = Array.isArray(result) ? result : (result?.data || []);
|
|
281
|
+
if (!Array.isArray(entries)) entries = [];
|
|
282
|
+
const failed = entries.find((entry) => entry && entry.success === false);
|
|
283
|
+
if (failed) {
|
|
284
|
+
const message = `${(failed.command || []).join(' ')}: ${failed.error || 'failed'}`;
|
|
285
|
+
const pinFailed = pin && Array.isArray(failed.command) && failed.command[0] === 'tab';
|
|
286
|
+
throw new Error(pinFailed ? `surface target lost (${message})` : message);
|
|
287
|
+
}
|
|
288
|
+
if (pin) entries = entries.slice(1);
|
|
289
|
+
return single ? { success: true, data: entries[0]?.result } : entries;
|
|
290
|
+
};
|
|
291
|
+
|
|
228
292
|
try {
|
|
229
|
-
return await
|
|
293
|
+
return await attempt();
|
|
230
294
|
} catch (err) {
|
|
231
295
|
if (backend.mode() === 'attached' && attachedSessions.has(session) && looksLikeLostConnection(err)) {
|
|
232
296
|
// The app restarted or the webview remounted; re-attach once.
|
|
233
297
|
attachedSessions.delete(session);
|
|
234
298
|
await ensureReady(session, context);
|
|
235
|
-
return
|
|
299
|
+
return attempt();
|
|
236
300
|
}
|
|
237
301
|
throw err;
|
|
238
302
|
}
|
|
239
303
|
}
|
|
240
304
|
|
|
305
|
+
async function run(session, commandArgs, options = {}) {
|
|
306
|
+
return execute(session, [commandArgs], options, true);
|
|
307
|
+
}
|
|
308
|
+
|
|
241
309
|
/**
|
|
242
310
|
* Several commands, one daemon round trip — composites (mouse click = move +
|
|
243
311
|
* down + up) cost one process spawn instead of three. Commands go as JSON on
|
|
@@ -246,32 +314,101 @@ async function run(session, commandArgs, options = {}) {
|
|
|
246
314
|
* named. Returns the per-command result array.
|
|
247
315
|
*/
|
|
248
316
|
async function runBatch(session, commands, options = {}) {
|
|
249
|
-
|
|
250
|
-
...options,
|
|
251
|
-
stdin: JSON.stringify(commands.map((command) => command.map(String))),
|
|
252
|
-
});
|
|
253
|
-
const entries = Array.isArray(result) ? result : (result?.data || []);
|
|
254
|
-
const failed = Array.isArray(entries) ? entries.find((entry) => entry && entry.success === false) : null;
|
|
255
|
-
if (failed) {
|
|
256
|
-
throw new Error(`${(failed.command || []).join(' ')}: ${failed.error || 'failed'}`);
|
|
257
|
-
}
|
|
258
|
-
return entries;
|
|
317
|
+
return execute(session, commands, options, false);
|
|
259
318
|
}
|
|
260
319
|
|
|
261
320
|
async function readText(session, commandArgs, timeoutMs) {
|
|
262
|
-
await
|
|
263
|
-
|
|
321
|
+
return cli.asText(await run(session, commandArgs, { timeoutMs }));
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* The CDP target of this session's <webview>, for raster capture. Requires
|
|
326
|
+
* the surface on screen (the one physical precondition for guest pixels) —
|
|
327
|
+
* ensureSurfaceVisible self-heals a backgrounded tab first. Identity:
|
|
328
|
+
* marker in the target url/title, else the live src the embedder DOM
|
|
329
|
+
* reports for the session's marked element. When several webviews share
|
|
330
|
+
* that url, a nonce planted through the session's own (pinned) daemon picks
|
|
331
|
+
* the right one — never a guess.
|
|
332
|
+
*/
|
|
333
|
+
async function guestTarget(session, context) {
|
|
334
|
+
const surface = await ensureSurfaceVisible(session, '', context);
|
|
335
|
+
if (!surface) throw new Error(SURFACE_HIDDEN_MESSAGE);
|
|
336
|
+
|
|
337
|
+
const markerText = marker(session);
|
|
338
|
+
const targets = await cdp.listTargets(backend.cdpEndpoint());
|
|
339
|
+
const webviews = targets.filter((t) => t.type === 'webview' && t.webSocketDebuggerUrl);
|
|
340
|
+
|
|
341
|
+
let candidates = webviews.filter((t) => String(t.url || '').includes(markerText)
|
|
342
|
+
|| String(t.title || '').includes(markerText));
|
|
343
|
+
if (!candidates.length && surface.rect?.src) {
|
|
344
|
+
candidates = webviews.filter((t) => String(t.url || '') === surface.rect.src);
|
|
345
|
+
}
|
|
346
|
+
if (candidates.length > 1) candidates = await confirmGuestsByNonce(session, candidates, context);
|
|
347
|
+
if (candidates.length !== 1) {
|
|
348
|
+
throw new Error(
|
|
349
|
+
'Could not identify this session\'s browser surface among the app\'s webviews. '
|
|
350
|
+
+ 'Close this browser session and open again (a fresh surface will mount).',
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
return candidates[0];
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
async function confirmGuestsByNonce(session, candidates, context) {
|
|
357
|
+
const nonce = `amalgm-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
358
|
+
await run(session, ['eval', `void (window.__amalgmSurfaceNonce = ${JSON.stringify(nonce)})`], { context });
|
|
359
|
+
const confirmed = [];
|
|
360
|
+
for (const candidate of candidates) {
|
|
361
|
+
const client = await cdp.connectCdp(candidate.webSocketDebuggerUrl).catch(() => null);
|
|
362
|
+
if (!client) continue;
|
|
363
|
+
try {
|
|
364
|
+
const { result } = await client.call('Runtime.evaluate', {
|
|
365
|
+
expression: 'window.__amalgmSurfaceNonce || null',
|
|
366
|
+
returnByValue: true,
|
|
367
|
+
}, 3_000);
|
|
368
|
+
if (result && result.value === nonce) confirmed.push(candidate);
|
|
369
|
+
} catch {
|
|
370
|
+
// Unreadable candidate — not ours.
|
|
371
|
+
} finally {
|
|
372
|
+
client.close();
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return confirmed;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Tell the UI this session's surface is done — the renderer unmounts the
|
|
380
|
+
* splitview/tab, so ended sessions stop accumulating webviews (each one is
|
|
381
|
+
* a full renderer process).
|
|
382
|
+
*/
|
|
383
|
+
function requestSurfaceClose(session) {
|
|
384
|
+
try {
|
|
385
|
+
const { appendStateEvent } = require('../state/events');
|
|
386
|
+
appendStateEvent({
|
|
387
|
+
resource: SURFACE_EVENT_RESOURCE,
|
|
388
|
+
op: 'delete', // the live client's delete op — drops the entry from the projection
|
|
389
|
+
id: session,
|
|
390
|
+
source: 'browser-engine',
|
|
391
|
+
});
|
|
392
|
+
return true;
|
|
393
|
+
} catch {
|
|
394
|
+
return false;
|
|
395
|
+
}
|
|
264
396
|
}
|
|
265
397
|
|
|
266
398
|
module.exports = {
|
|
267
399
|
SURFACE_EVENT_RESOURCE,
|
|
400
|
+
SURFACE_HIDDEN_MESSAGE,
|
|
268
401
|
WEBVIEW_MARKER_PREFIX,
|
|
269
402
|
cliOptions,
|
|
270
403
|
ensureReady,
|
|
271
404
|
ensureSurfaceVisible,
|
|
405
|
+
guestTarget,
|
|
406
|
+
looksLikeLostConnection,
|
|
272
407
|
marker,
|
|
273
408
|
matchWebview,
|
|
409
|
+
pinFailure,
|
|
274
410
|
readText,
|
|
411
|
+
requestSurfaceClose,
|
|
275
412
|
requestSurfaceOpen,
|
|
276
413
|
run,
|
|
277
414
|
runBatch,
|
|
@@ -59,7 +59,7 @@ module.exports = [
|
|
|
59
59
|
},
|
|
60
60
|
{
|
|
61
61
|
name: 'auth_link_create',
|
|
62
|
-
description: '
|
|
62
|
+
description: 'Start a login the user completes themselves when a site needs them to sign in. In the Amalgm desktop app the login page opens directly in the visible browser surface — tell the user to sign in there. From the web, share the returned link instead. Follow with auth_save to keep the session for later runs.',
|
|
63
63
|
inputSchema: {
|
|
64
64
|
type: 'object',
|
|
65
65
|
properties: {
|
|
@@ -82,16 +82,35 @@ module.exports = [
|
|
|
82
82
|
|| ctx?.callerSessionId
|
|
83
83
|
|| `login-${Date.now().toString(36)}`,
|
|
84
84
|
);
|
|
85
|
+
// In the desktop app, the login happens on the visible surface this
|
|
86
|
+
// session already drives — open the page there and say so. The link
|
|
87
|
+
// is the web/remote path, where a separate live view (noVNC today)
|
|
88
|
+
// shows the headless browser.
|
|
89
|
+
const attached = engine.mode() === 'attached';
|
|
85
90
|
const result = createBrowserLoginSession({
|
|
86
91
|
...args,
|
|
92
|
+
transport: args.transport || (attached ? 'electron-splitview' : undefined),
|
|
87
93
|
browserProfileId: profileId,
|
|
88
94
|
profileId,
|
|
89
95
|
}, { source: 'browser-auth-link-tool' });
|
|
96
|
+
|
|
97
|
+
let opened = false;
|
|
98
|
+
if (attached && args.targetUrl) {
|
|
99
|
+
try {
|
|
100
|
+
await engine.open({ session: profileId, url: args.targetUrl }, ctx);
|
|
101
|
+
opened = true;
|
|
102
|
+
} catch {
|
|
103
|
+
// The surface could not open — the link below still works.
|
|
104
|
+
}
|
|
105
|
+
}
|
|
90
106
|
return jsonText({
|
|
91
107
|
session: result.session,
|
|
92
108
|
authLink: result.authLink,
|
|
93
109
|
authLinkPath: result.authLinkPath,
|
|
94
|
-
|
|
110
|
+
openedInApp: opened,
|
|
111
|
+
message: opened
|
|
112
|
+
? `The login page is open in the Amalgm app's browser panel — ask the user to sign in there, then call auth_save with browserProfileId "${profileId}".`
|
|
113
|
+
: `Share this temporary login link with the user: ${result.authLink}`,
|
|
95
114
|
});
|
|
96
115
|
} catch (err) {
|
|
97
116
|
return errorResult(`auth_link_create failed: ${err.message}`);
|
|
@@ -3,22 +3,24 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* Raster capture — screenshots and recording frames, over raw CDP.
|
|
5
5
|
*
|
|
6
|
-
* Pixels
|
|
7
|
-
*
|
|
6
|
+
* Pixels come straight from the session's own page:
|
|
7
|
+
* - cli (headless) sessions capture their page target;
|
|
8
|
+
* - attached sessions capture the <webview> guest target itself.
|
|
8
9
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
10
|
+
* The one physical law (verified live in all four states): a guest rasters
|
|
11
|
+
* iff its <webview> is composited on-screen in the app's layout. The app
|
|
12
|
+
* window may be hidden or minimized — the desktop app disables background
|
|
13
|
+
* throttling while a surface is attached, so frames keep flowing. Only a
|
|
14
|
+
* surface in a background tab (display:none) cannot produce pixels, and
|
|
15
|
+
* ensureSurfaceVisible self-heals that by asking the UI to refocus the
|
|
16
|
+
* surface tab before any capture is attempted.
|
|
15
17
|
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* History: an earlier revision captured the app window clipped to the
|
|
19
|
+
* webview rectangle, believing guests could not raster at all. That belief
|
|
20
|
+
* came from probing panel-hidden guests, and the clip math corrupted frames
|
|
21
|
+
* whenever the splitview rect moved mid-animation. Guest-direct capture has
|
|
22
|
+
* no rect math to get wrong and returns exactly the page — never the app
|
|
23
|
+
* around it — in both screenshots and recordings.
|
|
22
24
|
*
|
|
23
25
|
* Every exchange carries a hard deadline (cdp.js): a capture may fail, it
|
|
24
26
|
* can never wedge a session.
|
|
@@ -29,51 +31,11 @@ const cli = require('./cli');
|
|
|
29
31
|
const attach = require('./attach');
|
|
30
32
|
const cdp = require('./cdp');
|
|
31
33
|
|
|
32
|
-
//
|
|
33
|
-
// live);
|
|
34
|
-
// deadline turns "compositor is off" from a 20s mystery into a fast, named
|
|
35
|
-
// failure.
|
|
34
|
+
// An on-screen surface answers captureScreenshot in well under 100ms
|
|
35
|
+
// (measured 54–100ms live); one that has not answered in 4s is not going to.
|
|
36
36
|
const SCREENSHOT_TIMEOUT_MS = 4_000;
|
|
37
37
|
const FULL_PAGE_MAX_PX = 16_384;
|
|
38
38
|
|
|
39
|
-
/**
|
|
40
|
-
* Page.captureScreenshot clip (CSS px) for a webview rectangle, clamped to
|
|
41
|
-
* the embedder viewport — a splitview mid-animation or mid-resize can report
|
|
42
|
-
* a rect that overhangs the window, and the overhang captures as black. The
|
|
43
|
-
* capture renders at the device scale factor, so scale divides it back out —
|
|
44
|
-
* the image is CSS-pixel sized and its coordinates map 1:1 onto cua_click/
|
|
45
|
-
* cua_move input coordinates (the contract vision loops rely on).
|
|
46
|
-
*/
|
|
47
|
-
function clipFor(rect) {
|
|
48
|
-
const x = Math.min(Math.max(0, Math.round(rect.x)), Math.max(0, Math.round(rect.vw || rect.x + rect.width) - 1));
|
|
49
|
-
const y = Math.min(Math.max(0, Math.round(rect.y)), Math.max(0, Math.round(rect.vh || rect.y + rect.height) - 1));
|
|
50
|
-
const maxW = Math.max(1, Math.round(rect.vw || rect.x + rect.width) - x);
|
|
51
|
-
const maxH = Math.max(1, Math.round(rect.vh || rect.y + rect.height) - y);
|
|
52
|
-
return {
|
|
53
|
-
x,
|
|
54
|
-
y,
|
|
55
|
-
width: Math.min(Math.max(1, Math.round(rect.width)), maxW),
|
|
56
|
-
height: Math.min(Math.max(1, Math.round(rect.height)), maxH),
|
|
57
|
-
scale: 1 / (rect.dpr || 1),
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* ffmpeg crop for a webview rectangle, expressed in viewport fractions so it
|
|
63
|
-
* holds for whatever resolution the screencast emits (frames arrive in
|
|
64
|
-
* device pixels, the rect is measured in CSS pixels — fractions cancel the
|
|
65
|
-
* scale). Crop happens before any frame reaches disk, so the recording file
|
|
66
|
-
* only ever contains the page, not the app around it.
|
|
67
|
-
*/
|
|
68
|
-
function cropFilterFor(rect) {
|
|
69
|
-
const fraction = (value, total) => Math.min(Math.max(value / total, 0), 1).toFixed(6);
|
|
70
|
-
const w = fraction(rect.width, rect.vw);
|
|
71
|
-
const h = fraction(rect.height, rect.vh);
|
|
72
|
-
const x = fraction(rect.x, rect.vw);
|
|
73
|
-
const y = fraction(rect.y, rect.vh);
|
|
74
|
-
return `crop=floor(iw*${w}/2)*2:floor(ih*${h}/2)*2:floor(iw*${x}):floor(ih*${y})`;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
39
|
/** The cli-mode daemon's CDP url (spawns the daemon on first use). */
|
|
78
40
|
async function daemonCdpUrl(session) {
|
|
79
41
|
const result = await cli.runCli(['get', 'cdp-url'], { session, timeoutMs: 15_000 });
|
|
@@ -82,8 +44,8 @@ async function daemonCdpUrl(session) {
|
|
|
82
44
|
|
|
83
45
|
/**
|
|
84
46
|
* Where to capture from for this session:
|
|
85
|
-
* { wsUrl,
|
|
86
|
-
*
|
|
47
|
+
* { wsUrl, guest } — guest marks an attached <webview> target, whose output
|
|
48
|
+
* is normalized to CSS pixels so coordinates map 1:1 onto cua_* input.
|
|
87
49
|
*/
|
|
88
50
|
async function plan(session, context) {
|
|
89
51
|
if (backend.mode() !== 'attached') {
|
|
@@ -95,7 +57,7 @@ async function plan(session, context) {
|
|
|
95
57
|
if (!page) throw new Error('No capturable page target on the browser endpoint.');
|
|
96
58
|
wsUrl = page.webSocketDebuggerUrl;
|
|
97
59
|
}
|
|
98
|
-
return { wsUrl,
|
|
60
|
+
return { wsUrl, guest: false };
|
|
99
61
|
}
|
|
100
62
|
|
|
101
63
|
await attach.ensureReady(session, context);
|
|
@@ -103,63 +65,57 @@ async function plan(session, context) {
|
|
|
103
65
|
const targets = await cdp.listTargets(endpoint);
|
|
104
66
|
|
|
105
67
|
if (!targets.some((t) => t.type === 'webview')) {
|
|
106
|
-
// Explicit external CDP endpoint (plain chromium):
|
|
107
|
-
//
|
|
68
|
+
// Explicit external CDP endpoint (plain chromium): capture the driven
|
|
69
|
+
// page directly.
|
|
108
70
|
const page = cdp.pageTargets(targets)[0];
|
|
109
71
|
if (!page) throw new Error('No capturable page target on the CDP endpoint.');
|
|
110
|
-
return { wsUrl: page.webSocketDebuggerUrl,
|
|
72
|
+
return { wsUrl: page.webSocketDebuggerUrl, guest: false };
|
|
111
73
|
}
|
|
112
74
|
|
|
113
|
-
// Desktop app: capture
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
)
|
|
124
|
-
|
|
125
|
-
return {
|
|
126
|
-
wsUrl: surface.pageWsUrl,
|
|
127
|
-
clip: clipFor(surface.rect),
|
|
128
|
-
crop: cropFilterFor(surface.rect),
|
|
129
|
-
embedderVisibility: surface.rect.embedderVisibility || null,
|
|
130
|
-
};
|
|
75
|
+
// Desktop app: capture this session's own webview target. The surface tab
|
|
76
|
+
// must be on screen for the guest to raster — ask the UI to refocus it
|
|
77
|
+
// (self-heal) rather than failing on a backgrounded panel.
|
|
78
|
+
const target = await attach.guestTarget(session, context);
|
|
79
|
+
return { wsUrl: target.webSocketDebuggerUrl, guest: true };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Why a guest capture starved, in words an agent can act on. */
|
|
83
|
+
function starvationReason() {
|
|
84
|
+
return 'the browser surface is not rendering. Its tab in the Amalgm app is hidden '
|
|
85
|
+
+ '(the app refocuses it automatically — retry), or the app is gone. '
|
|
86
|
+
+ 'AMALGM_BROWSER_BACKEND=cli forces a headless browser instead';
|
|
131
87
|
}
|
|
132
88
|
|
|
133
89
|
/**
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
90
|
+
* The guest's viewport as a CSS-pixel clip. Captures render at the device
|
|
91
|
+
* scale factor; scale divides it back out so the image is CSS-pixel sized
|
|
92
|
+
* and its coordinates map 1:1 onto cua_click/cua_move input.
|
|
137
93
|
*/
|
|
138
|
-
function
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
|
|
94
|
+
async function cssViewportClip(client) {
|
|
95
|
+
const { result } = await client.call('Runtime.evaluate', {
|
|
96
|
+
expression: 'JSON.stringify({w: innerWidth, h: innerHeight, dpr: devicePixelRatio || 1})',
|
|
97
|
+
returnByValue: true,
|
|
98
|
+
}, SCREENSHOT_TIMEOUT_MS);
|
|
99
|
+
const { w, h, dpr } = JSON.parse(result.value);
|
|
100
|
+
if (!w || !h) throw new Error('The browser surface has no viewport yet — open a page first.');
|
|
101
|
+
return { x: 0, y: 0, width: w, height: h, scale: 1 / dpr };
|
|
145
102
|
}
|
|
146
103
|
|
|
147
104
|
/**
|
|
148
105
|
* Prove the source can produce a pixel before committing to work that needs
|
|
149
106
|
* a stream of them (recording). One tiny capture, hard deadline, named
|
|
150
|
-
* failure —
|
|
151
|
-
* actionable error at start.
|
|
107
|
+
* failure — an actionable refusal at start instead of an empty file at stop.
|
|
152
108
|
*/
|
|
153
109
|
async function assertCapturable(client, source) {
|
|
154
|
-
if (!source.
|
|
110
|
+
if (!source.guest) return; // headless page targets raster unconditionally
|
|
155
111
|
try {
|
|
156
112
|
await client.call('Page.captureScreenshot', {
|
|
157
113
|
format: 'jpeg',
|
|
158
114
|
quality: 10,
|
|
159
|
-
clip: { x:
|
|
115
|
+
clip: { x: 0, y: 0, width: 8, height: 8, scale: 1 },
|
|
160
116
|
}, SCREENSHOT_TIMEOUT_MS);
|
|
161
117
|
} catch (err) {
|
|
162
|
-
throw new Error(`Cannot capture this surface: ${starvationReason(
|
|
118
|
+
throw new Error(`Cannot capture this surface: ${starvationReason()}. (${err.message})`);
|
|
163
119
|
}
|
|
164
120
|
}
|
|
165
121
|
|
|
@@ -169,8 +125,11 @@ async function screenshot(session, args = {}, context) {
|
|
|
169
125
|
const client = await cdp.connectCdp(source.wsUrl);
|
|
170
126
|
try {
|
|
171
127
|
const params = { format: 'png' };
|
|
172
|
-
if (source.
|
|
173
|
-
|
|
128
|
+
if (source.guest || !fullPage) {
|
|
129
|
+
// Every backend normalizes to CSS pixels — a dpr-2 display would
|
|
130
|
+
// otherwise return a 2x image whose coordinates no longer match the
|
|
131
|
+
// cua_* input space.
|
|
132
|
+
params.clip = await cssViewportClip(client);
|
|
174
133
|
} else if (fullPage) {
|
|
175
134
|
const metrics = await client.call('Page.getLayoutMetrics');
|
|
176
135
|
const size = metrics.cssContentSize || metrics.contentSize;
|
|
@@ -189,8 +148,8 @@ async function screenshot(session, args = {}, context) {
|
|
|
189
148
|
try {
|
|
190
149
|
({ data } = await client.call('Page.captureScreenshot', params, SCREENSHOT_TIMEOUT_MS));
|
|
191
150
|
} catch (err) {
|
|
192
|
-
if (source.
|
|
193
|
-
throw new Error(`Screenshot starved: ${starvationReason(
|
|
151
|
+
if (source.guest && /timed out/i.test(String(err.message || ''))) {
|
|
152
|
+
throw new Error(`Screenshot starved: ${starvationReason()}.`);
|
|
194
153
|
}
|
|
195
154
|
throw err;
|
|
196
155
|
}
|
|
@@ -198,7 +157,7 @@ async function screenshot(session, args = {}, context) {
|
|
|
198
157
|
return {
|
|
199
158
|
base64: data,
|
|
200
159
|
bytes: Buffer.byteLength(data, 'base64'),
|
|
201
|
-
mode: source.
|
|
160
|
+
mode: source.guest ? 'visible surface' : (fullPage ? 'full page' : 'viewport'),
|
|
202
161
|
};
|
|
203
162
|
} finally {
|
|
204
163
|
client.close();
|
|
@@ -207,9 +166,8 @@ async function screenshot(session, args = {}, context) {
|
|
|
207
166
|
|
|
208
167
|
module.exports = {
|
|
209
168
|
assertCapturable,
|
|
210
|
-
clipFor,
|
|
211
169
|
connectCdp: cdp.connectCdp,
|
|
212
|
-
|
|
170
|
+
cssViewportClip,
|
|
213
171
|
deadline: cdp.deadline,
|
|
214
172
|
listTargets: cdp.listTargets,
|
|
215
173
|
pageTargets: cdp.pageTargets,
|
|
@@ -123,23 +123,30 @@ function pageTargets(targets) {
|
|
|
123
123
|
* Never "the only webview": adopting an unidentified surface is how one
|
|
124
124
|
* chat's session ends up driving another chat's page.
|
|
125
125
|
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
126
|
+
* A session can match several elements — remounts (chat-tab switches) leave
|
|
127
|
+
* stale 0x0 copies behind — so all matches are collected and the one with
|
|
128
|
+
* on-screen geometry wins. Returning the first match here is how a live,
|
|
129
|
+
* capturable surface used to be reported as "hidden".
|
|
130
|
+
*
|
|
131
|
+
* Returns layout geometry only when a matched element has on-screen size,
|
|
132
|
+
* plus the embedder's visibilityState and the webview's live URL (to re-find
|
|
133
|
+
* the daemon-side target after a restart).
|
|
130
134
|
*/
|
|
131
135
|
function rectScript(markerText, currentUrl) {
|
|
132
136
|
const urlLiteral = JSON.stringify(String(currentUrl || ''));
|
|
133
137
|
return `(() => {
|
|
134
138
|
const views = [...document.querySelectorAll('webview')];
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
+
const matches = [
|
|
140
|
+
...views.filter((w) => (w.getAttribute('data-amalgm-surface') || '').includes(${JSON.stringify(markerText)})),
|
|
141
|
+
...views.filter((w) => (w.getAttribute('src') || '').includes(${JSON.stringify(markerText)})),
|
|
142
|
+
...(${urlLiteral} ? views.filter((w) => (w.src || '') === ${urlLiteral}) : []),
|
|
143
|
+
];
|
|
144
|
+
if (!matches.length) return null;
|
|
145
|
+
const sized = matches.map((w) => ({ w, r: w.getBoundingClientRect() }));
|
|
146
|
+
const match = sized.find(({ r }) => r.width && r.height);
|
|
139
147
|
if (!match) return null;
|
|
140
|
-
const r = match
|
|
141
|
-
|
|
142
|
-
return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, vw: window.innerWidth, vh: window.innerHeight, dpr: window.devicePixelRatio || 1, src: match.src || '', embedderVisibility: document.visibilityState });
|
|
148
|
+
const { w, r } = match;
|
|
149
|
+
return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, vw: window.innerWidth, vh: window.innerHeight, dpr: window.devicePixelRatio || 1, src: w.src || '', embedderVisibility: document.visibilityState });
|
|
143
150
|
})()`;
|
|
144
151
|
}
|
|
145
152
|
|