amalgm 0.1.94 → 0.1.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/browser/attach.js +251 -181
- package/runtime/scripts/amalgm-mcp/browser/capture.js +53 -33
- package/runtime/scripts/amalgm-mcp/browser/cdp.js +3 -0
- package/runtime/scripts/amalgm-mcp/browser/cli.js +10 -3
- package/runtime/scripts/amalgm-mcp/browser/engine.js +2 -2
- package/runtime/scripts/amalgm-mcp/browser/recorder-tools.js +1 -1
- package/runtime/scripts/amalgm-mcp/browser/recorder.js +12 -16
- package/runtime/scripts/amalgm-mcp/browser/sessions.js +8 -3
- package/runtime/scripts/amalgm-mcp/tests/browser-recorder.test.js +6 -2
- package/runtime/scripts/amalgm-mcp/tests/browser-routing.test.js +13 -14
- package/runtime/scripts/amalgm-mcp/tests/browser-transport.test.js +47 -38
package/package.json
CHANGED
|
@@ -4,36 +4,43 @@
|
|
|
4
4
|
* Attached mode — drive the browser surface the user is watching.
|
|
5
5
|
*
|
|
6
6
|
* The desktop app advertises its chromium over CDP; the UI mounts the in-tab
|
|
7
|
-
* split-view <webview> with a marker URL. This module owns the
|
|
7
|
+
* split-view <webview> with a marker URL. This module owns the attach
|
|
8
|
+
* handshake and the command transport, built around one rule:
|
|
8
9
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* 2. Select the session's webview target by marker (fallbacks: previously
|
|
12
|
-
* selected target, then the only webview). Fail closed if none appears —
|
|
13
|
-
* never silently drive the outer app window, never silently go headless.
|
|
14
|
-
* 3. One focus pass — chromium drops CDP Input.* events to a <webview>
|
|
15
|
-
* guest until the guest widget has focus (reads work unfocused; clicks
|
|
16
|
-
* would "succeed" without landing).
|
|
10
|
+
* A session's identity is established ONCE at attach and VERIFIED on every
|
|
11
|
+
* command — never re-asserted by switching tabs.
|
|
17
12
|
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
13
|
+
* Why: the daemon's tab selection silently falls back to "the active tab"
|
|
14
|
+
* when the selected target dies (how one session ends up reading another
|
|
15
|
+
* session's page), but re-selecting the tab per command resets the daemon's
|
|
16
|
+
* snapshot ref table — `click @e4` right after `snapshot` fails "Unknown
|
|
17
|
+
* ref". Both verified live. So instead of a per-command `tab` pin, attach:
|
|
18
|
+
*
|
|
19
|
+
* 1. Asks the UI to open the surface (a browser_surfaces state event).
|
|
20
|
+
* 2. Selects the session's webview in the daemon ONCE (refs don't exist yet).
|
|
21
|
+
* 3. Opens a persistent raw-CDP connection to the guest target and stamps it:
|
|
22
|
+
* `window.__amalgmBsid = session` now, plus the same script on every new
|
|
23
|
+
* document — the stamp survives navigation and dies with the target.
|
|
24
|
+
* 4. Prepends a guard eval to every batch: stamp mismatch throws, the batch
|
|
25
|
+
* bails before the action runs, and the session re-attaches once. eval
|
|
26
|
+
* does not reset refs, so @refs stay live across calls.
|
|
27
|
+
*
|
|
28
|
+
* The persistent guest connection doubles as the capture anchor: screenshots
|
|
29
|
+
* and recordings connect straight to the verified target instead of hunting
|
|
30
|
+
* the target list per capture (the hunt's "no webviews yet → first page
|
|
31
|
+
* target" fallback is how cua_screenshot once captured the app's own chat UI).
|
|
26
32
|
*/
|
|
27
33
|
|
|
28
34
|
const cli = require('./cli');
|
|
29
35
|
const backend = require('./backend');
|
|
30
36
|
const cdp = require('./cdp');
|
|
31
|
-
const {
|
|
37
|
+
const { surfaces, knownSessions } = require('./sessions');
|
|
32
38
|
|
|
33
39
|
const SURFACE_EVENT_RESOURCE = 'browser_surfaces';
|
|
34
40
|
const WEBVIEW_MARKER_PREFIX = 'amalgm-bsid-';
|
|
35
41
|
const WEBVIEW_WAIT_MS = 15_000;
|
|
36
42
|
const WEBVIEW_POLL_MS = 500;
|
|
43
|
+
const GUARD_ERROR = 'amalgm-surface-guard';
|
|
37
44
|
const SURFACE_HIDDEN_MESSAGE = 'The browser surface is hidden in the desktop app and did not reopen. '
|
|
38
45
|
+ 'Retry; if it keeps failing, close this browser session and open again '
|
|
39
46
|
+ '(a fresh surface will mount), or force a headless browser with AMALGM_BROWSER_BACKEND=cli.';
|
|
@@ -42,6 +49,15 @@ function marker(session) {
|
|
|
42
49
|
return `${WEBVIEW_MARKER_PREFIX}${session}`;
|
|
43
50
|
}
|
|
44
51
|
|
|
52
|
+
function stampScript(session) {
|
|
53
|
+
return `window.__amalgmBsid = ${JSON.stringify(session)};`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function guardCommand(session) {
|
|
57
|
+
return ['eval', `(() => { if (window.__amalgmBsid !== ${JSON.stringify(session)}) `
|
|
58
|
+
+ `throw new Error(${JSON.stringify(GUARD_ERROR)}); })()`];
|
|
59
|
+
}
|
|
60
|
+
|
|
45
61
|
/** Options every cli call needs for the current backend. */
|
|
46
62
|
function cliOptions(session, extra = {}) {
|
|
47
63
|
const options = { session, ...extra };
|
|
@@ -70,12 +86,6 @@ function requestSurfaceOpen(session, context) {
|
|
|
70
86
|
}
|
|
71
87
|
}
|
|
72
88
|
|
|
73
|
-
async function listWebviewTabs(session) {
|
|
74
|
-
const tabs = await cli.runCli(['tab'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
75
|
-
const list = tabs?.data?.tabs || tabs?.tabs || [];
|
|
76
|
-
return list.filter((tab) => tab.type === 'webview');
|
|
77
|
-
}
|
|
78
|
-
|
|
79
89
|
/**
|
|
80
90
|
* The session's webview rectangle in the app window, or null when the
|
|
81
91
|
* splitview is hidden (0x0) or gone. Raw CDP, deadline-bounded.
|
|
@@ -111,64 +121,154 @@ async function ensureSurfaceVisible(session, hintUrl, context, waitMs = WEBVIEW_
|
|
|
111
121
|
}
|
|
112
122
|
|
|
113
123
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
124
|
+
* This session's candidate webviews out of a target/tab list. Marker in the
|
|
125
|
+
* url or title identifies fresh surfaces exactly; the live URL the embedder
|
|
126
|
+
* DOM reports for the session's marked element covers surfaces that
|
|
127
|
+
* navigated away from the marker (e.g. after an MCP restart) — but two
|
|
128
|
+
* sessions can browse the same URL, so URL matches are candidates to be
|
|
129
|
+
* confirmed by stamp, never picked blindly. And never "the only webview":
|
|
130
|
+
* a displaced session adopting whatever surface remains is how one chat's
|
|
131
|
+
* browser ends up reading another chat's page.
|
|
122
132
|
*/
|
|
123
|
-
function
|
|
133
|
+
function identityCandidates(list, session, domSrc) {
|
|
124
134
|
const markerText = marker(session);
|
|
125
|
-
const byMarker =
|
|
126
|
-
|| String(
|
|
127
|
-
if (byMarker) return byMarker;
|
|
128
|
-
if (domSrc)
|
|
129
|
-
|
|
130
|
-
|
|
135
|
+
const byMarker = list.filter((item) => String(item.url || '').includes(markerText)
|
|
136
|
+
|| String(item.title || '').includes(markerText));
|
|
137
|
+
if (byMarker.length) return byMarker;
|
|
138
|
+
if (domSrc) return list.filter((item) => String(item.url || '') === domSrc);
|
|
139
|
+
return [];
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** The stamp a target's page carries, read over raw CDP; null if unreadable. */
|
|
143
|
+
async function readStamp(wsUrl) {
|
|
144
|
+
const client = await cdp.connectCdp(wsUrl).catch(() => null);
|
|
145
|
+
if (!client) return null;
|
|
146
|
+
try {
|
|
147
|
+
const { result } = await client.call('Runtime.evaluate', {
|
|
148
|
+
expression: 'window.__amalgmBsid || null',
|
|
149
|
+
returnByValue: true,
|
|
150
|
+
}, 3_000);
|
|
151
|
+
return result?.value ?? null;
|
|
152
|
+
} catch {
|
|
153
|
+
return null;
|
|
154
|
+
} finally {
|
|
155
|
+
client.close();
|
|
131
156
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Resolve same-URL ambiguity: the candidate whose page carries this
|
|
160
|
+
* session's stamp wins. A unique candidate stands on its own. */
|
|
161
|
+
async function confirmGuestCandidates(candidates, session) {
|
|
162
|
+
if (candidates.length <= 1) return candidates[0] || null;
|
|
163
|
+
for (const candidate of candidates) {
|
|
164
|
+
if (await readStamp(candidate.webSocketDebuggerUrl) === session) return candidate;
|
|
136
165
|
}
|
|
137
166
|
return null;
|
|
138
167
|
}
|
|
139
168
|
|
|
140
169
|
/**
|
|
141
|
-
*
|
|
142
|
-
*
|
|
170
|
+
* Stamp the guest and keep the connection open. The on-new-document script
|
|
171
|
+
* lives exactly as long as this CDP session — which is exactly as long as
|
|
172
|
+
* the stamp should live: the connection drops when the target dies (webview
|
|
173
|
+
* remount, app restart), the surface record is dropped with it, and the next
|
|
174
|
+
* command re-attaches against the new reality.
|
|
175
|
+
*/
|
|
176
|
+
async function stampGuest(session, wsUrl) {
|
|
177
|
+
const client = await cdp.connectCdp(wsUrl);
|
|
178
|
+
try {
|
|
179
|
+
await client.call('Page.enable').catch(() => {});
|
|
180
|
+
await client.call('Page.addScriptToEvaluateOnNewDocument', { source: stampScript(session) });
|
|
181
|
+
await client.call('Runtime.evaluate', { expression: stampScript(session), returnByValue: true });
|
|
182
|
+
} catch (err) {
|
|
183
|
+
client.close();
|
|
184
|
+
throw err;
|
|
185
|
+
}
|
|
186
|
+
return client;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Chromium drops CDP Input events to an Electron <webview> guest until the
|
|
191
|
+
* guest widget has focus (verified live: clicks report success but never
|
|
192
|
+
* land). One focus pass over raw CDP — the embedder focuses the element, the
|
|
193
|
+
* guest focuses its window. Best-effort: reads work unfocused regardless.
|
|
194
|
+
*/
|
|
195
|
+
async function focusSurface(session, surface) {
|
|
196
|
+
try {
|
|
197
|
+
if (surface.embedderWsUrl) {
|
|
198
|
+
const embedder = await cdp.connectCdp(surface.embedderWsUrl);
|
|
199
|
+
try {
|
|
200
|
+
await embedder.call('Runtime.evaluate', {
|
|
201
|
+
expression: `(() => {
|
|
202
|
+
const view = [...document.querySelectorAll('webview')]
|
|
203
|
+
.find((w) => (w.getAttribute('data-amalgm-surface') || '').includes(${JSON.stringify(marker(session))}));
|
|
204
|
+
if (view) view.focus();
|
|
205
|
+
return Boolean(view);
|
|
206
|
+
})()`,
|
|
207
|
+
returnByValue: true,
|
|
208
|
+
});
|
|
209
|
+
} finally {
|
|
210
|
+
embedder.close();
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
await surface.client.call('Runtime.evaluate', { expression: 'window.focus()', returnByValue: true });
|
|
214
|
+
} catch {
|
|
215
|
+
// Snapshot/eval still work without focus; input may need a retry.
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Connect the session to the advertised chromium on first use: select the
|
|
221
|
+
* in-tab webview in the daemon, stamp the guest over a persistent raw-CDP
|
|
222
|
+
* connection, focus it. The returned record is the session's one source of
|
|
223
|
+
* identity — commands verify against the stamp, captures connect to the
|
|
224
|
+
* target it names.
|
|
143
225
|
*/
|
|
144
226
|
async function ensureReady(session, context) {
|
|
145
|
-
if (backend.mode() !== 'attached'
|
|
227
|
+
if (backend.mode() !== 'attached') return null;
|
|
228
|
+
const live = surfaces.get(session);
|
|
229
|
+
if (live) return live;
|
|
230
|
+
|
|
231
|
+
// Explicit external CDP endpoints (plain chromium) have page targets, not
|
|
232
|
+
// webviews — drive whatever the daemon selected; no surface, no stamp.
|
|
233
|
+
if (process.env.AMALGM_BROWSER_CDP_URL) {
|
|
234
|
+
const targets = await cdp.listTargets(backend.cdpEndpoint()).catch(() => []);
|
|
235
|
+
if (!targets.some((t) => t.type === 'webview')) {
|
|
236
|
+
const external = { external: true, client: null, tabId: null, wsUrl: null };
|
|
237
|
+
surfaces.set(session, external);
|
|
238
|
+
return external;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
146
241
|
|
|
147
242
|
// The embedder DOM can identify the session's element even after it
|
|
148
243
|
// navigated away from the marker URL (data-amalgm-surface survives where
|
|
149
|
-
// target URLs do not)
|
|
244
|
+
// target URLs do not).
|
|
150
245
|
const domSrc = async () => (await surfaceRect(session, ''))?.rect?.src || null;
|
|
246
|
+
const webviewTabs = async () => {
|
|
247
|
+
const tabs = await cli.runCli(['tab'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
248
|
+
return (tabs?.data?.tabs || tabs?.tabs || []).filter((tab) => tab.type === 'webview');
|
|
249
|
+
};
|
|
151
250
|
|
|
152
|
-
let
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
if (!selected) {
|
|
251
|
+
let candidates = identityCandidates(await webviewTabs(), session, await domSrc());
|
|
252
|
+
if (!candidates.length) {
|
|
156
253
|
requestSurfaceOpen(session, context);
|
|
157
|
-
const
|
|
158
|
-
while (!
|
|
254
|
+
const deadlineAt = Date.now() + WEBVIEW_WAIT_MS;
|
|
255
|
+
while (!candidates.length && Date.now() < deadlineAt) {
|
|
159
256
|
await new Promise((resolve) => setTimeout(resolve, WEBVIEW_POLL_MS));
|
|
160
|
-
|
|
161
|
-
selected = matchWebview(webviews, session, await domSrc());
|
|
257
|
+
candidates = identityCandidates(await webviewTabs(), session, await domSrc());
|
|
162
258
|
}
|
|
163
259
|
}
|
|
164
260
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
261
|
+
// Same-URL candidates are told apart by the stamp their pages carry from a
|
|
262
|
+
// previous attach: pin each in turn and read it back through the daemon.
|
|
263
|
+
let selected = candidates.length === 1 ? candidates[0] : null;
|
|
264
|
+
for (const candidate of candidates) {
|
|
265
|
+
if (selected) break;
|
|
266
|
+
await cli.runCli(['tab', candidate.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
267
|
+
const probe = await cli.runCli(['eval', 'window.__amalgmBsid || null'], cliOptions(session, { timeoutMs: 15_000 }))
|
|
268
|
+
.catch(() => null);
|
|
269
|
+
const stampValue = probe?.data?.result ?? probe?.result ?? null;
|
|
270
|
+
if (stampValue === session) selected = candidate;
|
|
170
271
|
}
|
|
171
|
-
|
|
172
272
|
if (!selected) {
|
|
173
273
|
throw new Error(
|
|
174
274
|
'Browser surface did not open in the desktop app. '
|
|
@@ -177,74 +277,81 @@ async function ensureReady(session, context) {
|
|
|
177
277
|
);
|
|
178
278
|
}
|
|
179
279
|
|
|
180
|
-
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
181
|
-
|
|
182
280
|
// The webview target existing is not enough — the splitview panel must be
|
|
183
281
|
// on screen, or input drops and captures have nothing to render. (The app
|
|
184
282
|
// window itself may be hidden or minimized; that no longer matters — the
|
|
185
283
|
// app keeps an attached surface compositing wherever the window is.)
|
|
186
284
|
const visible = await ensureSurfaceVisible(session, selected.url, context);
|
|
187
|
-
if (!visible)
|
|
188
|
-
throw new Error(SURFACE_HIDDEN_MESSAGE);
|
|
189
|
-
}
|
|
285
|
+
if (!visible) throw new Error(SURFACE_HIDDEN_MESSAGE);
|
|
190
286
|
|
|
191
|
-
|
|
287
|
+
// Select the session's webview in the daemon — once. Refs don't exist yet,
|
|
288
|
+
// so this is the one place a `tab` command is allowed.
|
|
289
|
+
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
290
|
+
|
|
291
|
+
// The guest's raw-CDP target, by the same identity as the daemon tab —
|
|
292
|
+
// stamp-confirmed when several webviews share the URL.
|
|
293
|
+
const guest = await confirmGuestCandidates(identityCandidates(
|
|
294
|
+
(await cdp.listTargets(backend.cdpEndpoint()))
|
|
295
|
+
.filter((t) => t.type === 'webview' && t.webSocketDebuggerUrl),
|
|
296
|
+
session,
|
|
297
|
+
visible.rect?.src || null,
|
|
298
|
+
), session);
|
|
299
|
+
if (!guest) throw new Error(SURFACE_HIDDEN_MESSAGE);
|
|
300
|
+
|
|
301
|
+
const client = await stampGuest(session, guest.webSocketDebuggerUrl);
|
|
302
|
+
const surface = {
|
|
303
|
+
tabId: selected.tabId,
|
|
304
|
+
targetId: guest.id,
|
|
305
|
+
wsUrl: guest.webSocketDebuggerUrl,
|
|
306
|
+
embedderWsUrl: visible.pageWsUrl || null,
|
|
307
|
+
client,
|
|
308
|
+
external: false,
|
|
309
|
+
};
|
|
310
|
+
// The connection dropping IS the detach signal: webview remount, app
|
|
311
|
+
// restart, surface close — all of them invalidate this identity at once.
|
|
312
|
+
client.onClose(() => {
|
|
313
|
+
if (surfaces.get(session) === surface) surfaces.delete(session);
|
|
314
|
+
});
|
|
315
|
+
surfaces.set(session, surface);
|
|
192
316
|
knownSessions.set(session, { ...knownSessions.get(session), webviewTabId: selected.tabId });
|
|
193
|
-
|
|
317
|
+
|
|
318
|
+
await focusSurface(session, surface);
|
|
319
|
+
return surface;
|
|
194
320
|
}
|
|
195
321
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
* focus the <webview> element from the embedder page, then self-focus the
|
|
201
|
-
* guest. Best-effort — failures leave snapshot/eval working regardless.
|
|
202
|
-
*/
|
|
203
|
-
async function focusWebviewHost(session, selected) {
|
|
204
|
-
const markerText = marker(session);
|
|
205
|
-
try {
|
|
206
|
-
const tabs = await cli.runCli(['tab'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
207
|
-
const list = tabs?.data?.tabs || tabs?.tabs || [];
|
|
208
|
-
const outer = list.find((tab) => tab.type === 'page');
|
|
209
|
-
if (outer) {
|
|
210
|
-
const script = `(() => {
|
|
211
|
-
const views = [...document.querySelectorAll('webview')];
|
|
212
|
-
const target = views.find((w) => (w.getAttribute('src') || '').includes(${JSON.stringify(markerText)}))
|
|
213
|
-
|| views.find((w) => (w.src || '') === ${JSON.stringify(String(selected.url || ''))})
|
|
214
|
-
|| views[views.length - 1];
|
|
215
|
-
if (target) target.focus();
|
|
216
|
-
return views.length;
|
|
217
|
-
})()`;
|
|
218
|
-
await cli.runCli(['tab', outer.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
219
|
-
await cli.runCli(['eval', script], cliOptions(session, { timeoutMs: 15_000 }));
|
|
220
|
-
await cli.runCli(['tab', selected.tabId], cliOptions(session, { timeoutMs: 15_000 }));
|
|
221
|
-
}
|
|
222
|
-
await cli.runCli(['eval', 'window.focus()'], cliOptions(session, { timeoutMs: 15_000 }));
|
|
223
|
-
} catch {
|
|
224
|
-
// Snapshot/eval still work without focus; input may need a retry.
|
|
225
|
-
}
|
|
322
|
+
function detach(session) {
|
|
323
|
+
const surface = surfaces.get(session);
|
|
324
|
+
surfaces.delete(session);
|
|
325
|
+
if (surface?.client) { try { surface.client.close(); } catch {} }
|
|
226
326
|
}
|
|
227
327
|
|
|
228
328
|
function looksLikeLostConnection(err) {
|
|
229
|
-
|
|
329
|
+
const message = String(err?.message || err || '');
|
|
330
|
+
return message.includes(GUARD_ERROR) || /clos|connect|target|socket|detach/i.test(message);
|
|
230
331
|
}
|
|
231
332
|
|
|
232
333
|
/**
|
|
233
|
-
*
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
* the
|
|
334
|
+
* Reduce a failed batch to the step that failed. agent-browser reports batch
|
|
335
|
+
* failures two ways — a rejection whose message is the serialized entries, or
|
|
336
|
+
* a success envelope with per-entry failures — and the serialized form
|
|
337
|
+
* contains every command's source text, including the guard's. Classify by
|
|
338
|
+
* the failing entry's error, never by the blob: a failing guard step means
|
|
339
|
+
* the daemon is not on this session's page (re-attach); anything else is
|
|
340
|
+
* that step's own error, reported by name.
|
|
238
341
|
*/
|
|
239
|
-
function
|
|
240
|
-
|
|
342
|
+
function batchFailure(failedEntry, guarded) {
|
|
343
|
+
const stepError = String(failedEntry?.error || 'failed');
|
|
344
|
+
if (guarded && stepError.includes(GUARD_ERROR)) {
|
|
345
|
+
return new Error(`surface target lost (${GUARD_ERROR})`);
|
|
346
|
+
}
|
|
347
|
+
return new Error(`${(failedEntry?.command || []).join(' ')}: ${stepError}`);
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function thrownBatchFailure(err, guarded) {
|
|
241
351
|
try {
|
|
242
352
|
const entries = JSON.parse(String(err?.message || ''));
|
|
243
|
-
const
|
|
244
|
-
|
|
245
|
-
if (failedPin) {
|
|
246
|
-
return new Error(`surface target lost (${failedPin.command.join(' ')}: ${failedPin.error})`);
|
|
247
|
-
}
|
|
353
|
+
const failed = Array.isArray(entries) && entries.find((entry) => entry && entry.success === false);
|
|
354
|
+
if (failed) return batchFailure(failed, guarded);
|
|
248
355
|
} catch {
|
|
249
356
|
// Not a serialized batch — leave the error as-is.
|
|
250
357
|
}
|
|
@@ -253,18 +360,18 @@ function pinFailure(err, pin) {
|
|
|
253
360
|
|
|
254
361
|
/**
|
|
255
362
|
* Execute commands against the session's browser, re-attaching once on a
|
|
256
|
-
* lost
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
363
|
+
* lost surface. In attached mode every batch leads with the stamp guard —
|
|
364
|
+
* the daemon driving any page but this session's fails the batch before the
|
|
365
|
+
* action runs, instead of acting on the wrong page. The guard is an eval, so
|
|
366
|
+
* snapshot @refs survive across calls.
|
|
260
367
|
*/
|
|
261
368
|
async function execute(session, commands, options = {}, single) {
|
|
262
369
|
const { context, ...cliExtra } = options;
|
|
263
|
-
await ensureReady(session, context);
|
|
370
|
+
const surface = await ensureReady(session, context);
|
|
371
|
+
const guarded = Boolean(surface) && !surface.external;
|
|
264
372
|
|
|
265
373
|
const attempt = async () => {
|
|
266
|
-
const
|
|
267
|
-
const batchCommands = pin ? [['tab', pin], ...commands] : commands;
|
|
374
|
+
const batchCommands = guarded ? [guardCommand(session), ...commands] : commands;
|
|
268
375
|
if (batchCommands.length === 1 && single) {
|
|
269
376
|
return cli.runCli(batchCommands[0], cliOptions(session, cliExtra));
|
|
270
377
|
}
|
|
@@ -275,26 +382,23 @@ async function execute(session, commands, options = {}, single) {
|
|
|
275
382
|
stdin: JSON.stringify(batchCommands.map((command) => command.map(String))),
|
|
276
383
|
}));
|
|
277
384
|
} catch (err) {
|
|
278
|
-
throw
|
|
385
|
+
throw thrownBatchFailure(err, guarded);
|
|
279
386
|
}
|
|
280
387
|
let entries = Array.isArray(result) ? result : (result?.data || []);
|
|
281
388
|
if (!Array.isArray(entries)) entries = [];
|
|
282
389
|
const failed = entries.find((entry) => entry && entry.success === false);
|
|
283
|
-
if (failed)
|
|
284
|
-
|
|
285
|
-
const pinFailed = pin && Array.isArray(failed.command) && failed.command[0] === 'tab';
|
|
286
|
-
throw new Error(pinFailed ? `surface target lost (${message})` : message);
|
|
287
|
-
}
|
|
288
|
-
if (pin) entries = entries.slice(1);
|
|
390
|
+
if (failed) throw batchFailure(failed, guarded);
|
|
391
|
+
if (guarded) entries = entries.slice(1);
|
|
289
392
|
return single ? { success: true, data: entries[0]?.result } : entries;
|
|
290
393
|
};
|
|
291
394
|
|
|
292
395
|
try {
|
|
293
396
|
return await attempt();
|
|
294
397
|
} catch (err) {
|
|
295
|
-
if (
|
|
296
|
-
// The app restarted
|
|
297
|
-
|
|
398
|
+
if (guarded && looksLikeLostConnection(err)) {
|
|
399
|
+
// The app restarted, the webview remounted, or the daemon drifted off
|
|
400
|
+
// this session's page; re-attach once against current reality.
|
|
401
|
+
detach(session);
|
|
298
402
|
await ensureReady(session, context);
|
|
299
403
|
return attempt();
|
|
300
404
|
}
|
|
@@ -322,57 +426,20 @@ async function readText(session, commandArgs, timeoutMs) {
|
|
|
322
426
|
}
|
|
323
427
|
|
|
324
428
|
/**
|
|
325
|
-
* The CDP
|
|
326
|
-
* the surface on screen (the one physical
|
|
327
|
-
* ensureSurfaceVisible self-heals a
|
|
328
|
-
*
|
|
329
|
-
* reports for the session's marked element. When several webviews share
|
|
330
|
-
* that url, a nonce planted through the session's own (pinned) daemon picks
|
|
331
|
-
* the right one — never a guess.
|
|
429
|
+
* The CDP websocket url for raster capture — the surface's own guest target,
|
|
430
|
+
* established at attach. Requires the surface on screen (the one physical
|
|
431
|
+
* precondition for guest pixels); ensureSurfaceVisible self-heals a
|
|
432
|
+
* backgrounded tab first. Null for external plain-chromium endpoints.
|
|
332
433
|
*/
|
|
333
|
-
async function
|
|
334
|
-
|
|
335
|
-
if (
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|| String(t.title || '').includes(markerText));
|
|
343
|
-
if (!candidates.length && surface.rect?.src) {
|
|
344
|
-
candidates = webviews.filter((t) => String(t.url || '') === surface.rect.src);
|
|
345
|
-
}
|
|
346
|
-
if (candidates.length > 1) candidates = await confirmGuestsByNonce(session, candidates, context);
|
|
347
|
-
if (candidates.length !== 1) {
|
|
348
|
-
throw new Error(
|
|
349
|
-
'Could not identify this session\'s browser surface among the app\'s webviews. '
|
|
350
|
-
+ 'Close this browser session and open again (a fresh surface will mount).',
|
|
351
|
-
);
|
|
352
|
-
}
|
|
353
|
-
return candidates[0];
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
async function confirmGuestsByNonce(session, candidates, context) {
|
|
357
|
-
const nonce = `amalgm-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
358
|
-
await run(session, ['eval', `void (window.__amalgmSurfaceNonce = ${JSON.stringify(nonce)})`], { context });
|
|
359
|
-
const confirmed = [];
|
|
360
|
-
for (const candidate of candidates) {
|
|
361
|
-
const client = await cdp.connectCdp(candidate.webSocketDebuggerUrl).catch(() => null);
|
|
362
|
-
if (!client) continue;
|
|
363
|
-
try {
|
|
364
|
-
const { result } = await client.call('Runtime.evaluate', {
|
|
365
|
-
expression: 'window.__amalgmSurfaceNonce || null',
|
|
366
|
-
returnByValue: true,
|
|
367
|
-
}, 3_000);
|
|
368
|
-
if (result && result.value === nonce) confirmed.push(candidate);
|
|
369
|
-
} catch {
|
|
370
|
-
// Unreadable candidate — not ours.
|
|
371
|
-
} finally {
|
|
372
|
-
client.close();
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
return confirmed;
|
|
434
|
+
async function guestWsUrl(session, context) {
|
|
435
|
+
let surface = await ensureReady(session, context);
|
|
436
|
+
if (surface?.external) return null;
|
|
437
|
+
const visible = await ensureSurfaceVisible(session, '', context);
|
|
438
|
+
if (!visible) throw new Error(SURFACE_HIDDEN_MESSAGE);
|
|
439
|
+
// The target may have died between ensureReady and now — the close handler
|
|
440
|
+
// clears the record, so a second look re-attaches against the new target.
|
|
441
|
+
surface = surfaces.get(session) || await ensureReady(session, context);
|
|
442
|
+
return surface.wsUrl;
|
|
376
443
|
}
|
|
377
444
|
|
|
378
445
|
/**
|
|
@@ -396,19 +463,22 @@ function requestSurfaceClose(session) {
|
|
|
396
463
|
}
|
|
397
464
|
|
|
398
465
|
module.exports = {
|
|
466
|
+
GUARD_ERROR,
|
|
399
467
|
SURFACE_EVENT_RESOURCE,
|
|
400
468
|
SURFACE_HIDDEN_MESSAGE,
|
|
401
469
|
WEBVIEW_MARKER_PREFIX,
|
|
402
470
|
cliOptions,
|
|
471
|
+
detach,
|
|
403
472
|
ensureReady,
|
|
404
473
|
ensureSurfaceVisible,
|
|
405
|
-
|
|
474
|
+
guardCommand,
|
|
475
|
+
guestWsUrl,
|
|
406
476
|
looksLikeLostConnection,
|
|
407
477
|
marker,
|
|
408
|
-
|
|
409
|
-
pinFailure,
|
|
478
|
+
identityCandidates,
|
|
410
479
|
readText,
|
|
411
480
|
requestSurfaceClose,
|
|
481
|
+
thrownBatchFailure,
|
|
412
482
|
requestSurfaceOpen,
|
|
413
483
|
run,
|
|
414
484
|
runBatch,
|
|
@@ -5,7 +5,10 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Pixels come straight from the session's own page:
|
|
7
7
|
* - cli (headless) sessions capture their page target;
|
|
8
|
-
* - attached sessions capture the <webview> guest target
|
|
8
|
+
* - attached sessions capture the <webview> guest target whose identity was
|
|
9
|
+
* established at attach (attach.guestWsUrl) — never a target-list hunt,
|
|
10
|
+
* never a "first page target" fallback (that fallback is how
|
|
11
|
+
* cua_screenshot once returned the app's own chat UI).
|
|
9
12
|
*
|
|
10
13
|
* The one physical law (verified live in all four states): a guest rasters
|
|
11
14
|
* iff its <webview> is composited on-screen in the app's layout. The app
|
|
@@ -15,13 +18,6 @@
|
|
|
15
18
|
* ensureSurfaceVisible self-heals that by asking the UI to refocus the
|
|
16
19
|
* surface tab before any capture is attempted.
|
|
17
20
|
*
|
|
18
|
-
* History: an earlier revision captured the app window clipped to the
|
|
19
|
-
* webview rectangle, believing guests could not raster at all. That belief
|
|
20
|
-
* came from probing panel-hidden guests, and the clip math corrupted frames
|
|
21
|
-
* whenever the splitview rect moved mid-animation. Guest-direct capture has
|
|
22
|
-
* no rect math to get wrong and returns exactly the page — never the app
|
|
23
|
-
* around it — in both screenshots and recordings.
|
|
24
|
-
*
|
|
25
21
|
* Every exchange carries a hard deadline (cdp.js): a capture may fail, it
|
|
26
22
|
* can never wedge a session.
|
|
27
23
|
*/
|
|
@@ -42,6 +38,13 @@ async function daemonCdpUrl(session) {
|
|
|
42
38
|
return result?.data?.cdpUrl || result?.cdpUrl || null;
|
|
43
39
|
}
|
|
44
40
|
|
|
41
|
+
/** First capturable page target on a CDP endpoint (browser-level url or port). */
|
|
42
|
+
async function pageTargetWsUrl(endpoint) {
|
|
43
|
+
const page = cdp.pageTargets(await cdp.listTargets(endpoint))[0];
|
|
44
|
+
if (!page) throw new Error('No capturable page target on the CDP endpoint.');
|
|
45
|
+
return page.webSocketDebuggerUrl;
|
|
46
|
+
}
|
|
47
|
+
|
|
45
48
|
/**
|
|
46
49
|
* Where to capture from for this session:
|
|
47
50
|
* { wsUrl, guest } — guest marks an attached <webview> target, whose output
|
|
@@ -51,32 +54,16 @@ async function plan(session, context) {
|
|
|
51
54
|
if (backend.mode() !== 'attached') {
|
|
52
55
|
const raw = await daemonCdpUrl(session);
|
|
53
56
|
if (!raw) throw new Error('Could not resolve a CDP endpoint for this browser session.');
|
|
54
|
-
|
|
55
|
-
if (/\/devtools\/browser\//.test(raw)) {
|
|
56
|
-
const page = cdp.pageTargets(await cdp.listTargets(raw))[0];
|
|
57
|
-
if (!page) throw new Error('No capturable page target on the browser endpoint.');
|
|
58
|
-
wsUrl = page.webSocketDebuggerUrl;
|
|
59
|
-
}
|
|
57
|
+
const wsUrl = /\/devtools\/browser\//.test(raw) ? await pageTargetWsUrl(raw) : raw;
|
|
60
58
|
return { wsUrl, guest: false };
|
|
61
59
|
}
|
|
62
60
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if (
|
|
68
|
-
|
|
69
|
-
// page directly.
|
|
70
|
-
const page = cdp.pageTargets(targets)[0];
|
|
71
|
-
if (!page) throw new Error('No capturable page target on the CDP endpoint.');
|
|
72
|
-
return { wsUrl: page.webSocketDebuggerUrl, guest: false };
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// Desktop app: capture this session's own webview target. The surface tab
|
|
76
|
-
// must be on screen for the guest to raster — ask the UI to refocus it
|
|
77
|
-
// (self-heal) rather than failing on a backgrounded panel.
|
|
78
|
-
const target = await attach.guestTarget(session, context);
|
|
79
|
-
return { wsUrl: target.webSocketDebuggerUrl, guest: true };
|
|
61
|
+
// Desktop app: the session's own webview guest, identity from attach.
|
|
62
|
+
// Null only for explicit external CDP endpoints (plain chromium) — there
|
|
63
|
+
// the driven page is the capture source.
|
|
64
|
+
const wsUrl = await attach.guestWsUrl(session, context);
|
|
65
|
+
if (wsUrl) return { wsUrl, guest: true };
|
|
66
|
+
return { wsUrl: await pageTargetWsUrl(backend.cdpEndpoint()), guest: false };
|
|
80
67
|
}
|
|
81
68
|
|
|
82
69
|
/** Why a guest capture starved, in words an agent can act on. */
|
|
@@ -119,6 +106,35 @@ async function assertCapturable(client, source) {
|
|
|
119
106
|
}
|
|
120
107
|
}
|
|
121
108
|
|
|
109
|
+
/**
|
|
110
|
+
* Electron <webview> guests raster over a transparent base, so a page
|
|
111
|
+
* without its own background captures with alpha — which viewers and
|
|
112
|
+
* encoders composite as black. (CDP's background override is a no-op on
|
|
113
|
+
* guests; verified live.) Flatten guest PNGs onto white with the bundled
|
|
114
|
+
* ffmpeg — the pixels are intact under the alpha. Headless page targets
|
|
115
|
+
* raster opaque and skip this entirely.
|
|
116
|
+
*/
|
|
117
|
+
function flattenPng(buffer) {
|
|
118
|
+
const { spawn } = require('child_process');
|
|
119
|
+
return cdp.deadline(new Promise((resolve) => {
|
|
120
|
+
const child = spawn(cli.ffmpegBinary(), [
|
|
121
|
+
'-y', '-f', 'image2pipe', '-c:v', 'png', '-i', 'pipe:0',
|
|
122
|
+
'-filter_complex', '[0:v]format=rgba,split[fg1][fg2];[fg1]drawbox=color=white:t=fill[bg];'
|
|
123
|
+
+ '[bg][fg2]overlay=format=auto,format=rgb24',
|
|
124
|
+
'-frames:v', '1', '-f', 'image2pipe', '-c:v', 'png', 'pipe:1',
|
|
125
|
+
], { stdio: ['pipe', 'pipe', 'ignore'] });
|
|
126
|
+
const chunks = [];
|
|
127
|
+
child.stdout.on('data', (chunk) => chunks.push(chunk));
|
|
128
|
+
child.on('error', () => resolve(buffer)); // no ffmpeg — alpha beats no image
|
|
129
|
+
child.on('close', (code) => {
|
|
130
|
+
const out = Buffer.concat(chunks);
|
|
131
|
+
resolve(code === 0 && out.length ? out : buffer);
|
|
132
|
+
});
|
|
133
|
+
child.stdin.on('error', () => {});
|
|
134
|
+
child.stdin.end(buffer);
|
|
135
|
+
}), 10_000, 'screenshot flatten').catch(() => buffer);
|
|
136
|
+
}
|
|
137
|
+
|
|
122
138
|
async function screenshot(session, args = {}, context) {
|
|
123
139
|
const fullPage = Boolean(args.fullPage);
|
|
124
140
|
const source = await plan(session, context);
|
|
@@ -154,9 +170,12 @@ async function screenshot(session, args = {}, context) {
|
|
|
154
170
|
throw err;
|
|
155
171
|
}
|
|
156
172
|
if (!data) throw new Error('Empty screenshot response');
|
|
173
|
+
const image = source.guest
|
|
174
|
+
? await flattenPng(Buffer.from(data, 'base64'))
|
|
175
|
+
: Buffer.from(data, 'base64');
|
|
157
176
|
return {
|
|
158
|
-
base64:
|
|
159
|
-
bytes:
|
|
177
|
+
base64: image.toString('base64'),
|
|
178
|
+
bytes: image.length,
|
|
160
179
|
mode: source.guest ? 'visible surface' : (fullPage ? 'full page' : 'viewport'),
|
|
161
180
|
};
|
|
162
181
|
} finally {
|
|
@@ -169,6 +188,7 @@ module.exports = {
|
|
|
169
188
|
connectCdp: cdp.connectCdp,
|
|
170
189
|
cssViewportClip,
|
|
171
190
|
deadline: cdp.deadline,
|
|
191
|
+
flattenPng,
|
|
172
192
|
listTargets: cdp.listTargets,
|
|
173
193
|
pageTargets: cdp.pageTargets,
|
|
174
194
|
plan,
|
|
@@ -57,9 +57,11 @@ async function connectCdp(wsUrl) {
|
|
|
57
57
|
}
|
|
58
58
|
if (msg.method && listeners.has(msg.method)) listeners.get(msg.method)(msg.params || {});
|
|
59
59
|
});
|
|
60
|
+
const closeHandlers = [];
|
|
60
61
|
ws.on('close', () => {
|
|
61
62
|
for (const entry of pending.values()) entry.reject(new Error('CDP connection closed'));
|
|
62
63
|
pending.clear();
|
|
64
|
+
for (const handler of closeHandlers.splice(0)) { try { handler(); } catch {} }
|
|
63
65
|
});
|
|
64
66
|
ws.on('error', () => {});
|
|
65
67
|
|
|
@@ -80,6 +82,7 @@ async function connectCdp(wsUrl) {
|
|
|
80
82
|
try { ws.send(JSON.stringify({ id: ++nextId, method, params })); } catch {}
|
|
81
83
|
},
|
|
82
84
|
on(method, handler) { listeners.set(method, handler); },
|
|
85
|
+
onClose(handler) { closeHandlers.push(handler); },
|
|
83
86
|
close() { try { ws.close(); } catch {} },
|
|
84
87
|
};
|
|
85
88
|
}
|
|
@@ -60,12 +60,18 @@ function socketDir() {
|
|
|
60
60
|
return ensureDir(path.join(os.tmpdir(), 'amalgm-ab'));
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
function
|
|
63
|
+
function ffmpegBinary() {
|
|
64
|
+
if (process.env.AMALGM_FFMPEG) return process.env.AMALGM_FFMPEG;
|
|
64
65
|
try {
|
|
65
66
|
const resolved = require('ffmpeg-static');
|
|
66
|
-
if (resolved && fs.existsSync(resolved)) return
|
|
67
|
+
if (resolved && fs.existsSync(resolved)) return resolved;
|
|
67
68
|
} catch {}
|
|
68
|
-
return
|
|
69
|
+
return 'ffmpeg';
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function ffmpegDir() {
|
|
73
|
+
const binary = ffmpegBinary();
|
|
74
|
+
return binary === 'ffmpeg' ? null : path.dirname(binary);
|
|
69
75
|
}
|
|
70
76
|
|
|
71
77
|
/**
|
|
@@ -235,6 +241,7 @@ module.exports = {
|
|
|
235
241
|
asText,
|
|
236
242
|
browserRoot,
|
|
237
243
|
ensureDir,
|
|
244
|
+
ffmpegBinary,
|
|
238
245
|
ffmpegDir,
|
|
239
246
|
profileDir,
|
|
240
247
|
readText,
|
|
@@ -317,10 +317,10 @@ async function close(args = {}, context) {
|
|
|
317
317
|
// Tell the UI to unmount the splitview/tab. Each surface is a full
|
|
318
318
|
// renderer process; ended sessions must not accumulate them (ten zombie
|
|
319
319
|
// webviews were live in the app when this was written).
|
|
320
|
-
if (sessions.
|
|
320
|
+
if (sessions.surfaces.has(session) || sessions.knownSessions.has(session)) {
|
|
321
321
|
attach.requestSurfaceClose(session);
|
|
322
322
|
}
|
|
323
|
-
|
|
323
|
+
attach.detach(session);
|
|
324
324
|
sessions.knownSessions.delete(session);
|
|
325
325
|
}
|
|
326
326
|
|
|
@@ -50,7 +50,7 @@ module.exports = [
|
|
|
50
50
|
content: [{
|
|
51
51
|
type: 'text',
|
|
52
52
|
text: `Recording saved: ${result.path}\n`
|
|
53
|
-
+ `${result.frames} frames @ ${result.fps} fps
|
|
53
|
+
+ `${result.frames} frames @ ${result.fps} fps — ${result.videoSeconds}s of video over ${result.wallSeconds}s, ${result.bytes} bytes`
|
|
54
54
|
+ (result.warning ? `\nWarning: ${result.warning}` : ''),
|
|
55
55
|
}],
|
|
56
56
|
metadata: {
|
|
@@ -89,15 +89,6 @@ function createSampler({ fps, write }) {
|
|
|
89
89
|
return sampler;
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
function ffmpegBinary() {
|
|
93
|
-
if (process.env.AMALGM_FFMPEG) return process.env.AMALGM_FFMPEG;
|
|
94
|
-
try {
|
|
95
|
-
const resolved = require('ffmpeg-static');
|
|
96
|
-
if (resolved && fs.existsSync(resolved)) return resolved;
|
|
97
|
-
} catch {}
|
|
98
|
-
return 'ffmpeg';
|
|
99
|
-
}
|
|
100
|
-
|
|
101
92
|
let verifiedBinary = null;
|
|
102
93
|
|
|
103
94
|
/**
|
|
@@ -109,7 +100,7 @@ let verifiedBinary = null;
|
|
|
109
100
|
* server and every session's tools with it.
|
|
110
101
|
*/
|
|
111
102
|
function ensureFfmpeg() {
|
|
112
|
-
const binary = ffmpegBinary();
|
|
103
|
+
const binary = cli.ffmpegBinary();
|
|
113
104
|
if (binary === verifiedBinary) return binary;
|
|
114
105
|
const probe = require('child_process').spawnSync(binary, ['-version'], { stdio: 'ignore', timeout: 10_000 });
|
|
115
106
|
if (probe.error || probe.status !== 0) {
|
|
@@ -152,8 +143,8 @@ function recordingFile(args, context, session) {
|
|
|
152
143
|
|
|
153
144
|
function ffmpegArgs({ fps, filters, file }) {
|
|
154
145
|
return [
|
|
155
|
-
'-y', '-f', 'image2pipe', '-c:v', '
|
|
156
|
-
'-
|
|
146
|
+
'-y', '-f', 'image2pipe', '-c:v', 'png', '-framerate', String(fps), '-i', 'pipe:0',
|
|
147
|
+
'-filter_complex', filters,
|
|
157
148
|
'-c:v', 'libvpx',
|
|
158
149
|
'-deadline', 'realtime',
|
|
159
150
|
'-cpu-used', '8',
|
|
@@ -161,7 +152,6 @@ function ffmpegArgs({ fps, filters, file }) {
|
|
|
161
152
|
'-auto-alt-ref', '0',
|
|
162
153
|
'-threads', '4',
|
|
163
154
|
'-b:v', '1M',
|
|
164
|
-
'-pix_fmt', 'yuv420p',
|
|
165
155
|
file,
|
|
166
156
|
];
|
|
167
157
|
}
|
|
@@ -182,8 +172,13 @@ async function start(args = {}, context = {}) {
|
|
|
182
172
|
|
|
183
173
|
const file = recordingFile(args, context, session);
|
|
184
174
|
const fps = clampFps(args.fps);
|
|
185
|
-
//
|
|
186
|
-
|
|
175
|
+
// Frames arrive as PNG with alpha: Electron <webview> guests raster over a
|
|
176
|
+
// transparent base, so a page without its own background would encode as
|
|
177
|
+
// black. Composite every frame onto white before the encoder sees it
|
|
178
|
+
// (opaque headless frames pass through unchanged), then pad to the even
|
|
179
|
+
// dimensions VP8 wants.
|
|
180
|
+
const filters = '[0:v]format=rgba,split[fg1][fg2];[fg1]drawbox=color=white:t=fill[bg];'
|
|
181
|
+
+ '[bg][fg2]overlay=format=auto,pad=ceil(iw/2)*2:ceil(ih/2)*2,format=yuv420p';
|
|
187
182
|
|
|
188
183
|
const ffmpeg = spawn(binary, ffmpegArgs({ fps, filters, file }), { stdio: ['pipe', 'ignore', 'pipe'] });
|
|
189
184
|
let ffmpegErr = '';
|
|
@@ -230,12 +225,13 @@ async function start(args = {}, context = {}) {
|
|
|
230
225
|
// A surface that cannot produce one pixel now will not produce a stream
|
|
231
226
|
// either — refuse at start with the reason, not at stop with an empty file.
|
|
232
227
|
await capture.assertCapturable(recording.cdp, source);
|
|
228
|
+
|
|
233
229
|
recording.cdp.on('Page.screencastFrame', (params) => {
|
|
234
230
|
recording.sampler.push(Buffer.from(params.data, 'base64'));
|
|
235
231
|
recording.cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId });
|
|
236
232
|
});
|
|
237
233
|
await recording.cdp.call('Page.enable').catch(() => {});
|
|
238
|
-
await recording.cdp.call('Page.startScreencast', { format: '
|
|
234
|
+
await recording.cdp.call('Page.startScreencast', { format: 'png', everyNthFrame: 1 });
|
|
239
235
|
recording.sampler.start();
|
|
240
236
|
} catch (err) {
|
|
241
237
|
recording.sampler.stop();
|
|
@@ -12,8 +12,13 @@
|
|
|
12
12
|
const cli = require('./cli');
|
|
13
13
|
const backend = require('./backend');
|
|
14
14
|
|
|
15
|
-
/**
|
|
16
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Attached surfaces: session -> { tabId, targetId, wsUrl, embedderWsUrl,
|
|
17
|
+
* client, external }. An entry exists exactly while its guest CDP connection
|
|
18
|
+
* is open — the connection's close handler removes it, so presence means
|
|
19
|
+
* live. attach.js owns the lifecycle.
|
|
20
|
+
*/
|
|
21
|
+
const surfaces = new Map();
|
|
17
22
|
|
|
18
23
|
/** Per-session bookkeeping: webviewTabId, lastActiveAt, last url/title. */
|
|
19
24
|
const knownSessions = new Map();
|
|
@@ -56,9 +61,9 @@ function sessionInfo(session, extra = {}) {
|
|
|
56
61
|
}
|
|
57
62
|
|
|
58
63
|
module.exports = {
|
|
59
|
-
attachedSessions,
|
|
60
64
|
knownSessions,
|
|
61
65
|
rememberProfile,
|
|
62
66
|
sessionFor,
|
|
63
67
|
sessionInfo,
|
|
68
|
+
surfaces,
|
|
64
69
|
};
|
|
@@ -91,10 +91,14 @@ test('sampler skips ticks under encoder backpressure and resumes on drain', () =
|
|
|
91
91
|
assert.equal(sampler.stats.encoded, 2);
|
|
92
92
|
});
|
|
93
93
|
|
|
94
|
-
test('recording encoder uses realtime VP8 settings', () => {
|
|
94
|
+
test('recording encoder uses realtime VP8 settings on PNG frames', () => {
|
|
95
95
|
const { _test } = require('../browser/recorder');
|
|
96
|
-
const args = _test.ffmpegArgs({ fps: 12, filters: '
|
|
96
|
+
const args = _test.ffmpegArgs({ fps: 12, filters: 'null', file: '/tmp/out.webm' });
|
|
97
97
|
|
|
98
|
+
// PNG input keeps the alpha channel so the white underlay in the
|
|
99
|
+
// filtergraph can composite it (guests raster transparent backgrounds).
|
|
100
|
+
assert.equal(args[args.indexOf('-c:v') + 1], 'png');
|
|
101
|
+
assert.ok(args.includes('-filter_complex'));
|
|
98
102
|
assert.ok(args.includes('-deadline'));
|
|
99
103
|
assert.equal(args[args.indexOf('-deadline') + 1], 'realtime');
|
|
100
104
|
assert.ok(args.includes('-cpu-used'));
|
|
@@ -138,28 +138,27 @@ test('entrypoint routing: candidate order is state dir, derived label dir, AMALG
|
|
|
138
138
|
// a chat-tab switch) fell back to "the only webview" — someone else's.
|
|
139
139
|
// ---------------------------------------------------------------------------
|
|
140
140
|
|
|
141
|
-
test('surface routing: marker, embedder DOM src
|
|
141
|
+
test('surface routing: marker first, then the embedder DOM src as candidates', () => {
|
|
142
142
|
const attach = require('../browser/attach');
|
|
143
|
-
const { knownSessions } = require('../browser/sessions');
|
|
144
143
|
|
|
145
144
|
const fresh = { tabId: 't2', type: 'webview', url: 'about:blank#amalgm-bsid-s1' };
|
|
146
145
|
const navigated = { tabId: 't3', type: 'webview', url: 'https://example.com/' };
|
|
146
|
+
const twin = { tabId: 't5', type: 'webview', url: 'https://example.com/' };
|
|
147
147
|
const other = { tabId: 't4', type: 'webview', url: 'https://other.test/' };
|
|
148
148
|
|
|
149
|
-
// Fresh surface: the marker URL identifies it.
|
|
150
|
-
assert.
|
|
149
|
+
// Fresh surface: the marker URL identifies it exactly.
|
|
150
|
+
assert.deepEqual(attach.identityCandidates([other, fresh], 's1', null), [fresh]);
|
|
151
151
|
|
|
152
152
|
// Navigated surface: the embedder DOM (data-amalgm-surface element) reports
|
|
153
153
|
// its live src — that re-keys the daemon's tab list after an MCP restart.
|
|
154
|
-
assert.
|
|
154
|
+
assert.deepEqual(attach.identityCandidates([other, navigated], 's1', 'https://example.com/'), [navigated]);
|
|
155
155
|
|
|
156
|
-
//
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
}
|
|
156
|
+
// Two sessions on the same URL: both are candidates — ensureReady confirms
|
|
157
|
+
// the right one by the stamp its page carries, never by list order.
|
|
158
|
+
assert.deepEqual(
|
|
159
|
+
attach.identityCandidates([other, navigated, twin], 's1', 'https://example.com/'),
|
|
160
|
+
[navigated, twin],
|
|
161
|
+
);
|
|
163
162
|
});
|
|
164
163
|
|
|
165
164
|
test('surface routing: an unidentified lone webview is never adopted', () => {
|
|
@@ -167,6 +166,6 @@ test('surface routing: an unidentified lone webview is never adopted', () => {
|
|
|
167
166
|
// One webview left and it is not provably ours — that is the cross-chat
|
|
168
167
|
// leak shape. Fail closed; ensureReady reopens a marked surface instead.
|
|
169
168
|
const stranger = { tabId: 't9', type: 'webview', url: 'https://another-chats-page.test/' };
|
|
170
|
-
assert.
|
|
171
|
-
assert.
|
|
169
|
+
assert.deepEqual(attach.identityCandidates([stranger], 'displaced-session', null), []);
|
|
170
|
+
assert.deepEqual(attach.identityCandidates([], 'displaced-session', null), []);
|
|
172
171
|
});
|
|
@@ -3,14 +3,17 @@
|
|
|
3
3
|
const assert = require('node:assert/strict');
|
|
4
4
|
const test = require('node:test');
|
|
5
5
|
|
|
6
|
-
// Transport semantics
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
6
|
+
// Transport semantics: in attached mode a session's identity is established
|
|
7
|
+
// once at attach (daemon tab selection + a stamp on the guest document) and
|
|
8
|
+
// VERIFIED on every command by a guard eval leading the batch. The previous
|
|
9
|
+
// design re-pinned with a `tab` step per batch — but `tab` resets the
|
|
10
|
+
// daemon's snapshot ref table (verified live, even re-selecting the same
|
|
11
|
+
// tab), which broke the documented snapshot → click @ref flow. eval does not
|
|
12
|
+
// touch refs, so the guard keeps them alive across calls while still failing
|
|
13
|
+
// closed when the daemon drifts onto another page.
|
|
11
14
|
|
|
12
15
|
const attach = require('../browser/attach');
|
|
13
|
-
const {
|
|
16
|
+
const { surfaces, knownSessions } = require('../browser/sessions');
|
|
14
17
|
|
|
15
18
|
// Patch attach.run before engine destructures it (engine binds at require).
|
|
16
19
|
const calls = [];
|
|
@@ -75,15 +78,15 @@ test('passthrough refuses tab hops in attached mode, allows listing', async () =
|
|
|
75
78
|
}
|
|
76
79
|
});
|
|
77
80
|
|
|
78
|
-
// The
|
|
79
|
-
//
|
|
80
|
-
//
|
|
81
|
+
// The guard itself: execute() turns every attached call into a batch that
|
|
82
|
+
// leads with the stamp-check eval, and maps the single-command result back to
|
|
83
|
+
// its unbatched shape. The guard never appears in caller-visible results.
|
|
81
84
|
|
|
82
|
-
test('attached commands
|
|
85
|
+
test('attached commands lead with the guard eval inside the batch', async (t) => {
|
|
83
86
|
const fs = require('node:fs');
|
|
84
87
|
const os = require('node:os');
|
|
85
88
|
const path = require('node:path');
|
|
86
|
-
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'amalgm-
|
|
89
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'amalgm-guard-test-'));
|
|
87
90
|
const stub = path.join(dir, 'stub.sh');
|
|
88
91
|
// The stub answers `batch` by echoing stdin back as batch entries.
|
|
89
92
|
fs.writeFileSync(stub, `#!/bin/sh
|
|
@@ -100,18 +103,19 @@ console.log(JSON.stringify({ success: true, data: cmds.map((c) => ({ command: c,
|
|
|
100
103
|
process.env.AMALGM_AGENT_BROWSER_BIN = stub;
|
|
101
104
|
process.env.AMALGM_BROWSER_BACKEND = 'attached';
|
|
102
105
|
process.env.AMALGM_BROWSER_CDP_URL = '9242';
|
|
103
|
-
|
|
104
|
-
|
|
106
|
+
// A live surface record makes ensureReady a no-op and turns the guard on.
|
|
107
|
+
surfaces.set('guard-test', { tabId: 't7', wsUrl: 'ws://x/guest', client: { close() {} }, external: false });
|
|
108
|
+
knownSessions.set('guard-test', { webviewTabId: 't7' });
|
|
105
109
|
try {
|
|
106
|
-
const single = await attach.run('
|
|
110
|
+
const single = await attach.run('guard-test', ['get', 'url'], { timeoutMs: 5000 });
|
|
107
111
|
assert.deepEqual(single.data, { echoed: ['get', 'url'] }, 'single result keeps its unbatched shape');
|
|
108
112
|
|
|
109
|
-
const entries = await attach.runBatch('
|
|
110
|
-
assert.equal(entries.length, 2, 'the
|
|
113
|
+
const entries = await attach.runBatch('guard-test', [['open', 'https://example.com'], ['get', 'url']], { timeoutMs: 5000 });
|
|
114
|
+
assert.equal(entries.length, 2, 'the guard step is internal — callers see their own commands only');
|
|
111
115
|
assert.deepEqual(entries[0].command, ['open', 'https://example.com']);
|
|
112
116
|
} finally {
|
|
113
|
-
|
|
114
|
-
knownSessions.delete('
|
|
117
|
+
surfaces.delete('guard-test');
|
|
118
|
+
knownSessions.delete('guard-test');
|
|
115
119
|
if (savedBin === undefined) delete process.env.AMALGM_AGENT_BROWSER_BIN; else process.env.AMALGM_AGENT_BROWSER_BIN = savedBin;
|
|
116
120
|
if (savedBackend === undefined) delete process.env.AMALGM_BROWSER_BACKEND; else process.env.AMALGM_BROWSER_BACKEND = savedBackend;
|
|
117
121
|
if (savedCdp === undefined) delete process.env.AMALGM_BROWSER_CDP_URL; else process.env.AMALGM_BROWSER_CDP_URL = savedCdp;
|
|
@@ -119,37 +123,42 @@ console.log(JSON.stringify({ success: true, data: cmds.map((c) => ({ command: c,
|
|
|
119
123
|
}
|
|
120
124
|
});
|
|
121
125
|
|
|
122
|
-
test('a
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
|
|
127
|
-
assert.equal(attach.looksLikeLostConnection(new Error('surface target lost (tab t7: tab not found)')), true);
|
|
126
|
+
test('a guard failure is classified as a lost target; page errors are not', () => {
|
|
127
|
+
// Guard failures and dead connections re-attach. Ordinary page errors must
|
|
128
|
+
// NOT look like lost connections — a spurious re-attach on "Element not
|
|
129
|
+
// found" would tear down and rebuild the surface on every typo'd selector.
|
|
130
|
+
assert.equal(attach.looksLikeLostConnection(new Error(`surface target lost (${attach.GUARD_ERROR})`)), true);
|
|
128
131
|
assert.equal(attach.looksLikeLostConnection(new Error('WebSocket connection closed')), true);
|
|
129
132
|
assert.equal(attach.looksLikeLostConnection(new Error('Element not found: #submit')), false);
|
|
130
133
|
assert.equal(attach.looksLikeLostConnection(new Error('Timed out waiting for selector')), false);
|
|
131
134
|
});
|
|
132
135
|
|
|
133
|
-
test('
|
|
136
|
+
test('batch failures classify by the failing step, never by the serialized blob', () => {
|
|
134
137
|
// agent-browser --bail rejects with the serialized entries as the message
|
|
135
|
-
// (live-verified shape)
|
|
136
|
-
//
|
|
137
|
-
|
|
138
|
-
|
|
138
|
+
// (live-verified shape) — and that serialization contains the guard
|
|
139
|
+
// command's own source text. Only a guard step that actually FAILED may
|
|
140
|
+
// re-attach; any other failing step is that step's own error.
|
|
141
|
+
const guard = attach.guardCommand('s1');
|
|
142
|
+
|
|
143
|
+
const guardFailed = new Error(JSON.stringify([
|
|
144
|
+
{ command: guard, error: `Evaluation error: Error: ${attach.GUARD_ERROR}\n at <anonymous>:1:54`, result: null, success: false },
|
|
139
145
|
]));
|
|
140
|
-
const classified = attach.
|
|
146
|
+
const classified = attach.thrownBatchFailure(guardFailed, true);
|
|
141
147
|
assert.match(classified.message, /surface target lost/);
|
|
142
148
|
assert.equal(attach.looksLikeLostConnection(classified), true);
|
|
143
149
|
|
|
144
|
-
// A failing page step
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
{ command:
|
|
150
|
+
// A failing page step after a PASSING guard stays a page error — even
|
|
151
|
+
// though the blob contains the guard text.
|
|
152
|
+
const pageFailed = new Error(JSON.stringify([
|
|
153
|
+
{ command: guard, error: null, result: { result: null }, success: true },
|
|
154
|
+
{ command: ['eval', 'return 21 * 2'], error: 'Evaluation error: SyntaxError: Illegal return statement', result: null, success: false },
|
|
148
155
|
]));
|
|
149
|
-
|
|
156
|
+
const page = attach.thrownBatchFailure(pageFailed, true);
|
|
157
|
+
assert.match(page.message, /Illegal return statement/);
|
|
158
|
+
assert.doesNotMatch(page.message, /surface target lost/);
|
|
159
|
+
assert.equal(attach.looksLikeLostConnection(page), false);
|
|
150
160
|
|
|
151
|
-
//
|
|
161
|
+
// Non-JSON messages pass through untouched.
|
|
152
162
|
const plain = new Error('agent-browser exited with code 1');
|
|
153
|
-
assert.equal(attach.
|
|
154
|
-
assert.equal(attach.pinFailure(plain, 't16'), plain);
|
|
163
|
+
assert.equal(attach.thrownBatchFailure(plain, true), plain);
|
|
155
164
|
});
|