amalgm 0.1.92 → 0.1.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/browser/attach.js +45 -5
- package/runtime/scripts/amalgm-mcp/browser/capture.js +63 -9
- package/runtime/scripts/amalgm-mcp/browser/cdp.js +17 -7
- package/runtime/scripts/amalgm-mcp/browser/cli.js +29 -5
- package/runtime/scripts/amalgm-mcp/browser/engine.js +76 -41
- package/runtime/scripts/amalgm-mcp/browser/recorder-tools.js +2 -1
- package/runtime/scripts/amalgm-mcp/browser/recorder.js +136 -16
- package/runtime/scripts/amalgm-mcp/tests/browser-capture.test.js +35 -2
- package/runtime/scripts/amalgm-mcp/tests/browser-cli.test.js +58 -0
- package/runtime/scripts/amalgm-mcp/tests/browser-recorder.test.js +65 -0
- package/runtime/scripts/amalgm-mcp/tests/browser-routing.test.js +39 -0
package/package.json
CHANGED
|
@@ -102,17 +102,30 @@ async function ensureSurfaceVisible(session, hintUrl, context, waitMs = WEBVIEW_
|
|
|
102
102
|
return null;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
|
|
105
|
+
/**
|
|
106
|
+
* Pick this session's webview target from the daemon's tab list. Marker URL
|
|
107
|
+
* (fresh surfaces), previously selected tab id (live surfaces), or the live
|
|
108
|
+
* URL the embedder DOM reports for the session's marked element (surfaces
|
|
109
|
+
* that navigated away from the marker — e.g. after an MCP restart).
|
|
110
|
+
*
|
|
111
|
+
* Never "the only webview". A displaced session adopting whatever surface
|
|
112
|
+
* remains is how one chat's browser ends up reading another chat's page —
|
|
113
|
+
* fail closed and let ensureReady reopen a marked surface instead.
|
|
114
|
+
*/
|
|
115
|
+
function matchWebview(webviews, session, domSrc) {
|
|
106
116
|
const markerText = marker(session);
|
|
107
117
|
const byMarker = webviews.find((tab) => String(tab.url || '').includes(markerText)
|
|
108
118
|
|| String(tab.title || '').includes(markerText));
|
|
109
119
|
if (byMarker) return byMarker;
|
|
120
|
+
if (domSrc) {
|
|
121
|
+
const byDom = webviews.find((tab) => String(tab.url || '') === domSrc);
|
|
122
|
+
if (byDom) return byDom;
|
|
123
|
+
}
|
|
110
124
|
const previous = knownSessions.get(session)?.webviewTabId;
|
|
111
125
|
if (previous) {
|
|
112
126
|
const byPrevious = webviews.find((tab) => tab.tabId === previous);
|
|
113
127
|
if (byPrevious) return byPrevious;
|
|
114
128
|
}
|
|
115
|
-
if (webviews.length === 1) return webviews[0];
|
|
116
129
|
return null;
|
|
117
130
|
}
|
|
118
131
|
|
|
@@ -123,8 +136,13 @@ function matchWebview(webviews, session) {
|
|
|
123
136
|
async function ensureReady(session, context) {
|
|
124
137
|
if (backend.mode() !== 'attached' || attachedSessions.has(session)) return;
|
|
125
138
|
|
|
139
|
+
// The embedder DOM can identify the session's element even after it
|
|
140
|
+
// navigated away from the marker URL (data-amalgm-surface survives where
|
|
141
|
+
// target URLs do not) — its live src re-keys the daemon's tab list.
|
|
142
|
+
const domSrc = async () => (await surfaceRect(session, ''))?.rect?.src || null;
|
|
143
|
+
|
|
126
144
|
let webviews = await listWebviewTabs(session);
|
|
127
|
-
let selected = matchWebview(webviews, session);
|
|
145
|
+
let selected = matchWebview(webviews, session, await domSrc());
|
|
128
146
|
|
|
129
147
|
if (!selected) {
|
|
130
148
|
requestSurfaceOpen(session, context);
|
|
@@ -132,7 +150,7 @@ async function ensureReady(session, context) {
|
|
|
132
150
|
while (!selected && Date.now() < deadline) {
|
|
133
151
|
await new Promise((resolve) => setTimeout(resolve, WEBVIEW_POLL_MS));
|
|
134
152
|
webviews = await listWebviewTabs(session);
|
|
135
|
-
selected = matchWebview(webviews, session);
|
|
153
|
+
selected = matchWebview(webviews, session, await domSrc());
|
|
136
154
|
}
|
|
137
155
|
}
|
|
138
156
|
|
|
@@ -146,7 +164,8 @@ async function ensureReady(session, context) {
|
|
|
146
164
|
if (!selected) {
|
|
147
165
|
throw new Error(
|
|
148
166
|
'Browser surface did not open in the desktop app. '
|
|
149
|
-
+ 'Retry
|
|
167
|
+
+ 'Retry; if it keeps failing, close this browser session and open again '
|
|
168
|
+
+ '(a fresh surface will mount), or force a headless browser with AMALGM_BROWSER_BACKEND=cli.',
|
|
150
169
|
);
|
|
151
170
|
}
|
|
152
171
|
|
|
@@ -219,6 +238,26 @@ async function run(session, commandArgs, options = {}) {
|
|
|
219
238
|
}
|
|
220
239
|
}
|
|
221
240
|
|
|
241
|
+
/**
|
|
242
|
+
* Several commands, one daemon round trip — composites (mouse click = move +
|
|
243
|
+
* down + up) cost one process spawn instead of three. Commands go as JSON on
|
|
244
|
+
* stdin (argv-quoting of nested commands is how arguments get eaten).
|
|
245
|
+
* --bail: a failed step fails the batch; its error is thrown with the step
|
|
246
|
+
* named. Returns the per-command result array.
|
|
247
|
+
*/
|
|
248
|
+
async function runBatch(session, commands, options = {}) {
|
|
249
|
+
const result = await run(session, ['batch', '--bail'], {
|
|
250
|
+
...options,
|
|
251
|
+
stdin: JSON.stringify(commands.map((command) => command.map(String))),
|
|
252
|
+
});
|
|
253
|
+
const entries = Array.isArray(result) ? result : (result?.data || []);
|
|
254
|
+
const failed = Array.isArray(entries) ? entries.find((entry) => entry && entry.success === false) : null;
|
|
255
|
+
if (failed) {
|
|
256
|
+
throw new Error(`${(failed.command || []).join(' ')}: ${failed.error || 'failed'}`);
|
|
257
|
+
}
|
|
258
|
+
return entries;
|
|
259
|
+
}
|
|
260
|
+
|
|
222
261
|
async function readText(session, commandArgs, timeoutMs) {
|
|
223
262
|
await ensureReady(session);
|
|
224
263
|
return cli.readText(commandArgs, session, timeoutMs, backend.mode() === 'attached' ? backend.cdpEndpoint() : null);
|
|
@@ -235,4 +274,5 @@ module.exports = {
|
|
|
235
274
|
readText,
|
|
236
275
|
requestSurfaceOpen,
|
|
237
276
|
run,
|
|
277
|
+
runBatch,
|
|
238
278
|
};
|
|
@@ -29,21 +29,31 @@ const cli = require('./cli');
|
|
|
29
29
|
const attach = require('./attach');
|
|
30
30
|
const cdp = require('./cdp');
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
// Visible surfaces answer captureScreenshot in well under 100ms (verified
|
|
33
|
+
// live); a surface that has not answered in 4s is not going to. The short
|
|
34
|
+
// deadline turns "compositor is off" from a 20s mystery into a fast, named
|
|
35
|
+
// failure.
|
|
36
|
+
const SCREENSHOT_TIMEOUT_MS = 4_000;
|
|
33
37
|
const FULL_PAGE_MAX_PX = 16_384;
|
|
34
38
|
|
|
35
39
|
/**
|
|
36
|
-
* Page.captureScreenshot clip (CSS px) for a webview rectangle
|
|
37
|
-
*
|
|
38
|
-
*
|
|
40
|
+
* Page.captureScreenshot clip (CSS px) for a webview rectangle, clamped to
|
|
41
|
+
* the embedder viewport — a splitview mid-animation or mid-resize can report
|
|
42
|
+
* a rect that overhangs the window, and the overhang captures as black. The
|
|
43
|
+
* capture renders at the device scale factor, so scale divides it back out —
|
|
44
|
+
* the image is CSS-pixel sized and its coordinates map 1:1 onto cua_click/
|
|
39
45
|
* cua_move input coordinates (the contract vision loops rely on).
|
|
40
46
|
*/
|
|
41
47
|
function clipFor(rect) {
|
|
48
|
+
const x = Math.min(Math.max(0, Math.round(rect.x)), Math.max(0, Math.round(rect.vw || rect.x + rect.width) - 1));
|
|
49
|
+
const y = Math.min(Math.max(0, Math.round(rect.y)), Math.max(0, Math.round(rect.vh || rect.y + rect.height) - 1));
|
|
50
|
+
const maxW = Math.max(1, Math.round(rect.vw || rect.x + rect.width) - x);
|
|
51
|
+
const maxH = Math.max(1, Math.round(rect.vh || rect.y + rect.height) - y);
|
|
42
52
|
return {
|
|
43
|
-
x
|
|
44
|
-
y
|
|
45
|
-
width: Math.max(1, Math.round(rect.width)),
|
|
46
|
-
height: Math.max(1, Math.round(rect.height)),
|
|
53
|
+
x,
|
|
54
|
+
y,
|
|
55
|
+
width: Math.min(Math.max(1, Math.round(rect.width)), maxW),
|
|
56
|
+
height: Math.min(Math.max(1, Math.round(rect.height)), maxH),
|
|
47
57
|
scale: 1 / (rect.dpr || 1),
|
|
48
58
|
};
|
|
49
59
|
}
|
|
@@ -116,9 +126,43 @@ async function plan(session, context) {
|
|
|
116
126
|
wsUrl: surface.pageWsUrl,
|
|
117
127
|
clip: clipFor(surface.rect),
|
|
118
128
|
crop: cropFilterFor(surface.rect),
|
|
129
|
+
embedderVisibility: surface.rect.embedderVisibility || null,
|
|
119
130
|
};
|
|
120
131
|
}
|
|
121
132
|
|
|
133
|
+
/**
|
|
134
|
+
* Why a capture on this source just starved, in words an agent can act on.
|
|
135
|
+
* Geometry alone cannot tell: a hidden or minimized app window keeps full
|
|
136
|
+
* layout while the compositor draws nothing.
|
|
137
|
+
*/
|
|
138
|
+
function starvationReason(source) {
|
|
139
|
+
if (source.embedderVisibility === 'hidden') {
|
|
140
|
+
return 'the Amalgm app window is hidden or minimized, so the page is not rendering. '
|
|
141
|
+
+ 'Bring the Amalgm window back on screen and retry';
|
|
142
|
+
}
|
|
143
|
+
return 'the surface stopped rendering (the app window may be hidden, minimized, or on another desktop). '
|
|
144
|
+
+ 'Bring the Amalgm window forward and retry';
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Prove the source can produce a pixel before committing to work that needs
|
|
149
|
+
* a stream of them (recording). One tiny capture, hard deadline, named
|
|
150
|
+
* failure — this is the difference between "0-frame file after 145s" and an
|
|
151
|
+
* actionable error at start.
|
|
152
|
+
*/
|
|
153
|
+
async function assertCapturable(client, source) {
|
|
154
|
+
if (!source.clip) return; // page targets raster unconditionally
|
|
155
|
+
try {
|
|
156
|
+
await client.call('Page.captureScreenshot', {
|
|
157
|
+
format: 'jpeg',
|
|
158
|
+
quality: 10,
|
|
159
|
+
clip: { x: source.clip.x, y: source.clip.y, width: 8, height: 8, scale: 1 },
|
|
160
|
+
}, SCREENSHOT_TIMEOUT_MS);
|
|
161
|
+
} catch (err) {
|
|
162
|
+
throw new Error(`Cannot capture this surface: ${starvationReason(source)}. (${err.message})`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
122
166
|
async function screenshot(session, args = {}, context) {
|
|
123
167
|
const fullPage = Boolean(args.fullPage);
|
|
124
168
|
const source = await plan(session, context);
|
|
@@ -141,7 +185,15 @@ async function screenshot(session, args = {}, context) {
|
|
|
141
185
|
params.captureBeyondViewport = true;
|
|
142
186
|
}
|
|
143
187
|
}
|
|
144
|
-
|
|
188
|
+
let data;
|
|
189
|
+
try {
|
|
190
|
+
({ data } = await client.call('Page.captureScreenshot', params, SCREENSHOT_TIMEOUT_MS));
|
|
191
|
+
} catch (err) {
|
|
192
|
+
if (source.clip && /timed out/i.test(String(err.message || ''))) {
|
|
193
|
+
throw new Error(`Screenshot starved: ${starvationReason(source)}.`);
|
|
194
|
+
}
|
|
195
|
+
throw err;
|
|
196
|
+
}
|
|
145
197
|
if (!data) throw new Error('Empty screenshot response');
|
|
146
198
|
return {
|
|
147
199
|
base64: data,
|
|
@@ -154,6 +206,7 @@ async function screenshot(session, args = {}, context) {
|
|
|
154
206
|
}
|
|
155
207
|
|
|
156
208
|
module.exports = {
|
|
209
|
+
assertCapturable,
|
|
157
210
|
clipFor,
|
|
158
211
|
connectCdp: cdp.connectCdp,
|
|
159
212
|
cropFilterFor,
|
|
@@ -163,5 +216,6 @@ module.exports = {
|
|
|
163
216
|
plan,
|
|
164
217
|
rectScript: cdp.rectScript,
|
|
165
218
|
screenshot,
|
|
219
|
+
starvationReason,
|
|
166
220
|
targetListUrl: cdp.targetListUrl,
|
|
167
221
|
};
|
|
@@ -113,23 +113,33 @@ function pageTargets(targets) {
|
|
|
113
113
|
}
|
|
114
114
|
|
|
115
115
|
/**
|
|
116
|
-
* Locate the session's <webview> element inside a candidate app window
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
116
|
+
* Locate the session's <webview> element inside a candidate app window.
|
|
117
|
+
* Identity, strongest first:
|
|
118
|
+
* - data-amalgm-surface attribute — stamped once at mount, survives
|
|
119
|
+
* navigation (the src attribute is rewritten on every navigate, so the
|
|
120
|
+
* marker URL it starts with does not last);
|
|
121
|
+
* - marker still in the src attribute — fresh, never-navigated surfaces;
|
|
122
|
+
* - exact current URL — old-app fallback, needs a fresh hint.
|
|
123
|
+
* Never "the only webview": adopting an unidentified surface is how one
|
|
124
|
+
* chat's session ends up driving another chat's page.
|
|
125
|
+
*
|
|
126
|
+
* Returns layout geometry only when the element has on-screen size, plus the
|
|
127
|
+
* embedder's visibilityState (a hidden app window keeps full geometry while
|
|
128
|
+
* rastering nothing) and the webview's live URL (to re-find the daemon-side
|
|
129
|
+
* target after a restart).
|
|
121
130
|
*/
|
|
122
131
|
function rectScript(markerText, currentUrl) {
|
|
123
132
|
const urlLiteral = JSON.stringify(String(currentUrl || ''));
|
|
124
133
|
return `(() => {
|
|
125
134
|
const views = [...document.querySelectorAll('webview')];
|
|
135
|
+
const byAttr = views.find((w) => (w.getAttribute('data-amalgm-surface') || '').includes(${JSON.stringify(markerText)}));
|
|
126
136
|
const byMarker = views.find((w) => (w.getAttribute('src') || '').includes(${JSON.stringify(markerText)}));
|
|
127
137
|
const byUrl = ${urlLiteral} ? views.find((w) => (w.src || '') === ${urlLiteral}) : null;
|
|
128
|
-
const match =
|
|
138
|
+
const match = byAttr || byMarker || byUrl;
|
|
129
139
|
if (!match) return null;
|
|
130
140
|
const r = match.getBoundingClientRect();
|
|
131
141
|
if (!r.width || !r.height) return null;
|
|
132
|
-
return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, vw: window.innerWidth, vh: window.innerHeight, dpr: window.devicePixelRatio || 1 });
|
|
142
|
+
return JSON.stringify({ x: r.x, y: r.y, width: r.width, height: r.height, vw: window.innerWidth, vh: window.innerHeight, dpr: window.devicePixelRatio || 1, src: match.src || '', embedderVisibility: document.visibilityState });
|
|
133
143
|
})()`;
|
|
134
144
|
}
|
|
135
145
|
|
|
@@ -68,15 +68,34 @@ function ffmpegDir() {
|
|
|
68
68
|
return null;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* agent-browser ships prebuilt native binaries next to its node wrapper; the
|
|
73
|
+
* wrapper only exists to pick one. Spawning the binary directly saves a node
|
|
74
|
+
* startup (~100ms) on every command — on the hot path, that is the tool's
|
|
75
|
+
* floor latency.
|
|
76
|
+
*/
|
|
77
|
+
function nativeAgentBrowser(wrapperPath) {
|
|
78
|
+
const platforms = { darwin: 'darwin', linux: 'linux', win32: 'win32' };
|
|
79
|
+
const arches = { x64: 'x64', arm64: 'arm64' };
|
|
80
|
+
const platform = platforms[os.platform()];
|
|
81
|
+
const arch = arches[os.arch()];
|
|
82
|
+
if (!platform || !arch) return null;
|
|
83
|
+
const musl = platform === 'linux'
|
|
84
|
+
&& (fs.existsSync('/lib/ld-musl-x86_64.so.1') || fs.existsSync('/lib/ld-musl-aarch64.so.1'));
|
|
85
|
+
const name = `agent-browser-${platform}${musl ? '-musl' : ''}-${arch}${platform === 'win32' ? '.exe' : ''}`;
|
|
86
|
+
const candidate = path.join(path.dirname(wrapperPath), name);
|
|
87
|
+
return fs.existsSync(candidate) ? candidate : null;
|
|
88
|
+
}
|
|
89
|
+
|
|
71
90
|
function agentBrowserCommand() {
|
|
72
91
|
if (process.env.AMALGM_AGENT_BROWSER_BIN) {
|
|
73
92
|
return { command: process.env.AMALGM_AGENT_BROWSER_BIN, prefix: [] };
|
|
74
93
|
}
|
|
75
94
|
try {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
};
|
|
95
|
+
const wrapper = require.resolve('agent-browser/bin/agent-browser.js');
|
|
96
|
+
const native = nativeAgentBrowser(wrapper);
|
|
97
|
+
if (native) return { command: native, prefix: [] };
|
|
98
|
+
return { command: process.execPath, prefix: [wrapper] };
|
|
80
99
|
} catch {
|
|
81
100
|
return { command: 'agent-browser', prefix: [] };
|
|
82
101
|
}
|
|
@@ -167,8 +186,12 @@ function runCli(commandArgs, options = {}) {
|
|
|
167
186
|
const child = spawn(bin.command, args, {
|
|
168
187
|
cwd: os.homedir(),
|
|
169
188
|
env,
|
|
170
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
189
|
+
stdio: [options.stdin != null ? 'pipe' : 'ignore', 'pipe', 'pipe'],
|
|
171
190
|
});
|
|
191
|
+
if (options.stdin != null) {
|
|
192
|
+
child.stdin.on('error', () => {});
|
|
193
|
+
child.stdin.end(String(options.stdin));
|
|
194
|
+
}
|
|
172
195
|
|
|
173
196
|
let stdout = '';
|
|
174
197
|
let stderr = '';
|
|
@@ -207,6 +230,7 @@ async function readText(commandArgs, session, timeoutMs = 30_000, cdp = null) {
|
|
|
207
230
|
|
|
208
231
|
module.exports = {
|
|
209
232
|
DEFAULT_SESSION,
|
|
233
|
+
agentBrowserCommand,
|
|
210
234
|
amalgmDir,
|
|
211
235
|
asText,
|
|
212
236
|
browserRoot,
|
|
@@ -22,7 +22,7 @@ const capture = require('./capture');
|
|
|
22
22
|
const sessions = require('./sessions');
|
|
23
23
|
|
|
24
24
|
const { sessionFor, sessionInfo } = sessions;
|
|
25
|
-
const { run, readText } = attach;
|
|
25
|
+
const { run, runBatch, readText } = attach;
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Target grammar shared by click/fill:
|
|
@@ -41,26 +41,40 @@ function findArgs(target, action, value) {
|
|
|
41
41
|
return command;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/** url + title in one daemon round trip, batchable onto any verb. */
|
|
45
|
+
const STATE_COMMAND = ['eval', 'JSON.stringify({url: location.href, title: document.title})'];
|
|
46
|
+
|
|
47
|
+
function parseState(entryOrResult) {
|
|
48
|
+
const data = entryOrResult?.result ?? entryOrResult?.data ?? entryOrResult;
|
|
49
|
+
const raw = data && typeof data === 'object' && 'result' in data ? data.result : data;
|
|
50
|
+
try {
|
|
51
|
+
const { url = '', title = '' } = JSON.parse(String(raw));
|
|
52
|
+
return { url, title };
|
|
53
|
+
} catch {
|
|
54
|
+
return { url: '', title: '' };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
44
58
|
async function state(args = {}, context) {
|
|
45
59
|
const session = sessionFor(args, context);
|
|
46
|
-
let
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
try { title = await readText(session, ['get', 'title']); } catch {}
|
|
50
|
-
return sessionInfo(session, { url, title });
|
|
60
|
+
let current = { url: '', title: '' };
|
|
61
|
+
try { current = parseState(await run(session, STATE_COMMAND, { context })); } catch {}
|
|
62
|
+
return sessionInfo(session, current);
|
|
51
63
|
}
|
|
52
64
|
|
|
53
65
|
async function open(args = {}, context) {
|
|
54
66
|
const session = sessionFor(args, context);
|
|
55
|
-
await
|
|
56
|
-
return
|
|
67
|
+
const entries = await runBatch(session, [['open', args.url], STATE_COMMAND], { context });
|
|
68
|
+
return sessionInfo(session, parseState(entries[1]));
|
|
57
69
|
}
|
|
58
70
|
|
|
59
71
|
async function snapshot(args = {}, context) {
|
|
60
72
|
const session = sessionFor(args, context);
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
|
|
73
|
+
const entries = await runBatch(session, [['snapshot', '-i'], STATE_COMMAND], { timeoutMs: 30_000, context });
|
|
74
|
+
return {
|
|
75
|
+
...sessionInfo(session, parseState(entries[1])),
|
|
76
|
+
snapshotText: cli.asText(entries[0]?.result ?? entries[0]),
|
|
77
|
+
};
|
|
64
78
|
}
|
|
65
79
|
|
|
66
80
|
async function screenshot(args = {}, context) {
|
|
@@ -82,10 +96,12 @@ async function fill(args = {}, context) {
|
|
|
82
96
|
const session = sessionFor(args, context);
|
|
83
97
|
const target = String(args.target || '');
|
|
84
98
|
const text = String(args.text ?? '');
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
99
|
+
const fillCommand = target.startsWith('text=') ? findArgs(target, 'fill', text) : ['fill', target, text];
|
|
100
|
+
if (args.submit) {
|
|
101
|
+
await runBatch(session, [fillCommand, ['press', 'Enter']], { context });
|
|
102
|
+
} else {
|
|
103
|
+
await run(session, fillCommand, { context });
|
|
104
|
+
}
|
|
89
105
|
return sessionInfo(session, { filled: target, submitted: Boolean(args.submit) });
|
|
90
106
|
}
|
|
91
107
|
|
|
@@ -157,32 +173,38 @@ async function cua(action, args = {}, context) {
|
|
|
157
173
|
|
|
158
174
|
const x = Math.round(Number(args.x || 0));
|
|
159
175
|
const y = Math.round(Number(args.y || 0));
|
|
176
|
+
// Composites go as one batch — one daemon round trip per gesture, not one
|
|
177
|
+
// per mouse event.
|
|
160
178
|
if (action === 'cua_move') await run(session, ['mouse', 'move', x, y], options);
|
|
161
179
|
if (action === 'cua_click') {
|
|
162
180
|
const button = mouseButton(args.button);
|
|
163
|
-
await
|
|
164
|
-
|
|
165
|
-
|
|
181
|
+
await runBatch(session, [
|
|
182
|
+
['mouse', 'move', x, y],
|
|
183
|
+
['mouse', 'down', button],
|
|
184
|
+
['mouse', 'up', button],
|
|
185
|
+
], options);
|
|
166
186
|
}
|
|
167
187
|
if (action === 'cua_double_click') {
|
|
168
|
-
await run(session, ['mouse', 'move', x, y], options);
|
|
169
|
-
for (let i = 0; i < 2; i += 1) {
|
|
170
|
-
await run(session, ['mouse', 'down', 'left'], options);
|
|
171
|
-
await run(session, ['mouse', 'up', 'left'], options);
|
|
172
|
-
}
|
|
173
188
|
// agent-browser's mouse events always carry clickCount:1, so the renderer
|
|
174
|
-
// never coalesces the
|
|
189
|
+
// never coalesces the two real clicks into a native dblclick. Dispatch the
|
|
175
190
|
// dblclick event directly at the point; the two real clicks before it
|
|
176
191
|
// keep focus/selection side effects honest.
|
|
177
|
-
await
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
192
|
+
await runBatch(session, [
|
|
193
|
+
['mouse', 'move', x, y],
|
|
194
|
+
['mouse', 'down', 'left'],
|
|
195
|
+
['mouse', 'up', 'left'],
|
|
196
|
+
['mouse', 'down', 'left'],
|
|
197
|
+
['mouse', 'up', 'left'],
|
|
198
|
+
['eval', `(() => {
|
|
199
|
+
const el = document.elementFromPoint(${x}, ${y});
|
|
200
|
+
if (!el) return false;
|
|
201
|
+
el.dispatchEvent(new MouseEvent('dblclick', {
|
|
202
|
+
bubbles: true, cancelable: true, view: window,
|
|
203
|
+
clientX: ${x}, clientY: ${y}, button: 0, detail: 2,
|
|
204
|
+
}));
|
|
205
|
+
return true;
|
|
206
|
+
})()`],
|
|
207
|
+
], options);
|
|
186
208
|
}
|
|
187
209
|
if (action === 'cua_scroll') await run(session, ['mouse', 'wheel', args.scrollY || 0, args.scrollX || 0], options);
|
|
188
210
|
if (action === 'cua_type') await run(session, ['keyboard', 'type', args.text || ''], options);
|
|
@@ -190,10 +212,12 @@ async function cua(action, args = {}, context) {
|
|
|
190
212
|
if (action === 'cua_drag') {
|
|
191
213
|
const points = Array.isArray(args.path) ? args.path : [];
|
|
192
214
|
if (points.length < 2) throw new Error('drag path must contain at least two points');
|
|
193
|
-
await
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
215
|
+
await runBatch(session, [
|
|
216
|
+
['mouse', 'move', points[0].x, points[0].y],
|
|
217
|
+
['mouse', 'down', 'left'],
|
|
218
|
+
...points.slice(1).map((point) => ['mouse', 'move', point.x, point.y]),
|
|
219
|
+
['mouse', 'up', 'left'],
|
|
220
|
+
], options);
|
|
197
221
|
}
|
|
198
222
|
return sessionInfo(session, { ok: true });
|
|
199
223
|
}
|
|
@@ -246,10 +270,21 @@ async function loadState(args = {}, context) {
|
|
|
246
270
|
|
|
247
271
|
async function close(args = {}, context) {
|
|
248
272
|
const session = sessionFor(args, context);
|
|
249
|
-
//
|
|
250
|
-
//
|
|
251
|
-
|
|
252
|
-
|
|
273
|
+
// A recording keeps a CDP connection and an encoder alive — closing the
|
|
274
|
+
// session finalizes it (saving what was captured) rather than orphaning it.
|
|
275
|
+
const recorder = require('./recorder');
|
|
276
|
+
if (recorder.isActive(session)) {
|
|
277
|
+
try { await recorder.stop({ ...args, session }, context); } catch {}
|
|
278
|
+
}
|
|
279
|
+
// Tear down this session's daemon. In attached mode this only disconnects
|
|
280
|
+
// from the app's chromium (verified live: the app, its webviews, and other
|
|
281
|
+
// sessions' daemons survive) — the visible surface belongs to the UI and
|
|
282
|
+
// stays open for the user. Without this, every attached session leaks a
|
|
283
|
+
// daemon process for the life of the machine.
|
|
284
|
+
try {
|
|
285
|
+
await cli.runCli(['close'], attach.cliOptions(session, { timeoutMs: 15_000 }));
|
|
286
|
+
} catch {
|
|
287
|
+
// No daemon (never started, or already gone) — nothing to tear down.
|
|
253
288
|
}
|
|
254
289
|
sessions.attachedSessions.delete(session);
|
|
255
290
|
sessions.knownSessions.delete(session);
|
|
@@ -50,7 +50,8 @@ module.exports = [
|
|
|
50
50
|
content: [{
|
|
51
51
|
type: 'text',
|
|
52
52
|
text: `Recording saved: ${result.path}\n`
|
|
53
|
-
+ `${result.frames} frames, ${Math.round(result.durationMs / 1000)}s, ${result.bytes} bytes
|
|
53
|
+
+ `${result.frames} frames @ ${result.fps} fps, ${Math.round(result.durationMs / 1000)}s, ${result.bytes} bytes`
|
|
54
|
+
+ (result.warning ? `\nWarning: ${result.warning}` : ''),
|
|
54
55
|
}],
|
|
55
56
|
metadata: {
|
|
56
57
|
browserVideo: {
|
|
@@ -30,10 +30,66 @@ const backend = require('./backend');
|
|
|
30
30
|
const capture = require('./capture');
|
|
31
31
|
const sessions = require('./sessions');
|
|
32
32
|
|
|
33
|
-
const active = new Map(); // session -> { cdp, ffmpeg, file, frames, startedAt }
|
|
33
|
+
const active = new Map(); // session -> { cdp, ffmpeg, file, frames, droppedFrames, startedAt }
|
|
34
34
|
|
|
35
35
|
const FFMPEG_CLOSE_TIMEOUT_MS = 10_000;
|
|
36
36
|
|
|
37
|
+
function clampFps(value) {
|
|
38
|
+
return Math.min(Math.max(Number(value) || 10, 1), 30);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Paint-on-change, encode-on-clock.
|
|
43
|
+
*
|
|
44
|
+
* The screencast only emits on visual damage: a static page yields one frame
|
|
45
|
+
* per minute, an animated canvas yields sixty per second. Feeding that
|
|
46
|
+
* stream to ffmpeg directly produces dishonest files at both extremes — a
|
|
47
|
+
* "145-second" recording holding one frame, or an encoder drowning in input
|
|
48
|
+
* and getting killed at stop. So frames land in a one-deep latest slot, and
|
|
49
|
+
* a clock writes the slot to ffmpeg at exactly fps: static pages get real
|
|
50
|
+
* duration, floods cost nothing (the slot just gets overwritten), and the
|
|
51
|
+
* encoder sees constant-rate input matching its -framerate.
|
|
52
|
+
*
|
|
53
|
+
* Ticks are skipped (and counted) while ffmpeg's stdin is backpressured —
|
|
54
|
+
* with realtime VP8 settings that should stay at zero.
|
|
55
|
+
*/
|
|
56
|
+
function createSampler({ fps, write }) {
|
|
57
|
+
const intervalMs = 1000 / clampFps(fps);
|
|
58
|
+
const stats = { received: 0, encoded: 0, skippedTicks: 0 };
|
|
59
|
+
let latest = null;
|
|
60
|
+
let blocked = false;
|
|
61
|
+
let timer = null;
|
|
62
|
+
|
|
63
|
+
const sampler = {
|
|
64
|
+
stats,
|
|
65
|
+
push(frame) {
|
|
66
|
+
stats.received += 1;
|
|
67
|
+
latest = frame;
|
|
68
|
+
},
|
|
69
|
+
tick() {
|
|
70
|
+
if (!latest) return false;
|
|
71
|
+
if (blocked) {
|
|
72
|
+
stats.skippedTicks += 1;
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
const ready = write(latest, () => { blocked = false; });
|
|
76
|
+
stats.encoded += 1;
|
|
77
|
+
if (ready === false) blocked = true;
|
|
78
|
+
return true;
|
|
79
|
+
},
|
|
80
|
+
start() {
|
|
81
|
+
if (timer) return;
|
|
82
|
+
timer = setInterval(() => sampler.tick(), intervalMs);
|
|
83
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
84
|
+
},
|
|
85
|
+
stop() {
|
|
86
|
+
if (timer) clearInterval(timer);
|
|
87
|
+
timer = null;
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
return sampler;
|
|
91
|
+
}
|
|
92
|
+
|
|
37
93
|
function ffmpegBinary() {
|
|
38
94
|
if (process.env.AMALGM_FFMPEG) return process.env.AMALGM_FFMPEG;
|
|
39
95
|
try {
|
|
@@ -95,6 +151,22 @@ function recordingFile(args, context, session) {
|
|
|
95
151
|
return path.join(recordingDir(args, context, session), `${name}-${stamp}.webm`);
|
|
96
152
|
}
|
|
97
153
|
|
|
154
|
+
function ffmpegArgs({ fps, filters, file }) {
|
|
155
|
+
return [
|
|
156
|
+
'-y', '-f', 'image2pipe', '-c:v', 'mjpeg', '-framerate', String(fps), '-i', 'pipe:0',
|
|
157
|
+
'-vf', filters,
|
|
158
|
+
'-c:v', 'libvpx',
|
|
159
|
+
'-deadline', 'realtime',
|
|
160
|
+
'-cpu-used', '8',
|
|
161
|
+
'-lag-in-frames', '0',
|
|
162
|
+
'-auto-alt-ref', '0',
|
|
163
|
+
'-threads', '4',
|
|
164
|
+
'-b:v', '1M',
|
|
165
|
+
'-pix_fmt', 'yuv420p',
|
|
166
|
+
file,
|
|
167
|
+
];
|
|
168
|
+
}
|
|
169
|
+
|
|
98
170
|
async function start(args = {}, context = {}) {
|
|
99
171
|
const session = sessions.sessionFor(args, context);
|
|
100
172
|
if (active.has(session)) {
|
|
@@ -110,22 +182,40 @@ async function start(args = {}, context = {}) {
|
|
|
110
182
|
const source = await capture.plan(session, context);
|
|
111
183
|
|
|
112
184
|
const file = recordingFile(args, context, session);
|
|
113
|
-
const fps =
|
|
185
|
+
const fps = clampFps(args.fps);
|
|
114
186
|
const filters = [source.crop, 'pad=ceil(iw/2)*2:ceil(ih/2)*2'].filter(Boolean).join(',');
|
|
115
187
|
|
|
116
|
-
const ffmpeg = spawn(binary, [
|
|
117
|
-
'-y', '-f', 'image2pipe', '-c:v', 'mjpeg', '-framerate', String(fps), '-i', 'pipe:0',
|
|
118
|
-
'-vf', filters, '-c:v', 'libvpx', '-pix_fmt', 'yuv420p', file,
|
|
119
|
-
], { stdio: ['pipe', 'ignore', 'pipe'] });
|
|
188
|
+
const ffmpeg = spawn(binary, ffmpegArgs({ fps, filters, file }), { stdio: ['pipe', 'ignore', 'pipe'] });
|
|
120
189
|
let ffmpegErr = '';
|
|
121
190
|
ffmpeg.stderr.on('data', (chunk) => { ffmpegErr = (ffmpegErr + chunk.toString()).slice(-2000); });
|
|
122
191
|
|
|
123
|
-
const recording = {
|
|
192
|
+
const recording = {
|
|
193
|
+
cdp: null,
|
|
194
|
+
ffmpeg,
|
|
195
|
+
file,
|
|
196
|
+
fps,
|
|
197
|
+
sampler: null,
|
|
198
|
+
startedAt: Date.now(),
|
|
199
|
+
ffmpegErr: () => ffmpegErr,
|
|
200
|
+
};
|
|
201
|
+
recording.sampler = createSampler({
|
|
202
|
+
fps,
|
|
203
|
+
write(frame, onDrain) {
|
|
204
|
+
try {
|
|
205
|
+
const ready = ffmpeg.stdin.write(frame);
|
|
206
|
+
if (!ready) ffmpeg.stdin.once('drain', onDrain);
|
|
207
|
+
return ready;
|
|
208
|
+
} catch {
|
|
209
|
+
return true; // stdin gone; stop() reports via exit code + stderr
|
|
210
|
+
}
|
|
211
|
+
},
|
|
212
|
+
});
|
|
124
213
|
|
|
125
214
|
// Failures after the preflight (binary swapped out, EMFILE, ffmpeg dying
|
|
126
215
|
// mid-recording and EPIPE-ing stdin) end this recording, not the server.
|
|
127
216
|
ffmpeg.on('error', (err) => {
|
|
128
217
|
ffmpegErr = `${err.message}${ffmpegErr ? ` ${ffmpegErr}` : ''}`;
|
|
218
|
+
recording.sampler.stop();
|
|
129
219
|
if (active.get(session) === recording) active.delete(session);
|
|
130
220
|
try { recording.cdp?.close(); } catch {}
|
|
131
221
|
});
|
|
@@ -137,14 +227,18 @@ async function start(args = {}, context = {}) {
|
|
|
137
227
|
|
|
138
228
|
try {
|
|
139
229
|
recording.cdp = await capture.connectCdp(source.wsUrl);
|
|
230
|
+
// A surface that cannot produce one pixel now will not produce a stream
|
|
231
|
+
// either — refuse at start with the reason, not at stop with an empty file.
|
|
232
|
+
await capture.assertCapturable(recording.cdp, source);
|
|
140
233
|
recording.cdp.on('Page.screencastFrame', (params) => {
|
|
141
|
-
recording.
|
|
142
|
-
try { ffmpeg.stdin.write(Buffer.from(params.data, 'base64')); } catch {}
|
|
234
|
+
recording.sampler.push(Buffer.from(params.data, 'base64'));
|
|
143
235
|
recording.cdp.send('Page.screencastFrameAck', { sessionId: params.sessionId });
|
|
144
236
|
});
|
|
145
237
|
await recording.cdp.call('Page.enable').catch(() => {});
|
|
146
238
|
await recording.cdp.call('Page.startScreencast', { format: 'jpeg', quality: 70, everyNthFrame: 1 });
|
|
239
|
+
recording.sampler.start();
|
|
147
240
|
} catch (err) {
|
|
241
|
+
recording.sampler.stop();
|
|
148
242
|
active.delete(session);
|
|
149
243
|
try { recording.cdp?.close(); } catch {}
|
|
150
244
|
try { ffmpeg.kill('SIGKILL'); } catch {}
|
|
@@ -167,13 +261,16 @@ async function stop(args = {}, context = {}) {
|
|
|
167
261
|
if (!recording) throw new Error(`No active recording for session "${session}".`);
|
|
168
262
|
active.delete(session);
|
|
169
263
|
|
|
264
|
+
recording.sampler.stop();
|
|
170
265
|
try { recording.cdp?.send('Page.stopScreencast'); } catch {}
|
|
171
266
|
try { recording.cdp?.close(); } catch {}
|
|
172
267
|
|
|
173
268
|
// ffmpeg gets a bounded window to flush and finalize the container; a
|
|
174
269
|
// wedged encoder is killed rather than wedging record_stop.
|
|
270
|
+
let killed = false;
|
|
175
271
|
const exitCode = await new Promise((resolve) => {
|
|
176
272
|
const killTimer = setTimeout(() => {
|
|
273
|
+
killed = true;
|
|
177
274
|
try { recording.ffmpeg.kill('SIGKILL'); } catch {}
|
|
178
275
|
}, FFMPEG_CLOSE_TIMEOUT_MS);
|
|
179
276
|
recording.ffmpeg.on('close', (code) => {
|
|
@@ -187,23 +284,38 @@ async function stop(args = {}, context = {}) {
|
|
|
187
284
|
});
|
|
188
285
|
|
|
189
286
|
const durationMs = Date.now() - recording.startedAt;
|
|
190
|
-
|
|
287
|
+
const { received, encoded, skippedTicks } = recording.sampler.stats;
|
|
288
|
+
const bytes = fs.existsSync(recording.file) ? fs.statSync(recording.file).size : 0;
|
|
289
|
+
|
|
290
|
+
if (encoded === 0 || bytes === 0) {
|
|
191
291
|
try { fs.rmSync(recording.file, { force: true }); } catch {}
|
|
192
|
-
throw new Error(
|
|
193
|
-
? 'No frames captured —
|
|
292
|
+
throw new Error(encoded === 0
|
|
293
|
+
? 'No frames captured — the surface never produced a pixel while recording '
|
|
294
|
+
+ '(it likely went off screen; in the desktop app the Amalgm window must stay on screen).'
|
|
194
295
|
: `ffmpeg failed (exit ${exitCode}): ${recording.ffmpegErr()}`);
|
|
195
296
|
}
|
|
196
297
|
|
|
197
|
-
|
|
198
|
-
return
|
|
298
|
+
// WebM is cluster-streamed: even a force-killed encode leaves the frames
|
|
299
|
+
// written so far playable. Captured video beats a deleted file — return it
|
|
300
|
+
// and say what happened instead of throwing it away.
|
|
301
|
+
const result = {
|
|
199
302
|
session,
|
|
200
303
|
path: recording.file,
|
|
201
304
|
relativePath: path.relative(cli.amalgmDir(), recording.file),
|
|
202
305
|
bytes,
|
|
203
|
-
frames:
|
|
306
|
+
frames: encoded,
|
|
307
|
+
framesReceived: received,
|
|
308
|
+
skippedTicks,
|
|
309
|
+
fps: recording.fps,
|
|
204
310
|
durationMs,
|
|
205
311
|
mimeType: 'video/webm',
|
|
206
312
|
};
|
|
313
|
+
if (exitCode !== 0) {
|
|
314
|
+
result.warning = killed
|
|
315
|
+
? `Encoder was force-stopped after ${FFMPEG_CLOSE_TIMEOUT_MS / 1000}s; the file plays but may lack duration metadata.`
|
|
316
|
+
: `Encoder exited ${exitCode} (${recording.ffmpegErr().slice(-300)}); the file holds the frames written before that.`;
|
|
317
|
+
}
|
|
318
|
+
return result;
|
|
207
319
|
}
|
|
208
320
|
|
|
209
321
|
function list(args = {}, context = {}) {
|
|
@@ -227,4 +339,12 @@ function isActive(session) {
|
|
|
227
339
|
return active.has(session);
|
|
228
340
|
}
|
|
229
341
|
|
|
230
|
-
module.exports = {
|
|
342
|
+
module.exports = {
|
|
343
|
+
isActive,
|
|
344
|
+
list,
|
|
345
|
+
recordingDir,
|
|
346
|
+
resolveProjectPath,
|
|
347
|
+
start,
|
|
348
|
+
stop,
|
|
349
|
+
_test: { clampFps, createSampler, ffmpegArgs },
|
|
350
|
+
};
|
|
@@ -7,10 +7,15 @@ const http = require('node:http');
|
|
|
7
7
|
const capture = require('../browser/capture');
|
|
8
8
|
|
|
9
9
|
test('clipFor turns a webview rect into integer CSS-px clip bounds', () => {
|
|
10
|
+
// 411.5 + 868.75 overhangs the 1280px viewport by a rounding hair — the
|
|
11
|
+
// clip clamps to the viewport edge (the overhang captures as a black bar).
|
|
10
12
|
const clip = capture.clipFor({ x: 411.5, y: 52.25, width: 868.75, height: 781.5, vw: 1280, vh: 834 });
|
|
11
|
-
assert.deepEqual(clip, { x: 412, y: 52, width:
|
|
13
|
+
assert.deepEqual(clip, { x: 412, y: 52, width: 868, height: 782, scale: 1 });
|
|
12
14
|
// Negative origins (mid-animation surfaces) clamp to the viewport.
|
|
13
15
|
assert.equal(capture.clipFor({ x: -4, y: 0, width: 100, height: 100 }).x, 0);
|
|
16
|
+
// A splitview sliding open can report a rect mostly past the right edge.
|
|
17
|
+
const sliding = capture.clipFor({ x: 1200, y: 0, width: 700, height: 600, vw: 1280, vh: 800 });
|
|
18
|
+
assert.deepEqual(sliding, { x: 1200, y: 0, width: 80, height: 600, scale: 1 });
|
|
14
19
|
// Retina displays render captures at devicePixelRatio; scale divides it
|
|
15
20
|
// back out so image pixels map 1:1 onto cua input coordinates.
|
|
16
21
|
assert.equal(capture.clipFor({ x: 0, y: 0, width: 100, height: 100, dpr: 2 }).scale, 0.5);
|
|
@@ -35,13 +40,41 @@ test('pageTargets keeps only capturable page targets', () => {
|
|
|
35
40
|
assert.deepEqual(capture.pageTargets(targets).map((t) => t.webSocketDebuggerUrl), ['ws://x/3']);
|
|
36
41
|
});
|
|
37
42
|
|
|
38
|
-
test('rectScript
|
|
43
|
+
test('rectScript identifies the session surface and never guesses', () => {
|
|
39
44
|
const script = capture.rectScript('amalgm-bsid-abc', 'https://example.com/');
|
|
45
|
+
// Strongest identity first: the data attribute stamped at mount survives
|
|
46
|
+
// navigation; the marker in src only exists until the first navigate.
|
|
47
|
+
assert.match(script, /data-amalgm-surface/);
|
|
40
48
|
assert.match(script, /amalgm-bsid-abc/);
|
|
41
49
|
assert.match(script, /https:\/\/example\.com\//);
|
|
42
50
|
// No current URL → the byUrl branch is disabled, not matching everything.
|
|
43
51
|
const bare = capture.rectScript('amalgm-bsid-abc', '');
|
|
44
52
|
assert.match(bare, /"" \? views\.find/);
|
|
53
|
+
// "The only webview" is how one chat's capture reads another chat's page —
|
|
54
|
+
// an unidentified surface must resolve to null, never to a guess.
|
|
55
|
+
assert.doesNotMatch(script, /views\.length === 1/);
|
|
56
|
+
// The embedder's visibilityState rides along: full geometry with a hidden
|
|
57
|
+
// window is exactly the state captures must be able to name.
|
|
58
|
+
assert.match(script, /embedderVisibility/);
|
|
59
|
+
assert.match(script, /src: match\.src/);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('starvationReason names the app window state', () => {
|
|
63
|
+
assert.match(capture.starvationReason({ embedderVisibility: 'hidden' }), /hidden or minimized/);
|
|
64
|
+
assert.match(capture.starvationReason({ embedderVisibility: null }), /stopped rendering/);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('assertCapturable converts a starved probe into a named refusal', async () => {
|
|
68
|
+
const starvedClient = { call: () => new Promise(() => {}) };
|
|
69
|
+
// capture.js wraps the probe in its own deadline via cdp.call timeout — here
|
|
70
|
+
// the fake client never resolves, so rely on the call-site deadline shape:
|
|
71
|
+
const probe = capture.assertCapturable(
|
|
72
|
+
{ call: (m, p, t) => capture.deadline(new Promise(() => {}), 50, m) },
|
|
73
|
+
{ clip: { x: 0, y: 0, width: 10, height: 10 }, embedderVisibility: 'hidden' },
|
|
74
|
+
);
|
|
75
|
+
await assert.rejects(() => probe, /Cannot capture this surface: .*hidden or minimized/);
|
|
76
|
+
// Headless page targets raster unconditionally — no probe, no rejection.
|
|
77
|
+
await capture.assertCapturable(starvedClient, { clip: null });
|
|
45
78
|
});
|
|
46
79
|
|
|
47
80
|
test('targetListUrl accepts bare ports and ws urls', () => {
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const assert = require('node:assert/strict');
|
|
4
|
+
const test = require('node:test');
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const os = require('node:os');
|
|
7
|
+
const path = require('node:path');
|
|
8
|
+
|
|
9
|
+
const cli = require('../browser/cli');
|
|
10
|
+
|
|
11
|
+
function withoutBinOverride(fn) {
|
|
12
|
+
const saved = process.env.AMALGM_AGENT_BROWSER_BIN;
|
|
13
|
+
delete process.env.AMALGM_AGENT_BROWSER_BIN;
|
|
14
|
+
try {
|
|
15
|
+
return fn();
|
|
16
|
+
} finally {
|
|
17
|
+
if (saved !== undefined) process.env.AMALGM_AGENT_BROWSER_BIN = saved;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
test('agent-browser spawns the native binary directly when it exists', () => {
|
|
22
|
+
// The node wrapper exists only to pick a platform binary — spawning it adds
|
|
23
|
+
// a node startup (~100ms) to every command, which is the verb floor latency.
|
|
24
|
+
let wrapper;
|
|
25
|
+
try {
|
|
26
|
+
wrapper = require.resolve('agent-browser/bin/agent-browser.js');
|
|
27
|
+
} catch {
|
|
28
|
+
return; // checkout without agent-browser installed — nothing to assert
|
|
29
|
+
}
|
|
30
|
+
const { command, prefix } = withoutBinOverride(() => cli.agentBrowserCommand());
|
|
31
|
+
const dir = path.dirname(wrapper);
|
|
32
|
+
const hasNative = fs.readdirSync(dir).some((name) => name.startsWith(`agent-browser-${os.platform()}`));
|
|
33
|
+
if (hasNative) {
|
|
34
|
+
assert.notEqual(command, process.execPath, 'must not pay a node startup per command');
|
|
35
|
+
assert.match(command, /agent-browser-/);
|
|
36
|
+
assert.deepEqual(prefix, []);
|
|
37
|
+
} else {
|
|
38
|
+
assert.equal(command, process.execPath);
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('runCli forwards stdin for batch JSON commands', async () => {
|
|
43
|
+
// `batch` reads JSON command arrays from stdin — argv-quoting nested
|
|
44
|
+
// commands is exactly how arguments got eaten in the wild.
|
|
45
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'amalgm-cli-test-'));
|
|
46
|
+
const stub = path.join(dir, 'echo-stdin.sh');
|
|
47
|
+
fs.writeFileSync(stub, '#!/bin/sh\ncat\n', { mode: 0o755 });
|
|
48
|
+
const saved = process.env.AMALGM_AGENT_BROWSER_BIN;
|
|
49
|
+
process.env.AMALGM_AGENT_BROWSER_BIN = stub;
|
|
50
|
+
try {
|
|
51
|
+
const result = await cli.runCli([], { session: 'stdin-test', json: false, stdin: '[["get","url"]]', timeoutMs: 5000 });
|
|
52
|
+
assert.match(result.text, /\[\["get","url"\]\]/);
|
|
53
|
+
} finally {
|
|
54
|
+
if (saved === undefined) delete process.env.AMALGM_AGENT_BROWSER_BIN;
|
|
55
|
+
else process.env.AMALGM_AGENT_BROWSER_BIN = saved;
|
|
56
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
57
|
+
}
|
|
58
|
+
});
|
|
@@ -38,3 +38,68 @@ test('ffmpeg ships with the published package', (t) => {
|
|
|
38
38
|
'packages/amalgm must depend on ffmpeg-static — PATH ffmpeg does not exist on fresh machines',
|
|
39
39
|
);
|
|
40
40
|
});
|
|
41
|
+
|
|
42
|
+
// The sampler is the recorder's honesty mechanism: paint-on-change,
|
|
43
|
+
// encode-on-clock. Both 2026-06-10 stress failures live here — a "145s"
|
|
44
|
+
// recording holding one frame (static page, damage-driven ingest) and
|
|
45
|
+
// ffmpeg drowning under a 60fps animation then being killed at stop.
|
|
46
|
+
|
|
47
|
+
test('sampler gives static pages honest duration: one frame, many ticks', () => {
|
|
48
|
+
const { _test } = require('../browser/recorder');
|
|
49
|
+
const written = [];
|
|
50
|
+
const sampler = _test.createSampler({ fps: 10, write: (frame) => { written.push(frame); return true; } });
|
|
51
|
+
|
|
52
|
+
sampler.tick(); // nothing captured yet — nothing to encode
|
|
53
|
+
assert.equal(written.length, 0);
|
|
54
|
+
|
|
55
|
+
sampler.push(Buffer.from('frame-1')); // the single repaint a static page emits
|
|
56
|
+
for (let i = 0; i < 30; i += 1) sampler.tick(); // 3 seconds of clock at 10fps
|
|
57
|
+
|
|
58
|
+
assert.equal(written.length, 30, 'the latest frame re-encodes every tick — 3s of video, not 1 frame');
|
|
59
|
+
assert.equal(sampler.stats.received, 1);
|
|
60
|
+
assert.equal(sampler.stats.encoded, 30);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('sampler absorbs animation floods: encode rate is the clock, not the damage rate', () => {
|
|
64
|
+
const { _test } = require('../browser/recorder');
|
|
65
|
+
let writes = 0;
|
|
66
|
+
const sampler = _test.createSampler({ fps: 10, write: () => { writes += 1; return true; } });
|
|
67
|
+
|
|
68
|
+
for (let burst = 0; burst < 10; burst += 1) {
|
|
69
|
+
for (let i = 0; i < 60; i += 1) sampler.push(Buffer.from(`f${burst}-${i}`)); // 60fps canvas
|
|
70
|
+
sampler.tick();
|
|
71
|
+
}
|
|
72
|
+
assert.equal(writes, 10, 'ffmpeg sees fps ticks, never the 600 damage frames');
|
|
73
|
+
assert.equal(sampler.stats.received, 600);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test('sampler skips ticks under encoder backpressure and resumes on drain', () => {
|
|
77
|
+
const { _test } = require('../browser/recorder');
|
|
78
|
+
let drains = [];
|
|
79
|
+
const sampler = _test.createSampler({
|
|
80
|
+
fps: 10,
|
|
81
|
+
write: (frame, onDrain) => { drains.push(onDrain); return false; }, // stdin buffer full
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
sampler.push(Buffer.from('frame'));
|
|
85
|
+
assert.equal(sampler.tick(), true);
|
|
86
|
+
assert.equal(sampler.tick(), false, 'blocked while ffmpeg has not drained');
|
|
87
|
+
assert.equal(sampler.stats.skippedTicks, 1);
|
|
88
|
+
|
|
89
|
+
drains[0](); // ffmpeg drained
|
|
90
|
+
assert.equal(sampler.tick(), true);
|
|
91
|
+
assert.equal(sampler.stats.encoded, 2);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test('recording encoder uses realtime VP8 settings', () => {
|
|
95
|
+
const { _test } = require('../browser/recorder');
|
|
96
|
+
const args = _test.ffmpegArgs({ fps: 12, filters: 'pad=ceil(iw/2)*2:ceil(ih/2)*2', file: '/tmp/out.webm' });
|
|
97
|
+
|
|
98
|
+
assert.ok(args.includes('-deadline'));
|
|
99
|
+
assert.equal(args[args.indexOf('-deadline') + 1], 'realtime');
|
|
100
|
+
assert.ok(args.includes('-cpu-used'));
|
|
101
|
+
assert.equal(args[args.indexOf('-cpu-used') + 1], '8');
|
|
102
|
+
assert.ok(args.includes('-lag-in-frames'));
|
|
103
|
+
assert.equal(args[args.indexOf('-lag-in-frames') + 1], '0');
|
|
104
|
+
assert.equal(args[args.length - 1], '/tmp/out.webm');
|
|
105
|
+
});
|
|
@@ -131,3 +131,42 @@ test('entrypoint routing: candidate order is state dir, derived label dir, AMALG
|
|
|
131
131
|
assert.equal(candidates[3], path.join(os.homedir(), '.amalgm', 'electron-browser-bridge.json'));
|
|
132
132
|
});
|
|
133
133
|
});
|
|
134
|
+
|
|
135
|
+
// ---------------------------------------------------------------------------
|
|
136
|
+
// Session → surface routing. The 2026-06-10 stress tests caught two parallel
|
|
137
|
+
// chats reading each other's pages: a displaced session (its webview died on
|
|
138
|
+
// a chat-tab switch) fell back to "the only webview" — someone else's.
|
|
139
|
+
// ---------------------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
test('surface routing: marker, embedder DOM src, then previously selected tab', () => {
|
|
142
|
+
const attach = require('../browser/attach');
|
|
143
|
+
const { knownSessions } = require('../browser/sessions');
|
|
144
|
+
|
|
145
|
+
const fresh = { tabId: 't2', type: 'webview', url: 'about:blank#amalgm-bsid-s1' };
|
|
146
|
+
const navigated = { tabId: 't3', type: 'webview', url: 'https://example.com/' };
|
|
147
|
+
const other = { tabId: 't4', type: 'webview', url: 'https://other.test/' };
|
|
148
|
+
|
|
149
|
+
// Fresh surface: the marker URL identifies it.
|
|
150
|
+
assert.equal(attach.matchWebview([other, fresh], 's1', null), fresh);
|
|
151
|
+
|
|
152
|
+
// Navigated surface: the embedder DOM (data-amalgm-surface element) reports
|
|
153
|
+
// its live src — that re-keys the daemon's tab list after an MCP restart.
|
|
154
|
+
assert.equal(attach.matchWebview([other, navigated], 's1', 'https://example.com/'), navigated);
|
|
155
|
+
|
|
156
|
+
// Same surface, no DOM hint, but selected earlier in this process.
|
|
157
|
+
knownSessions.set('s1', { webviewTabId: 't3' });
|
|
158
|
+
try {
|
|
159
|
+
assert.equal(attach.matchWebview([other, navigated], 's1', null), navigated);
|
|
160
|
+
} finally {
|
|
161
|
+
knownSessions.delete('s1');
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
test('surface routing: an unidentified lone webview is never adopted', () => {
|
|
166
|
+
const attach = require('../browser/attach');
|
|
167
|
+
// One webview left and it is not provably ours — that is the cross-chat
|
|
168
|
+
// leak shape. Fail closed; ensureReady reopens a marked surface instead.
|
|
169
|
+
const stranger = { tabId: 't9', type: 'webview', url: 'https://another-chats-page.test/' };
|
|
170
|
+
assert.equal(attach.matchWebview([stranger], 'displaced-session', null), null);
|
|
171
|
+
assert.equal(attach.matchWebview([], 'displaced-session', null), null);
|
|
172
|
+
});
|