@peakflo/agent-harness 0.2.19 → 0.2.20-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -0
- package/dist/browser-broker/agent-actions.mjs +799 -0
- package/dist/browser-broker/browsers.mjs +1699 -0
- package/dist/browser-broker/credentials.mjs +211 -0
- package/dist/browser-broker/disk-guard.mjs +84 -0
- package/dist/browser-broker/egress.mjs +183 -0
- package/dist/browser-broker/input.mjs +270 -0
- package/dist/browser-broker/lease.mjs +547 -0
- package/dist/browser-broker/local-network.mjs +173 -0
- package/dist/browser-broker/local.mjs +158 -0
- package/dist/browser-broker/log.mjs +28 -0
- package/dist/browser-broker/network-policy.mjs +144 -0
- package/dist/browser-broker/os-users.mjs +493 -0
- package/dist/browser-broker/package.json +1 -0
- package/dist/browser-broker/protocol.mjs +321 -0
- package/dist/browser-broker/queue.mjs +37 -0
- package/dist/browser-broker/recorder-scripts.mjs +221 -0
- package/dist/browser-broker/recorder.mjs +438 -0
- package/dist/browser-broker/recording-store.mjs +493 -0
- package/dist/browser-broker/relay.mjs +399 -0
- package/dist/browser-broker/screencast.mjs +198 -0
- package/dist/browser-broker/state-store.mjs +112 -0
- package/dist/browser-broker/transfers.mjs +460 -0
- package/dist/control-plane.d.ts +6 -0
- package/dist/control-plane.js +10 -0
- package/dist/github-proxy.d.ts +1 -1
- package/dist/github-workspace.d.ts +2 -0
- package/dist/github-workspace.js +4 -0
- package/dist/harness.d.ts +1 -0
- package/dist/harness.js +27 -1
- package/dist/runtime/knowledge-tools.d.ts +1 -0
- package/dist/runtime/knowledge-tools.js +3 -0
- package/dist/runtime/local-browser.d.ts +33 -0
- package/dist/runtime/local-browser.js +135 -0
- package/package.json +5 -3
|
@@ -0,0 +1,799 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The agent's browser tools, with 20x semantics (src/main/panel-browser-broker.ts),
|
|
3
|
+
* driven through Playwright so clicks, typing and keys are REAL input events
|
|
4
|
+
* (isTrusted) rather than 20x's injected el.click()/value setters.
|
|
5
|
+
*
|
|
6
|
+
* Result values are 20x's shapes. A tool-level failure throws a BrokerError
|
|
7
|
+
* whose `message` is the verbatim 20x error string, so the gateway can answer
|
|
8
|
+
* `{ error: message }` exactly as 20x did:
|
|
9
|
+
*
|
|
10
|
+
* stale_ref Stale or unknown target ... — run browser_snapshot again.
|
|
11
|
+
* invalid_args not editable / unsupported key / bad `what` / bad selector
|
|
12
|
+
* timeout Timed out after ... (wait, page did not answer)
|
|
13
|
+
* navigation_failed Navigation failed: ...
|
|
14
|
+
*
|
|
15
|
+
* The snapshot script is 20x's VERBATIM: it labels visible interactive
|
|
16
|
+
* elements with data-bx-ref="eN" in the page's main world.
|
|
17
|
+
*
|
|
18
|
+
* `fillCredential` (protocol v3) is the one command that types a value the
|
|
19
|
+
* agent never sees: see its header.
|
|
20
|
+
*/
|
|
21
|
+
import {
|
|
22
|
+
BrokerError,
|
|
23
|
+
BROWSER_MAX_TEXT_MESSAGE_BYTES,
|
|
24
|
+
TOOL_ERRORS
|
|
25
|
+
} from './protocol.mjs';
|
|
26
|
+
import { errorText } from './log.mjs';
|
|
27
|
+
import { SENSITIVE_FIELD_RE, normalizeFieldText } from './recorder-scripts.mjs';
|
|
28
|
+
import {
|
|
29
|
+
CREDENTIAL_ID_RE,
|
|
30
|
+
FIELD_RE,
|
|
31
|
+
elementAllowsField
|
|
32
|
+
} from './credentials.mjs';
|
|
33
|
+
|
|
34
|
+
const SNAPSHOT_ELEMENT_CAP = 120;
|
|
35
|
+
|
|
36
|
+
/** 20x buildSnapshotScript, verbatim except the sensitive-field pattern (SENSITIVE_FIELD_RE adds card / bank fields and word boundaries). */
|
|
37
|
+
export function buildSnapshotScript() {
|
|
38
|
+
return `(() => {
|
|
39
|
+
const SEL = 'a[href],button,input,select,textarea,[role=button],[role=link],[role=checkbox],[role=tab],[role=switch],[onclick],[contenteditable=true],summary';
|
|
40
|
+
const sensitive = (el) => ${SENSITIVE_FIELD_RE}.test((${normalizeFieldText})([el.type, el.name, el.id, el.autocomplete, el.getAttribute('aria-label'), el.getAttribute('placeholder')].join(' ')));
|
|
41
|
+
const out = [];
|
|
42
|
+
let i = 0;
|
|
43
|
+
for (const el of document.querySelectorAll(SEL)) {
|
|
44
|
+
if (out.length >= ${SNAPSHOT_ELEMENT_CAP}) break;
|
|
45
|
+
const r = el.getBoundingClientRect();
|
|
46
|
+
if (r.width < 2 || r.height < 2) continue;
|
|
47
|
+
const st = getComputedStyle(el);
|
|
48
|
+
if (st.visibility === 'hidden' || st.display === 'none' || Number(st.opacity) === 0) continue;
|
|
49
|
+
const ref = 'e' + (++i);
|
|
50
|
+
el.setAttribute('data-bx-ref', ref);
|
|
51
|
+
out.push({
|
|
52
|
+
ref,
|
|
53
|
+
role: el.getAttribute('role') || el.tagName.toLowerCase(),
|
|
54
|
+
name: (el.getAttribute('aria-label') || el.innerText || el.getAttribute('placeholder') || el.getAttribute('title') || '').trim().replace(/\\s+/g, ' ').slice(0, 80),
|
|
55
|
+
value: ['INPUT','SELECT','TEXTAREA'].includes(el.tagName) ? (sensitive(el) || el.type === 'file' ? '[redacted]' : String(el.value ?? '').slice(0, 60)) : undefined,
|
|
56
|
+
href: el.tagName === 'A' ? (el.getAttribute('href') || '').slice(0, 120) : undefined
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
return JSON.stringify({ url: location.href, title: document.title, elements: out });
|
|
60
|
+
})()`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Removes every agent ref: run when a person takes the lease. */
|
|
64
|
+
export const CLEAR_REFS_SCRIPT = `(() => {
|
|
65
|
+
for (const el of document.querySelectorAll('[data-bx-ref]')) el.removeAttribute('data-bx-ref');
|
|
66
|
+
return true;
|
|
67
|
+
})()`;
|
|
68
|
+
|
|
69
|
+
/** 20x parseSnapshotResult. */
|
|
70
|
+
export function parseSnapshotResult(raw) {
|
|
71
|
+
let parsed = raw;
|
|
72
|
+
if (typeof raw === 'string') {
|
|
73
|
+
try {
|
|
74
|
+
parsed = JSON.parse(raw);
|
|
75
|
+
} catch {
|
|
76
|
+
return { error: TOOL_ERRORS.malformedSnapshot };
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
if (
|
|
80
|
+
!parsed ||
|
|
81
|
+
typeof parsed !== 'object' ||
|
|
82
|
+
!Array.isArray(parsed.elements)
|
|
83
|
+
) {
|
|
84
|
+
return { error: TOOL_ERRORS.malformedSnapshot };
|
|
85
|
+
}
|
|
86
|
+
return parsed;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function isRef(target) {
|
|
90
|
+
return /^@?e\d+$/.test(String(target).trim());
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function normalizeRef(ref) {
|
|
94
|
+
return ref.startsWith('@') ? ref.slice(1) : ref;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** The CSS a target resolves to (20x queryFor). */
|
|
98
|
+
export function targetSelector(target) {
|
|
99
|
+
const trimmed = String(target).trim();
|
|
100
|
+
return isRef(trimmed)
|
|
101
|
+
? `[data-bx-ref="${normalizeRef(trimmed)}"]`
|
|
102
|
+
: String(target);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const KEY_CODES = {
|
|
106
|
+
enter: { key: 'Enter', code: 'Enter', keyCode: 13 },
|
|
107
|
+
tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
|
|
108
|
+
escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
|
|
109
|
+
backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
|
|
110
|
+
delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
|
|
111
|
+
space: { key: ' ', code: 'Space', keyCode: 32 },
|
|
112
|
+
arrowup: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
|
|
113
|
+
arrowdown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
|
|
114
|
+
arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
|
|
115
|
+
arrowright: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
|
|
116
|
+
home: { key: 'Home', code: 'Home', keyCode: 36 },
|
|
117
|
+
end: { key: 'End', code: 'End', keyCode: 35 },
|
|
118
|
+
pageup: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
|
|
119
|
+
pagedown: { key: 'PageDown', code: 'PageDown', keyCode: 34 }
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
/** 20x resolveKeyName. */
|
|
123
|
+
export function resolveKeyName(key) {
|
|
124
|
+
const k = KEY_CODES[String(key).toLowerCase()];
|
|
125
|
+
if (k) return k;
|
|
126
|
+
if (/^[a-z0-9]$/i.test(key)) {
|
|
127
|
+
return {
|
|
128
|
+
key: key.toUpperCase(),
|
|
129
|
+
code: `Key${key.toUpperCase()}`,
|
|
130
|
+
keyCode: key.toUpperCase().charCodeAt(0)
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** 20x: http(s) only; anything else gets https:// prefixed. */
|
|
137
|
+
export function normaliseUrl(url) {
|
|
138
|
+
let target = String(url || '').trim();
|
|
139
|
+
if (!/^https?:\/\//i.test(target)) target = 'https://' + target;
|
|
140
|
+
return target;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** The 20x recording label for an agent action (never contains typed text). */
|
|
144
|
+
export function actionLabel(name, args) {
|
|
145
|
+
switch (name) {
|
|
146
|
+
case 'click':
|
|
147
|
+
return 'click';
|
|
148
|
+
case 'type':
|
|
149
|
+
return args.submit ? 'type-and-submit' : 'type';
|
|
150
|
+
case 'press_key': {
|
|
151
|
+
const key = String(args.key ?? '');
|
|
152
|
+
return resolveKeyName(key) && key.length > 1
|
|
153
|
+
? `press-key:${key}`
|
|
154
|
+
: 'press-key (character omitted)';
|
|
155
|
+
}
|
|
156
|
+
case 'scroll': {
|
|
157
|
+
const direction = ['up', 'down', 'left', 'right'].includes(args.direction)
|
|
158
|
+
? args.direction
|
|
159
|
+
: 'down';
|
|
160
|
+
const amount = Math.max(0, Math.min(Number(args.amount) || 600, 100000));
|
|
161
|
+
return `scroll:${direction}:${amount}`;
|
|
162
|
+
}
|
|
163
|
+
case 'navigate':
|
|
164
|
+
return 'navigate';
|
|
165
|
+
case 'back':
|
|
166
|
+
case 'forward':
|
|
167
|
+
case 'reload':
|
|
168
|
+
return 'history-or-reload';
|
|
169
|
+
default:
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function withTimeout(promise, ms, onTimeout) {
|
|
175
|
+
let timer;
|
|
176
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
177
|
+
timer = setTimeout(() => reject(onTimeout()), Math.max(ms, 0));
|
|
178
|
+
});
|
|
179
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function requireString(args, name) {
|
|
183
|
+
const value = args[name];
|
|
184
|
+
if (typeof value !== 'string' || !value) {
|
|
185
|
+
throw new BrokerError('invalid_args', `${name} is required`);
|
|
186
|
+
}
|
|
187
|
+
return value;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** page.evaluate with 20x's 30 s guard and one retry across a navigation. */
|
|
191
|
+
async function evaluate(page, script) {
|
|
192
|
+
const once = () =>
|
|
193
|
+
withTimeout(
|
|
194
|
+
page.evaluate(script),
|
|
195
|
+
30000,
|
|
196
|
+
() => new BrokerError('timeout', TOOL_ERRORS.evalTimeout)
|
|
197
|
+
);
|
|
198
|
+
try {
|
|
199
|
+
return await once();
|
|
200
|
+
} catch (error) {
|
|
201
|
+
if (error instanceof BrokerError) throw error;
|
|
202
|
+
if (/context was destroyed|navigat/i.test(errorText(error))) {
|
|
203
|
+
await page
|
|
204
|
+
.waitForLoadState('domcontentloaded', { timeout: 10000 })
|
|
205
|
+
.catch(() => undefined);
|
|
206
|
+
return once();
|
|
207
|
+
}
|
|
208
|
+
throw new BrokerError('internal', errorText(error));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function isTimeout(error) {
|
|
213
|
+
return (
|
|
214
|
+
error?.name === 'TimeoutError' ||
|
|
215
|
+
/Timeout \d+ms exceeded/.test(errorText(error))
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Resolve a target to a Playwright locator, or throw the 20x stale error. */
|
|
220
|
+
async function locate(page, target, staleMessage) {
|
|
221
|
+
const selector = targetSelector(target);
|
|
222
|
+
const locator = page.locator(`css=${selector}`).first();
|
|
223
|
+
let count;
|
|
224
|
+
try {
|
|
225
|
+
count = await locator.count();
|
|
226
|
+
} catch (error) {
|
|
227
|
+
throw new BrokerError('invalid_args', errorText(error));
|
|
228
|
+
}
|
|
229
|
+
if (count === 0) throw new BrokerError('stale_ref', staleMessage);
|
|
230
|
+
return locator;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/* -------------------------------------------------------------------------- */
|
|
234
|
+
/* Commands */
|
|
235
|
+
/* -------------------------------------------------------------------------- */
|
|
236
|
+
|
|
237
|
+
export async function snapshot(page) {
|
|
238
|
+
const raw = await evaluate(page, buildSnapshotScript());
|
|
239
|
+
const result = parseSnapshotResult(raw);
|
|
240
|
+
if (result.error) throw new BrokerError('internal', result.error);
|
|
241
|
+
return result;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export async function navigate(page, args) {
|
|
245
|
+
const target = normaliseUrl(requireString(args, 'url'));
|
|
246
|
+
try {
|
|
247
|
+
await page.goto(target, { waitUntil: 'load', timeout: 30000 });
|
|
248
|
+
return {
|
|
249
|
+
ok: true,
|
|
250
|
+
url: page.url(),
|
|
251
|
+
title: await page.title().catch(() => '')
|
|
252
|
+
};
|
|
253
|
+
} catch (error) {
|
|
254
|
+
// 20x: the page may still have navigated (client redirect, ERR_ABORTED).
|
|
255
|
+
const current = page.url();
|
|
256
|
+
if (current && current !== 'about:blank') {
|
|
257
|
+
const loading = await page
|
|
258
|
+
.evaluate(() => document.readyState !== 'complete')
|
|
259
|
+
.catch(() => true);
|
|
260
|
+
if (!loading) {
|
|
261
|
+
return {
|
|
262
|
+
ok: true,
|
|
263
|
+
url: current,
|
|
264
|
+
title: await page.title().catch(() => ''),
|
|
265
|
+
note: errorText(error)
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
throw new BrokerError(
|
|
270
|
+
'navigation_failed',
|
|
271
|
+
TOOL_ERRORS.navigationFailed(errorText(error))
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
export async function click(page, args) {
|
|
277
|
+
const target = requireString(args, 'target');
|
|
278
|
+
const locator = await locate(page, target, TOOL_ERRORS.staleTarget(target));
|
|
279
|
+
try {
|
|
280
|
+
await locator.click({ timeout: 5000 });
|
|
281
|
+
return { ok: true };
|
|
282
|
+
} catch (error) {
|
|
283
|
+
if (/detached|not attached/i.test(errorText(error))) {
|
|
284
|
+
throw new BrokerError('stale_ref', TOOL_ERRORS.staleTarget(target));
|
|
285
|
+
}
|
|
286
|
+
// Covered or never "stable": fall back to 20x's own DOM click.
|
|
287
|
+
try {
|
|
288
|
+
await locator.evaluate((el) => {
|
|
289
|
+
el.scrollIntoView({ block: 'center' });
|
|
290
|
+
el.click();
|
|
291
|
+
});
|
|
292
|
+
return {
|
|
293
|
+
ok: true,
|
|
294
|
+
note: 'Clicked through the DOM; the element did not accept pointer input.'
|
|
295
|
+
};
|
|
296
|
+
} catch {
|
|
297
|
+
if (isTimeout(error)) throw new BrokerError('timeout', errorText(error));
|
|
298
|
+
throw new BrokerError('internal', errorText(error));
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export async function type(page, args) {
|
|
304
|
+
const target = requireString(args, 'target');
|
|
305
|
+
if (typeof args.text !== 'string')
|
|
306
|
+
throw new BrokerError('invalid_args', 'text is required');
|
|
307
|
+
const text = args.text;
|
|
308
|
+
const locator = await locate(page, target, TOOL_ERRORS.staleTargetShort);
|
|
309
|
+
const info = await locator.evaluate((el) => ({
|
|
310
|
+
tag: el.tagName,
|
|
311
|
+
readOnly: !!el.readOnly,
|
|
312
|
+
editable: !!el.isContentEditable,
|
|
313
|
+
role: el.getAttribute('role') || el.tagName.toLowerCase(),
|
|
314
|
+
inForm: !!el.closest('form')
|
|
315
|
+
}));
|
|
316
|
+
const domSet = (value) =>
|
|
317
|
+
locator.evaluate((el, v) => {
|
|
318
|
+
if (el.tagName === 'SELECT') {
|
|
319
|
+
el.value = v;
|
|
320
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
el.focus();
|
|
324
|
+
const proto =
|
|
325
|
+
el.tagName === 'TEXTAREA'
|
|
326
|
+
? HTMLTextAreaElement.prototype
|
|
327
|
+
: HTMLInputElement.prototype;
|
|
328
|
+
const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
|
|
329
|
+
if (setter) setter.call(el, v);
|
|
330
|
+
else el.value = v;
|
|
331
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
332
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
333
|
+
}, value);
|
|
334
|
+
|
|
335
|
+
if (info.tag === 'SELECT') {
|
|
336
|
+
try {
|
|
337
|
+
await locator.selectOption(text, { timeout: 3000 });
|
|
338
|
+
} catch {
|
|
339
|
+
await domSet(text);
|
|
340
|
+
}
|
|
341
|
+
} else if (
|
|
342
|
+
(info.tag === 'INPUT' || info.tag === 'TEXTAREA') &&
|
|
343
|
+
!info.readOnly
|
|
344
|
+
) {
|
|
345
|
+
try {
|
|
346
|
+
await locator.fill(text, { timeout: 10000 });
|
|
347
|
+
} catch (error) {
|
|
348
|
+
if (isTimeout(error)) throw new BrokerError('timeout', errorText(error));
|
|
349
|
+
await domSet(text);
|
|
350
|
+
}
|
|
351
|
+
} else if (info.editable) {
|
|
352
|
+
await locator.fill(text, { timeout: 10000 });
|
|
353
|
+
} else {
|
|
354
|
+
throw new BrokerError(
|
|
355
|
+
'invalid_args',
|
|
356
|
+
TOOL_ERRORS.notEditable(target, info.role)
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
if (args.submit) {
|
|
361
|
+
if (info.inForm) {
|
|
362
|
+
await locator
|
|
363
|
+
.evaluate((el) => {
|
|
364
|
+
const form = el.closest('form');
|
|
365
|
+
form.__pfSubmitted = false;
|
|
366
|
+
form.addEventListener('submit', () => (form.__pfSubmitted = true), {
|
|
367
|
+
once: true,
|
|
368
|
+
capture: true
|
|
369
|
+
});
|
|
370
|
+
})
|
|
371
|
+
.catch(() => undefined);
|
|
372
|
+
}
|
|
373
|
+
await locator.press('Enter', { timeout: 5000 }).catch(() => undefined);
|
|
374
|
+
if (info.inForm) {
|
|
375
|
+
// Enter submits a form natively only in some shapes; 20x always
|
|
376
|
+
// requested submission. Do it once, only if Enter did not.
|
|
377
|
+
await locator
|
|
378
|
+
.evaluate((el) => {
|
|
379
|
+
const form = el.closest('form');
|
|
380
|
+
if (form && !form.__pfSubmitted) {
|
|
381
|
+
if (form.requestSubmit) form.requestSubmit();
|
|
382
|
+
else form.submit();
|
|
383
|
+
}
|
|
384
|
+
})
|
|
385
|
+
.catch(() => undefined); // navigation destroyed the context: it submitted
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
return { ok: true };
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
export async function pressKey(page, args) {
|
|
392
|
+
const key = requireString(args, 'key');
|
|
393
|
+
const resolved = resolveKeyName(key);
|
|
394
|
+
if (!resolved)
|
|
395
|
+
throw new BrokerError('invalid_args', TOOL_ERRORS.unsupportedKey(key));
|
|
396
|
+
// Named keys by their code; a single character as itself (keeps its case).
|
|
397
|
+
await page.keyboard.press(key.length > 1 ? resolved.code : key);
|
|
398
|
+
return { ok: true };
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
export async function scroll(page, args) {
|
|
402
|
+
if (args.target) {
|
|
403
|
+
const locator = await locate(
|
|
404
|
+
page,
|
|
405
|
+
String(args.target),
|
|
406
|
+
TOOL_ERRORS.staleTargetShort
|
|
407
|
+
);
|
|
408
|
+
await locator.evaluate((el) => el.scrollIntoView({ block: 'center' }));
|
|
409
|
+
return { ok: true };
|
|
410
|
+
}
|
|
411
|
+
const delta =
|
|
412
|
+
typeof args.amount === 'number' && args.amount > 0
|
|
413
|
+
? Math.round(args.amount)
|
|
414
|
+
: 600;
|
|
415
|
+
const direction = String(args.direction || 'down').toLowerCase();
|
|
416
|
+
const dx = direction === 'left' ? -delta : direction === 'right' ? delta : 0;
|
|
417
|
+
const dy = direction === 'up' ? -delta : dx === 0 ? delta : 0;
|
|
418
|
+
const scrollY = await evaluate(
|
|
419
|
+
page,
|
|
420
|
+
`(() => { window.scrollBy(${dx}, ${dy}); return window.scrollY; })()`
|
|
421
|
+
);
|
|
422
|
+
return { ok: true, scrollY };
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
export async function get(page, args) {
|
|
426
|
+
switch (args.what) {
|
|
427
|
+
case 'url':
|
|
428
|
+
return { url: page.url() };
|
|
429
|
+
case 'title':
|
|
430
|
+
return { title: await evaluate(page, 'document.title') };
|
|
431
|
+
case 'text':
|
|
432
|
+
return {
|
|
433
|
+
text: await evaluate(
|
|
434
|
+
page,
|
|
435
|
+
`(document.body?.innerText || '').slice(0, 20000)`
|
|
436
|
+
)
|
|
437
|
+
};
|
|
438
|
+
default:
|
|
439
|
+
throw new BrokerError('invalid_args', TOOL_ERRORS.badGet);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
export async function wait(page, args) {
|
|
444
|
+
const mode = args.mode;
|
|
445
|
+
if (!['selector', 'text', 'url'].includes(mode)) {
|
|
446
|
+
throw new BrokerError(
|
|
447
|
+
'invalid_args',
|
|
448
|
+
'mode must be "selector", "text" or "url"'
|
|
449
|
+
);
|
|
450
|
+
}
|
|
451
|
+
const value = typeof args.value === 'string' ? args.value : '';
|
|
452
|
+
const budget = Math.max(
|
|
453
|
+
500,
|
|
454
|
+
Math.min(Number(args.timeout_ms) || 10000, 60000)
|
|
455
|
+
);
|
|
456
|
+
const started = Date.now();
|
|
457
|
+
for (;;) {
|
|
458
|
+
let hit = false;
|
|
459
|
+
if (mode === 'url') {
|
|
460
|
+
hit = page.url().includes(value);
|
|
461
|
+
} else {
|
|
462
|
+
try {
|
|
463
|
+
hit =
|
|
464
|
+
mode === 'selector'
|
|
465
|
+
? await page.evaluate((s) => !!document.querySelector(s), value)
|
|
466
|
+
: await page.evaluate(
|
|
467
|
+
(t) => (document.body?.innerText || '').includes(t),
|
|
468
|
+
value
|
|
469
|
+
);
|
|
470
|
+
} catch (error) {
|
|
471
|
+
if (/SyntaxError|not a valid selector/i.test(errorText(error))) {
|
|
472
|
+
throw new BrokerError('invalid_args', errorText(error));
|
|
473
|
+
}
|
|
474
|
+
hit = false; // navigating: try again
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
if (hit) {
|
|
478
|
+
return mode === 'url'
|
|
479
|
+
? { ok: true, waitedMs: Date.now() - started, url: page.url() }
|
|
480
|
+
: { ok: true, waitedMs: Date.now() - started };
|
|
481
|
+
}
|
|
482
|
+
if (Date.now() - started >= budget) {
|
|
483
|
+
throw new BrokerError(
|
|
484
|
+
'timeout',
|
|
485
|
+
TOOL_ERRORS.waitTimeout(budget, mode, value)
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/** Base64 must fit a text message with room for the JSON envelope. */
|
|
493
|
+
const SCREENSHOT_BASE64_LIMIT = BROWSER_MAX_TEXT_MESSAGE_BYTES - 8 * 1024;
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* `masks` (protocol v3): the elements `fill_credential` typed into, as
|
|
497
|
+
* `{ locators, boxes }`. They are painted over in every agent screenshot
|
|
498
|
+
* while they exist, whatever their type is now: a page that flips a password
|
|
499
|
+
* field to type=text ("show password") must not put the value in a PNG the
|
|
500
|
+
* model reads. The normal path uses Playwright's own `mask`; the scaled-down
|
|
501
|
+
* CDP fallback paints the same boxes with a temporary overlay.
|
|
502
|
+
*/
|
|
503
|
+
export async function screenshot(page, args, { cdp, viewport, masks = null }) {
|
|
504
|
+
const locators = masks?.locators ?? [];
|
|
505
|
+
const boxes = masks?.boxes ?? [];
|
|
506
|
+
try {
|
|
507
|
+
const png = await page.screenshot({
|
|
508
|
+
type: 'png',
|
|
509
|
+
timeout: 15000,
|
|
510
|
+
...(locators.length > 0 ? { mask: locators, maskColor: '#000000' } : {})
|
|
511
|
+
});
|
|
512
|
+
let data = png.toString('base64');
|
|
513
|
+
let scale = 1;
|
|
514
|
+
// A busy 1280x800 page can exceed one socket message as PNG; scale down
|
|
515
|
+
// rather than fail. The agent sees `scale` and the pixel size.
|
|
516
|
+
let overlaid = false;
|
|
517
|
+
try {
|
|
518
|
+
for (const next of [0.75, 0.5, 0.35, 0.25]) {
|
|
519
|
+
if (data.length <= SCREENSHOT_BASE64_LIMIT) break;
|
|
520
|
+
if (boxes.length > 0 && !overlaid) {
|
|
521
|
+
await page.evaluate(paintMaskOverlay, boxes);
|
|
522
|
+
overlaid = true;
|
|
523
|
+
} else if (locators.length > 0 && !overlaid) {
|
|
524
|
+
// Masked elements but no box (hidden/zero-size): nothing to paint,
|
|
525
|
+
// yet never fall back to an unmasked capture of a live field.
|
|
526
|
+
throw new Error('the page image is too large to send');
|
|
527
|
+
}
|
|
528
|
+
const shot = await cdp.send('Page.captureScreenshot', {
|
|
529
|
+
format: 'png',
|
|
530
|
+
clip: {
|
|
531
|
+
x: 0,
|
|
532
|
+
y: 0,
|
|
533
|
+
width: viewport.width,
|
|
534
|
+
height: viewport.height,
|
|
535
|
+
scale: next
|
|
536
|
+
}
|
|
537
|
+
});
|
|
538
|
+
data = shot.data;
|
|
539
|
+
scale = next;
|
|
540
|
+
}
|
|
541
|
+
} finally {
|
|
542
|
+
if (overlaid)
|
|
543
|
+
await page.evaluate(removeMaskOverlay).catch(() => undefined);
|
|
544
|
+
}
|
|
545
|
+
if (data.length > SCREENSHOT_BASE64_LIMIT) {
|
|
546
|
+
throw new Error('the page image is too large to send');
|
|
547
|
+
}
|
|
548
|
+
return {
|
|
549
|
+
ok: true,
|
|
550
|
+
mimeType: 'image/png',
|
|
551
|
+
data,
|
|
552
|
+
width: Math.round(viewport.width * scale),
|
|
553
|
+
height: Math.round(viewport.height * scale),
|
|
554
|
+
scale
|
|
555
|
+
};
|
|
556
|
+
} catch (error) {
|
|
557
|
+
throw new BrokerError(
|
|
558
|
+
'internal',
|
|
559
|
+
TOOL_ERRORS.screenshotFailed(errorText(error))
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/** Black boxes over `boxes` (viewport CSS px) for the CDP fallback capture. */
|
|
565
|
+
export function paintMaskOverlay(boxes) {
|
|
566
|
+
for (const b of boxes) {
|
|
567
|
+
const d = document.createElement('div');
|
|
568
|
+
d.setAttribute('data-pf-credential-overlay', '');
|
|
569
|
+
d.style.cssText =
|
|
570
|
+
'position:fixed;z-index:2147483647;pointer-events:none;background:#000;' +
|
|
571
|
+
`left:${b.x}px;top:${b.y}px;width:${b.width}px;height:${b.height}px`;
|
|
572
|
+
document.documentElement.appendChild(d);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
export function removeMaskOverlay() {
|
|
576
|
+
for (const d of document.querySelectorAll('[data-pf-credential-overlay]'))
|
|
577
|
+
d.remove();
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
/* -------------------------------------------------------------------------- */
|
|
581
|
+
/* fill_credential (protocol v3) */
|
|
582
|
+
/* -------------------------------------------------------------------------- */
|
|
583
|
+
|
|
584
|
+
export const FILL_ERRORS = Object.freeze({
|
|
585
|
+
notHttps: 'fill_credential needs an https page.',
|
|
586
|
+
originNotAllowed: (host) =>
|
|
587
|
+
`This credential is not allowed on ${String(host).slice(0, 253)}.`,
|
|
588
|
+
denied: (code) =>
|
|
589
|
+
`The credential could not be used here (${String(code).slice(0, 40)}).`,
|
|
590
|
+
originChanged:
|
|
591
|
+
'The page changed origin while the credential was being filled; nothing was kept in the field.',
|
|
592
|
+
elementNotAllowed: (field) =>
|
|
593
|
+
`This element cannot take the "${String(field).slice(0, 64)}" field. Pick the matching input from browser_snapshot.`,
|
|
594
|
+
notFocused: 'The field did not keep focus; nothing was typed.'
|
|
595
|
+
});
|
|
596
|
+
|
|
597
|
+
/** The hostname of an https: URL, else null. */
|
|
598
|
+
function httpsHost(url) {
|
|
599
|
+
try {
|
|
600
|
+
const parsed = new URL(url);
|
|
601
|
+
return parsed.protocol === 'https:' ? parsed.hostname.toLowerCase() : null;
|
|
602
|
+
} catch {
|
|
603
|
+
return null;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
async function frameUrlOf(handle, page) {
|
|
608
|
+
try {
|
|
609
|
+
const frame = await handle?.ownerFrame?.();
|
|
610
|
+
if (frame) return frame.url();
|
|
611
|
+
} catch {
|
|
612
|
+
/* detached: the caller re-checks */
|
|
613
|
+
}
|
|
614
|
+
return page.url();
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
async function clearField(locator) {
|
|
618
|
+
try {
|
|
619
|
+
await locator.fill('', { timeout: 3000 });
|
|
620
|
+
return;
|
|
621
|
+
} catch {
|
|
622
|
+
/* fall through to the DOM setter */
|
|
623
|
+
}
|
|
624
|
+
await locator
|
|
625
|
+
.evaluate((el) => {
|
|
626
|
+
const proto =
|
|
627
|
+
el.tagName === 'TEXTAREA'
|
|
628
|
+
? HTMLTextAreaElement.prototype
|
|
629
|
+
: HTMLInputElement.prototype;
|
|
630
|
+
const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
|
|
631
|
+
if (setter) setter.call(el, '');
|
|
632
|
+
else el.value = '';
|
|
633
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
634
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
635
|
+
})
|
|
636
|
+
.catch(() => undefined);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* Type a credential field into a page element without the value ever
|
|
641
|
+
* reaching the agent (contract "fill_credential in the broker"):
|
|
642
|
+
*
|
|
643
|
+
* 2. the target must be an element that fits the field (a password goes
|
|
644
|
+
* only into a password input, ...), else `element_not_allowed`;
|
|
645
|
+
* 3. the top-level page must be https; `host` / `frameHost` are read from
|
|
646
|
+
* Playwright (CDP navigation state), not from page script;
|
|
647
|
+
* 4. only then is the value asked for (`resolve`, the API's `cred_resolve`
|
|
648
|
+
* with the single-use nonce). A refusal is `origin_not_allowed`
|
|
649
|
+
* (host_not_allowed) or `credential_denied`, and NO value was sent;
|
|
650
|
+
* 5. the origin is read AGAIN after the value arrives: a page that
|
|
651
|
+
* navigated during the round trip gets nothing (`origin_changed`).
|
|
652
|
+
* Then focus, clear, CDP `Input.insertText` (one trusted insertion, no
|
|
653
|
+
* per-key events a page could log as keystrokes beyond what any typing
|
|
654
|
+
* gives it);
|
|
655
|
+
* 6. and again after typing: if the origin moved, the field is cleared and
|
|
656
|
+
* the result is `origin_changed`.
|
|
657
|
+
*
|
|
658
|
+
* `onValue(value)` adds the value to the chat's redaction set as soon as it
|
|
659
|
+
* exists; `onTyped(handle)` registers the element for screenshot masking.
|
|
660
|
+
* The result never carries the value.
|
|
661
|
+
*
|
|
662
|
+
* @param {import('playwright-core').Page} page
|
|
663
|
+
* @param {{ target: string, credentialId: string, field: string, nonce: string }} args
|
|
664
|
+
* @param {{ cdp: { send: Function },
|
|
665
|
+
* resolve: (req: { credentialId, field, nonce, host, frameHost }) => Promise<{ values?: Record<string,string> }>,
|
|
666
|
+
* onValue?: (value: string) => void, onTyped?: (handle) => void }} deps
|
|
667
|
+
*/
|
|
668
|
+
export async function fillCredential(page, args, deps) {
|
|
669
|
+
const target = requireString(args, 'target');
|
|
670
|
+
const credentialId = requireString(args, 'credentialId');
|
|
671
|
+
const field = requireString(args, 'field');
|
|
672
|
+
const nonce = requireString(args, 'nonce');
|
|
673
|
+
if (!CREDENTIAL_ID_RE.test(credentialId))
|
|
674
|
+
throw new BrokerError('invalid_args', 'credentialId must be a uuid');
|
|
675
|
+
if (!FIELD_RE.test(field))
|
|
676
|
+
throw new BrokerError('invalid_args', 'field is not a credential field');
|
|
677
|
+
if (nonce.length > 2048)
|
|
678
|
+
throw new BrokerError('invalid_args', 'nonce is too long');
|
|
679
|
+
|
|
680
|
+
// 2. The element.
|
|
681
|
+
const locator = await locate(page, target, TOOL_ERRORS.staleTargetShort);
|
|
682
|
+
let info;
|
|
683
|
+
try {
|
|
684
|
+
info = await locator.evaluate((el) => ({
|
|
685
|
+
tag: el.tagName,
|
|
686
|
+
type: el.tagName === 'INPUT' ? String(el.type || '').toLowerCase() : '',
|
|
687
|
+
autocomplete: el.getAttribute('autocomplete') || '',
|
|
688
|
+
readOnly: !!el.readOnly,
|
|
689
|
+
disabled: !!el.disabled
|
|
690
|
+
}));
|
|
691
|
+
} catch (error) {
|
|
692
|
+
if (/detached|not attached/i.test(errorText(error)))
|
|
693
|
+
throw new BrokerError('stale_ref', TOOL_ERRORS.staleTargetShort);
|
|
694
|
+
throw new BrokerError('internal', errorText(error));
|
|
695
|
+
}
|
|
696
|
+
if (!elementAllowsField(field, info))
|
|
697
|
+
throw new BrokerError(
|
|
698
|
+
'element_not_allowed',
|
|
699
|
+
FILL_ERRORS.elementNotAllowed(field)
|
|
700
|
+
);
|
|
701
|
+
|
|
702
|
+
// 3. The origin, before anything is asked for.
|
|
703
|
+
const host = httpsHost(page.url());
|
|
704
|
+
if (!host) throw new BrokerError('origin_not_allowed', FILL_ERRORS.notHttps);
|
|
705
|
+
// The handle pins THIS element for the origin re-checks and the
|
|
706
|
+
// screenshot mask, whatever the page later does to its attributes.
|
|
707
|
+
const handle = await locator
|
|
708
|
+
.elementHandle({ timeout: 3000 })
|
|
709
|
+
.catch(() => null);
|
|
710
|
+
if (!handle) throw new BrokerError('stale_ref', TOOL_ERRORS.staleTargetShort);
|
|
711
|
+
const frameHost = httpsHost(await frameUrlOf(handle, page));
|
|
712
|
+
if (!frameHost)
|
|
713
|
+
throw new BrokerError('origin_not_allowed', FILL_ERRORS.notHttps);
|
|
714
|
+
|
|
715
|
+
// 4. The value.
|
|
716
|
+
let value;
|
|
717
|
+
try {
|
|
718
|
+
const result = await deps.resolve({
|
|
719
|
+
credentialId,
|
|
720
|
+
field,
|
|
721
|
+
nonce,
|
|
722
|
+
host,
|
|
723
|
+
frameHost
|
|
724
|
+
});
|
|
725
|
+
value = result?.values?.[field];
|
|
726
|
+
} catch (error) {
|
|
727
|
+
const code = typeof error?.code === 'string' ? error.code : 'unavailable';
|
|
728
|
+
if (code === 'host_not_allowed')
|
|
729
|
+
throw new BrokerError(
|
|
730
|
+
'origin_not_allowed',
|
|
731
|
+
FILL_ERRORS.originNotAllowed(host)
|
|
732
|
+
);
|
|
733
|
+
if (code === 'timeout')
|
|
734
|
+
throw new BrokerError('timeout', FILL_ERRORS.denied(code));
|
|
735
|
+
throw new BrokerError('credential_denied', FILL_ERRORS.denied(code));
|
|
736
|
+
}
|
|
737
|
+
if (typeof value !== 'string' || value.length === 0 || value.length > 16384)
|
|
738
|
+
throw new BrokerError('credential_denied', FILL_ERRORS.denied('no_value'));
|
|
739
|
+
deps.onValue?.(value);
|
|
740
|
+
|
|
741
|
+
// 5. Re-check the origin, then type.
|
|
742
|
+
const stillHere = async () =>
|
|
743
|
+
httpsHost(page.url()) === host &&
|
|
744
|
+
httpsHost(await frameUrlOf(handle, page)) === frameHost;
|
|
745
|
+
if (!(await stillHere())) {
|
|
746
|
+
value = null;
|
|
747
|
+
throw new BrokerError('origin_changed', FILL_ERRORS.originChanged);
|
|
748
|
+
}
|
|
749
|
+
try {
|
|
750
|
+
await locator.focus({ timeout: 5000 });
|
|
751
|
+
await clearField(locator);
|
|
752
|
+
const focused = await locator.evaluate((el) => {
|
|
753
|
+
const root = el.getRootNode();
|
|
754
|
+
return (root.activeElement ?? document.activeElement) === el;
|
|
755
|
+
});
|
|
756
|
+
if (!focused)
|
|
757
|
+
throw new BrokerError('element_not_allowed', FILL_ERRORS.notFocused);
|
|
758
|
+
} catch (error) {
|
|
759
|
+
value = null;
|
|
760
|
+
if (error instanceof BrokerError) throw error;
|
|
761
|
+
if (/detached|not attached/i.test(errorText(error)))
|
|
762
|
+
throw new BrokerError('stale_ref', TOOL_ERRORS.staleTargetShort);
|
|
763
|
+
throw new BrokerError('internal', errorText(error));
|
|
764
|
+
}
|
|
765
|
+
// Re-checked once more right before the insertion: focus/clear can run
|
|
766
|
+
// page handlers that navigate.
|
|
767
|
+
if (!(await stillHere())) {
|
|
768
|
+
value = null;
|
|
769
|
+
throw new BrokerError('origin_changed', FILL_ERRORS.originChanged);
|
|
770
|
+
}
|
|
771
|
+
deps.onTyped?.(handle);
|
|
772
|
+
try {
|
|
773
|
+
await deps.cdp.send('Input.insertText', { text: value });
|
|
774
|
+
} finally {
|
|
775
|
+
value = null; // drop the only reference held here
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
// 6. After typing.
|
|
779
|
+
if (!(await stillHere())) {
|
|
780
|
+
await clearField(locator);
|
|
781
|
+
throw new BrokerError('origin_changed', FILL_ERRORS.originChanged);
|
|
782
|
+
}
|
|
783
|
+
return { filled: true, credentialId, field, host };
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
export async function history(page, name, args, { cdp }) {
|
|
787
|
+
const options = { timeout: 15000, waitUntil: 'load' };
|
|
788
|
+
try {
|
|
789
|
+
if (name === 'back') await page.goBack(options);
|
|
790
|
+
else if (name === 'forward') await page.goForward(options);
|
|
791
|
+
else if (args.hard) {
|
|
792
|
+
await cdp.send('Page.reload', { ignoreCache: true });
|
|
793
|
+
await page.waitForLoadState('load', { timeout: 15000 });
|
|
794
|
+
} else await page.reload(options);
|
|
795
|
+
} catch {
|
|
796
|
+
/* 20x: settle after 15 s or on the first navigation, whichever is first */
|
|
797
|
+
}
|
|
798
|
+
return { ok: true, url: page.url() };
|
|
799
|
+
}
|