@peakflo/agent-harness 0.2.20-rc.1 → 0.2.20-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +78 -0
  2. package/dist/browser-broker/agent-actions.mjs +799 -0
  3. package/dist/browser-broker/browsers.mjs +1699 -0
  4. package/dist/browser-broker/credentials.mjs +211 -0
  5. package/dist/browser-broker/disk-guard.mjs +84 -0
  6. package/dist/browser-broker/egress.mjs +183 -0
  7. package/dist/browser-broker/input.mjs +270 -0
  8. package/dist/browser-broker/lease.mjs +547 -0
  9. package/dist/browser-broker/local-network.mjs +173 -0
  10. package/dist/browser-broker/local.mjs +158 -0
  11. package/dist/browser-broker/log.mjs +28 -0
  12. package/dist/browser-broker/network-policy.mjs +144 -0
  13. package/dist/browser-broker/os-users.mjs +493 -0
  14. package/dist/browser-broker/package.json +1 -0
  15. package/dist/browser-broker/protocol.mjs +321 -0
  16. package/dist/browser-broker/queue.mjs +37 -0
  17. package/dist/browser-broker/recorder-scripts.mjs +221 -0
  18. package/dist/browser-broker/recorder.mjs +438 -0
  19. package/dist/browser-broker/recording-store.mjs +493 -0
  20. package/dist/browser-broker/relay.mjs +399 -0
  21. package/dist/browser-broker/screencast.mjs +198 -0
  22. package/dist/browser-broker/state-store.mjs +112 -0
  23. package/dist/browser-broker/transfers.mjs +460 -0
  24. package/dist/control-plane.d.ts +6 -0
  25. package/dist/control-plane.js +10 -0
  26. package/dist/github-workspace.d.ts +2 -0
  27. package/dist/github-workspace.js +4 -0
  28. package/dist/harness.d.ts +1 -0
  29. package/dist/harness.js +24 -0
  30. package/dist/runtime/knowledge-tools.d.ts +1 -0
  31. package/dist/runtime/knowledge-tools.js +3 -0
  32. package/dist/runtime/local-browser.d.ts +33 -0
  33. package/dist/runtime/local-browser.js +135 -0
  34. package/package.json +5 -3
@@ -0,0 +1,799 @@
1
+ /**
2
+ * The agent's browser tools, with 20x semantics (src/main/panel-browser-broker.ts),
3
+ * driven through Playwright so clicks, typing and keys are REAL input events
4
+ * (isTrusted) rather than 20x's injected el.click()/value setters.
5
+ *
6
+ * Result values are 20x's shapes. A tool-level failure throws a BrokerError
7
+ * whose `message` is the verbatim 20x error string, so the gateway can answer
8
+ * `{ error: message }` exactly as 20x did:
9
+ *
10
+ * stale_ref Stale or unknown target ... — run browser_snapshot again.
11
+ * invalid_args not editable / unsupported key / bad `what` / bad selector
12
+ * timeout Timed out after ... (wait, page did not answer)
13
+ * navigation_failed Navigation failed: ...
14
+ *
15
+ * The snapshot script is 20x's VERBATIM: it labels visible interactive
16
+ * elements with data-bx-ref="eN" in the page's main world.
17
+ *
18
+ * `fillCredential` (protocol v3) is the one command that types a value the
19
+ * agent never sees: see its header.
20
+ */
21
+ import {
22
+ BrokerError,
23
+ BROWSER_MAX_TEXT_MESSAGE_BYTES,
24
+ TOOL_ERRORS
25
+ } from './protocol.mjs';
26
+ import { errorText } from './log.mjs';
27
+ import { SENSITIVE_FIELD_RE, normalizeFieldText } from './recorder-scripts.mjs';
28
+ import {
29
+ CREDENTIAL_ID_RE,
30
+ FIELD_RE,
31
+ elementAllowsField
32
+ } from './credentials.mjs';
33
+
34
+ const SNAPSHOT_ELEMENT_CAP = 120;
35
+
36
+ /** 20x buildSnapshotScript, verbatim except the sensitive-field pattern (SENSITIVE_FIELD_RE adds card / bank fields and word boundaries). */
37
+ export function buildSnapshotScript() {
38
+ return `(() => {
39
+ const SEL = 'a[href],button,input,select,textarea,[role=button],[role=link],[role=checkbox],[role=tab],[role=switch],[onclick],[contenteditable=true],summary';
40
+ const sensitive = (el) => ${SENSITIVE_FIELD_RE}.test((${normalizeFieldText})([el.type, el.name, el.id, el.autocomplete, el.getAttribute('aria-label'), el.getAttribute('placeholder')].join(' ')));
41
+ const out = [];
42
+ let i = 0;
43
+ for (const el of document.querySelectorAll(SEL)) {
44
+ if (out.length >= ${SNAPSHOT_ELEMENT_CAP}) break;
45
+ const r = el.getBoundingClientRect();
46
+ if (r.width < 2 || r.height < 2) continue;
47
+ const st = getComputedStyle(el);
48
+ if (st.visibility === 'hidden' || st.display === 'none' || Number(st.opacity) === 0) continue;
49
+ const ref = 'e' + (++i);
50
+ el.setAttribute('data-bx-ref', ref);
51
+ out.push({
52
+ ref,
53
+ role: el.getAttribute('role') || el.tagName.toLowerCase(),
54
+ name: (el.getAttribute('aria-label') || el.innerText || el.getAttribute('placeholder') || el.getAttribute('title') || '').trim().replace(/\\s+/g, ' ').slice(0, 80),
55
+ value: ['INPUT','SELECT','TEXTAREA'].includes(el.tagName) ? (sensitive(el) || el.type === 'file' ? '[redacted]' : String(el.value ?? '').slice(0, 60)) : undefined,
56
+ href: el.tagName === 'A' ? (el.getAttribute('href') || '').slice(0, 120) : undefined
57
+ });
58
+ }
59
+ return JSON.stringify({ url: location.href, title: document.title, elements: out });
60
+ })()`;
61
+ }
62
+
63
+ /** Removes every agent ref: run when a person takes the lease. */
64
+ export const CLEAR_REFS_SCRIPT = `(() => {
65
+ for (const el of document.querySelectorAll('[data-bx-ref]')) el.removeAttribute('data-bx-ref');
66
+ return true;
67
+ })()`;
68
+
69
+ /** 20x parseSnapshotResult. */
70
+ export function parseSnapshotResult(raw) {
71
+ let parsed = raw;
72
+ if (typeof raw === 'string') {
73
+ try {
74
+ parsed = JSON.parse(raw);
75
+ } catch {
76
+ return { error: TOOL_ERRORS.malformedSnapshot };
77
+ }
78
+ }
79
+ if (
80
+ !parsed ||
81
+ typeof parsed !== 'object' ||
82
+ !Array.isArray(parsed.elements)
83
+ ) {
84
+ return { error: TOOL_ERRORS.malformedSnapshot };
85
+ }
86
+ return parsed;
87
+ }
88
+
89
+ export function isRef(target) {
90
+ return /^@?e\d+$/.test(String(target).trim());
91
+ }
92
+
93
+ export function normalizeRef(ref) {
94
+ return ref.startsWith('@') ? ref.slice(1) : ref;
95
+ }
96
+
97
+ /** The CSS a target resolves to (20x queryFor). */
98
+ export function targetSelector(target) {
99
+ const trimmed = String(target).trim();
100
+ return isRef(trimmed)
101
+ ? `[data-bx-ref="${normalizeRef(trimmed)}"]`
102
+ : String(target);
103
+ }
104
+
105
+ const KEY_CODES = {
106
+ enter: { key: 'Enter', code: 'Enter', keyCode: 13 },
107
+ tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
108
+ escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
109
+ backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
110
+ delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
111
+ space: { key: ' ', code: 'Space', keyCode: 32 },
112
+ arrowup: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
113
+ arrowdown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
114
+ arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
115
+ arrowright: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
116
+ home: { key: 'Home', code: 'Home', keyCode: 36 },
117
+ end: { key: 'End', code: 'End', keyCode: 35 },
118
+ pageup: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
119
+ pagedown: { key: 'PageDown', code: 'PageDown', keyCode: 34 }
120
+ };
121
+
122
+ /** 20x resolveKeyName. */
123
+ export function resolveKeyName(key) {
124
+ const k = KEY_CODES[String(key).toLowerCase()];
125
+ if (k) return k;
126
+ if (/^[a-z0-9]$/i.test(key)) {
127
+ return {
128
+ key: key.toUpperCase(),
129
+ code: `Key${key.toUpperCase()}`,
130
+ keyCode: key.toUpperCase().charCodeAt(0)
131
+ };
132
+ }
133
+ return null;
134
+ }
135
+
136
+ /** 20x: http(s) only; anything else gets https:// prefixed. */
137
+ export function normaliseUrl(url) {
138
+ let target = String(url || '').trim();
139
+ if (!/^https?:\/\//i.test(target)) target = 'https://' + target;
140
+ return target;
141
+ }
142
+
143
+ /** The 20x recording label for an agent action (never contains typed text). */
144
+ export function actionLabel(name, args) {
145
+ switch (name) {
146
+ case 'click':
147
+ return 'click';
148
+ case 'type':
149
+ return args.submit ? 'type-and-submit' : 'type';
150
+ case 'press_key': {
151
+ const key = String(args.key ?? '');
152
+ return resolveKeyName(key) && key.length > 1
153
+ ? `press-key:${key}`
154
+ : 'press-key (character omitted)';
155
+ }
156
+ case 'scroll': {
157
+ const direction = ['up', 'down', 'left', 'right'].includes(args.direction)
158
+ ? args.direction
159
+ : 'down';
160
+ const amount = Math.max(0, Math.min(Number(args.amount) || 600, 100000));
161
+ return `scroll:${direction}:${amount}`;
162
+ }
163
+ case 'navigate':
164
+ return 'navigate';
165
+ case 'back':
166
+ case 'forward':
167
+ case 'reload':
168
+ return 'history-or-reload';
169
+ default:
170
+ return null;
171
+ }
172
+ }
173
+
174
+ function withTimeout(promise, ms, onTimeout) {
175
+ let timer;
176
+ const timeout = new Promise((_resolve, reject) => {
177
+ timer = setTimeout(() => reject(onTimeout()), Math.max(ms, 0));
178
+ });
179
+ return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
180
+ }
181
+
182
+ function requireString(args, name) {
183
+ const value = args[name];
184
+ if (typeof value !== 'string' || !value) {
185
+ throw new BrokerError('invalid_args', `${name} is required`);
186
+ }
187
+ return value;
188
+ }
189
+
190
+ /** page.evaluate with 20x's 30 s guard and one retry across a navigation. */
191
+ async function evaluate(page, script) {
192
+ const once = () =>
193
+ withTimeout(
194
+ page.evaluate(script),
195
+ 30000,
196
+ () => new BrokerError('timeout', TOOL_ERRORS.evalTimeout)
197
+ );
198
+ try {
199
+ return await once();
200
+ } catch (error) {
201
+ if (error instanceof BrokerError) throw error;
202
+ if (/context was destroyed|navigat/i.test(errorText(error))) {
203
+ await page
204
+ .waitForLoadState('domcontentloaded', { timeout: 10000 })
205
+ .catch(() => undefined);
206
+ return once();
207
+ }
208
+ throw new BrokerError('internal', errorText(error));
209
+ }
210
+ }
211
+
212
+ function isTimeout(error) {
213
+ return (
214
+ error?.name === 'TimeoutError' ||
215
+ /Timeout \d+ms exceeded/.test(errorText(error))
216
+ );
217
+ }
218
+
219
+ /** Resolve a target to a Playwright locator, or throw the 20x stale error. */
220
+ async function locate(page, target, staleMessage) {
221
+ const selector = targetSelector(target);
222
+ const locator = page.locator(`css=${selector}`).first();
223
+ let count;
224
+ try {
225
+ count = await locator.count();
226
+ } catch (error) {
227
+ throw new BrokerError('invalid_args', errorText(error));
228
+ }
229
+ if (count === 0) throw new BrokerError('stale_ref', staleMessage);
230
+ return locator;
231
+ }
232
+
233
+ /* -------------------------------------------------------------------------- */
234
+ /* Commands */
235
+ /* -------------------------------------------------------------------------- */
236
+
237
+ export async function snapshot(page) {
238
+ const raw = await evaluate(page, buildSnapshotScript());
239
+ const result = parseSnapshotResult(raw);
240
+ if (result.error) throw new BrokerError('internal', result.error);
241
+ return result;
242
+ }
243
+
244
+ export async function navigate(page, args) {
245
+ const target = normaliseUrl(requireString(args, 'url'));
246
+ try {
247
+ await page.goto(target, { waitUntil: 'load', timeout: 30000 });
248
+ return {
249
+ ok: true,
250
+ url: page.url(),
251
+ title: await page.title().catch(() => '')
252
+ };
253
+ } catch (error) {
254
+ // 20x: the page may still have navigated (client redirect, ERR_ABORTED).
255
+ const current = page.url();
256
+ if (current && current !== 'about:blank') {
257
+ const loading = await page
258
+ .evaluate(() => document.readyState !== 'complete')
259
+ .catch(() => true);
260
+ if (!loading) {
261
+ return {
262
+ ok: true,
263
+ url: current,
264
+ title: await page.title().catch(() => ''),
265
+ note: errorText(error)
266
+ };
267
+ }
268
+ }
269
+ throw new BrokerError(
270
+ 'navigation_failed',
271
+ TOOL_ERRORS.navigationFailed(errorText(error))
272
+ );
273
+ }
274
+ }
275
+
276
+ export async function click(page, args) {
277
+ const target = requireString(args, 'target');
278
+ const locator = await locate(page, target, TOOL_ERRORS.staleTarget(target));
279
+ try {
280
+ await locator.click({ timeout: 5000 });
281
+ return { ok: true };
282
+ } catch (error) {
283
+ if (/detached|not attached/i.test(errorText(error))) {
284
+ throw new BrokerError('stale_ref', TOOL_ERRORS.staleTarget(target));
285
+ }
286
+ // Covered or never "stable": fall back to 20x's own DOM click.
287
+ try {
288
+ await locator.evaluate((el) => {
289
+ el.scrollIntoView({ block: 'center' });
290
+ el.click();
291
+ });
292
+ return {
293
+ ok: true,
294
+ note: 'Clicked through the DOM; the element did not accept pointer input.'
295
+ };
296
+ } catch {
297
+ if (isTimeout(error)) throw new BrokerError('timeout', errorText(error));
298
+ throw new BrokerError('internal', errorText(error));
299
+ }
300
+ }
301
+ }
302
+
303
+ export async function type(page, args) {
304
+ const target = requireString(args, 'target');
305
+ if (typeof args.text !== 'string')
306
+ throw new BrokerError('invalid_args', 'text is required');
307
+ const text = args.text;
308
+ const locator = await locate(page, target, TOOL_ERRORS.staleTargetShort);
309
+ const info = await locator.evaluate((el) => ({
310
+ tag: el.tagName,
311
+ readOnly: !!el.readOnly,
312
+ editable: !!el.isContentEditable,
313
+ role: el.getAttribute('role') || el.tagName.toLowerCase(),
314
+ inForm: !!el.closest('form')
315
+ }));
316
+ const domSet = (value) =>
317
+ locator.evaluate((el, v) => {
318
+ if (el.tagName === 'SELECT') {
319
+ el.value = v;
320
+ el.dispatchEvent(new Event('change', { bubbles: true }));
321
+ return;
322
+ }
323
+ el.focus();
324
+ const proto =
325
+ el.tagName === 'TEXTAREA'
326
+ ? HTMLTextAreaElement.prototype
327
+ : HTMLInputElement.prototype;
328
+ const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
329
+ if (setter) setter.call(el, v);
330
+ else el.value = v;
331
+ el.dispatchEvent(new Event('input', { bubbles: true }));
332
+ el.dispatchEvent(new Event('change', { bubbles: true }));
333
+ }, value);
334
+
335
+ if (info.tag === 'SELECT') {
336
+ try {
337
+ await locator.selectOption(text, { timeout: 3000 });
338
+ } catch {
339
+ await domSet(text);
340
+ }
341
+ } else if (
342
+ (info.tag === 'INPUT' || info.tag === 'TEXTAREA') &&
343
+ !info.readOnly
344
+ ) {
345
+ try {
346
+ await locator.fill(text, { timeout: 10000 });
347
+ } catch (error) {
348
+ if (isTimeout(error)) throw new BrokerError('timeout', errorText(error));
349
+ await domSet(text);
350
+ }
351
+ } else if (info.editable) {
352
+ await locator.fill(text, { timeout: 10000 });
353
+ } else {
354
+ throw new BrokerError(
355
+ 'invalid_args',
356
+ TOOL_ERRORS.notEditable(target, info.role)
357
+ );
358
+ }
359
+
360
+ if (args.submit) {
361
+ if (info.inForm) {
362
+ await locator
363
+ .evaluate((el) => {
364
+ const form = el.closest('form');
365
+ form.__pfSubmitted = false;
366
+ form.addEventListener('submit', () => (form.__pfSubmitted = true), {
367
+ once: true,
368
+ capture: true
369
+ });
370
+ })
371
+ .catch(() => undefined);
372
+ }
373
+ await locator.press('Enter', { timeout: 5000 }).catch(() => undefined);
374
+ if (info.inForm) {
375
+ // Enter submits a form natively only in some shapes; 20x always
376
+ // requested submission. Do it once, only if Enter did not.
377
+ await locator
378
+ .evaluate((el) => {
379
+ const form = el.closest('form');
380
+ if (form && !form.__pfSubmitted) {
381
+ if (form.requestSubmit) form.requestSubmit();
382
+ else form.submit();
383
+ }
384
+ })
385
+ .catch(() => undefined); // navigation destroyed the context: it submitted
386
+ }
387
+ }
388
+ return { ok: true };
389
+ }
390
+
391
+ export async function pressKey(page, args) {
392
+ const key = requireString(args, 'key');
393
+ const resolved = resolveKeyName(key);
394
+ if (!resolved)
395
+ throw new BrokerError('invalid_args', TOOL_ERRORS.unsupportedKey(key));
396
+ // Named keys by their code; a single character as itself (keeps its case).
397
+ await page.keyboard.press(key.length > 1 ? resolved.code : key);
398
+ return { ok: true };
399
+ }
400
+
401
+ export async function scroll(page, args) {
402
+ if (args.target) {
403
+ const locator = await locate(
404
+ page,
405
+ String(args.target),
406
+ TOOL_ERRORS.staleTargetShort
407
+ );
408
+ await locator.evaluate((el) => el.scrollIntoView({ block: 'center' }));
409
+ return { ok: true };
410
+ }
411
+ const delta =
412
+ typeof args.amount === 'number' && args.amount > 0
413
+ ? Math.round(args.amount)
414
+ : 600;
415
+ const direction = String(args.direction || 'down').toLowerCase();
416
+ const dx = direction === 'left' ? -delta : direction === 'right' ? delta : 0;
417
+ const dy = direction === 'up' ? -delta : dx === 0 ? delta : 0;
418
+ const scrollY = await evaluate(
419
+ page,
420
+ `(() => { window.scrollBy(${dx}, ${dy}); return window.scrollY; })()`
421
+ );
422
+ return { ok: true, scrollY };
423
+ }
424
+
425
+ export async function get(page, args) {
426
+ switch (args.what) {
427
+ case 'url':
428
+ return { url: page.url() };
429
+ case 'title':
430
+ return { title: await evaluate(page, 'document.title') };
431
+ case 'text':
432
+ return {
433
+ text: await evaluate(
434
+ page,
435
+ `(document.body?.innerText || '').slice(0, 20000)`
436
+ )
437
+ };
438
+ default:
439
+ throw new BrokerError('invalid_args', TOOL_ERRORS.badGet);
440
+ }
441
+ }
442
+
443
+ export async function wait(page, args) {
444
+ const mode = args.mode;
445
+ if (!['selector', 'text', 'url'].includes(mode)) {
446
+ throw new BrokerError(
447
+ 'invalid_args',
448
+ 'mode must be "selector", "text" or "url"'
449
+ );
450
+ }
451
+ const value = typeof args.value === 'string' ? args.value : '';
452
+ const budget = Math.max(
453
+ 500,
454
+ Math.min(Number(args.timeout_ms) || 10000, 60000)
455
+ );
456
+ const started = Date.now();
457
+ for (;;) {
458
+ let hit = false;
459
+ if (mode === 'url') {
460
+ hit = page.url().includes(value);
461
+ } else {
462
+ try {
463
+ hit =
464
+ mode === 'selector'
465
+ ? await page.evaluate((s) => !!document.querySelector(s), value)
466
+ : await page.evaluate(
467
+ (t) => (document.body?.innerText || '').includes(t),
468
+ value
469
+ );
470
+ } catch (error) {
471
+ if (/SyntaxError|not a valid selector/i.test(errorText(error))) {
472
+ throw new BrokerError('invalid_args', errorText(error));
473
+ }
474
+ hit = false; // navigating: try again
475
+ }
476
+ }
477
+ if (hit) {
478
+ return mode === 'url'
479
+ ? { ok: true, waitedMs: Date.now() - started, url: page.url() }
480
+ : { ok: true, waitedMs: Date.now() - started };
481
+ }
482
+ if (Date.now() - started >= budget) {
483
+ throw new BrokerError(
484
+ 'timeout',
485
+ TOOL_ERRORS.waitTimeout(budget, mode, value)
486
+ );
487
+ }
488
+ await new Promise((resolve) => setTimeout(resolve, 250));
489
+ }
490
+ }
491
+
492
+ /** Base64 must fit a text message with room for the JSON envelope. */
493
+ const SCREENSHOT_BASE64_LIMIT = BROWSER_MAX_TEXT_MESSAGE_BYTES - 8 * 1024;
494
+
495
+ /**
496
+ * `masks` (protocol v3): the elements `fill_credential` typed into, as
497
+ * `{ locators, boxes }`. They are painted over in every agent screenshot
498
+ * while they exist, whatever their type is now: a page that flips a password
499
+ * field to type=text ("show password") must not put the value in a PNG the
500
+ * model reads. The normal path uses Playwright's own `mask`; the scaled-down
501
+ * CDP fallback paints the same boxes with a temporary overlay.
502
+ */
503
+ export async function screenshot(page, args, { cdp, viewport, masks = null }) {
504
+ const locators = masks?.locators ?? [];
505
+ const boxes = masks?.boxes ?? [];
506
+ try {
507
+ const png = await page.screenshot({
508
+ type: 'png',
509
+ timeout: 15000,
510
+ ...(locators.length > 0 ? { mask: locators, maskColor: '#000000' } : {})
511
+ });
512
+ let data = png.toString('base64');
513
+ let scale = 1;
514
+ // A busy 1280x800 page can exceed one socket message as PNG; scale down
515
+ // rather than fail. The agent sees `scale` and the pixel size.
516
+ let overlaid = false;
517
+ try {
518
+ for (const next of [0.75, 0.5, 0.35, 0.25]) {
519
+ if (data.length <= SCREENSHOT_BASE64_LIMIT) break;
520
+ if (boxes.length > 0 && !overlaid) {
521
+ await page.evaluate(paintMaskOverlay, boxes);
522
+ overlaid = true;
523
+ } else if (locators.length > 0 && !overlaid) {
524
+ // Masked elements but no box (hidden/zero-size): nothing to paint,
525
+ // yet never fall back to an unmasked capture of a live field.
526
+ throw new Error('the page image is too large to send');
527
+ }
528
+ const shot = await cdp.send('Page.captureScreenshot', {
529
+ format: 'png',
530
+ clip: {
531
+ x: 0,
532
+ y: 0,
533
+ width: viewport.width,
534
+ height: viewport.height,
535
+ scale: next
536
+ }
537
+ });
538
+ data = shot.data;
539
+ scale = next;
540
+ }
541
+ } finally {
542
+ if (overlaid)
543
+ await page.evaluate(removeMaskOverlay).catch(() => undefined);
544
+ }
545
+ if (data.length > SCREENSHOT_BASE64_LIMIT) {
546
+ throw new Error('the page image is too large to send');
547
+ }
548
+ return {
549
+ ok: true,
550
+ mimeType: 'image/png',
551
+ data,
552
+ width: Math.round(viewport.width * scale),
553
+ height: Math.round(viewport.height * scale),
554
+ scale
555
+ };
556
+ } catch (error) {
557
+ throw new BrokerError(
558
+ 'internal',
559
+ TOOL_ERRORS.screenshotFailed(errorText(error))
560
+ );
561
+ }
562
+ }
563
+
564
+ /** Black boxes over `boxes` (viewport CSS px) for the CDP fallback capture. */
565
+ export function paintMaskOverlay(boxes) {
566
+ for (const b of boxes) {
567
+ const d = document.createElement('div');
568
+ d.setAttribute('data-pf-credential-overlay', '');
569
+ d.style.cssText =
570
+ 'position:fixed;z-index:2147483647;pointer-events:none;background:#000;' +
571
+ `left:${b.x}px;top:${b.y}px;width:${b.width}px;height:${b.height}px`;
572
+ document.documentElement.appendChild(d);
573
+ }
574
+ }
575
+ export function removeMaskOverlay() {
576
+ for (const d of document.querySelectorAll('[data-pf-credential-overlay]'))
577
+ d.remove();
578
+ }
579
+
580
+ /* -------------------------------------------------------------------------- */
581
+ /* fill_credential (protocol v3) */
582
+ /* -------------------------------------------------------------------------- */
583
+
584
+ export const FILL_ERRORS = Object.freeze({
585
+ notHttps: 'fill_credential needs an https page.',
586
+ originNotAllowed: (host) =>
587
+ `This credential is not allowed on ${String(host).slice(0, 253)}.`,
588
+ denied: (code) =>
589
+ `The credential could not be used here (${String(code).slice(0, 40)}).`,
590
+ originChanged:
591
+ 'The page changed origin while the credential was being filled; nothing was kept in the field.',
592
+ elementNotAllowed: (field) =>
593
+ `This element cannot take the "${String(field).slice(0, 64)}" field. Pick the matching input from browser_snapshot.`,
594
+ notFocused: 'The field did not keep focus; nothing was typed.'
595
+ });
596
+
597
+ /** The hostname of an https: URL, else null. */
598
+ function httpsHost(url) {
599
+ try {
600
+ const parsed = new URL(url);
601
+ return parsed.protocol === 'https:' ? parsed.hostname.toLowerCase() : null;
602
+ } catch {
603
+ return null;
604
+ }
605
+ }
606
+
607
+ async function frameUrlOf(handle, page) {
608
+ try {
609
+ const frame = await handle?.ownerFrame?.();
610
+ if (frame) return frame.url();
611
+ } catch {
612
+ /* detached: the caller re-checks */
613
+ }
614
+ return page.url();
615
+ }
616
+
617
+ async function clearField(locator) {
618
+ try {
619
+ await locator.fill('', { timeout: 3000 });
620
+ return;
621
+ } catch {
622
+ /* fall through to the DOM setter */
623
+ }
624
+ await locator
625
+ .evaluate((el) => {
626
+ const proto =
627
+ el.tagName === 'TEXTAREA'
628
+ ? HTMLTextAreaElement.prototype
629
+ : HTMLInputElement.prototype;
630
+ const setter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
631
+ if (setter) setter.call(el, '');
632
+ else el.value = '';
633
+ el.dispatchEvent(new Event('input', { bubbles: true }));
634
+ el.dispatchEvent(new Event('change', { bubbles: true }));
635
+ })
636
+ .catch(() => undefined);
637
+ }
638
+
639
+ /**
640
+ * Type a credential field into a page element without the value ever
641
+ * reaching the agent (contract "fill_credential in the broker"):
642
+ *
643
+ * 2. the target must be an element that fits the field (a password goes
644
+ * only into a password input, ...), else `element_not_allowed`;
645
+ * 3. the top-level page must be https; `host` / `frameHost` are read from
646
+ * Playwright (CDP navigation state), not from page script;
647
+ * 4. only then is the value asked for (`resolve`, the API's `cred_resolve`
648
+ * with the single-use nonce). A refusal is `origin_not_allowed`
649
+ * (host_not_allowed) or `credential_denied`, and NO value was sent;
650
+ * 5. the origin is read AGAIN after the value arrives: a page that
651
+ * navigated during the round trip gets nothing (`origin_changed`).
652
+ * Then focus, clear, CDP `Input.insertText` (one trusted insertion, no
653
+ * per-key events a page could log as keystrokes beyond what any typing
654
+ * gives it);
655
+ * 6. and again after typing: if the origin moved, the field is cleared and
656
+ * the result is `origin_changed`.
657
+ *
658
+ * `onValue(value)` adds the value to the chat's redaction set as soon as it
659
+ * exists; `onTyped(handle)` registers the element for screenshot masking.
660
+ * The result never carries the value.
661
+ *
662
+ * @param {import('playwright-core').Page} page
663
+ * @param {{ target: string, credentialId: string, field: string, nonce: string }} args
664
+ * @param {{ cdp: { send: Function },
665
+ * resolve: (req: { credentialId, field, nonce, host, frameHost }) => Promise<{ values?: Record<string,string> }>,
666
+ * onValue?: (value: string) => void, onTyped?: (handle) => void }} deps
667
+ */
668
+ export async function fillCredential(page, args, deps) {
669
+ const target = requireString(args, 'target');
670
+ const credentialId = requireString(args, 'credentialId');
671
+ const field = requireString(args, 'field');
672
+ const nonce = requireString(args, 'nonce');
673
+ if (!CREDENTIAL_ID_RE.test(credentialId))
674
+ throw new BrokerError('invalid_args', 'credentialId must be a uuid');
675
+ if (!FIELD_RE.test(field))
676
+ throw new BrokerError('invalid_args', 'field is not a credential field');
677
+ if (nonce.length > 2048)
678
+ throw new BrokerError('invalid_args', 'nonce is too long');
679
+
680
+ // 2. The element.
681
+ const locator = await locate(page, target, TOOL_ERRORS.staleTargetShort);
682
+ let info;
683
+ try {
684
+ info = await locator.evaluate((el) => ({
685
+ tag: el.tagName,
686
+ type: el.tagName === 'INPUT' ? String(el.type || '').toLowerCase() : '',
687
+ autocomplete: el.getAttribute('autocomplete') || '',
688
+ readOnly: !!el.readOnly,
689
+ disabled: !!el.disabled
690
+ }));
691
+ } catch (error) {
692
+ if (/detached|not attached/i.test(errorText(error)))
693
+ throw new BrokerError('stale_ref', TOOL_ERRORS.staleTargetShort);
694
+ throw new BrokerError('internal', errorText(error));
695
+ }
696
+ if (!elementAllowsField(field, info))
697
+ throw new BrokerError(
698
+ 'element_not_allowed',
699
+ FILL_ERRORS.elementNotAllowed(field)
700
+ );
701
+
702
+ // 3. The origin, before anything is asked for.
703
+ const host = httpsHost(page.url());
704
+ if (!host) throw new BrokerError('origin_not_allowed', FILL_ERRORS.notHttps);
705
+ // The handle pins THIS element for the origin re-checks and the
706
+ // screenshot mask, whatever the page later does to its attributes.
707
+ const handle = await locator
708
+ .elementHandle({ timeout: 3000 })
709
+ .catch(() => null);
710
+ if (!handle) throw new BrokerError('stale_ref', TOOL_ERRORS.staleTargetShort);
711
+ const frameHost = httpsHost(await frameUrlOf(handle, page));
712
+ if (!frameHost)
713
+ throw new BrokerError('origin_not_allowed', FILL_ERRORS.notHttps);
714
+
715
+ // 4. The value.
716
+ let value;
717
+ try {
718
+ const result = await deps.resolve({
719
+ credentialId,
720
+ field,
721
+ nonce,
722
+ host,
723
+ frameHost
724
+ });
725
+ value = result?.values?.[field];
726
+ } catch (error) {
727
+ const code = typeof error?.code === 'string' ? error.code : 'unavailable';
728
+ if (code === 'host_not_allowed')
729
+ throw new BrokerError(
730
+ 'origin_not_allowed',
731
+ FILL_ERRORS.originNotAllowed(host)
732
+ );
733
+ if (code === 'timeout')
734
+ throw new BrokerError('timeout', FILL_ERRORS.denied(code));
735
+ throw new BrokerError('credential_denied', FILL_ERRORS.denied(code));
736
+ }
737
+ if (typeof value !== 'string' || value.length === 0 || value.length > 16384)
738
+ throw new BrokerError('credential_denied', FILL_ERRORS.denied('no_value'));
739
+ deps.onValue?.(value);
740
+
741
+ // 5. Re-check the origin, then type.
742
+ const stillHere = async () =>
743
+ httpsHost(page.url()) === host &&
744
+ httpsHost(await frameUrlOf(handle, page)) === frameHost;
745
+ if (!(await stillHere())) {
746
+ value = null;
747
+ throw new BrokerError('origin_changed', FILL_ERRORS.originChanged);
748
+ }
749
+ try {
750
+ await locator.focus({ timeout: 5000 });
751
+ await clearField(locator);
752
+ const focused = await locator.evaluate((el) => {
753
+ const root = el.getRootNode();
754
+ return (root.activeElement ?? document.activeElement) === el;
755
+ });
756
+ if (!focused)
757
+ throw new BrokerError('element_not_allowed', FILL_ERRORS.notFocused);
758
+ } catch (error) {
759
+ value = null;
760
+ if (error instanceof BrokerError) throw error;
761
+ if (/detached|not attached/i.test(errorText(error)))
762
+ throw new BrokerError('stale_ref', TOOL_ERRORS.staleTargetShort);
763
+ throw new BrokerError('internal', errorText(error));
764
+ }
765
+ // Re-checked once more right before the insertion: focus/clear can run
766
+ // page handlers that navigate.
767
+ if (!(await stillHere())) {
768
+ value = null;
769
+ throw new BrokerError('origin_changed', FILL_ERRORS.originChanged);
770
+ }
771
+ deps.onTyped?.(handle);
772
+ try {
773
+ await deps.cdp.send('Input.insertText', { text: value });
774
+ } finally {
775
+ value = null; // drop the only reference held here
776
+ }
777
+
778
+ // 6. After typing.
779
+ if (!(await stillHere())) {
780
+ await clearField(locator);
781
+ throw new BrokerError('origin_changed', FILL_ERRORS.originChanged);
782
+ }
783
+ return { filled: true, credentialId, field, host };
784
+ }
785
+
786
+ export async function history(page, name, args, { cdp }) {
787
+ const options = { timeout: 15000, waitUntil: 'load' };
788
+ try {
789
+ if (name === 'back') await page.goBack(options);
790
+ else if (name === 'forward') await page.goForward(options);
791
+ else if (args.hard) {
792
+ await cdp.send('Page.reload', { ignoreCache: true });
793
+ await page.waitForLoadState('load', { timeout: 15000 });
794
+ } else await page.reload(options);
795
+ } catch {
796
+ /* 20x: settle after 15 s or on the first navigation, whichever is first */
797
+ }
798
+ return { ok: true, url: page.url() };
799
+ }