@namzu/browser 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/classifier.d.ts +22 -1
  3. package/dist/classifier.d.ts.map +1 -1
  4. package/dist/classifier.js +30 -7
  5. package/dist/classifier.js.map +1 -1
  6. package/dist/errors.d.ts +19 -1
  7. package/dist/errors.d.ts.map +1 -1
  8. package/dist/errors.js +26 -0
  9. package/dist/errors.js.map +1 -1
  10. package/dist/host.d.ts +42 -0
  11. package/dist/host.d.ts.map +1 -1
  12. package/dist/host.js +487 -70
  13. package/dist/host.js.map +1 -1
  14. package/dist/index.d.ts +3 -3
  15. package/dist/index.d.ts.map +1 -1
  16. package/dist/index.js +2 -2
  17. package/dist/index.js.map +1 -1
  18. package/dist/page-scripts.d.ts +20 -0
  19. package/dist/page-scripts.d.ts.map +1 -1
  20. package/dist/page-scripts.js +110 -0
  21. package/dist/page-scripts.js.map +1 -1
  22. package/dist/reach.d.ts +66 -0
  23. package/dist/reach.d.ts.map +1 -0
  24. package/dist/reach.js +189 -0
  25. package/dist/reach.js.map +1 -0
  26. package/dist/settle.d.ts +40 -0
  27. package/dist/settle.d.ts.map +1 -0
  28. package/dist/settle.js +30 -0
  29. package/dist/settle.js.map +1 -0
  30. package/dist/snapshot.d.ts +80 -2
  31. package/dist/snapshot.d.ts.map +1 -1
  32. package/dist/snapshot.js +326 -20
  33. package/dist/snapshot.js.map +1 -1
  34. package/dist/windows-engine.d.ts +1 -1
  35. package/dist/windows-engine.d.ts.map +1 -1
  36. package/dist/windows-engine.js +3 -1
  37. package/dist/windows-engine.js.map +1 -1
  38. package/package.json +3 -2
  39. package/src/classifier.ts +44 -7
  40. package/src/errors.ts +32 -0
  41. package/src/host.ts +608 -89
  42. package/src/index.ts +4 -0
  43. package/src/page-scripts.ts +108 -0
  44. package/src/reach.ts +211 -0
  45. package/src/settle.ts +60 -0
  46. package/src/snapshot.ts +402 -20
  47. package/src/windows-engine.ts +4 -1
package/dist/host.js CHANGED
@@ -1,13 +1,14 @@
1
1
  import { randomBytes } from 'node:crypto';
2
- import { canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome } from '@namzu/sdk';
3
- import { chromium, } from 'playwright-core';
4
- import { classifyHumanRequired, isCredentialField, } from './classifier.js';
2
+ import { BROWSER_FIND_MAX_MATCHES, canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome, } from '@namzu/sdk';
3
+ import { classifyHumanRequired, isCredentialField, signInNotice, } from './classifier.js';
5
4
  import { detectBrowserEnvironment, runnableBrowserPlan, } from './detect.js';
6
- import { BrowserHumanRequiredError, BrowserOriginMismatchError, BrowserOutcomeUnknownError, BrowserSiteDeniedError, BrowserStaleRefError, BrowserUnavailableError, ProfileBusyError, } from './errors.js';
7
- import { countCredentialFields, fieldFacts } from './page-scripts.js';
5
+ import { BrowserHumanRequiredError, BrowserNoEffectError, BrowserOriginMismatchError, BrowserOutcomeUnknownError, BrowserSiteDeniedError, BrowserStaleRefError, BrowserTargetUnclearError, BrowserUnavailableError, ProfileBusyError, } from './errors.js';
6
+ import { countCredentialFields, fieldFacts, hasVisibleContent, installMutationClock, msSinceLastChange, valueFacts, } from './page-scripts.js';
8
7
  import { BrowserSitePolicy, DEFAULT_BROWSER_SITE_RULES } from './policy.js';
9
8
  import { BrowserLeaseStore, BrowserProfileError, BrowserProfileStore, DEFAULT_BROWSER_PROFILE, } from './profiles.js';
10
- import { PLAYWRIGHT_CORE_VERSION, SnapshotPager, captureAriaTree, locateRef, renderAriaTree, } from './snapshot.js';
9
+ import { CANDIDATES_MAX, KEY_DELAY_MS, NAMED_KEYS, aimInside, deepFocus, describeCandidate, failedBeforeActing, focusIsOpaque, keyCanCommit, namedKey, noEffectReason, } from './reach.js';
10
+ import { waitUntilQuiet } from './settle.js';
11
+ import { PLAYWRIGHT_CORE_VERSION, SnapshotPager, buildEffect, captureAriaTree, diffLines, findInTree, locateRef, renderAriaTree, renderInteractive, renderReading, subtreeOf, } from './snapshot.js';
11
12
  import { WindowsBridgeError } from './windows-bridge.js';
12
13
  import { connectWindowsBrowser } from './windows-engine.js';
13
14
  const DEFAULT_NAVIGATION_TIMEOUT_MS = 30_000;
@@ -15,12 +16,38 @@ const DEFAULT_ACTION_TIMEOUT_MS = 10_000;
15
16
  /** How long a history move waits for the document after it committed. */
16
17
  const HISTORY_LOAD_WAIT_MS = 5_000;
17
18
  const SNAPSHOT_PAGE_MAX = 20_000;
18
- /** How long the host lets events (popups, downloads, dialogs) arrive after an action. */
19
- const EVENT_SETTLE_MS = 150;
19
+ /**
20
+ * A page is quiet after this long with no change to the document and no
21
+ * request in flight. It also bounds how long an event the browser has not
22
+ * announced yet (a popup, a download, a dialog) has to arrive.
23
+ */
24
+ const QUIET_MS = 250;
25
+ /** The most an action waits for quiet. A page that never settles is reported, not waited on. */
26
+ const ACT_SETTLE_CAP_MS = 2_000;
27
+ /** The same for a navigation, which loads more. */
28
+ const NAVIGATE_SETTLE_CAP_MS = 5_000;
29
+ /** The wait before an action, for events left over from the last one. */
30
+ const PRE_ACT_QUIET_MS = 100;
31
+ const PRE_ACT_CAP_MS = 500;
32
+ /** `wait_for` with `stable` and no time: how long it waits. */
33
+ const STABLE_DEFAULT_WAIT_MS = 5_000;
34
+ /** Request kinds that can change what the page shows. */
35
+ const PAGE_REQUEST_KINDS = new Set(['document', 'script', 'stylesheet', 'fetch', 'xhr']);
36
+ /** Lines of a `delta` snapshot that are listed in full. */
37
+ const DELTA_LINES_SHOWN = 80;
38
+ /** Lines of an effect the model is shown. */
39
+ const EFFECT_LINES_SHOWN = 10;
20
40
  /** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
21
41
  const CAPTCHA_FRAME_MIN_PX = 30;
22
42
  const REMEMBERED_DOCUMENTS_MAX = 64;
43
+ function evaluateOn(target, fn) {
44
+ return 'asElement' in target
45
+ ? target.evaluate(fn)
46
+ : target.evaluate(fn);
47
+ }
23
48
  const SCREENSHOT_INPUT_MASK_COLOR = '#666666';
49
+ /** Most inputs one screenshot checks for a credential. */
50
+ const MASK_CANDIDATES_MAX = 200;
24
51
  function firstLine(error) {
25
52
  const text = error instanceof Error ? error.message : String(error);
26
53
  return (text.split('\n')[0] ?? '').slice(0, 300);
@@ -156,6 +183,8 @@ export class PlaywrightBrowserHost {
156
183
  this.context = context;
157
184
  context.setDefaultTimeout(this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
158
185
  context.setDefaultNavigationTimeout(this.options.navigationTimeoutMs ?? DEFAULT_NAVIGATION_TIMEOUT_MS);
186
+ // Every new document keeps a clock of its last change, for the quiet-page wait.
187
+ await context.addInitScript(installMutationClock).catch(() => undefined);
159
188
  context.on('page', (page) => this.adopt(page));
160
189
  context.on('close', () => this.forget(context));
161
190
  context.browser()?.on('disconnected', () => this.forget(context));
@@ -163,8 +192,10 @@ export class PlaywrightBrowserHost {
163
192
  // a site the rules do not allow is stopped here, so its address (and
164
193
  // anything a page smuggled into it) never reaches the network.
165
194
  await context.route(() => true, (route) => this.screen(route));
166
- for (const page of context.pages())
195
+ for (const page of context.pages()) {
167
196
  this.adopt(page, false);
197
+ void page.evaluate(installMutationClock).catch(() => undefined);
198
+ }
168
199
  return context;
169
200
  }
170
201
  home() {
@@ -179,6 +210,10 @@ export class PlaywrightBrowserHost {
179
210
  });
180
211
  let context;
181
212
  try {
213
+ // Loaded here, on the first browser launch: playwright-core is several
214
+ // megabytes of source that a session which never opens a browser must
215
+ // not compile at startup.
216
+ const { chromium } = await import('playwright-core');
182
217
  context = await chromium.launchPersistentContext(descriptor.userDataDir, {
183
218
  headless: plan.headless,
184
219
  ...(plan.channel ? { channel: plan.channel } : {}),
@@ -338,6 +373,8 @@ export class PlaywrightBrowserHost {
338
373
  mainFrameCommits: 0,
339
374
  mainDocumentResponses: 0,
340
375
  responsesByLoader: new Map(),
376
+ inflight: new Set(),
377
+ lastNetworkAt: Date.now(),
341
378
  };
342
379
  this.tabs.set(tab.id, tab);
343
380
  this.tabOf.set(page, tab);
@@ -366,6 +403,19 @@ export class PlaywrightBrowserHost {
366
403
  // A response without a frame (a service worker's) says nothing about the page.
367
404
  }
368
405
  });
406
+ const kind = (request) => PAGE_REQUEST_KINDS.has(request.resourceType());
407
+ page.on('request', (request) => {
408
+ if (!kind(request))
409
+ return;
410
+ tab.inflight.add(request);
411
+ tab.lastNetworkAt = Date.now();
412
+ });
413
+ const ended = (request) => {
414
+ if (tab.inflight.delete(request))
415
+ tab.lastNetworkAt = Date.now();
416
+ };
417
+ page.on('requestfinished', ended);
418
+ page.on('requestfailed', ended);
369
419
  page.on('dialog', (dialog) => this.onDialog(tab, dialog));
370
420
  page.on('download', (download) => {
371
421
  const name = download.suggestedFilename();
@@ -435,10 +485,37 @@ export class PlaywrightBrowserHost {
435
485
  .finally(() => this.blanking.delete(blank));
436
486
  this.blanking.add(blank);
437
487
  }
438
- async settle() {
439
- await new Promise((resolve) => setTimeout(resolve, EVENT_SETTLE_MS));
488
+ /**
489
+ * Wait until the tab's page is quiet (nothing changed in it and no request
490
+ * in flight for `quietMs`), or `capMs` has passed. Events the browser
491
+ * announces late (a popup, a download, a dialog) arrive inside the window.
492
+ * Then let any tab that was cleared to `about:blank` finish clearing.
493
+ */
494
+ async settle(tab, options = {
495
+ quietMs: QUIET_MS,
496
+ capMs: ACT_SETTLE_CAP_MS,
497
+ }) {
498
+ const outcome = await waitUntilQuiet({ now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }, this.quietProbe(tab), options);
440
499
  while (this.blanking.size > 0)
441
500
  await Promise.all([...this.blanking]);
501
+ return outcome;
502
+ }
503
+ quietProbe(tab) {
504
+ return {
505
+ stillForMs: async () => {
506
+ // A dialog pauses the page; nothing more will happen until it is answered.
507
+ if (this.pendingDialog?.tab === tab || tab.page.isClosed())
508
+ return Number.MAX_SAFE_INTEGER;
509
+ if (tab.inflight.size > 0)
510
+ return 0;
511
+ const sinceNetwork = Date.now() - tab.lastNetworkAt;
512
+ const sincePage = await Promise.race([
513
+ tab.page.evaluate(msSinceLastChange).catch(() => 0),
514
+ new Promise((resolve) => setTimeout(() => resolve(0), 1000)),
515
+ ]);
516
+ return Math.min(sinceNetwork, sincePage);
517
+ },
518
+ };
442
519
  }
443
520
  async activeTab() {
444
521
  const context = await this.ensureContext();
@@ -527,19 +604,29 @@ export class PlaywrightBrowserHost {
527
604
  // A frame that went away while we looked is not a challenge.
528
605
  }
529
606
  }
530
- const reason = classifyHumanRequired({
607
+ const signals = {
531
608
  url,
532
609
  title: await this.title(tab),
533
610
  ...(tab.status !== undefined ? { status: tab.status } : {}),
534
611
  ...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
535
- ...(tab.proxyAuthenticate !== undefined
536
- ? { proxyAuthenticate: tab.proxyAuthenticate }
537
- : {}),
612
+ ...(tab.proxyAuthenticate !== undefined ? { proxyAuthenticate: tab.proxyAuthenticate } : {}),
538
613
  ...counts,
539
614
  frameUrls,
540
- }, this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {});
615
+ };
616
+ const classifier = {
617
+ ...(this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {}),
618
+ ...(this.options.signInDetection ? { signInDetection: this.options.signInDetection } : {}),
619
+ };
620
+ const reason = classifyHumanRequired(signals, classifier);
541
621
  if (reason)
542
622
  throw this.humanRequired(reason, tab);
623
+ // A page with a password field that did not stop the turn: say so once
624
+ // per address, so the model reports it instead of trying to type.
625
+ const notice = signInNotice(signals, classifier, this.loginCommandFor(url));
626
+ if (notice && tab.signInNoticeFor !== withoutQuery(url)) {
627
+ tab.signInNoticeFor = withoutQuery(url);
628
+ this.notes.push(notice);
629
+ }
543
630
  if (tab.status === 401) {
544
631
  throw new Error('The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.');
545
632
  }
@@ -676,7 +763,7 @@ export class PlaywrightBrowserHost {
676
763
  case 'reload':
677
764
  return this.history(action.action);
678
765
  case 'snapshot':
679
- return this.snapshot(action.ref, action.cursor);
766
+ return this.snapshot(action);
680
767
  case 'screenshot':
681
768
  return this.screenshot(action.ref, action.fullPage);
682
769
  case 'scroll':
@@ -685,6 +772,8 @@ export class PlaywrightBrowserHost {
685
772
  return this.waitFor(action);
686
773
  case 'tabs':
687
774
  return this.tabsCall(action);
775
+ case 'value':
776
+ return this.readValue(action.ref);
688
777
  }
689
778
  }
690
779
  async navigate(tab, rawUrl) {
@@ -705,7 +794,7 @@ export class PlaywrightBrowserHost {
705
794
  catch (error) {
706
795
  await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined);
707
796
  if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
708
- await this.settle();
797
+ await this.settle(tab);
709
798
  throw new Error('The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.');
710
799
  }
711
800
  if (!/ERR_BLOCKED_BY_CLIENT|ERR_ABORTED|Download is starting/.test(firstLine(error))) {
@@ -716,10 +805,35 @@ export class PlaywrightBrowserHost {
716
805
  return this.afterNavigation(tab);
717
806
  }
718
807
  async afterNavigation(tab) {
719
- await this.settle();
808
+ // The document first (a page that has not finished loading has not
809
+ // finished building itself), then the same quiet the actions wait for.
810
+ await tab.page
811
+ .waitForLoadState('load', { timeout: NAVIGATE_SETTLE_CAP_MS })
812
+ .catch(() => undefined);
813
+ const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: NAVIGATE_SETTLE_CAP_MS });
720
814
  this.checkLanding(tab);
721
- await this.settle();
815
+ while (this.blanking.size > 0)
816
+ await Promise.all([...this.blanking]);
722
817
  await this.checkHuman(tab);
818
+ if (!settled.quiet) {
819
+ this.notes.push(`The page was still changing after ${settled.waitedMs} ms, so what it shows may be unfinished. Call browser with wait_for and stable, or take a snapshot.`);
820
+ }
821
+ const populated = originOf(tab.page.url()) === 'null' ||
822
+ (await tab.page.evaluate(hasVisibleContent).catch(() => true));
823
+ if (!populated) {
824
+ this.notes.push(`The page has nothing to read yet (waited ${settled.waitedMs} ms). Call browser with wait_for and stable, or reload.`);
825
+ }
826
+ // The model's next step after opening a page is to read it, so the
827
+ // reading comes with the open: one request fewer on every page visit.
828
+ if (populated && originOf(tab.page.url()) !== 'null') {
829
+ try {
830
+ const snapshot = await this.takeSnapshot(tab);
831
+ return this.result({ page: snapshot.page, snapshot });
832
+ }
833
+ catch {
834
+ // An unreadable page still opened; say where we are and let the model ask again.
835
+ }
836
+ }
723
837
  return this.result({ page: await this.pageInfo(tab) });
724
838
  }
725
839
  async history(which) {
@@ -768,7 +882,16 @@ export class PlaywrightBrowserHost {
768
882
  await pending.dialog.dismiss().catch(() => undefined);
769
883
  this.notes.push('The open dialog was dismissed.');
770
884
  }
771
- async takeSnapshot(tab, ref) {
885
+ /** Read the page's tree and remember its refs. */
886
+ async readTree(tab) {
887
+ const { nodes, hidden, redact } = await captureAriaTree(tab.page, this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
888
+ const whole = renderAriaTree(nodes, hidden, undefined, redact);
889
+ if (!whole)
890
+ throw new Error('The page could not be read.');
891
+ this.refs = whole.refs;
892
+ return { nodes, hidden, redact, whole };
893
+ }
894
+ async takeSnapshot(tab, request = {}) {
772
895
  const pending = this.pendingDialog;
773
896
  if (pending && pending.tab === tab) {
774
897
  const dialog = pending.dialog;
@@ -780,17 +903,67 @@ export class PlaywrightBrowserHost {
780
903
  lines.push(' - (the page is paused until the dialog is answered)');
781
904
  return { page: await this.pageInfo(tab), text: lines.join('\n') };
782
905
  }
783
- const { nodes, hidden, redact } = await captureAriaTree(tab.page, this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
784
- const whole = renderAriaTree(nodes, hidden, undefined, redact);
785
- if (!whole)
786
- throw new Error('The page could not be read.');
787
- this.refs = whole.refs;
788
- let text = whole.text;
789
- if (ref !== undefined) {
790
- const region = renderAriaTree(nodes, hidden, ref, redact);
791
- if (!region || !whole.refs.has(ref))
792
- throw new BrowserStaleRefError(ref);
793
- text = region.text;
906
+ const { ref, view = 'full', query, role } = request;
907
+ const { nodes, hidden, redact, whole } = await this.readTree(tab);
908
+ if (ref !== undefined && !whole.refs.has(ref))
909
+ throw new BrowserStaleRefError(ref);
910
+ // A region is cut from the whole-page tree: a snapshot of one element would replace the ref table.
911
+ const scope = ref === undefined ? nodes : (subtreeOf(nodes, ref) ?? []);
912
+ let text;
913
+ if (query !== undefined || role !== undefined) {
914
+ const found = findInTree(scope, hidden, redact, { ...(query !== undefined ? { query } : {}), ...(role !== undefined ? { role } : {}) }, BROWSER_FIND_MAX_MATCHES);
915
+ text = found.lines.join('\n');
916
+ const what = [
917
+ query !== undefined ? `text ${quoteShort(query, 60)}` : '',
918
+ role !== undefined ? `role ${quoteShort(role, 30)}` : '',
919
+ ]
920
+ .filter(Boolean)
921
+ .join(' and ');
922
+ this.notes.unshift(found.total === 0
923
+ ? `Nothing on the page matches ${what}.`
924
+ : found.total > found.lines.length
925
+ ? `${found.total} elements match ${what}; the first ${found.lines.length} are listed. Narrow the search.`
926
+ : `${found.total} element${found.total === 1 ? '' : 's'} match${found.total === 1 ? 'es' : ''} ${what}.`);
927
+ }
928
+ else if (view === 'interactive') {
929
+ text = renderInteractive(scope, hidden, redact);
930
+ tab.lastSeen = whole.text;
931
+ }
932
+ else if (view === 'reading') {
933
+ text = renderReading(scope, hidden, redact);
934
+ tab.lastSeen = whole.text;
935
+ }
936
+ else if (view === 'delta') {
937
+ const before = tab.lastSeen;
938
+ if (before === undefined) {
939
+ this.notes.unshift('There is no earlier snapshot of this tab to compare with; this is the whole page.');
940
+ text =
941
+ ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '');
942
+ }
943
+ else {
944
+ const diff = diffLines(before, whole.text, DELTA_LINES_SHOWN);
945
+ text =
946
+ diff.added === 0 && diff.removed === 0
947
+ ? '(no change since the last snapshot)'
948
+ : [
949
+ ...diff.addedLines.map((l) => `+ ${l}`),
950
+ ...(diff.added > diff.addedLines.length
951
+ ? [` (${diff.added - diff.addedLines.length} more lines added)`]
952
+ : []),
953
+ ...diff.removedLines.map((l) => `- ${l}`),
954
+ ...(diff.removed > diff.removedLines.length
955
+ ? [` (${diff.removed - diff.removedLines.length} more lines removed)`]
956
+ : []),
957
+ ].join('\n');
958
+ this.notes.unshift(`Changes since the last snapshot: ${diff.added} lines added, ${diff.removed} removed. Take a full snapshot to see the whole page again.`);
959
+ }
960
+ tab.lastSeen = whole.text;
961
+ }
962
+ else {
963
+ text =
964
+ ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '');
965
+ if (ref === undefined)
966
+ tab.lastSeen = whole.text;
794
967
  }
795
968
  if (text.trim() === '')
796
969
  text = '(nothing readable on this page)';
@@ -801,10 +974,10 @@ export class PlaywrightBrowserHost {
801
974
  ...(first.nextCursor !== undefined ? { nextCursor: first.nextCursor } : {}),
802
975
  };
803
976
  }
804
- async snapshot(ref, cursor) {
977
+ async snapshot(request) {
805
978
  const tab = await this.activeTab();
806
- if (cursor !== undefined) {
807
- const next = this.pager.resume(cursor);
979
+ if (request.cursor !== undefined) {
980
+ const next = this.pager.resume(request.cursor);
808
981
  if (!next) {
809
982
  throw new Error('That cursor belongs to an older snapshot. Take a new snapshot and page through it.');
810
983
  }
@@ -816,8 +989,15 @@ export class PlaywrightBrowserHost {
816
989
  },
817
990
  });
818
991
  }
819
- await this.settle();
820
- return this.result({ snapshot: await this.takeSnapshot(tab, ref) });
992
+ await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS });
993
+ return this.result({
994
+ snapshot: await this.takeSnapshot(tab, {
995
+ ...(request.ref !== undefined ? { ref: request.ref } : {}),
996
+ ...(request.view !== undefined ? { view: request.view } : {}),
997
+ ...(request.query !== undefined ? { query: request.query } : {}),
998
+ ...(request.role !== undefined ? { role: request.role } : {}),
999
+ }),
1000
+ });
821
1001
  }
822
1002
  async resolveRef(tab, ref) {
823
1003
  if (!this.refs.has(ref))
@@ -839,10 +1019,12 @@ export class PlaywrightBrowserHost {
839
1019
  if (this.pendingDialog?.tab === tab) {
840
1020
  throw new Error('A dialog is open; dismiss it with browser_act dialog before taking a screenshot.');
841
1021
  }
842
- // Screenshots are image blocks sent to the model. Mask DOM-addressable
843
- // inputs regardless of their names, including child frames and refs.
844
- // Closed shadow roots and page-drawn text remain outside this mask.
845
- const mask = tab.page.frames().map((frame) => frame.locator('input, textarea'));
1022
+ // Screenshots are image blocks sent to the model. Mask the fields that
1023
+ // take a password or a one-time code, in every frame; the other inputs
1024
+ // stay visible, because a search box or an address field the model
1025
+ // cannot read cannot be checked. Closed shadow roots and page-drawn
1026
+ // text remain outside this mask.
1027
+ const mask = await this.credentialMask(tab);
846
1028
  const data = ref !== undefined
847
1029
  ? await (await this.resolveRef(tab, ref)).screenshot({
848
1030
  type: 'png',
@@ -867,6 +1049,59 @@ export class PlaywrightBrowserHost {
867
1049
  },
868
1050
  });
869
1051
  }
1052
+ /** Locators of the credential fields (password, one-time code) of every frame of the tab. */
1053
+ async credentialMask(tab) {
1054
+ const out = [];
1055
+ for (const frame of tab.page.frames()) {
1056
+ const inputs = frame.locator('input, textarea');
1057
+ let count = 0;
1058
+ try {
1059
+ count = Math.min(await inputs.count(), MASK_CANDIDATES_MAX);
1060
+ }
1061
+ catch {
1062
+ // A frame that went away while we looked has nothing to mask.
1063
+ continue;
1064
+ }
1065
+ const verdicts = await Promise.all(Array.from({ length: count }, async (_, index) => {
1066
+ try {
1067
+ return isCredentialField(await inputs.nth(index).evaluate(fieldFacts, undefined, { timeout: 2000 }));
1068
+ }
1069
+ catch {
1070
+ // A field that cannot be asked about is hidden to be safe.
1071
+ return true;
1072
+ }
1073
+ }));
1074
+ verdicts.forEach((hide, index) => {
1075
+ if (hide)
1076
+ out.push(inputs.nth(index));
1077
+ });
1078
+ }
1079
+ return out;
1080
+ }
1081
+ /** One element's current value and state, read from the page itself. */
1082
+ async readValue(ref) {
1083
+ const tab = await this.activeTab();
1084
+ if (this.pendingDialog?.tab === tab) {
1085
+ throw new Error('A dialog is open; dismiss it with browser_act dialog before reading a value.');
1086
+ }
1087
+ const locator = await this.resolveRef(tab, ref);
1088
+ const facts = await locator.evaluate(fieldFacts);
1089
+ const described = this.refs.get(ref);
1090
+ const page = await this.pageInfo(tab);
1091
+ const hidden = isCredentialField(facts);
1092
+ const read = hidden ? undefined : await locator.evaluate(valueFacts);
1093
+ const value = {
1094
+ page,
1095
+ ref,
1096
+ ...(described?.role !== undefined ? { role: described.role } : {}),
1097
+ ...(described?.name !== undefined ? { name: described.name } : {}),
1098
+ ...(read?.value !== undefined ? { value: read.value } : {}),
1099
+ state: read?.state ?? [],
1100
+ ...(read?.options !== undefined ? { options: read.options } : {}),
1101
+ ...(hidden ? { hidden: true } : {}),
1102
+ };
1103
+ return this.result({ value });
1104
+ }
870
1105
  async scroll(direction, ref, amount) {
871
1106
  const tab = await this.activeTab();
872
1107
  const screens = amount ?? 1;
@@ -913,10 +1148,17 @@ export class PlaywrightBrowserHost {
913
1148
  : `${quoteShort(text)} was still there after ${timeout} ms.`);
914
1149
  }
915
1150
  }
916
- else {
1151
+ else if (action.stable !== true) {
917
1152
  await tab.page.waitForTimeout(timeout);
918
1153
  this.notes.unshift(`Waited ${timeout} ms.`);
919
1154
  }
1155
+ if (action.stable === true) {
1156
+ const cap = action.timeMs !== undefined ? timeout : STABLE_DEFAULT_WAIT_MS;
1157
+ const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: cap });
1158
+ this.notes.push(settled.quiet
1159
+ ? `The page is quiet (nothing changed for ${QUIET_MS} ms; waited ${settled.waitedMs} ms).`
1160
+ : `The page was still changing after ${settled.waitedMs} ms.`);
1161
+ }
920
1162
  return this.result({ page: await this.pageInfo(tab) });
921
1163
  }
922
1164
  async tabsCall(action) {
@@ -974,15 +1216,15 @@ export class PlaywrightBrowserHost {
974
1216
  // Acting
975
1217
  // -------------------------------------------------------------------------
976
1218
  async guardField(locator) {
977
- const facts = await locator.evaluate(fieldFacts);
1219
+ const facts = await evaluateOn(locator, fieldFacts);
978
1220
  if (isCredentialField(facts)) {
979
- throw new Error('This action targets a password or one-time-code control. Ask the operator to use the browser window.');
1221
+ throw new Error('This action targets a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.');
980
1222
  }
981
1223
  return facts;
982
1224
  }
983
1225
  /** Refuse direct form submission while a form holds a password or code input. */
984
1226
  async guardCredentialForm(locator, action, key) {
985
- const context = await locator.evaluate((element) => {
1227
+ const context = await evaluateOn(locator, (element) => {
986
1228
  const label = element.closest('label');
987
1229
  const labelledField = label?.control
988
1230
  ? {
@@ -1023,16 +1265,62 @@ export class PlaywrightBrowserHost {
1023
1265
  if ((action === 'click' && context.isFormButton) ||
1024
1266
  (action === 'press' && (context.isFormButton || /enter|return/i.test(key ?? ''))) ||
1025
1267
  action === 'type-submit') {
1026
- throw new Error('This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.');
1268
+ throw new Error('This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.');
1269
+ }
1270
+ }
1271
+ /**
1272
+ * The page's tree as the model could have read it just before an action,
1273
+ * or `undefined` when it cannot be read now (a dialog holds the page, or
1274
+ * the read failed). An effect is a bonus; it never fails the action.
1275
+ */
1276
+ async effectBaseline(tab) {
1277
+ if (this.pendingDialog?.tab === tab)
1278
+ return undefined;
1279
+ try {
1280
+ return (await this.readTree(tab)).whole.text;
1281
+ }
1282
+ catch {
1283
+ return undefined;
1284
+ }
1285
+ }
1286
+ /** What the action did, from the tree before and the tree now. */
1287
+ async effectSince(before, urlBefore, tabsBefore, tab, settled) {
1288
+ if (before === undefined || this.pendingDialog?.tab === tab || tab.page.isClosed())
1289
+ return undefined;
1290
+ let after;
1291
+ try {
1292
+ after = (await this.readTree(tab)).whole.text;
1027
1293
  }
1294
+ catch {
1295
+ return undefined;
1296
+ }
1297
+ return buildEffect({
1298
+ beforeTree: before,
1299
+ afterTree: after,
1300
+ urlBefore,
1301
+ urlNow: tab.page.url(),
1302
+ tabsBefore,
1303
+ tabsNow: this.tabs.size,
1304
+ settle: { quiet: settled.quiet, waitedMs: settled.waitedMs, capMs: settled.capMs },
1305
+ }, EFFECT_LINES_SHOWN);
1028
1306
  }
1029
1307
  async runAct(action) {
1030
1308
  const tab = await this.activeTab();
1031
- await this.settle();
1309
+ await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS });
1032
1310
  const expected = canonicalizeBrowserOrigin(action.origin);
1033
1311
  const live = originOf(tab.page.url());
1034
1312
  if (!expected.ok || expected.origin !== live) {
1035
- throw new BrowserOriginMismatchError(expected.ok ? expected.origin : action.origin, live === 'null' ? 'about:blank' : live);
1313
+ // The page moved since the snapshot. Going on is fine only when the
1314
+ // site rules treat the new origin exactly as the named one: both at
1315
+ // `act`, where no review stood between the model and the page.
1316
+ const sameRules = expected.ok &&
1317
+ live !== 'null' &&
1318
+ this.policy.levelOf(expected.origin) === 'act' &&
1319
+ this.policy.levelOf(live) === 'act';
1320
+ if (!sameRules) {
1321
+ throw new BrowserOriginMismatchError(expected.ok ? expected.origin : action.origin, live === 'null' ? 'about:blank' : live);
1322
+ }
1323
+ this.notes.unshift(`The page is now at ${live}, not ${expected.origin}; the site rules treat both alike, so the action went ahead there.`);
1036
1324
  }
1037
1325
  if (!this.policy.canAct(live)) {
1038
1326
  throw new BrowserSiteDeniedError(live, `${live} may be read but not changed under the site rules.`);
@@ -1043,11 +1331,16 @@ export class PlaywrightBrowserHost {
1043
1331
  throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.');
1044
1332
  }
1045
1333
  const checkpoint = await this.beginDocumentResponseChange(tab);
1334
+ const urlBefore = tab.page.url();
1335
+ const tabsBefore = this.tabs.size;
1336
+ const treeBefore = await this.effectBaseline(tab);
1337
+ let settled;
1046
1338
  try {
1047
1339
  await this.perform(tab, action);
1048
- await this.settle();
1340
+ settled = await this.settle(tab);
1049
1341
  this.checkLanding(tab);
1050
- await this.settle();
1342
+ while (this.blanking.size > 0)
1343
+ await Promise.all([...this.blanking]);
1051
1344
  }
1052
1345
  catch (error) {
1053
1346
  await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined);
@@ -1056,9 +1349,14 @@ export class PlaywrightBrowserHost {
1056
1349
  await this.finishDocumentResponseChange(tab, checkpoint);
1057
1350
  await this.checkHuman(tab);
1058
1351
  const after = this.tabs.get(tab.id) ?? tab;
1059
- if (action.snapshot)
1060
- return this.result({ snapshot: await this.takeSnapshot(after) });
1061
- return this.result({ page: await this.pageInfo(after) });
1352
+ const effect = await this.effectSince(treeBefore, urlBefore, tabsBefore, after, settled);
1353
+ if (action.snapshot) {
1354
+ return this.result({
1355
+ snapshot: await this.takeSnapshot(after),
1356
+ ...(effect ? { effect } : {}),
1357
+ });
1358
+ }
1359
+ return this.result({ page: await this.pageInfo(after), ...(effect ? { effect } : {}) });
1062
1360
  }
1063
1361
  async answerDialog(tab, action) {
1064
1362
  const pending = this.pendingDialog;
@@ -1082,9 +1380,10 @@ export class PlaywrightBrowserHost {
1082
1380
  throw new BrowserOutcomeUnknownError('dialog', error);
1083
1381
  }
1084
1382
  this.notes.unshift('The dialog was dismissed.');
1085
- await this.settle();
1383
+ await this.settle(tab);
1086
1384
  this.checkLanding(tab);
1087
- await this.settle();
1385
+ while (this.blanking.size > 0)
1386
+ await Promise.all([...this.blanking]);
1088
1387
  await this.finishDocumentResponseChange(tab, checkpoint);
1089
1388
  await this.checkHuman(tab);
1090
1389
  if (action.snapshot)
@@ -1099,18 +1398,20 @@ export class PlaywrightBrowserHost {
1099
1398
  async perform(tab, action) {
1100
1399
  switch (action.action) {
1101
1400
  case 'click': {
1102
- const locator = await this.resolveRef(tab, action.ref);
1103
- await this.guardField(locator);
1104
- await this.guardCredentialForm(locator, 'click');
1105
- await locator.click({ trial: true });
1106
- await this.unknownOnFailure('click', () => action.doubleClick ? locator.dblclick() : locator.click());
1401
+ const target = await this.reach(tab, action);
1402
+ await this.guardField(target.locator);
1403
+ await this.guardCredentialForm(target.locator, 'click');
1404
+ const at = target.position ? { position: target.position } : {};
1405
+ await this.noEffectOnFailure('click', () => target.locator.click({ trial: true, ...at }));
1406
+ await this.unknownOnFailure('click', () => action.doubleClick ? target.locator.dblclick(at) : target.locator.click(at));
1107
1407
  return;
1108
1408
  }
1109
1409
  case 'hover': {
1110
- const locator = await this.resolveRef(tab, action.ref);
1111
- await this.guardField(locator);
1112
- await locator.hover({ trial: true });
1113
- await this.unknownOnFailure('hover', () => locator.hover());
1410
+ const target = await this.reach(tab, action);
1411
+ await this.guardField(target.locator);
1412
+ const at = target.position ? { position: target.position } : {};
1413
+ await this.noEffectOnFailure('hover', () => target.locator.hover({ trial: true, ...at }));
1414
+ await this.unknownOnFailure('hover', () => target.locator.hover(at));
1114
1415
  return;
1115
1416
  }
1116
1417
  case 'type': {
@@ -1118,9 +1419,27 @@ export class PlaywrightBrowserHost {
1118
1419
  await this.guardField(locator);
1119
1420
  if (action.submit)
1120
1421
  await this.guardCredentialForm(locator, 'type-submit');
1121
- await locator.click({ trial: true });
1122
- await this.unknownOnFailure('type', async () => {
1123
- await locator.fill(action.text);
1422
+ await this.noEffectOnFailure('type', () => locator.click({ trial: true }));
1423
+ await this.unknownOnFailure('type', async (begun) => {
1424
+ if (action.keys) {
1425
+ await locator.focus();
1426
+ // Clearing and typing change the field; from here a failure is not harmless.
1427
+ begun();
1428
+ if (action.append)
1429
+ await locator.press('Control+End');
1430
+ else
1431
+ await locator.fill('');
1432
+ await locator.pressSequentially(action.text, { delay: KEY_DELAY_MS });
1433
+ }
1434
+ else if (action.append) {
1435
+ const held = await locator
1436
+ .inputValue()
1437
+ .catch(() => locator.evaluate((el) => el.textContent ?? ''));
1438
+ await locator.fill(held + action.text);
1439
+ }
1440
+ else {
1441
+ await locator.fill(action.text);
1442
+ }
1124
1443
  if (action.submit)
1125
1444
  await locator.press('Enter');
1126
1445
  });
@@ -1166,7 +1485,13 @@ export class PlaywrightBrowserHost {
1166
1485
  await this.unknownOnFailure('press', () => locator.press(action.key));
1167
1486
  return;
1168
1487
  }
1169
- throw new Error('An untargeted key could reach a password or one-time-code control, including one inside a closed shadow root. Use a specific control instead.');
1488
+ const key = namedKey(action.key);
1489
+ if (!key) {
1490
+ throw new Error(`Without a ref, press takes only ${NAMED_KEYS.join(', ')}. Any other key goes to a specific control: give its ref.`);
1491
+ }
1492
+ await this.guardFocus(tab, key);
1493
+ await this.unknownOnFailure('press', () => tab.page.keyboard.press(key));
1494
+ return;
1170
1495
  }
1171
1496
  case 'upload': {
1172
1497
  const locator = await this.resolveRef(tab, action.ref);
@@ -1191,13 +1516,102 @@ export class PlaywrightBrowserHost {
1191
1516
  return;
1192
1517
  }
1193
1518
  }
1519
+ /**
1520
+ * The element a click or hover means, and where on it to aim. A ref names
1521
+ * it directly; a role with a name, or visible text, is looked up and
1522
+ * refused unless exactly one element matches. A container (a section, a
1523
+ * list item holding a menu title) is not clicked in its middle, which is
1524
+ * often empty: the aim goes to its first control or to its own text.
1525
+ */
1526
+ async reach(tab, target) {
1527
+ const found = target.ref !== undefined
1528
+ ? await this.resolveRef(tab, target.ref)
1529
+ : await this.findTarget(tab, target);
1530
+ let aim = { kind: 'centre' };
1531
+ try {
1532
+ aim = await found.evaluate(aimInside);
1533
+ }
1534
+ catch {
1535
+ // Aim at the middle, as before.
1536
+ }
1537
+ if (aim.kind === 'descendant')
1538
+ return { locator: found.locator(`xpath=${aim.path}`) };
1539
+ if (aim.kind === 'text')
1540
+ return { locator: found, position: { x: aim.x, y: aim.y } };
1541
+ return { locator: found };
1542
+ }
1543
+ /** Find the one element a role/name or text target means, or refuse with the candidates. */
1544
+ async findTarget(tab, target) {
1545
+ const page = tab.page;
1546
+ const role = target.role;
1547
+ const wanted = role !== undefined ? `${role} "${target.name ?? ''}"` : `text "${target.text ?? ''}"`;
1548
+ const build = (exact) => role !== undefined
1549
+ ? page.getByRole(role, {
1550
+ name: target.name ?? '',
1551
+ exact,
1552
+ })
1553
+ : page.getByText(target.text ?? '', { exact }).filter({ visible: true });
1554
+ let locator = build(false);
1555
+ let count = await locator.count();
1556
+ if (count > 1) {
1557
+ // A short name also fits longer ones ("Save" in "Save draft"): the exact matches decide.
1558
+ const exact = build(true);
1559
+ const exactCount = await exact.count();
1560
+ if (exactCount > 0) {
1561
+ locator = exact;
1562
+ count = exactCount;
1563
+ }
1564
+ }
1565
+ if (count === 1)
1566
+ return locator;
1567
+ const pool = count > 0 ? locator : role !== undefined ? page.getByRole(role) : undefined;
1568
+ const candidates = [];
1569
+ if (pool) {
1570
+ const total = Math.min(await pool.count(), CANDIDATES_MAX);
1571
+ for (let i = 0; i < total; i++) {
1572
+ candidates.push(await pool.nth(i).evaluate(describeCandidate));
1573
+ }
1574
+ }
1575
+ throw new BrowserTargetUnclearError(wanted, count, candidates);
1576
+ }
1577
+ /**
1578
+ * Refuse a key sent to the page when the focus is on a credential, or
1579
+ * when it cannot be told. Every frame is asked, because the key goes to
1580
+ * whichever one holds the focus and a stale focus in another frame is
1581
+ * still a reason to be careful.
1582
+ */
1583
+ async guardFocus(tab, key) {
1584
+ for (const frame of tab.page.frames()) {
1585
+ const handle = await frame.evaluateHandle(deepFocus).catch(() => undefined);
1586
+ const element = handle?.asElement() ?? null;
1587
+ if (!element)
1588
+ continue;
1589
+ if (keyCanCommit(key) && (await element.evaluate(focusIsOpaque))) {
1590
+ throw new Error(`The focus is on a component whose inside cannot be read, so ${key} could reach a password or code field. Use browser_act with the ref of a specific control.`);
1591
+ }
1592
+ await this.guardField(element);
1593
+ await this.guardCredentialForm(element, 'press', key);
1594
+ }
1595
+ }
1596
+ /** A check Playwright runs without acting (`trial`) failed: nothing on the page changed. */
1597
+ async noEffectOnFailure(action, run) {
1598
+ try {
1599
+ await run();
1600
+ }
1601
+ catch (error) {
1602
+ throw new BrowserNoEffectError(action, noEffectReason(error));
1603
+ }
1604
+ }
1194
1605
  /**
1195
1606
  * Run the change. A failure once it has started is `outcome unknown`. A
1196
1607
  * dialog the action opens ends the wait: the page is paused until the
1197
1608
  * dialog is answered, and so is anything Playwright was still waiting for.
1198
1609
  */
1199
1610
  async unknownOnFailure(action, run) {
1200
- const running = run();
1611
+ let touched = false;
1612
+ const running = run(() => {
1613
+ touched = true;
1614
+ });
1201
1615
  let timer;
1202
1616
  const dialogOpened = new Promise((resolve) => {
1203
1617
  timer = setInterval(() => {
@@ -1211,7 +1625,10 @@ export class PlaywrightBrowserHost {
1211
1625
  running.catch(() => undefined);
1212
1626
  }
1213
1627
  catch (error) {
1214
- throw new BrowserOutcomeUnknownError(action, error);
1628
+ // A timeout whose log never reached the input step left the page alone.
1629
+ throw !touched && failedBeforeActing(error)
1630
+ ? new BrowserNoEffectError(action, noEffectReason(error))
1631
+ : new BrowserOutcomeUnknownError(action, error);
1215
1632
  }
1216
1633
  finally {
1217
1634
  clearInterval(timer);