@namzu/browser 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/host.js CHANGED
@@ -1,12 +1,14 @@
1
1
  import { randomBytes } from 'node:crypto';
2
- import { canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome } from '@namzu/sdk';
3
- import { classifyHumanRequired, isCredentialField, } from './classifier.js';
2
+ import { BROWSER_FIND_MAX_MATCHES, canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome, } from '@namzu/sdk';
3
+ import { classifyHumanRequired, isCredentialField, signInNotice, } from './classifier.js';
4
4
  import { detectBrowserEnvironment, runnableBrowserPlan, } from './detect.js';
5
- import { BrowserHumanRequiredError, BrowserOriginMismatchError, BrowserOutcomeUnknownError, BrowserSiteDeniedError, BrowserStaleRefError, BrowserUnavailableError, ProfileBusyError, } from './errors.js';
6
- import { countCredentialFields, fieldFacts } from './page-scripts.js';
5
+ import { BrowserHumanRequiredError, BrowserNoEffectError, BrowserOriginMismatchError, BrowserOutcomeUnknownError, BrowserSiteDeniedError, BrowserStaleRefError, BrowserTargetUnclearError, BrowserUnavailableError, ProfileBusyError, } from './errors.js';
6
+ import { countCredentialFields, fieldFacts, hasVisibleContent, installMutationClock, msSinceLastChange, valueFacts, } from './page-scripts.js';
7
7
  import { BrowserSitePolicy, DEFAULT_BROWSER_SITE_RULES } from './policy.js';
8
8
  import { BrowserLeaseStore, BrowserProfileError, BrowserProfileStore, DEFAULT_BROWSER_PROFILE, } from './profiles.js';
9
- import { PLAYWRIGHT_CORE_VERSION, SnapshotPager, captureAriaTree, locateRef, renderAriaTree, } from './snapshot.js';
9
+ import { CANDIDATES_MAX, KEY_DELAY_MS, NAMED_KEYS, aimInside, deepFocus, describeCandidate, failedBeforeActing, focusIsOpaque, keyCanCommit, namedKey, noEffectReason, } from './reach.js';
10
+ import { waitUntilQuiet } from './settle.js';
11
+ import { PLAYWRIGHT_CORE_VERSION, SnapshotPager, buildEffect, captureAriaTree, diffLines, findInTree, locateRef, renderAriaTree, renderInteractive, renderReading, subtreeOf, } from './snapshot.js';
10
12
  import { WindowsBridgeError } from './windows-bridge.js';
11
13
  import { connectWindowsBrowser } from './windows-engine.js';
12
14
  const DEFAULT_NAVIGATION_TIMEOUT_MS = 30_000;
@@ -14,12 +16,38 @@ const DEFAULT_ACTION_TIMEOUT_MS = 10_000;
14
16
  /** How long a history move waits for the document after it committed. */
15
17
  const HISTORY_LOAD_WAIT_MS = 5_000;
16
18
  const SNAPSHOT_PAGE_MAX = 20_000;
17
- /** How long the host lets events (popups, downloads, dialogs) arrive after an action. */
18
- const EVENT_SETTLE_MS = 150;
19
+ /**
20
+ * A page is quiet after this long with no change to the document and no
21
+ * request in flight. It also bounds how long an event the browser has not
22
+ * announced yet (a popup, a download, a dialog) has to arrive.
23
+ */
24
+ const QUIET_MS = 250;
25
+ /** The most an action waits for quiet. A page that never settles is reported, not waited on. */
26
+ const ACT_SETTLE_CAP_MS = 2_000;
27
+ /** The same for a navigation, which loads more. */
28
+ const NAVIGATE_SETTLE_CAP_MS = 5_000;
29
+ /** The wait before an action, for events left over from the last one. */
30
+ const PRE_ACT_QUIET_MS = 100;
31
+ const PRE_ACT_CAP_MS = 500;
32
+ /** `wait_for` with `stable` and no time: how long it waits. */
33
+ const STABLE_DEFAULT_WAIT_MS = 5_000;
34
+ /** Request kinds that can change what the page shows. */
35
+ const PAGE_REQUEST_KINDS = new Set(['document', 'script', 'stylesheet', 'fetch', 'xhr']);
36
+ /** Lines of a `delta` snapshot that are listed in full. */
37
+ const DELTA_LINES_SHOWN = 80;
38
+ /** Lines of an effect the model is shown. */
39
+ const EFFECT_LINES_SHOWN = 10;
19
40
  /** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
20
41
  const CAPTCHA_FRAME_MIN_PX = 30;
21
42
  const REMEMBERED_DOCUMENTS_MAX = 64;
43
+ function evaluateOn(target, fn) {
44
+ return 'asElement' in target
45
+ ? target.evaluate(fn)
46
+ : target.evaluate(fn);
47
+ }
22
48
  const SCREENSHOT_INPUT_MASK_COLOR = '#666666';
49
+ /** Most inputs one screenshot checks for a credential. */
50
+ const MASK_CANDIDATES_MAX = 200;
23
51
  function firstLine(error) {
24
52
  const text = error instanceof Error ? error.message : String(error);
25
53
  return (text.split('\n')[0] ?? '').slice(0, 300);
@@ -155,6 +183,8 @@ export class PlaywrightBrowserHost {
155
183
  this.context = context;
156
184
  context.setDefaultTimeout(this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
157
185
  context.setDefaultNavigationTimeout(this.options.navigationTimeoutMs ?? DEFAULT_NAVIGATION_TIMEOUT_MS);
186
+ // Every new document keeps a clock of its last change, for the quiet-page wait.
187
+ await context.addInitScript(installMutationClock).catch(() => undefined);
158
188
  context.on('page', (page) => this.adopt(page));
159
189
  context.on('close', () => this.forget(context));
160
190
  context.browser()?.on('disconnected', () => this.forget(context));
@@ -162,8 +192,10 @@ export class PlaywrightBrowserHost {
162
192
  // a site the rules do not allow is stopped here, so its address (and
163
193
  // anything a page smuggled into it) never reaches the network.
164
194
  await context.route(() => true, (route) => this.screen(route));
165
- for (const page of context.pages())
195
+ for (const page of context.pages()) {
166
196
  this.adopt(page, false);
197
+ void page.evaluate(installMutationClock).catch(() => undefined);
198
+ }
167
199
  return context;
168
200
  }
169
201
  home() {
@@ -341,6 +373,8 @@ export class PlaywrightBrowserHost {
341
373
  mainFrameCommits: 0,
342
374
  mainDocumentResponses: 0,
343
375
  responsesByLoader: new Map(),
376
+ inflight: new Set(),
377
+ lastNetworkAt: Date.now(),
344
378
  };
345
379
  this.tabs.set(tab.id, tab);
346
380
  this.tabOf.set(page, tab);
@@ -369,6 +403,19 @@ export class PlaywrightBrowserHost {
369
403
  // A response without a frame (a service worker's) says nothing about the page.
370
404
  }
371
405
  });
406
+ const kind = (request) => PAGE_REQUEST_KINDS.has(request.resourceType());
407
+ page.on('request', (request) => {
408
+ if (!kind(request))
409
+ return;
410
+ tab.inflight.add(request);
411
+ tab.lastNetworkAt = Date.now();
412
+ });
413
+ const ended = (request) => {
414
+ if (tab.inflight.delete(request))
415
+ tab.lastNetworkAt = Date.now();
416
+ };
417
+ page.on('requestfinished', ended);
418
+ page.on('requestfailed', ended);
372
419
  page.on('dialog', (dialog) => this.onDialog(tab, dialog));
373
420
  page.on('download', (download) => {
374
421
  const name = download.suggestedFilename();
@@ -438,10 +485,37 @@ export class PlaywrightBrowserHost {
438
485
  .finally(() => this.blanking.delete(blank));
439
486
  this.blanking.add(blank);
440
487
  }
441
- async settle() {
442
- await new Promise((resolve) => setTimeout(resolve, EVENT_SETTLE_MS));
488
+ /**
489
+ * Wait until the tab's page is quiet (nothing changed in it and no request
490
+ * in flight for `quietMs`), or `capMs` has passed. Events the browser
491
+ * announces late (a popup, a download, a dialog) arrive inside the window.
492
+ * Then let any tab that was cleared to `about:blank` finish clearing.
493
+ */
494
+ async settle(tab, options = {
495
+ quietMs: QUIET_MS,
496
+ capMs: ACT_SETTLE_CAP_MS,
497
+ }) {
498
+ const outcome = await waitUntilQuiet({ now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }, this.quietProbe(tab), options);
443
499
  while (this.blanking.size > 0)
444
500
  await Promise.all([...this.blanking]);
501
+ return outcome;
502
+ }
503
+ quietProbe(tab) {
504
+ return {
505
+ stillForMs: async () => {
506
+ // A dialog pauses the page; nothing more will happen until it is answered.
507
+ if (this.pendingDialog?.tab === tab || tab.page.isClosed())
508
+ return Number.MAX_SAFE_INTEGER;
509
+ if (tab.inflight.size > 0)
510
+ return 0;
511
+ const sinceNetwork = Date.now() - tab.lastNetworkAt;
512
+ const sincePage = await Promise.race([
513
+ tab.page.evaluate(msSinceLastChange).catch(() => 0),
514
+ new Promise((resolve) => setTimeout(() => resolve(0), 1000)),
515
+ ]);
516
+ return Math.min(sinceNetwork, sincePage);
517
+ },
518
+ };
445
519
  }
446
520
  async activeTab() {
447
521
  const context = await this.ensureContext();
@@ -530,19 +604,29 @@ export class PlaywrightBrowserHost {
530
604
  // A frame that went away while we looked is not a challenge.
531
605
  }
532
606
  }
533
- const reason = classifyHumanRequired({
607
+ const signals = {
534
608
  url,
535
609
  title: await this.title(tab),
536
610
  ...(tab.status !== undefined ? { status: tab.status } : {}),
537
611
  ...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
538
- ...(tab.proxyAuthenticate !== undefined
539
- ? { proxyAuthenticate: tab.proxyAuthenticate }
540
- : {}),
612
+ ...(tab.proxyAuthenticate !== undefined ? { proxyAuthenticate: tab.proxyAuthenticate } : {}),
541
613
  ...counts,
542
614
  frameUrls,
543
- }, this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {});
615
+ };
616
+ const classifier = {
617
+ ...(this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {}),
618
+ ...(this.options.signInDetection ? { signInDetection: this.options.signInDetection } : {}),
619
+ };
620
+ const reason = classifyHumanRequired(signals, classifier);
544
621
  if (reason)
545
622
  throw this.humanRequired(reason, tab);
623
+ // A page with a password field that did not stop the turn: say so once
624
+ // per address, so the model reports it instead of trying to type.
625
+ const notice = signInNotice(signals, classifier, this.loginCommandFor(url));
626
+ if (notice && tab.signInNoticeFor !== withoutQuery(url)) {
627
+ tab.signInNoticeFor = withoutQuery(url);
628
+ this.notes.push(notice);
629
+ }
546
630
  if (tab.status === 401) {
547
631
  throw new Error('The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.');
548
632
  }
@@ -679,7 +763,7 @@ export class PlaywrightBrowserHost {
679
763
  case 'reload':
680
764
  return this.history(action.action);
681
765
  case 'snapshot':
682
- return this.snapshot(action.ref, action.cursor);
766
+ return this.snapshot(action);
683
767
  case 'screenshot':
684
768
  return this.screenshot(action.ref, action.fullPage);
685
769
  case 'scroll':
@@ -688,6 +772,8 @@ export class PlaywrightBrowserHost {
688
772
  return this.waitFor(action);
689
773
  case 'tabs':
690
774
  return this.tabsCall(action);
775
+ case 'value':
776
+ return this.readValue(action.ref);
691
777
  }
692
778
  }
693
779
  async navigate(tab, rawUrl) {
@@ -708,7 +794,7 @@ export class PlaywrightBrowserHost {
708
794
  catch (error) {
709
795
  await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined);
710
796
  if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
711
- await this.settle();
797
+ await this.settle(tab);
712
798
  throw new Error('The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.');
713
799
  }
714
800
  if (!/ERR_BLOCKED_BY_CLIENT|ERR_ABORTED|Download is starting/.test(firstLine(error))) {
@@ -719,10 +805,35 @@ export class PlaywrightBrowserHost {
719
805
  return this.afterNavigation(tab);
720
806
  }
721
807
  async afterNavigation(tab) {
722
- await this.settle();
808
+ // The document first (a page that has not finished loading has not
809
+ // finished building itself), then the same quiet the actions wait for.
810
+ await tab.page
811
+ .waitForLoadState('load', { timeout: NAVIGATE_SETTLE_CAP_MS })
812
+ .catch(() => undefined);
813
+ const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: NAVIGATE_SETTLE_CAP_MS });
723
814
  this.checkLanding(tab);
724
- await this.settle();
815
+ while (this.blanking.size > 0)
816
+ await Promise.all([...this.blanking]);
725
817
  await this.checkHuman(tab);
818
+ if (!settled.quiet) {
819
+ this.notes.push(`The page was still changing after ${settled.waitedMs} ms, so what it shows may be unfinished. Call browser with wait_for and stable, or take a snapshot.`);
820
+ }
821
+ const populated = originOf(tab.page.url()) === 'null' ||
822
+ (await tab.page.evaluate(hasVisibleContent).catch(() => true));
823
+ if (!populated) {
824
+ this.notes.push(`The page has nothing to read yet (waited ${settled.waitedMs} ms). Call browser with wait_for and stable, or reload.`);
825
+ }
826
+ // The model's next step after opening a page is to read it, so the
827
+ // reading comes with the open: one request fewer on every page visit.
828
+ if (populated && originOf(tab.page.url()) !== 'null') {
829
+ try {
830
+ const snapshot = await this.takeSnapshot(tab);
831
+ return this.result({ page: snapshot.page, snapshot });
832
+ }
833
+ catch {
834
+ // An unreadable page still opened; say where we are and let the model ask again.
835
+ }
836
+ }
726
837
  return this.result({ page: await this.pageInfo(tab) });
727
838
  }
728
839
  async history(which) {
@@ -771,7 +882,16 @@ export class PlaywrightBrowserHost {
771
882
  await pending.dialog.dismiss().catch(() => undefined);
772
883
  this.notes.push('The open dialog was dismissed.');
773
884
  }
774
- async takeSnapshot(tab, ref) {
885
+ /** Read the page's tree and remember its refs. */
886
+ async readTree(tab) {
887
+ const { nodes, hidden, redact } = await captureAriaTree(tab.page, this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
888
+ const whole = renderAriaTree(nodes, hidden, undefined, redact);
889
+ if (!whole)
890
+ throw new Error('The page could not be read.');
891
+ this.refs = whole.refs;
892
+ return { nodes, hidden, redact, whole };
893
+ }
894
+ async takeSnapshot(tab, request = {}) {
775
895
  const pending = this.pendingDialog;
776
896
  if (pending && pending.tab === tab) {
777
897
  const dialog = pending.dialog;
@@ -783,17 +903,67 @@ export class PlaywrightBrowserHost {
783
903
  lines.push(' - (the page is paused until the dialog is answered)');
784
904
  return { page: await this.pageInfo(tab), text: lines.join('\n') };
785
905
  }
786
- const { nodes, hidden, redact } = await captureAriaTree(tab.page, this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
787
- const whole = renderAriaTree(nodes, hidden, undefined, redact);
788
- if (!whole)
789
- throw new Error('The page could not be read.');
790
- this.refs = whole.refs;
791
- let text = whole.text;
792
- if (ref !== undefined) {
793
- const region = renderAriaTree(nodes, hidden, ref, redact);
794
- if (!region || !whole.refs.has(ref))
795
- throw new BrowserStaleRefError(ref);
796
- text = region.text;
906
+ const { ref, view = 'full', query, role } = request;
907
+ const { nodes, hidden, redact, whole } = await this.readTree(tab);
908
+ if (ref !== undefined && !whole.refs.has(ref))
909
+ throw new BrowserStaleRefError(ref);
910
+ // A region is cut from the whole-page tree: a snapshot of one element would replace the ref table.
911
+ const scope = ref === undefined ? nodes : (subtreeOf(nodes, ref) ?? []);
912
+ let text;
913
+ if (query !== undefined || role !== undefined) {
914
+ const found = findInTree(scope, hidden, redact, { ...(query !== undefined ? { query } : {}), ...(role !== undefined ? { role } : {}) }, BROWSER_FIND_MAX_MATCHES);
915
+ text = found.lines.join('\n');
916
+ const what = [
917
+ query !== undefined ? `text ${quoteShort(query, 60)}` : '',
918
+ role !== undefined ? `role ${quoteShort(role, 30)}` : '',
919
+ ]
920
+ .filter(Boolean)
921
+ .join(' and ');
922
+ this.notes.unshift(found.total === 0
923
+ ? `Nothing on the page matches ${what}.`
924
+ : found.total > found.lines.length
925
+ ? `${found.total} elements match ${what}; the first ${found.lines.length} are listed. Narrow the search.`
926
+ : `${found.total} element${found.total === 1 ? '' : 's'} match${found.total === 1 ? 'es' : ''} ${what}.`);
927
+ }
928
+ else if (view === 'interactive') {
929
+ text = renderInteractive(scope, hidden, redact);
930
+ tab.lastSeen = whole.text;
931
+ }
932
+ else if (view === 'reading') {
933
+ text = renderReading(scope, hidden, redact);
934
+ tab.lastSeen = whole.text;
935
+ }
936
+ else if (view === 'delta') {
937
+ const before = tab.lastSeen;
938
+ if (before === undefined) {
939
+ this.notes.unshift('There is no earlier snapshot of this tab to compare with; this is the whole page.');
940
+ text =
941
+ ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '');
942
+ }
943
+ else {
944
+ const diff = diffLines(before, whole.text, DELTA_LINES_SHOWN);
945
+ text =
946
+ diff.added === 0 && diff.removed === 0
947
+ ? '(no change since the last snapshot)'
948
+ : [
949
+ ...diff.addedLines.map((l) => `+ ${l}`),
950
+ ...(diff.added > diff.addedLines.length
951
+ ? [` (${diff.added - diff.addedLines.length} more lines added)`]
952
+ : []),
953
+ ...diff.removedLines.map((l) => `- ${l}`),
954
+ ...(diff.removed > diff.removedLines.length
955
+ ? [` (${diff.removed - diff.removedLines.length} more lines removed)`]
956
+ : []),
957
+ ].join('\n');
958
+ this.notes.unshift(`Changes since the last snapshot: ${diff.added} lines added, ${diff.removed} removed. Take a full snapshot to see the whole page again.`);
959
+ }
960
+ tab.lastSeen = whole.text;
961
+ }
962
+ else {
963
+ text =
964
+ ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '');
965
+ if (ref === undefined)
966
+ tab.lastSeen = whole.text;
797
967
  }
798
968
  if (text.trim() === '')
799
969
  text = '(nothing readable on this page)';
@@ -804,10 +974,10 @@ export class PlaywrightBrowserHost {
804
974
  ...(first.nextCursor !== undefined ? { nextCursor: first.nextCursor } : {}),
805
975
  };
806
976
  }
807
- async snapshot(ref, cursor) {
977
+ async snapshot(request) {
808
978
  const tab = await this.activeTab();
809
- if (cursor !== undefined) {
810
- const next = this.pager.resume(cursor);
979
+ if (request.cursor !== undefined) {
980
+ const next = this.pager.resume(request.cursor);
811
981
  if (!next) {
812
982
  throw new Error('That cursor belongs to an older snapshot. Take a new snapshot and page through it.');
813
983
  }
@@ -819,8 +989,15 @@ export class PlaywrightBrowserHost {
819
989
  },
820
990
  });
821
991
  }
822
- await this.settle();
823
- return this.result({ snapshot: await this.takeSnapshot(tab, ref) });
992
+ await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS });
993
+ return this.result({
994
+ snapshot: await this.takeSnapshot(tab, {
995
+ ...(request.ref !== undefined ? { ref: request.ref } : {}),
996
+ ...(request.view !== undefined ? { view: request.view } : {}),
997
+ ...(request.query !== undefined ? { query: request.query } : {}),
998
+ ...(request.role !== undefined ? { role: request.role } : {}),
999
+ }),
1000
+ });
824
1001
  }
825
1002
  async resolveRef(tab, ref) {
826
1003
  if (!this.refs.has(ref))
@@ -842,10 +1019,12 @@ export class PlaywrightBrowserHost {
842
1019
  if (this.pendingDialog?.tab === tab) {
843
1020
  throw new Error('A dialog is open; dismiss it with browser_act dialog before taking a screenshot.');
844
1021
  }
845
- // Screenshots are image blocks sent to the model. Mask DOM-addressable
846
- // inputs regardless of their names, including child frames and refs.
847
- // Closed shadow roots and page-drawn text remain outside this mask.
848
- const mask = tab.page.frames().map((frame) => frame.locator('input, textarea'));
1022
+ // Screenshots are image blocks sent to the model. Mask the fields that
1023
+ // take a password or a one-time code, in every frame; the other inputs
1024
+ // stay visible, because a search box or an address field the model
1025
+ // cannot read cannot be checked. Closed shadow roots and page-drawn
1026
+ // text remain outside this mask.
1027
+ const mask = await this.credentialMask(tab);
849
1028
  const data = ref !== undefined
850
1029
  ? await (await this.resolveRef(tab, ref)).screenshot({
851
1030
  type: 'png',
@@ -870,6 +1049,59 @@ export class PlaywrightBrowserHost {
870
1049
  },
871
1050
  });
872
1051
  }
1052
+ /** Locators of the credential fields (password, one-time code) of every frame of the tab. */
1053
+ async credentialMask(tab) {
1054
+ const out = [];
1055
+ for (const frame of tab.page.frames()) {
1056
+ const inputs = frame.locator('input, textarea');
1057
+ let count = 0;
1058
+ try {
1059
+ count = Math.min(await inputs.count(), MASK_CANDIDATES_MAX);
1060
+ }
1061
+ catch {
1062
+ // A frame that went away while we looked has nothing to mask.
1063
+ continue;
1064
+ }
1065
+ const verdicts = await Promise.all(Array.from({ length: count }, async (_, index) => {
1066
+ try {
1067
+ return isCredentialField(await inputs.nth(index).evaluate(fieldFacts, undefined, { timeout: 2000 }));
1068
+ }
1069
+ catch {
1070
+ // A field that cannot be asked about is hidden to be safe.
1071
+ return true;
1072
+ }
1073
+ }));
1074
+ verdicts.forEach((hide, index) => {
1075
+ if (hide)
1076
+ out.push(inputs.nth(index));
1077
+ });
1078
+ }
1079
+ return out;
1080
+ }
1081
+ /** One element's current value and state, read from the page itself. */
1082
+ async readValue(ref) {
1083
+ const tab = await this.activeTab();
1084
+ if (this.pendingDialog?.tab === tab) {
1085
+ throw new Error('A dialog is open; dismiss it with browser_act dialog before reading a value.');
1086
+ }
1087
+ const locator = await this.resolveRef(tab, ref);
1088
+ const facts = await locator.evaluate(fieldFacts);
1089
+ const described = this.refs.get(ref);
1090
+ const page = await this.pageInfo(tab);
1091
+ const hidden = isCredentialField(facts);
1092
+ const read = hidden ? undefined : await locator.evaluate(valueFacts);
1093
+ const value = {
1094
+ page,
1095
+ ref,
1096
+ ...(described?.role !== undefined ? { role: described.role } : {}),
1097
+ ...(described?.name !== undefined ? { name: described.name } : {}),
1098
+ ...(read?.value !== undefined ? { value: read.value } : {}),
1099
+ state: read?.state ?? [],
1100
+ ...(read?.options !== undefined ? { options: read.options } : {}),
1101
+ ...(hidden ? { hidden: true } : {}),
1102
+ };
1103
+ return this.result({ value });
1104
+ }
873
1105
  async scroll(direction, ref, amount) {
874
1106
  const tab = await this.activeTab();
875
1107
  const screens = amount ?? 1;
@@ -916,10 +1148,17 @@ export class PlaywrightBrowserHost {
916
1148
  : `${quoteShort(text)} was still there after ${timeout} ms.`);
917
1149
  }
918
1150
  }
919
- else {
1151
+ else if (action.stable !== true) {
920
1152
  await tab.page.waitForTimeout(timeout);
921
1153
  this.notes.unshift(`Waited ${timeout} ms.`);
922
1154
  }
1155
+ if (action.stable === true) {
1156
+ const cap = action.timeMs !== undefined ? timeout : STABLE_DEFAULT_WAIT_MS;
1157
+ const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: cap });
1158
+ this.notes.push(settled.quiet
1159
+ ? `The page is quiet (nothing changed for ${QUIET_MS} ms; waited ${settled.waitedMs} ms).`
1160
+ : `The page was still changing after ${settled.waitedMs} ms.`);
1161
+ }
923
1162
  return this.result({ page: await this.pageInfo(tab) });
924
1163
  }
925
1164
  async tabsCall(action) {
@@ -977,15 +1216,15 @@ export class PlaywrightBrowserHost {
977
1216
  // Acting
978
1217
  // -------------------------------------------------------------------------
979
1218
  async guardField(locator) {
980
- const facts = await locator.evaluate(fieldFacts);
1219
+ const facts = await evaluateOn(locator, fieldFacts);
981
1220
  if (isCredentialField(facts)) {
982
- throw new Error('This action targets a password or one-time-code control. Ask the operator to use the browser window.');
1221
+ throw new Error('This action targets a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.');
983
1222
  }
984
1223
  return facts;
985
1224
  }
986
1225
  /** Refuse direct form submission while a form holds a password or code input. */
987
1226
  async guardCredentialForm(locator, action, key) {
988
- const context = await locator.evaluate((element) => {
1227
+ const context = await evaluateOn(locator, (element) => {
989
1228
  const label = element.closest('label');
990
1229
  const labelledField = label?.control
991
1230
  ? {
@@ -1026,16 +1265,62 @@ export class PlaywrightBrowserHost {
1026
1265
  if ((action === 'click' && context.isFormButton) ||
1027
1266
  (action === 'press' && (context.isFormButton || /enter|return/i.test(key ?? ''))) ||
1028
1267
  action === 'type-submit') {
1029
- throw new Error('This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.');
1268
+ throw new Error('This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.');
1269
+ }
1270
+ }
1271
+ /**
1272
+ * The page's tree as the model could have read it just before an action,
1273
+ * or `undefined` when it cannot be read now (a dialog holds the page, or
1274
+ * the read failed). An effect is a bonus; it never fails the action.
1275
+ */
1276
+ async effectBaseline(tab) {
1277
+ if (this.pendingDialog?.tab === tab)
1278
+ return undefined;
1279
+ try {
1280
+ return (await this.readTree(tab)).whole.text;
1281
+ }
1282
+ catch {
1283
+ return undefined;
1284
+ }
1285
+ }
1286
+ /** What the action did, from the tree before and the tree now. */
1287
+ async effectSince(before, urlBefore, tabsBefore, tab, settled) {
1288
+ if (before === undefined || this.pendingDialog?.tab === tab || tab.page.isClosed())
1289
+ return undefined;
1290
+ let after;
1291
+ try {
1292
+ after = (await this.readTree(tab)).whole.text;
1030
1293
  }
1294
+ catch {
1295
+ return undefined;
1296
+ }
1297
+ return buildEffect({
1298
+ beforeTree: before,
1299
+ afterTree: after,
1300
+ urlBefore,
1301
+ urlNow: tab.page.url(),
1302
+ tabsBefore,
1303
+ tabsNow: this.tabs.size,
1304
+ settle: { quiet: settled.quiet, waitedMs: settled.waitedMs, capMs: settled.capMs },
1305
+ }, EFFECT_LINES_SHOWN);
1031
1306
  }
1032
1307
  async runAct(action) {
1033
1308
  const tab = await this.activeTab();
1034
- await this.settle();
1309
+ await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS });
1035
1310
  const expected = canonicalizeBrowserOrigin(action.origin);
1036
1311
  const live = originOf(tab.page.url());
1037
1312
  if (!expected.ok || expected.origin !== live) {
1038
- throw new BrowserOriginMismatchError(expected.ok ? expected.origin : action.origin, live === 'null' ? 'about:blank' : live);
1313
+ // The page moved since the snapshot. Going on is fine only when the
1314
+ // site rules treat the new origin exactly as the named one: both at
1315
+ // `act`, where no review stood between the model and the page.
1316
+ const sameRules = expected.ok &&
1317
+ live !== 'null' &&
1318
+ this.policy.levelOf(expected.origin) === 'act' &&
1319
+ this.policy.levelOf(live) === 'act';
1320
+ if (!sameRules) {
1321
+ throw new BrowserOriginMismatchError(expected.ok ? expected.origin : action.origin, live === 'null' ? 'about:blank' : live);
1322
+ }
1323
+ this.notes.unshift(`The page is now at ${live}, not ${expected.origin}; the site rules treat both alike, so the action went ahead there.`);
1039
1324
  }
1040
1325
  if (!this.policy.canAct(live)) {
1041
1326
  throw new BrowserSiteDeniedError(live, `${live} may be read but not changed under the site rules.`);
@@ -1046,11 +1331,16 @@ export class PlaywrightBrowserHost {
1046
1331
  throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.');
1047
1332
  }
1048
1333
  const checkpoint = await this.beginDocumentResponseChange(tab);
1334
+ const urlBefore = tab.page.url();
1335
+ const tabsBefore = this.tabs.size;
1336
+ const treeBefore = await this.effectBaseline(tab);
1337
+ let settled;
1049
1338
  try {
1050
1339
  await this.perform(tab, action);
1051
- await this.settle();
1340
+ settled = await this.settle(tab);
1052
1341
  this.checkLanding(tab);
1053
- await this.settle();
1342
+ while (this.blanking.size > 0)
1343
+ await Promise.all([...this.blanking]);
1054
1344
  }
1055
1345
  catch (error) {
1056
1346
  await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined);
@@ -1059,9 +1349,14 @@ export class PlaywrightBrowserHost {
1059
1349
  await this.finishDocumentResponseChange(tab, checkpoint);
1060
1350
  await this.checkHuman(tab);
1061
1351
  const after = this.tabs.get(tab.id) ?? tab;
1062
- if (action.snapshot)
1063
- return this.result({ snapshot: await this.takeSnapshot(after) });
1064
- return this.result({ page: await this.pageInfo(after) });
1352
+ const effect = await this.effectSince(treeBefore, urlBefore, tabsBefore, after, settled);
1353
+ if (action.snapshot) {
1354
+ return this.result({
1355
+ snapshot: await this.takeSnapshot(after),
1356
+ ...(effect ? { effect } : {}),
1357
+ });
1358
+ }
1359
+ return this.result({ page: await this.pageInfo(after), ...(effect ? { effect } : {}) });
1065
1360
  }
1066
1361
  async answerDialog(tab, action) {
1067
1362
  const pending = this.pendingDialog;
@@ -1085,9 +1380,10 @@ export class PlaywrightBrowserHost {
1085
1380
  throw new BrowserOutcomeUnknownError('dialog', error);
1086
1381
  }
1087
1382
  this.notes.unshift('The dialog was dismissed.');
1088
- await this.settle();
1383
+ await this.settle(tab);
1089
1384
  this.checkLanding(tab);
1090
- await this.settle();
1385
+ while (this.blanking.size > 0)
1386
+ await Promise.all([...this.blanking]);
1091
1387
  await this.finishDocumentResponseChange(tab, checkpoint);
1092
1388
  await this.checkHuman(tab);
1093
1389
  if (action.snapshot)
@@ -1102,18 +1398,20 @@ export class PlaywrightBrowserHost {
1102
1398
  async perform(tab, action) {
1103
1399
  switch (action.action) {
1104
1400
  case 'click': {
1105
- const locator = await this.resolveRef(tab, action.ref);
1106
- await this.guardField(locator);
1107
- await this.guardCredentialForm(locator, 'click');
1108
- await locator.click({ trial: true });
1109
- await this.unknownOnFailure('click', () => action.doubleClick ? locator.dblclick() : locator.click());
1401
+ const target = await this.reach(tab, action);
1402
+ await this.guardField(target.locator);
1403
+ await this.guardCredentialForm(target.locator, 'click');
1404
+ const at = target.position ? { position: target.position } : {};
1405
+ await this.noEffectOnFailure('click', () => target.locator.click({ trial: true, ...at }));
1406
+ await this.unknownOnFailure('click', () => action.doubleClick ? target.locator.dblclick(at) : target.locator.click(at));
1110
1407
  return;
1111
1408
  }
1112
1409
  case 'hover': {
1113
- const locator = await this.resolveRef(tab, action.ref);
1114
- await this.guardField(locator);
1115
- await locator.hover({ trial: true });
1116
- await this.unknownOnFailure('hover', () => locator.hover());
1410
+ const target = await this.reach(tab, action);
1411
+ await this.guardField(target.locator);
1412
+ const at = target.position ? { position: target.position } : {};
1413
+ await this.noEffectOnFailure('hover', () => target.locator.hover({ trial: true, ...at }));
1414
+ await this.unknownOnFailure('hover', () => target.locator.hover(at));
1117
1415
  return;
1118
1416
  }
1119
1417
  case 'type': {
@@ -1121,9 +1419,27 @@ export class PlaywrightBrowserHost {
1121
1419
  await this.guardField(locator);
1122
1420
  if (action.submit)
1123
1421
  await this.guardCredentialForm(locator, 'type-submit');
1124
- await locator.click({ trial: true });
1125
- await this.unknownOnFailure('type', async () => {
1126
- await locator.fill(action.text);
1422
+ await this.noEffectOnFailure('type', () => locator.click({ trial: true }));
1423
+ await this.unknownOnFailure('type', async (begun) => {
1424
+ if (action.keys) {
1425
+ await locator.focus();
1426
+ // Clearing and typing change the field; from here a failure is not harmless.
1427
+ begun();
1428
+ if (action.append)
1429
+ await locator.press('Control+End');
1430
+ else
1431
+ await locator.fill('');
1432
+ await locator.pressSequentially(action.text, { delay: KEY_DELAY_MS });
1433
+ }
1434
+ else if (action.append) {
1435
+ const held = await locator
1436
+ .inputValue()
1437
+ .catch(() => locator.evaluate((el) => el.textContent ?? ''));
1438
+ await locator.fill(held + action.text);
1439
+ }
1440
+ else {
1441
+ await locator.fill(action.text);
1442
+ }
1127
1443
  if (action.submit)
1128
1444
  await locator.press('Enter');
1129
1445
  });
@@ -1169,7 +1485,13 @@ export class PlaywrightBrowserHost {
1169
1485
  await this.unknownOnFailure('press', () => locator.press(action.key));
1170
1486
  return;
1171
1487
  }
1172
- throw new Error('An untargeted key could reach a password or one-time-code control, including one inside a closed shadow root. Use a specific control instead.');
1488
+ const key = namedKey(action.key);
1489
+ if (!key) {
1490
+ throw new Error(`Without a ref, press takes only ${NAMED_KEYS.join(', ')}. Any other key goes to a specific control: give its ref.`);
1491
+ }
1492
+ await this.guardFocus(tab, key);
1493
+ await this.unknownOnFailure('press', () => tab.page.keyboard.press(key));
1494
+ return;
1173
1495
  }
1174
1496
  case 'upload': {
1175
1497
  const locator = await this.resolveRef(tab, action.ref);
@@ -1194,13 +1516,102 @@ export class PlaywrightBrowserHost {
1194
1516
  return;
1195
1517
  }
1196
1518
  }
1519
+ /**
1520
+ * The element a click or hover means, and where on it to aim. A ref names
1521
+ * it directly; a role with a name, or visible text, is looked up and
1522
+ * refused unless exactly one element matches. A container (a section, a
1523
+ * list item holding a menu title) is not clicked in its middle, which is
1524
+ * often empty: the aim goes to its first control or to its own text.
1525
+ */
1526
+ async reach(tab, target) {
1527
+ const found = target.ref !== undefined
1528
+ ? await this.resolveRef(tab, target.ref)
1529
+ : await this.findTarget(tab, target);
1530
+ let aim = { kind: 'centre' };
1531
+ try {
1532
+ aim = await found.evaluate(aimInside);
1533
+ }
1534
+ catch {
1535
+ // Aim at the middle, as before.
1536
+ }
1537
+ if (aim.kind === 'descendant')
1538
+ return { locator: found.locator(`xpath=${aim.path}`) };
1539
+ if (aim.kind === 'text')
1540
+ return { locator: found, position: { x: aim.x, y: aim.y } };
1541
+ return { locator: found };
1542
+ }
1543
+ /** Find the one element a role/name or text target means, or refuse with the candidates. */
1544
+ async findTarget(tab, target) {
1545
+ const page = tab.page;
1546
+ const role = target.role;
1547
+ const wanted = role !== undefined ? `${role} "${target.name ?? ''}"` : `text "${target.text ?? ''}"`;
1548
+ const build = (exact) => role !== undefined
1549
+ ? page.getByRole(role, {
1550
+ name: target.name ?? '',
1551
+ exact,
1552
+ })
1553
+ : page.getByText(target.text ?? '', { exact }).filter({ visible: true });
1554
+ let locator = build(false);
1555
+ let count = await locator.count();
1556
+ if (count > 1) {
1557
+ // A short name also fits longer ones ("Save" in "Save draft"): the exact matches decide.
1558
+ const exact = build(true);
1559
+ const exactCount = await exact.count();
1560
+ if (exactCount > 0) {
1561
+ locator = exact;
1562
+ count = exactCount;
1563
+ }
1564
+ }
1565
+ if (count === 1)
1566
+ return locator;
1567
+ const pool = count > 0 ? locator : role !== undefined ? page.getByRole(role) : undefined;
1568
+ const candidates = [];
1569
+ if (pool) {
1570
+ const total = Math.min(await pool.count(), CANDIDATES_MAX);
1571
+ for (let i = 0; i < total; i++) {
1572
+ candidates.push(await pool.nth(i).evaluate(describeCandidate));
1573
+ }
1574
+ }
1575
+ throw new BrowserTargetUnclearError(wanted, count, candidates);
1576
+ }
1577
+ /**
1578
+ * Refuse a key sent to the page when the focus is on a credential, or
1579
+ * when it cannot be told. Every frame is asked, because the key goes to
1580
+ * whichever one holds the focus and a stale focus in another frame is
1581
+ * still a reason to be careful.
1582
+ */
1583
+ async guardFocus(tab, key) {
1584
+ for (const frame of tab.page.frames()) {
1585
+ const handle = await frame.evaluateHandle(deepFocus).catch(() => undefined);
1586
+ const element = handle?.asElement() ?? null;
1587
+ if (!element)
1588
+ continue;
1589
+ if (keyCanCommit(key) && (await element.evaluate(focusIsOpaque))) {
1590
+ throw new Error(`The focus is on a component whose inside cannot be read, so ${key} could reach a password or code field. Use browser_act with the ref of a specific control.`);
1591
+ }
1592
+ await this.guardField(element);
1593
+ await this.guardCredentialForm(element, 'press', key);
1594
+ }
1595
+ }
1596
+ /** A check Playwright runs without acting (`trial`) failed: nothing on the page changed. */
1597
+ async noEffectOnFailure(action, run) {
1598
+ try {
1599
+ await run();
1600
+ }
1601
+ catch (error) {
1602
+ throw new BrowserNoEffectError(action, noEffectReason(error));
1603
+ }
1604
+ }
1197
1605
  /**
1198
1606
  * Run the change. A failure once it has started is `outcome unknown`. A
1199
1607
  * dialog the action opens ends the wait: the page is paused until the
1200
1608
  * dialog is answered, and so is anything Playwright was still waiting for.
1201
1609
  */
1202
1610
  async unknownOnFailure(action, run) {
1203
- const running = run();
1611
+ let touched = false;
1612
+ const running = run(() => {
1613
+ touched = true;
1614
+ });
1204
1615
  let timer;
1205
1616
  const dialogOpened = new Promise((resolve) => {
1206
1617
  timer = setInterval(() => {
@@ -1214,7 +1625,10 @@ export class PlaywrightBrowserHost {
1214
1625
  running.catch(() => undefined);
1215
1626
  }
1216
1627
  catch (error) {
1217
- throw new BrowserOutcomeUnknownError(action, error);
1628
+ // A timeout whose log never reached the input step left the page alone.
1629
+ throw !touched && failedBeforeActing(error)
1630
+ ? new BrowserNoEffectError(action, noEffectReason(error))
1631
+ : new BrowserOutcomeUnknownError(action, error);
1218
1632
  }
1219
1633
  finally {
1220
1634
  clearInterval(timer);