@namzu/browser 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/classifier.d.ts +22 -1
  3. package/dist/classifier.d.ts.map +1 -1
  4. package/dist/classifier.js +30 -7
  5. package/dist/classifier.js.map +1 -1
  6. package/dist/errors.d.ts +19 -1
  7. package/dist/errors.d.ts.map +1 -1
  8. package/dist/errors.js +26 -0
  9. package/dist/errors.js.map +1 -1
  10. package/dist/host.d.ts +42 -0
  11. package/dist/host.d.ts.map +1 -1
  12. package/dist/host.js +487 -70
  13. package/dist/host.js.map +1 -1
  14. package/dist/index.d.ts +3 -3
  15. package/dist/index.d.ts.map +1 -1
  16. package/dist/index.js +2 -2
  17. package/dist/index.js.map +1 -1
  18. package/dist/page-scripts.d.ts +20 -0
  19. package/dist/page-scripts.d.ts.map +1 -1
  20. package/dist/page-scripts.js +110 -0
  21. package/dist/page-scripts.js.map +1 -1
  22. package/dist/reach.d.ts +66 -0
  23. package/dist/reach.d.ts.map +1 -0
  24. package/dist/reach.js +189 -0
  25. package/dist/reach.js.map +1 -0
  26. package/dist/settle.d.ts +40 -0
  27. package/dist/settle.d.ts.map +1 -0
  28. package/dist/settle.js +30 -0
  29. package/dist/settle.js.map +1 -0
  30. package/dist/snapshot.d.ts +80 -2
  31. package/dist/snapshot.d.ts.map +1 -1
  32. package/dist/snapshot.js +326 -20
  33. package/dist/snapshot.js.map +1 -1
  34. package/dist/windows-engine.d.ts +1 -1
  35. package/dist/windows-engine.d.ts.map +1 -1
  36. package/dist/windows-engine.js +3 -1
  37. package/dist/windows-engine.js.map +1 -1
  38. package/package.json +3 -2
  39. package/src/classifier.ts +44 -7
  40. package/src/errors.ts +32 -0
  41. package/src/host.ts +608 -89
  42. package/src/index.ts +4 -0
  43. package/src/page-scripts.ts +108 -0
  44. package/src/reach.ts +211 -0
  45. package/src/settle.ts +60 -0
  46. package/src/snapshot.ts +402 -20
  47. package/src/windows-engine.ts +4 -1
package/src/host.ts CHANGED
@@ -3,6 +3,7 @@ import type {
3
3
  BrowserActAction,
4
4
  BrowserCallOptions,
5
5
  BrowserCapabilities,
6
+ BrowserEffect,
6
7
  BrowserHost,
7
8
  BrowserObserveAction,
8
9
  BrowserPageInfo,
@@ -10,22 +11,33 @@ import type {
10
11
  BrowserResult,
11
12
  BrowserSessionInfo,
12
13
  BrowserSnapshot,
14
+ BrowserSnapshotView,
13
15
  BrowserTabInfo,
16
+ BrowserTarget,
17
+ BrowserValue,
14
18
  } from '@namzu/sdk'
15
- import { canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome } from '@namzu/sdk'
16
19
  import {
17
- type BrowserContext,
18
- type CDPSession,
19
- type Dialog,
20
- type Locator,
21
- type Page,
22
- type Route,
23
- chromium,
20
+ BROWSER_FIND_MAX_MATCHES,
21
+ canonicalizeBrowserOrigin,
22
+ canonicalizeBrowserUrl,
23
+ resolveNamzuHome,
24
+ } from '@namzu/sdk'
25
+ import type {
26
+ BrowserContext,
27
+ CDPSession,
28
+ Dialog,
29
+ ElementHandle,
30
+ Locator,
31
+ Page,
32
+ Request,
33
+ Route,
24
34
  } from 'playwright-core'
25
35
  import {
26
36
  type BrowserHumanClassifierOptions,
37
+ type BrowserPageSignals,
27
38
  classifyHumanRequired,
28
39
  isCredentialField,
40
+ signInNotice,
29
41
  } from './classifier.js'
30
42
  import {
31
43
  type BrowserEnginePlan,
@@ -39,14 +51,24 @@ import {
39
51
  } from './detect.js'
40
52
  import {
41
53
  BrowserHumanRequiredError,
54
+ BrowserNoEffectError,
42
55
  BrowserOriginMismatchError,
43
56
  BrowserOutcomeUnknownError,
44
57
  BrowserSiteDeniedError,
45
58
  BrowserStaleRefError,
59
+ BrowserTargetUnclearError,
46
60
  BrowserUnavailableError,
47
61
  ProfileBusyError,
48
62
  } from './errors.js'
49
- import { type ElementFieldFacts, countCredentialFields, fieldFacts } from './page-scripts.js'
63
+ import {
64
+ type ElementFieldFacts,
65
+ countCredentialFields,
66
+ fieldFacts,
67
+ hasVisibleContent,
68
+ installMutationClock,
69
+ msSinceLastChange,
70
+ valueFacts,
71
+ } from './page-scripts.js'
50
72
  import { BrowserSitePolicy, type BrowserSiteRules, DEFAULT_BROWSER_SITE_RULES } from './policy.js'
51
73
  import {
52
74
  type BrowserLease,
@@ -55,12 +77,34 @@ import {
55
77
  BrowserProfileStore,
56
78
  DEFAULT_BROWSER_PROFILE,
57
79
  } from './profiles.js'
80
+ import {
81
+ type Aim,
82
+ CANDIDATES_MAX,
83
+ KEY_DELAY_MS,
84
+ NAMED_KEYS,
85
+ type NamedKey,
86
+ aimInside,
87
+ deepFocus,
88
+ describeCandidate,
89
+ failedBeforeActing,
90
+ focusIsOpaque,
91
+ keyCanCommit,
92
+ namedKey,
93
+ noEffectReason,
94
+ } from './reach.js'
95
+ import { type SettleOutcome, type SettleProbe, waitUntilQuiet } from './settle.js'
58
96
  import {
59
97
  PLAYWRIGHT_CORE_VERSION,
60
98
  SnapshotPager,
99
+ buildEffect,
61
100
  captureAriaTree,
101
+ diffLines,
102
+ findInTree,
62
103
  locateRef,
63
104
  renderAriaTree,
105
+ renderInteractive,
106
+ renderReading,
107
+ subtreeOf,
64
108
  } from './snapshot.js'
65
109
  import { WindowsBridgeError } from './windows-bridge.js'
66
110
  import { type WindowsBrowserConnection, connectWindowsBrowser } from './windows-engine.js'
@@ -96,6 +140,8 @@ export interface PlaywrightBrowserHostOptions {
96
140
  readonly executablePath?: string
97
141
  /** More sign-in addresses for the classifier. */
98
142
  readonly signInAddresses?: BrowserHumanClassifierOptions['signInAddresses']
143
+ /** What makes a page a sign-in page: `address` (default) or `page`. See {@link BrowserHumanClassifierOptions.signInDetection}. */
144
+ readonly signInDetection?: BrowserHumanClassifierOptions['signInDetection']
99
145
  /** The command that opens a visible window for signing in. Default `namzu browser login <profile> <url>`. */
100
146
  readonly loginCommand?: (profile: string, url: string) => string
101
147
  /** Windows engine: how long starting and connecting to the browser may take. Default 60 000. */
@@ -110,10 +156,18 @@ interface Tab {
110
156
  mainDocumentResponses: number
111
157
  cdp?: CDPSession
112
158
  readonly responsesByLoader: Map<string, DocumentResponse>
159
+ /** Requests that can still change the page: documents, scripts, styles, fetch and XHR. */
160
+ readonly inflight: Set<Request>
161
+ /** When a request of that kind last started or ended. */
162
+ lastNetworkAt: number
163
+ /** The whole-page tree the model was last shown for this tab, for `delta` and effects. */
164
+ lastSeen?: string
113
165
  /** HTTP status of the main document's last response. */
114
166
  status?: number
115
167
  wwwAuthenticate?: string
116
168
  proxyAuthenticate?: string
169
+ /** The address the sign-in notice was last given for, so it is said once. */
170
+ signInNoticeFor?: string
117
171
  }
118
172
 
119
173
  type DocumentResponse = Pick<Tab, 'status' | 'wwwAuthenticate' | 'proxyAuthenticate'>
@@ -136,12 +190,42 @@ const DEFAULT_ACTION_TIMEOUT_MS = 10_000
136
190
  /** How long a history move waits for the document after it committed. */
137
191
  const HISTORY_LOAD_WAIT_MS = 5_000
138
192
  const SNAPSHOT_PAGE_MAX = 20_000
139
- /** How long the host lets events (popups, downloads, dialogs) arrive after an action. */
140
- const EVENT_SETTLE_MS = 150
193
+ /**
194
+ * A page is quiet after this long with no change to the document and no
195
+ * request in flight. It also bounds how long an event the browser has not
196
+ * announced yet (a popup, a download, a dialog) has to arrive.
197
+ */
198
+ const QUIET_MS = 250
199
+ /** The most an action waits for quiet. A page that never settles is reported, not waited on. */
200
+ const ACT_SETTLE_CAP_MS = 2_000
201
+ /** The same for a navigation, which loads more. */
202
+ const NAVIGATE_SETTLE_CAP_MS = 5_000
203
+ /** The wait before an action, for events left over from the last one. */
204
+ const PRE_ACT_QUIET_MS = 100
205
+ const PRE_ACT_CAP_MS = 500
206
+ /** `wait_for` with `stable` and no time: how long it waits. */
207
+ const STABLE_DEFAULT_WAIT_MS = 5_000
208
+ /** Request kinds that can change what the page shows. */
209
+ const PAGE_REQUEST_KINDS = new Set(['document', 'script', 'stylesheet', 'fetch', 'xhr'])
210
+ /** Lines of a `delta` snapshot that are listed in full. */
211
+ const DELTA_LINES_SHOWN = 80
212
+ /** Lines of an effect the model is shown. */
213
+ const EFFECT_LINES_SHOWN = 10
141
214
  /** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
142
215
  const CAPTCHA_FRAME_MIN_PX = 30
143
216
  const REMEMBERED_DOCUMENTS_MAX = 64
217
+
218
+ /** Something the credential guards can ask about: a located element or a handle to one. */
219
+ type Probe = Locator | ElementHandle
220
+
221
+ function evaluateOn<R>(target: Probe, fn: (element: Element) => R): Promise<R> {
222
+ return 'asElement' in target
223
+ ? (target as ElementHandle).evaluate(fn)
224
+ : (target as Locator).evaluate(fn)
225
+ }
144
226
  const SCREENSHOT_INPUT_MASK_COLOR = '#666666'
227
+ /** Most inputs one screenshot checks for a credential. */
228
+ const MASK_CANDIDATES_MAX = 200
145
229
 
146
230
  function firstLine(error: unknown): string {
147
231
  const text = error instanceof Error ? error.message : String(error)
@@ -295,6 +379,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
295
379
  context.setDefaultNavigationTimeout(
296
380
  this.options.navigationTimeoutMs ?? DEFAULT_NAVIGATION_TIMEOUT_MS,
297
381
  )
382
+ // Every new document keeps a clock of its last change, for the quiet-page wait.
383
+ await context.addInitScript(installMutationClock).catch(() => undefined)
298
384
  context.on('page', (page) => this.adopt(page))
299
385
  context.on('close', () => this.forget(context))
300
386
  context.browser()?.on('disconnected', () => this.forget(context))
@@ -305,7 +391,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
305
391
  () => true,
306
392
  (route) => this.screen(route),
307
393
  )
308
- for (const page of context.pages()) this.adopt(page, false)
394
+ for (const page of context.pages()) {
395
+ this.adopt(page, false)
396
+ void page.evaluate(installMutationClock).catch(() => undefined)
397
+ }
309
398
  return context
310
399
  }
311
400
 
@@ -322,6 +411,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
322
411
  })
323
412
  let context: BrowserContext
324
413
  try {
414
+ // Loaded here, on the first browser launch: playwright-core is several
415
+ // megabytes of source that a session which never opens a browser must
416
+ // not compile at startup.
417
+ const { chromium } = await import('playwright-core')
325
418
  context = await chromium.launchPersistentContext(descriptor.userDataDir, {
326
419
  headless: plan.headless,
327
420
  ...(plan.channel ? { channel: plan.channel } : {}),
@@ -496,6 +589,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
496
589
  mainFrameCommits: 0,
497
590
  mainDocumentResponses: 0,
498
591
  responsesByLoader: new Map(),
592
+ inflight: new Set(),
593
+ lastNetworkAt: Date.now(),
499
594
  }
500
595
  this.tabs.set(tab.id, tab)
501
596
  this.tabOf.set(page, tab)
@@ -524,6 +619,17 @@ export class PlaywrightBrowserHost implements BrowserHost {
524
619
  // A response without a frame (a service worker's) says nothing about the page.
525
620
  }
526
621
  })
622
+ const kind = (request: Request): boolean => PAGE_REQUEST_KINDS.has(request.resourceType())
623
+ page.on('request', (request) => {
624
+ if (!kind(request)) return
625
+ tab.inflight.add(request)
626
+ tab.lastNetworkAt = Date.now()
627
+ })
628
+ const ended = (request: Request): void => {
629
+ if (tab.inflight.delete(request)) tab.lastNetworkAt = Date.now()
630
+ }
631
+ page.on('requestfinished', ended)
632
+ page.on('requestfailed', ended)
527
633
  page.on('dialog', (dialog) => this.onDialog(tab, dialog))
528
634
  page.on('download', (download) => {
529
635
  const name = download.suggestedFilename()
@@ -601,9 +707,42 @@ export class PlaywrightBrowserHost implements BrowserHost {
601
707
  this.blanking.add(blank)
602
708
  }
603
709
 
604
- private async settle(): Promise<void> {
605
- await new Promise((resolve) => setTimeout(resolve, EVENT_SETTLE_MS))
710
+ /**
711
+ * Wait until the tab's page is quiet (nothing changed in it and no request
712
+ * in flight for `quietMs`), or `capMs` has passed. Events the browser
713
+ * announces late (a popup, a download, a dialog) arrive inside the window.
714
+ * Then let any tab that was cleared to `about:blank` finish clearing.
715
+ */
716
+ private async settle(
717
+ tab: Tab,
718
+ options: { readonly quietMs: number; readonly capMs: number } = {
719
+ quietMs: QUIET_MS,
720
+ capMs: ACT_SETTLE_CAP_MS,
721
+ },
722
+ ): Promise<SettleOutcome> {
723
+ const outcome = await waitUntilQuiet(
724
+ { now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) },
725
+ this.quietProbe(tab),
726
+ options,
727
+ )
606
728
  while (this.blanking.size > 0) await Promise.all([...this.blanking])
729
+ return outcome
730
+ }
731
+
732
+ private quietProbe(tab: Tab): SettleProbe {
733
+ return {
734
+ stillForMs: async () => {
735
+ // A dialog pauses the page; nothing more will happen until it is answered.
736
+ if (this.pendingDialog?.tab === tab || tab.page.isClosed()) return Number.MAX_SAFE_INTEGER
737
+ if (tab.inflight.size > 0) return 0
738
+ const sinceNetwork = Date.now() - tab.lastNetworkAt
739
+ const sincePage = await Promise.race([
740
+ tab.page.evaluate(msSinceLastChange).catch(() => 0),
741
+ new Promise<number>((resolve) => setTimeout(() => resolve(0), 1000)),
742
+ ])
743
+ return Math.min(sinceNetwork, sincePage)
744
+ },
745
+ }
607
746
  }
608
747
 
609
748
  private async activeTab(): Promise<Tab> {
@@ -698,21 +837,28 @@ export class PlaywrightBrowserHost implements BrowserHost {
698
837
  // A frame that went away while we looked is not a challenge.
699
838
  }
700
839
  }
701
- const reason = classifyHumanRequired(
702
- {
703
- url,
704
- title: await this.title(tab),
705
- ...(tab.status !== undefined ? { status: tab.status } : {}),
706
- ...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
707
- ...(tab.proxyAuthenticate !== undefined
708
- ? { proxyAuthenticate: tab.proxyAuthenticate }
709
- : {}),
710
- ...counts,
711
- frameUrls,
712
- },
713
- this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {},
714
- )
840
+ const signals: BrowserPageSignals = {
841
+ url,
842
+ title: await this.title(tab),
843
+ ...(tab.status !== undefined ? { status: tab.status } : {}),
844
+ ...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
845
+ ...(tab.proxyAuthenticate !== undefined ? { proxyAuthenticate: tab.proxyAuthenticate } : {}),
846
+ ...counts,
847
+ frameUrls,
848
+ }
849
+ const classifier: BrowserHumanClassifierOptions = {
850
+ ...(this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {}),
851
+ ...(this.options.signInDetection ? { signInDetection: this.options.signInDetection } : {}),
852
+ }
853
+ const reason = classifyHumanRequired(signals, classifier)
715
854
  if (reason) throw this.humanRequired(reason, tab)
855
+ // A page with a password field that did not stop the turn: say so once
856
+ // per address, so the model reports it instead of trying to type.
857
+ const notice = signInNotice(signals, classifier, this.loginCommandFor(url))
858
+ if (notice && tab.signInNoticeFor !== withoutQuery(url)) {
859
+ tab.signInNoticeFor = withoutQuery(url)
860
+ this.notes.push(notice)
861
+ }
716
862
  if (tab.status === 401) {
717
863
  throw new Error(
718
864
  'The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.',
@@ -866,7 +1012,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
866
1012
  case 'reload':
867
1013
  return this.history(action.action)
868
1014
  case 'snapshot':
869
- return this.snapshot(action.ref, action.cursor)
1015
+ return this.snapshot(action)
870
1016
  case 'screenshot':
871
1017
  return this.screenshot(action.ref, action.fullPage)
872
1018
  case 'scroll':
@@ -875,6 +1021,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
875
1021
  return this.waitFor(action)
876
1022
  case 'tabs':
877
1023
  return this.tabsCall(action)
1024
+ case 'value':
1025
+ return this.readValue(action.ref)
878
1026
  }
879
1027
  }
880
1028
 
@@ -894,7 +1042,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
894
1042
  } catch (error) {
895
1043
  await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
896
1044
  if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
897
- await this.settle()
1045
+ await this.settle(tab)
898
1046
  throw new Error(
899
1047
  'The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.',
900
1048
  )
@@ -908,10 +1056,38 @@ export class PlaywrightBrowserHost implements BrowserHost {
908
1056
  }
909
1057
 
910
1058
  private async afterNavigation(tab: Tab): Promise<BrowserResult> {
911
- await this.settle()
1059
+ // The document first (a page that has not finished loading has not
1060
+ // finished building itself), then the same quiet the actions wait for.
1061
+ await tab.page
1062
+ .waitForLoadState('load', { timeout: NAVIGATE_SETTLE_CAP_MS })
1063
+ .catch(() => undefined)
1064
+ const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: NAVIGATE_SETTLE_CAP_MS })
912
1065
  this.checkLanding(tab)
913
- await this.settle()
1066
+ while (this.blanking.size > 0) await Promise.all([...this.blanking])
914
1067
  await this.checkHuman(tab)
1068
+ if (!settled.quiet) {
1069
+ this.notes.push(
1070
+ `The page was still changing after ${settled.waitedMs} ms, so what it shows may be unfinished. Call browser with wait_for and stable, or take a snapshot.`,
1071
+ )
1072
+ }
1073
+ const populated =
1074
+ originOf(tab.page.url()) === 'null' ||
1075
+ (await tab.page.evaluate(hasVisibleContent).catch(() => true))
1076
+ if (!populated) {
1077
+ this.notes.push(
1078
+ `The page has nothing to read yet (waited ${settled.waitedMs} ms). Call browser with wait_for and stable, or reload.`,
1079
+ )
1080
+ }
1081
+ // The model's next step after opening a page is to read it, so the
1082
+ // reading comes with the open: one request fewer on every page visit.
1083
+ if (populated && originOf(tab.page.url()) !== 'null') {
1084
+ try {
1085
+ const snapshot = await this.takeSnapshot(tab)
1086
+ return this.result({ page: snapshot.page, snapshot })
1087
+ } catch {
1088
+ // An unreadable page still opened; say where we are and let the model ask again.
1089
+ }
1090
+ }
915
1091
  return this.result({ page: await this.pageInfo(tab) })
916
1092
  }
917
1093
 
@@ -960,7 +1136,27 @@ export class PlaywrightBrowserHost implements BrowserHost {
960
1136
  this.notes.push('The open dialog was dismissed.')
961
1137
  }
962
1138
 
963
- private async takeSnapshot(tab: Tab, ref?: string): Promise<BrowserSnapshot> {
1139
+ /** Read the page's tree and remember its refs. */
1140
+ private async readTree(tab: Tab) {
1141
+ const { nodes, hidden, redact } = await captureAriaTree(
1142
+ tab.page,
1143
+ this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS,
1144
+ )
1145
+ const whole = renderAriaTree(nodes, hidden, undefined, redact)
1146
+ if (!whole) throw new Error('The page could not be read.')
1147
+ this.refs = whole.refs
1148
+ return { nodes, hidden, redact, whole }
1149
+ }
1150
+
1151
+ private async takeSnapshot(
1152
+ tab: Tab,
1153
+ request: {
1154
+ readonly ref?: string
1155
+ readonly view?: BrowserSnapshotView
1156
+ readonly query?: string
1157
+ readonly role?: string
1158
+ } = {},
1159
+ ): Promise<BrowserSnapshot> {
964
1160
  const pending = this.pendingDialog
965
1161
  if (pending && pending.tab === tab) {
966
1162
  const dialog = pending.dialog
@@ -972,18 +1168,72 @@ export class PlaywrightBrowserHost implements BrowserHost {
972
1168
  lines.push(' - (the page is paused until the dialog is answered)')
973
1169
  return { page: await this.pageInfo(tab), text: lines.join('\n') }
974
1170
  }
975
- const { nodes, hidden, redact } = await captureAriaTree(
976
- tab.page,
977
- this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS,
978
- )
979
- const whole = renderAriaTree(nodes, hidden, undefined, redact)
980
- if (!whole) throw new Error('The page could not be read.')
981
- this.refs = whole.refs
982
- let text = whole.text
983
- if (ref !== undefined) {
984
- const region = renderAriaTree(nodes, hidden, ref, redact)
985
- if (!region || !whole.refs.has(ref)) throw new BrowserStaleRefError(ref)
986
- text = region.text
1171
+ const { ref, view = 'full', query, role } = request
1172
+ const { nodes, hidden, redact, whole } = await this.readTree(tab)
1173
+ if (ref !== undefined && !whole.refs.has(ref)) throw new BrowserStaleRefError(ref)
1174
+ // A region is cut from the whole-page tree: a snapshot of one element would replace the ref table.
1175
+ const scope = ref === undefined ? nodes : (subtreeOf(nodes, ref) ?? [])
1176
+ let text: string
1177
+ if (query !== undefined || role !== undefined) {
1178
+ const found = findInTree(
1179
+ scope,
1180
+ hidden,
1181
+ redact,
1182
+ { ...(query !== undefined ? { query } : {}), ...(role !== undefined ? { role } : {}) },
1183
+ BROWSER_FIND_MAX_MATCHES,
1184
+ )
1185
+ text = found.lines.join('\n')
1186
+ const what = [
1187
+ query !== undefined ? `text ${quoteShort(query, 60)}` : '',
1188
+ role !== undefined ? `role ${quoteShort(role, 30)}` : '',
1189
+ ]
1190
+ .filter(Boolean)
1191
+ .join(' and ')
1192
+ this.notes.unshift(
1193
+ found.total === 0
1194
+ ? `Nothing on the page matches ${what}.`
1195
+ : found.total > found.lines.length
1196
+ ? `${found.total} elements match ${what}; the first ${found.lines.length} are listed. Narrow the search.`
1197
+ : `${found.total} element${found.total === 1 ? '' : 's'} match${found.total === 1 ? 'es' : ''} ${what}.`,
1198
+ )
1199
+ } else if (view === 'interactive') {
1200
+ text = renderInteractive(scope, hidden, redact)
1201
+ tab.lastSeen = whole.text
1202
+ } else if (view === 'reading') {
1203
+ text = renderReading(scope, hidden, redact)
1204
+ tab.lastSeen = whole.text
1205
+ } else if (view === 'delta') {
1206
+ const before = tab.lastSeen
1207
+ if (before === undefined) {
1208
+ this.notes.unshift(
1209
+ 'There is no earlier snapshot of this tab to compare with; this is the whole page.',
1210
+ )
1211
+ text =
1212
+ ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '')
1213
+ } else {
1214
+ const diff = diffLines(before, whole.text, DELTA_LINES_SHOWN)
1215
+ text =
1216
+ diff.added === 0 && diff.removed === 0
1217
+ ? '(no change since the last snapshot)'
1218
+ : [
1219
+ ...diff.addedLines.map((l) => `+ ${l}`),
1220
+ ...(diff.added > diff.addedLines.length
1221
+ ? [` (${diff.added - diff.addedLines.length} more lines added)`]
1222
+ : []),
1223
+ ...diff.removedLines.map((l) => `- ${l}`),
1224
+ ...(diff.removed > diff.removedLines.length
1225
+ ? [` (${diff.removed - diff.removedLines.length} more lines removed)`]
1226
+ : []),
1227
+ ].join('\n')
1228
+ this.notes.unshift(
1229
+ `Changes since the last snapshot: ${diff.added} lines added, ${diff.removed} removed. Take a full snapshot to see the whole page again.`,
1230
+ )
1231
+ }
1232
+ tab.lastSeen = whole.text
1233
+ } else {
1234
+ text =
1235
+ ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '')
1236
+ if (ref === undefined) tab.lastSeen = whole.text
987
1237
  }
988
1238
  if (text.trim() === '') text = '(nothing readable on this page)'
989
1239
  const first = this.pager.start(text)
@@ -994,10 +1244,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
994
1244
  }
995
1245
  }
996
1246
 
997
- private async snapshot(ref?: string, cursor?: string): Promise<BrowserResult> {
1247
+ private async snapshot(
1248
+ request: Extract<BrowserObserveAction, { action: 'snapshot' }>,
1249
+ ): Promise<BrowserResult> {
998
1250
  const tab = await this.activeTab()
999
- if (cursor !== undefined) {
1000
- const next = this.pager.resume(cursor)
1251
+ if (request.cursor !== undefined) {
1252
+ const next = this.pager.resume(request.cursor)
1001
1253
  if (!next) {
1002
1254
  throw new Error(
1003
1255
  'That cursor belongs to an older snapshot. Take a new snapshot and page through it.',
@@ -1011,8 +1263,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
1011
1263
  },
1012
1264
  })
1013
1265
  }
1014
- await this.settle()
1015
- return this.result({ snapshot: await this.takeSnapshot(tab, ref) })
1266
+ await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS })
1267
+ return this.result({
1268
+ snapshot: await this.takeSnapshot(tab, {
1269
+ ...(request.ref !== undefined ? { ref: request.ref } : {}),
1270
+ ...(request.view !== undefined ? { view: request.view } : {}),
1271
+ ...(request.query !== undefined ? { query: request.query } : {}),
1272
+ ...(request.role !== undefined ? { role: request.role } : {}),
1273
+ }),
1274
+ })
1016
1275
  }
1017
1276
 
1018
1277
  private async resolveRef(tab: Tab, ref: string): Promise<Locator> {
@@ -1035,10 +1294,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
1035
1294
  'A dialog is open; dismiss it with browser_act dialog before taking a screenshot.',
1036
1295
  )
1037
1296
  }
1038
- // Screenshots are image blocks sent to the model. Mask DOM-addressable
1039
- // inputs regardless of their names, including child frames and refs.
1040
- // Closed shadow roots and page-drawn text remain outside this mask.
1041
- const mask = tab.page.frames().map((frame) => frame.locator('input, textarea'))
1297
+ // Screenshots are image blocks sent to the model. Mask the fields that
1298
+ // take a password or a one-time code, in every frame; the other inputs
1299
+ // stay visible, because a search box or an address field the model
1300
+ // cannot read cannot be checked. Closed shadow roots and page-drawn
1301
+ // text remain outside this mask.
1302
+ const mask = await this.credentialMask(tab)
1042
1303
  const data =
1043
1304
  ref !== undefined
1044
1305
  ? await (await this.resolveRef(tab, ref)).screenshot({
@@ -1065,6 +1326,64 @@ export class PlaywrightBrowserHost implements BrowserHost {
1065
1326
  })
1066
1327
  }
1067
1328
 
1329
+ /** Locators of the credential fields (password, one-time code) of every frame of the tab. */
1330
+ private async credentialMask(tab: Tab): Promise<Locator[]> {
1331
+ const out: Locator[] = []
1332
+ for (const frame of tab.page.frames()) {
1333
+ const inputs = frame.locator('input, textarea')
1334
+ let count = 0
1335
+ try {
1336
+ count = Math.min(await inputs.count(), MASK_CANDIDATES_MAX)
1337
+ } catch {
1338
+ // A frame that went away while we looked has nothing to mask.
1339
+ continue
1340
+ }
1341
+ const verdicts = await Promise.all(
1342
+ Array.from({ length: count }, async (_, index) => {
1343
+ try {
1344
+ return isCredentialField(
1345
+ await inputs.nth(index).evaluate(fieldFacts, undefined, { timeout: 2000 }),
1346
+ )
1347
+ } catch {
1348
+ // A field that cannot be asked about is hidden to be safe.
1349
+ return true
1350
+ }
1351
+ }),
1352
+ )
1353
+ verdicts.forEach((hide, index) => {
1354
+ if (hide) out.push(inputs.nth(index))
1355
+ })
1356
+ }
1357
+ return out
1358
+ }
1359
+
1360
+ /** One element's current value and state, read from the page itself. */
1361
+ private async readValue(ref: string): Promise<BrowserResult> {
1362
+ const tab = await this.activeTab()
1363
+ if (this.pendingDialog?.tab === tab) {
1364
+ throw new Error(
1365
+ 'A dialog is open; dismiss it with browser_act dialog before reading a value.',
1366
+ )
1367
+ }
1368
+ const locator = await this.resolveRef(tab, ref)
1369
+ const facts = await locator.evaluate(fieldFacts)
1370
+ const described = this.refs.get(ref)
1371
+ const page = await this.pageInfo(tab)
1372
+ const hidden = isCredentialField(facts)
1373
+ const read = hidden ? undefined : await locator.evaluate(valueFacts)
1374
+ const value: BrowserValue = {
1375
+ page,
1376
+ ref,
1377
+ ...(described?.role !== undefined ? { role: described.role } : {}),
1378
+ ...(described?.name !== undefined ? { name: described.name } : {}),
1379
+ ...(read?.value !== undefined ? { value: read.value } : {}),
1380
+ state: read?.state ?? [],
1381
+ ...(read?.options !== undefined ? { options: read.options } : {}),
1382
+ ...(hidden ? { hidden: true } : {}),
1383
+ }
1384
+ return this.result({ value })
1385
+ }
1386
+
1068
1387
  private async scroll(
1069
1388
  direction: 'up' | 'down' | 'left' | 'right',
1070
1389
  ref?: string,
@@ -1125,10 +1444,19 @@ export class PlaywrightBrowserHost implements BrowserHost {
1125
1444
  : `${quoteShort(text)} was still there after ${timeout} ms.`,
1126
1445
  )
1127
1446
  }
1128
- } else {
1447
+ } else if (action.stable !== true) {
1129
1448
  await tab.page.waitForTimeout(timeout)
1130
1449
  this.notes.unshift(`Waited ${timeout} ms.`)
1131
1450
  }
1451
+ if (action.stable === true) {
1452
+ const cap = action.timeMs !== undefined ? timeout : STABLE_DEFAULT_WAIT_MS
1453
+ const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: cap })
1454
+ this.notes.push(
1455
+ settled.quiet
1456
+ ? `The page is quiet (nothing changed for ${QUIET_MS} ms; waited ${settled.waitedMs} ms).`
1457
+ : `The page was still changing after ${settled.waitedMs} ms.`,
1458
+ )
1459
+ }
1132
1460
  return this.result({ page: await this.pageInfo(tab) })
1133
1461
  }
1134
1462
 
@@ -1188,11 +1516,11 @@ export class PlaywrightBrowserHost implements BrowserHost {
1188
1516
  // Acting
1189
1517
  // -------------------------------------------------------------------------
1190
1518
 
1191
- private async guardField(locator: Locator): Promise<ElementFieldFacts> {
1192
- const facts = await locator.evaluate(fieldFacts)
1519
+ private async guardField(locator: Probe): Promise<ElementFieldFacts> {
1520
+ const facts = await evaluateOn(locator, fieldFacts)
1193
1521
  if (isCredentialField(facts)) {
1194
1522
  throw new Error(
1195
- 'This action targets a password or one-time-code control. Ask the operator to use the browser window.',
1523
+ 'This action targets a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.',
1196
1524
  )
1197
1525
  }
1198
1526
  return facts
@@ -1200,11 +1528,11 @@ export class PlaywrightBrowserHost implements BrowserHost {
1200
1528
 
1201
1529
  /** Refuse direct form submission while a form holds a password or code input. */
1202
1530
  private async guardCredentialForm(
1203
- locator: Locator,
1531
+ locator: Probe,
1204
1532
  action: 'click' | 'press' | 'type-submit',
1205
1533
  key?: string,
1206
1534
  ): Promise<void> {
1207
- const context = await locator.evaluate((element) => {
1535
+ const context = await evaluateOn(locator, (element) => {
1208
1536
  const label = element.closest('label') as HTMLLabelElement | null
1209
1537
  const labelledField = label?.control
1210
1538
  ? {
@@ -1251,20 +1579,77 @@ export class PlaywrightBrowserHost implements BrowserHost {
1251
1579
  action === 'type-submit'
1252
1580
  ) {
1253
1581
  throw new Error(
1254
- 'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.',
1582
+ 'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.',
1255
1583
  )
1256
1584
  }
1257
1585
  }
1258
1586
 
1587
+ /**
1588
+ * The page's tree as the model could have read it just before an action,
1589
+ * or `undefined` when it cannot be read now (a dialog holds the page, or
1590
+ * the read failed). An effect is a bonus; it never fails the action.
1591
+ */
1592
+ private async effectBaseline(tab: Tab): Promise<string | undefined> {
1593
+ if (this.pendingDialog?.tab === tab) return undefined
1594
+ try {
1595
+ return (await this.readTree(tab)).whole.text
1596
+ } catch {
1597
+ return undefined
1598
+ }
1599
+ }
1600
+
1601
+ /** What the action did, from the tree before and the tree now. */
1602
+ private async effectSince(
1603
+ before: string | undefined,
1604
+ urlBefore: string,
1605
+ tabsBefore: number,
1606
+ tab: Tab,
1607
+ settled: SettleOutcome,
1608
+ ): Promise<BrowserEffect | undefined> {
1609
+ if (before === undefined || this.pendingDialog?.tab === tab || tab.page.isClosed())
1610
+ return undefined
1611
+ let after: string
1612
+ try {
1613
+ after = (await this.readTree(tab)).whole.text
1614
+ } catch {
1615
+ return undefined
1616
+ }
1617
+ return buildEffect(
1618
+ {
1619
+ beforeTree: before,
1620
+ afterTree: after,
1621
+ urlBefore,
1622
+ urlNow: tab.page.url(),
1623
+ tabsBefore,
1624
+ tabsNow: this.tabs.size,
1625
+ settle: { quiet: settled.quiet, waitedMs: settled.waitedMs, capMs: settled.capMs },
1626
+ },
1627
+ EFFECT_LINES_SHOWN,
1628
+ )
1629
+ }
1630
+
1259
1631
  private async runAct(action: BrowserActAction): Promise<BrowserResult> {
1260
1632
  const tab = await this.activeTab()
1261
- await this.settle()
1633
+ await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS })
1262
1634
  const expected = canonicalizeBrowserOrigin(action.origin)
1263
1635
  const live = originOf(tab.page.url())
1264
1636
  if (!expected.ok || expected.origin !== live) {
1265
- throw new BrowserOriginMismatchError(
1266
- expected.ok ? expected.origin : action.origin,
1267
- live === 'null' ? 'about:blank' : live,
1637
+ // The page moved since the snapshot. Going on is fine only when the
1638
+ // site rules treat the new origin exactly as the named one: both at
1639
+ // `act`, where no review stood between the model and the page.
1640
+ const sameRules =
1641
+ expected.ok &&
1642
+ live !== 'null' &&
1643
+ this.policy.levelOf(expected.origin) === 'act' &&
1644
+ this.policy.levelOf(live) === 'act'
1645
+ if (!sameRules) {
1646
+ throw new BrowserOriginMismatchError(
1647
+ expected.ok ? expected.origin : action.origin,
1648
+ live === 'null' ? 'about:blank' : live,
1649
+ )
1650
+ }
1651
+ this.notes.unshift(
1652
+ `The page is now at ${live}, not ${expected.origin}; the site rules treat both alike, so the action went ahead there.`,
1268
1653
  )
1269
1654
  }
1270
1655
  if (!this.policy.canAct(live)) {
@@ -1278,11 +1663,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
1278
1663
  throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.')
1279
1664
  }
1280
1665
  const checkpoint = await this.beginDocumentResponseChange(tab)
1666
+ const urlBefore = tab.page.url()
1667
+ const tabsBefore = this.tabs.size
1668
+ const treeBefore = await this.effectBaseline(tab)
1669
+ let settled: SettleOutcome
1281
1670
  try {
1282
1671
  await this.perform(tab, action)
1283
- await this.settle()
1672
+ settled = await this.settle(tab)
1284
1673
  this.checkLanding(tab)
1285
- await this.settle()
1674
+ while (this.blanking.size > 0) await Promise.all([...this.blanking])
1286
1675
  } catch (error) {
1287
1676
  await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
1288
1677
  throw error
@@ -1290,8 +1679,14 @@ export class PlaywrightBrowserHost implements BrowserHost {
1290
1679
  await this.finishDocumentResponseChange(tab, checkpoint)
1291
1680
  await this.checkHuman(tab)
1292
1681
  const after = this.tabs.get(tab.id) ?? tab
1293
- if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(after) })
1294
- return this.result({ page: await this.pageInfo(after) })
1682
+ const effect = await this.effectSince(treeBefore, urlBefore, tabsBefore, after, settled)
1683
+ if (action.snapshot) {
1684
+ return this.result({
1685
+ snapshot: await this.takeSnapshot(after),
1686
+ ...(effect ? { effect } : {}),
1687
+ })
1688
+ }
1689
+ return this.result({ page: await this.pageInfo(after), ...(effect ? { effect } : {}) })
1295
1690
  }
1296
1691
 
1297
1692
  private async answerDialog(
@@ -1319,9 +1714,9 @@ export class PlaywrightBrowserHost implements BrowserHost {
1319
1714
  throw new BrowserOutcomeUnknownError('dialog', error)
1320
1715
  }
1321
1716
  this.notes.unshift('The dialog was dismissed.')
1322
- await this.settle()
1717
+ await this.settle(tab)
1323
1718
  this.checkLanding(tab)
1324
- await this.settle()
1719
+ while (this.blanking.size > 0) await Promise.all([...this.blanking])
1325
1720
  await this.finishDocumentResponseChange(tab, checkpoint)
1326
1721
  await this.checkHuman(tab)
1327
1722
  if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(tab) })
@@ -1336,29 +1731,45 @@ export class PlaywrightBrowserHost implements BrowserHost {
1336
1731
  private async perform(tab: Tab, action: BrowserActAction): Promise<void> {
1337
1732
  switch (action.action) {
1338
1733
  case 'click': {
1339
- const locator = await this.resolveRef(tab, action.ref)
1340
- await this.guardField(locator)
1341
- await this.guardCredentialForm(locator, 'click')
1342
- await locator.click({ trial: true })
1734
+ const target = await this.reach(tab, action)
1735
+ await this.guardField(target.locator)
1736
+ await this.guardCredentialForm(target.locator, 'click')
1737
+ const at = target.position ? { position: target.position } : {}
1738
+ await this.noEffectOnFailure('click', () => target.locator.click({ trial: true, ...at }))
1343
1739
  await this.unknownOnFailure('click', () =>
1344
- action.doubleClick ? locator.dblclick() : locator.click(),
1740
+ action.doubleClick ? target.locator.dblclick(at) : target.locator.click(at),
1345
1741
  )
1346
1742
  return
1347
1743
  }
1348
1744
  case 'hover': {
1349
- const locator = await this.resolveRef(tab, action.ref)
1350
- await this.guardField(locator)
1351
- await locator.hover({ trial: true })
1352
- await this.unknownOnFailure('hover', () => locator.hover())
1745
+ const target = await this.reach(tab, action)
1746
+ await this.guardField(target.locator)
1747
+ const at = target.position ? { position: target.position } : {}
1748
+ await this.noEffectOnFailure('hover', () => target.locator.hover({ trial: true, ...at }))
1749
+ await this.unknownOnFailure('hover', () => target.locator.hover(at))
1353
1750
  return
1354
1751
  }
1355
1752
  case 'type': {
1356
1753
  const locator = await this.resolveRef(tab, action.ref)
1357
1754
  await this.guardField(locator)
1358
1755
  if (action.submit) await this.guardCredentialForm(locator, 'type-submit')
1359
- await locator.click({ trial: true })
1360
- await this.unknownOnFailure('type', async () => {
1361
- await locator.fill(action.text)
1756
+ await this.noEffectOnFailure('type', () => locator.click({ trial: true }))
1757
+ await this.unknownOnFailure('type', async (begun) => {
1758
+ if (action.keys) {
1759
+ await locator.focus()
1760
+ // Clearing and typing change the field; from here a failure is not harmless.
1761
+ begun()
1762
+ if (action.append) await locator.press('Control+End')
1763
+ else await locator.fill('')
1764
+ await locator.pressSequentially(action.text, { delay: KEY_DELAY_MS })
1765
+ } else if (action.append) {
1766
+ const held = await locator
1767
+ .inputValue()
1768
+ .catch(() => locator.evaluate((el) => el.textContent ?? ''))
1769
+ await locator.fill(held + action.text)
1770
+ } else {
1771
+ await locator.fill(action.text)
1772
+ }
1362
1773
  if (action.submit) await locator.press('Enter')
1363
1774
  })
1364
1775
  return
@@ -1405,9 +1816,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
1405
1816
  await this.unknownOnFailure('press', () => locator.press(action.key))
1406
1817
  return
1407
1818
  }
1408
- throw new Error(
1409
- 'An untargeted key could reach a password or one-time-code control, including one inside a closed shadow root. Use a specific control instead.',
1410
- )
1819
+ const key = namedKey(action.key)
1820
+ if (!key) {
1821
+ throw new Error(
1822
+ `Without a ref, press takes only ${NAMED_KEYS.join(', ')}. Any other key goes to a specific control: give its ref.`,
1823
+ )
1824
+ }
1825
+ await this.guardFocus(tab, key)
1826
+ await this.unknownOnFailure('press', () => tab.page.keyboard.press(key))
1827
+ return
1411
1828
  }
1412
1829
  case 'upload': {
1413
1830
  const locator = await this.resolveRef(tab, action.ref)
@@ -1433,6 +1850,102 @@ export class PlaywrightBrowserHost implements BrowserHost {
1433
1850
  }
1434
1851
  }
1435
1852
 
1853
+ /**
1854
+ * The element a click or hover means, and where on it to aim. A ref names
1855
+ * it directly; a role with a name, or visible text, is looked up and
1856
+ * refused unless exactly one element matches. A container (a section, a
1857
+ * list item holding a menu title) is not clicked in its middle, which is
1858
+ * often empty: the aim goes to its first control or to its own text.
1859
+ */
1860
+ private async reach(
1861
+ tab: Tab,
1862
+ target: BrowserTarget,
1863
+ ): Promise<{ locator: Locator; position?: { x: number; y: number } }> {
1864
+ const found =
1865
+ target.ref !== undefined
1866
+ ? await this.resolveRef(tab, target.ref)
1867
+ : await this.findTarget(tab, target)
1868
+ let aim: Aim = { kind: 'centre' }
1869
+ try {
1870
+ aim = await found.evaluate(aimInside)
1871
+ } catch {
1872
+ // Aim at the middle, as before.
1873
+ }
1874
+ if (aim.kind === 'descendant') return { locator: found.locator(`xpath=${aim.path}`) }
1875
+ if (aim.kind === 'text') return { locator: found, position: { x: aim.x, y: aim.y } }
1876
+ return { locator: found }
1877
+ }
1878
+
1879
+ /** Find the one element a role/name or text target means, or refuse with the candidates. */
1880
+ private async findTarget(tab: Tab, target: BrowserTarget): Promise<Locator> {
1881
+ const page = tab.page
1882
+ const role = target.role
1883
+ const wanted =
1884
+ role !== undefined ? `${role} "${target.name ?? ''}"` : `text "${target.text ?? ''}"`
1885
+ const build = (exact: boolean): Locator =>
1886
+ role !== undefined
1887
+ ? page.getByRole(role as Parameters<Page['getByRole']>[0], {
1888
+ name: target.name ?? '',
1889
+ exact,
1890
+ })
1891
+ : page.getByText(target.text ?? '', { exact }).filter({ visible: true })
1892
+ let locator = build(false)
1893
+ let count = await locator.count()
1894
+ if (count > 1) {
1895
+ // A short name also fits longer ones ("Save" in "Save draft"): the exact matches decide.
1896
+ const exact = build(true)
1897
+ const exactCount = await exact.count()
1898
+ if (exactCount > 0) {
1899
+ locator = exact
1900
+ count = exactCount
1901
+ }
1902
+ }
1903
+ if (count === 1) return locator
1904
+ const pool =
1905
+ count > 0 ? locator : role !== undefined ? page.getByRole(role as never) : undefined
1906
+ const candidates = []
1907
+ if (pool) {
1908
+ const total = Math.min(await pool.count(), CANDIDATES_MAX)
1909
+ for (let i = 0; i < total; i++) {
1910
+ candidates.push(await pool.nth(i).evaluate(describeCandidate))
1911
+ }
1912
+ }
1913
+ throw new BrowserTargetUnclearError(wanted, count, candidates)
1914
+ }
1915
+
1916
+ /**
1917
+ * Refuse a key sent to the page when the focus is on a credential, or
1918
+ * when it cannot be told. Every frame is asked, because the key goes to
1919
+ * whichever one holds the focus and a stale focus in another frame is
1920
+ * still a reason to be careful.
1921
+ */
1922
+ private async guardFocus(tab: Tab, key: NamedKey): Promise<void> {
1923
+ for (const frame of tab.page.frames()) {
1924
+ const handle = await frame.evaluateHandle(deepFocus).catch(() => undefined)
1925
+ const element = handle?.asElement() ?? null
1926
+ if (!element) continue
1927
+ if (keyCanCommit(key) && (await element.evaluate(focusIsOpaque))) {
1928
+ throw new Error(
1929
+ `The focus is on a component whose inside cannot be read, so ${key} could reach a password or code field. Use browser_act with the ref of a specific control.`,
1930
+ )
1931
+ }
1932
+ await this.guardField(element)
1933
+ await this.guardCredentialForm(element, 'press', key)
1934
+ }
1935
+ }
1936
+
1937
+ /** A check Playwright runs without acting (`trial`) failed: nothing on the page changed. */
1938
+ private async noEffectOnFailure(
1939
+ action: BrowserActAction['action'],
1940
+ run: () => Promise<unknown>,
1941
+ ): Promise<void> {
1942
+ try {
1943
+ await run()
1944
+ } catch (error) {
1945
+ throw new BrowserNoEffectError(action, noEffectReason(error))
1946
+ }
1947
+ }
1948
+
1436
1949
  /**
1437
1950
  * Run the change. A failure once it has started is `outcome unknown`. A
1438
1951
  * dialog the action opens ends the wait: the page is paused until the
@@ -1440,9 +1953,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
1440
1953
  */
1441
1954
  private async unknownOnFailure(
1442
1955
  action: BrowserActAction['action'],
1443
- run: () => Promise<unknown>,
1956
+ run: (begun: () => void) => Promise<unknown>,
1444
1957
  ): Promise<void> {
1445
- const running = run()
1958
+ let touched = false
1959
+ const running = run(() => {
1960
+ touched = true
1961
+ })
1446
1962
  let timer: ReturnType<typeof setInterval> | undefined
1447
1963
  const dialogOpened = new Promise<'dialog'>((resolve) => {
1448
1964
  timer = setInterval(() => {
@@ -1453,7 +1969,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
1453
1969
  const outcome = await Promise.race([running, dialogOpened])
1454
1970
  if (outcome === 'dialog') running.catch(() => undefined)
1455
1971
  } catch (error) {
1456
- throw new BrowserOutcomeUnknownError(action, error)
1972
+ // A timeout whose log never reached the input step left the page alone.
1973
+ throw !touched && failedBeforeActing(error)
1974
+ ? new BrowserNoEffectError(action, noEffectReason(error))
1975
+ : new BrowserOutcomeUnknownError(action, error)
1457
1976
  } finally {
1458
1977
  clearInterval(timer)
1459
1978
  }