@namzu/browser 0.1.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/host.ts CHANGED
@@ -15,6 +15,7 @@ import type {
15
15
  import { canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome } from '@namzu/sdk'
16
16
  import {
17
17
  type BrowserContext,
18
+ type CDPSession,
18
19
  type Dialog,
19
20
  type Locator,
20
21
  type Page,
@@ -45,12 +46,7 @@ import {
45
46
  BrowserUnavailableError,
46
47
  ProfileBusyError,
47
48
  } from './errors.js'
48
- import {
49
- type ElementFieldFacts,
50
- countCredentialFields,
51
- fieldFacts,
52
- focusedFieldFacts,
53
- } from './page-scripts.js'
49
+ import { type ElementFieldFacts, countCredentialFields, fieldFacts } from './page-scripts.js'
54
50
  import { BrowserSitePolicy, type BrowserSiteRules, DEFAULT_BROWSER_SITE_RULES } from './policy.js'
55
51
  import {
56
52
  type BrowserLease,
@@ -109,13 +105,30 @@ export interface PlaywrightBrowserHostOptions {
109
105
  interface Tab {
110
106
  readonly id: string
111
107
  readonly page: Page
108
+ /** Main-frame navigation events, including same-document history moves. */
109
+ mainFrameCommits: number
110
+ mainDocumentResponses: number
111
+ cdp?: CDPSession
112
+ readonly responsesByLoader: Map<string, DocumentResponse>
112
113
  /** HTTP status of the main document's last response. */
113
114
  status?: number
115
+ wwwAuthenticate?: string
116
+ proxyAuthenticate?: string
117
+ }
118
+
119
+ type DocumentResponse = Pick<Tab, 'status' | 'wwwAuthenticate' | 'proxyAuthenticate'>
120
+
121
+ interface DocumentResponseCheckpoint {
122
+ readonly previous: DocumentResponse
123
+ readonly loaderId?: string
124
+ readonly responsesBefore: number
125
+ readonly commitsBefore: number
114
126
  }
115
127
 
116
128
  interface PendingDialog {
117
129
  readonly tab: Tab
118
130
  readonly dialog: Dialog
131
+ checkpoint?: DocumentResponseCheckpoint
119
132
  }
120
133
 
121
134
  const DEFAULT_NAVIGATION_TIMEOUT_MS = 30_000
@@ -127,6 +140,8 @@ const SNAPSHOT_PAGE_MAX = 20_000
127
140
  const EVENT_SETTLE_MS = 150
128
141
  /** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
129
142
  const CAPTCHA_FRAME_MIN_PX = 30
143
+ const REMEMBERED_DOCUMENTS_MAX = 64
144
+ const SCREENSHOT_INPUT_MASK_COLOR = '#666666'
130
145
 
131
146
  function firstLine(error: unknown): string {
132
147
  const text = error instanceof Error ? error.message : String(error)
@@ -161,9 +176,6 @@ function quoteShort(value: string, max = 80): string {
161
176
  return JSON.stringify(flat.length > max ? `${flat.slice(0, max - 1)}…` : flat)
162
177
  }
163
178
 
164
- /** Keys that type nothing: allowed while a credential field has focus. */
165
- const NON_TYPING_KEYS = new Set(['Tab', 'Shift+Tab', 'Escape', 'Enter'])
166
-
167
179
  /**
168
180
  * A {@link BrowserHost} that runs Chromium (or an installed Chrome or Edge)
169
181
  * in this process with Playwright, on a persistent profile. Inside WSL it
@@ -184,7 +196,7 @@ const NON_TYPING_KEYS = new Set(['Tab', 'Shift+Tab', 'Escape', 'Enter'])
184
196
  * anything else is blocked before it is sent; a redirect is caught when it
185
197
  * lands, and the tab is cleared to `about:blank`;
186
198
  * - acting needs the live origin at `ask` or `act`;
187
- * - a sign-in, second factor, CAPTCHA, bot wall or HTTP credential prompt
199
+ * - a sign-in, second factor, CAPTCHA, bot wall or HTTP authentication challenge
188
200
  * stops the call with `browser_human_required`;
189
201
  * - nothing is ever typed into a password or one-time-code field;
190
202
  * - downloads are cancelled and reported; a leave-page prompt is answered so
@@ -478,7 +490,13 @@ export class PlaywrightBrowserHost implements BrowserHost {
478
490
  const known = this.tabOf.get(page)
479
491
  if (known) return known
480
492
  this.tabCounter += 1
481
- const tab: Tab = { id: `t${this.tabCounter}`, page }
493
+ const tab: Tab = {
494
+ id: `t${this.tabCounter}`,
495
+ page,
496
+ mainFrameCommits: 0,
497
+ mainDocumentResponses: 0,
498
+ responsesByLoader: new Map(),
499
+ }
482
500
  this.tabs.set(tab.id, tab)
483
501
  this.tabOf.set(page, tab)
484
502
  if (this.activeTabId === undefined) this.activeTabId = tab.id
@@ -488,12 +506,19 @@ export class PlaywrightBrowserHost implements BrowserHost {
488
506
  )
489
507
  }
490
508
  page.on('framenavigated', (frame) => {
491
- if (frame === page.mainFrame()) this.checkLanding(tab)
509
+ if (frame === page.mainFrame()) {
510
+ tab.mainFrameCommits += 1
511
+ this.checkLanding(tab)
512
+ }
492
513
  })
493
514
  page.on('response', (response) => {
494
515
  try {
495
516
  if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
517
+ const headers = response.headers()
518
+ tab.mainDocumentResponses += 1
496
519
  tab.status = response.status()
520
+ tab.wwwAuthenticate = headers['www-authenticate']
521
+ tab.proxyAuthenticate = headers['proxy-authenticate']
497
522
  }
498
523
  } catch {
499
524
  // A response without a frame (a service worker's) says nothing about the page.
@@ -529,7 +554,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
529
554
  }
530
555
  this.pendingDialog = { tab, dialog }
531
556
  this.notes.push(
532
- `A ${dialog.type()} dialog is open on tab ${tab.id}. Take a snapshot to read it, then answer it with browser_act dialog.`,
557
+ `A ${dialog.type()} dialog is open on tab ${tab.id}. Take a snapshot to read it, then dismiss it with browser_act dialog or ask the operator to use the browser window.`,
533
558
  )
534
559
  }
535
560
 
@@ -617,7 +642,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
617
642
 
618
643
  private async pageInfo(tab: Tab): Promise<BrowserPageInfo> {
619
644
  const url = tab.page.url()
620
- return { origin: originOf(url), url, title: await this.title(tab), tab: tab.id }
645
+ return {
646
+ origin: originOf(url),
647
+ url,
648
+ title: await this.title(tab),
649
+ tab: tab.id,
650
+ }
621
651
  }
622
652
 
623
653
  private result(extra: Omit<BrowserResult, 'message'>): BrowserResult {
@@ -673,12 +703,128 @@ export class PlaywrightBrowserHost implements BrowserHost {
673
703
  url,
674
704
  title: await this.title(tab),
675
705
  ...(tab.status !== undefined ? { status: tab.status } : {}),
706
+ ...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
707
+ ...(tab.proxyAuthenticate !== undefined
708
+ ? { proxyAuthenticate: tab.proxyAuthenticate }
709
+ : {}),
676
710
  ...counts,
677
711
  frameUrls,
678
712
  },
679
713
  this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {},
680
714
  )
681
715
  if (reason) throw this.humanRequired(reason, tab)
716
+ if (tab.status === 401) {
717
+ throw new Error(
718
+ 'The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.',
719
+ )
720
+ }
721
+ if (tab.status === 407) {
722
+ throw new Error(
723
+ 'The proxy returned HTTP 407 without a Proxy-Authenticate challenge. Access was denied or the response is incomplete; no proxy password prompt was identified. Tell the user the proxy refused this request.',
724
+ )
725
+ }
726
+ }
727
+
728
+ private documentResponse(tab: Tab): DocumentResponse {
729
+ return {
730
+ status: tab.status,
731
+ wwwAuthenticate: tab.wwwAuthenticate,
732
+ proxyAuthenticate: tab.proxyAuthenticate,
733
+ }
734
+ }
735
+
736
+ private clearDocumentResponse(tab: Tab): void {
737
+ tab.status = undefined
738
+ tab.wwwAuthenticate = undefined
739
+ tab.proxyAuthenticate = undefined
740
+ }
741
+
742
+ private setDocumentResponse(tab: Tab, response: DocumentResponse): void {
743
+ tab.status = response.status
744
+ tab.wwwAuthenticate = response.wwwAuthenticate
745
+ tab.proxyAuthenticate = response.proxyAuthenticate
746
+ }
747
+
748
+ /** Chromium's loader ID belongs to a document, and survives same-document history moves. */
749
+ private async loaderId(tab: Tab): Promise<string | undefined> {
750
+ try {
751
+ tab.cdp ??= await tab.page.context().newCDPSession(tab.page)
752
+ const tree = (await tab.cdp.send('Page.getFrameTree')) as {
753
+ frameTree?: { frame?: { loaderId?: unknown } }
754
+ }
755
+ const id = tree.frameTree?.frame?.loaderId
756
+ return typeof id === 'string' && id.length > 0 ? id : undefined
757
+ } catch {
758
+ tab.cdp = undefined
759
+ return undefined
760
+ }
761
+ }
762
+
763
+ private rememberDocumentResponse(tab: Tab, loaderId: string, response: DocumentResponse): void {
764
+ tab.responsesByLoader.delete(loaderId)
765
+ tab.responsesByLoader.set(loaderId, response)
766
+ if (tab.responsesByLoader.size > REMEMBERED_DOCUMENTS_MAX) {
767
+ const oldest = tab.responsesByLoader.keys().next().value
768
+ if (oldest !== undefined) tab.responsesByLoader.delete(oldest)
769
+ }
770
+ }
771
+
772
+ private async beginDocumentResponseChange(tab: Tab): Promise<DocumentResponseCheckpoint> {
773
+ const loaderId = await this.loaderId(tab)
774
+ const previous = this.documentResponse(tab)
775
+ if (loaderId !== undefined) this.rememberDocumentResponse(tab, loaderId, previous)
776
+ const checkpoint = {
777
+ previous,
778
+ ...(loaderId !== undefined ? { loaderId } : {}),
779
+ responsesBefore: tab.mainDocumentResponses,
780
+ commitsBefore: tab.mainFrameCommits,
781
+ }
782
+ this.clearDocumentResponse(tab)
783
+ return checkpoint
784
+ }
785
+
786
+ private async finishDocumentResponseChange(
787
+ tab: Tab,
788
+ checkpoint: DocumentResponseCheckpoint,
789
+ ): Promise<void> {
790
+ if (this.pendingDialog?.tab === tab) {
791
+ // CDP page queries wait behind an open JavaScript dialog. The page
792
+ // cannot commit a navigation until the dialog is answered.
793
+ this.pendingDialog.checkpoint = checkpoint
794
+ this.setDocumentResponse(tab, checkpoint.previous)
795
+ return
796
+ }
797
+ const currentLoader = await this.loaderId(tab)
798
+ const freshResponse = tab.mainDocumentResponses !== checkpoint.responsesBefore
799
+ const committed =
800
+ (currentLoader !== undefined &&
801
+ checkpoint.loaderId !== undefined &&
802
+ currentLoader !== checkpoint.loaderId) ||
803
+ tab.mainFrameCommits !== checkpoint.commitsBefore
804
+ if (currentLoader !== undefined && currentLoader === checkpoint.loaderId) {
805
+ // A hash change, pushState, no-op, or failed request left this document in place.
806
+ this.setDocumentResponse(tab, checkpoint.previous)
807
+ } else if (freshResponse && committed) {
808
+ // A new main-document response and a committed navigation agree.
809
+ } else if (freshResponse) {
810
+ this.setDocumentResponse(tab, checkpoint.previous)
811
+ throw new Error(
812
+ 'The browser received a document response but could not confirm that it replaced the current page. Its access status is unknown; do not treat this as a successful page load.',
813
+ )
814
+ } else if (!freshResponse && currentLoader && tab.responsesByLoader.has(currentLoader)) {
815
+ // Back-forward cache restores a document without another response event.
816
+ this.setDocumentResponse(tab, tab.responsesByLoader.get(currentLoader) as DocumentResponse)
817
+ } else if (!freshResponse && !committed) {
818
+ // CDP can be unavailable while the browser is closing; no frame moved.
819
+ this.setDocumentResponse(tab, checkpoint.previous)
820
+ } else if (!freshResponse && originOf(tab.page.url()) !== 'null') {
821
+ throw new Error(
822
+ 'The browser moved to a page without a response that could be verified. Its access status is unknown; do not treat this as a successful page load.',
823
+ )
824
+ }
825
+ if (currentLoader !== undefined) {
826
+ this.rememberDocumentResponse(tab, currentLoader, this.documentResponse(tab))
827
+ }
682
828
  }
683
829
 
684
830
  // -------------------------------------------------------------------------
@@ -742,14 +888,22 @@ export class PlaywrightBrowserHost implements BrowserHost {
742
888
  this.policy.approve(verdict.origin)
743
889
  }
744
890
  await this.dismissPendingDialog(tab)
745
- tab.status = undefined
891
+ const checkpoint = await this.beginDocumentResponseChange(tab)
746
892
  try {
747
893
  await tab.page.goto(verdict.url, { waitUntil: 'domcontentloaded' })
748
894
  } catch (error) {
895
+ await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
896
+ if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
897
+ await this.settle()
898
+ throw new Error(
899
+ 'The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.',
900
+ )
901
+ }
749
902
  if (!/ERR_BLOCKED_BY_CLIENT|ERR_ABORTED|Download is starting/.test(firstLine(error))) {
750
903
  throw new Error(`Could not open ${verdict.url}: ${firstLine(error)}`)
751
904
  }
752
905
  }
906
+ await this.finishDocumentResponseChange(tab, checkpoint)
753
907
  return this.afterNavigation(tab)
754
908
  }
755
909
 
@@ -764,7 +918,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
764
918
  private async history(which: 'back' | 'forward' | 'reload'): Promise<BrowserResult> {
765
919
  const tab = await this.activeTab()
766
920
  await this.dismissPendingDialog(tab)
767
- tab.status = undefined
921
+ const checkpoint = await this.beginDocumentResponseChange(tab)
768
922
  // `commit`, then the document as a bonus: a page restored from the
769
923
  // back-forward cache fires no `domcontentloaded`, so waiting for it
770
924
  // timed out after a history move that had long finished (measured
@@ -781,8 +935,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
781
935
  ? await tab.page.goForward(options)
782
936
  : await tab.page.reload(options)
783
937
  } catch (error) {
938
+ await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
784
939
  if (!/ERR_BLOCKED_BY_CLIENT|ERR_ABORTED/.test(firstLine(error))) throw error
785
940
  }
941
+ await this.finishDocumentResponseChange(tab, checkpoint)
786
942
  if (response !== null) {
787
943
  await tab.page
788
944
  .waitForLoadState('domcontentloaded', { timeout: HISTORY_LOAD_WAIT_MS })
@@ -876,13 +1032,26 @@ export class PlaywrightBrowserHost implements BrowserHost {
876
1032
  const tab = await this.activeTab()
877
1033
  if (this.pendingDialog?.tab === tab) {
878
1034
  throw new Error(
879
- 'A dialog is open; answer it with browser_act dialog before taking a screenshot.',
1035
+ 'A dialog is open; dismiss it with browser_act dialog before taking a screenshot.',
880
1036
  )
881
1037
  }
1038
+ // Screenshots are image blocks sent to the model. Mask DOM-addressable
1039
+ // inputs regardless of their names, including child frames and refs.
1040
+ // Closed shadow roots and page-drawn text remain outside this mask.
1041
+ const mask = tab.page.frames().map((frame) => frame.locator('input, textarea'))
882
1042
  const data =
883
1043
  ref !== undefined
884
- ? await (await this.resolveRef(tab, ref)).screenshot({ type: 'png' })
885
- : await tab.page.screenshot({ type: 'png', fullPage: fullPage === true })
1044
+ ? await (await this.resolveRef(tab, ref)).screenshot({
1045
+ type: 'png',
1046
+ mask,
1047
+ maskColor: SCREENSHOT_INPUT_MASK_COLOR,
1048
+ })
1049
+ : await tab.page.screenshot({
1050
+ type: 'png',
1051
+ fullPage: fullPage === true,
1052
+ mask,
1053
+ maskColor: SCREENSHOT_INPUT_MASK_COLOR,
1054
+ })
886
1055
  const bytes = new Uint8Array(data)
887
1056
  const size = pngSize(bytes)
888
1057
  return this.result({
@@ -1007,7 +1176,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
1007
1176
  const out: BrowserTabInfo[] = []
1008
1177
  for (const tab of this.tabs.values()) {
1009
1178
  if (tab.page.isClosed()) continue
1010
- out.push({ ...(await this.pageInfo(tab)), active: tab.id === this.activeTabId })
1179
+ out.push({
1180
+ ...(await this.pageInfo(tab)),
1181
+ active: tab.id === this.activeTabId,
1182
+ })
1011
1183
  }
1012
1184
  return out
1013
1185
  }
@@ -1016,12 +1188,74 @@ export class PlaywrightBrowserHost implements BrowserHost {
1016
1188
  // Acting
1017
1189
  // -------------------------------------------------------------------------
1018
1190
 
1019
- private async guardField(tab: Tab, locator: Locator): Promise<ElementFieldFacts> {
1191
+ private async guardField(locator: Locator): Promise<ElementFieldFacts> {
1020
1192
  const facts = await locator.evaluate(fieldFacts)
1021
- if (isCredentialField(facts)) throw this.humanRequired('credential-field', tab)
1193
+ if (isCredentialField(facts)) {
1194
+ throw new Error(
1195
+ 'This action targets a password or one-time-code control. Ask the operator to use the browser window.',
1196
+ )
1197
+ }
1022
1198
  return facts
1023
1199
  }
1024
1200
 
1201
+ /** Refuse direct form submission while a form holds a password or code input. */
1202
+ private async guardCredentialForm(
1203
+ locator: Locator,
1204
+ action: 'click' | 'press' | 'type-submit',
1205
+ key?: string,
1206
+ ): Promise<void> {
1207
+ const context = await locator.evaluate((element) => {
1208
+ const label = element.closest('label') as HTMLLabelElement | null
1209
+ const labelledField = label?.control
1210
+ ? {
1211
+ tag: label.control.tagName.toLowerCase(),
1212
+ type: label.control instanceof HTMLInputElement ? label.control.type.toLowerCase() : '',
1213
+ autocomplete: label.control.getAttribute('autocomplete') ?? '',
1214
+ name: label.control.getAttribute('name') ?? '',
1215
+ id: label.control.id,
1216
+ label: label.textContent ?? '',
1217
+ }
1218
+ : undefined
1219
+ const button = element.closest('button, input[type="submit"], input[type="image"]')
1220
+ const associated = (button ?? element) as HTMLInputElement
1221
+ const form = associated.form ?? element.closest('form')
1222
+ const fields = form
1223
+ ? Array.from(form.elements)
1224
+ .filter(
1225
+ (field) => field instanceof HTMLInputElement || field instanceof HTMLTextAreaElement,
1226
+ )
1227
+ .map((field) => ({
1228
+ tag: field.tagName.toLowerCase(),
1229
+ type: field instanceof HTMLInputElement ? field.type.toLowerCase() : '',
1230
+ autocomplete: field.getAttribute('autocomplete') ?? '',
1231
+ name: field.getAttribute('name') ?? '',
1232
+ id: field.id,
1233
+ label: [
1234
+ field.getAttribute('aria-label') ?? '',
1235
+ field.getAttribute('placeholder') ?? '',
1236
+ ...Array.from((field as HTMLInputElement).labels ?? []).map(
1237
+ (label) => label.textContent ?? '',
1238
+ ),
1239
+ ].join(' '),
1240
+ }))
1241
+ : []
1242
+ return { isFormButton: button !== null, labelledField, fields }
1243
+ })
1244
+ if (context.labelledField && isCredentialField(context.labelledField)) {
1245
+ throw new Error('This action targets a password or one-time-code control.')
1246
+ }
1247
+ if (!context.fields.some(isCredentialField)) return
1248
+ if (
1249
+ (action === 'click' && context.isFormButton) ||
1250
+ (action === 'press' && (context.isFormButton || /enter|return/i.test(key ?? ''))) ||
1251
+ action === 'type-submit'
1252
+ ) {
1253
+ throw new Error(
1254
+ 'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.',
1255
+ )
1256
+ }
1257
+ }
1258
+
1025
1259
  private async runAct(action: BrowserActAction): Promise<BrowserResult> {
1026
1260
  const tab = await this.activeTab()
1027
1261
  await this.settle()
@@ -1041,13 +1275,19 @@ export class PlaywrightBrowserHost implements BrowserHost {
1041
1275
  }
1042
1276
  if (action.action === 'dialog') return this.answerDialog(tab, action)
1043
1277
  if (this.pendingDialog?.tab === tab) {
1044
- throw new Error('A dialog is open on this tab; answer it with browser_act dialog first.')
1278
+ throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.')
1045
1279
  }
1046
-
1047
- await this.perform(tab, action)
1048
- await this.settle()
1049
- this.checkLanding(tab)
1050
- await this.settle()
1280
+ const checkpoint = await this.beginDocumentResponseChange(tab)
1281
+ try {
1282
+ await this.perform(tab, action)
1283
+ await this.settle()
1284
+ this.checkLanding(tab)
1285
+ await this.settle()
1286
+ } catch (error) {
1287
+ await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
1288
+ throw error
1289
+ }
1290
+ await this.finishDocumentResponseChange(tab, checkpoint)
1051
1291
  await this.checkHuman(tab)
1052
1292
  const after = this.tabs.get(tab.id) ?? tab
1053
1293
  if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(after) })
@@ -1060,17 +1300,30 @@ export class PlaywrightBrowserHost implements BrowserHost {
1060
1300
  ): Promise<BrowserResult> {
1061
1301
  const pending = this.pendingDialog
1062
1302
  if (!pending || pending.tab !== tab) throw new Error('No dialog is open on this tab.')
1303
+ if (action.accept) {
1304
+ throw new Error(
1305
+ 'The agent cannot accept a page-created dialog. Dismiss it, or ask the operator to use the browser window.',
1306
+ )
1307
+ }
1063
1308
  this.pendingDialog = undefined
1309
+ const checkpoint = pending.checkpoint ?? {
1310
+ previous: this.documentResponse(tab),
1311
+ responsesBefore: tab.mainDocumentResponses,
1312
+ commitsBefore: tab.mainFrameCommits,
1313
+ }
1314
+ this.clearDocumentResponse(tab)
1064
1315
  try {
1065
- if (action.accept) await pending.dialog.accept(action.promptText)
1066
- else await pending.dialog.dismiss()
1316
+ await pending.dialog.dismiss()
1067
1317
  } catch (error) {
1318
+ this.setDocumentResponse(tab, checkpoint.previous)
1068
1319
  throw new BrowserOutcomeUnknownError('dialog', error)
1069
1320
  }
1070
- this.notes.unshift(action.accept ? 'The dialog was accepted.' : 'The dialog was dismissed.')
1321
+ this.notes.unshift('The dialog was dismissed.')
1071
1322
  await this.settle()
1072
1323
  this.checkLanding(tab)
1073
1324
  await this.settle()
1325
+ await this.finishDocumentResponseChange(tab, checkpoint)
1326
+ await this.checkHuman(tab)
1074
1327
  if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(tab) })
1075
1328
  return this.result({ page: await this.pageInfo(tab) })
1076
1329
  }
@@ -1084,6 +1337,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
1084
1337
  switch (action.action) {
1085
1338
  case 'click': {
1086
1339
  const locator = await this.resolveRef(tab, action.ref)
1340
+ await this.guardField(locator)
1341
+ await this.guardCredentialForm(locator, 'click')
1087
1342
  await locator.click({ trial: true })
1088
1343
  await this.unknownOnFailure('click', () =>
1089
1344
  action.doubleClick ? locator.dblclick() : locator.click(),
@@ -1092,13 +1347,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
1092
1347
  }
1093
1348
  case 'hover': {
1094
1349
  const locator = await this.resolveRef(tab, action.ref)
1350
+ await this.guardField(locator)
1095
1351
  await locator.hover({ trial: true })
1096
1352
  await this.unknownOnFailure('hover', () => locator.hover())
1097
1353
  return
1098
1354
  }
1099
1355
  case 'type': {
1100
1356
  const locator = await this.resolveRef(tab, action.ref)
1101
- await this.guardField(tab, locator)
1357
+ await this.guardField(locator)
1358
+ if (action.submit) await this.guardCredentialForm(locator, 'type-submit')
1102
1359
  await locator.click({ trial: true })
1103
1360
  await this.unknownOnFailure('type', async () => {
1104
1361
  await locator.fill(action.text)
@@ -1107,10 +1364,14 @@ export class PlaywrightBrowserHost implements BrowserHost {
1107
1364
  return
1108
1365
  }
1109
1366
  case 'fill_form': {
1110
- const plan: { locator: Locator; facts: ElementFieldFacts; value: string }[] = []
1367
+ const plan: {
1368
+ locator: Locator
1369
+ facts: ElementFieldFacts
1370
+ value: string
1371
+ }[] = []
1111
1372
  for (const field of action.fields) {
1112
1373
  const locator = await this.resolveRef(tab, field.ref)
1113
- const facts = await this.guardField(tab, locator)
1374
+ const facts = await this.guardField(locator)
1114
1375
  if (facts.kind === 'file') {
1115
1376
  throw new Error(`${field.ref} is a file input; use browser_act upload for it.`)
1116
1377
  }
@@ -1139,27 +1400,24 @@ export class PlaywrightBrowserHost implements BrowserHost {
1139
1400
  case 'press': {
1140
1401
  if (action.ref !== undefined) {
1141
1402
  const locator = await this.resolveRef(tab, action.ref)
1142
- const facts = await locator.evaluate(fieldFacts)
1143
- if (isCredentialField(facts) && !NON_TYPING_KEYS.has(action.key)) {
1144
- throw this.humanRequired('credential-field', tab)
1145
- }
1403
+ await this.guardField(locator)
1404
+ await this.guardCredentialForm(locator, 'press', action.key)
1146
1405
  await this.unknownOnFailure('press', () => locator.press(action.key))
1147
1406
  return
1148
1407
  }
1149
- const focused = await tab.page.evaluate(focusedFieldFacts).catch(() => null)
1150
- if (focused && isCredentialField(focused) && !NON_TYPING_KEYS.has(action.key)) {
1151
- throw this.humanRequired('credential-field', tab)
1152
- }
1153
- await this.unknownOnFailure('press', () => tab.page.keyboard.press(action.key))
1154
- return
1408
+ throw new Error(
1409
+ 'An untargeted key could reach a password or one-time-code control, including one inside a closed shadow root. Use a specific control instead.',
1410
+ )
1155
1411
  }
1156
1412
  case 'upload': {
1157
1413
  const locator = await this.resolveRef(tab, action.ref)
1414
+ await this.guardField(locator)
1158
1415
  const facts = await locator.evaluate(fieldFacts)
1159
1416
  if (facts.kind === 'file') {
1160
1417
  await this.unknownOnFailure('upload', () => locator.setInputFiles(action.path))
1161
1418
  return
1162
1419
  }
1420
+ await this.guardCredentialForm(locator, 'click')
1163
1421
  await locator.click({ trial: true })
1164
1422
  await this.unknownOnFailure('upload', async () => {
1165
1423
  const [chooser] = await Promise.all([