@namzu/browser 0.1.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +10 -4
- package/dist/classifier.d.ts +9 -5
- package/dist/classifier.d.ts.map +1 -1
- package/dist/classifier.js +11 -8
- package/dist/classifier.js.map +1 -1
- package/dist/host.d.ts +11 -1
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +247 -39
- package/dist/host.js.map +1 -1
- package/package.json +2 -2
- package/src/classifier.ts +21 -8
- package/src/host.ts +302 -44
package/src/host.ts
CHANGED
|
@@ -15,6 +15,7 @@ import type {
|
|
|
15
15
|
import { canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome } from '@namzu/sdk'
|
|
16
16
|
import {
|
|
17
17
|
type BrowserContext,
|
|
18
|
+
type CDPSession,
|
|
18
19
|
type Dialog,
|
|
19
20
|
type Locator,
|
|
20
21
|
type Page,
|
|
@@ -45,12 +46,7 @@ import {
|
|
|
45
46
|
BrowserUnavailableError,
|
|
46
47
|
ProfileBusyError,
|
|
47
48
|
} from './errors.js'
|
|
48
|
-
import {
|
|
49
|
-
type ElementFieldFacts,
|
|
50
|
-
countCredentialFields,
|
|
51
|
-
fieldFacts,
|
|
52
|
-
focusedFieldFacts,
|
|
53
|
-
} from './page-scripts.js'
|
|
49
|
+
import { type ElementFieldFacts, countCredentialFields, fieldFacts } from './page-scripts.js'
|
|
54
50
|
import { BrowserSitePolicy, type BrowserSiteRules, DEFAULT_BROWSER_SITE_RULES } from './policy.js'
|
|
55
51
|
import {
|
|
56
52
|
type BrowserLease,
|
|
@@ -109,13 +105,30 @@ export interface PlaywrightBrowserHostOptions {
|
|
|
109
105
|
interface Tab {
|
|
110
106
|
readonly id: string
|
|
111
107
|
readonly page: Page
|
|
108
|
+
/** Main-frame navigation events, including same-document history moves. */
|
|
109
|
+
mainFrameCommits: number
|
|
110
|
+
mainDocumentResponses: number
|
|
111
|
+
cdp?: CDPSession
|
|
112
|
+
readonly responsesByLoader: Map<string, DocumentResponse>
|
|
112
113
|
/** HTTP status of the main document's last response. */
|
|
113
114
|
status?: number
|
|
115
|
+
wwwAuthenticate?: string
|
|
116
|
+
proxyAuthenticate?: string
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
type DocumentResponse = Pick<Tab, 'status' | 'wwwAuthenticate' | 'proxyAuthenticate'>
|
|
120
|
+
|
|
121
|
+
interface DocumentResponseCheckpoint {
|
|
122
|
+
readonly previous: DocumentResponse
|
|
123
|
+
readonly loaderId?: string
|
|
124
|
+
readonly responsesBefore: number
|
|
125
|
+
readonly commitsBefore: number
|
|
114
126
|
}
|
|
115
127
|
|
|
116
128
|
interface PendingDialog {
|
|
117
129
|
readonly tab: Tab
|
|
118
130
|
readonly dialog: Dialog
|
|
131
|
+
checkpoint?: DocumentResponseCheckpoint
|
|
119
132
|
}
|
|
120
133
|
|
|
121
134
|
const DEFAULT_NAVIGATION_TIMEOUT_MS = 30_000
|
|
@@ -127,6 +140,8 @@ const SNAPSHOT_PAGE_MAX = 20_000
|
|
|
127
140
|
const EVENT_SETTLE_MS = 150
|
|
128
141
|
/** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
|
|
129
142
|
const CAPTCHA_FRAME_MIN_PX = 30
|
|
143
|
+
const REMEMBERED_DOCUMENTS_MAX = 64
|
|
144
|
+
const SCREENSHOT_INPUT_MASK_COLOR = '#666666'
|
|
130
145
|
|
|
131
146
|
function firstLine(error: unknown): string {
|
|
132
147
|
const text = error instanceof Error ? error.message : String(error)
|
|
@@ -161,9 +176,6 @@ function quoteShort(value: string, max = 80): string {
|
|
|
161
176
|
return JSON.stringify(flat.length > max ? `${flat.slice(0, max - 1)}…` : flat)
|
|
162
177
|
}
|
|
163
178
|
|
|
164
|
-
/** Keys that type nothing: allowed while a credential field has focus. */
|
|
165
|
-
const NON_TYPING_KEYS = new Set(['Tab', 'Shift+Tab', 'Escape', 'Enter'])
|
|
166
|
-
|
|
167
179
|
/**
|
|
168
180
|
* A {@link BrowserHost} that runs Chromium (or an installed Chrome or Edge)
|
|
169
181
|
* in this process with Playwright, on a persistent profile. Inside WSL it
|
|
@@ -184,7 +196,7 @@ const NON_TYPING_KEYS = new Set(['Tab', 'Shift+Tab', 'Escape', 'Enter'])
|
|
|
184
196
|
* anything else is blocked before it is sent; a redirect is caught when it
|
|
185
197
|
* lands, and the tab is cleared to `about:blank`;
|
|
186
198
|
* - acting needs the live origin at `ask` or `act`;
|
|
187
|
-
* - a sign-in, second factor, CAPTCHA, bot wall or HTTP
|
|
199
|
+
* - a sign-in, second factor, CAPTCHA, bot wall or HTTP authentication challenge
|
|
188
200
|
* stops the call with `browser_human_required`;
|
|
189
201
|
* - nothing is ever typed into a password or one-time-code field;
|
|
190
202
|
* - downloads are cancelled and reported; a leave-page prompt is answered so
|
|
@@ -478,7 +490,13 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
478
490
|
const known = this.tabOf.get(page)
|
|
479
491
|
if (known) return known
|
|
480
492
|
this.tabCounter += 1
|
|
481
|
-
const tab: Tab = {
|
|
493
|
+
const tab: Tab = {
|
|
494
|
+
id: `t${this.tabCounter}`,
|
|
495
|
+
page,
|
|
496
|
+
mainFrameCommits: 0,
|
|
497
|
+
mainDocumentResponses: 0,
|
|
498
|
+
responsesByLoader: new Map(),
|
|
499
|
+
}
|
|
482
500
|
this.tabs.set(tab.id, tab)
|
|
483
501
|
this.tabOf.set(page, tab)
|
|
484
502
|
if (this.activeTabId === undefined) this.activeTabId = tab.id
|
|
@@ -488,12 +506,19 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
488
506
|
)
|
|
489
507
|
}
|
|
490
508
|
page.on('framenavigated', (frame) => {
|
|
491
|
-
if (frame === page.mainFrame())
|
|
509
|
+
if (frame === page.mainFrame()) {
|
|
510
|
+
tab.mainFrameCommits += 1
|
|
511
|
+
this.checkLanding(tab)
|
|
512
|
+
}
|
|
492
513
|
})
|
|
493
514
|
page.on('response', (response) => {
|
|
494
515
|
try {
|
|
495
516
|
if (response.request().isNavigationRequest() && response.frame() === page.mainFrame()) {
|
|
517
|
+
const headers = response.headers()
|
|
518
|
+
tab.mainDocumentResponses += 1
|
|
496
519
|
tab.status = response.status()
|
|
520
|
+
tab.wwwAuthenticate = headers['www-authenticate']
|
|
521
|
+
tab.proxyAuthenticate = headers['proxy-authenticate']
|
|
497
522
|
}
|
|
498
523
|
} catch {
|
|
499
524
|
// A response without a frame (a service worker's) says nothing about the page.
|
|
@@ -529,7 +554,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
529
554
|
}
|
|
530
555
|
this.pendingDialog = { tab, dialog }
|
|
531
556
|
this.notes.push(
|
|
532
|
-
`A ${dialog.type()} dialog is open on tab ${tab.id}. Take a snapshot to read it, then
|
|
557
|
+
`A ${dialog.type()} dialog is open on tab ${tab.id}. Take a snapshot to read it, then dismiss it with browser_act dialog or ask the operator to use the browser window.`,
|
|
533
558
|
)
|
|
534
559
|
}
|
|
535
560
|
|
|
@@ -617,7 +642,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
617
642
|
|
|
618
643
|
private async pageInfo(tab: Tab): Promise<BrowserPageInfo> {
|
|
619
644
|
const url = tab.page.url()
|
|
620
|
-
return {
|
|
645
|
+
return {
|
|
646
|
+
origin: originOf(url),
|
|
647
|
+
url,
|
|
648
|
+
title: await this.title(tab),
|
|
649
|
+
tab: tab.id,
|
|
650
|
+
}
|
|
621
651
|
}
|
|
622
652
|
|
|
623
653
|
private result(extra: Omit<BrowserResult, 'message'>): BrowserResult {
|
|
@@ -673,12 +703,128 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
673
703
|
url,
|
|
674
704
|
title: await this.title(tab),
|
|
675
705
|
...(tab.status !== undefined ? { status: tab.status } : {}),
|
|
706
|
+
...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
|
|
707
|
+
...(tab.proxyAuthenticate !== undefined
|
|
708
|
+
? { proxyAuthenticate: tab.proxyAuthenticate }
|
|
709
|
+
: {}),
|
|
676
710
|
...counts,
|
|
677
711
|
frameUrls,
|
|
678
712
|
},
|
|
679
713
|
this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {},
|
|
680
714
|
)
|
|
681
715
|
if (reason) throw this.humanRequired(reason, tab)
|
|
716
|
+
if (tab.status === 401) {
|
|
717
|
+
throw new Error(
|
|
718
|
+
'The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.',
|
|
719
|
+
)
|
|
720
|
+
}
|
|
721
|
+
if (tab.status === 407) {
|
|
722
|
+
throw new Error(
|
|
723
|
+
'The proxy returned HTTP 407 without a Proxy-Authenticate challenge. Access was denied or the response is incomplete; no proxy password prompt was identified. Tell the user the proxy refused this request.',
|
|
724
|
+
)
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
private documentResponse(tab: Tab): DocumentResponse {
|
|
729
|
+
return {
|
|
730
|
+
status: tab.status,
|
|
731
|
+
wwwAuthenticate: tab.wwwAuthenticate,
|
|
732
|
+
proxyAuthenticate: tab.proxyAuthenticate,
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
private clearDocumentResponse(tab: Tab): void {
|
|
737
|
+
tab.status = undefined
|
|
738
|
+
tab.wwwAuthenticate = undefined
|
|
739
|
+
tab.proxyAuthenticate = undefined
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
private setDocumentResponse(tab: Tab, response: DocumentResponse): void {
|
|
743
|
+
tab.status = response.status
|
|
744
|
+
tab.wwwAuthenticate = response.wwwAuthenticate
|
|
745
|
+
tab.proxyAuthenticate = response.proxyAuthenticate
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/** Chromium's loader ID belongs to a document, and survives same-document history moves. */
|
|
749
|
+
private async loaderId(tab: Tab): Promise<string | undefined> {
|
|
750
|
+
try {
|
|
751
|
+
tab.cdp ??= await tab.page.context().newCDPSession(tab.page)
|
|
752
|
+
const tree = (await tab.cdp.send('Page.getFrameTree')) as {
|
|
753
|
+
frameTree?: { frame?: { loaderId?: unknown } }
|
|
754
|
+
}
|
|
755
|
+
const id = tree.frameTree?.frame?.loaderId
|
|
756
|
+
return typeof id === 'string' && id.length > 0 ? id : undefined
|
|
757
|
+
} catch {
|
|
758
|
+
tab.cdp = undefined
|
|
759
|
+
return undefined
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
private rememberDocumentResponse(tab: Tab, loaderId: string, response: DocumentResponse): void {
|
|
764
|
+
tab.responsesByLoader.delete(loaderId)
|
|
765
|
+
tab.responsesByLoader.set(loaderId, response)
|
|
766
|
+
if (tab.responsesByLoader.size > REMEMBERED_DOCUMENTS_MAX) {
|
|
767
|
+
const oldest = tab.responsesByLoader.keys().next().value
|
|
768
|
+
if (oldest !== undefined) tab.responsesByLoader.delete(oldest)
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
private async beginDocumentResponseChange(tab: Tab): Promise<DocumentResponseCheckpoint> {
|
|
773
|
+
const loaderId = await this.loaderId(tab)
|
|
774
|
+
const previous = this.documentResponse(tab)
|
|
775
|
+
if (loaderId !== undefined) this.rememberDocumentResponse(tab, loaderId, previous)
|
|
776
|
+
const checkpoint = {
|
|
777
|
+
previous,
|
|
778
|
+
...(loaderId !== undefined ? { loaderId } : {}),
|
|
779
|
+
responsesBefore: tab.mainDocumentResponses,
|
|
780
|
+
commitsBefore: tab.mainFrameCommits,
|
|
781
|
+
}
|
|
782
|
+
this.clearDocumentResponse(tab)
|
|
783
|
+
return checkpoint
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
private async finishDocumentResponseChange(
|
|
787
|
+
tab: Tab,
|
|
788
|
+
checkpoint: DocumentResponseCheckpoint,
|
|
789
|
+
): Promise<void> {
|
|
790
|
+
if (this.pendingDialog?.tab === tab) {
|
|
791
|
+
// CDP page queries wait behind an open JavaScript dialog. The page
|
|
792
|
+
// cannot commit a navigation until the dialog is answered.
|
|
793
|
+
this.pendingDialog.checkpoint = checkpoint
|
|
794
|
+
this.setDocumentResponse(tab, checkpoint.previous)
|
|
795
|
+
return
|
|
796
|
+
}
|
|
797
|
+
const currentLoader = await this.loaderId(tab)
|
|
798
|
+
const freshResponse = tab.mainDocumentResponses !== checkpoint.responsesBefore
|
|
799
|
+
const committed =
|
|
800
|
+
(currentLoader !== undefined &&
|
|
801
|
+
checkpoint.loaderId !== undefined &&
|
|
802
|
+
currentLoader !== checkpoint.loaderId) ||
|
|
803
|
+
tab.mainFrameCommits !== checkpoint.commitsBefore
|
|
804
|
+
if (currentLoader !== undefined && currentLoader === checkpoint.loaderId) {
|
|
805
|
+
// A hash change, pushState, no-op, or failed request left this document in place.
|
|
806
|
+
this.setDocumentResponse(tab, checkpoint.previous)
|
|
807
|
+
} else if (freshResponse && committed) {
|
|
808
|
+
// A new main-document response and a committed navigation agree.
|
|
809
|
+
} else if (freshResponse) {
|
|
810
|
+
this.setDocumentResponse(tab, checkpoint.previous)
|
|
811
|
+
throw new Error(
|
|
812
|
+
'The browser received a document response but could not confirm that it replaced the current page. Its access status is unknown; do not treat this as a successful page load.',
|
|
813
|
+
)
|
|
814
|
+
} else if (!freshResponse && currentLoader && tab.responsesByLoader.has(currentLoader)) {
|
|
815
|
+
// Back-forward cache restores a document without another response event.
|
|
816
|
+
this.setDocumentResponse(tab, tab.responsesByLoader.get(currentLoader) as DocumentResponse)
|
|
817
|
+
} else if (!freshResponse && !committed) {
|
|
818
|
+
// CDP can be unavailable while the browser is closing; no frame moved.
|
|
819
|
+
this.setDocumentResponse(tab, checkpoint.previous)
|
|
820
|
+
} else if (!freshResponse && originOf(tab.page.url()) !== 'null') {
|
|
821
|
+
throw new Error(
|
|
822
|
+
'The browser moved to a page without a response that could be verified. Its access status is unknown; do not treat this as a successful page load.',
|
|
823
|
+
)
|
|
824
|
+
}
|
|
825
|
+
if (currentLoader !== undefined) {
|
|
826
|
+
this.rememberDocumentResponse(tab, currentLoader, this.documentResponse(tab))
|
|
827
|
+
}
|
|
682
828
|
}
|
|
683
829
|
|
|
684
830
|
// -------------------------------------------------------------------------
|
|
@@ -742,14 +888,22 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
742
888
|
this.policy.approve(verdict.origin)
|
|
743
889
|
}
|
|
744
890
|
await this.dismissPendingDialog(tab)
|
|
745
|
-
|
|
891
|
+
const checkpoint = await this.beginDocumentResponseChange(tab)
|
|
746
892
|
try {
|
|
747
893
|
await tab.page.goto(verdict.url, { waitUntil: 'domcontentloaded' })
|
|
748
894
|
} catch (error) {
|
|
895
|
+
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
|
|
896
|
+
if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
|
|
897
|
+
await this.settle()
|
|
898
|
+
throw new Error(
|
|
899
|
+
'The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.',
|
|
900
|
+
)
|
|
901
|
+
}
|
|
749
902
|
if (!/ERR_BLOCKED_BY_CLIENT|ERR_ABORTED|Download is starting/.test(firstLine(error))) {
|
|
750
903
|
throw new Error(`Could not open ${verdict.url}: ${firstLine(error)}`)
|
|
751
904
|
}
|
|
752
905
|
}
|
|
906
|
+
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
753
907
|
return this.afterNavigation(tab)
|
|
754
908
|
}
|
|
755
909
|
|
|
@@ -764,7 +918,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
764
918
|
private async history(which: 'back' | 'forward' | 'reload'): Promise<BrowserResult> {
|
|
765
919
|
const tab = await this.activeTab()
|
|
766
920
|
await this.dismissPendingDialog(tab)
|
|
767
|
-
|
|
921
|
+
const checkpoint = await this.beginDocumentResponseChange(tab)
|
|
768
922
|
// `commit`, then the document as a bonus: a page restored from the
|
|
769
923
|
// back-forward cache fires no `domcontentloaded`, so waiting for it
|
|
770
924
|
// timed out after a history move that had long finished (measured
|
|
@@ -781,8 +935,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
781
935
|
? await tab.page.goForward(options)
|
|
782
936
|
: await tab.page.reload(options)
|
|
783
937
|
} catch (error) {
|
|
938
|
+
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
|
|
784
939
|
if (!/ERR_BLOCKED_BY_CLIENT|ERR_ABORTED/.test(firstLine(error))) throw error
|
|
785
940
|
}
|
|
941
|
+
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
786
942
|
if (response !== null) {
|
|
787
943
|
await tab.page
|
|
788
944
|
.waitForLoadState('domcontentloaded', { timeout: HISTORY_LOAD_WAIT_MS })
|
|
@@ -876,13 +1032,26 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
876
1032
|
const tab = await this.activeTab()
|
|
877
1033
|
if (this.pendingDialog?.tab === tab) {
|
|
878
1034
|
throw new Error(
|
|
879
|
-
'A dialog is open;
|
|
1035
|
+
'A dialog is open; dismiss it with browser_act dialog before taking a screenshot.',
|
|
880
1036
|
)
|
|
881
1037
|
}
|
|
1038
|
+
// Screenshots are image blocks sent to the model. Mask DOM-addressable
|
|
1039
|
+
// inputs regardless of their names, including child frames and refs.
|
|
1040
|
+
// Closed shadow roots and page-drawn text remain outside this mask.
|
|
1041
|
+
const mask = tab.page.frames().map((frame) => frame.locator('input, textarea'))
|
|
882
1042
|
const data =
|
|
883
1043
|
ref !== undefined
|
|
884
|
-
? await (await this.resolveRef(tab, ref)).screenshot({
|
|
885
|
-
|
|
1044
|
+
? await (await this.resolveRef(tab, ref)).screenshot({
|
|
1045
|
+
type: 'png',
|
|
1046
|
+
mask,
|
|
1047
|
+
maskColor: SCREENSHOT_INPUT_MASK_COLOR,
|
|
1048
|
+
})
|
|
1049
|
+
: await tab.page.screenshot({
|
|
1050
|
+
type: 'png',
|
|
1051
|
+
fullPage: fullPage === true,
|
|
1052
|
+
mask,
|
|
1053
|
+
maskColor: SCREENSHOT_INPUT_MASK_COLOR,
|
|
1054
|
+
})
|
|
886
1055
|
const bytes = new Uint8Array(data)
|
|
887
1056
|
const size = pngSize(bytes)
|
|
888
1057
|
return this.result({
|
|
@@ -1007,7 +1176,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1007
1176
|
const out: BrowserTabInfo[] = []
|
|
1008
1177
|
for (const tab of this.tabs.values()) {
|
|
1009
1178
|
if (tab.page.isClosed()) continue
|
|
1010
|
-
out.push({
|
|
1179
|
+
out.push({
|
|
1180
|
+
...(await this.pageInfo(tab)),
|
|
1181
|
+
active: tab.id === this.activeTabId,
|
|
1182
|
+
})
|
|
1011
1183
|
}
|
|
1012
1184
|
return out
|
|
1013
1185
|
}
|
|
@@ -1016,12 +1188,74 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1016
1188
|
// Acting
|
|
1017
1189
|
// -------------------------------------------------------------------------
|
|
1018
1190
|
|
|
1019
|
-
private async guardField(
|
|
1191
|
+
private async guardField(locator: Locator): Promise<ElementFieldFacts> {
|
|
1020
1192
|
const facts = await locator.evaluate(fieldFacts)
|
|
1021
|
-
if (isCredentialField(facts))
|
|
1193
|
+
if (isCredentialField(facts)) {
|
|
1194
|
+
throw new Error(
|
|
1195
|
+
'This action targets a password or one-time-code control. Ask the operator to use the browser window.',
|
|
1196
|
+
)
|
|
1197
|
+
}
|
|
1022
1198
|
return facts
|
|
1023
1199
|
}
|
|
1024
1200
|
|
|
1201
|
+
/** Refuse direct form submission while a form holds a password or code input. */
|
|
1202
|
+
private async guardCredentialForm(
|
|
1203
|
+
locator: Locator,
|
|
1204
|
+
action: 'click' | 'press' | 'type-submit',
|
|
1205
|
+
key?: string,
|
|
1206
|
+
): Promise<void> {
|
|
1207
|
+
const context = await locator.evaluate((element) => {
|
|
1208
|
+
const label = element.closest('label') as HTMLLabelElement | null
|
|
1209
|
+
const labelledField = label?.control
|
|
1210
|
+
? {
|
|
1211
|
+
tag: label.control.tagName.toLowerCase(),
|
|
1212
|
+
type: label.control instanceof HTMLInputElement ? label.control.type.toLowerCase() : '',
|
|
1213
|
+
autocomplete: label.control.getAttribute('autocomplete') ?? '',
|
|
1214
|
+
name: label.control.getAttribute('name') ?? '',
|
|
1215
|
+
id: label.control.id,
|
|
1216
|
+
label: label.textContent ?? '',
|
|
1217
|
+
}
|
|
1218
|
+
: undefined
|
|
1219
|
+
const button = element.closest('button, input[type="submit"], input[type="image"]')
|
|
1220
|
+
const associated = (button ?? element) as HTMLInputElement
|
|
1221
|
+
const form = associated.form ?? element.closest('form')
|
|
1222
|
+
const fields = form
|
|
1223
|
+
? Array.from(form.elements)
|
|
1224
|
+
.filter(
|
|
1225
|
+
(field) => field instanceof HTMLInputElement || field instanceof HTMLTextAreaElement,
|
|
1226
|
+
)
|
|
1227
|
+
.map((field) => ({
|
|
1228
|
+
tag: field.tagName.toLowerCase(),
|
|
1229
|
+
type: field instanceof HTMLInputElement ? field.type.toLowerCase() : '',
|
|
1230
|
+
autocomplete: field.getAttribute('autocomplete') ?? '',
|
|
1231
|
+
name: field.getAttribute('name') ?? '',
|
|
1232
|
+
id: field.id,
|
|
1233
|
+
label: [
|
|
1234
|
+
field.getAttribute('aria-label') ?? '',
|
|
1235
|
+
field.getAttribute('placeholder') ?? '',
|
|
1236
|
+
...Array.from((field as HTMLInputElement).labels ?? []).map(
|
|
1237
|
+
(label) => label.textContent ?? '',
|
|
1238
|
+
),
|
|
1239
|
+
].join(' '),
|
|
1240
|
+
}))
|
|
1241
|
+
: []
|
|
1242
|
+
return { isFormButton: button !== null, labelledField, fields }
|
|
1243
|
+
})
|
|
1244
|
+
if (context.labelledField && isCredentialField(context.labelledField)) {
|
|
1245
|
+
throw new Error('This action targets a password or one-time-code control.')
|
|
1246
|
+
}
|
|
1247
|
+
if (!context.fields.some(isCredentialField)) return
|
|
1248
|
+
if (
|
|
1249
|
+
(action === 'click' && context.isFormButton) ||
|
|
1250
|
+
(action === 'press' && (context.isFormButton || /enter|return/i.test(key ?? ''))) ||
|
|
1251
|
+
action === 'type-submit'
|
|
1252
|
+
) {
|
|
1253
|
+
throw new Error(
|
|
1254
|
+
'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.',
|
|
1255
|
+
)
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1025
1259
|
private async runAct(action: BrowserActAction): Promise<BrowserResult> {
|
|
1026
1260
|
const tab = await this.activeTab()
|
|
1027
1261
|
await this.settle()
|
|
@@ -1041,13 +1275,19 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1041
1275
|
}
|
|
1042
1276
|
if (action.action === 'dialog') return this.answerDialog(tab, action)
|
|
1043
1277
|
if (this.pendingDialog?.tab === tab) {
|
|
1044
|
-
throw new Error('A dialog is open on this tab;
|
|
1278
|
+
throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.')
|
|
1045
1279
|
}
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1280
|
+
const checkpoint = await this.beginDocumentResponseChange(tab)
|
|
1281
|
+
try {
|
|
1282
|
+
await this.perform(tab, action)
|
|
1283
|
+
await this.settle()
|
|
1284
|
+
this.checkLanding(tab)
|
|
1285
|
+
await this.settle()
|
|
1286
|
+
} catch (error) {
|
|
1287
|
+
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
|
|
1288
|
+
throw error
|
|
1289
|
+
}
|
|
1290
|
+
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
1051
1291
|
await this.checkHuman(tab)
|
|
1052
1292
|
const after = this.tabs.get(tab.id) ?? tab
|
|
1053
1293
|
if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(after) })
|
|
@@ -1060,17 +1300,30 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1060
1300
|
): Promise<BrowserResult> {
|
|
1061
1301
|
const pending = this.pendingDialog
|
|
1062
1302
|
if (!pending || pending.tab !== tab) throw new Error('No dialog is open on this tab.')
|
|
1303
|
+
if (action.accept) {
|
|
1304
|
+
throw new Error(
|
|
1305
|
+
'The agent cannot accept a page-created dialog. Dismiss it, or ask the operator to use the browser window.',
|
|
1306
|
+
)
|
|
1307
|
+
}
|
|
1063
1308
|
this.pendingDialog = undefined
|
|
1309
|
+
const checkpoint = pending.checkpoint ?? {
|
|
1310
|
+
previous: this.documentResponse(tab),
|
|
1311
|
+
responsesBefore: tab.mainDocumentResponses,
|
|
1312
|
+
commitsBefore: tab.mainFrameCommits,
|
|
1313
|
+
}
|
|
1314
|
+
this.clearDocumentResponse(tab)
|
|
1064
1315
|
try {
|
|
1065
|
-
|
|
1066
|
-
else await pending.dialog.dismiss()
|
|
1316
|
+
await pending.dialog.dismiss()
|
|
1067
1317
|
} catch (error) {
|
|
1318
|
+
this.setDocumentResponse(tab, checkpoint.previous)
|
|
1068
1319
|
throw new BrowserOutcomeUnknownError('dialog', error)
|
|
1069
1320
|
}
|
|
1070
|
-
this.notes.unshift(
|
|
1321
|
+
this.notes.unshift('The dialog was dismissed.')
|
|
1071
1322
|
await this.settle()
|
|
1072
1323
|
this.checkLanding(tab)
|
|
1073
1324
|
await this.settle()
|
|
1325
|
+
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
1326
|
+
await this.checkHuman(tab)
|
|
1074
1327
|
if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(tab) })
|
|
1075
1328
|
return this.result({ page: await this.pageInfo(tab) })
|
|
1076
1329
|
}
|
|
@@ -1084,6 +1337,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1084
1337
|
switch (action.action) {
|
|
1085
1338
|
case 'click': {
|
|
1086
1339
|
const locator = await this.resolveRef(tab, action.ref)
|
|
1340
|
+
await this.guardField(locator)
|
|
1341
|
+
await this.guardCredentialForm(locator, 'click')
|
|
1087
1342
|
await locator.click({ trial: true })
|
|
1088
1343
|
await this.unknownOnFailure('click', () =>
|
|
1089
1344
|
action.doubleClick ? locator.dblclick() : locator.click(),
|
|
@@ -1092,13 +1347,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1092
1347
|
}
|
|
1093
1348
|
case 'hover': {
|
|
1094
1349
|
const locator = await this.resolveRef(tab, action.ref)
|
|
1350
|
+
await this.guardField(locator)
|
|
1095
1351
|
await locator.hover({ trial: true })
|
|
1096
1352
|
await this.unknownOnFailure('hover', () => locator.hover())
|
|
1097
1353
|
return
|
|
1098
1354
|
}
|
|
1099
1355
|
case 'type': {
|
|
1100
1356
|
const locator = await this.resolveRef(tab, action.ref)
|
|
1101
|
-
await this.guardField(
|
|
1357
|
+
await this.guardField(locator)
|
|
1358
|
+
if (action.submit) await this.guardCredentialForm(locator, 'type-submit')
|
|
1102
1359
|
await locator.click({ trial: true })
|
|
1103
1360
|
await this.unknownOnFailure('type', async () => {
|
|
1104
1361
|
await locator.fill(action.text)
|
|
@@ -1107,10 +1364,14 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1107
1364
|
return
|
|
1108
1365
|
}
|
|
1109
1366
|
case 'fill_form': {
|
|
1110
|
-
const plan: {
|
|
1367
|
+
const plan: {
|
|
1368
|
+
locator: Locator
|
|
1369
|
+
facts: ElementFieldFacts
|
|
1370
|
+
value: string
|
|
1371
|
+
}[] = []
|
|
1111
1372
|
for (const field of action.fields) {
|
|
1112
1373
|
const locator = await this.resolveRef(tab, field.ref)
|
|
1113
|
-
const facts = await this.guardField(
|
|
1374
|
+
const facts = await this.guardField(locator)
|
|
1114
1375
|
if (facts.kind === 'file') {
|
|
1115
1376
|
throw new Error(`${field.ref} is a file input; use browser_act upload for it.`)
|
|
1116
1377
|
}
|
|
@@ -1139,27 +1400,24 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1139
1400
|
case 'press': {
|
|
1140
1401
|
if (action.ref !== undefined) {
|
|
1141
1402
|
const locator = await this.resolveRef(tab, action.ref)
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
throw this.humanRequired('credential-field', tab)
|
|
1145
|
-
}
|
|
1403
|
+
await this.guardField(locator)
|
|
1404
|
+
await this.guardCredentialForm(locator, 'press', action.key)
|
|
1146
1405
|
await this.unknownOnFailure('press', () => locator.press(action.key))
|
|
1147
1406
|
return
|
|
1148
1407
|
}
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
}
|
|
1153
|
-
await this.unknownOnFailure('press', () => tab.page.keyboard.press(action.key))
|
|
1154
|
-
return
|
|
1408
|
+
throw new Error(
|
|
1409
|
+
'An untargeted key could reach a password or one-time-code control, including one inside a closed shadow root. Use a specific control instead.',
|
|
1410
|
+
)
|
|
1155
1411
|
}
|
|
1156
1412
|
case 'upload': {
|
|
1157
1413
|
const locator = await this.resolveRef(tab, action.ref)
|
|
1414
|
+
await this.guardField(locator)
|
|
1158
1415
|
const facts = await locator.evaluate(fieldFacts)
|
|
1159
1416
|
if (facts.kind === 'file') {
|
|
1160
1417
|
await this.unknownOnFailure('upload', () => locator.setInputFiles(action.path))
|
|
1161
1418
|
return
|
|
1162
1419
|
}
|
|
1420
|
+
await this.guardCredentialForm(locator, 'click')
|
|
1163
1421
|
await locator.click({ trial: true })
|
|
1164
1422
|
await this.unknownOnFailure('upload', async () => {
|
|
1165
1423
|
const [chooser] = await Promise.all([
|