@namzu/browser 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/classifier.d.ts +22 -1
- package/dist/classifier.d.ts.map +1 -1
- package/dist/classifier.js +30 -7
- package/dist/classifier.js.map +1 -1
- package/dist/errors.d.ts +19 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +26 -0
- package/dist/errors.js.map +1 -1
- package/dist/host.d.ts +42 -0
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +483 -69
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/page-scripts.d.ts +20 -0
- package/dist/page-scripts.d.ts.map +1 -1
- package/dist/page-scripts.js +110 -0
- package/dist/page-scripts.js.map +1 -1
- package/dist/reach.d.ts +66 -0
- package/dist/reach.d.ts.map +1 -0
- package/dist/reach.js +189 -0
- package/dist/reach.js.map +1 -0
- package/dist/settle.d.ts +40 -0
- package/dist/settle.d.ts.map +1 -0
- package/dist/settle.js +30 -0
- package/dist/settle.js.map +1 -0
- package/dist/snapshot.d.ts +80 -2
- package/dist/snapshot.d.ts.map +1 -1
- package/dist/snapshot.js +326 -20
- package/dist/snapshot.js.map +1 -1
- package/package.json +3 -2
- package/src/classifier.ts +44 -7
- package/src/errors.ts +32 -0
- package/src/host.ts +606 -83
- package/src/index.ts +4 -0
- package/src/page-scripts.ts +108 -0
- package/src/reach.ts +211 -0
- package/src/settle.ts +60 -0
- package/src/snapshot.ts +402 -20
package/src/host.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type {
|
|
|
3
3
|
BrowserActAction,
|
|
4
4
|
BrowserCallOptions,
|
|
5
5
|
BrowserCapabilities,
|
|
6
|
+
BrowserEffect,
|
|
6
7
|
BrowserHost,
|
|
7
8
|
BrowserObserveAction,
|
|
8
9
|
BrowserPageInfo,
|
|
@@ -10,14 +11,33 @@ import type {
|
|
|
10
11
|
BrowserResult,
|
|
11
12
|
BrowserSessionInfo,
|
|
12
13
|
BrowserSnapshot,
|
|
14
|
+
BrowserSnapshotView,
|
|
13
15
|
BrowserTabInfo,
|
|
16
|
+
BrowserTarget,
|
|
17
|
+
BrowserValue,
|
|
14
18
|
} from '@namzu/sdk'
|
|
15
|
-
import {
|
|
16
|
-
|
|
19
|
+
import {
|
|
20
|
+
BROWSER_FIND_MAX_MATCHES,
|
|
21
|
+
canonicalizeBrowserOrigin,
|
|
22
|
+
canonicalizeBrowserUrl,
|
|
23
|
+
resolveNamzuHome,
|
|
24
|
+
} from '@namzu/sdk'
|
|
25
|
+
import type {
|
|
26
|
+
BrowserContext,
|
|
27
|
+
CDPSession,
|
|
28
|
+
Dialog,
|
|
29
|
+
ElementHandle,
|
|
30
|
+
Locator,
|
|
31
|
+
Page,
|
|
32
|
+
Request,
|
|
33
|
+
Route,
|
|
34
|
+
} from 'playwright-core'
|
|
17
35
|
import {
|
|
18
36
|
type BrowserHumanClassifierOptions,
|
|
37
|
+
type BrowserPageSignals,
|
|
19
38
|
classifyHumanRequired,
|
|
20
39
|
isCredentialField,
|
|
40
|
+
signInNotice,
|
|
21
41
|
} from './classifier.js'
|
|
22
42
|
import {
|
|
23
43
|
type BrowserEnginePlan,
|
|
@@ -31,14 +51,24 @@ import {
|
|
|
31
51
|
} from './detect.js'
|
|
32
52
|
import {
|
|
33
53
|
BrowserHumanRequiredError,
|
|
54
|
+
BrowserNoEffectError,
|
|
34
55
|
BrowserOriginMismatchError,
|
|
35
56
|
BrowserOutcomeUnknownError,
|
|
36
57
|
BrowserSiteDeniedError,
|
|
37
58
|
BrowserStaleRefError,
|
|
59
|
+
BrowserTargetUnclearError,
|
|
38
60
|
BrowserUnavailableError,
|
|
39
61
|
ProfileBusyError,
|
|
40
62
|
} from './errors.js'
|
|
41
|
-
import {
|
|
63
|
+
import {
|
|
64
|
+
type ElementFieldFacts,
|
|
65
|
+
countCredentialFields,
|
|
66
|
+
fieldFacts,
|
|
67
|
+
hasVisibleContent,
|
|
68
|
+
installMutationClock,
|
|
69
|
+
msSinceLastChange,
|
|
70
|
+
valueFacts,
|
|
71
|
+
} from './page-scripts.js'
|
|
42
72
|
import { BrowserSitePolicy, type BrowserSiteRules, DEFAULT_BROWSER_SITE_RULES } from './policy.js'
|
|
43
73
|
import {
|
|
44
74
|
type BrowserLease,
|
|
@@ -47,12 +77,34 @@ import {
|
|
|
47
77
|
BrowserProfileStore,
|
|
48
78
|
DEFAULT_BROWSER_PROFILE,
|
|
49
79
|
} from './profiles.js'
|
|
80
|
+
import {
|
|
81
|
+
type Aim,
|
|
82
|
+
CANDIDATES_MAX,
|
|
83
|
+
KEY_DELAY_MS,
|
|
84
|
+
NAMED_KEYS,
|
|
85
|
+
type NamedKey,
|
|
86
|
+
aimInside,
|
|
87
|
+
deepFocus,
|
|
88
|
+
describeCandidate,
|
|
89
|
+
failedBeforeActing,
|
|
90
|
+
focusIsOpaque,
|
|
91
|
+
keyCanCommit,
|
|
92
|
+
namedKey,
|
|
93
|
+
noEffectReason,
|
|
94
|
+
} from './reach.js'
|
|
95
|
+
import { type SettleOutcome, type SettleProbe, waitUntilQuiet } from './settle.js'
|
|
50
96
|
import {
|
|
51
97
|
PLAYWRIGHT_CORE_VERSION,
|
|
52
98
|
SnapshotPager,
|
|
99
|
+
buildEffect,
|
|
53
100
|
captureAriaTree,
|
|
101
|
+
diffLines,
|
|
102
|
+
findInTree,
|
|
54
103
|
locateRef,
|
|
55
104
|
renderAriaTree,
|
|
105
|
+
renderInteractive,
|
|
106
|
+
renderReading,
|
|
107
|
+
subtreeOf,
|
|
56
108
|
} from './snapshot.js'
|
|
57
109
|
import { WindowsBridgeError } from './windows-bridge.js'
|
|
58
110
|
import { type WindowsBrowserConnection, connectWindowsBrowser } from './windows-engine.js'
|
|
@@ -88,6 +140,8 @@ export interface PlaywrightBrowserHostOptions {
|
|
|
88
140
|
readonly executablePath?: string
|
|
89
141
|
/** More sign-in addresses for the classifier. */
|
|
90
142
|
readonly signInAddresses?: BrowserHumanClassifierOptions['signInAddresses']
|
|
143
|
+
/** What makes a page a sign-in page: `address` (default) or `page`. See {@link BrowserHumanClassifierOptions.signInDetection}. */
|
|
144
|
+
readonly signInDetection?: BrowserHumanClassifierOptions['signInDetection']
|
|
91
145
|
/** The command that opens a visible window for signing in. Default `namzu browser login <profile> <url>`. */
|
|
92
146
|
readonly loginCommand?: (profile: string, url: string) => string
|
|
93
147
|
/** Windows engine: how long starting and connecting to the browser may take. Default 60 000. */
|
|
@@ -102,10 +156,18 @@ interface Tab {
|
|
|
102
156
|
mainDocumentResponses: number
|
|
103
157
|
cdp?: CDPSession
|
|
104
158
|
readonly responsesByLoader: Map<string, DocumentResponse>
|
|
159
|
+
/** Requests that can still change the page: documents, scripts, styles, fetch and XHR. */
|
|
160
|
+
readonly inflight: Set<Request>
|
|
161
|
+
/** When a request of that kind last started or ended. */
|
|
162
|
+
lastNetworkAt: number
|
|
163
|
+
/** The whole-page tree the model was last shown for this tab, for `delta` and effects. */
|
|
164
|
+
lastSeen?: string
|
|
105
165
|
/** HTTP status of the main document's last response. */
|
|
106
166
|
status?: number
|
|
107
167
|
wwwAuthenticate?: string
|
|
108
168
|
proxyAuthenticate?: string
|
|
169
|
+
/** The address the sign-in notice was last given for, so it is said once. */
|
|
170
|
+
signInNoticeFor?: string
|
|
109
171
|
}
|
|
110
172
|
|
|
111
173
|
type DocumentResponse = Pick<Tab, 'status' | 'wwwAuthenticate' | 'proxyAuthenticate'>
|
|
@@ -128,12 +190,42 @@ const DEFAULT_ACTION_TIMEOUT_MS = 10_000
|
|
|
128
190
|
/** How long a history move waits for the document after it committed. */
|
|
129
191
|
const HISTORY_LOAD_WAIT_MS = 5_000
|
|
130
192
|
const SNAPSHOT_PAGE_MAX = 20_000
|
|
131
|
-
/**
|
|
132
|
-
|
|
193
|
+
/**
|
|
194
|
+
* A page is quiet after this long with no change to the document and no
|
|
195
|
+
* request in flight. It also bounds how long an event the browser has not
|
|
196
|
+
* announced yet (a popup, a download, a dialog) has to arrive.
|
|
197
|
+
*/
|
|
198
|
+
const QUIET_MS = 250
|
|
199
|
+
/** The most an action waits for quiet. A page that never settles is reported, not waited on. */
|
|
200
|
+
const ACT_SETTLE_CAP_MS = 2_000
|
|
201
|
+
/** The same for a navigation, which loads more. */
|
|
202
|
+
const NAVIGATE_SETTLE_CAP_MS = 5_000
|
|
203
|
+
/** The wait before an action, for events left over from the last one. */
|
|
204
|
+
const PRE_ACT_QUIET_MS = 100
|
|
205
|
+
const PRE_ACT_CAP_MS = 500
|
|
206
|
+
/** `wait_for` with `stable` and no time: how long it waits. */
|
|
207
|
+
const STABLE_DEFAULT_WAIT_MS = 5_000
|
|
208
|
+
/** Request kinds that can change what the page shows. */
|
|
209
|
+
const PAGE_REQUEST_KINDS = new Set(['document', 'script', 'stylesheet', 'fetch', 'xhr'])
|
|
210
|
+
/** Lines of a `delta` snapshot that are listed in full. */
|
|
211
|
+
const DELTA_LINES_SHOWN = 80
|
|
212
|
+
/** Lines of an effect the model is shown. */
|
|
213
|
+
const EFFECT_LINES_SHOWN = 10
|
|
133
214
|
/** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
|
|
134
215
|
const CAPTCHA_FRAME_MIN_PX = 30
|
|
135
216
|
const REMEMBERED_DOCUMENTS_MAX = 64
|
|
217
|
+
|
|
218
|
+
/** Something the credential guards can ask about: a located element or a handle to one. */
|
|
219
|
+
type Probe = Locator | ElementHandle
|
|
220
|
+
|
|
221
|
+
function evaluateOn<R>(target: Probe, fn: (element: Element) => R): Promise<R> {
|
|
222
|
+
return 'asElement' in target
|
|
223
|
+
? (target as ElementHandle).evaluate(fn)
|
|
224
|
+
: (target as Locator).evaluate(fn)
|
|
225
|
+
}
|
|
136
226
|
const SCREENSHOT_INPUT_MASK_COLOR = '#666666'
|
|
227
|
+
/** Most inputs one screenshot checks for a credential. */
|
|
228
|
+
const MASK_CANDIDATES_MAX = 200
|
|
137
229
|
|
|
138
230
|
function firstLine(error: unknown): string {
|
|
139
231
|
const text = error instanceof Error ? error.message : String(error)
|
|
@@ -287,6 +379,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
287
379
|
context.setDefaultNavigationTimeout(
|
|
288
380
|
this.options.navigationTimeoutMs ?? DEFAULT_NAVIGATION_TIMEOUT_MS,
|
|
289
381
|
)
|
|
382
|
+
// Every new document keeps a clock of its last change, for the quiet-page wait.
|
|
383
|
+
await context.addInitScript(installMutationClock).catch(() => undefined)
|
|
290
384
|
context.on('page', (page) => this.adopt(page))
|
|
291
385
|
context.on('close', () => this.forget(context))
|
|
292
386
|
context.browser()?.on('disconnected', () => this.forget(context))
|
|
@@ -297,7 +391,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
297
391
|
() => true,
|
|
298
392
|
(route) => this.screen(route),
|
|
299
393
|
)
|
|
300
|
-
for (const page of context.pages())
|
|
394
|
+
for (const page of context.pages()) {
|
|
395
|
+
this.adopt(page, false)
|
|
396
|
+
void page.evaluate(installMutationClock).catch(() => undefined)
|
|
397
|
+
}
|
|
301
398
|
return context
|
|
302
399
|
}
|
|
303
400
|
|
|
@@ -492,6 +589,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
492
589
|
mainFrameCommits: 0,
|
|
493
590
|
mainDocumentResponses: 0,
|
|
494
591
|
responsesByLoader: new Map(),
|
|
592
|
+
inflight: new Set(),
|
|
593
|
+
lastNetworkAt: Date.now(),
|
|
495
594
|
}
|
|
496
595
|
this.tabs.set(tab.id, tab)
|
|
497
596
|
this.tabOf.set(page, tab)
|
|
@@ -520,6 +619,17 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
520
619
|
// A response without a frame (a service worker's) says nothing about the page.
|
|
521
620
|
}
|
|
522
621
|
})
|
|
622
|
+
const kind = (request: Request): boolean => PAGE_REQUEST_KINDS.has(request.resourceType())
|
|
623
|
+
page.on('request', (request) => {
|
|
624
|
+
if (!kind(request)) return
|
|
625
|
+
tab.inflight.add(request)
|
|
626
|
+
tab.lastNetworkAt = Date.now()
|
|
627
|
+
})
|
|
628
|
+
const ended = (request: Request): void => {
|
|
629
|
+
if (tab.inflight.delete(request)) tab.lastNetworkAt = Date.now()
|
|
630
|
+
}
|
|
631
|
+
page.on('requestfinished', ended)
|
|
632
|
+
page.on('requestfailed', ended)
|
|
523
633
|
page.on('dialog', (dialog) => this.onDialog(tab, dialog))
|
|
524
634
|
page.on('download', (download) => {
|
|
525
635
|
const name = download.suggestedFilename()
|
|
@@ -597,9 +707,42 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
597
707
|
this.blanking.add(blank)
|
|
598
708
|
}
|
|
599
709
|
|
|
600
|
-
|
|
601
|
-
|
|
710
|
+
/**
|
|
711
|
+
* Wait until the tab's page is quiet (nothing changed in it and no request
|
|
712
|
+
* in flight for `quietMs`), or `capMs` has passed. Events the browser
|
|
713
|
+
* announces late (a popup, a download, a dialog) arrive inside the window.
|
|
714
|
+
* Then let any tab that was cleared to `about:blank` finish clearing.
|
|
715
|
+
*/
|
|
716
|
+
private async settle(
|
|
717
|
+
tab: Tab,
|
|
718
|
+
options: { readonly quietMs: number; readonly capMs: number } = {
|
|
719
|
+
quietMs: QUIET_MS,
|
|
720
|
+
capMs: ACT_SETTLE_CAP_MS,
|
|
721
|
+
},
|
|
722
|
+
): Promise<SettleOutcome> {
|
|
723
|
+
const outcome = await waitUntilQuiet(
|
|
724
|
+
{ now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) },
|
|
725
|
+
this.quietProbe(tab),
|
|
726
|
+
options,
|
|
727
|
+
)
|
|
602
728
|
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
729
|
+
return outcome
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
private quietProbe(tab: Tab): SettleProbe {
|
|
733
|
+
return {
|
|
734
|
+
stillForMs: async () => {
|
|
735
|
+
// A dialog pauses the page; nothing more will happen until it is answered.
|
|
736
|
+
if (this.pendingDialog?.tab === tab || tab.page.isClosed()) return Number.MAX_SAFE_INTEGER
|
|
737
|
+
if (tab.inflight.size > 0) return 0
|
|
738
|
+
const sinceNetwork = Date.now() - tab.lastNetworkAt
|
|
739
|
+
const sincePage = await Promise.race([
|
|
740
|
+
tab.page.evaluate(msSinceLastChange).catch(() => 0),
|
|
741
|
+
new Promise<number>((resolve) => setTimeout(() => resolve(0), 1000)),
|
|
742
|
+
])
|
|
743
|
+
return Math.min(sinceNetwork, sincePage)
|
|
744
|
+
},
|
|
745
|
+
}
|
|
603
746
|
}
|
|
604
747
|
|
|
605
748
|
private async activeTab(): Promise<Tab> {
|
|
@@ -694,21 +837,28 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
694
837
|
// A frame that went away while we looked is not a challenge.
|
|
695
838
|
}
|
|
696
839
|
}
|
|
697
|
-
const
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
},
|
|
709
|
-
|
|
710
|
-
)
|
|
840
|
+
const signals: BrowserPageSignals = {
|
|
841
|
+
url,
|
|
842
|
+
title: await this.title(tab),
|
|
843
|
+
...(tab.status !== undefined ? { status: tab.status } : {}),
|
|
844
|
+
...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
|
|
845
|
+
...(tab.proxyAuthenticate !== undefined ? { proxyAuthenticate: tab.proxyAuthenticate } : {}),
|
|
846
|
+
...counts,
|
|
847
|
+
frameUrls,
|
|
848
|
+
}
|
|
849
|
+
const classifier: BrowserHumanClassifierOptions = {
|
|
850
|
+
...(this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {}),
|
|
851
|
+
...(this.options.signInDetection ? { signInDetection: this.options.signInDetection } : {}),
|
|
852
|
+
}
|
|
853
|
+
const reason = classifyHumanRequired(signals, classifier)
|
|
711
854
|
if (reason) throw this.humanRequired(reason, tab)
|
|
855
|
+
// A page with a password field that did not stop the turn: say so once
|
|
856
|
+
// per address, so the model reports it instead of trying to type.
|
|
857
|
+
const notice = signInNotice(signals, classifier, this.loginCommandFor(url))
|
|
858
|
+
if (notice && tab.signInNoticeFor !== withoutQuery(url)) {
|
|
859
|
+
tab.signInNoticeFor = withoutQuery(url)
|
|
860
|
+
this.notes.push(notice)
|
|
861
|
+
}
|
|
712
862
|
if (tab.status === 401) {
|
|
713
863
|
throw new Error(
|
|
714
864
|
'The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.',
|
|
@@ -862,7 +1012,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
862
1012
|
case 'reload':
|
|
863
1013
|
return this.history(action.action)
|
|
864
1014
|
case 'snapshot':
|
|
865
|
-
return this.snapshot(action
|
|
1015
|
+
return this.snapshot(action)
|
|
866
1016
|
case 'screenshot':
|
|
867
1017
|
return this.screenshot(action.ref, action.fullPage)
|
|
868
1018
|
case 'scroll':
|
|
@@ -871,6 +1021,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
871
1021
|
return this.waitFor(action)
|
|
872
1022
|
case 'tabs':
|
|
873
1023
|
return this.tabsCall(action)
|
|
1024
|
+
case 'value':
|
|
1025
|
+
return this.readValue(action.ref)
|
|
874
1026
|
}
|
|
875
1027
|
}
|
|
876
1028
|
|
|
@@ -890,7 +1042,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
890
1042
|
} catch (error) {
|
|
891
1043
|
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
|
|
892
1044
|
if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
|
|
893
|
-
await this.settle()
|
|
1045
|
+
await this.settle(tab)
|
|
894
1046
|
throw new Error(
|
|
895
1047
|
'The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.',
|
|
896
1048
|
)
|
|
@@ -904,10 +1056,38 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
904
1056
|
}
|
|
905
1057
|
|
|
906
1058
|
private async afterNavigation(tab: Tab): Promise<BrowserResult> {
|
|
907
|
-
|
|
1059
|
+
// The document first (a page that has not finished loading has not
|
|
1060
|
+
// finished building itself), then the same quiet the actions wait for.
|
|
1061
|
+
await tab.page
|
|
1062
|
+
.waitForLoadState('load', { timeout: NAVIGATE_SETTLE_CAP_MS })
|
|
1063
|
+
.catch(() => undefined)
|
|
1064
|
+
const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: NAVIGATE_SETTLE_CAP_MS })
|
|
908
1065
|
this.checkLanding(tab)
|
|
909
|
-
await this.
|
|
1066
|
+
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
910
1067
|
await this.checkHuman(tab)
|
|
1068
|
+
if (!settled.quiet) {
|
|
1069
|
+
this.notes.push(
|
|
1070
|
+
`The page was still changing after ${settled.waitedMs} ms, so what it shows may be unfinished. Call browser with wait_for and stable, or take a snapshot.`,
|
|
1071
|
+
)
|
|
1072
|
+
}
|
|
1073
|
+
const populated =
|
|
1074
|
+
originOf(tab.page.url()) === 'null' ||
|
|
1075
|
+
(await tab.page.evaluate(hasVisibleContent).catch(() => true))
|
|
1076
|
+
if (!populated) {
|
|
1077
|
+
this.notes.push(
|
|
1078
|
+
`The page has nothing to read yet (waited ${settled.waitedMs} ms). Call browser with wait_for and stable, or reload.`,
|
|
1079
|
+
)
|
|
1080
|
+
}
|
|
1081
|
+
// The model's next step after opening a page is to read it, so the
|
|
1082
|
+
// reading comes with the open: one request fewer on every page visit.
|
|
1083
|
+
if (populated && originOf(tab.page.url()) !== 'null') {
|
|
1084
|
+
try {
|
|
1085
|
+
const snapshot = await this.takeSnapshot(tab)
|
|
1086
|
+
return this.result({ page: snapshot.page, snapshot })
|
|
1087
|
+
} catch {
|
|
1088
|
+
// An unreadable page still opened; say where we are and let the model ask again.
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
911
1091
|
return this.result({ page: await this.pageInfo(tab) })
|
|
912
1092
|
}
|
|
913
1093
|
|
|
@@ -956,7 +1136,27 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
956
1136
|
this.notes.push('The open dialog was dismissed.')
|
|
957
1137
|
}
|
|
958
1138
|
|
|
959
|
-
|
|
1139
|
+
/** Read the page's tree and remember its refs. */
|
|
1140
|
+
private async readTree(tab: Tab) {
|
|
1141
|
+
const { nodes, hidden, redact } = await captureAriaTree(
|
|
1142
|
+
tab.page,
|
|
1143
|
+
this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS,
|
|
1144
|
+
)
|
|
1145
|
+
const whole = renderAriaTree(nodes, hidden, undefined, redact)
|
|
1146
|
+
if (!whole) throw new Error('The page could not be read.')
|
|
1147
|
+
this.refs = whole.refs
|
|
1148
|
+
return { nodes, hidden, redact, whole }
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
private async takeSnapshot(
|
|
1152
|
+
tab: Tab,
|
|
1153
|
+
request: {
|
|
1154
|
+
readonly ref?: string
|
|
1155
|
+
readonly view?: BrowserSnapshotView
|
|
1156
|
+
readonly query?: string
|
|
1157
|
+
readonly role?: string
|
|
1158
|
+
} = {},
|
|
1159
|
+
): Promise<BrowserSnapshot> {
|
|
960
1160
|
const pending = this.pendingDialog
|
|
961
1161
|
if (pending && pending.tab === tab) {
|
|
962
1162
|
const dialog = pending.dialog
|
|
@@ -968,18 +1168,72 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
968
1168
|
lines.push(' - (the page is paused until the dialog is answered)')
|
|
969
1169
|
return { page: await this.pageInfo(tab), text: lines.join('\n') }
|
|
970
1170
|
}
|
|
971
|
-
const {
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
const
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
1171
|
+
const { ref, view = 'full', query, role } = request
|
|
1172
|
+
const { nodes, hidden, redact, whole } = await this.readTree(tab)
|
|
1173
|
+
if (ref !== undefined && !whole.refs.has(ref)) throw new BrowserStaleRefError(ref)
|
|
1174
|
+
// A region is cut from the whole-page tree: a snapshot of one element would replace the ref table.
|
|
1175
|
+
const scope = ref === undefined ? nodes : (subtreeOf(nodes, ref) ?? [])
|
|
1176
|
+
let text: string
|
|
1177
|
+
if (query !== undefined || role !== undefined) {
|
|
1178
|
+
const found = findInTree(
|
|
1179
|
+
scope,
|
|
1180
|
+
hidden,
|
|
1181
|
+
redact,
|
|
1182
|
+
{ ...(query !== undefined ? { query } : {}), ...(role !== undefined ? { role } : {}) },
|
|
1183
|
+
BROWSER_FIND_MAX_MATCHES,
|
|
1184
|
+
)
|
|
1185
|
+
text = found.lines.join('\n')
|
|
1186
|
+
const what = [
|
|
1187
|
+
query !== undefined ? `text ${quoteShort(query, 60)}` : '',
|
|
1188
|
+
role !== undefined ? `role ${quoteShort(role, 30)}` : '',
|
|
1189
|
+
]
|
|
1190
|
+
.filter(Boolean)
|
|
1191
|
+
.join(' and ')
|
|
1192
|
+
this.notes.unshift(
|
|
1193
|
+
found.total === 0
|
|
1194
|
+
? `Nothing on the page matches ${what}.`
|
|
1195
|
+
: found.total > found.lines.length
|
|
1196
|
+
? `${found.total} elements match ${what}; the first ${found.lines.length} are listed. Narrow the search.`
|
|
1197
|
+
: `${found.total} element${found.total === 1 ? '' : 's'} match${found.total === 1 ? 'es' : ''} ${what}.`,
|
|
1198
|
+
)
|
|
1199
|
+
} else if (view === 'interactive') {
|
|
1200
|
+
text = renderInteractive(scope, hidden, redact)
|
|
1201
|
+
tab.lastSeen = whole.text
|
|
1202
|
+
} else if (view === 'reading') {
|
|
1203
|
+
text = renderReading(scope, hidden, redact)
|
|
1204
|
+
tab.lastSeen = whole.text
|
|
1205
|
+
} else if (view === 'delta') {
|
|
1206
|
+
const before = tab.lastSeen
|
|
1207
|
+
if (before === undefined) {
|
|
1208
|
+
this.notes.unshift(
|
|
1209
|
+
'There is no earlier snapshot of this tab to compare with; this is the whole page.',
|
|
1210
|
+
)
|
|
1211
|
+
text =
|
|
1212
|
+
ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '')
|
|
1213
|
+
} else {
|
|
1214
|
+
const diff = diffLines(before, whole.text, DELTA_LINES_SHOWN)
|
|
1215
|
+
text =
|
|
1216
|
+
diff.added === 0 && diff.removed === 0
|
|
1217
|
+
? '(no change since the last snapshot)'
|
|
1218
|
+
: [
|
|
1219
|
+
...diff.addedLines.map((l) => `+ ${l}`),
|
|
1220
|
+
...(diff.added > diff.addedLines.length
|
|
1221
|
+
? [` (${diff.added - diff.addedLines.length} more lines added)`]
|
|
1222
|
+
: []),
|
|
1223
|
+
...diff.removedLines.map((l) => `- ${l}`),
|
|
1224
|
+
...(diff.removed > diff.removedLines.length
|
|
1225
|
+
? [` (${diff.removed - diff.removedLines.length} more lines removed)`]
|
|
1226
|
+
: []),
|
|
1227
|
+
].join('\n')
|
|
1228
|
+
this.notes.unshift(
|
|
1229
|
+
`Changes since the last snapshot: ${diff.added} lines added, ${diff.removed} removed. Take a full snapshot to see the whole page again.`,
|
|
1230
|
+
)
|
|
1231
|
+
}
|
|
1232
|
+
tab.lastSeen = whole.text
|
|
1233
|
+
} else {
|
|
1234
|
+
text =
|
|
1235
|
+
ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '')
|
|
1236
|
+
if (ref === undefined) tab.lastSeen = whole.text
|
|
983
1237
|
}
|
|
984
1238
|
if (text.trim() === '') text = '(nothing readable on this page)'
|
|
985
1239
|
const first = this.pager.start(text)
|
|
@@ -990,10 +1244,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
990
1244
|
}
|
|
991
1245
|
}
|
|
992
1246
|
|
|
993
|
-
private async snapshot(
|
|
1247
|
+
private async snapshot(
|
|
1248
|
+
request: Extract<BrowserObserveAction, { action: 'snapshot' }>,
|
|
1249
|
+
): Promise<BrowserResult> {
|
|
994
1250
|
const tab = await this.activeTab()
|
|
995
|
-
if (cursor !== undefined) {
|
|
996
|
-
const next = this.pager.resume(cursor)
|
|
1251
|
+
if (request.cursor !== undefined) {
|
|
1252
|
+
const next = this.pager.resume(request.cursor)
|
|
997
1253
|
if (!next) {
|
|
998
1254
|
throw new Error(
|
|
999
1255
|
'That cursor belongs to an older snapshot. Take a new snapshot and page through it.',
|
|
@@ -1007,8 +1263,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1007
1263
|
},
|
|
1008
1264
|
})
|
|
1009
1265
|
}
|
|
1010
|
-
await this.settle()
|
|
1011
|
-
return this.result({
|
|
1266
|
+
await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS })
|
|
1267
|
+
return this.result({
|
|
1268
|
+
snapshot: await this.takeSnapshot(tab, {
|
|
1269
|
+
...(request.ref !== undefined ? { ref: request.ref } : {}),
|
|
1270
|
+
...(request.view !== undefined ? { view: request.view } : {}),
|
|
1271
|
+
...(request.query !== undefined ? { query: request.query } : {}),
|
|
1272
|
+
...(request.role !== undefined ? { role: request.role } : {}),
|
|
1273
|
+
}),
|
|
1274
|
+
})
|
|
1012
1275
|
}
|
|
1013
1276
|
|
|
1014
1277
|
private async resolveRef(tab: Tab, ref: string): Promise<Locator> {
|
|
@@ -1031,10 +1294,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1031
1294
|
'A dialog is open; dismiss it with browser_act dialog before taking a screenshot.',
|
|
1032
1295
|
)
|
|
1033
1296
|
}
|
|
1034
|
-
// Screenshots are image blocks sent to the model. Mask
|
|
1035
|
-
//
|
|
1036
|
-
//
|
|
1037
|
-
|
|
1297
|
+
// Screenshots are image blocks sent to the model. Mask the fields that
|
|
1298
|
+
// take a password or a one-time code, in every frame; the other inputs
|
|
1299
|
+
// stay visible, because a search box or an address field the model
|
|
1300
|
+
// cannot read cannot be checked. Closed shadow roots and page-drawn
|
|
1301
|
+
// text remain outside this mask.
|
|
1302
|
+
const mask = await this.credentialMask(tab)
|
|
1038
1303
|
const data =
|
|
1039
1304
|
ref !== undefined
|
|
1040
1305
|
? await (await this.resolveRef(tab, ref)).screenshot({
|
|
@@ -1061,6 +1326,64 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1061
1326
|
})
|
|
1062
1327
|
}
|
|
1063
1328
|
|
|
1329
|
+
/** Locators of the credential fields (password, one-time code) of every frame of the tab. */
|
|
1330
|
+
private async credentialMask(tab: Tab): Promise<Locator[]> {
|
|
1331
|
+
const out: Locator[] = []
|
|
1332
|
+
for (const frame of tab.page.frames()) {
|
|
1333
|
+
const inputs = frame.locator('input, textarea')
|
|
1334
|
+
let count = 0
|
|
1335
|
+
try {
|
|
1336
|
+
count = Math.min(await inputs.count(), MASK_CANDIDATES_MAX)
|
|
1337
|
+
} catch {
|
|
1338
|
+
// A frame that went away while we looked has nothing to mask.
|
|
1339
|
+
continue
|
|
1340
|
+
}
|
|
1341
|
+
const verdicts = await Promise.all(
|
|
1342
|
+
Array.from({ length: count }, async (_, index) => {
|
|
1343
|
+
try {
|
|
1344
|
+
return isCredentialField(
|
|
1345
|
+
await inputs.nth(index).evaluate(fieldFacts, undefined, { timeout: 2000 }),
|
|
1346
|
+
)
|
|
1347
|
+
} catch {
|
|
1348
|
+
// A field that cannot be asked about is hidden to be safe.
|
|
1349
|
+
return true
|
|
1350
|
+
}
|
|
1351
|
+
}),
|
|
1352
|
+
)
|
|
1353
|
+
verdicts.forEach((hide, index) => {
|
|
1354
|
+
if (hide) out.push(inputs.nth(index))
|
|
1355
|
+
})
|
|
1356
|
+
}
|
|
1357
|
+
return out
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/** One element's current value and state, read from the page itself. */
|
|
1361
|
+
private async readValue(ref: string): Promise<BrowserResult> {
|
|
1362
|
+
const tab = await this.activeTab()
|
|
1363
|
+
if (this.pendingDialog?.tab === tab) {
|
|
1364
|
+
throw new Error(
|
|
1365
|
+
'A dialog is open; dismiss it with browser_act dialog before reading a value.',
|
|
1366
|
+
)
|
|
1367
|
+
}
|
|
1368
|
+
const locator = await this.resolveRef(tab, ref)
|
|
1369
|
+
const facts = await locator.evaluate(fieldFacts)
|
|
1370
|
+
const described = this.refs.get(ref)
|
|
1371
|
+
const page = await this.pageInfo(tab)
|
|
1372
|
+
const hidden = isCredentialField(facts)
|
|
1373
|
+
const read = hidden ? undefined : await locator.evaluate(valueFacts)
|
|
1374
|
+
const value: BrowserValue = {
|
|
1375
|
+
page,
|
|
1376
|
+
ref,
|
|
1377
|
+
...(described?.role !== undefined ? { role: described.role } : {}),
|
|
1378
|
+
...(described?.name !== undefined ? { name: described.name } : {}),
|
|
1379
|
+
...(read?.value !== undefined ? { value: read.value } : {}),
|
|
1380
|
+
state: read?.state ?? [],
|
|
1381
|
+
...(read?.options !== undefined ? { options: read.options } : {}),
|
|
1382
|
+
...(hidden ? { hidden: true } : {}),
|
|
1383
|
+
}
|
|
1384
|
+
return this.result({ value })
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1064
1387
|
private async scroll(
|
|
1065
1388
|
direction: 'up' | 'down' | 'left' | 'right',
|
|
1066
1389
|
ref?: string,
|
|
@@ -1121,10 +1444,19 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1121
1444
|
: `${quoteShort(text)} was still there after ${timeout} ms.`,
|
|
1122
1445
|
)
|
|
1123
1446
|
}
|
|
1124
|
-
} else {
|
|
1447
|
+
} else if (action.stable !== true) {
|
|
1125
1448
|
await tab.page.waitForTimeout(timeout)
|
|
1126
1449
|
this.notes.unshift(`Waited ${timeout} ms.`)
|
|
1127
1450
|
}
|
|
1451
|
+
if (action.stable === true) {
|
|
1452
|
+
const cap = action.timeMs !== undefined ? timeout : STABLE_DEFAULT_WAIT_MS
|
|
1453
|
+
const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: cap })
|
|
1454
|
+
this.notes.push(
|
|
1455
|
+
settled.quiet
|
|
1456
|
+
? `The page is quiet (nothing changed for ${QUIET_MS} ms; waited ${settled.waitedMs} ms).`
|
|
1457
|
+
: `The page was still changing after ${settled.waitedMs} ms.`,
|
|
1458
|
+
)
|
|
1459
|
+
}
|
|
1128
1460
|
return this.result({ page: await this.pageInfo(tab) })
|
|
1129
1461
|
}
|
|
1130
1462
|
|
|
@@ -1184,11 +1516,11 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1184
1516
|
// Acting
|
|
1185
1517
|
// -------------------------------------------------------------------------
|
|
1186
1518
|
|
|
1187
|
-
private async guardField(locator:
|
|
1188
|
-
const facts = await locator
|
|
1519
|
+
private async guardField(locator: Probe): Promise<ElementFieldFacts> {
|
|
1520
|
+
const facts = await evaluateOn(locator, fieldFacts)
|
|
1189
1521
|
if (isCredentialField(facts)) {
|
|
1190
1522
|
throw new Error(
|
|
1191
|
-
'This action targets a password or one-time-code control. Ask the operator to use the browser window.',
|
|
1523
|
+
'This action targets a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.',
|
|
1192
1524
|
)
|
|
1193
1525
|
}
|
|
1194
1526
|
return facts
|
|
@@ -1196,11 +1528,11 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1196
1528
|
|
|
1197
1529
|
/** Refuse direct form submission while a form holds a password or code input. */
|
|
1198
1530
|
private async guardCredentialForm(
|
|
1199
|
-
locator:
|
|
1531
|
+
locator: Probe,
|
|
1200
1532
|
action: 'click' | 'press' | 'type-submit',
|
|
1201
1533
|
key?: string,
|
|
1202
1534
|
): Promise<void> {
|
|
1203
|
-
const context = await locator
|
|
1535
|
+
const context = await evaluateOn(locator, (element) => {
|
|
1204
1536
|
const label = element.closest('label') as HTMLLabelElement | null
|
|
1205
1537
|
const labelledField = label?.control
|
|
1206
1538
|
? {
|
|
@@ -1247,20 +1579,77 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1247
1579
|
action === 'type-submit'
|
|
1248
1580
|
) {
|
|
1249
1581
|
throw new Error(
|
|
1250
|
-
'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.',
|
|
1582
|
+
'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.',
|
|
1251
1583
|
)
|
|
1252
1584
|
}
|
|
1253
1585
|
}
|
|
1254
1586
|
|
|
1587
|
+
/**
|
|
1588
|
+
* The page's tree as the model could have read it just before an action,
|
|
1589
|
+
* or `undefined` when it cannot be read now (a dialog holds the page, or
|
|
1590
|
+
* the read failed). An effect is a bonus; it never fails the action.
|
|
1591
|
+
*/
|
|
1592
|
+
private async effectBaseline(tab: Tab): Promise<string | undefined> {
|
|
1593
|
+
if (this.pendingDialog?.tab === tab) return undefined
|
|
1594
|
+
try {
|
|
1595
|
+
return (await this.readTree(tab)).whole.text
|
|
1596
|
+
} catch {
|
|
1597
|
+
return undefined
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
/** What the action did, from the tree before and the tree now. */
|
|
1602
|
+
private async effectSince(
|
|
1603
|
+
before: string | undefined,
|
|
1604
|
+
urlBefore: string,
|
|
1605
|
+
tabsBefore: number,
|
|
1606
|
+
tab: Tab,
|
|
1607
|
+
settled: SettleOutcome,
|
|
1608
|
+
): Promise<BrowserEffect | undefined> {
|
|
1609
|
+
if (before === undefined || this.pendingDialog?.tab === tab || tab.page.isClosed())
|
|
1610
|
+
return undefined
|
|
1611
|
+
let after: string
|
|
1612
|
+
try {
|
|
1613
|
+
after = (await this.readTree(tab)).whole.text
|
|
1614
|
+
} catch {
|
|
1615
|
+
return undefined
|
|
1616
|
+
}
|
|
1617
|
+
return buildEffect(
|
|
1618
|
+
{
|
|
1619
|
+
beforeTree: before,
|
|
1620
|
+
afterTree: after,
|
|
1621
|
+
urlBefore,
|
|
1622
|
+
urlNow: tab.page.url(),
|
|
1623
|
+
tabsBefore,
|
|
1624
|
+
tabsNow: this.tabs.size,
|
|
1625
|
+
settle: { quiet: settled.quiet, waitedMs: settled.waitedMs, capMs: settled.capMs },
|
|
1626
|
+
},
|
|
1627
|
+
EFFECT_LINES_SHOWN,
|
|
1628
|
+
)
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1255
1631
|
private async runAct(action: BrowserActAction): Promise<BrowserResult> {
|
|
1256
1632
|
const tab = await this.activeTab()
|
|
1257
|
-
await this.settle()
|
|
1633
|
+
await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS })
|
|
1258
1634
|
const expected = canonicalizeBrowserOrigin(action.origin)
|
|
1259
1635
|
const live = originOf(tab.page.url())
|
|
1260
1636
|
if (!expected.ok || expected.origin !== live) {
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1637
|
+
// The page moved since the snapshot. Going on is fine only when the
|
|
1638
|
+
// site rules treat the new origin exactly as the named one: both at
|
|
1639
|
+
// `act`, where no review stood between the model and the page.
|
|
1640
|
+
const sameRules =
|
|
1641
|
+
expected.ok &&
|
|
1642
|
+
live !== 'null' &&
|
|
1643
|
+
this.policy.levelOf(expected.origin) === 'act' &&
|
|
1644
|
+
this.policy.levelOf(live) === 'act'
|
|
1645
|
+
if (!sameRules) {
|
|
1646
|
+
throw new BrowserOriginMismatchError(
|
|
1647
|
+
expected.ok ? expected.origin : action.origin,
|
|
1648
|
+
live === 'null' ? 'about:blank' : live,
|
|
1649
|
+
)
|
|
1650
|
+
}
|
|
1651
|
+
this.notes.unshift(
|
|
1652
|
+
`The page is now at ${live}, not ${expected.origin}; the site rules treat both alike, so the action went ahead there.`,
|
|
1264
1653
|
)
|
|
1265
1654
|
}
|
|
1266
1655
|
if (!this.policy.canAct(live)) {
|
|
@@ -1274,11 +1663,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1274
1663
|
throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.')
|
|
1275
1664
|
}
|
|
1276
1665
|
const checkpoint = await this.beginDocumentResponseChange(tab)
|
|
1666
|
+
const urlBefore = tab.page.url()
|
|
1667
|
+
const tabsBefore = this.tabs.size
|
|
1668
|
+
const treeBefore = await this.effectBaseline(tab)
|
|
1669
|
+
let settled: SettleOutcome
|
|
1277
1670
|
try {
|
|
1278
1671
|
await this.perform(tab, action)
|
|
1279
|
-
await this.settle()
|
|
1672
|
+
settled = await this.settle(tab)
|
|
1280
1673
|
this.checkLanding(tab)
|
|
1281
|
-
await this.
|
|
1674
|
+
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
1282
1675
|
} catch (error) {
|
|
1283
1676
|
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
|
|
1284
1677
|
throw error
|
|
@@ -1286,8 +1679,14 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1286
1679
|
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
1287
1680
|
await this.checkHuman(tab)
|
|
1288
1681
|
const after = this.tabs.get(tab.id) ?? tab
|
|
1289
|
-
|
|
1290
|
-
|
|
1682
|
+
const effect = await this.effectSince(treeBefore, urlBefore, tabsBefore, after, settled)
|
|
1683
|
+
if (action.snapshot) {
|
|
1684
|
+
return this.result({
|
|
1685
|
+
snapshot: await this.takeSnapshot(after),
|
|
1686
|
+
...(effect ? { effect } : {}),
|
|
1687
|
+
})
|
|
1688
|
+
}
|
|
1689
|
+
return this.result({ page: await this.pageInfo(after), ...(effect ? { effect } : {}) })
|
|
1291
1690
|
}
|
|
1292
1691
|
|
|
1293
1692
|
private async answerDialog(
|
|
@@ -1315,9 +1714,9 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1315
1714
|
throw new BrowserOutcomeUnknownError('dialog', error)
|
|
1316
1715
|
}
|
|
1317
1716
|
this.notes.unshift('The dialog was dismissed.')
|
|
1318
|
-
await this.settle()
|
|
1717
|
+
await this.settle(tab)
|
|
1319
1718
|
this.checkLanding(tab)
|
|
1320
|
-
await this.
|
|
1719
|
+
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
1321
1720
|
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
1322
1721
|
await this.checkHuman(tab)
|
|
1323
1722
|
if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(tab) })
|
|
@@ -1332,29 +1731,45 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1332
1731
|
private async perform(tab: Tab, action: BrowserActAction): Promise<void> {
|
|
1333
1732
|
switch (action.action) {
|
|
1334
1733
|
case 'click': {
|
|
1335
|
-
const
|
|
1336
|
-
await this.guardField(locator)
|
|
1337
|
-
await this.guardCredentialForm(locator, 'click')
|
|
1338
|
-
|
|
1734
|
+
const target = await this.reach(tab, action)
|
|
1735
|
+
await this.guardField(target.locator)
|
|
1736
|
+
await this.guardCredentialForm(target.locator, 'click')
|
|
1737
|
+
const at = target.position ? { position: target.position } : {}
|
|
1738
|
+
await this.noEffectOnFailure('click', () => target.locator.click({ trial: true, ...at }))
|
|
1339
1739
|
await this.unknownOnFailure('click', () =>
|
|
1340
|
-
action.doubleClick ? locator.dblclick() : locator.click(),
|
|
1740
|
+
action.doubleClick ? target.locator.dblclick(at) : target.locator.click(at),
|
|
1341
1741
|
)
|
|
1342
1742
|
return
|
|
1343
1743
|
}
|
|
1344
1744
|
case 'hover': {
|
|
1345
|
-
const
|
|
1346
|
-
await this.guardField(locator)
|
|
1347
|
-
|
|
1348
|
-
await this.
|
|
1745
|
+
const target = await this.reach(tab, action)
|
|
1746
|
+
await this.guardField(target.locator)
|
|
1747
|
+
const at = target.position ? { position: target.position } : {}
|
|
1748
|
+
await this.noEffectOnFailure('hover', () => target.locator.hover({ trial: true, ...at }))
|
|
1749
|
+
await this.unknownOnFailure('hover', () => target.locator.hover(at))
|
|
1349
1750
|
return
|
|
1350
1751
|
}
|
|
1351
1752
|
case 'type': {
|
|
1352
1753
|
const locator = await this.resolveRef(tab, action.ref)
|
|
1353
1754
|
await this.guardField(locator)
|
|
1354
1755
|
if (action.submit) await this.guardCredentialForm(locator, 'type-submit')
|
|
1355
|
-
await locator.click({ trial: true })
|
|
1356
|
-
await this.unknownOnFailure('type', async () => {
|
|
1357
|
-
|
|
1756
|
+
await this.noEffectOnFailure('type', () => locator.click({ trial: true }))
|
|
1757
|
+
await this.unknownOnFailure('type', async (begun) => {
|
|
1758
|
+
if (action.keys) {
|
|
1759
|
+
await locator.focus()
|
|
1760
|
+
// Clearing and typing change the field; from here a failure is not harmless.
|
|
1761
|
+
begun()
|
|
1762
|
+
if (action.append) await locator.press('Control+End')
|
|
1763
|
+
else await locator.fill('')
|
|
1764
|
+
await locator.pressSequentially(action.text, { delay: KEY_DELAY_MS })
|
|
1765
|
+
} else if (action.append) {
|
|
1766
|
+
const held = await locator
|
|
1767
|
+
.inputValue()
|
|
1768
|
+
.catch(() => locator.evaluate((el) => el.textContent ?? ''))
|
|
1769
|
+
await locator.fill(held + action.text)
|
|
1770
|
+
} else {
|
|
1771
|
+
await locator.fill(action.text)
|
|
1772
|
+
}
|
|
1358
1773
|
if (action.submit) await locator.press('Enter')
|
|
1359
1774
|
})
|
|
1360
1775
|
return
|
|
@@ -1401,9 +1816,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1401
1816
|
await this.unknownOnFailure('press', () => locator.press(action.key))
|
|
1402
1817
|
return
|
|
1403
1818
|
}
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1819
|
+
const key = namedKey(action.key)
|
|
1820
|
+
if (!key) {
|
|
1821
|
+
throw new Error(
|
|
1822
|
+
`Without a ref, press takes only ${NAMED_KEYS.join(', ')}. Any other key goes to a specific control: give its ref.`,
|
|
1823
|
+
)
|
|
1824
|
+
}
|
|
1825
|
+
await this.guardFocus(tab, key)
|
|
1826
|
+
await this.unknownOnFailure('press', () => tab.page.keyboard.press(key))
|
|
1827
|
+
return
|
|
1407
1828
|
}
|
|
1408
1829
|
case 'upload': {
|
|
1409
1830
|
const locator = await this.resolveRef(tab, action.ref)
|
|
@@ -1429,6 +1850,102 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1429
1850
|
}
|
|
1430
1851
|
}
|
|
1431
1852
|
|
|
1853
|
+
/**
|
|
1854
|
+
* The element a click or hover means, and where on it to aim. A ref names
|
|
1855
|
+
* it directly; a role with a name, or visible text, is looked up and
|
|
1856
|
+
* refused unless exactly one element matches. A container (a section, a
|
|
1857
|
+
* list item holding a menu title) is not clicked in its middle, which is
|
|
1858
|
+
* often empty: the aim goes to its first control or to its own text.
|
|
1859
|
+
*/
|
|
1860
|
+
private async reach(
|
|
1861
|
+
tab: Tab,
|
|
1862
|
+
target: BrowserTarget,
|
|
1863
|
+
): Promise<{ locator: Locator; position?: { x: number; y: number } }> {
|
|
1864
|
+
const found =
|
|
1865
|
+
target.ref !== undefined
|
|
1866
|
+
? await this.resolveRef(tab, target.ref)
|
|
1867
|
+
: await this.findTarget(tab, target)
|
|
1868
|
+
let aim: Aim = { kind: 'centre' }
|
|
1869
|
+
try {
|
|
1870
|
+
aim = await found.evaluate(aimInside)
|
|
1871
|
+
} catch {
|
|
1872
|
+
// Aim at the middle, as before.
|
|
1873
|
+
}
|
|
1874
|
+
if (aim.kind === 'descendant') return { locator: found.locator(`xpath=${aim.path}`) }
|
|
1875
|
+
if (aim.kind === 'text') return { locator: found, position: { x: aim.x, y: aim.y } }
|
|
1876
|
+
return { locator: found }
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
/** Find the one element a role/name or text target means, or refuse with the candidates. */
|
|
1880
|
+
private async findTarget(tab: Tab, target: BrowserTarget): Promise<Locator> {
|
|
1881
|
+
const page = tab.page
|
|
1882
|
+
const role = target.role
|
|
1883
|
+
const wanted =
|
|
1884
|
+
role !== undefined ? `${role} "${target.name ?? ''}"` : `text "${target.text ?? ''}"`
|
|
1885
|
+
const build = (exact: boolean): Locator =>
|
|
1886
|
+
role !== undefined
|
|
1887
|
+
? page.getByRole(role as Parameters<Page['getByRole']>[0], {
|
|
1888
|
+
name: target.name ?? '',
|
|
1889
|
+
exact,
|
|
1890
|
+
})
|
|
1891
|
+
: page.getByText(target.text ?? '', { exact }).filter({ visible: true })
|
|
1892
|
+
let locator = build(false)
|
|
1893
|
+
let count = await locator.count()
|
|
1894
|
+
if (count > 1) {
|
|
1895
|
+
// A short name also fits longer ones ("Save" in "Save draft"): the exact matches decide.
|
|
1896
|
+
const exact = build(true)
|
|
1897
|
+
const exactCount = await exact.count()
|
|
1898
|
+
if (exactCount > 0) {
|
|
1899
|
+
locator = exact
|
|
1900
|
+
count = exactCount
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
if (count === 1) return locator
|
|
1904
|
+
const pool =
|
|
1905
|
+
count > 0 ? locator : role !== undefined ? page.getByRole(role as never) : undefined
|
|
1906
|
+
const candidates = []
|
|
1907
|
+
if (pool) {
|
|
1908
|
+
const total = Math.min(await pool.count(), CANDIDATES_MAX)
|
|
1909
|
+
for (let i = 0; i < total; i++) {
|
|
1910
|
+
candidates.push(await pool.nth(i).evaluate(describeCandidate))
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
throw new BrowserTargetUnclearError(wanted, count, candidates)
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
/**
|
|
1917
|
+
* Refuse a key sent to the page when the focus is on a credential, or
|
|
1918
|
+
* when it cannot be told. Every frame is asked, because the key goes to
|
|
1919
|
+
* whichever one holds the focus and a stale focus in another frame is
|
|
1920
|
+
* still a reason to be careful.
|
|
1921
|
+
*/
|
|
1922
|
+
private async guardFocus(tab: Tab, key: NamedKey): Promise<void> {
|
|
1923
|
+
for (const frame of tab.page.frames()) {
|
|
1924
|
+
const handle = await frame.evaluateHandle(deepFocus).catch(() => undefined)
|
|
1925
|
+
const element = handle?.asElement() ?? null
|
|
1926
|
+
if (!element) continue
|
|
1927
|
+
if (keyCanCommit(key) && (await element.evaluate(focusIsOpaque))) {
|
|
1928
|
+
throw new Error(
|
|
1929
|
+
`The focus is on a component whose inside cannot be read, so ${key} could reach a password or code field. Use browser_act with the ref of a specific control.`,
|
|
1930
|
+
)
|
|
1931
|
+
}
|
|
1932
|
+
await this.guardField(element)
|
|
1933
|
+
await this.guardCredentialForm(element, 'press', key)
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
/** A check Playwright runs without acting (`trial`) failed: nothing on the page changed. */
|
|
1938
|
+
private async noEffectOnFailure(
|
|
1939
|
+
action: BrowserActAction['action'],
|
|
1940
|
+
run: () => Promise<unknown>,
|
|
1941
|
+
): Promise<void> {
|
|
1942
|
+
try {
|
|
1943
|
+
await run()
|
|
1944
|
+
} catch (error) {
|
|
1945
|
+
throw new BrowserNoEffectError(action, noEffectReason(error))
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1432
1949
|
/**
|
|
1433
1950
|
* Run the change. A failure once it has started is `outcome unknown`. A
|
|
1434
1951
|
* dialog the action opens ends the wait: the page is paused until the
|
|
@@ -1436,9 +1953,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1436
1953
|
*/
|
|
1437
1954
|
private async unknownOnFailure(
|
|
1438
1955
|
action: BrowserActAction['action'],
|
|
1439
|
-
run: () => Promise<unknown>,
|
|
1956
|
+
run: (begun: () => void) => Promise<unknown>,
|
|
1440
1957
|
): Promise<void> {
|
|
1441
|
-
|
|
1958
|
+
let touched = false
|
|
1959
|
+
const running = run(() => {
|
|
1960
|
+
touched = true
|
|
1961
|
+
})
|
|
1442
1962
|
let timer: ReturnType<typeof setInterval> | undefined
|
|
1443
1963
|
const dialogOpened = new Promise<'dialog'>((resolve) => {
|
|
1444
1964
|
timer = setInterval(() => {
|
|
@@ -1449,7 +1969,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1449
1969
|
const outcome = await Promise.race([running, dialogOpened])
|
|
1450
1970
|
if (outcome === 'dialog') running.catch(() => undefined)
|
|
1451
1971
|
} catch (error) {
|
|
1452
|
-
|
|
1972
|
+
// A timeout whose log never reached the input step left the page alone.
|
|
1973
|
+
throw !touched && failedBeforeActing(error)
|
|
1974
|
+
? new BrowserNoEffectError(action, noEffectReason(error))
|
|
1975
|
+
: new BrowserOutcomeUnknownError(action, error)
|
|
1453
1976
|
} finally {
|
|
1454
1977
|
clearInterval(timer)
|
|
1455
1978
|
}
|