@namzu/browser 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/classifier.d.ts +22 -1
- package/dist/classifier.d.ts.map +1 -1
- package/dist/classifier.js +30 -7
- package/dist/classifier.js.map +1 -1
- package/dist/errors.d.ts +19 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +26 -0
- package/dist/errors.js.map +1 -1
- package/dist/host.d.ts +42 -0
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +487 -70
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/page-scripts.d.ts +20 -0
- package/dist/page-scripts.d.ts.map +1 -1
- package/dist/page-scripts.js +110 -0
- package/dist/page-scripts.js.map +1 -1
- package/dist/reach.d.ts +66 -0
- package/dist/reach.d.ts.map +1 -0
- package/dist/reach.js +189 -0
- package/dist/reach.js.map +1 -0
- package/dist/settle.d.ts +40 -0
- package/dist/settle.d.ts.map +1 -0
- package/dist/settle.js +30 -0
- package/dist/settle.js.map +1 -0
- package/dist/snapshot.d.ts +80 -2
- package/dist/snapshot.d.ts.map +1 -1
- package/dist/snapshot.js +326 -20
- package/dist/snapshot.js.map +1 -1
- package/dist/windows-engine.d.ts +1 -1
- package/dist/windows-engine.d.ts.map +1 -1
- package/dist/windows-engine.js +3 -1
- package/dist/windows-engine.js.map +1 -1
- package/package.json +3 -2
- package/src/classifier.ts +44 -7
- package/src/errors.ts +32 -0
- package/src/host.ts +608 -89
- package/src/index.ts +4 -0
- package/src/page-scripts.ts +108 -0
- package/src/reach.ts +211 -0
- package/src/settle.ts +60 -0
- package/src/snapshot.ts +402 -20
- package/src/windows-engine.ts +4 -1
package/src/host.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type {
|
|
|
3
3
|
BrowserActAction,
|
|
4
4
|
BrowserCallOptions,
|
|
5
5
|
BrowserCapabilities,
|
|
6
|
+
BrowserEffect,
|
|
6
7
|
BrowserHost,
|
|
7
8
|
BrowserObserveAction,
|
|
8
9
|
BrowserPageInfo,
|
|
@@ -10,22 +11,33 @@ import type {
|
|
|
10
11
|
BrowserResult,
|
|
11
12
|
BrowserSessionInfo,
|
|
12
13
|
BrowserSnapshot,
|
|
14
|
+
BrowserSnapshotView,
|
|
13
15
|
BrowserTabInfo,
|
|
16
|
+
BrowserTarget,
|
|
17
|
+
BrowserValue,
|
|
14
18
|
} from '@namzu/sdk'
|
|
15
|
-
import { canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome } from '@namzu/sdk'
|
|
16
19
|
import {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
BROWSER_FIND_MAX_MATCHES,
|
|
21
|
+
canonicalizeBrowserOrigin,
|
|
22
|
+
canonicalizeBrowserUrl,
|
|
23
|
+
resolveNamzuHome,
|
|
24
|
+
} from '@namzu/sdk'
|
|
25
|
+
import type {
|
|
26
|
+
BrowserContext,
|
|
27
|
+
CDPSession,
|
|
28
|
+
Dialog,
|
|
29
|
+
ElementHandle,
|
|
30
|
+
Locator,
|
|
31
|
+
Page,
|
|
32
|
+
Request,
|
|
33
|
+
Route,
|
|
24
34
|
} from 'playwright-core'
|
|
25
35
|
import {
|
|
26
36
|
type BrowserHumanClassifierOptions,
|
|
37
|
+
type BrowserPageSignals,
|
|
27
38
|
classifyHumanRequired,
|
|
28
39
|
isCredentialField,
|
|
40
|
+
signInNotice,
|
|
29
41
|
} from './classifier.js'
|
|
30
42
|
import {
|
|
31
43
|
type BrowserEnginePlan,
|
|
@@ -39,14 +51,24 @@ import {
|
|
|
39
51
|
} from './detect.js'
|
|
40
52
|
import {
|
|
41
53
|
BrowserHumanRequiredError,
|
|
54
|
+
BrowserNoEffectError,
|
|
42
55
|
BrowserOriginMismatchError,
|
|
43
56
|
BrowserOutcomeUnknownError,
|
|
44
57
|
BrowserSiteDeniedError,
|
|
45
58
|
BrowserStaleRefError,
|
|
59
|
+
BrowserTargetUnclearError,
|
|
46
60
|
BrowserUnavailableError,
|
|
47
61
|
ProfileBusyError,
|
|
48
62
|
} from './errors.js'
|
|
49
|
-
import {
|
|
63
|
+
import {
|
|
64
|
+
type ElementFieldFacts,
|
|
65
|
+
countCredentialFields,
|
|
66
|
+
fieldFacts,
|
|
67
|
+
hasVisibleContent,
|
|
68
|
+
installMutationClock,
|
|
69
|
+
msSinceLastChange,
|
|
70
|
+
valueFacts,
|
|
71
|
+
} from './page-scripts.js'
|
|
50
72
|
import { BrowserSitePolicy, type BrowserSiteRules, DEFAULT_BROWSER_SITE_RULES } from './policy.js'
|
|
51
73
|
import {
|
|
52
74
|
type BrowserLease,
|
|
@@ -55,12 +77,34 @@ import {
|
|
|
55
77
|
BrowserProfileStore,
|
|
56
78
|
DEFAULT_BROWSER_PROFILE,
|
|
57
79
|
} from './profiles.js'
|
|
80
|
+
import {
|
|
81
|
+
type Aim,
|
|
82
|
+
CANDIDATES_MAX,
|
|
83
|
+
KEY_DELAY_MS,
|
|
84
|
+
NAMED_KEYS,
|
|
85
|
+
type NamedKey,
|
|
86
|
+
aimInside,
|
|
87
|
+
deepFocus,
|
|
88
|
+
describeCandidate,
|
|
89
|
+
failedBeforeActing,
|
|
90
|
+
focusIsOpaque,
|
|
91
|
+
keyCanCommit,
|
|
92
|
+
namedKey,
|
|
93
|
+
noEffectReason,
|
|
94
|
+
} from './reach.js'
|
|
95
|
+
import { type SettleOutcome, type SettleProbe, waitUntilQuiet } from './settle.js'
|
|
58
96
|
import {
|
|
59
97
|
PLAYWRIGHT_CORE_VERSION,
|
|
60
98
|
SnapshotPager,
|
|
99
|
+
buildEffect,
|
|
61
100
|
captureAriaTree,
|
|
101
|
+
diffLines,
|
|
102
|
+
findInTree,
|
|
62
103
|
locateRef,
|
|
63
104
|
renderAriaTree,
|
|
105
|
+
renderInteractive,
|
|
106
|
+
renderReading,
|
|
107
|
+
subtreeOf,
|
|
64
108
|
} from './snapshot.js'
|
|
65
109
|
import { WindowsBridgeError } from './windows-bridge.js'
|
|
66
110
|
import { type WindowsBrowserConnection, connectWindowsBrowser } from './windows-engine.js'
|
|
@@ -96,6 +140,8 @@ export interface PlaywrightBrowserHostOptions {
|
|
|
96
140
|
readonly executablePath?: string
|
|
97
141
|
/** More sign-in addresses for the classifier. */
|
|
98
142
|
readonly signInAddresses?: BrowserHumanClassifierOptions['signInAddresses']
|
|
143
|
+
/** What makes a page a sign-in page: `address` (default) or `page`. See {@link BrowserHumanClassifierOptions.signInDetection}. */
|
|
144
|
+
readonly signInDetection?: BrowserHumanClassifierOptions['signInDetection']
|
|
99
145
|
/** The command that opens a visible window for signing in. Default `namzu browser login <profile> <url>`. */
|
|
100
146
|
readonly loginCommand?: (profile: string, url: string) => string
|
|
101
147
|
/** Windows engine: how long starting and connecting to the browser may take. Default 60 000. */
|
|
@@ -110,10 +156,18 @@ interface Tab {
|
|
|
110
156
|
mainDocumentResponses: number
|
|
111
157
|
cdp?: CDPSession
|
|
112
158
|
readonly responsesByLoader: Map<string, DocumentResponse>
|
|
159
|
+
/** Requests that can still change the page: documents, scripts, styles, fetch and XHR. */
|
|
160
|
+
readonly inflight: Set<Request>
|
|
161
|
+
/** When a request of that kind last started or ended. */
|
|
162
|
+
lastNetworkAt: number
|
|
163
|
+
/** The whole-page tree the model was last shown for this tab, for `delta` and effects. */
|
|
164
|
+
lastSeen?: string
|
|
113
165
|
/** HTTP status of the main document's last response. */
|
|
114
166
|
status?: number
|
|
115
167
|
wwwAuthenticate?: string
|
|
116
168
|
proxyAuthenticate?: string
|
|
169
|
+
/** The address the sign-in notice was last given for, so it is said once. */
|
|
170
|
+
signInNoticeFor?: string
|
|
117
171
|
}
|
|
118
172
|
|
|
119
173
|
type DocumentResponse = Pick<Tab, 'status' | 'wwwAuthenticate' | 'proxyAuthenticate'>
|
|
@@ -136,12 +190,42 @@ const DEFAULT_ACTION_TIMEOUT_MS = 10_000
|
|
|
136
190
|
/** How long a history move waits for the document after it committed. */
|
|
137
191
|
const HISTORY_LOAD_WAIT_MS = 5_000
|
|
138
192
|
const SNAPSHOT_PAGE_MAX = 20_000
|
|
139
|
-
/**
|
|
140
|
-
|
|
193
|
+
/**
|
|
194
|
+
* A page is quiet after this long with no change to the document and no
|
|
195
|
+
* request in flight. It also bounds how long an event the browser has not
|
|
196
|
+
* announced yet (a popup, a download, a dialog) has to arrive.
|
|
197
|
+
*/
|
|
198
|
+
const QUIET_MS = 250
|
|
199
|
+
/** The most an action waits for quiet. A page that never settles is reported, not waited on. */
|
|
200
|
+
const ACT_SETTLE_CAP_MS = 2_000
|
|
201
|
+
/** The same for a navigation, which loads more. */
|
|
202
|
+
const NAVIGATE_SETTLE_CAP_MS = 5_000
|
|
203
|
+
/** The wait before an action, for events left over from the last one. */
|
|
204
|
+
const PRE_ACT_QUIET_MS = 100
|
|
205
|
+
const PRE_ACT_CAP_MS = 500
|
|
206
|
+
/** `wait_for` with `stable` and no time: how long it waits. */
|
|
207
|
+
const STABLE_DEFAULT_WAIT_MS = 5_000
|
|
208
|
+
/** Request kinds that can change what the page shows. */
|
|
209
|
+
const PAGE_REQUEST_KINDS = new Set(['document', 'script', 'stylesheet', 'fetch', 'xhr'])
|
|
210
|
+
/** Lines of a `delta` snapshot that are listed in full. */
|
|
211
|
+
const DELTA_LINES_SHOWN = 80
|
|
212
|
+
/** Lines of an effect the model is shown. */
|
|
213
|
+
const EFFECT_LINES_SHOWN = 10
|
|
141
214
|
/** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
|
|
142
215
|
const CAPTCHA_FRAME_MIN_PX = 30
|
|
143
216
|
const REMEMBERED_DOCUMENTS_MAX = 64
|
|
217
|
+
|
|
218
|
+
/** Something the credential guards can ask about: a located element or a handle to one. */
|
|
219
|
+
type Probe = Locator | ElementHandle
|
|
220
|
+
|
|
221
|
+
function evaluateOn<R>(target: Probe, fn: (element: Element) => R): Promise<R> {
|
|
222
|
+
return 'asElement' in target
|
|
223
|
+
? (target as ElementHandle).evaluate(fn)
|
|
224
|
+
: (target as Locator).evaluate(fn)
|
|
225
|
+
}
|
|
144
226
|
const SCREENSHOT_INPUT_MASK_COLOR = '#666666'
|
|
227
|
+
/** Most inputs one screenshot checks for a credential. */
|
|
228
|
+
const MASK_CANDIDATES_MAX = 200
|
|
145
229
|
|
|
146
230
|
function firstLine(error: unknown): string {
|
|
147
231
|
const text = error instanceof Error ? error.message : String(error)
|
|
@@ -295,6 +379,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
295
379
|
context.setDefaultNavigationTimeout(
|
|
296
380
|
this.options.navigationTimeoutMs ?? DEFAULT_NAVIGATION_TIMEOUT_MS,
|
|
297
381
|
)
|
|
382
|
+
// Every new document keeps a clock of its last change, for the quiet-page wait.
|
|
383
|
+
await context.addInitScript(installMutationClock).catch(() => undefined)
|
|
298
384
|
context.on('page', (page) => this.adopt(page))
|
|
299
385
|
context.on('close', () => this.forget(context))
|
|
300
386
|
context.browser()?.on('disconnected', () => this.forget(context))
|
|
@@ -305,7 +391,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
305
391
|
() => true,
|
|
306
392
|
(route) => this.screen(route),
|
|
307
393
|
)
|
|
308
|
-
for (const page of context.pages())
|
|
394
|
+
for (const page of context.pages()) {
|
|
395
|
+
this.adopt(page, false)
|
|
396
|
+
void page.evaluate(installMutationClock).catch(() => undefined)
|
|
397
|
+
}
|
|
309
398
|
return context
|
|
310
399
|
}
|
|
311
400
|
|
|
@@ -322,6 +411,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
322
411
|
})
|
|
323
412
|
let context: BrowserContext
|
|
324
413
|
try {
|
|
414
|
+
// Loaded here, on the first browser launch: playwright-core is several
|
|
415
|
+
// megabytes of source that a session which never opens a browser must
|
|
416
|
+
// not compile at startup.
|
|
417
|
+
const { chromium } = await import('playwright-core')
|
|
325
418
|
context = await chromium.launchPersistentContext(descriptor.userDataDir, {
|
|
326
419
|
headless: plan.headless,
|
|
327
420
|
...(plan.channel ? { channel: plan.channel } : {}),
|
|
@@ -496,6 +589,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
496
589
|
mainFrameCommits: 0,
|
|
497
590
|
mainDocumentResponses: 0,
|
|
498
591
|
responsesByLoader: new Map(),
|
|
592
|
+
inflight: new Set(),
|
|
593
|
+
lastNetworkAt: Date.now(),
|
|
499
594
|
}
|
|
500
595
|
this.tabs.set(tab.id, tab)
|
|
501
596
|
this.tabOf.set(page, tab)
|
|
@@ -524,6 +619,17 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
524
619
|
// A response without a frame (a service worker's) says nothing about the page.
|
|
525
620
|
}
|
|
526
621
|
})
|
|
622
|
+
const kind = (request: Request): boolean => PAGE_REQUEST_KINDS.has(request.resourceType())
|
|
623
|
+
page.on('request', (request) => {
|
|
624
|
+
if (!kind(request)) return
|
|
625
|
+
tab.inflight.add(request)
|
|
626
|
+
tab.lastNetworkAt = Date.now()
|
|
627
|
+
})
|
|
628
|
+
const ended = (request: Request): void => {
|
|
629
|
+
if (tab.inflight.delete(request)) tab.lastNetworkAt = Date.now()
|
|
630
|
+
}
|
|
631
|
+
page.on('requestfinished', ended)
|
|
632
|
+
page.on('requestfailed', ended)
|
|
527
633
|
page.on('dialog', (dialog) => this.onDialog(tab, dialog))
|
|
528
634
|
page.on('download', (download) => {
|
|
529
635
|
const name = download.suggestedFilename()
|
|
@@ -601,9 +707,42 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
601
707
|
this.blanking.add(blank)
|
|
602
708
|
}
|
|
603
709
|
|
|
604
|
-
|
|
605
|
-
|
|
710
|
+
/**
|
|
711
|
+
* Wait until the tab's page is quiet (nothing changed in it and no request
|
|
712
|
+
* in flight for `quietMs`), or `capMs` has passed. Events the browser
|
|
713
|
+
* announces late (a popup, a download, a dialog) arrive inside the window.
|
|
714
|
+
* Then let any tab that was cleared to `about:blank` finish clearing.
|
|
715
|
+
*/
|
|
716
|
+
private async settle(
|
|
717
|
+
tab: Tab,
|
|
718
|
+
options: { readonly quietMs: number; readonly capMs: number } = {
|
|
719
|
+
quietMs: QUIET_MS,
|
|
720
|
+
capMs: ACT_SETTLE_CAP_MS,
|
|
721
|
+
},
|
|
722
|
+
): Promise<SettleOutcome> {
|
|
723
|
+
const outcome = await waitUntilQuiet(
|
|
724
|
+
{ now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) },
|
|
725
|
+
this.quietProbe(tab),
|
|
726
|
+
options,
|
|
727
|
+
)
|
|
606
728
|
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
729
|
+
return outcome
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
private quietProbe(tab: Tab): SettleProbe {
|
|
733
|
+
return {
|
|
734
|
+
stillForMs: async () => {
|
|
735
|
+
// A dialog pauses the page; nothing more will happen until it is answered.
|
|
736
|
+
if (this.pendingDialog?.tab === tab || tab.page.isClosed()) return Number.MAX_SAFE_INTEGER
|
|
737
|
+
if (tab.inflight.size > 0) return 0
|
|
738
|
+
const sinceNetwork = Date.now() - tab.lastNetworkAt
|
|
739
|
+
const sincePage = await Promise.race([
|
|
740
|
+
tab.page.evaluate(msSinceLastChange).catch(() => 0),
|
|
741
|
+
new Promise<number>((resolve) => setTimeout(() => resolve(0), 1000)),
|
|
742
|
+
])
|
|
743
|
+
return Math.min(sinceNetwork, sincePage)
|
|
744
|
+
},
|
|
745
|
+
}
|
|
607
746
|
}
|
|
608
747
|
|
|
609
748
|
private async activeTab(): Promise<Tab> {
|
|
@@ -698,21 +837,28 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
698
837
|
// A frame that went away while we looked is not a challenge.
|
|
699
838
|
}
|
|
700
839
|
}
|
|
701
|
-
const
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
},
|
|
713
|
-
|
|
714
|
-
)
|
|
840
|
+
const signals: BrowserPageSignals = {
|
|
841
|
+
url,
|
|
842
|
+
title: await this.title(tab),
|
|
843
|
+
...(tab.status !== undefined ? { status: tab.status } : {}),
|
|
844
|
+
...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
|
|
845
|
+
...(tab.proxyAuthenticate !== undefined ? { proxyAuthenticate: tab.proxyAuthenticate } : {}),
|
|
846
|
+
...counts,
|
|
847
|
+
frameUrls,
|
|
848
|
+
}
|
|
849
|
+
const classifier: BrowserHumanClassifierOptions = {
|
|
850
|
+
...(this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {}),
|
|
851
|
+
...(this.options.signInDetection ? { signInDetection: this.options.signInDetection } : {}),
|
|
852
|
+
}
|
|
853
|
+
const reason = classifyHumanRequired(signals, classifier)
|
|
715
854
|
if (reason) throw this.humanRequired(reason, tab)
|
|
855
|
+
// A page with a password field that did not stop the turn: say so once
|
|
856
|
+
// per address, so the model reports it instead of trying to type.
|
|
857
|
+
const notice = signInNotice(signals, classifier, this.loginCommandFor(url))
|
|
858
|
+
if (notice && tab.signInNoticeFor !== withoutQuery(url)) {
|
|
859
|
+
tab.signInNoticeFor = withoutQuery(url)
|
|
860
|
+
this.notes.push(notice)
|
|
861
|
+
}
|
|
716
862
|
if (tab.status === 401) {
|
|
717
863
|
throw new Error(
|
|
718
864
|
'The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.',
|
|
@@ -866,7 +1012,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
866
1012
|
case 'reload':
|
|
867
1013
|
return this.history(action.action)
|
|
868
1014
|
case 'snapshot':
|
|
869
|
-
return this.snapshot(action
|
|
1015
|
+
return this.snapshot(action)
|
|
870
1016
|
case 'screenshot':
|
|
871
1017
|
return this.screenshot(action.ref, action.fullPage)
|
|
872
1018
|
case 'scroll':
|
|
@@ -875,6 +1021,8 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
875
1021
|
return this.waitFor(action)
|
|
876
1022
|
case 'tabs':
|
|
877
1023
|
return this.tabsCall(action)
|
|
1024
|
+
case 'value':
|
|
1025
|
+
return this.readValue(action.ref)
|
|
878
1026
|
}
|
|
879
1027
|
}
|
|
880
1028
|
|
|
@@ -894,7 +1042,7 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
894
1042
|
} catch (error) {
|
|
895
1043
|
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
|
|
896
1044
|
if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
|
|
897
|
-
await this.settle()
|
|
1045
|
+
await this.settle(tab)
|
|
898
1046
|
throw new Error(
|
|
899
1047
|
'The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.',
|
|
900
1048
|
)
|
|
@@ -908,10 +1056,38 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
908
1056
|
}
|
|
909
1057
|
|
|
910
1058
|
private async afterNavigation(tab: Tab): Promise<BrowserResult> {
|
|
911
|
-
|
|
1059
|
+
// The document first (a page that has not finished loading has not
|
|
1060
|
+
// finished building itself), then the same quiet the actions wait for.
|
|
1061
|
+
await tab.page
|
|
1062
|
+
.waitForLoadState('load', { timeout: NAVIGATE_SETTLE_CAP_MS })
|
|
1063
|
+
.catch(() => undefined)
|
|
1064
|
+
const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: NAVIGATE_SETTLE_CAP_MS })
|
|
912
1065
|
this.checkLanding(tab)
|
|
913
|
-
await this.
|
|
1066
|
+
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
914
1067
|
await this.checkHuman(tab)
|
|
1068
|
+
if (!settled.quiet) {
|
|
1069
|
+
this.notes.push(
|
|
1070
|
+
`The page was still changing after ${settled.waitedMs} ms, so what it shows may be unfinished. Call browser with wait_for and stable, or take a snapshot.`,
|
|
1071
|
+
)
|
|
1072
|
+
}
|
|
1073
|
+
const populated =
|
|
1074
|
+
originOf(tab.page.url()) === 'null' ||
|
|
1075
|
+
(await tab.page.evaluate(hasVisibleContent).catch(() => true))
|
|
1076
|
+
if (!populated) {
|
|
1077
|
+
this.notes.push(
|
|
1078
|
+
`The page has nothing to read yet (waited ${settled.waitedMs} ms). Call browser with wait_for and stable, or reload.`,
|
|
1079
|
+
)
|
|
1080
|
+
}
|
|
1081
|
+
// The model's next step after opening a page is to read it, so the
|
|
1082
|
+
// reading comes with the open: one request fewer on every page visit.
|
|
1083
|
+
if (populated && originOf(tab.page.url()) !== 'null') {
|
|
1084
|
+
try {
|
|
1085
|
+
const snapshot = await this.takeSnapshot(tab)
|
|
1086
|
+
return this.result({ page: snapshot.page, snapshot })
|
|
1087
|
+
} catch {
|
|
1088
|
+
// An unreadable page still opened; say where we are and let the model ask again.
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
915
1091
|
return this.result({ page: await this.pageInfo(tab) })
|
|
916
1092
|
}
|
|
917
1093
|
|
|
@@ -960,7 +1136,27 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
960
1136
|
this.notes.push('The open dialog was dismissed.')
|
|
961
1137
|
}
|
|
962
1138
|
|
|
963
|
-
|
|
1139
|
+
/** Read the page's tree and remember its refs. */
|
|
1140
|
+
private async readTree(tab: Tab) {
|
|
1141
|
+
const { nodes, hidden, redact } = await captureAriaTree(
|
|
1142
|
+
tab.page,
|
|
1143
|
+
this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS,
|
|
1144
|
+
)
|
|
1145
|
+
const whole = renderAriaTree(nodes, hidden, undefined, redact)
|
|
1146
|
+
if (!whole) throw new Error('The page could not be read.')
|
|
1147
|
+
this.refs = whole.refs
|
|
1148
|
+
return { nodes, hidden, redact, whole }
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
private async takeSnapshot(
|
|
1152
|
+
tab: Tab,
|
|
1153
|
+
request: {
|
|
1154
|
+
readonly ref?: string
|
|
1155
|
+
readonly view?: BrowserSnapshotView
|
|
1156
|
+
readonly query?: string
|
|
1157
|
+
readonly role?: string
|
|
1158
|
+
} = {},
|
|
1159
|
+
): Promise<BrowserSnapshot> {
|
|
964
1160
|
const pending = this.pendingDialog
|
|
965
1161
|
if (pending && pending.tab === tab) {
|
|
966
1162
|
const dialog = pending.dialog
|
|
@@ -972,18 +1168,72 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
972
1168
|
lines.push(' - (the page is paused until the dialog is answered)')
|
|
973
1169
|
return { page: await this.pageInfo(tab), text: lines.join('\n') }
|
|
974
1170
|
}
|
|
975
|
-
const {
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
const
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
1171
|
+
const { ref, view = 'full', query, role } = request
|
|
1172
|
+
const { nodes, hidden, redact, whole } = await this.readTree(tab)
|
|
1173
|
+
if (ref !== undefined && !whole.refs.has(ref)) throw new BrowserStaleRefError(ref)
|
|
1174
|
+
// A region is cut from the whole-page tree: a snapshot of one element would replace the ref table.
|
|
1175
|
+
const scope = ref === undefined ? nodes : (subtreeOf(nodes, ref) ?? [])
|
|
1176
|
+
let text: string
|
|
1177
|
+
if (query !== undefined || role !== undefined) {
|
|
1178
|
+
const found = findInTree(
|
|
1179
|
+
scope,
|
|
1180
|
+
hidden,
|
|
1181
|
+
redact,
|
|
1182
|
+
{ ...(query !== undefined ? { query } : {}), ...(role !== undefined ? { role } : {}) },
|
|
1183
|
+
BROWSER_FIND_MAX_MATCHES,
|
|
1184
|
+
)
|
|
1185
|
+
text = found.lines.join('\n')
|
|
1186
|
+
const what = [
|
|
1187
|
+
query !== undefined ? `text ${quoteShort(query, 60)}` : '',
|
|
1188
|
+
role !== undefined ? `role ${quoteShort(role, 30)}` : '',
|
|
1189
|
+
]
|
|
1190
|
+
.filter(Boolean)
|
|
1191
|
+
.join(' and ')
|
|
1192
|
+
this.notes.unshift(
|
|
1193
|
+
found.total === 0
|
|
1194
|
+
? `Nothing on the page matches ${what}.`
|
|
1195
|
+
: found.total > found.lines.length
|
|
1196
|
+
? `${found.total} elements match ${what}; the first ${found.lines.length} are listed. Narrow the search.`
|
|
1197
|
+
: `${found.total} element${found.total === 1 ? '' : 's'} match${found.total === 1 ? 'es' : ''} ${what}.`,
|
|
1198
|
+
)
|
|
1199
|
+
} else if (view === 'interactive') {
|
|
1200
|
+
text = renderInteractive(scope, hidden, redact)
|
|
1201
|
+
tab.lastSeen = whole.text
|
|
1202
|
+
} else if (view === 'reading') {
|
|
1203
|
+
text = renderReading(scope, hidden, redact)
|
|
1204
|
+
tab.lastSeen = whole.text
|
|
1205
|
+
} else if (view === 'delta') {
|
|
1206
|
+
const before = tab.lastSeen
|
|
1207
|
+
if (before === undefined) {
|
|
1208
|
+
this.notes.unshift(
|
|
1209
|
+
'There is no earlier snapshot of this tab to compare with; this is the whole page.',
|
|
1210
|
+
)
|
|
1211
|
+
text =
|
|
1212
|
+
ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '')
|
|
1213
|
+
} else {
|
|
1214
|
+
const diff = diffLines(before, whole.text, DELTA_LINES_SHOWN)
|
|
1215
|
+
text =
|
|
1216
|
+
diff.added === 0 && diff.removed === 0
|
|
1217
|
+
? '(no change since the last snapshot)'
|
|
1218
|
+
: [
|
|
1219
|
+
...diff.addedLines.map((l) => `+ ${l}`),
|
|
1220
|
+
...(diff.added > diff.addedLines.length
|
|
1221
|
+
? [` (${diff.added - diff.addedLines.length} more lines added)`]
|
|
1222
|
+
: []),
|
|
1223
|
+
...diff.removedLines.map((l) => `- ${l}`),
|
|
1224
|
+
...(diff.removed > diff.removedLines.length
|
|
1225
|
+
? [` (${diff.removed - diff.removedLines.length} more lines removed)`]
|
|
1226
|
+
: []),
|
|
1227
|
+
].join('\n')
|
|
1228
|
+
this.notes.unshift(
|
|
1229
|
+
`Changes since the last snapshot: ${diff.added} lines added, ${diff.removed} removed. Take a full snapshot to see the whole page again.`,
|
|
1230
|
+
)
|
|
1231
|
+
}
|
|
1232
|
+
tab.lastSeen = whole.text
|
|
1233
|
+
} else {
|
|
1234
|
+
text =
|
|
1235
|
+
ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '')
|
|
1236
|
+
if (ref === undefined) tab.lastSeen = whole.text
|
|
987
1237
|
}
|
|
988
1238
|
if (text.trim() === '') text = '(nothing readable on this page)'
|
|
989
1239
|
const first = this.pager.start(text)
|
|
@@ -994,10 +1244,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
994
1244
|
}
|
|
995
1245
|
}
|
|
996
1246
|
|
|
997
|
-
private async snapshot(
|
|
1247
|
+
private async snapshot(
|
|
1248
|
+
request: Extract<BrowserObserveAction, { action: 'snapshot' }>,
|
|
1249
|
+
): Promise<BrowserResult> {
|
|
998
1250
|
const tab = await this.activeTab()
|
|
999
|
-
if (cursor !== undefined) {
|
|
1000
|
-
const next = this.pager.resume(cursor)
|
|
1251
|
+
if (request.cursor !== undefined) {
|
|
1252
|
+
const next = this.pager.resume(request.cursor)
|
|
1001
1253
|
if (!next) {
|
|
1002
1254
|
throw new Error(
|
|
1003
1255
|
'That cursor belongs to an older snapshot. Take a new snapshot and page through it.',
|
|
@@ -1011,8 +1263,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1011
1263
|
},
|
|
1012
1264
|
})
|
|
1013
1265
|
}
|
|
1014
|
-
await this.settle()
|
|
1015
|
-
return this.result({
|
|
1266
|
+
await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS })
|
|
1267
|
+
return this.result({
|
|
1268
|
+
snapshot: await this.takeSnapshot(tab, {
|
|
1269
|
+
...(request.ref !== undefined ? { ref: request.ref } : {}),
|
|
1270
|
+
...(request.view !== undefined ? { view: request.view } : {}),
|
|
1271
|
+
...(request.query !== undefined ? { query: request.query } : {}),
|
|
1272
|
+
...(request.role !== undefined ? { role: request.role } : {}),
|
|
1273
|
+
}),
|
|
1274
|
+
})
|
|
1016
1275
|
}
|
|
1017
1276
|
|
|
1018
1277
|
private async resolveRef(tab: Tab, ref: string): Promise<Locator> {
|
|
@@ -1035,10 +1294,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1035
1294
|
'A dialog is open; dismiss it with browser_act dialog before taking a screenshot.',
|
|
1036
1295
|
)
|
|
1037
1296
|
}
|
|
1038
|
-
// Screenshots are image blocks sent to the model. Mask
|
|
1039
|
-
//
|
|
1040
|
-
//
|
|
1041
|
-
|
|
1297
|
+
// Screenshots are image blocks sent to the model. Mask the fields that
|
|
1298
|
+
// take a password or a one-time code, in every frame; the other inputs
|
|
1299
|
+
// stay visible, because a search box or an address field the model
|
|
1300
|
+
// cannot read cannot be checked. Closed shadow roots and page-drawn
|
|
1301
|
+
// text remain outside this mask.
|
|
1302
|
+
const mask = await this.credentialMask(tab)
|
|
1042
1303
|
const data =
|
|
1043
1304
|
ref !== undefined
|
|
1044
1305
|
? await (await this.resolveRef(tab, ref)).screenshot({
|
|
@@ -1065,6 +1326,64 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1065
1326
|
})
|
|
1066
1327
|
}
|
|
1067
1328
|
|
|
1329
|
+
/** Locators of the credential fields (password, one-time code) of every frame of the tab. */
|
|
1330
|
+
private async credentialMask(tab: Tab): Promise<Locator[]> {
|
|
1331
|
+
const out: Locator[] = []
|
|
1332
|
+
for (const frame of tab.page.frames()) {
|
|
1333
|
+
const inputs = frame.locator('input, textarea')
|
|
1334
|
+
let count = 0
|
|
1335
|
+
try {
|
|
1336
|
+
count = Math.min(await inputs.count(), MASK_CANDIDATES_MAX)
|
|
1337
|
+
} catch {
|
|
1338
|
+
// A frame that went away while we looked has nothing to mask.
|
|
1339
|
+
continue
|
|
1340
|
+
}
|
|
1341
|
+
const verdicts = await Promise.all(
|
|
1342
|
+
Array.from({ length: count }, async (_, index) => {
|
|
1343
|
+
try {
|
|
1344
|
+
return isCredentialField(
|
|
1345
|
+
await inputs.nth(index).evaluate(fieldFacts, undefined, { timeout: 2000 }),
|
|
1346
|
+
)
|
|
1347
|
+
} catch {
|
|
1348
|
+
// A field that cannot be asked about is hidden to be safe.
|
|
1349
|
+
return true
|
|
1350
|
+
}
|
|
1351
|
+
}),
|
|
1352
|
+
)
|
|
1353
|
+
verdicts.forEach((hide, index) => {
|
|
1354
|
+
if (hide) out.push(inputs.nth(index))
|
|
1355
|
+
})
|
|
1356
|
+
}
|
|
1357
|
+
return out
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/** One element's current value and state, read from the page itself. */
|
|
1361
|
+
private async readValue(ref: string): Promise<BrowserResult> {
|
|
1362
|
+
const tab = await this.activeTab()
|
|
1363
|
+
if (this.pendingDialog?.tab === tab) {
|
|
1364
|
+
throw new Error(
|
|
1365
|
+
'A dialog is open; dismiss it with browser_act dialog before reading a value.',
|
|
1366
|
+
)
|
|
1367
|
+
}
|
|
1368
|
+
const locator = await this.resolveRef(tab, ref)
|
|
1369
|
+
const facts = await locator.evaluate(fieldFacts)
|
|
1370
|
+
const described = this.refs.get(ref)
|
|
1371
|
+
const page = await this.pageInfo(tab)
|
|
1372
|
+
const hidden = isCredentialField(facts)
|
|
1373
|
+
const read = hidden ? undefined : await locator.evaluate(valueFacts)
|
|
1374
|
+
const value: BrowserValue = {
|
|
1375
|
+
page,
|
|
1376
|
+
ref,
|
|
1377
|
+
...(described?.role !== undefined ? { role: described.role } : {}),
|
|
1378
|
+
...(described?.name !== undefined ? { name: described.name } : {}),
|
|
1379
|
+
...(read?.value !== undefined ? { value: read.value } : {}),
|
|
1380
|
+
state: read?.state ?? [],
|
|
1381
|
+
...(read?.options !== undefined ? { options: read.options } : {}),
|
|
1382
|
+
...(hidden ? { hidden: true } : {}),
|
|
1383
|
+
}
|
|
1384
|
+
return this.result({ value })
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1068
1387
|
private async scroll(
|
|
1069
1388
|
direction: 'up' | 'down' | 'left' | 'right',
|
|
1070
1389
|
ref?: string,
|
|
@@ -1125,10 +1444,19 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1125
1444
|
: `${quoteShort(text)} was still there after ${timeout} ms.`,
|
|
1126
1445
|
)
|
|
1127
1446
|
}
|
|
1128
|
-
} else {
|
|
1447
|
+
} else if (action.stable !== true) {
|
|
1129
1448
|
await tab.page.waitForTimeout(timeout)
|
|
1130
1449
|
this.notes.unshift(`Waited ${timeout} ms.`)
|
|
1131
1450
|
}
|
|
1451
|
+
if (action.stable === true) {
|
|
1452
|
+
const cap = action.timeMs !== undefined ? timeout : STABLE_DEFAULT_WAIT_MS
|
|
1453
|
+
const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: cap })
|
|
1454
|
+
this.notes.push(
|
|
1455
|
+
settled.quiet
|
|
1456
|
+
? `The page is quiet (nothing changed for ${QUIET_MS} ms; waited ${settled.waitedMs} ms).`
|
|
1457
|
+
: `The page was still changing after ${settled.waitedMs} ms.`,
|
|
1458
|
+
)
|
|
1459
|
+
}
|
|
1132
1460
|
return this.result({ page: await this.pageInfo(tab) })
|
|
1133
1461
|
}
|
|
1134
1462
|
|
|
@@ -1188,11 +1516,11 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1188
1516
|
// Acting
|
|
1189
1517
|
// -------------------------------------------------------------------------
|
|
1190
1518
|
|
|
1191
|
-
private async guardField(locator:
|
|
1192
|
-
const facts = await locator
|
|
1519
|
+
private async guardField(locator: Probe): Promise<ElementFieldFacts> {
|
|
1520
|
+
const facts = await evaluateOn(locator, fieldFacts)
|
|
1193
1521
|
if (isCredentialField(facts)) {
|
|
1194
1522
|
throw new Error(
|
|
1195
|
-
'This action targets a password or one-time-code control. Ask the operator to use the browser window.',
|
|
1523
|
+
'This action targets a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.',
|
|
1196
1524
|
)
|
|
1197
1525
|
}
|
|
1198
1526
|
return facts
|
|
@@ -1200,11 +1528,11 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1200
1528
|
|
|
1201
1529
|
/** Refuse direct form submission while a form holds a password or code input. */
|
|
1202
1530
|
private async guardCredentialForm(
|
|
1203
|
-
locator:
|
|
1531
|
+
locator: Probe,
|
|
1204
1532
|
action: 'click' | 'press' | 'type-submit',
|
|
1205
1533
|
key?: string,
|
|
1206
1534
|
): Promise<void> {
|
|
1207
|
-
const context = await locator
|
|
1535
|
+
const context = await evaluateOn(locator, (element) => {
|
|
1208
1536
|
const label = element.closest('label') as HTMLLabelElement | null
|
|
1209
1537
|
const labelledField = label?.control
|
|
1210
1538
|
? {
|
|
@@ -1251,20 +1579,77 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1251
1579
|
action === 'type-submit'
|
|
1252
1580
|
) {
|
|
1253
1581
|
throw new Error(
|
|
1254
|
-
'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.',
|
|
1582
|
+
'This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.',
|
|
1255
1583
|
)
|
|
1256
1584
|
}
|
|
1257
1585
|
}
|
|
1258
1586
|
|
|
1587
|
+
/**
|
|
1588
|
+
* The page's tree as the model could have read it just before an action,
|
|
1589
|
+
* or `undefined` when it cannot be read now (a dialog holds the page, or
|
|
1590
|
+
* the read failed). An effect is a bonus; it never fails the action.
|
|
1591
|
+
*/
|
|
1592
|
+
private async effectBaseline(tab: Tab): Promise<string | undefined> {
|
|
1593
|
+
if (this.pendingDialog?.tab === tab) return undefined
|
|
1594
|
+
try {
|
|
1595
|
+
return (await this.readTree(tab)).whole.text
|
|
1596
|
+
} catch {
|
|
1597
|
+
return undefined
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
/** What the action did, from the tree before and the tree now. */
|
|
1602
|
+
private async effectSince(
|
|
1603
|
+
before: string | undefined,
|
|
1604
|
+
urlBefore: string,
|
|
1605
|
+
tabsBefore: number,
|
|
1606
|
+
tab: Tab,
|
|
1607
|
+
settled: SettleOutcome,
|
|
1608
|
+
): Promise<BrowserEffect | undefined> {
|
|
1609
|
+
if (before === undefined || this.pendingDialog?.tab === tab || tab.page.isClosed())
|
|
1610
|
+
return undefined
|
|
1611
|
+
let after: string
|
|
1612
|
+
try {
|
|
1613
|
+
after = (await this.readTree(tab)).whole.text
|
|
1614
|
+
} catch {
|
|
1615
|
+
return undefined
|
|
1616
|
+
}
|
|
1617
|
+
return buildEffect(
|
|
1618
|
+
{
|
|
1619
|
+
beforeTree: before,
|
|
1620
|
+
afterTree: after,
|
|
1621
|
+
urlBefore,
|
|
1622
|
+
urlNow: tab.page.url(),
|
|
1623
|
+
tabsBefore,
|
|
1624
|
+
tabsNow: this.tabs.size,
|
|
1625
|
+
settle: { quiet: settled.quiet, waitedMs: settled.waitedMs, capMs: settled.capMs },
|
|
1626
|
+
},
|
|
1627
|
+
EFFECT_LINES_SHOWN,
|
|
1628
|
+
)
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1259
1631
|
private async runAct(action: BrowserActAction): Promise<BrowserResult> {
|
|
1260
1632
|
const tab = await this.activeTab()
|
|
1261
|
-
await this.settle()
|
|
1633
|
+
await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS })
|
|
1262
1634
|
const expected = canonicalizeBrowserOrigin(action.origin)
|
|
1263
1635
|
const live = originOf(tab.page.url())
|
|
1264
1636
|
if (!expected.ok || expected.origin !== live) {
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1637
|
+
// The page moved since the snapshot. Going on is fine only when the
|
|
1638
|
+
// site rules treat the new origin exactly as the named one: both at
|
|
1639
|
+
// `act`, where no review stood between the model and the page.
|
|
1640
|
+
const sameRules =
|
|
1641
|
+
expected.ok &&
|
|
1642
|
+
live !== 'null' &&
|
|
1643
|
+
this.policy.levelOf(expected.origin) === 'act' &&
|
|
1644
|
+
this.policy.levelOf(live) === 'act'
|
|
1645
|
+
if (!sameRules) {
|
|
1646
|
+
throw new BrowserOriginMismatchError(
|
|
1647
|
+
expected.ok ? expected.origin : action.origin,
|
|
1648
|
+
live === 'null' ? 'about:blank' : live,
|
|
1649
|
+
)
|
|
1650
|
+
}
|
|
1651
|
+
this.notes.unshift(
|
|
1652
|
+
`The page is now at ${live}, not ${expected.origin}; the site rules treat both alike, so the action went ahead there.`,
|
|
1268
1653
|
)
|
|
1269
1654
|
}
|
|
1270
1655
|
if (!this.policy.canAct(live)) {
|
|
@@ -1278,11 +1663,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1278
1663
|
throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.')
|
|
1279
1664
|
}
|
|
1280
1665
|
const checkpoint = await this.beginDocumentResponseChange(tab)
|
|
1666
|
+
const urlBefore = tab.page.url()
|
|
1667
|
+
const tabsBefore = this.tabs.size
|
|
1668
|
+
const treeBefore = await this.effectBaseline(tab)
|
|
1669
|
+
let settled: SettleOutcome
|
|
1281
1670
|
try {
|
|
1282
1671
|
await this.perform(tab, action)
|
|
1283
|
-
await this.settle()
|
|
1672
|
+
settled = await this.settle(tab)
|
|
1284
1673
|
this.checkLanding(tab)
|
|
1285
|
-
await this.
|
|
1674
|
+
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
1286
1675
|
} catch (error) {
|
|
1287
1676
|
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined)
|
|
1288
1677
|
throw error
|
|
@@ -1290,8 +1679,14 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1290
1679
|
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
1291
1680
|
await this.checkHuman(tab)
|
|
1292
1681
|
const after = this.tabs.get(tab.id) ?? tab
|
|
1293
|
-
|
|
1294
|
-
|
|
1682
|
+
const effect = await this.effectSince(treeBefore, urlBefore, tabsBefore, after, settled)
|
|
1683
|
+
if (action.snapshot) {
|
|
1684
|
+
return this.result({
|
|
1685
|
+
snapshot: await this.takeSnapshot(after),
|
|
1686
|
+
...(effect ? { effect } : {}),
|
|
1687
|
+
})
|
|
1688
|
+
}
|
|
1689
|
+
return this.result({ page: await this.pageInfo(after), ...(effect ? { effect } : {}) })
|
|
1295
1690
|
}
|
|
1296
1691
|
|
|
1297
1692
|
private async answerDialog(
|
|
@@ -1319,9 +1714,9 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1319
1714
|
throw new BrowserOutcomeUnknownError('dialog', error)
|
|
1320
1715
|
}
|
|
1321
1716
|
this.notes.unshift('The dialog was dismissed.')
|
|
1322
|
-
await this.settle()
|
|
1717
|
+
await this.settle(tab)
|
|
1323
1718
|
this.checkLanding(tab)
|
|
1324
|
-
await this.
|
|
1719
|
+
while (this.blanking.size > 0) await Promise.all([...this.blanking])
|
|
1325
1720
|
await this.finishDocumentResponseChange(tab, checkpoint)
|
|
1326
1721
|
await this.checkHuman(tab)
|
|
1327
1722
|
if (action.snapshot) return this.result({ snapshot: await this.takeSnapshot(tab) })
|
|
@@ -1336,29 +1731,45 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1336
1731
|
private async perform(tab: Tab, action: BrowserActAction): Promise<void> {
|
|
1337
1732
|
switch (action.action) {
|
|
1338
1733
|
case 'click': {
|
|
1339
|
-
const
|
|
1340
|
-
await this.guardField(locator)
|
|
1341
|
-
await this.guardCredentialForm(locator, 'click')
|
|
1342
|
-
|
|
1734
|
+
const target = await this.reach(tab, action)
|
|
1735
|
+
await this.guardField(target.locator)
|
|
1736
|
+
await this.guardCredentialForm(target.locator, 'click')
|
|
1737
|
+
const at = target.position ? { position: target.position } : {}
|
|
1738
|
+
await this.noEffectOnFailure('click', () => target.locator.click({ trial: true, ...at }))
|
|
1343
1739
|
await this.unknownOnFailure('click', () =>
|
|
1344
|
-
action.doubleClick ? locator.dblclick() : locator.click(),
|
|
1740
|
+
action.doubleClick ? target.locator.dblclick(at) : target.locator.click(at),
|
|
1345
1741
|
)
|
|
1346
1742
|
return
|
|
1347
1743
|
}
|
|
1348
1744
|
case 'hover': {
|
|
1349
|
-
const
|
|
1350
|
-
await this.guardField(locator)
|
|
1351
|
-
|
|
1352
|
-
await this.
|
|
1745
|
+
const target = await this.reach(tab, action)
|
|
1746
|
+
await this.guardField(target.locator)
|
|
1747
|
+
const at = target.position ? { position: target.position } : {}
|
|
1748
|
+
await this.noEffectOnFailure('hover', () => target.locator.hover({ trial: true, ...at }))
|
|
1749
|
+
await this.unknownOnFailure('hover', () => target.locator.hover(at))
|
|
1353
1750
|
return
|
|
1354
1751
|
}
|
|
1355
1752
|
case 'type': {
|
|
1356
1753
|
const locator = await this.resolveRef(tab, action.ref)
|
|
1357
1754
|
await this.guardField(locator)
|
|
1358
1755
|
if (action.submit) await this.guardCredentialForm(locator, 'type-submit')
|
|
1359
|
-
await locator.click({ trial: true })
|
|
1360
|
-
await this.unknownOnFailure('type', async () => {
|
|
1361
|
-
|
|
1756
|
+
await this.noEffectOnFailure('type', () => locator.click({ trial: true }))
|
|
1757
|
+
await this.unknownOnFailure('type', async (begun) => {
|
|
1758
|
+
if (action.keys) {
|
|
1759
|
+
await locator.focus()
|
|
1760
|
+
// Clearing and typing change the field; from here a failure is not harmless.
|
|
1761
|
+
begun()
|
|
1762
|
+
if (action.append) await locator.press('Control+End')
|
|
1763
|
+
else await locator.fill('')
|
|
1764
|
+
await locator.pressSequentially(action.text, { delay: KEY_DELAY_MS })
|
|
1765
|
+
} else if (action.append) {
|
|
1766
|
+
const held = await locator
|
|
1767
|
+
.inputValue()
|
|
1768
|
+
.catch(() => locator.evaluate((el) => el.textContent ?? ''))
|
|
1769
|
+
await locator.fill(held + action.text)
|
|
1770
|
+
} else {
|
|
1771
|
+
await locator.fill(action.text)
|
|
1772
|
+
}
|
|
1362
1773
|
if (action.submit) await locator.press('Enter')
|
|
1363
1774
|
})
|
|
1364
1775
|
return
|
|
@@ -1405,9 +1816,15 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1405
1816
|
await this.unknownOnFailure('press', () => locator.press(action.key))
|
|
1406
1817
|
return
|
|
1407
1818
|
}
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1819
|
+
const key = namedKey(action.key)
|
|
1820
|
+
if (!key) {
|
|
1821
|
+
throw new Error(
|
|
1822
|
+
`Without a ref, press takes only ${NAMED_KEYS.join(', ')}. Any other key goes to a specific control: give its ref.`,
|
|
1823
|
+
)
|
|
1824
|
+
}
|
|
1825
|
+
await this.guardFocus(tab, key)
|
|
1826
|
+
await this.unknownOnFailure('press', () => tab.page.keyboard.press(key))
|
|
1827
|
+
return
|
|
1411
1828
|
}
|
|
1412
1829
|
case 'upload': {
|
|
1413
1830
|
const locator = await this.resolveRef(tab, action.ref)
|
|
@@ -1433,6 +1850,102 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1433
1850
|
}
|
|
1434
1851
|
}
|
|
1435
1852
|
|
|
1853
|
+
/**
|
|
1854
|
+
* The element a click or hover means, and where on it to aim. A ref names
|
|
1855
|
+
* it directly; a role with a name, or visible text, is looked up and
|
|
1856
|
+
* refused unless exactly one element matches. A container (a section, a
|
|
1857
|
+
* list item holding a menu title) is not clicked in its middle, which is
|
|
1858
|
+
* often empty: the aim goes to its first control or to its own text.
|
|
1859
|
+
*/
|
|
1860
|
+
private async reach(
|
|
1861
|
+
tab: Tab,
|
|
1862
|
+
target: BrowserTarget,
|
|
1863
|
+
): Promise<{ locator: Locator; position?: { x: number; y: number } }> {
|
|
1864
|
+
const found =
|
|
1865
|
+
target.ref !== undefined
|
|
1866
|
+
? await this.resolveRef(tab, target.ref)
|
|
1867
|
+
: await this.findTarget(tab, target)
|
|
1868
|
+
let aim: Aim = { kind: 'centre' }
|
|
1869
|
+
try {
|
|
1870
|
+
aim = await found.evaluate(aimInside)
|
|
1871
|
+
} catch {
|
|
1872
|
+
// Aim at the middle, as before.
|
|
1873
|
+
}
|
|
1874
|
+
if (aim.kind === 'descendant') return { locator: found.locator(`xpath=${aim.path}`) }
|
|
1875
|
+
if (aim.kind === 'text') return { locator: found, position: { x: aim.x, y: aim.y } }
|
|
1876
|
+
return { locator: found }
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
/** Find the one element a role/name or text target means, or refuse with the candidates. */
|
|
1880
|
+
private async findTarget(tab: Tab, target: BrowserTarget): Promise<Locator> {
|
|
1881
|
+
const page = tab.page
|
|
1882
|
+
const role = target.role
|
|
1883
|
+
const wanted =
|
|
1884
|
+
role !== undefined ? `${role} "${target.name ?? ''}"` : `text "${target.text ?? ''}"`
|
|
1885
|
+
const build = (exact: boolean): Locator =>
|
|
1886
|
+
role !== undefined
|
|
1887
|
+
? page.getByRole(role as Parameters<Page['getByRole']>[0], {
|
|
1888
|
+
name: target.name ?? '',
|
|
1889
|
+
exact,
|
|
1890
|
+
})
|
|
1891
|
+
: page.getByText(target.text ?? '', { exact }).filter({ visible: true })
|
|
1892
|
+
let locator = build(false)
|
|
1893
|
+
let count = await locator.count()
|
|
1894
|
+
if (count > 1) {
|
|
1895
|
+
// A short name also fits longer ones ("Save" in "Save draft"): the exact matches decide.
|
|
1896
|
+
const exact = build(true)
|
|
1897
|
+
const exactCount = await exact.count()
|
|
1898
|
+
if (exactCount > 0) {
|
|
1899
|
+
locator = exact
|
|
1900
|
+
count = exactCount
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
if (count === 1) return locator
|
|
1904
|
+
const pool =
|
|
1905
|
+
count > 0 ? locator : role !== undefined ? page.getByRole(role as never) : undefined
|
|
1906
|
+
const candidates = []
|
|
1907
|
+
if (pool) {
|
|
1908
|
+
const total = Math.min(await pool.count(), CANDIDATES_MAX)
|
|
1909
|
+
for (let i = 0; i < total; i++) {
|
|
1910
|
+
candidates.push(await pool.nth(i).evaluate(describeCandidate))
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
throw new BrowserTargetUnclearError(wanted, count, candidates)
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
/**
|
|
1917
|
+
* Refuse a key sent to the page when the focus is on a credential, or
|
|
1918
|
+
* when it cannot be told. Every frame is asked, because the key goes to
|
|
1919
|
+
* whichever one holds the focus and a stale focus in another frame is
|
|
1920
|
+
* still a reason to be careful.
|
|
1921
|
+
*/
|
|
1922
|
+
private async guardFocus(tab: Tab, key: NamedKey): Promise<void> {
|
|
1923
|
+
for (const frame of tab.page.frames()) {
|
|
1924
|
+
const handle = await frame.evaluateHandle(deepFocus).catch(() => undefined)
|
|
1925
|
+
const element = handle?.asElement() ?? null
|
|
1926
|
+
if (!element) continue
|
|
1927
|
+
if (keyCanCommit(key) && (await element.evaluate(focusIsOpaque))) {
|
|
1928
|
+
throw new Error(
|
|
1929
|
+
`The focus is on a component whose inside cannot be read, so ${key} could reach a password or code field. Use browser_act with the ref of a specific control.`,
|
|
1930
|
+
)
|
|
1931
|
+
}
|
|
1932
|
+
await this.guardField(element)
|
|
1933
|
+
await this.guardCredentialForm(element, 'press', key)
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
/** A check Playwright runs without acting (`trial`) failed: nothing on the page changed. */
|
|
1938
|
+
private async noEffectOnFailure(
|
|
1939
|
+
action: BrowserActAction['action'],
|
|
1940
|
+
run: () => Promise<unknown>,
|
|
1941
|
+
): Promise<void> {
|
|
1942
|
+
try {
|
|
1943
|
+
await run()
|
|
1944
|
+
} catch (error) {
|
|
1945
|
+
throw new BrowserNoEffectError(action, noEffectReason(error))
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1436
1949
|
/**
|
|
1437
1950
|
* Run the change. A failure once it has started is `outcome unknown`. A
|
|
1438
1951
|
* dialog the action opens ends the wait: the page is paused until the
|
|
@@ -1440,9 +1953,12 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1440
1953
|
*/
|
|
1441
1954
|
private async unknownOnFailure(
|
|
1442
1955
|
action: BrowserActAction['action'],
|
|
1443
|
-
run: () => Promise<unknown>,
|
|
1956
|
+
run: (begun: () => void) => Promise<unknown>,
|
|
1444
1957
|
): Promise<void> {
|
|
1445
|
-
|
|
1958
|
+
let touched = false
|
|
1959
|
+
const running = run(() => {
|
|
1960
|
+
touched = true
|
|
1961
|
+
})
|
|
1446
1962
|
let timer: ReturnType<typeof setInterval> | undefined
|
|
1447
1963
|
const dialogOpened = new Promise<'dialog'>((resolve) => {
|
|
1448
1964
|
timer = setInterval(() => {
|
|
@@ -1453,7 +1969,10 @@ export class PlaywrightBrowserHost implements BrowserHost {
|
|
|
1453
1969
|
const outcome = await Promise.race([running, dialogOpened])
|
|
1454
1970
|
if (outcome === 'dialog') running.catch(() => undefined)
|
|
1455
1971
|
} catch (error) {
|
|
1456
|
-
|
|
1972
|
+
// A timeout whose log never reached the input step left the page alone.
|
|
1973
|
+
throw !touched && failedBeforeActing(error)
|
|
1974
|
+
? new BrowserNoEffectError(action, noEffectReason(error))
|
|
1975
|
+
: new BrowserOutcomeUnknownError(action, error)
|
|
1457
1976
|
} finally {
|
|
1458
1977
|
clearInterval(timer)
|
|
1459
1978
|
}
|