@namzu/browser 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/classifier.d.ts +22 -1
- package/dist/classifier.d.ts.map +1 -1
- package/dist/classifier.js +30 -7
- package/dist/classifier.js.map +1 -1
- package/dist/errors.d.ts +19 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +26 -0
- package/dist/errors.js.map +1 -1
- package/dist/host.d.ts +42 -0
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +487 -70
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/page-scripts.d.ts +20 -0
- package/dist/page-scripts.d.ts.map +1 -1
- package/dist/page-scripts.js +110 -0
- package/dist/page-scripts.js.map +1 -1
- package/dist/reach.d.ts +66 -0
- package/dist/reach.d.ts.map +1 -0
- package/dist/reach.js +189 -0
- package/dist/reach.js.map +1 -0
- package/dist/settle.d.ts +40 -0
- package/dist/settle.d.ts.map +1 -0
- package/dist/settle.js +30 -0
- package/dist/settle.js.map +1 -0
- package/dist/snapshot.d.ts +80 -2
- package/dist/snapshot.d.ts.map +1 -1
- package/dist/snapshot.js +326 -20
- package/dist/snapshot.js.map +1 -1
- package/dist/windows-engine.d.ts +1 -1
- package/dist/windows-engine.d.ts.map +1 -1
- package/dist/windows-engine.js +3 -1
- package/dist/windows-engine.js.map +1 -1
- package/package.json +3 -2
- package/src/classifier.ts +44 -7
- package/src/errors.ts +32 -0
- package/src/host.ts +608 -89
- package/src/index.ts +4 -0
- package/src/page-scripts.ts +108 -0
- package/src/reach.ts +211 -0
- package/src/settle.ts +60 -0
- package/src/snapshot.ts +402 -20
- package/src/windows-engine.ts +4 -1
package/dist/host.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto';
|
|
2
|
-
import { canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome } from '@namzu/sdk';
|
|
3
|
-
import {
|
|
4
|
-
import { classifyHumanRequired, isCredentialField, } from './classifier.js';
|
|
2
|
+
import { BROWSER_FIND_MAX_MATCHES, canonicalizeBrowserOrigin, canonicalizeBrowserUrl, resolveNamzuHome, } from '@namzu/sdk';
|
|
3
|
+
import { classifyHumanRequired, isCredentialField, signInNotice, } from './classifier.js';
|
|
5
4
|
import { detectBrowserEnvironment, runnableBrowserPlan, } from './detect.js';
|
|
6
|
-
import { BrowserHumanRequiredError, BrowserOriginMismatchError, BrowserOutcomeUnknownError, BrowserSiteDeniedError, BrowserStaleRefError, BrowserUnavailableError, ProfileBusyError, } from './errors.js';
|
|
7
|
-
import { countCredentialFields, fieldFacts } from './page-scripts.js';
|
|
5
|
+
import { BrowserHumanRequiredError, BrowserNoEffectError, BrowserOriginMismatchError, BrowserOutcomeUnknownError, BrowserSiteDeniedError, BrowserStaleRefError, BrowserTargetUnclearError, BrowserUnavailableError, ProfileBusyError, } from './errors.js';
|
|
6
|
+
import { countCredentialFields, fieldFacts, hasVisibleContent, installMutationClock, msSinceLastChange, valueFacts, } from './page-scripts.js';
|
|
8
7
|
import { BrowserSitePolicy, DEFAULT_BROWSER_SITE_RULES } from './policy.js';
|
|
9
8
|
import { BrowserLeaseStore, BrowserProfileError, BrowserProfileStore, DEFAULT_BROWSER_PROFILE, } from './profiles.js';
|
|
10
|
-
import {
|
|
9
|
+
import { CANDIDATES_MAX, KEY_DELAY_MS, NAMED_KEYS, aimInside, deepFocus, describeCandidate, failedBeforeActing, focusIsOpaque, keyCanCommit, namedKey, noEffectReason, } from './reach.js';
|
|
10
|
+
import { waitUntilQuiet } from './settle.js';
|
|
11
|
+
import { PLAYWRIGHT_CORE_VERSION, SnapshotPager, buildEffect, captureAriaTree, diffLines, findInTree, locateRef, renderAriaTree, renderInteractive, renderReading, subtreeOf, } from './snapshot.js';
|
|
11
12
|
import { WindowsBridgeError } from './windows-bridge.js';
|
|
12
13
|
import { connectWindowsBrowser } from './windows-engine.js';
|
|
13
14
|
const DEFAULT_NAVIGATION_TIMEOUT_MS = 30_000;
|
|
@@ -15,12 +16,38 @@ const DEFAULT_ACTION_TIMEOUT_MS = 10_000;
|
|
|
15
16
|
/** How long a history move waits for the document after it committed. */
|
|
16
17
|
const HISTORY_LOAD_WAIT_MS = 5_000;
|
|
17
18
|
const SNAPSHOT_PAGE_MAX = 20_000;
|
|
18
|
-
/**
|
|
19
|
-
|
|
19
|
+
/**
|
|
20
|
+
* A page is quiet after this long with no change to the document and no
|
|
21
|
+
* request in flight. It also bounds how long an event the browser has not
|
|
22
|
+
* announced yet (a popup, a download, a dialog) has to arrive.
|
|
23
|
+
*/
|
|
24
|
+
const QUIET_MS = 250;
|
|
25
|
+
/** The most an action waits for quiet. A page that never settles is reported, not waited on. */
|
|
26
|
+
const ACT_SETTLE_CAP_MS = 2_000;
|
|
27
|
+
/** The same for a navigation, which loads more. */
|
|
28
|
+
const NAVIGATE_SETTLE_CAP_MS = 5_000;
|
|
29
|
+
/** The wait before an action, for events left over from the last one. */
|
|
30
|
+
const PRE_ACT_QUIET_MS = 100;
|
|
31
|
+
const PRE_ACT_CAP_MS = 500;
|
|
32
|
+
/** `wait_for` with `stable` and no time: how long it waits. */
|
|
33
|
+
const STABLE_DEFAULT_WAIT_MS = 5_000;
|
|
34
|
+
/** Request kinds that can change what the page shows. */
|
|
35
|
+
const PAGE_REQUEST_KINDS = new Set(['document', 'script', 'stylesheet', 'fetch', 'xhr']);
|
|
36
|
+
/** Lines of a `delta` snapshot that are listed in full. */
|
|
37
|
+
const DELTA_LINES_SHOWN = 80;
|
|
38
|
+
/** Lines of an effect the model is shown. */
|
|
39
|
+
const EFFECT_LINES_SHOWN = 10;
|
|
20
40
|
/** Smallest frame that can hold a CAPTCHA a person is meant to solve. */
|
|
21
41
|
const CAPTCHA_FRAME_MIN_PX = 30;
|
|
22
42
|
const REMEMBERED_DOCUMENTS_MAX = 64;
|
|
43
|
+
function evaluateOn(target, fn) {
|
|
44
|
+
return 'asElement' in target
|
|
45
|
+
? target.evaluate(fn)
|
|
46
|
+
: target.evaluate(fn);
|
|
47
|
+
}
|
|
23
48
|
const SCREENSHOT_INPUT_MASK_COLOR = '#666666';
|
|
49
|
+
/** Most inputs one screenshot checks for a credential. */
|
|
50
|
+
const MASK_CANDIDATES_MAX = 200;
|
|
24
51
|
function firstLine(error) {
|
|
25
52
|
const text = error instanceof Error ? error.message : String(error);
|
|
26
53
|
return (text.split('\n')[0] ?? '').slice(0, 300);
|
|
@@ -156,6 +183,8 @@ export class PlaywrightBrowserHost {
|
|
|
156
183
|
this.context = context;
|
|
157
184
|
context.setDefaultTimeout(this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
|
|
158
185
|
context.setDefaultNavigationTimeout(this.options.navigationTimeoutMs ?? DEFAULT_NAVIGATION_TIMEOUT_MS);
|
|
186
|
+
// Every new document keeps a clock of its last change, for the quiet-page wait.
|
|
187
|
+
await context.addInitScript(installMutationClock).catch(() => undefined);
|
|
159
188
|
context.on('page', (page) => this.adopt(page));
|
|
160
189
|
context.on('close', () => this.forget(context));
|
|
161
190
|
context.browser()?.on('disconnected', () => this.forget(context));
|
|
@@ -163,8 +192,10 @@ export class PlaywrightBrowserHost {
|
|
|
163
192
|
// a site the rules do not allow is stopped here, so its address (and
|
|
164
193
|
// anything a page smuggled into it) never reaches the network.
|
|
165
194
|
await context.route(() => true, (route) => this.screen(route));
|
|
166
|
-
for (const page of context.pages())
|
|
195
|
+
for (const page of context.pages()) {
|
|
167
196
|
this.adopt(page, false);
|
|
197
|
+
void page.evaluate(installMutationClock).catch(() => undefined);
|
|
198
|
+
}
|
|
168
199
|
return context;
|
|
169
200
|
}
|
|
170
201
|
home() {
|
|
@@ -179,6 +210,10 @@ export class PlaywrightBrowserHost {
|
|
|
179
210
|
});
|
|
180
211
|
let context;
|
|
181
212
|
try {
|
|
213
|
+
// Loaded here, on the first browser launch: playwright-core is several
|
|
214
|
+
// megabytes of source that a session which never opens a browser must
|
|
215
|
+
// not compile at startup.
|
|
216
|
+
const { chromium } = await import('playwright-core');
|
|
182
217
|
context = await chromium.launchPersistentContext(descriptor.userDataDir, {
|
|
183
218
|
headless: plan.headless,
|
|
184
219
|
...(plan.channel ? { channel: plan.channel } : {}),
|
|
@@ -338,6 +373,8 @@ export class PlaywrightBrowserHost {
|
|
|
338
373
|
mainFrameCommits: 0,
|
|
339
374
|
mainDocumentResponses: 0,
|
|
340
375
|
responsesByLoader: new Map(),
|
|
376
|
+
inflight: new Set(),
|
|
377
|
+
lastNetworkAt: Date.now(),
|
|
341
378
|
};
|
|
342
379
|
this.tabs.set(tab.id, tab);
|
|
343
380
|
this.tabOf.set(page, tab);
|
|
@@ -366,6 +403,19 @@ export class PlaywrightBrowserHost {
|
|
|
366
403
|
// A response without a frame (a service worker's) says nothing about the page.
|
|
367
404
|
}
|
|
368
405
|
});
|
|
406
|
+
const kind = (request) => PAGE_REQUEST_KINDS.has(request.resourceType());
|
|
407
|
+
page.on('request', (request) => {
|
|
408
|
+
if (!kind(request))
|
|
409
|
+
return;
|
|
410
|
+
tab.inflight.add(request);
|
|
411
|
+
tab.lastNetworkAt = Date.now();
|
|
412
|
+
});
|
|
413
|
+
const ended = (request) => {
|
|
414
|
+
if (tab.inflight.delete(request))
|
|
415
|
+
tab.lastNetworkAt = Date.now();
|
|
416
|
+
};
|
|
417
|
+
page.on('requestfinished', ended);
|
|
418
|
+
page.on('requestfailed', ended);
|
|
369
419
|
page.on('dialog', (dialog) => this.onDialog(tab, dialog));
|
|
370
420
|
page.on('download', (download) => {
|
|
371
421
|
const name = download.suggestedFilename();
|
|
@@ -435,10 +485,37 @@ export class PlaywrightBrowserHost {
|
|
|
435
485
|
.finally(() => this.blanking.delete(blank));
|
|
436
486
|
this.blanking.add(blank);
|
|
437
487
|
}
|
|
438
|
-
|
|
439
|
-
|
|
488
|
+
/**
|
|
489
|
+
* Wait until the tab's page is quiet (nothing changed in it and no request
|
|
490
|
+
* in flight for `quietMs`), or `capMs` has passed. Events the browser
|
|
491
|
+
* announces late (a popup, a download, a dialog) arrive inside the window.
|
|
492
|
+
* Then let any tab that was cleared to `about:blank` finish clearing.
|
|
493
|
+
*/
|
|
494
|
+
async settle(tab, options = {
|
|
495
|
+
quietMs: QUIET_MS,
|
|
496
|
+
capMs: ACT_SETTLE_CAP_MS,
|
|
497
|
+
}) {
|
|
498
|
+
const outcome = await waitUntilQuiet({ now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }, this.quietProbe(tab), options);
|
|
440
499
|
while (this.blanking.size > 0)
|
|
441
500
|
await Promise.all([...this.blanking]);
|
|
501
|
+
return outcome;
|
|
502
|
+
}
|
|
503
|
+
quietProbe(tab) {
|
|
504
|
+
return {
|
|
505
|
+
stillForMs: async () => {
|
|
506
|
+
// A dialog pauses the page; nothing more will happen until it is answered.
|
|
507
|
+
if (this.pendingDialog?.tab === tab || tab.page.isClosed())
|
|
508
|
+
return Number.MAX_SAFE_INTEGER;
|
|
509
|
+
if (tab.inflight.size > 0)
|
|
510
|
+
return 0;
|
|
511
|
+
const sinceNetwork = Date.now() - tab.lastNetworkAt;
|
|
512
|
+
const sincePage = await Promise.race([
|
|
513
|
+
tab.page.evaluate(msSinceLastChange).catch(() => 0),
|
|
514
|
+
new Promise((resolve) => setTimeout(() => resolve(0), 1000)),
|
|
515
|
+
]);
|
|
516
|
+
return Math.min(sinceNetwork, sincePage);
|
|
517
|
+
},
|
|
518
|
+
};
|
|
442
519
|
}
|
|
443
520
|
async activeTab() {
|
|
444
521
|
const context = await this.ensureContext();
|
|
@@ -527,19 +604,29 @@ export class PlaywrightBrowserHost {
|
|
|
527
604
|
// A frame that went away while we looked is not a challenge.
|
|
528
605
|
}
|
|
529
606
|
}
|
|
530
|
-
const
|
|
607
|
+
const signals = {
|
|
531
608
|
url,
|
|
532
609
|
title: await this.title(tab),
|
|
533
610
|
...(tab.status !== undefined ? { status: tab.status } : {}),
|
|
534
611
|
...(tab.wwwAuthenticate !== undefined ? { wwwAuthenticate: tab.wwwAuthenticate } : {}),
|
|
535
|
-
...(tab.proxyAuthenticate !== undefined
|
|
536
|
-
? { proxyAuthenticate: tab.proxyAuthenticate }
|
|
537
|
-
: {}),
|
|
612
|
+
...(tab.proxyAuthenticate !== undefined ? { proxyAuthenticate: tab.proxyAuthenticate } : {}),
|
|
538
613
|
...counts,
|
|
539
614
|
frameUrls,
|
|
540
|
-
}
|
|
615
|
+
};
|
|
616
|
+
const classifier = {
|
|
617
|
+
...(this.options.signInAddresses ? { signInAddresses: this.options.signInAddresses } : {}),
|
|
618
|
+
...(this.options.signInDetection ? { signInDetection: this.options.signInDetection } : {}),
|
|
619
|
+
};
|
|
620
|
+
const reason = classifyHumanRequired(signals, classifier);
|
|
541
621
|
if (reason)
|
|
542
622
|
throw this.humanRequired(reason, tab);
|
|
623
|
+
// A page with a password field that did not stop the turn: say so once
|
|
624
|
+
// per address, so the model reports it instead of trying to type.
|
|
625
|
+
const notice = signInNotice(signals, classifier, this.loginCommandFor(url));
|
|
626
|
+
if (notice && tab.signInNoticeFor !== withoutQuery(url)) {
|
|
627
|
+
tab.signInNoticeFor = withoutQuery(url);
|
|
628
|
+
this.notes.push(notice);
|
|
629
|
+
}
|
|
543
630
|
if (tab.status === 401) {
|
|
544
631
|
throw new Error('The site returned HTTP 401 without a WWW-Authenticate challenge. Access was denied or the response is incomplete; no HTTP password prompt was identified. Tell the user the site refused this request.');
|
|
545
632
|
}
|
|
@@ -676,7 +763,7 @@ export class PlaywrightBrowserHost {
|
|
|
676
763
|
case 'reload':
|
|
677
764
|
return this.history(action.action);
|
|
678
765
|
case 'snapshot':
|
|
679
|
-
return this.snapshot(action
|
|
766
|
+
return this.snapshot(action);
|
|
680
767
|
case 'screenshot':
|
|
681
768
|
return this.screenshot(action.ref, action.fullPage);
|
|
682
769
|
case 'scroll':
|
|
@@ -685,6 +772,8 @@ export class PlaywrightBrowserHost {
|
|
|
685
772
|
return this.waitFor(action);
|
|
686
773
|
case 'tabs':
|
|
687
774
|
return this.tabsCall(action);
|
|
775
|
+
case 'value':
|
|
776
|
+
return this.readValue(action.ref);
|
|
688
777
|
}
|
|
689
778
|
}
|
|
690
779
|
async navigate(tab, rawUrl) {
|
|
@@ -705,7 +794,7 @@ export class PlaywrightBrowserHost {
|
|
|
705
794
|
catch (error) {
|
|
706
795
|
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined);
|
|
707
796
|
if (/ERR_UNEXPECTED_PROXY_AUTH/.test(firstLine(error))) {
|
|
708
|
-
await this.settle();
|
|
797
|
+
await this.settle(tab);
|
|
709
798
|
throw new Error('The browser refused an HTTP 407 proxy response before the page loaded. Proxy access is unavailable or its response is invalid; no usable password prompt was identified. Tell the user the proxy refused this request.');
|
|
710
799
|
}
|
|
711
800
|
if (!/ERR_BLOCKED_BY_CLIENT|ERR_ABORTED|Download is starting/.test(firstLine(error))) {
|
|
@@ -716,10 +805,35 @@ export class PlaywrightBrowserHost {
|
|
|
716
805
|
return this.afterNavigation(tab);
|
|
717
806
|
}
|
|
718
807
|
async afterNavigation(tab) {
|
|
719
|
-
|
|
808
|
+
// The document first (a page that has not finished loading has not
|
|
809
|
+
// finished building itself), then the same quiet the actions wait for.
|
|
810
|
+
await tab.page
|
|
811
|
+
.waitForLoadState('load', { timeout: NAVIGATE_SETTLE_CAP_MS })
|
|
812
|
+
.catch(() => undefined);
|
|
813
|
+
const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: NAVIGATE_SETTLE_CAP_MS });
|
|
720
814
|
this.checkLanding(tab);
|
|
721
|
-
|
|
815
|
+
while (this.blanking.size > 0)
|
|
816
|
+
await Promise.all([...this.blanking]);
|
|
722
817
|
await this.checkHuman(tab);
|
|
818
|
+
if (!settled.quiet) {
|
|
819
|
+
this.notes.push(`The page was still changing after ${settled.waitedMs} ms, so what it shows may be unfinished. Call browser with wait_for and stable, or take a snapshot.`);
|
|
820
|
+
}
|
|
821
|
+
const populated = originOf(tab.page.url()) === 'null' ||
|
|
822
|
+
(await tab.page.evaluate(hasVisibleContent).catch(() => true));
|
|
823
|
+
if (!populated) {
|
|
824
|
+
this.notes.push(`The page has nothing to read yet (waited ${settled.waitedMs} ms). Call browser with wait_for and stable, or reload.`);
|
|
825
|
+
}
|
|
826
|
+
// The model's next step after opening a page is to read it, so the
|
|
827
|
+
// reading comes with the open: one request fewer on every page visit.
|
|
828
|
+
if (populated && originOf(tab.page.url()) !== 'null') {
|
|
829
|
+
try {
|
|
830
|
+
const snapshot = await this.takeSnapshot(tab);
|
|
831
|
+
return this.result({ page: snapshot.page, snapshot });
|
|
832
|
+
}
|
|
833
|
+
catch {
|
|
834
|
+
// An unreadable page still opened; say where we are and let the model ask again.
|
|
835
|
+
}
|
|
836
|
+
}
|
|
723
837
|
return this.result({ page: await this.pageInfo(tab) });
|
|
724
838
|
}
|
|
725
839
|
async history(which) {
|
|
@@ -768,7 +882,16 @@ export class PlaywrightBrowserHost {
|
|
|
768
882
|
await pending.dialog.dismiss().catch(() => undefined);
|
|
769
883
|
this.notes.push('The open dialog was dismissed.');
|
|
770
884
|
}
|
|
771
|
-
|
|
885
|
+
/** Read the page's tree and remember its refs. */
|
|
886
|
+
async readTree(tab) {
|
|
887
|
+
const { nodes, hidden, redact } = await captureAriaTree(tab.page, this.options.actionTimeoutMs ?? DEFAULT_ACTION_TIMEOUT_MS);
|
|
888
|
+
const whole = renderAriaTree(nodes, hidden, undefined, redact);
|
|
889
|
+
if (!whole)
|
|
890
|
+
throw new Error('The page could not be read.');
|
|
891
|
+
this.refs = whole.refs;
|
|
892
|
+
return { nodes, hidden, redact, whole };
|
|
893
|
+
}
|
|
894
|
+
async takeSnapshot(tab, request = {}) {
|
|
772
895
|
const pending = this.pendingDialog;
|
|
773
896
|
if (pending && pending.tab === tab) {
|
|
774
897
|
const dialog = pending.dialog;
|
|
@@ -780,17 +903,67 @@ export class PlaywrightBrowserHost {
|
|
|
780
903
|
lines.push(' - (the page is paused until the dialog is answered)');
|
|
781
904
|
return { page: await this.pageInfo(tab), text: lines.join('\n') };
|
|
782
905
|
}
|
|
783
|
-
const {
|
|
784
|
-
const
|
|
785
|
-
if (!whole)
|
|
786
|
-
throw new
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
906
|
+
const { ref, view = 'full', query, role } = request;
|
|
907
|
+
const { nodes, hidden, redact, whole } = await this.readTree(tab);
|
|
908
|
+
if (ref !== undefined && !whole.refs.has(ref))
|
|
909
|
+
throw new BrowserStaleRefError(ref);
|
|
910
|
+
// A region is cut from the whole-page tree: a snapshot of one element would replace the ref table.
|
|
911
|
+
const scope = ref === undefined ? nodes : (subtreeOf(nodes, ref) ?? []);
|
|
912
|
+
let text;
|
|
913
|
+
if (query !== undefined || role !== undefined) {
|
|
914
|
+
const found = findInTree(scope, hidden, redact, { ...(query !== undefined ? { query } : {}), ...(role !== undefined ? { role } : {}) }, BROWSER_FIND_MAX_MATCHES);
|
|
915
|
+
text = found.lines.join('\n');
|
|
916
|
+
const what = [
|
|
917
|
+
query !== undefined ? `text ${quoteShort(query, 60)}` : '',
|
|
918
|
+
role !== undefined ? `role ${quoteShort(role, 30)}` : '',
|
|
919
|
+
]
|
|
920
|
+
.filter(Boolean)
|
|
921
|
+
.join(' and ');
|
|
922
|
+
this.notes.unshift(found.total === 0
|
|
923
|
+
? `Nothing on the page matches ${what}.`
|
|
924
|
+
: found.total > found.lines.length
|
|
925
|
+
? `${found.total} elements match ${what}; the first ${found.lines.length} are listed. Narrow the search.`
|
|
926
|
+
: `${found.total} element${found.total === 1 ? '' : 's'} match${found.total === 1 ? 'es' : ''} ${what}.`);
|
|
927
|
+
}
|
|
928
|
+
else if (view === 'interactive') {
|
|
929
|
+
text = renderInteractive(scope, hidden, redact);
|
|
930
|
+
tab.lastSeen = whole.text;
|
|
931
|
+
}
|
|
932
|
+
else if (view === 'reading') {
|
|
933
|
+
text = renderReading(scope, hidden, redact);
|
|
934
|
+
tab.lastSeen = whole.text;
|
|
935
|
+
}
|
|
936
|
+
else if (view === 'delta') {
|
|
937
|
+
const before = tab.lastSeen;
|
|
938
|
+
if (before === undefined) {
|
|
939
|
+
this.notes.unshift('There is no earlier snapshot of this tab to compare with; this is the whole page.');
|
|
940
|
+
text =
|
|
941
|
+
ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '');
|
|
942
|
+
}
|
|
943
|
+
else {
|
|
944
|
+
const diff = diffLines(before, whole.text, DELTA_LINES_SHOWN);
|
|
945
|
+
text =
|
|
946
|
+
diff.added === 0 && diff.removed === 0
|
|
947
|
+
? '(no change since the last snapshot)'
|
|
948
|
+
: [
|
|
949
|
+
...diff.addedLines.map((l) => `+ ${l}`),
|
|
950
|
+
...(diff.added > diff.addedLines.length
|
|
951
|
+
? [` (${diff.added - diff.addedLines.length} more lines added)`]
|
|
952
|
+
: []),
|
|
953
|
+
...diff.removedLines.map((l) => `- ${l}`),
|
|
954
|
+
...(diff.removed > diff.removedLines.length
|
|
955
|
+
? [` (${diff.removed - diff.removedLines.length} more lines removed)`]
|
|
956
|
+
: []),
|
|
957
|
+
].join('\n');
|
|
958
|
+
this.notes.unshift(`Changes since the last snapshot: ${diff.added} lines added, ${diff.removed} removed. Take a full snapshot to see the whole page again.`);
|
|
959
|
+
}
|
|
960
|
+
tab.lastSeen = whole.text;
|
|
961
|
+
}
|
|
962
|
+
else {
|
|
963
|
+
text =
|
|
964
|
+
ref === undefined ? whole.text : (renderAriaTree(nodes, hidden, ref, redact)?.text ?? '');
|
|
965
|
+
if (ref === undefined)
|
|
966
|
+
tab.lastSeen = whole.text;
|
|
794
967
|
}
|
|
795
968
|
if (text.trim() === '')
|
|
796
969
|
text = '(nothing readable on this page)';
|
|
@@ -801,10 +974,10 @@ export class PlaywrightBrowserHost {
|
|
|
801
974
|
...(first.nextCursor !== undefined ? { nextCursor: first.nextCursor } : {}),
|
|
802
975
|
};
|
|
803
976
|
}
|
|
804
|
-
async snapshot(
|
|
977
|
+
async snapshot(request) {
|
|
805
978
|
const tab = await this.activeTab();
|
|
806
|
-
if (cursor !== undefined) {
|
|
807
|
-
const next = this.pager.resume(cursor);
|
|
979
|
+
if (request.cursor !== undefined) {
|
|
980
|
+
const next = this.pager.resume(request.cursor);
|
|
808
981
|
if (!next) {
|
|
809
982
|
throw new Error('That cursor belongs to an older snapshot. Take a new snapshot and page through it.');
|
|
810
983
|
}
|
|
@@ -816,8 +989,15 @@ export class PlaywrightBrowserHost {
|
|
|
816
989
|
},
|
|
817
990
|
});
|
|
818
991
|
}
|
|
819
|
-
await this.settle();
|
|
820
|
-
return this.result({
|
|
992
|
+
await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS });
|
|
993
|
+
return this.result({
|
|
994
|
+
snapshot: await this.takeSnapshot(tab, {
|
|
995
|
+
...(request.ref !== undefined ? { ref: request.ref } : {}),
|
|
996
|
+
...(request.view !== undefined ? { view: request.view } : {}),
|
|
997
|
+
...(request.query !== undefined ? { query: request.query } : {}),
|
|
998
|
+
...(request.role !== undefined ? { role: request.role } : {}),
|
|
999
|
+
}),
|
|
1000
|
+
});
|
|
821
1001
|
}
|
|
822
1002
|
async resolveRef(tab, ref) {
|
|
823
1003
|
if (!this.refs.has(ref))
|
|
@@ -839,10 +1019,12 @@ export class PlaywrightBrowserHost {
|
|
|
839
1019
|
if (this.pendingDialog?.tab === tab) {
|
|
840
1020
|
throw new Error('A dialog is open; dismiss it with browser_act dialog before taking a screenshot.');
|
|
841
1021
|
}
|
|
842
|
-
// Screenshots are image blocks sent to the model. Mask
|
|
843
|
-
//
|
|
844
|
-
//
|
|
845
|
-
|
|
1022
|
+
// Screenshots are image blocks sent to the model. Mask the fields that
|
|
1023
|
+
// take a password or a one-time code, in every frame; the other inputs
|
|
1024
|
+
// stay visible, because a search box or an address field the model
|
|
1025
|
+
// cannot read cannot be checked. Closed shadow roots and page-drawn
|
|
1026
|
+
// text remain outside this mask.
|
|
1027
|
+
const mask = await this.credentialMask(tab);
|
|
846
1028
|
const data = ref !== undefined
|
|
847
1029
|
? await (await this.resolveRef(tab, ref)).screenshot({
|
|
848
1030
|
type: 'png',
|
|
@@ -867,6 +1049,59 @@ export class PlaywrightBrowserHost {
|
|
|
867
1049
|
},
|
|
868
1050
|
});
|
|
869
1051
|
}
|
|
1052
|
+
/** Locators of the credential fields (password, one-time code) of every frame of the tab. */
|
|
1053
|
+
async credentialMask(tab) {
|
|
1054
|
+
const out = [];
|
|
1055
|
+
for (const frame of tab.page.frames()) {
|
|
1056
|
+
const inputs = frame.locator('input, textarea');
|
|
1057
|
+
let count = 0;
|
|
1058
|
+
try {
|
|
1059
|
+
count = Math.min(await inputs.count(), MASK_CANDIDATES_MAX);
|
|
1060
|
+
}
|
|
1061
|
+
catch {
|
|
1062
|
+
// A frame that went away while we looked has nothing to mask.
|
|
1063
|
+
continue;
|
|
1064
|
+
}
|
|
1065
|
+
const verdicts = await Promise.all(Array.from({ length: count }, async (_, index) => {
|
|
1066
|
+
try {
|
|
1067
|
+
return isCredentialField(await inputs.nth(index).evaluate(fieldFacts, undefined, { timeout: 2000 }));
|
|
1068
|
+
}
|
|
1069
|
+
catch {
|
|
1070
|
+
// A field that cannot be asked about is hidden to be safe.
|
|
1071
|
+
return true;
|
|
1072
|
+
}
|
|
1073
|
+
}));
|
|
1074
|
+
verdicts.forEach((hide, index) => {
|
|
1075
|
+
if (hide)
|
|
1076
|
+
out.push(inputs.nth(index));
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
return out;
|
|
1080
|
+
}
|
|
1081
|
+
/** One element's current value and state, read from the page itself. */
|
|
1082
|
+
async readValue(ref) {
|
|
1083
|
+
const tab = await this.activeTab();
|
|
1084
|
+
if (this.pendingDialog?.tab === tab) {
|
|
1085
|
+
throw new Error('A dialog is open; dismiss it with browser_act dialog before reading a value.');
|
|
1086
|
+
}
|
|
1087
|
+
const locator = await this.resolveRef(tab, ref);
|
|
1088
|
+
const facts = await locator.evaluate(fieldFacts);
|
|
1089
|
+
const described = this.refs.get(ref);
|
|
1090
|
+
const page = await this.pageInfo(tab);
|
|
1091
|
+
const hidden = isCredentialField(facts);
|
|
1092
|
+
const read = hidden ? undefined : await locator.evaluate(valueFacts);
|
|
1093
|
+
const value = {
|
|
1094
|
+
page,
|
|
1095
|
+
ref,
|
|
1096
|
+
...(described?.role !== undefined ? { role: described.role } : {}),
|
|
1097
|
+
...(described?.name !== undefined ? { name: described.name } : {}),
|
|
1098
|
+
...(read?.value !== undefined ? { value: read.value } : {}),
|
|
1099
|
+
state: read?.state ?? [],
|
|
1100
|
+
...(read?.options !== undefined ? { options: read.options } : {}),
|
|
1101
|
+
...(hidden ? { hidden: true } : {}),
|
|
1102
|
+
};
|
|
1103
|
+
return this.result({ value });
|
|
1104
|
+
}
|
|
870
1105
|
async scroll(direction, ref, amount) {
|
|
871
1106
|
const tab = await this.activeTab();
|
|
872
1107
|
const screens = amount ?? 1;
|
|
@@ -913,10 +1148,17 @@ export class PlaywrightBrowserHost {
|
|
|
913
1148
|
: `${quoteShort(text)} was still there after ${timeout} ms.`);
|
|
914
1149
|
}
|
|
915
1150
|
}
|
|
916
|
-
else {
|
|
1151
|
+
else if (action.stable !== true) {
|
|
917
1152
|
await tab.page.waitForTimeout(timeout);
|
|
918
1153
|
this.notes.unshift(`Waited ${timeout} ms.`);
|
|
919
1154
|
}
|
|
1155
|
+
if (action.stable === true) {
|
|
1156
|
+
const cap = action.timeMs !== undefined ? timeout : STABLE_DEFAULT_WAIT_MS;
|
|
1157
|
+
const settled = await this.settle(tab, { quietMs: QUIET_MS, capMs: cap });
|
|
1158
|
+
this.notes.push(settled.quiet
|
|
1159
|
+
? `The page is quiet (nothing changed for ${QUIET_MS} ms; waited ${settled.waitedMs} ms).`
|
|
1160
|
+
: `The page was still changing after ${settled.waitedMs} ms.`);
|
|
1161
|
+
}
|
|
920
1162
|
return this.result({ page: await this.pageInfo(tab) });
|
|
921
1163
|
}
|
|
922
1164
|
async tabsCall(action) {
|
|
@@ -974,15 +1216,15 @@ export class PlaywrightBrowserHost {
|
|
|
974
1216
|
// Acting
|
|
975
1217
|
// -------------------------------------------------------------------------
|
|
976
1218
|
async guardField(locator) {
|
|
977
|
-
const facts = await locator
|
|
1219
|
+
const facts = await evaluateOn(locator, fieldFacts);
|
|
978
1220
|
if (isCredentialField(facts)) {
|
|
979
|
-
throw new Error('This action targets a password or one-time-code control. Ask the operator to use the browser window.');
|
|
1221
|
+
throw new Error('This action targets a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.');
|
|
980
1222
|
}
|
|
981
1223
|
return facts;
|
|
982
1224
|
}
|
|
983
1225
|
/** Refuse direct form submission while a form holds a password or code input. */
|
|
984
1226
|
async guardCredentialForm(locator, action, key) {
|
|
985
|
-
const context = await locator
|
|
1227
|
+
const context = await evaluateOn(locator, (element) => {
|
|
986
1228
|
const label = element.closest('label');
|
|
987
1229
|
const labelledField = label?.control
|
|
988
1230
|
? {
|
|
@@ -1023,16 +1265,62 @@ export class PlaywrightBrowserHost {
|
|
|
1023
1265
|
if ((action === 'click' && context.isFormButton) ||
|
|
1024
1266
|
(action === 'press' && (context.isFormButton || /enter|return/i.test(key ?? ''))) ||
|
|
1025
1267
|
action === 'type-submit') {
|
|
1026
|
-
throw new Error('This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window.');
|
|
1268
|
+
throw new Error('This action could submit a form containing a password or one-time-code control. Ask the operator to use the browser window. Do the other steps of the task that do not need it, then tell the user which step was left and why.');
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
/**
|
|
1272
|
+
* The page's tree as the model could have read it just before an action,
|
|
1273
|
+
* or `undefined` when it cannot be read now (a dialog holds the page, or
|
|
1274
|
+
* the read failed). An effect is a bonus; it never fails the action.
|
|
1275
|
+
*/
|
|
1276
|
+
async effectBaseline(tab) {
|
|
1277
|
+
if (this.pendingDialog?.tab === tab)
|
|
1278
|
+
return undefined;
|
|
1279
|
+
try {
|
|
1280
|
+
return (await this.readTree(tab)).whole.text;
|
|
1281
|
+
}
|
|
1282
|
+
catch {
|
|
1283
|
+
return undefined;
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
/** What the action did, from the tree before and the tree now. */
|
|
1287
|
+
async effectSince(before, urlBefore, tabsBefore, tab, settled) {
|
|
1288
|
+
if (before === undefined || this.pendingDialog?.tab === tab || tab.page.isClosed())
|
|
1289
|
+
return undefined;
|
|
1290
|
+
let after;
|
|
1291
|
+
try {
|
|
1292
|
+
after = (await this.readTree(tab)).whole.text;
|
|
1027
1293
|
}
|
|
1294
|
+
catch {
|
|
1295
|
+
return undefined;
|
|
1296
|
+
}
|
|
1297
|
+
return buildEffect({
|
|
1298
|
+
beforeTree: before,
|
|
1299
|
+
afterTree: after,
|
|
1300
|
+
urlBefore,
|
|
1301
|
+
urlNow: tab.page.url(),
|
|
1302
|
+
tabsBefore,
|
|
1303
|
+
tabsNow: this.tabs.size,
|
|
1304
|
+
settle: { quiet: settled.quiet, waitedMs: settled.waitedMs, capMs: settled.capMs },
|
|
1305
|
+
}, EFFECT_LINES_SHOWN);
|
|
1028
1306
|
}
|
|
1029
1307
|
async runAct(action) {
|
|
1030
1308
|
const tab = await this.activeTab();
|
|
1031
|
-
await this.settle();
|
|
1309
|
+
await this.settle(tab, { quietMs: PRE_ACT_QUIET_MS, capMs: PRE_ACT_CAP_MS });
|
|
1032
1310
|
const expected = canonicalizeBrowserOrigin(action.origin);
|
|
1033
1311
|
const live = originOf(tab.page.url());
|
|
1034
1312
|
if (!expected.ok || expected.origin !== live) {
|
|
1035
|
-
|
|
1313
|
+
// The page moved since the snapshot. Going on is fine only when the
|
|
1314
|
+
// site rules treat the new origin exactly as the named one: both at
|
|
1315
|
+
// `act`, where no review stood between the model and the page.
|
|
1316
|
+
const sameRules = expected.ok &&
|
|
1317
|
+
live !== 'null' &&
|
|
1318
|
+
this.policy.levelOf(expected.origin) === 'act' &&
|
|
1319
|
+
this.policy.levelOf(live) === 'act';
|
|
1320
|
+
if (!sameRules) {
|
|
1321
|
+
throw new BrowserOriginMismatchError(expected.ok ? expected.origin : action.origin, live === 'null' ? 'about:blank' : live);
|
|
1322
|
+
}
|
|
1323
|
+
this.notes.unshift(`The page is now at ${live}, not ${expected.origin}; the site rules treat both alike, so the action went ahead there.`);
|
|
1036
1324
|
}
|
|
1037
1325
|
if (!this.policy.canAct(live)) {
|
|
1038
1326
|
throw new BrowserSiteDeniedError(live, `${live} may be read but not changed under the site rules.`);
|
|
@@ -1043,11 +1331,16 @@ export class PlaywrightBrowserHost {
|
|
|
1043
1331
|
throw new Error('A dialog is open on this tab; dismiss it with browser_act dialog first.');
|
|
1044
1332
|
}
|
|
1045
1333
|
const checkpoint = await this.beginDocumentResponseChange(tab);
|
|
1334
|
+
const urlBefore = tab.page.url();
|
|
1335
|
+
const tabsBefore = this.tabs.size;
|
|
1336
|
+
const treeBefore = await this.effectBaseline(tab);
|
|
1337
|
+
let settled;
|
|
1046
1338
|
try {
|
|
1047
1339
|
await this.perform(tab, action);
|
|
1048
|
-
await this.settle();
|
|
1340
|
+
settled = await this.settle(tab);
|
|
1049
1341
|
this.checkLanding(tab);
|
|
1050
|
-
|
|
1342
|
+
while (this.blanking.size > 0)
|
|
1343
|
+
await Promise.all([...this.blanking]);
|
|
1051
1344
|
}
|
|
1052
1345
|
catch (error) {
|
|
1053
1346
|
await this.finishDocumentResponseChange(tab, checkpoint).catch(() => undefined);
|
|
@@ -1056,9 +1349,14 @@ export class PlaywrightBrowserHost {
|
|
|
1056
1349
|
await this.finishDocumentResponseChange(tab, checkpoint);
|
|
1057
1350
|
await this.checkHuman(tab);
|
|
1058
1351
|
const after = this.tabs.get(tab.id) ?? tab;
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1352
|
+
const effect = await this.effectSince(treeBefore, urlBefore, tabsBefore, after, settled);
|
|
1353
|
+
if (action.snapshot) {
|
|
1354
|
+
return this.result({
|
|
1355
|
+
snapshot: await this.takeSnapshot(after),
|
|
1356
|
+
...(effect ? { effect } : {}),
|
|
1357
|
+
});
|
|
1358
|
+
}
|
|
1359
|
+
return this.result({ page: await this.pageInfo(after), ...(effect ? { effect } : {}) });
|
|
1062
1360
|
}
|
|
1063
1361
|
async answerDialog(tab, action) {
|
|
1064
1362
|
const pending = this.pendingDialog;
|
|
@@ -1082,9 +1380,10 @@ export class PlaywrightBrowserHost {
|
|
|
1082
1380
|
throw new BrowserOutcomeUnknownError('dialog', error);
|
|
1083
1381
|
}
|
|
1084
1382
|
this.notes.unshift('The dialog was dismissed.');
|
|
1085
|
-
await this.settle();
|
|
1383
|
+
await this.settle(tab);
|
|
1086
1384
|
this.checkLanding(tab);
|
|
1087
|
-
|
|
1385
|
+
while (this.blanking.size > 0)
|
|
1386
|
+
await Promise.all([...this.blanking]);
|
|
1088
1387
|
await this.finishDocumentResponseChange(tab, checkpoint);
|
|
1089
1388
|
await this.checkHuman(tab);
|
|
1090
1389
|
if (action.snapshot)
|
|
@@ -1099,18 +1398,20 @@ export class PlaywrightBrowserHost {
|
|
|
1099
1398
|
async perform(tab, action) {
|
|
1100
1399
|
switch (action.action) {
|
|
1101
1400
|
case 'click': {
|
|
1102
|
-
const
|
|
1103
|
-
await this.guardField(locator);
|
|
1104
|
-
await this.guardCredentialForm(locator, 'click');
|
|
1105
|
-
|
|
1106
|
-
await this.
|
|
1401
|
+
const target = await this.reach(tab, action);
|
|
1402
|
+
await this.guardField(target.locator);
|
|
1403
|
+
await this.guardCredentialForm(target.locator, 'click');
|
|
1404
|
+
const at = target.position ? { position: target.position } : {};
|
|
1405
|
+
await this.noEffectOnFailure('click', () => target.locator.click({ trial: true, ...at }));
|
|
1406
|
+
await this.unknownOnFailure('click', () => action.doubleClick ? target.locator.dblclick(at) : target.locator.click(at));
|
|
1107
1407
|
return;
|
|
1108
1408
|
}
|
|
1109
1409
|
case 'hover': {
|
|
1110
|
-
const
|
|
1111
|
-
await this.guardField(locator);
|
|
1112
|
-
|
|
1113
|
-
await this.
|
|
1410
|
+
const target = await this.reach(tab, action);
|
|
1411
|
+
await this.guardField(target.locator);
|
|
1412
|
+
const at = target.position ? { position: target.position } : {};
|
|
1413
|
+
await this.noEffectOnFailure('hover', () => target.locator.hover({ trial: true, ...at }));
|
|
1414
|
+
await this.unknownOnFailure('hover', () => target.locator.hover(at));
|
|
1114
1415
|
return;
|
|
1115
1416
|
}
|
|
1116
1417
|
case 'type': {
|
|
@@ -1118,9 +1419,27 @@ export class PlaywrightBrowserHost {
|
|
|
1118
1419
|
await this.guardField(locator);
|
|
1119
1420
|
if (action.submit)
|
|
1120
1421
|
await this.guardCredentialForm(locator, 'type-submit');
|
|
1121
|
-
await locator.click({ trial: true });
|
|
1122
|
-
await this.unknownOnFailure('type', async () => {
|
|
1123
|
-
|
|
1422
|
+
await this.noEffectOnFailure('type', () => locator.click({ trial: true }));
|
|
1423
|
+
await this.unknownOnFailure('type', async (begun) => {
|
|
1424
|
+
if (action.keys) {
|
|
1425
|
+
await locator.focus();
|
|
1426
|
+
// Clearing and typing change the field; from here a failure is not harmless.
|
|
1427
|
+
begun();
|
|
1428
|
+
if (action.append)
|
|
1429
|
+
await locator.press('Control+End');
|
|
1430
|
+
else
|
|
1431
|
+
await locator.fill('');
|
|
1432
|
+
await locator.pressSequentially(action.text, { delay: KEY_DELAY_MS });
|
|
1433
|
+
}
|
|
1434
|
+
else if (action.append) {
|
|
1435
|
+
const held = await locator
|
|
1436
|
+
.inputValue()
|
|
1437
|
+
.catch(() => locator.evaluate((el) => el.textContent ?? ''));
|
|
1438
|
+
await locator.fill(held + action.text);
|
|
1439
|
+
}
|
|
1440
|
+
else {
|
|
1441
|
+
await locator.fill(action.text);
|
|
1442
|
+
}
|
|
1124
1443
|
if (action.submit)
|
|
1125
1444
|
await locator.press('Enter');
|
|
1126
1445
|
});
|
|
@@ -1166,7 +1485,13 @@ export class PlaywrightBrowserHost {
|
|
|
1166
1485
|
await this.unknownOnFailure('press', () => locator.press(action.key));
|
|
1167
1486
|
return;
|
|
1168
1487
|
}
|
|
1169
|
-
|
|
1488
|
+
const key = namedKey(action.key);
|
|
1489
|
+
if (!key) {
|
|
1490
|
+
throw new Error(`Without a ref, press takes only ${NAMED_KEYS.join(', ')}. Any other key goes to a specific control: give its ref.`);
|
|
1491
|
+
}
|
|
1492
|
+
await this.guardFocus(tab, key);
|
|
1493
|
+
await this.unknownOnFailure('press', () => tab.page.keyboard.press(key));
|
|
1494
|
+
return;
|
|
1170
1495
|
}
|
|
1171
1496
|
case 'upload': {
|
|
1172
1497
|
const locator = await this.resolveRef(tab, action.ref);
|
|
@@ -1191,13 +1516,102 @@ export class PlaywrightBrowserHost {
|
|
|
1191
1516
|
return;
|
|
1192
1517
|
}
|
|
1193
1518
|
}
|
|
1519
|
+
/**
|
|
1520
|
+
* The element a click or hover means, and where on it to aim. A ref names
|
|
1521
|
+
* it directly; a role with a name, or visible text, is looked up and
|
|
1522
|
+
* refused unless exactly one element matches. A container (a section, a
|
|
1523
|
+
* list item holding a menu title) is not clicked in its middle, which is
|
|
1524
|
+
* often empty: the aim goes to its first control or to its own text.
|
|
1525
|
+
*/
|
|
1526
|
+
async reach(tab, target) {
|
|
1527
|
+
const found = target.ref !== undefined
|
|
1528
|
+
? await this.resolveRef(tab, target.ref)
|
|
1529
|
+
: await this.findTarget(tab, target);
|
|
1530
|
+
let aim = { kind: 'centre' };
|
|
1531
|
+
try {
|
|
1532
|
+
aim = await found.evaluate(aimInside);
|
|
1533
|
+
}
|
|
1534
|
+
catch {
|
|
1535
|
+
// Aim at the middle, as before.
|
|
1536
|
+
}
|
|
1537
|
+
if (aim.kind === 'descendant')
|
|
1538
|
+
return { locator: found.locator(`xpath=${aim.path}`) };
|
|
1539
|
+
if (aim.kind === 'text')
|
|
1540
|
+
return { locator: found, position: { x: aim.x, y: aim.y } };
|
|
1541
|
+
return { locator: found };
|
|
1542
|
+
}
|
|
1543
|
+
/** Find the one element a role/name or text target means, or refuse with the candidates. */
|
|
1544
|
+
async findTarget(tab, target) {
|
|
1545
|
+
const page = tab.page;
|
|
1546
|
+
const role = target.role;
|
|
1547
|
+
const wanted = role !== undefined ? `${role} "${target.name ?? ''}"` : `text "${target.text ?? ''}"`;
|
|
1548
|
+
const build = (exact) => role !== undefined
|
|
1549
|
+
? page.getByRole(role, {
|
|
1550
|
+
name: target.name ?? '',
|
|
1551
|
+
exact,
|
|
1552
|
+
})
|
|
1553
|
+
: page.getByText(target.text ?? '', { exact }).filter({ visible: true });
|
|
1554
|
+
let locator = build(false);
|
|
1555
|
+
let count = await locator.count();
|
|
1556
|
+
if (count > 1) {
|
|
1557
|
+
// A short name also fits longer ones ("Save" in "Save draft"): the exact matches decide.
|
|
1558
|
+
const exact = build(true);
|
|
1559
|
+
const exactCount = await exact.count();
|
|
1560
|
+
if (exactCount > 0) {
|
|
1561
|
+
locator = exact;
|
|
1562
|
+
count = exactCount;
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
if (count === 1)
|
|
1566
|
+
return locator;
|
|
1567
|
+
const pool = count > 0 ? locator : role !== undefined ? page.getByRole(role) : undefined;
|
|
1568
|
+
const candidates = [];
|
|
1569
|
+
if (pool) {
|
|
1570
|
+
const total = Math.min(await pool.count(), CANDIDATES_MAX);
|
|
1571
|
+
for (let i = 0; i < total; i++) {
|
|
1572
|
+
candidates.push(await pool.nth(i).evaluate(describeCandidate));
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
throw new BrowserTargetUnclearError(wanted, count, candidates);
|
|
1576
|
+
}
|
|
1577
|
+
/**
|
|
1578
|
+
* Refuse a key sent to the page when the focus is on a credential, or
|
|
1579
|
+
* when it cannot be told. Every frame is asked, because the key goes to
|
|
1580
|
+
* whichever one holds the focus and a stale focus in another frame is
|
|
1581
|
+
* still a reason to be careful.
|
|
1582
|
+
*/
|
|
1583
|
+
async guardFocus(tab, key) {
|
|
1584
|
+
for (const frame of tab.page.frames()) {
|
|
1585
|
+
const handle = await frame.evaluateHandle(deepFocus).catch(() => undefined);
|
|
1586
|
+
const element = handle?.asElement() ?? null;
|
|
1587
|
+
if (!element)
|
|
1588
|
+
continue;
|
|
1589
|
+
if (keyCanCommit(key) && (await element.evaluate(focusIsOpaque))) {
|
|
1590
|
+
throw new Error(`The focus is on a component whose inside cannot be read, so ${key} could reach a password or code field. Use browser_act with the ref of a specific control.`);
|
|
1591
|
+
}
|
|
1592
|
+
await this.guardField(element);
|
|
1593
|
+
await this.guardCredentialForm(element, 'press', key);
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
/** A check Playwright runs without acting (`trial`) failed: nothing on the page changed. */
|
|
1597
|
+
async noEffectOnFailure(action, run) {
|
|
1598
|
+
try {
|
|
1599
|
+
await run();
|
|
1600
|
+
}
|
|
1601
|
+
catch (error) {
|
|
1602
|
+
throw new BrowserNoEffectError(action, noEffectReason(error));
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1194
1605
|
/**
|
|
1195
1606
|
* Run the change. A failure once it has started is `outcome unknown`. A
|
|
1196
1607
|
* dialog the action opens ends the wait: the page is paused until the
|
|
1197
1608
|
* dialog is answered, and so is anything Playwright was still waiting for.
|
|
1198
1609
|
*/
|
|
1199
1610
|
async unknownOnFailure(action, run) {
|
|
1200
|
-
|
|
1611
|
+
let touched = false;
|
|
1612
|
+
const running = run(() => {
|
|
1613
|
+
touched = true;
|
|
1614
|
+
});
|
|
1201
1615
|
let timer;
|
|
1202
1616
|
const dialogOpened = new Promise((resolve) => {
|
|
1203
1617
|
timer = setInterval(() => {
|
|
@@ -1211,7 +1625,10 @@ export class PlaywrightBrowserHost {
|
|
|
1211
1625
|
running.catch(() => undefined);
|
|
1212
1626
|
}
|
|
1213
1627
|
catch (error) {
|
|
1214
|
-
|
|
1628
|
+
// A timeout whose log never reached the input step left the page alone.
|
|
1629
|
+
throw !touched && failedBeforeActing(error)
|
|
1630
|
+
? new BrowserNoEffectError(action, noEffectReason(error))
|
|
1631
|
+
: new BrowserOutcomeUnknownError(action, error);
|
|
1215
1632
|
}
|
|
1216
1633
|
finally {
|
|
1217
1634
|
clearInterval(timer);
|