autokap 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/chrome/ios-statusbar-comparison-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-dark-reference.jpg +0 -0
- package/assets/chrome/ios-statusbar-light-reference.jpg +0 -0
- package/assets/devices/ipad-pro-11-m4.json +52 -0
- package/assets/devices/iphone-16-pro.json +53 -0
- package/assets/devices/macbook-air-13.json +45 -0
- package/assets/frames/MacBook Air 13.svg +242 -0
- package/assets/frames/Status bar - iPhone.png +0 -0
- Menu bar- iPad.png +0 -0
- package/assets/frames/iPad Pro M4 11_.png +0 -0
- package/assets/frames/iPhone 16 Pro.png +0 -0
- package/assets/icons/Cellular Connection.svg +3 -0
- package/assets/icons/Union.svg +6 -0
- package/assets/icons/Wifi.svg +3 -0
- package/assets/icons/battery.svg +5 -0
- package/assets/icons/battery_charging.svg +8 -0
- package/assets/skill/SKILL.md +575 -0
- package/dist/abort.d.ts +5 -0
- package/dist/abort.js +44 -0
- package/dist/agent.d.ts +142 -0
- package/dist/agent.js +4504 -0
- package/dist/browser-bar.d.ts +40 -0
- package/dist/browser-bar.js +147 -0
- package/dist/browser-pool.d.ts +34 -0
- package/dist/browser-pool.js +122 -0
- package/dist/browser.d.ts +279 -0
- package/dist/browser.js +2902 -0
- package/dist/cli-utils.d.ts +25 -0
- package/dist/cli-utils.js +80 -0
- package/dist/cli.d.ts +4 -0
- package/dist/cli.js +365 -0
- package/dist/clip-orchestrator.d.ts +148 -0
- package/dist/clip-orchestrator.js +950 -0
- package/dist/clip-postprocess.d.ts +42 -0
- package/dist/clip-postprocess.js +192 -0
- package/dist/cookie-dismiss.d.ts +5 -0
- package/dist/cookie-dismiss.js +172 -0
- package/dist/credential-templates.d.ts +5 -0
- package/dist/credential-templates.js +60 -0
- package/dist/element-capture.d.ts +53 -0
- package/dist/element-capture.js +766 -0
- package/dist/hybrid-navigator.d.ts +138 -0
- package/dist/hybrid-navigator.js +468 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +11 -0
- package/dist/llm-usage.d.ts +17 -0
- package/dist/llm-usage.js +45 -0
- package/dist/logger.d.ts +46 -0
- package/dist/logger.js +79 -0
- package/dist/mockup-html.d.ts +119 -0
- package/dist/mockup-html.js +253 -0
- package/dist/mockup.d.ts +94 -0
- package/dist/mockup.js +604 -0
- package/dist/mouse-animation.d.ts +46 -0
- package/dist/mouse-animation.js +100 -0
- package/dist/overlay-utils.d.ts +14 -0
- package/dist/overlay-utils.js +13 -0
- package/dist/posthog.d.ts +4 -0
- package/dist/posthog.js +26 -0
- package/dist/prompt-cache.d.ts +10 -0
- package/dist/prompt-cache.js +24 -0
- package/dist/prompts.d.ts +167 -0
- package/dist/prompts.js +1165 -0
- package/dist/security.d.ts +20 -0
- package/dist/security.js +569 -0
- package/dist/session-profile.d.ts +86 -0
- package/dist/session-profile.js +1471 -0
- package/dist/sf-pro-fonts.d.ts +4 -0
- package/dist/sf-pro-fonts.js +7 -0
- package/dist/status-bar-l10n.d.ts +14 -0
- package/dist/status-bar-l10n.js +177 -0
- package/dist/status-bar.d.ts +44 -0
- package/dist/status-bar.js +336 -0
- package/dist/tools.d.ts +4 -0
- package/dist/tools.js +578 -0
- package/dist/types.d.ts +796 -0
- package/dist/types.js +2 -0
- package/dist/video-agent.d.ts +143 -0
- package/dist/video-agent.js +4783 -0
- package/dist/video-observation.d.ts +36 -0
- package/dist/video-observation.js +192 -0
- package/dist/video-planner.d.ts +12 -0
- package/dist/video-planner.js +500 -0
- package/dist/video-prompts.d.ts +37 -0
- package/dist/video-prompts.js +554 -0
- package/dist/video-tools.d.ts +3 -0
- package/dist/video-tools.js +59 -0
- package/dist/video-variant-state.d.ts +29 -0
- package/dist/video-variant-state.js +80 -0
- package/dist/vision-model.d.ts +17 -0
- package/dist/vision-model.js +74 -0
- package/package.json +165 -0
- package/readme.md +61 -0
package/dist/browser.js
ADDED
|
@@ -0,0 +1,2902 @@
|
|
|
1
|
+
import { chromium } from 'playwright';
|
|
2
|
+
import sharp from 'sharp';
|
|
3
|
+
import { createHash } from 'crypto';
|
|
4
|
+
/**
|
|
5
|
+
* Set-of-Marks (SoM) annotation: overlays colored [N] badges on each visible
|
|
6
|
+
* interactive element so the vision model can reference elements by their badge index.
|
|
7
|
+
* Inspired by SoM prompting (Yang et al., 2310.11441) used in production web agents.
|
|
8
|
+
*
|
|
9
|
+
* Color scheme by role:
|
|
10
|
+
* button → blue input/textarea/select → green link → orange other → gray
|
|
11
|
+
*/
|
|
12
|
+
async function annotateWithSetOfMarks(screenshot, elements) {
|
|
13
|
+
// Only annotate visible elements that have a valid bounding box
|
|
14
|
+
const toAnnotate = elements.filter(el => el.visible && el.boundingBox && el.boundingBox.width > 4 && el.boundingBox.height > 4);
|
|
15
|
+
if (toAnnotate.length === 0)
|
|
16
|
+
return screenshot;
|
|
17
|
+
const meta = await sharp(screenshot).metadata();
|
|
18
|
+
const imgWidth = meta.width ?? 1280;
|
|
19
|
+
const imgHeight = meta.height ?? 800;
|
|
20
|
+
function roleColor(el) {
|
|
21
|
+
const role = el.role?.toLowerCase() ?? '';
|
|
22
|
+
const tag = el.tag?.toLowerCase() ?? '';
|
|
23
|
+
if (role === 'button' || tag === 'button')
|
|
24
|
+
return '#2563eb'; // blue
|
|
25
|
+
if (tag === 'input' || tag === 'textarea' || tag === 'select' || role === 'textbox' || role === 'combobox' || role === 'spinbutton')
|
|
26
|
+
return '#16a34a'; // green
|
|
27
|
+
if (role === 'link' || tag === 'a')
|
|
28
|
+
return '#ea580c'; // orange
|
|
29
|
+
return '#6b7280'; // gray
|
|
30
|
+
}
|
|
31
|
+
// Build an SVG overlay with one rectangle + badge per element
|
|
32
|
+
const svgParts = [];
|
|
33
|
+
for (const el of toAnnotate) {
|
|
34
|
+
const bb = el.boundingBox;
|
|
35
|
+
// Clamp bounding box to image dimensions
|
|
36
|
+
const x = Math.max(0, Math.min(Math.round(bb.x), imgWidth - 2));
|
|
37
|
+
const y = Math.max(0, Math.min(Math.round(bb.y), imgHeight - 2));
|
|
38
|
+
const w = Math.min(Math.round(bb.width), imgWidth - x);
|
|
39
|
+
const h = Math.min(Math.round(bb.height), imgHeight - y);
|
|
40
|
+
if (w < 2 || h < 2)
|
|
41
|
+
continue;
|
|
42
|
+
const color = roleColor(el);
|
|
43
|
+
const label = String(el.index);
|
|
44
|
+
const fontSize = Math.max(9, Math.min(13, Math.round(h * 0.55)));
|
|
45
|
+
const badgeW = Math.max(label.length * fontSize * 0.65 + 4, 14);
|
|
46
|
+
const badgeH = fontSize + 4;
|
|
47
|
+
const bx = Math.min(x, imgWidth - badgeW);
|
|
48
|
+
const by = Math.max(0, y - badgeH);
|
|
49
|
+
svgParts.push(
|
|
50
|
+
// Element outline rectangle
|
|
51
|
+
`<rect x="${x}" y="${y}" width="${w}" height="${h}" fill="none" stroke="${color}" stroke-width="1.5" opacity="0.75"/>`,
|
|
52
|
+
// Badge background
|
|
53
|
+
`<rect x="${bx}" y="${by}" width="${badgeW}" height="${badgeH}" fill="${color}" rx="2"/>`,
|
|
54
|
+
// Badge text
|
|
55
|
+
`<text x="${bx + badgeW / 2}" y="${by + badgeH - 3}" font-family="monospace,sans-serif" font-size="${fontSize}" font-weight="bold" fill="white" text-anchor="middle">${label}</text>`);
|
|
56
|
+
}
|
|
57
|
+
if (svgParts.length === 0)
|
|
58
|
+
return screenshot;
|
|
59
|
+
const svgOverlay = `<svg xmlns="http://www.w3.org/2000/svg" width="${imgWidth}" height="${imgHeight}">${svgParts.join('')}</svg>`;
|
|
60
|
+
try {
|
|
61
|
+
return await sharp(screenshot)
|
|
62
|
+
.composite([{ input: Buffer.from(svgOverlay), top: 0, left: 0 }])
|
|
63
|
+
.jpeg({ quality: 72 })
|
|
64
|
+
.toBuffer();
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// If annotation fails (e.g. coordinate overflow), return original screenshot
|
|
68
|
+
return screenshot;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function normalizeViewportDimension(value) {
|
|
72
|
+
if (!Number.isFinite(value))
|
|
73
|
+
return 1;
|
|
74
|
+
return Math.max(1, Math.round(value));
|
|
75
|
+
}
|
|
76
|
+
function resolveEffectivePadding(config, bbox) {
|
|
77
|
+
let top = 0, right = 0, bottom = 0, left = 0;
|
|
78
|
+
// Layer 1: percentage-based
|
|
79
|
+
if (config.paddingPercent != null) {
|
|
80
|
+
top = bottom = (config.paddingPercent / 100) * bbox.height;
|
|
81
|
+
left = right = (config.paddingPercent / 100) * bbox.width;
|
|
82
|
+
}
|
|
83
|
+
// Layer 2: uniform (overrides percent)
|
|
84
|
+
if (config.padding != null) {
|
|
85
|
+
top = right = bottom = left = config.padding;
|
|
86
|
+
}
|
|
87
|
+
// Layer 3: per-side (overrides uniform)
|
|
88
|
+
if (config.paddingTop != null)
|
|
89
|
+
top = config.paddingTop;
|
|
90
|
+
if (config.paddingRight != null)
|
|
91
|
+
right = config.paddingRight;
|
|
92
|
+
if (config.paddingBottom != null)
|
|
93
|
+
bottom = config.paddingBottom;
|
|
94
|
+
if (config.paddingLeft != null)
|
|
95
|
+
left = config.paddingLeft;
|
|
96
|
+
return { top, right, bottom, left };
|
|
97
|
+
}
|
|
98
|
+
import { dismissCookiesAndWidgets } from './cookie-dismiss.js';
|
|
99
|
+
import { CHROMIUM_ARGS, browserPool } from './browser-pool.js';
|
|
100
|
+
async function withHelperTimeout(label, timeoutMs, work) {
|
|
101
|
+
if (!timeoutMs || timeoutMs <= 0) {
|
|
102
|
+
return work();
|
|
103
|
+
}
|
|
104
|
+
let timer = null;
|
|
105
|
+
try {
|
|
106
|
+
return await Promise.race([
|
|
107
|
+
work(),
|
|
108
|
+
new Promise((_, reject) => {
|
|
109
|
+
timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
110
|
+
}),
|
|
111
|
+
]);
|
|
112
|
+
}
|
|
113
|
+
finally {
|
|
114
|
+
if (timer)
|
|
115
|
+
clearTimeout(timer);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Map a BCP-47 language tag to a Playwright-compatible locale string.
|
|
120
|
+
* Playwright accepts both "fr" and "fr-FR". We normalize 2-char codes to their
|
|
121
|
+
* primary regional variant so JS Intl APIs behave correctly.
|
|
122
|
+
*/
|
|
123
|
+
function langToLocale(lang) {
|
|
124
|
+
if (!lang)
|
|
125
|
+
return 'en-US';
|
|
126
|
+
if (lang.includes('-') || lang.includes('_'))
|
|
127
|
+
return lang;
|
|
128
|
+
const map = {
|
|
129
|
+
en: 'en-US', fr: 'fr-FR', de: 'de-DE', es: 'es-ES',
|
|
130
|
+
it: 'it-IT', pt: 'pt-PT', nl: 'nl-NL', ja: 'ja-JP',
|
|
131
|
+
zh: 'zh-CN', ko: 'ko-KR', ar: 'ar-SA', ru: 'ru-RU',
|
|
132
|
+
pl: 'pl-PL', sv: 'sv-SE', da: 'da-DK', fi: 'fi-FI',
|
|
133
|
+
no: 'nb-NO', tr: 'tr-TR', cs: 'cs-CZ', hu: 'hu-HU',
|
|
134
|
+
};
|
|
135
|
+
return map[lang] ?? lang;
|
|
136
|
+
}
|
|
137
|
+
function normalizeDeviceScaleFactor(value) {
|
|
138
|
+
if (!Number.isFinite(value))
|
|
139
|
+
return 2;
|
|
140
|
+
return Math.max(0.5, Math.min(4, Number(value)));
|
|
141
|
+
}
|
|
142
|
+
function resolveRecordedVideoSize(viewport) {
|
|
143
|
+
// Playwright's video recorder expects a canvas size close to the CSS viewport.
|
|
144
|
+
// Using viewport × deviceScaleFactor can produce a larger recording surface
|
|
145
|
+
// with the page rendered only in the top-left corner, leaving the remainder
|
|
146
|
+
// as empty matte. Keep the recorded frame size aligned to the viewport and
|
|
147
|
+
// let the browser's deviceScaleFactor handle HiDPI rendering internally.
|
|
148
|
+
const width = Math.max(2, Math.round(viewport.width)) & ~1;
|
|
149
|
+
const height = Math.max(2, Math.round(viewport.height)) & ~1;
|
|
150
|
+
return { width, height };
|
|
151
|
+
}
|
|
152
|
+
function escapeCssAttributeValue(value) {
|
|
153
|
+
return value
|
|
154
|
+
.replace(/\\/g, '\\\\')
|
|
155
|
+
.replace(/"/g, '\\"');
|
|
156
|
+
}
|
|
157
|
+
function escapeCssIdentifier(value) {
|
|
158
|
+
return value.replace(/(^-?\d)|[^a-zA-Z0-9_-]/g, (match) => `\\${match}`);
|
|
159
|
+
}
|
|
160
|
+
export function buildStableCssSelectorCandidates(snapshot) {
|
|
161
|
+
const tag = snapshot.tag.toLowerCase();
|
|
162
|
+
const candidates = [];
|
|
163
|
+
const push = (selector) => {
|
|
164
|
+
if (selector && !candidates.includes(selector)) {
|
|
165
|
+
candidates.push(selector);
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
if (snapshot.id) {
|
|
169
|
+
push(`#${escapeCssIdentifier(snapshot.id)}`);
|
|
170
|
+
}
|
|
171
|
+
for (const [attr, value] of [
|
|
172
|
+
['data-testid', snapshot.dataTestId],
|
|
173
|
+
['data-test', snapshot.dataTest],
|
|
174
|
+
['data-qa', snapshot.dataQa],
|
|
175
|
+
['data-cy', snapshot.dataCy],
|
|
176
|
+
]) {
|
|
177
|
+
if (!value)
|
|
178
|
+
continue;
|
|
179
|
+
push(`[${attr}="${escapeCssAttributeValue(value)}"]`);
|
|
180
|
+
push(`${tag}[${attr}="${escapeCssAttributeValue(value)}"]`);
|
|
181
|
+
}
|
|
182
|
+
if (snapshot.name) {
|
|
183
|
+
push(`${tag}[name="${escapeCssAttributeValue(snapshot.name)}"]`);
|
|
184
|
+
if (snapshot.type) {
|
|
185
|
+
push(`${tag}[name="${escapeCssAttributeValue(snapshot.name)}"][type="${escapeCssAttributeValue(snapshot.type)}"]`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (tag === 'a' && snapshot.href) {
|
|
189
|
+
push(`a[href="${escapeCssAttributeValue(snapshot.href)}"]`);
|
|
190
|
+
if (snapshot.hreflang) {
|
|
191
|
+
push(`a[href="${escapeCssAttributeValue(snapshot.href)}"][hreflang="${escapeCssAttributeValue(snapshot.hreflang)}"]`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
for (const [attr, value] of [
|
|
195
|
+
['aria-label', snapshot.ariaLabel],
|
|
196
|
+
['title', snapshot.title],
|
|
197
|
+
['placeholder', snapshot.placeholder],
|
|
198
|
+
['data-lang', snapshot.dataLang],
|
|
199
|
+
['data-language', snapshot.dataLanguage],
|
|
200
|
+
['data-locale', snapshot.dataLocale],
|
|
201
|
+
['data-theme', snapshot.dataTheme],
|
|
202
|
+
['data-color-scheme', snapshot.dataColorScheme],
|
|
203
|
+
['hreflang', snapshot.hreflang],
|
|
204
|
+
]) {
|
|
205
|
+
if (!value)
|
|
206
|
+
continue;
|
|
207
|
+
push(`${tag}[${attr}="${escapeCssAttributeValue(value)}"]`);
|
|
208
|
+
if (snapshot.role) {
|
|
209
|
+
push(`${tag}[role="${escapeCssAttributeValue(snapshot.role)}"][${attr}="${escapeCssAttributeValue(value)}"]`);
|
|
210
|
+
}
|
|
211
|
+
if (snapshot.type) {
|
|
212
|
+
push(`${tag}[type="${escapeCssAttributeValue(snapshot.type)}"][${attr}="${escapeCssAttributeValue(value)}"]`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
if (snapshot.type) {
|
|
216
|
+
push(`${tag}[type="${escapeCssAttributeValue(snapshot.type)}"]`);
|
|
217
|
+
}
|
|
218
|
+
return candidates;
|
|
219
|
+
}
|
|
220
|
+
export function selectStableCssSelector(snapshot, isUniqueSelector) {
|
|
221
|
+
for (const selector of buildStableCssSelectorCandidates(snapshot)) {
|
|
222
|
+
if (isUniqueSelector(selector)) {
|
|
223
|
+
return selector;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
function formatUrlForSummary(rawUrl) {
|
|
229
|
+
try {
|
|
230
|
+
const url = new URL(rawUrl);
|
|
231
|
+
return `${url.pathname || '/'}${url.search}${url.hash}` || rawUrl;
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
return rawUrl;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
export function describeObservationChange(before, after) {
|
|
238
|
+
const changes = [];
|
|
239
|
+
if (after.url !== before.url) {
|
|
240
|
+
changes.push(`URL -> ${formatUrlForSummary(after.url)}`);
|
|
241
|
+
}
|
|
242
|
+
if (after.title !== before.title) {
|
|
243
|
+
changes.push(`title -> "${after.title || '(untitled)'}"`);
|
|
244
|
+
}
|
|
245
|
+
const scrollDeltaY = after.scrollY - before.scrollY;
|
|
246
|
+
if (Math.abs(scrollDeltaY) >= 40) {
|
|
247
|
+
changes.push(`scroll ${scrollDeltaY > 0 ? 'down' : 'up'} ${Math.abs(scrollDeltaY)}px`);
|
|
248
|
+
}
|
|
249
|
+
const interactiveDelta = after.interactiveCount - before.interactiveCount;
|
|
250
|
+
if (Math.abs(interactiveDelta) >= 1) {
|
|
251
|
+
changes.push(`interactive ${before.interactiveCount} -> ${after.interactiveCount}`);
|
|
252
|
+
}
|
|
253
|
+
if (after.dialogCount !== before.dialogCount) {
|
|
254
|
+
changes.push(`dialogs ${before.dialogCount} -> ${after.dialogCount}`);
|
|
255
|
+
}
|
|
256
|
+
if (after.expandedCount !== before.expandedCount) {
|
|
257
|
+
changes.push(`expanded controls ${before.expandedCount} -> ${after.expandedCount}`);
|
|
258
|
+
}
|
|
259
|
+
if (after.loadingIndicatorCount !== before.loadingIndicatorCount) {
|
|
260
|
+
if (after.loadingIndicatorCount === 0 && before.loadingIndicatorCount > 0) {
|
|
261
|
+
changes.push('loading indicators cleared');
|
|
262
|
+
}
|
|
263
|
+
else {
|
|
264
|
+
changes.push(`loading indicators ${before.loadingIndicatorCount} -> ${after.loadingIndicatorCount}`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
if (after.scrollHeight !== before.scrollHeight && Math.abs(after.scrollHeight - before.scrollHeight) > 100) {
|
|
268
|
+
changes.push(`page height ${before.scrollHeight}px -> ${after.scrollHeight}px`);
|
|
269
|
+
}
|
|
270
|
+
if (after.textSample
|
|
271
|
+
&& after.textSample !== before.textSample
|
|
272
|
+
&& !changes.some((entry) => entry.startsWith('URL ->'))) {
|
|
273
|
+
changes.push(`visible text changed to "${after.textSample.slice(0, 80)}"`);
|
|
274
|
+
}
|
|
275
|
+
return {
|
|
276
|
+
before,
|
|
277
|
+
after,
|
|
278
|
+
changed: changes.length > 0,
|
|
279
|
+
summary: changes.length > 0
|
|
280
|
+
? changes.join('; ')
|
|
281
|
+
: 'No visible state change detected after the action.',
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
export class Browser {
|
|
285
|
+
options;
|
|
286
|
+
browser = null;
|
|
287
|
+
context = null;
|
|
288
|
+
page = null;
|
|
289
|
+
elementMap = new Map();
|
|
290
|
+
poolContext = false;
|
|
291
|
+
constructor(options) {
|
|
292
|
+
this.options = options;
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Create a Browser using the shared pool (server/web API mode).
|
|
296
|
+
* The Chromium process is reused across captures; only the context is isolated.
|
|
297
|
+
* Call browser.close() when done — it releases the context back to the pool.
|
|
298
|
+
*/
|
|
299
|
+
static async fromPool(options) {
|
|
300
|
+
const instance = new Browser(options);
|
|
301
|
+
instance.context = await browserPool.acquireContext(options.viewport, normalizeDeviceScaleFactor(options.deviceScaleFactor), {
|
|
302
|
+
lang: langToLocale(options.lang ?? 'en'),
|
|
303
|
+
colorScheme: options.colorScheme ?? 'light',
|
|
304
|
+
storageState: options.storageState,
|
|
305
|
+
});
|
|
306
|
+
instance.page = await instance.context.newPage();
|
|
307
|
+
instance.poolContext = true;
|
|
308
|
+
return instance;
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Create a Browser with a dedicated Chromium process and video recording enabled.
|
|
312
|
+
* The context is configured with `recordVideo` so Playwright records the session.
|
|
313
|
+
*
|
|
314
|
+
* IMPORTANT: After calling `browser.close()`, retrieve the video file path via
|
|
315
|
+
* `browser.currentPage.video()?.path()` BEFORE closing (save ref beforehand).
|
|
316
|
+
* Or call `browser.saveVideo(outputPath)` before closing.
|
|
317
|
+
*
|
|
318
|
+
* @param videoDir - Directory where Playwright will write the WebM file.
|
|
319
|
+
*/
|
|
320
|
+
static async forVideoRecording(options, videoDir, cursorScript) {
|
|
321
|
+
const instance = new Browser(options);
|
|
322
|
+
const deviceScaleFactor = normalizeDeviceScaleFactor(options.deviceScaleFactor);
|
|
323
|
+
const recordedVideoSize = resolveRecordedVideoSize(options.viewport);
|
|
324
|
+
// Dedicated browser process for video — cannot use the pool because
|
|
325
|
+
// `recordVideo` must be set at context creation time.
|
|
326
|
+
instance.browser = await chromium.launch({
|
|
327
|
+
headless: true,
|
|
328
|
+
args: CHROMIUM_ARGS,
|
|
329
|
+
});
|
|
330
|
+
instance.context = await instance.browser.newContext({
|
|
331
|
+
viewport: options.viewport,
|
|
332
|
+
deviceScaleFactor,
|
|
333
|
+
locale: langToLocale(options.lang ?? 'en'),
|
|
334
|
+
colorScheme: options.colorScheme ?? 'light',
|
|
335
|
+
storageState: options.storageState,
|
|
336
|
+
recordVideo: {
|
|
337
|
+
dir: videoDir,
|
|
338
|
+
size: recordedVideoSize,
|
|
339
|
+
},
|
|
340
|
+
});
|
|
341
|
+
// Inject cursor overlay at context level — survives all navigations in this session
|
|
342
|
+
await instance.context.addInitScript(cursorScript);
|
|
343
|
+
instance.page = await instance.context.newPage();
|
|
344
|
+
return instance;
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Close only the browser context (not the browser process).
|
|
348
|
+
* Use this for video recording: closing the context finalizes the WebM on disk
|
|
349
|
+
* while keeping the browser process alive so that saveAs() can still use the IPC channel.
|
|
350
|
+
* Call browser.close() afterwards to shut down the browser process.
|
|
351
|
+
*/
|
|
352
|
+
async closeContext() {
|
|
353
|
+
if (this.context) {
|
|
354
|
+
try {
|
|
355
|
+
await this.context.close();
|
|
356
|
+
}
|
|
357
|
+
catch { /* ignore */ }
|
|
358
|
+
this.context = null;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
async launch() {
|
|
362
|
+
// Pool mode: context and page are already initialized via fromPool()
|
|
363
|
+
if (this.poolContext)
|
|
364
|
+
return;
|
|
365
|
+
this.browser = await chromium.launch({
|
|
366
|
+
headless: !this.options.headed,
|
|
367
|
+
args: CHROMIUM_ARGS,
|
|
368
|
+
});
|
|
369
|
+
this.context = await this.browser.newContext({
|
|
370
|
+
viewport: this.options.viewport,
|
|
371
|
+
deviceScaleFactor: normalizeDeviceScaleFactor(this.options.deviceScaleFactor),
|
|
372
|
+
locale: langToLocale(this.options.lang ?? 'en'),
|
|
373
|
+
colorScheme: this.options.colorScheme ?? 'light',
|
|
374
|
+
storageState: this.options.storageState,
|
|
375
|
+
});
|
|
376
|
+
this.page = await this.context.newPage();
|
|
377
|
+
}
|
|
378
|
+
async addCookies(cookies) {
|
|
379
|
+
const context = this.ensureContext();
|
|
380
|
+
await context.addCookies(cookies.map(c => ({
|
|
381
|
+
name: c.name,
|
|
382
|
+
value: c.value,
|
|
383
|
+
domain: c.domain,
|
|
384
|
+
path: c.path || '/',
|
|
385
|
+
httpOnly: c.httpOnly ?? false,
|
|
386
|
+
secure: c.secure ?? true,
|
|
387
|
+
sameSite: c.sameSite || 'Lax',
|
|
388
|
+
})));
|
|
389
|
+
}
|
|
390
|
+
async close() {
|
|
391
|
+
if (this.poolContext) {
|
|
392
|
+
// Pool mode: close just the page, then release the context back to the pool
|
|
393
|
+
if (this.page) {
|
|
394
|
+
try {
|
|
395
|
+
await this.page.close();
|
|
396
|
+
}
|
|
397
|
+
catch { /* ignore */ }
|
|
398
|
+
this.page = null;
|
|
399
|
+
}
|
|
400
|
+
if (this.context) {
|
|
401
|
+
await browserPool.releaseContext(this.context);
|
|
402
|
+
this.context = null;
|
|
403
|
+
}
|
|
404
|
+
this.poolContext = false;
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
// Standalone mode (CLI): close the entire browser process
|
|
408
|
+
if (this.browser) {
|
|
409
|
+
await this.browser.close();
|
|
410
|
+
this.browser = null;
|
|
411
|
+
this.context = null;
|
|
412
|
+
this.page = null;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
async navigateTo(url) {
|
|
416
|
+
const page = this.ensurePage();
|
|
417
|
+
// Fix common URL typos (htpps, hhtps, htps, etc.)
|
|
418
|
+
url = url.replace(/^h+t+p+s?:\/\//i, (m) => /s/i.test(m) ? 'https://' : 'http://');
|
|
419
|
+
if (!/^https?:\/\//i.test(url)) {
|
|
420
|
+
url = 'https://' + url;
|
|
421
|
+
}
|
|
422
|
+
try {
|
|
423
|
+
await page.goto(url, { waitUntil: 'load', timeout: 20000 });
|
|
424
|
+
}
|
|
425
|
+
catch {
|
|
426
|
+
// Fallback for SPAs that never fully load
|
|
427
|
+
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
|
|
428
|
+
}
|
|
429
|
+
// Wait for network to settle (no new requests for 300ms, max 3s)
|
|
430
|
+
await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => { });
|
|
431
|
+
// Wait for DOM to stabilize (no mutations for 200ms, max 2s)
|
|
432
|
+
await this.waitForDomStability(page);
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Wait for DOM to stabilize: no MutationObserver events for `settleMs`,
|
|
436
|
+
* with an overall timeout of `timeoutMs`.
|
|
437
|
+
*/
|
|
438
|
+
async waitForDomStability(page, settleMs = 200, timeoutMs = 2000) {
|
|
439
|
+
try {
|
|
440
|
+
await page.evaluate(([settle, timeout]) => {
|
|
441
|
+
return new Promise((resolve) => {
|
|
442
|
+
let timer;
|
|
443
|
+
const observer = new MutationObserver(() => {
|
|
444
|
+
clearTimeout(timer);
|
|
445
|
+
timer = setTimeout(() => { observer.disconnect(); resolve(); }, settle);
|
|
446
|
+
});
|
|
447
|
+
observer.observe(document.documentElement, {
|
|
448
|
+
childList: true, subtree: true, attributes: true,
|
|
449
|
+
});
|
|
450
|
+
// Start the settle timer immediately (page might already be stable)
|
|
451
|
+
timer = setTimeout(() => { observer.disconnect(); resolve(); }, settle);
|
|
452
|
+
// Hard timeout
|
|
453
|
+
setTimeout(() => { observer.disconnect(); resolve(); }, timeout);
|
|
454
|
+
});
|
|
455
|
+
}, [settleMs, timeoutMs]);
|
|
456
|
+
}
|
|
457
|
+
catch {
|
|
458
|
+
// Page may have navigated during wait
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
async takeScreenshot() {
|
|
462
|
+
const page = this.ensurePage();
|
|
463
|
+
// Move cursor off-screen to avoid hover effects in screenshots
|
|
464
|
+
await page.mouse.move(0, 0);
|
|
465
|
+
return Buffer.from(await page.screenshot({ type: 'png', fullPage: false }));
|
|
466
|
+
}
|
|
467
|
+
async takeScreenshotForAI(options = {}) {
|
|
468
|
+
const page = this.ensurePage();
|
|
469
|
+
return withHelperTimeout('takeScreenshotForAI', options.timeoutMs, async () => {
|
|
470
|
+
await page.mouse.move(0, 0);
|
|
471
|
+
// Normalize the screenshot to viewport CSS pixels via browser canvas.
|
|
472
|
+
const viewport = page.viewportSize();
|
|
473
|
+
if (viewport) {
|
|
474
|
+
const fullBuf = await page.screenshot({ type: 'png', fullPage: false });
|
|
475
|
+
const resized = await page.evaluate(async ({ pngBase64, w, h }) => {
|
|
476
|
+
const img = new Image();
|
|
477
|
+
img.src = `data:image/png;base64,${pngBase64}`;
|
|
478
|
+
await new Promise((resolve) => { img.onload = () => resolve(); });
|
|
479
|
+
const canvas = document.createElement('canvas');
|
|
480
|
+
canvas.width = w;
|
|
481
|
+
canvas.height = h;
|
|
482
|
+
const ctx = canvas.getContext('2d');
|
|
483
|
+
ctx.drawImage(img, 0, 0, w, h);
|
|
484
|
+
// Return as JPEG base64 (strip the data:image/jpeg;base64, prefix)
|
|
485
|
+
return canvas.toDataURL('image/jpeg', 0.70).split(',')[1];
|
|
486
|
+
}, { pngBase64: fullBuf.toString('base64'), w: viewport.width, h: viewport.height });
|
|
487
|
+
return Buffer.from(resized, 'base64');
|
|
488
|
+
}
|
|
489
|
+
return Buffer.from(await page.screenshot({ type: 'jpeg', quality: 70, fullPage: false }));
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
async getAccessibilityTree(options = {}) {
|
|
493
|
+
const page = this.ensurePage();
|
|
494
|
+
return withHelperTimeout('getAccessibilityTree', options.timeoutMs, async () => {
|
|
495
|
+
try {
|
|
496
|
+
return await page.locator('body').ariaSnapshot();
|
|
497
|
+
}
|
|
498
|
+
catch {
|
|
499
|
+
// Fallback: build a simple text tree from the DOM
|
|
500
|
+
return await page.evaluate(() => {
|
|
501
|
+
function walk(node, depth) {
|
|
502
|
+
const role = node.getAttribute('role') || node.tagName.toLowerCase();
|
|
503
|
+
const text = (node.textContent || '').trim().slice(0, 60);
|
|
504
|
+
const indent = ' '.repeat(depth);
|
|
505
|
+
let result = `${indent}${role}`;
|
|
506
|
+
if (text && node.children.length === 0) {
|
|
507
|
+
result += `: "${text}"`;
|
|
508
|
+
}
|
|
509
|
+
result += '\n';
|
|
510
|
+
for (const child of Array.from(node.children)) {
|
|
511
|
+
result += walk(child, depth + 1);
|
|
512
|
+
}
|
|
513
|
+
return result;
|
|
514
|
+
}
|
|
515
|
+
return walk(document.body, 0);
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
async getInteractiveElements(options = {}) {
|
|
521
|
+
const page = this.ensurePage();
|
|
522
|
+
const elements = await withHelperTimeout('getInteractiveElements', options.timeoutMs, () => page.evaluate(() => {
|
|
523
|
+
document
|
|
524
|
+
.querySelectorAll('[data-ak-interactive-index]')
|
|
525
|
+
.forEach(el => el.removeAttribute('data-ak-interactive-index'));
|
|
526
|
+
const INTERACTIVE_SELECTORS = [
|
|
527
|
+
'a[href]', 'button', 'input', 'select', 'textarea', 'summary',
|
|
528
|
+
'[role="button"]', '[role="link"]', '[role="tab"]',
|
|
529
|
+
'[role="menuitem"]', '[role="menuitemcheckbox"]', '[role="menuitemradio"]',
|
|
530
|
+
'[role="checkbox"]', '[role="radio"]',
|
|
531
|
+
'[role="switch"]', '[role="combobox"]', '[role="option"]',
|
|
532
|
+
'[role="slider"]', '[role="spinbutton"]', '[role="listbox"]',
|
|
533
|
+
'[role="treeitem"]', '[role="gridcell"]',
|
|
534
|
+
'[tabindex]:not([tabindex="-1"])', '[onclick]',
|
|
535
|
+
'[contenteditable="true"]',
|
|
536
|
+
'[aria-haspopup]',
|
|
537
|
+
'label[for]',
|
|
538
|
+
// Bootstrap / common framework toggles
|
|
539
|
+
'[data-toggle]', '[data-bs-toggle]',
|
|
540
|
+
// Language/theme selectors are often custom elements
|
|
541
|
+
'[data-lang]', '[data-language]', '[data-locale]',
|
|
542
|
+
'[hreflang]',
|
|
543
|
+
];
|
|
544
|
+
const allElements = document.querySelectorAll(INTERACTIVE_SELECTORS.join(','));
|
|
545
|
+
// Also find implicitly clickable elements — covers icon buttons, React/Vue
|
|
546
|
+
// components with JS event listeners, and other non-semantic interactive elements.
|
|
547
|
+
const clickableElements = new Set();
|
|
548
|
+
const SKIP_TAGS = new Set(['A', 'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA', 'HTML', 'BODY']);
|
|
549
|
+
const candidateSelectors = 'div, span, li, label, p, img, svg, i, em, strong, figure, figcaption, nav > *, header > *, footer > *, section > *, article > *, main > *, aside > *';
|
|
550
|
+
const candidates = document.body.querySelectorAll(candidateSelectors);
|
|
551
|
+
for (const el of candidates) {
|
|
552
|
+
if (SKIP_TAGS.has(el.tagName))
|
|
553
|
+
continue;
|
|
554
|
+
const rect = el.getBoundingClientRect();
|
|
555
|
+
if (rect.width === 0 || rect.height === 0)
|
|
556
|
+
continue;
|
|
557
|
+
if (rect.width > 800 && rect.height > 600)
|
|
558
|
+
continue;
|
|
559
|
+
const style = window.getComputedStyle(el);
|
|
560
|
+
if (style.display === 'none' || style.visibility === 'hidden')
|
|
561
|
+
continue;
|
|
562
|
+
const isClickable =
|
|
563
|
+
// Standard cursor:pointer detection
|
|
564
|
+
style.cursor === 'pointer'
|
|
565
|
+
// React/framework patterns: data-* attributes that suggest interactivity
|
|
566
|
+
|| el.hasAttribute('data-state')
|
|
567
|
+
|| el.hasAttribute('data-radix-collection-item')
|
|
568
|
+
|| el.hasAttribute('data-slot')
|
|
569
|
+
// Inline event handlers (beyond [onclick] already covered above)
|
|
570
|
+
|| el.hasAttribute('onmousedown')
|
|
571
|
+
|| el.hasAttribute('onpointerdown')
|
|
572
|
+
|| el.hasAttribute('ontouchstart')
|
|
573
|
+
// Elements with tabindex that might have been missed
|
|
574
|
+
|| (el.tabIndex >= 0 && el.tagName !== 'DIV')
|
|
575
|
+
// Small icon-like elements (SVG icons, font icons) inside interactive contexts
|
|
576
|
+
|| (rect.width <= 48 && rect.height <= 48 && (el.closest('button, [role="button"], a[href]') !== null
|
|
577
|
+
|| style.cursor === 'pointer'
|
|
578
|
+
|| el.hasAttribute('title')));
|
|
579
|
+
if (isClickable) {
|
|
580
|
+
// For small icons inside buttons, prefer the parent button
|
|
581
|
+
const parentButton = el.closest('button, [role="button"]');
|
|
582
|
+
if (parentButton && parentButton !== el) {
|
|
583
|
+
clickableElements.add(parentButton);
|
|
584
|
+
}
|
|
585
|
+
else {
|
|
586
|
+
clickableElements.add(el);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
// Merge both sets (avoid duplicates)
|
|
591
|
+
const combined = new Set(allElements);
|
|
592
|
+
for (const el of clickableElements)
|
|
593
|
+
combined.add(el);
|
|
594
|
+
// Viewport dimensions for visibility check
|
|
595
|
+
const vw = window.innerWidth;
|
|
596
|
+
const vh = window.innerHeight;
|
|
597
|
+
const scrollX = window.scrollX;
|
|
598
|
+
const scrollY = window.scrollY;
|
|
599
|
+
// Helper: get the direct/own text of an element (not its children's text)
|
|
600
|
+
function getOwnText(el) {
|
|
601
|
+
let text = '';
|
|
602
|
+
for (const child of el.childNodes) {
|
|
603
|
+
if (child.nodeType === Node.TEXT_NODE) {
|
|
604
|
+
text += (child.textContent || '').trim() + ' ';
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
return text.trim();
|
|
608
|
+
}
|
|
609
|
+
// Helper: get the shallow text (own text + immediate child text, not deep)
|
|
610
|
+
function getShallowText(el) {
|
|
611
|
+
const own = getOwnText(el);
|
|
612
|
+
if (own)
|
|
613
|
+
return own;
|
|
614
|
+
// If no own text, get text from direct children only (1 level deep)
|
|
615
|
+
let childText = '';
|
|
616
|
+
for (const child of el.children) {
|
|
617
|
+
const ct = getOwnText(child);
|
|
618
|
+
if (ct)
|
|
619
|
+
childText += ct + ' ';
|
|
620
|
+
}
|
|
621
|
+
return childText.trim();
|
|
622
|
+
}
|
|
623
|
+
const quoteForSelector = (value) => value
|
|
624
|
+
.replace(/\\/g, '\\\\')
|
|
625
|
+
.replace(/"/g, '\\"');
|
|
626
|
+
const isUniqueSelector = (selector, expected) => {
|
|
627
|
+
try {
|
|
628
|
+
const matches = document.querySelectorAll(selector);
|
|
629
|
+
return matches.length === 1 && matches[0] === expected;
|
|
630
|
+
}
|
|
631
|
+
catch {
|
|
632
|
+
return false;
|
|
633
|
+
}
|
|
634
|
+
};
|
|
635
|
+
const buildStableCssSelector = (node) => {
|
|
636
|
+
const tag = node.tagName.toLowerCase();
|
|
637
|
+
const candidates = [];
|
|
638
|
+
const push = (selector) => {
|
|
639
|
+
if (selector && !candidates.includes(selector)) {
|
|
640
|
+
candidates.push(selector);
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
const role = node.getAttribute('role');
|
|
644
|
+
const type = node.getAttribute('type');
|
|
645
|
+
const id = node.getAttribute('id');
|
|
646
|
+
if (id) {
|
|
647
|
+
push(`#${CSS.escape(id)}`);
|
|
648
|
+
}
|
|
649
|
+
for (const attr of ['data-testid', 'data-test', 'data-qa', 'data-cy']) {
|
|
650
|
+
const value = node.getAttribute(attr);
|
|
651
|
+
if (!value)
|
|
652
|
+
continue;
|
|
653
|
+
push(`[${attr}="${quoteForSelector(value)}"]`);
|
|
654
|
+
push(`${tag}[${attr}="${quoteForSelector(value)}"]`);
|
|
655
|
+
}
|
|
656
|
+
const name = node.getAttribute('name');
|
|
657
|
+
if (name) {
|
|
658
|
+
push(`${tag}[name="${quoteForSelector(name)}"]`);
|
|
659
|
+
if (type) {
|
|
660
|
+
push(`${tag}[name="${quoteForSelector(name)}"][type="${quoteForSelector(type)}"]`);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
const rawHref = node.getAttribute('href');
|
|
664
|
+
if (tag === 'a' && rawHref) {
|
|
665
|
+
push(`a[href="${quoteForSelector(rawHref)}"]`);
|
|
666
|
+
const hreflang = node.getAttribute('hreflang');
|
|
667
|
+
if (hreflang) {
|
|
668
|
+
push(`a[href="${quoteForSelector(rawHref)}"][hreflang="${quoteForSelector(hreflang)}"]`);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
for (const attr of ['aria-label', 'title', 'placeholder', 'data-lang', 'data-language', 'data-locale', 'data-theme', 'data-color-scheme', 'hreflang']) {
|
|
672
|
+
const value = node.getAttribute(attr);
|
|
673
|
+
if (!value)
|
|
674
|
+
continue;
|
|
675
|
+
push(`${tag}[${attr}="${quoteForSelector(value)}"]`);
|
|
676
|
+
if (role) {
|
|
677
|
+
push(`${tag}[role="${quoteForSelector(role)}"][${attr}="${quoteForSelector(value)}"]`);
|
|
678
|
+
}
|
|
679
|
+
if (type) {
|
|
680
|
+
push(`${tag}[type="${quoteForSelector(type)}"][${attr}="${quoteForSelector(value)}"]`);
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
if (type) {
|
|
684
|
+
push(`${tag}[type="${quoteForSelector(type)}"]`);
|
|
685
|
+
}
|
|
686
|
+
for (const selector of candidates) {
|
|
687
|
+
if (isUniqueSelector(selector, node)) {
|
|
688
|
+
return selector;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
return null;
|
|
692
|
+
};
|
|
693
|
+
const results = [];
|
|
694
|
+
for (const el of combined) {
|
|
695
|
+
const htmlEl = el;
|
|
696
|
+
// Skip hidden elements (but allow fixed-position elements which may have null offsetParent)
|
|
697
|
+
const style = window.getComputedStyle(htmlEl);
|
|
698
|
+
const isFixed = style.position === 'fixed' || style.position === 'sticky';
|
|
699
|
+
if (!isFixed && htmlEl.offsetParent === null && htmlEl.tagName !== 'BODY')
|
|
700
|
+
continue;
|
|
701
|
+
if (style.display === 'none' || style.visibility === 'hidden')
|
|
702
|
+
continue;
|
|
703
|
+
if (parseFloat(style.opacity) === 0)
|
|
704
|
+
continue;
|
|
705
|
+
const rect = htmlEl.getBoundingClientRect();
|
|
706
|
+
// Skip elements with no size
|
|
707
|
+
if (rect.width === 0 && rect.height === 0)
|
|
708
|
+
continue;
|
|
709
|
+
// Skip extremely large container elements that happen to have cursor:pointer
|
|
710
|
+
if (rect.width > vw * 0.9 && rect.height > vh * 0.9)
|
|
711
|
+
continue;
|
|
712
|
+
// Check if element is in viewport
|
|
713
|
+
const visibleWidth = Math.max(0, Math.min(rect.right, vw) - Math.max(rect.left, 0));
|
|
714
|
+
const visibleHeight = Math.max(0, Math.min(rect.bottom, vh) - Math.max(rect.top, 0));
|
|
715
|
+
const inViewport = visibleWidth > 0 && visibleHeight > 0;
|
|
716
|
+
const fullyVisible = rect.top >= 0 && rect.left >= 0 && rect.bottom <= vh && rect.right <= vw;
|
|
717
|
+
const visibilityState = !inViewport ? 'offscreen' : fullyVisible ? 'full' : 'partial';
|
|
718
|
+
const elementIndex = results.length;
|
|
719
|
+
htmlEl.setAttribute('data-ak-interactive-index', String(elementIndex));
|
|
720
|
+
const selector = buildStableCssSelector(htmlEl) ?? `[data-ak-interactive-index="${elementIndex}"]`;
|
|
721
|
+
// Get text content — prefer shallow/own text to avoid parent element pollution
|
|
722
|
+
let text = '';
|
|
723
|
+
if (htmlEl.tagName === 'INPUT' || htmlEl.tagName === 'TEXTAREA') {
|
|
724
|
+
text = htmlEl.placeholder || htmlEl.value || '';
|
|
725
|
+
}
|
|
726
|
+
else {
|
|
727
|
+
// Use shallow text first (more specific), fall back to full textContent
|
|
728
|
+
text = getShallowText(htmlEl);
|
|
729
|
+
if (!text) {
|
|
730
|
+
text = (htmlEl.textContent || '').trim();
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
// For empty-text elements, try multiple fallbacks (common for icon buttons)
|
|
734
|
+
if (!text) {
|
|
735
|
+
text = htmlEl.getAttribute('title')
|
|
736
|
+
|| htmlEl.getAttribute('aria-label')
|
|
737
|
+
// Check child SVG title element (common pattern: <button><svg><title>Edit</title></svg></button>)
|
|
738
|
+
|| htmlEl.querySelector('svg > title')?.textContent?.trim()
|
|
739
|
+
// Check child img alt
|
|
740
|
+
|| htmlEl.querySelector('img')?.getAttribute('alt')
|
|
741
|
+
// Check aria-labelledby reference
|
|
742
|
+
|| (() => {
|
|
743
|
+
const labelledBy = htmlEl.getAttribute('aria-labelledby');
|
|
744
|
+
if (labelledBy) {
|
|
745
|
+
const labelEl = document.getElementById(labelledBy);
|
|
746
|
+
if (labelEl)
|
|
747
|
+
return labelEl.textContent?.trim() || '';
|
|
748
|
+
}
|
|
749
|
+
return '';
|
|
750
|
+
})()
|
|
751
|
+
|| '';
|
|
752
|
+
}
|
|
753
|
+
text = text.slice(0, 80);
|
|
754
|
+
// Get role
|
|
755
|
+
const role = htmlEl.getAttribute('role')
|
|
756
|
+
|| (htmlEl.tagName === 'A' ? 'link' : '')
|
|
757
|
+
|| (htmlEl.tagName === 'BUTTON' ? 'button' : '')
|
|
758
|
+
|| (htmlEl.tagName === 'INPUT' ? `input[${htmlEl.type}]` : '')
|
|
759
|
+
|| htmlEl.tagName.toLowerCase();
|
|
760
|
+
// Get href for links
|
|
761
|
+
const href = htmlEl.tagName === 'A' ? htmlEl.href : null;
|
|
762
|
+
// Get input type
|
|
763
|
+
const inputType = (htmlEl.tagName === 'INPUT' || htmlEl.tagName === 'SELECT' || htmlEl.tagName === 'TEXTAREA')
|
|
764
|
+
? htmlEl.type || htmlEl.tagName.toLowerCase()
|
|
765
|
+
: null;
|
|
766
|
+
const titleAttr = htmlEl.getAttribute('title') || null;
|
|
767
|
+
results.push({
|
|
768
|
+
tag: htmlEl.tagName.toLowerCase(),
|
|
769
|
+
role,
|
|
770
|
+
text,
|
|
771
|
+
ariaLabel: htmlEl.getAttribute('aria-label'),
|
|
772
|
+
title: titleAttr,
|
|
773
|
+
ariaControls: htmlEl.getAttribute('aria-controls'),
|
|
774
|
+
ariaExpanded: htmlEl.getAttribute('aria-expanded'),
|
|
775
|
+
ariaHasPopup: htmlEl.getAttribute('aria-haspopup'),
|
|
776
|
+
href,
|
|
777
|
+
inputType,
|
|
778
|
+
boundingBox: {
|
|
779
|
+
x: Math.round(rect.x),
|
|
780
|
+
y: Math.round(rect.y),
|
|
781
|
+
width: Math.round(rect.width),
|
|
782
|
+
height: Math.round(rect.height),
|
|
783
|
+
},
|
|
784
|
+
pageBox: {
|
|
785
|
+
x: Math.round(rect.x + scrollX),
|
|
786
|
+
y: Math.round(rect.y + scrollY),
|
|
787
|
+
width: Math.round(rect.width),
|
|
788
|
+
height: Math.round(rect.height),
|
|
789
|
+
},
|
|
790
|
+
selector,
|
|
791
|
+
visible: inViewport,
|
|
792
|
+
visibilityState,
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
return results;
|
|
796
|
+
}));
|
|
797
|
+
// Capture scroll position at the time elements were observed
|
|
798
|
+
const scrollPos = await page.evaluate(() => ({
|
|
799
|
+
x: window.scrollX,
|
|
800
|
+
y: window.scrollY,
|
|
801
|
+
}));
|
|
802
|
+
// Assign indices and store mapping (including scroll position for reliable screenshots)
|
|
803
|
+
this.elementMap.clear();
|
|
804
|
+
return elements.map((el, i) => {
|
|
805
|
+
this.elementMap.set(i, {
|
|
806
|
+
selector: el.selector,
|
|
807
|
+
boundingBox: el.boundingBox,
|
|
808
|
+
pageBox: el.pageBox,
|
|
809
|
+
scrollPos,
|
|
810
|
+
tag: el.tag,
|
|
811
|
+
text: el.text,
|
|
812
|
+
ariaLabel: el.ariaLabel,
|
|
813
|
+
role: el.role,
|
|
814
|
+
inputType: el.inputType,
|
|
815
|
+
});
|
|
816
|
+
const { pageBox: _pageBox, ...publicElement } = el;
|
|
817
|
+
return { index: i, ...publicElement };
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* Extract a simplified DOM representation of the page.
|
|
822
|
+
* Strips scripts, styles, SVGs, class/style attributes, and collapses empty containers.
|
|
823
|
+
* Returns clean indented HTML-like text, budget-capped at ~4000 chars.
|
|
824
|
+
*/
|
|
825
|
+
async getSimplifiedDOM() {
|
|
826
|
+
const page = this.ensurePage();
|
|
827
|
+
try {
|
|
828
|
+
return await page.evaluate(() => {
|
|
829
|
+
const SKIP_TAGS = new Set(['STYLE', 'SCRIPT', 'SVG', 'NOSCRIPT', 'LINK', 'META', 'BR', 'HR', 'IFRAME', 'CANVAS', 'VIDEO', 'AUDIO', 'SOURCE', 'PICTURE', 'TEMPLATE']);
|
|
830
|
+
const KEEP_ATTRS = new Set(['id', 'role', 'aria-label', 'aria-expanded', 'aria-haspopup', 'aria-controls', 'aria-selected', 'aria-checked', 'aria-disabled', 'href', 'type', 'name', 'placeholder', 'alt', 'title', 'value', 'for', 'action', 'method', 'data-testid']);
|
|
831
|
+
const MAX_CHARS = 4000;
|
|
832
|
+
let output = '';
|
|
833
|
+
let stopped = false;
|
|
834
|
+
function append(text) {
|
|
835
|
+
if (stopped)
|
|
836
|
+
return;
|
|
837
|
+
if (output.length + text.length > MAX_CHARS) {
|
|
838
|
+
output += text.slice(0, MAX_CHARS - output.length);
|
|
839
|
+
output += '\n... (truncated)';
|
|
840
|
+
stopped = true;
|
|
841
|
+
return;
|
|
842
|
+
}
|
|
843
|
+
output += text;
|
|
844
|
+
}
|
|
845
|
+
function getDirectText(node) {
|
|
846
|
+
let text = '';
|
|
847
|
+
for (const child of node.childNodes) {
|
|
848
|
+
if (child.nodeType === 3) { // TEXT_NODE
|
|
849
|
+
const t = (child.textContent || '').trim();
|
|
850
|
+
if (t)
|
|
851
|
+
text += (text ? ' ' : '') + t;
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
return text.slice(0, 80);
|
|
855
|
+
}
|
|
856
|
+
function walk(node, depth) {
|
|
857
|
+
if (stopped)
|
|
858
|
+
return;
|
|
859
|
+
if (node.nodeType === 8)
|
|
860
|
+
return; // COMMENT_NODE
|
|
861
|
+
const tag = node.tagName;
|
|
862
|
+
if (!tag)
|
|
863
|
+
return;
|
|
864
|
+
if (SKIP_TAGS.has(tag))
|
|
865
|
+
return;
|
|
866
|
+
// Collect kept attributes
|
|
867
|
+
const attrs = [];
|
|
868
|
+
for (const attr of node.attributes) {
|
|
869
|
+
if (KEEP_ATTRS.has(attr.name)) {
|
|
870
|
+
let val = attr.value.trim();
|
|
871
|
+
if (val.length > 80)
|
|
872
|
+
val = val.slice(0, 77) + '...';
|
|
873
|
+
if (val)
|
|
874
|
+
attrs.push(`${attr.name}="${val}"`);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
const directText = getDirectText(node);
|
|
878
|
+
const childElements = Array.from(node.children).filter(c => c.nodeType === 1 && !SKIP_TAGS.has(c.tagName));
|
|
879
|
+
// Collapse empty containers (no text, no meaningful children)
|
|
880
|
+
if (!directText && childElements.length === 0 && attrs.length === 0)
|
|
881
|
+
return;
|
|
882
|
+
const indent = ' '.repeat(Math.min(depth, 10));
|
|
883
|
+
const tagLower = tag.toLowerCase();
|
|
884
|
+
const attrStr = attrs.length > 0 ? ' ' + attrs.join(' ') : '';
|
|
885
|
+
if (childElements.length === 0) {
|
|
886
|
+
// Leaf node — inline
|
|
887
|
+
if (directText) {
|
|
888
|
+
append(`${indent}<${tagLower}${attrStr}>${directText}</${tagLower}>\n`);
|
|
889
|
+
}
|
|
890
|
+
else if (attrStr) {
|
|
891
|
+
append(`${indent}<${tagLower}${attrStr} />\n`);
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
else {
|
|
895
|
+
// Container with children
|
|
896
|
+
if (directText) {
|
|
897
|
+
append(`${indent}<${tagLower}${attrStr}>${directText}\n`);
|
|
898
|
+
}
|
|
899
|
+
else {
|
|
900
|
+
append(`${indent}<${tagLower}${attrStr}>\n`);
|
|
901
|
+
}
|
|
902
|
+
for (const child of childElements) {
|
|
903
|
+
walk(child, depth + 1);
|
|
904
|
+
}
|
|
905
|
+
append(`${indent}</${tagLower}>\n`);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
// Start from body, skip head
|
|
909
|
+
const body = document.body;
|
|
910
|
+
if (body) {
|
|
911
|
+
// Include title
|
|
912
|
+
const title = document.title;
|
|
913
|
+
if (title)
|
|
914
|
+
append(`<title>${title.slice(0, 100)}</title>\n`);
|
|
915
|
+
walk(body, 0);
|
|
916
|
+
}
|
|
917
|
+
return output;
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
catch (err) {
|
|
921
|
+
return ''; // Fallback: empty DOM, accessibility tree still available
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
async getPageState(opts) {
|
|
925
|
+
const page = this.ensurePage();
|
|
926
|
+
const [cleanScreenshot, accessibilityTree, interactiveElements, scrollInfo, simplifiedDOM] = await Promise.all([
|
|
927
|
+
this.takeScreenshotForAI(),
|
|
928
|
+
this.getAccessibilityTree(),
|
|
929
|
+
this.getInteractiveElements(),
|
|
930
|
+
page.evaluate(() => ({
|
|
931
|
+
scrollY: Math.round(window.scrollY),
|
|
932
|
+
scrollHeight: document.documentElement.scrollHeight,
|
|
933
|
+
viewportHeight: window.innerHeight,
|
|
934
|
+
})),
|
|
935
|
+
this.getSimplifiedDOM(),
|
|
936
|
+
]);
|
|
937
|
+
const annotatedScreenshot = opts?.skipAnnotation
|
|
938
|
+
? cleanScreenshot
|
|
939
|
+
: await annotateWithSetOfMarks(cleanScreenshot, interactiveElements);
|
|
940
|
+
return {
|
|
941
|
+
cleanScreenshot,
|
|
942
|
+
screenshot: annotatedScreenshot,
|
|
943
|
+
accessibilityTree,
|
|
944
|
+
interactiveElements,
|
|
945
|
+
simplifiedDOM,
|
|
946
|
+
scrollInfo,
|
|
947
|
+
};
|
|
948
|
+
}
|
|
949
|
+
/** Lightweight page state without screenshots — skips takeScreenshotForAI + SoM annotation. */
|
|
950
|
+
async getPageStateLite() {
|
|
951
|
+
const page = this.ensurePage();
|
|
952
|
+
const [accessibilityTree, interactiveElements, scrollInfo, simplifiedDOM] = await Promise.all([
|
|
953
|
+
this.getAccessibilityTree(),
|
|
954
|
+
this.getInteractiveElements(),
|
|
955
|
+
page.evaluate(() => ({
|
|
956
|
+
scrollY: Math.round(window.scrollY),
|
|
957
|
+
scrollHeight: document.documentElement.scrollHeight,
|
|
958
|
+
viewportHeight: window.innerHeight,
|
|
959
|
+
})),
|
|
960
|
+
this.getSimplifiedDOM(),
|
|
961
|
+
]);
|
|
962
|
+
return { accessibilityTree, interactiveElements, simplifiedDOM, scrollInfo };
|
|
963
|
+
}
|
|
964
|
+
async exportStorageState() {
|
|
965
|
+
const context = this.ensureContext();
|
|
966
|
+
return context.storageState();
|
|
967
|
+
}
|
|
968
|
+
async exportSessionStorage() {
|
|
969
|
+
const page = this.ensurePage();
|
|
970
|
+
const snapshot = await page.evaluate(() => {
|
|
971
|
+
const entries = {};
|
|
972
|
+
for (let index = 0; index < window.sessionStorage.length; index += 1) {
|
|
973
|
+
const key = window.sessionStorage.key(index);
|
|
974
|
+
if (!key)
|
|
975
|
+
continue;
|
|
976
|
+
entries[key] = window.sessionStorage.getItem(key) ?? '';
|
|
977
|
+
}
|
|
978
|
+
return {
|
|
979
|
+
origin: window.location.origin,
|
|
980
|
+
entries,
|
|
981
|
+
};
|
|
982
|
+
});
|
|
983
|
+
if (!snapshot.origin || Object.keys(snapshot.entries).length === 0) {
|
|
984
|
+
return {};
|
|
985
|
+
}
|
|
986
|
+
return {
|
|
987
|
+
[snapshot.origin]: snapshot.entries,
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
async prepareSessionStorage(bundle, options = {}) {
|
|
991
|
+
if (!bundle || Object.keys(bundle).length === 0)
|
|
992
|
+
return;
|
|
993
|
+
const context = this.ensureContext();
|
|
994
|
+
const page = this.ensurePage();
|
|
995
|
+
await context.addInitScript(({ storageByOrigin, replace }) => {
|
|
996
|
+
try {
|
|
997
|
+
const entries = storageByOrigin[window.location.origin];
|
|
998
|
+
if (!entries)
|
|
999
|
+
return;
|
|
1000
|
+
if (replace)
|
|
1001
|
+
window.sessionStorage.clear();
|
|
1002
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
1003
|
+
window.sessionStorage.setItem(key, String(value));
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
catch {
|
|
1007
|
+
// Ignore storage restoration issues on restricted pages.
|
|
1008
|
+
}
|
|
1009
|
+
}, { storageByOrigin: bundle, replace: options.replace ?? false });
|
|
1010
|
+
await page.evaluate(({ storageByOrigin, replace }) => {
|
|
1011
|
+
try {
|
|
1012
|
+
const entries = storageByOrigin[window.location.origin];
|
|
1013
|
+
if (!entries)
|
|
1014
|
+
return;
|
|
1015
|
+
if (replace)
|
|
1016
|
+
window.sessionStorage.clear();
|
|
1017
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
1018
|
+
window.sessionStorage.setItem(key, String(value));
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
catch {
|
|
1022
|
+
// Ignore storage restoration issues on restricted pages.
|
|
1023
|
+
}
|
|
1024
|
+
}, { storageByOrigin: bundle, replace: options.replace ?? false }).catch(() => {
|
|
1025
|
+
// The current page may not yet be navigated to the target origin.
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
async capturePageSignals(options = {}) {
|
|
1029
|
+
const page = this.ensurePage();
|
|
1030
|
+
return withHelperTimeout('capturePageSignals', options.timeoutMs, () => page.evaluate(() => {
|
|
1031
|
+
const dedupe = (values, limit = 12) => {
|
|
1032
|
+
const seen = new Set();
|
|
1033
|
+
const result = [];
|
|
1034
|
+
for (const raw of values) {
|
|
1035
|
+
const value = (raw || '').replace(/\s+/g, ' ').trim();
|
|
1036
|
+
if (!value)
|
|
1037
|
+
continue;
|
|
1038
|
+
const key = value.toLowerCase();
|
|
1039
|
+
if (seen.has(key))
|
|
1040
|
+
continue;
|
|
1041
|
+
seen.add(key);
|
|
1042
|
+
result.push(value);
|
|
1043
|
+
if (result.length >= limit)
|
|
1044
|
+
break;
|
|
1045
|
+
}
|
|
1046
|
+
return result;
|
|
1047
|
+
};
|
|
1048
|
+
const dedupeObjects = (values, makeKey, limit = 12) => {
|
|
1049
|
+
const seen = new Set();
|
|
1050
|
+
const result = [];
|
|
1051
|
+
for (const value of values) {
|
|
1052
|
+
const key = makeKey(value);
|
|
1053
|
+
if (!key || seen.has(key))
|
|
1054
|
+
continue;
|
|
1055
|
+
seen.add(key);
|
|
1056
|
+
result.push(value);
|
|
1057
|
+
if (result.length >= limit)
|
|
1058
|
+
break;
|
|
1059
|
+
}
|
|
1060
|
+
return result;
|
|
1061
|
+
};
|
|
1062
|
+
const isVisible = (node) => {
|
|
1063
|
+
if (!(node instanceof HTMLElement))
|
|
1064
|
+
return false;
|
|
1065
|
+
const style = window.getComputedStyle(node);
|
|
1066
|
+
if (style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity || '1') === 0) {
|
|
1067
|
+
return false;
|
|
1068
|
+
}
|
|
1069
|
+
const rect = node.getBoundingClientRect();
|
|
1070
|
+
return rect.width > 0 && rect.height > 0;
|
|
1071
|
+
};
|
|
1072
|
+
const textFor = (node) => {
|
|
1073
|
+
if (!(node instanceof HTMLElement))
|
|
1074
|
+
return '';
|
|
1075
|
+
const aria = node.getAttribute('aria-label') || node.getAttribute('title') || '';
|
|
1076
|
+
const text = (node.innerText || node.textContent || '').replace(/\s+/g, ' ').trim();
|
|
1077
|
+
return (text || aria).slice(0, 120);
|
|
1078
|
+
};
|
|
1079
|
+
const quoteForSelector = (value) => value
|
|
1080
|
+
.replace(/\\/g, '\\\\')
|
|
1081
|
+
.replace(/"/g, '\\"');
|
|
1082
|
+
const buildSelector = (node) => {
|
|
1083
|
+
if (!(node instanceof HTMLElement))
|
|
1084
|
+
return '';
|
|
1085
|
+
const tag = node.tagName.toLowerCase();
|
|
1086
|
+
const id = node.getAttribute('id');
|
|
1087
|
+
if (id)
|
|
1088
|
+
return `#${CSS.escape(id)}`;
|
|
1089
|
+
for (const attr of ['data-testid', 'data-test', 'data-qa', 'data-cy']) {
|
|
1090
|
+
const value = node.getAttribute(attr);
|
|
1091
|
+
if (value)
|
|
1092
|
+
return `[${attr}="${quoteForSelector(value)}"]`;
|
|
1093
|
+
}
|
|
1094
|
+
for (const attr of ['name', 'aria-label', 'data-lang', 'data-language', 'data-locale', 'data-theme', 'data-color-scheme', 'hreflang']) {
|
|
1095
|
+
const value = node.getAttribute(attr);
|
|
1096
|
+
if (value)
|
|
1097
|
+
return `${tag}[${attr}="${quoteForSelector(value)}"]`;
|
|
1098
|
+
}
|
|
1099
|
+
if (node instanceof HTMLAnchorElement) {
|
|
1100
|
+
const href = node.getAttribute('href');
|
|
1101
|
+
if (href)
|
|
1102
|
+
return `a[href*="${quoteForSelector(href.slice(0, 80))}"]`;
|
|
1103
|
+
}
|
|
1104
|
+
const text = textFor(node).slice(0, 60);
|
|
1105
|
+
if (text) {
|
|
1106
|
+
if (tag === 'button' || tag === 'a' || node.getAttribute('role')) {
|
|
1107
|
+
return `${tag}:has-text("${quoteForSelector(text)}")`;
|
|
1108
|
+
}
|
|
1109
|
+
return `${tag}:has-text("${quoteForSelector(text)}")`;
|
|
1110
|
+
}
|
|
1111
|
+
return tag;
|
|
1112
|
+
};
|
|
1113
|
+
const LOCALE_HINT_RE = /\b(lang|locale|language|idioma|sprache|langue|lingua|english|fran[cç]ais|deutsch|espa[nñ]ol|italiano|portugu[eê]s|nederlands|polski|svenska|dansk|suomi|norsk|japanese|japonais|中文|日本語)\b/i;
|
|
1114
|
+
const THEME_HINT_RE = /\b(theme|appearance|mode|color[- ]?scheme|dark|light|system|clair|sombre|oscuro|claro|hell|dunkel)\b/i;
|
|
1115
|
+
const STORAGE_LOCALE_RE = /\b(lang|locale|language|i18n|intl|translation|translations)\b/i;
|
|
1116
|
+
const STORAGE_THEME_RE = /\b(theme|appearance|color[-_ ]?scheme|darkmode|dark_mode|mode)\b/i;
|
|
1117
|
+
const LOGIN_BUTTON_RE = /\b(sign[\s-]?in|log[\s-]?in|connexion|se connecter|continue with email|continue|access account|login)\b/i;
|
|
1118
|
+
const LOGOUT_BUTTON_RE = /\b(sign[\s-]?out|log[\s-]?out|d[eé]connexion|se d[eé]connecter|logout)\b/i;
|
|
1119
|
+
const ACCOUNT_MENU_RE = /\b(account|profile|profil|compte|workspace|organization|organisation|team|billing|settings)\b/i;
|
|
1120
|
+
const EMAIL_LIKE_RE = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i;
|
|
1121
|
+
const PERSON_LIKE_RE = /^[A-ZÀ-Ý][A-Za-zÀ-ÿ'’-]{1,20}(?:\s+[A-ZÀ-Ý][A-Za-zÀ-ÿ'’-]{1,20}){1,2}$/;
|
|
1122
|
+
const CORPORATE_SUFFIX_RE = /\b(inc|llc|ltd|corp|gmbh|sas|sarl|company|studio|agency)\b/i;
|
|
1123
|
+
const classifyVariantKind = (combined) => {
|
|
1124
|
+
if (LOCALE_HINT_RE.test(combined))
|
|
1125
|
+
return 'locale';
|
|
1126
|
+
if (THEME_HINT_RE.test(combined))
|
|
1127
|
+
return 'theme';
|
|
1128
|
+
return 'unknown';
|
|
1129
|
+
};
|
|
1130
|
+
const classifyStorageKind = (key, value) => {
|
|
1131
|
+
const combined = `${key} ${value}`;
|
|
1132
|
+
if (STORAGE_LOCALE_RE.test(combined))
|
|
1133
|
+
return 'locale';
|
|
1134
|
+
if (STORAGE_THEME_RE.test(combined))
|
|
1135
|
+
return 'theme';
|
|
1136
|
+
return 'unknown';
|
|
1137
|
+
};
|
|
1138
|
+
const collectText = (selector, limit = 8) => {
|
|
1139
|
+
return dedupe(Array.from(document.querySelectorAll(selector))
|
|
1140
|
+
.filter(isVisible)
|
|
1141
|
+
.map((node) => textFor(node))
|
|
1142
|
+
.slice(0, limit), limit);
|
|
1143
|
+
};
|
|
1144
|
+
const looksLikeAccountText = (value) => {
|
|
1145
|
+
const normalized = (value || '').replace(/\s+/g, ' ').trim();
|
|
1146
|
+
if (!normalized)
|
|
1147
|
+
return false;
|
|
1148
|
+
if (LOGIN_BUTTON_RE.test(normalized) || LOGOUT_BUTTON_RE.test(normalized))
|
|
1149
|
+
return false;
|
|
1150
|
+
if (EMAIL_LIKE_RE.test(normalized))
|
|
1151
|
+
return true;
|
|
1152
|
+
return PERSON_LIKE_RE.test(normalized) && !CORPORATE_SUFFIX_RE.test(normalized);
|
|
1153
|
+
};
|
|
1154
|
+
const collectActionLabels = (selector, matcher, limit = 8) => dedupe(Array.from(document.querySelectorAll(selector))
|
|
1155
|
+
.filter(isVisible)
|
|
1156
|
+
.map((node) => textFor(node))
|
|
1157
|
+
.filter((text) => matcher.test(text))
|
|
1158
|
+
.slice(0, limit), limit);
|
|
1159
|
+
const body = document.body ?? document.documentElement;
|
|
1160
|
+
const walker = document.createTreeWalker(body, NodeFilter.SHOW_TEXT);
|
|
1161
|
+
let visibleText = '';
|
|
1162
|
+
while (walker.nextNode() && visibleText.length < 800) {
|
|
1163
|
+
const parent = walker.currentNode.parentElement;
|
|
1164
|
+
if (!isVisible(parent))
|
|
1165
|
+
continue;
|
|
1166
|
+
const nextText = (walker.currentNode.textContent || '').replace(/\s+/g, ' ').trim();
|
|
1167
|
+
if (!nextText)
|
|
1168
|
+
continue;
|
|
1169
|
+
visibleText += `${visibleText ? ' ' : ''}${nextText}`;
|
|
1170
|
+
}
|
|
1171
|
+
const html = document.documentElement;
|
|
1172
|
+
const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') ?? null;
|
|
1173
|
+
const hreflangs = dedupe(Array.from(document.querySelectorAll('link[rel="alternate"][hreflang]'))
|
|
1174
|
+
.map((node) => `${node.getAttribute('hreflang') || ''}:${node.getAttribute('href') || ''}`), 16);
|
|
1175
|
+
const headingTexts = collectText('h1, h2, h3');
|
|
1176
|
+
const navLabels = collectText('header a, header button, nav a, nav button, [role="navigation"] a, [role="navigation"] button');
|
|
1177
|
+
const breadcrumbLabels = dedupe(Array.from(document.querySelectorAll('nav[aria-label*="breadcrumb" i] a, nav[aria-label*="breadcrumb" i] li, [data-breadcrumb] a, .breadcrumb a, .breadcrumbs a, ol.breadcrumb li, ul.breadcrumb li'))
|
|
1178
|
+
.filter((node) => node instanceof HTMLElement && isVisible(node))
|
|
1179
|
+
.map((node) => textFor(node)), 10);
|
|
1180
|
+
const localeHints = dedupe([
|
|
1181
|
+
html.getAttribute('lang'),
|
|
1182
|
+
...Array.from(document.querySelectorAll('[hreflang], [lang], [data-lang], [data-locale]'))
|
|
1183
|
+
.flatMap((node) => [
|
|
1184
|
+
node.getAttribute('hreflang'),
|
|
1185
|
+
node.getAttribute('lang'),
|
|
1186
|
+
node.getAttribute('data-lang'),
|
|
1187
|
+
node.getAttribute('data-locale'),
|
|
1188
|
+
textFor(node),
|
|
1189
|
+
]),
|
|
1190
|
+
], 20);
|
|
1191
|
+
const themeRootHints = dedupe([
|
|
1192
|
+
html.className || '',
|
|
1193
|
+
document.body?.className || '',
|
|
1194
|
+
html.getAttribute('data-theme') || '',
|
|
1195
|
+
document.body?.getAttribute('data-theme') || '',
|
|
1196
|
+
html.getAttribute('theme') || '',
|
|
1197
|
+
document.body?.getAttribute('theme') || '',
|
|
1198
|
+
html.getAttribute('color-scheme') || '',
|
|
1199
|
+
document.body?.getAttribute('color-scheme') || '',
|
|
1200
|
+
document.querySelector('meta[name="color-scheme"]')?.getAttribute('content') || '',
|
|
1201
|
+
], 16);
|
|
1202
|
+
const themeRootTokens = themeRootHints.join(' ').toLowerCase();
|
|
1203
|
+
const parseColor = (value) => {
|
|
1204
|
+
if (!value)
|
|
1205
|
+
return null;
|
|
1206
|
+
const normalized = value.trim().toLowerCase();
|
|
1207
|
+
const rgbMatch = normalized.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([0-9.]+))?\)/);
|
|
1208
|
+
if (rgbMatch) {
|
|
1209
|
+
return {
|
|
1210
|
+
r: Number(rgbMatch[1]),
|
|
1211
|
+
g: Number(rgbMatch[2]),
|
|
1212
|
+
b: Number(rgbMatch[3]),
|
|
1213
|
+
a: rgbMatch[4] ? Number(rgbMatch[4]) : 1,
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
const hexMatch = normalized.match(/^#([0-9a-f]{6}|[0-9a-f]{3})$/i);
|
|
1217
|
+
if (hexMatch) {
|
|
1218
|
+
const hex = hexMatch[1];
|
|
1219
|
+
if (hex.length === 3) {
|
|
1220
|
+
return {
|
|
1221
|
+
r: Number.parseInt(`${hex[0]}${hex[0]}`, 16),
|
|
1222
|
+
g: Number.parseInt(`${hex[1]}${hex[1]}`, 16),
|
|
1223
|
+
b: Number.parseInt(`${hex[2]}${hex[2]}`, 16),
|
|
1224
|
+
a: 1,
|
|
1225
|
+
};
|
|
1226
|
+
}
|
|
1227
|
+
return {
|
|
1228
|
+
r: Number.parseInt(hex.slice(0, 2), 16),
|
|
1229
|
+
g: Number.parseInt(hex.slice(2, 4), 16),
|
|
1230
|
+
b: Number.parseInt(hex.slice(4, 6), 16),
|
|
1231
|
+
a: 1,
|
|
1232
|
+
};
|
|
1233
|
+
}
|
|
1234
|
+
return null;
|
|
1235
|
+
};
|
|
1236
|
+
const luminanceFor = (value) => {
|
|
1237
|
+
const parsed = parseColor(value);
|
|
1238
|
+
if (!parsed || parsed.a === 0)
|
|
1239
|
+
return null;
|
|
1240
|
+
const normalize = (channel) => {
|
|
1241
|
+
const srgb = channel / 255;
|
|
1242
|
+
return srgb <= 0.03928 ? srgb / 12.92 : ((srgb + 0.055) / 1.055) ** 2.4;
|
|
1243
|
+
};
|
|
1244
|
+
const r = normalize(parsed.r);
|
|
1245
|
+
const g = normalize(parsed.g);
|
|
1246
|
+
const b = normalize(parsed.b);
|
|
1247
|
+
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
|
1248
|
+
};
|
|
1249
|
+
const chromeThemeSamples = dedupeObjects(['body', 'header', 'nav', 'aside', '[role="navigation"]', '[data-toolbar]', '.toolbar']
|
|
1250
|
+
.flatMap((selector) => Array.from(document.querySelectorAll(selector))
|
|
1251
|
+
.filter(isVisible)
|
|
1252
|
+
.slice(0, selector === 'body' ? 1 : 2)
|
|
1253
|
+
.map((node) => {
|
|
1254
|
+
const style = window.getComputedStyle(node);
|
|
1255
|
+
const area = selector === 'body'
|
|
1256
|
+
? 'body'
|
|
1257
|
+
: selector === 'header'
|
|
1258
|
+
? 'header'
|
|
1259
|
+
: selector === 'nav' || selector === '[role="navigation"]'
|
|
1260
|
+
? 'nav'
|
|
1261
|
+
: selector === 'aside'
|
|
1262
|
+
? 'aside'
|
|
1263
|
+
: 'toolbar';
|
|
1264
|
+
return {
|
|
1265
|
+
area,
|
|
1266
|
+
selector,
|
|
1267
|
+
background: style.backgroundColor || null,
|
|
1268
|
+
color: style.color || null,
|
|
1269
|
+
luminance: luminanceFor(style.backgroundColor || null),
|
|
1270
|
+
};
|
|
1271
|
+
})), (sample) => `${sample.area}|${sample.selector}|${sample.background}|${sample.color}`, 10);
|
|
1272
|
+
let detectedTheme = null;
|
|
1273
|
+
const hasDarkToken = /\bdark\b/.test(themeRootTokens);
|
|
1274
|
+
const hasLightToken = /\blight\b/.test(themeRootTokens);
|
|
1275
|
+
const brightChrome = chromeThemeSamples.filter((sample) => sample.luminance !== null && sample.luminance >= 0.58).length;
|
|
1276
|
+
const darkChrome = chromeThemeSamples.filter((sample) => sample.luminance !== null && sample.luminance <= 0.28).length;
|
|
1277
|
+
if (hasDarkToken !== hasLightToken) {
|
|
1278
|
+
const tokenTheme = hasDarkToken ? 'dark' : 'light';
|
|
1279
|
+
if ((tokenTheme === 'dark' && brightChrome === 0)
|
|
1280
|
+
|| (tokenTheme === 'light' && darkChrome === 0)
|
|
1281
|
+
|| chromeThemeSamples.length === 0) {
|
|
1282
|
+
detectedTheme = tokenTheme;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
else if (brightChrome > 0 && darkChrome === 0) {
|
|
1286
|
+
detectedTheme = 'light';
|
|
1287
|
+
}
|
|
1288
|
+
else if (darkChrome > 0 && brightChrome === 0) {
|
|
1289
|
+
detectedTheme = 'dark';
|
|
1290
|
+
}
|
|
1291
|
+
const preferredColorScheme = window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
|
|
1292
|
+
const variantControls = dedupeObjects(Array.from(document.querySelectorAll('select, button, a[href], summary, [role="button"], [role="link"], [role="menuitem"], [role="menuitemradio"], [role="switch"], [role="radio"], [aria-haspopup], [data-lang], [data-language], [data-locale], [data-theme], [data-color-scheme], [theme], [color-scheme]'))
|
|
1293
|
+
.filter(isVisible)
|
|
1294
|
+
.flatMap((node) => {
|
|
1295
|
+
if (!(node instanceof HTMLElement))
|
|
1296
|
+
return [];
|
|
1297
|
+
const label = textFor(node);
|
|
1298
|
+
const tag = node.tagName.toLowerCase();
|
|
1299
|
+
const role = node.getAttribute('role') || '';
|
|
1300
|
+
const href = node instanceof HTMLAnchorElement ? node.href : node.getAttribute('href');
|
|
1301
|
+
const value = node instanceof HTMLInputElement || node instanceof HTMLSelectElement
|
|
1302
|
+
? node.value
|
|
1303
|
+
: node.getAttribute('value');
|
|
1304
|
+
const attrBag = [
|
|
1305
|
+
label,
|
|
1306
|
+
role,
|
|
1307
|
+
href,
|
|
1308
|
+
value,
|
|
1309
|
+
node.getAttribute('aria-label'),
|
|
1310
|
+
node.getAttribute('title'),
|
|
1311
|
+
node.getAttribute('name'),
|
|
1312
|
+
node.getAttribute('id'),
|
|
1313
|
+
node.getAttribute('data-lang'),
|
|
1314
|
+
node.getAttribute('data-language'),
|
|
1315
|
+
node.getAttribute('data-locale'),
|
|
1316
|
+
node.getAttribute('data-theme'),
|
|
1317
|
+
node.getAttribute('data-color-scheme'),
|
|
1318
|
+
node.getAttribute('theme'),
|
|
1319
|
+
node.getAttribute('color-scheme'),
|
|
1320
|
+
node.getAttribute('hreflang'),
|
|
1321
|
+
].filter(Boolean).join(' ');
|
|
1322
|
+
let kind = classifyVariantKind(attrBag);
|
|
1323
|
+
let mechanism = 'custom';
|
|
1324
|
+
let options;
|
|
1325
|
+
if (node instanceof HTMLSelectElement) {
|
|
1326
|
+
mechanism = 'select';
|
|
1327
|
+
options = Array.from(node.options)
|
|
1328
|
+
.map((option) => ({
|
|
1329
|
+
label: (option.textContent || option.label || '').replace(/\s+/g, ' ').trim().slice(0, 80),
|
|
1330
|
+
value: option.value || null,
|
|
1331
|
+
selected: option.selected,
|
|
1332
|
+
}))
|
|
1333
|
+
.filter((option) => option.label || option.value)
|
|
1334
|
+
.slice(0, 16);
|
|
1335
|
+
if (kind === 'unknown') {
|
|
1336
|
+
const optionText = options.map((option) => `${option.label} ${option.value || ''}`).join(' ');
|
|
1337
|
+
kind = classifyVariantKind(`${attrBag} ${optionText}`);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
else if (tag === 'a' || role === 'link') {
|
|
1341
|
+
mechanism = 'link';
|
|
1342
|
+
}
|
|
1343
|
+
else if (role === 'switch' || node.getAttribute('aria-pressed') != null || /toggle/i.test(attrBag)) {
|
|
1344
|
+
mechanism = 'toggle';
|
|
1345
|
+
}
|
|
1346
|
+
else if (role.startsWith('menuitem')) {
|
|
1347
|
+
mechanism = 'menuitem';
|
|
1348
|
+
}
|
|
1349
|
+
else if (role === 'radio' || node.getAttribute('type') === 'radio') {
|
|
1350
|
+
mechanism = 'radio';
|
|
1351
|
+
}
|
|
1352
|
+
else {
|
|
1353
|
+
mechanism = 'button';
|
|
1354
|
+
}
|
|
1355
|
+
const selector = buildSelector(node);
|
|
1356
|
+
if (!selector)
|
|
1357
|
+
return [];
|
|
1358
|
+
if (kind === 'unknown')
|
|
1359
|
+
return [];
|
|
1360
|
+
return [{
|
|
1361
|
+
kind,
|
|
1362
|
+
mechanism,
|
|
1363
|
+
selector,
|
|
1364
|
+
label: label || node.getAttribute('aria-label') || node.getAttribute('title') || selector,
|
|
1365
|
+
value: value || null,
|
|
1366
|
+
href: href || null,
|
|
1367
|
+
tag,
|
|
1368
|
+
role,
|
|
1369
|
+
options,
|
|
1370
|
+
}];
|
|
1371
|
+
}), (control) => `${control.kind}|${control.selector}|${control.label}`, 20);
|
|
1372
|
+
const collectStorageHints = (storage, storageName) => {
|
|
1373
|
+
const hints = [];
|
|
1374
|
+
for (let index = 0; index < storage.length; index += 1) {
|
|
1375
|
+
const key = storage.key(index);
|
|
1376
|
+
if (!key)
|
|
1377
|
+
continue;
|
|
1378
|
+
const value = storage.getItem(key) || '';
|
|
1379
|
+
const normalizedValue = value.replace(/\s+/g, ' ').trim().slice(0, 140);
|
|
1380
|
+
const kind = classifyStorageKind(key, normalizedValue);
|
|
1381
|
+
if (kind === 'unknown')
|
|
1382
|
+
continue;
|
|
1383
|
+
hints.push({
|
|
1384
|
+
storage: storageName,
|
|
1385
|
+
key,
|
|
1386
|
+
kind,
|
|
1387
|
+
valueSample: normalizedValue,
|
|
1388
|
+
});
|
|
1389
|
+
}
|
|
1390
|
+
return hints;
|
|
1391
|
+
};
|
|
1392
|
+
const storageHints = dedupeObjects([
|
|
1393
|
+
...collectStorageHints(window.localStorage, 'localStorage'),
|
|
1394
|
+
...collectStorageHints(window.sessionStorage, 'sessionStorage'),
|
|
1395
|
+
], (hint) => `${hint.storage}|${hint.kind}|${hint.key}`, 20);
|
|
1396
|
+
const hasPasswordField = Array.from(document.querySelectorAll('input[type="password"], input[autocomplete="current-password"], input[autocomplete="new-password"]')).some((node) => isVisible(node));
|
|
1397
|
+
const hasEmailField = Array.from(document.querySelectorAll('input[type="email"], input[autocomplete="email"], input[autocomplete="username"], input[name*="email" i], input[id*="email" i]')).some((node) => isVisible(node));
|
|
1398
|
+
const hasAuthForm = Array.from(document.querySelectorAll('form'))
|
|
1399
|
+
.filter(isVisible)
|
|
1400
|
+
.some((form) => {
|
|
1401
|
+
const formText = [
|
|
1402
|
+
textFor(form),
|
|
1403
|
+
...Array.from(form.querySelectorAll('button, a[href], input, [role="button"]'))
|
|
1404
|
+
.filter(isVisible)
|
|
1405
|
+
.map((node) => textFor(node)),
|
|
1406
|
+
].join(' ');
|
|
1407
|
+
const formHasPassword = Array.from(form.querySelectorAll('input[type="password"], input[autocomplete="current-password"], input[autocomplete="new-password"]')).some((node) => isVisible(node));
|
|
1408
|
+
const formHasEmail = Array.from(form.querySelectorAll('input[type="email"], input[autocomplete="email"], input[autocomplete="username"], input[name*="email" i], input[id*="email" i]')).some((node) => isVisible(node));
|
|
1409
|
+
return formHasPassword || (formHasEmail && LOGIN_BUTTON_RE.test(formText));
|
|
1410
|
+
});
|
|
1411
|
+
const loginButtons = collectActionLabels('button, a[href], input[type="submit"], [role="button"], [role="menuitem"]', LOGIN_BUTTON_RE);
|
|
1412
|
+
const logoutButtons = collectActionLabels('button, a[href], input[type="submit"], [role="button"], [role="menuitem"]', LOGOUT_BUTTON_RE);
|
|
1413
|
+
const accountMenuLabels = dedupe(Array.from(document.querySelectorAll('header a, header button, nav a, nav button, aside a, aside button, summary, [aria-haspopup], [role="menuitem"], [data-testid*="account" i], [data-testid*="profile" i]'))
|
|
1414
|
+
.filter(isVisible)
|
|
1415
|
+
.map((node) => textFor(node))
|
|
1416
|
+
.filter((text) => ACCOUNT_MENU_RE.test(text) || looksLikeAccountText(text)), 8);
|
|
1417
|
+
const accountLikeText = dedupe([
|
|
1418
|
+
...accountMenuLabels.filter((text) => looksLikeAccountText(text)),
|
|
1419
|
+
...navLabels.filter((text) => looksLikeAccountText(text)),
|
|
1420
|
+
...breadcrumbLabels.filter((text) => looksLikeAccountText(text)),
|
|
1421
|
+
], 8);
|
|
1422
|
+
return {
|
|
1423
|
+
url: window.location.href,
|
|
1424
|
+
title: document.title || '',
|
|
1425
|
+
htmlLang: html.getAttribute('lang'),
|
|
1426
|
+
canonicalUrl: canonical,
|
|
1427
|
+
hreflangs,
|
|
1428
|
+
headings: headingTexts,
|
|
1429
|
+
navLabels,
|
|
1430
|
+
breadcrumbLabels,
|
|
1431
|
+
visibleText: visibleText.slice(0, 800),
|
|
1432
|
+
localeHints,
|
|
1433
|
+
detectedTheme,
|
|
1434
|
+
preferredColorScheme,
|
|
1435
|
+
themeRootHints,
|
|
1436
|
+
chromeThemeSamples,
|
|
1437
|
+
authHints: {
|
|
1438
|
+
hasPasswordField,
|
|
1439
|
+
hasEmailField,
|
|
1440
|
+
hasAuthForm,
|
|
1441
|
+
loginButtons,
|
|
1442
|
+
logoutButtons,
|
|
1443
|
+
accountMenuLabels,
|
|
1444
|
+
accountLikeText,
|
|
1445
|
+
},
|
|
1446
|
+
variantControls,
|
|
1447
|
+
storageHints,
|
|
1448
|
+
};
|
|
1449
|
+
}));
|
|
1450
|
+
}
|
|
1451
|
+
async captureObservation() {
|
|
1452
|
+
const page = this.ensurePage();
|
|
1453
|
+
return page.evaluate(() => {
|
|
1454
|
+
const root = document.body ?? document.documentElement;
|
|
1455
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
|
|
1456
|
+
let textSample = '';
|
|
1457
|
+
while (walker.nextNode() && textSample.length < 240) {
|
|
1458
|
+
const nextText = (walker.currentNode.textContent || '').replace(/\s+/g, ' ').trim();
|
|
1459
|
+
if (!nextText)
|
|
1460
|
+
continue;
|
|
1461
|
+
textSample += `${textSample ? ' ' : ''}${nextText}`;
|
|
1462
|
+
}
|
|
1463
|
+
const interactiveSelectors = [
|
|
1464
|
+
'a[href]', 'button', 'input', 'select', 'textarea',
|
|
1465
|
+
'[role="button"]', '[role="link"]', '[role="tab"]',
|
|
1466
|
+
'[role="menuitem"]', '[role="checkbox"]', '[role="radio"]',
|
|
1467
|
+
'[role="switch"]', '[role="combobox"]', '[tabindex]:not([tabindex="-1"])',
|
|
1468
|
+
];
|
|
1469
|
+
const isVisible = (node) => {
|
|
1470
|
+
if (!(node instanceof HTMLElement))
|
|
1471
|
+
return false;
|
|
1472
|
+
const style = window.getComputedStyle(node);
|
|
1473
|
+
if (style.display === 'none' || style.visibility === 'hidden' || parseFloat(style.opacity || '1') === 0) {
|
|
1474
|
+
return false;
|
|
1475
|
+
}
|
|
1476
|
+
const rect = node.getBoundingClientRect();
|
|
1477
|
+
return rect.width > 0 && rect.height > 0;
|
|
1478
|
+
};
|
|
1479
|
+
const dialogCount = Array.from(document.querySelectorAll('dialog[open], [role="dialog"], [aria-modal="true"]')).filter(isVisible).length;
|
|
1480
|
+
const expandedCount = Array.from(document.querySelectorAll('[aria-expanded="true"], details[open]')).filter(isVisible).length;
|
|
1481
|
+
const loadingSelectors = [
|
|
1482
|
+
'[aria-busy="true"]',
|
|
1483
|
+
'[class*="spinner"]', '[class*="loading"]', '[class*="skeleton"]',
|
|
1484
|
+
'[class*="Spinner"]', '[class*="Loading"]', '[class*="Skeleton"]',
|
|
1485
|
+
];
|
|
1486
|
+
const loadingIndicatorCount = Array.from(document.querySelectorAll(loadingSelectors.join(','))).filter(isVisible).length;
|
|
1487
|
+
return {
|
|
1488
|
+
url: window.location.href,
|
|
1489
|
+
title: document.title || '',
|
|
1490
|
+
readyState: document.readyState,
|
|
1491
|
+
scrollX: Math.round(window.scrollX),
|
|
1492
|
+
scrollY: Math.round(window.scrollY),
|
|
1493
|
+
scrollHeight: document.documentElement.scrollHeight,
|
|
1494
|
+
interactiveCount: Array.from(document.querySelectorAll(interactiveSelectors.join(','))).filter(isVisible).length,
|
|
1495
|
+
dialogCount,
|
|
1496
|
+
expandedCount,
|
|
1497
|
+
loadingIndicatorCount,
|
|
1498
|
+
textSample: textSample.slice(0, 240),
|
|
1499
|
+
};
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
async captureVerificationBundle(options = {}) {
|
|
1503
|
+
const page = this.ensurePage();
|
|
1504
|
+
const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
|
|
1505
|
+
const settleMs = Math.max(120, options.settleMs ?? 180);
|
|
1506
|
+
let lastError = null;
|
|
1507
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
1508
|
+
// Run networkidle + DOM stability in parallel — they are independent signals
|
|
1509
|
+
await Promise.all([
|
|
1510
|
+
page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => { }),
|
|
1511
|
+
this.waitForDomStability(page, settleMs, 1800).catch(() => { }),
|
|
1512
|
+
]);
|
|
1513
|
+
const before = await this.captureObservation();
|
|
1514
|
+
const screenshot = await this.takeScreenshotForAI();
|
|
1515
|
+
const [pageSignals, after] = await Promise.all([
|
|
1516
|
+
this.capturePageSignals(),
|
|
1517
|
+
this.captureObservation(),
|
|
1518
|
+
]);
|
|
1519
|
+
const sameUrl = before.url === pageSignals.url && pageSignals.url === after.url;
|
|
1520
|
+
const pageReady = before.readyState !== 'loading' && after.readyState !== 'loading';
|
|
1521
|
+
if (sameUrl && pageReady) {
|
|
1522
|
+
const capturedAt = new Date().toISOString();
|
|
1523
|
+
const coherenceKey = createHash('sha1')
|
|
1524
|
+
.update(`${pageSignals.url}|${after.readyState}|${after.scrollY}|${capturedAt}`)
|
|
1525
|
+
.digest('hex')
|
|
1526
|
+
.slice(0, 16);
|
|
1527
|
+
return {
|
|
1528
|
+
screenshot,
|
|
1529
|
+
pageSignals,
|
|
1530
|
+
observation: after,
|
|
1531
|
+
url: pageSignals.url,
|
|
1532
|
+
title: pageSignals.title || after.title,
|
|
1533
|
+
capturedAt,
|
|
1534
|
+
coherenceKey,
|
|
1535
|
+
};
|
|
1536
|
+
}
|
|
1537
|
+
lastError = new Error(`stale verification snapshot (attempt ${attempt}/${maxAttempts}): `
|
|
1538
|
+
+ `url ${before.url} -> ${pageSignals.url} -> ${after.url}; `
|
|
1539
|
+
+ `readyState ${before.readyState} -> ${after.readyState}`);
|
|
1540
|
+
if (attempt < maxAttempts) {
|
|
1541
|
+
await page.waitForTimeout(120);
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
throw lastError ?? new Error('stale verification snapshot');
|
|
1545
|
+
}
|
|
1546
|
+
async captureVideoVerificationBundle(options = {}) {
|
|
1547
|
+
const page = this.ensurePage();
|
|
1548
|
+
const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
|
|
1549
|
+
const settleMs = Math.max(120, options.settleMs ?? 180);
|
|
1550
|
+
let lastError = null;
|
|
1551
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
1552
|
+
// Run networkidle + DOM stability in parallel — they are independent signals
|
|
1553
|
+
await Promise.all([
|
|
1554
|
+
page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => { }),
|
|
1555
|
+
this.waitForDomStability(page, settleMs, 1800).catch(() => { }),
|
|
1556
|
+
]);
|
|
1557
|
+
const before = await this.captureObservation();
|
|
1558
|
+
const screenshot = await this.takeScreenshotForAI({ timeoutMs: options.helperTimeoutMs });
|
|
1559
|
+
const [pageSignals, accessibilityTree, interactiveElements, scrollInfo, after] = await Promise.all([
|
|
1560
|
+
this.capturePageSignals({ timeoutMs: options.helperTimeoutMs }),
|
|
1561
|
+
this.getAccessibilityTree({ timeoutMs: options.helperTimeoutMs }),
|
|
1562
|
+
this.getInteractiveElements({ timeoutMs: options.helperTimeoutMs }),
|
|
1563
|
+
page.evaluate(() => ({
|
|
1564
|
+
scrollY: Math.round(window.scrollY),
|
|
1565
|
+
scrollHeight: document.documentElement.scrollHeight,
|
|
1566
|
+
viewportHeight: window.innerHeight,
|
|
1567
|
+
})),
|
|
1568
|
+
this.captureObservation(),
|
|
1569
|
+
]);
|
|
1570
|
+
const sameUrl = before.url === pageSignals.url && pageSignals.url === after.url;
|
|
1571
|
+
const pageReady = before.readyState !== 'loading' && after.readyState !== 'loading';
|
|
1572
|
+
if (sameUrl && pageReady) {
|
|
1573
|
+
const capturedAt = new Date().toISOString();
|
|
1574
|
+
const coherenceKey = createHash('sha1')
|
|
1575
|
+
.update(`${pageSignals.url}|${after.readyState}|${after.scrollY}|${capturedAt}`)
|
|
1576
|
+
.digest('hex')
|
|
1577
|
+
.slice(0, 16);
|
|
1578
|
+
return {
|
|
1579
|
+
screenshot,
|
|
1580
|
+
pageSignals,
|
|
1581
|
+
observation: after,
|
|
1582
|
+
accessibilityTree,
|
|
1583
|
+
interactiveElements,
|
|
1584
|
+
scrollInfo,
|
|
1585
|
+
url: pageSignals.url,
|
|
1586
|
+
title: pageSignals.title || after.title,
|
|
1587
|
+
capturedAt,
|
|
1588
|
+
coherenceKey,
|
|
1589
|
+
};
|
|
1590
|
+
}
|
|
1591
|
+
lastError = new Error(`stale verification snapshot (attempt ${attempt}/${maxAttempts}): `
|
|
1592
|
+
+ `url ${before.url} -> ${pageSignals.url} -> ${after.url}; `
|
|
1593
|
+
+ `readyState ${before.readyState} -> ${after.readyState}`);
|
|
1594
|
+
options.onRetry?.(lastError.message);
|
|
1595
|
+
if (attempt < maxAttempts) {
|
|
1596
|
+
await page.waitForTimeout(120);
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
throw lastError ?? new Error('stale verification snapshot');
|
|
1600
|
+
}
|
|
1601
|
+
async waitForPageReaction(before, options = {}) {
|
|
1602
|
+
const page = this.ensurePage();
|
|
1603
|
+
const baseline = before ?? await this.captureObservation();
|
|
1604
|
+
const timeoutMs = Math.max(250, options.timeoutMs ?? 2200);
|
|
1605
|
+
const settleMs = Math.max(150, options.settleMs ?? 350);
|
|
1606
|
+
const deadline = Date.now() + timeoutMs;
|
|
1607
|
+
let last = baseline;
|
|
1608
|
+
let lastMovementAt = Date.now();
|
|
1609
|
+
let observedChange = false;
|
|
1610
|
+
await page.waitForTimeout(100);
|
|
1611
|
+
while (Date.now() < deadline) {
|
|
1612
|
+
const current = await this.captureObservation().catch(() => last);
|
|
1613
|
+
const sinceBaseline = describeObservationChange(baseline, current);
|
|
1614
|
+
const sinceLast = describeObservationChange(last, current);
|
|
1615
|
+
if (sinceBaseline.changed) {
|
|
1616
|
+
observedChange = true;
|
|
1617
|
+
}
|
|
1618
|
+
if (sinceLast.changed || current.readyState !== last.readyState) {
|
|
1619
|
+
last = current;
|
|
1620
|
+
lastMovementAt = Date.now();
|
|
1621
|
+
}
|
|
1622
|
+
const quietForMs = Date.now() - lastMovementAt;
|
|
1623
|
+
const ready = current.readyState !== 'loading';
|
|
1624
|
+
const noChangeGraceMs = Math.max(150, options.idleGraceMs ?? Math.min(settleMs, 250));
|
|
1625
|
+
// Fast settle: when the page is fully stable (complete + no loading indicators + no URL change),
|
|
1626
|
+
// use a reduced settle threshold of 150ms instead of the full settleMs (typically 350ms).
|
|
1627
|
+
const isFullyStable = ready && current.readyState === 'complete'
|
|
1628
|
+
&& current.loadingIndicatorCount === 0
|
|
1629
|
+
&& current.url === baseline.url;
|
|
1630
|
+
const effectiveSettleMs = (observedChange && isFullyStable) ? Math.min(settleMs, 150) : settleMs;
|
|
1631
|
+
if ((observedChange && quietForMs >= effectiveSettleMs && ready) || (!observedChange && quietForMs >= noChangeGraceMs && ready)) {
|
|
1632
|
+
return describeObservationChange(baseline, current);
|
|
1633
|
+
}
|
|
1634
|
+
if (observedChange && current.url !== baseline.url) {
|
|
1635
|
+
const remaining = deadline - Date.now();
|
|
1636
|
+
if (remaining > 200) {
|
|
1637
|
+
await page.waitForLoadState('load', { timeout: Math.min(remaining, 900) }).catch(() => { });
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
await page.waitForTimeout(120);
|
|
1641
|
+
}
|
|
1642
|
+
const after = await this.captureObservation().catch(() => baseline);
|
|
1643
|
+
return describeObservationChange(baseline, after);
|
|
1644
|
+
}
|
|
1645
|
+
computeVisibilityState(box, viewport) {
|
|
1646
|
+
const visibleWidth = Math.max(0, Math.min(box.x + box.width, viewport.width) - Math.max(box.x, 0));
|
|
1647
|
+
const visibleHeight = Math.max(0, Math.min(box.y + box.height, viewport.height) - Math.max(box.y, 0));
|
|
1648
|
+
if (visibleWidth === 0 || visibleHeight === 0)
|
|
1649
|
+
return 'offscreen';
|
|
1650
|
+
if (box.x >= 0 && box.y >= 0 && box.x + box.width <= viewport.width && box.y + box.height <= viewport.height) {
|
|
1651
|
+
return 'full';
|
|
1652
|
+
}
|
|
1653
|
+
return 'partial';
|
|
1654
|
+
}
|
|
1655
|
+
getVisiblePoint(box, viewport) {
|
|
1656
|
+
const left = Math.max(1, box.x);
|
|
1657
|
+
const right = Math.min(viewport.width - 1, box.x + box.width);
|
|
1658
|
+
const top = Math.max(1, box.y);
|
|
1659
|
+
const bottom = Math.min(viewport.height - 1, box.y + box.height);
|
|
1660
|
+
if (right <= left || bottom <= top) {
|
|
1661
|
+
return {
|
|
1662
|
+
x: Math.max(1, Math.min(viewport.width - 1, Math.round(box.x + box.width / 2))),
|
|
1663
|
+
y: Math.max(1, Math.min(viewport.height - 1, Math.round(box.y + box.height / 2))),
|
|
1664
|
+
};
|
|
1665
|
+
}
|
|
1666
|
+
return {
|
|
1667
|
+
x: Math.round((left + right) / 2),
|
|
1668
|
+
y: Math.round((top + bottom) / 2),
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
async withResolvedLocator(element, fn) {
|
|
1672
|
+
const page = this.ensurePage();
|
|
1673
|
+
if (!element.selector)
|
|
1674
|
+
return null;
|
|
1675
|
+
const token = `ak-target-${Math.random().toString(36).slice(2, 10)}`;
|
|
1676
|
+
const matched = await page.evaluate(({ selector, pageBox, token, tag, text, ariaLabel, role, inputType }) => {
|
|
1677
|
+
document.querySelectorAll('[data-ak-target]').forEach(el => el.removeAttribute('data-ak-target'));
|
|
1678
|
+
const matches = Array.from(document.querySelectorAll(selector));
|
|
1679
|
+
if (matches.length === 0)
|
|
1680
|
+
return false;
|
|
1681
|
+
const normalizedText = text.trim().toLowerCase();
|
|
1682
|
+
const normalizedAria = (ariaLabel || '').trim().toLowerCase();
|
|
1683
|
+
const targetCenterX = pageBox ? pageBox.x + pageBox.width / 2 : null;
|
|
1684
|
+
const targetCenterY = pageBox ? pageBox.y + pageBox.height / 2 : null;
|
|
1685
|
+
const scored = matches
|
|
1686
|
+
.map((node) => {
|
|
1687
|
+
const el = node;
|
|
1688
|
+
const rect = el.getBoundingClientRect();
|
|
1689
|
+
if (rect.width === 0 && rect.height === 0)
|
|
1690
|
+
return null;
|
|
1691
|
+
const absX = rect.x + window.scrollX;
|
|
1692
|
+
const absY = rect.y + window.scrollY;
|
|
1693
|
+
const centerX = absX + rect.width / 2;
|
|
1694
|
+
const centerY = absY + rect.height / 2;
|
|
1695
|
+
const ownText = (el.textContent || '').trim().slice(0, 120).toLowerCase();
|
|
1696
|
+
const ownAria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
|
1697
|
+
const ownRole = el.getAttribute('role')
|
|
1698
|
+
|| (el.tagName === 'A' ? 'link' : '')
|
|
1699
|
+
|| (el.tagName === 'BUTTON' ? 'button' : '')
|
|
1700
|
+
|| (el.tagName === 'INPUT' ? `input[${el.type}]` : '')
|
|
1701
|
+
|| el.tagName.toLowerCase();
|
|
1702
|
+
const ownInputType = (el.tagName === 'INPUT' || el.tagName === 'SELECT' || el.tagName === 'TEXTAREA')
|
|
1703
|
+
? el.type || el.tagName.toLowerCase()
|
|
1704
|
+
: null;
|
|
1705
|
+
let score = 0;
|
|
1706
|
+
if (targetCenterX != null && targetCenterY != null) {
|
|
1707
|
+
score += Math.abs(centerX - targetCenterX) + Math.abs(centerY - targetCenterY);
|
|
1708
|
+
}
|
|
1709
|
+
if (pageBox) {
|
|
1710
|
+
score += Math.abs(rect.width - pageBox.width) * 1.5;
|
|
1711
|
+
score += Math.abs(rect.height - pageBox.height) * 1.5;
|
|
1712
|
+
}
|
|
1713
|
+
if (el.tagName.toLowerCase() !== tag)
|
|
1714
|
+
score += 200;
|
|
1715
|
+
if (ownRole !== role)
|
|
1716
|
+
score += 120;
|
|
1717
|
+
if ((ownInputType || null) !== (inputType || null))
|
|
1718
|
+
score += 80;
|
|
1719
|
+
if (normalizedAria && ownAria !== normalizedAria)
|
|
1720
|
+
score += 120;
|
|
1721
|
+
if (normalizedText) {
|
|
1722
|
+
if (ownText === normalizedText)
|
|
1723
|
+
score -= 120;
|
|
1724
|
+
else if (ownText.includes(normalizedText) || normalizedText.includes(ownText))
|
|
1725
|
+
score -= 60;
|
|
1726
|
+
else
|
|
1727
|
+
score += 160;
|
|
1728
|
+
}
|
|
1729
|
+
return { el, score };
|
|
1730
|
+
})
|
|
1731
|
+
.filter((entry) => entry !== null)
|
|
1732
|
+
.sort((a, b) => a.score - b.score);
|
|
1733
|
+
const best = scored[0]?.el;
|
|
1734
|
+
if (!best)
|
|
1735
|
+
return false;
|
|
1736
|
+
best.setAttribute('data-ak-target', token);
|
|
1737
|
+
return true;
|
|
1738
|
+
}, {
|
|
1739
|
+
selector: element.selector,
|
|
1740
|
+
pageBox: element.pageBox,
|
|
1741
|
+
token,
|
|
1742
|
+
tag: element.tag,
|
|
1743
|
+
text: element.text,
|
|
1744
|
+
ariaLabel: element.ariaLabel,
|
|
1745
|
+
role: element.role,
|
|
1746
|
+
inputType: element.inputType,
|
|
1747
|
+
});
|
|
1748
|
+
if (!matched)
|
|
1749
|
+
return null;
|
|
1750
|
+
try {
|
|
1751
|
+
const locator = page.locator(`[data-ak-target="${token}"]`).first();
|
|
1752
|
+
return await fn(locator);
|
|
1753
|
+
}
|
|
1754
|
+
finally {
|
|
1755
|
+
await page.evaluate((targetToken) => {
|
|
1756
|
+
document
|
|
1757
|
+
.querySelectorAll(`[data-ak-target="${targetToken}"]`)
|
|
1758
|
+
.forEach(el => el.removeAttribute('data-ak-target'));
|
|
1759
|
+
}, token).catch(() => { });
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
async measureElement(index) {
|
|
1763
|
+
const page = this.ensurePage();
|
|
1764
|
+
const element = this.elementMap.get(index);
|
|
1765
|
+
if (!element)
|
|
1766
|
+
throw new Error(`Element index ${index} not found`);
|
|
1767
|
+
const measuredFromDom = await this.withResolvedLocator(element, async (locator) => {
|
|
1768
|
+
const box = await locator.boundingBox();
|
|
1769
|
+
if (!box)
|
|
1770
|
+
return null;
|
|
1771
|
+
const viewport = page.viewportSize();
|
|
1772
|
+
if (!viewport)
|
|
1773
|
+
return null;
|
|
1774
|
+
const scroll = await page.evaluate(() => ({ x: window.scrollX, y: window.scrollY }));
|
|
1775
|
+
const roundedBox = {
|
|
1776
|
+
x: Math.round(box.x),
|
|
1777
|
+
y: Math.round(box.y),
|
|
1778
|
+
width: Math.round(box.width),
|
|
1779
|
+
height: Math.round(box.height),
|
|
1780
|
+
};
|
|
1781
|
+
return {
|
|
1782
|
+
boundingBox: roundedBox,
|
|
1783
|
+
pageBox: {
|
|
1784
|
+
x: Math.round(box.x + scroll.x),
|
|
1785
|
+
y: Math.round(box.y + scroll.y),
|
|
1786
|
+
width: Math.round(box.width),
|
|
1787
|
+
height: Math.round(box.height),
|
|
1788
|
+
},
|
|
1789
|
+
visibilityState: this.computeVisibilityState(roundedBox, viewport),
|
|
1790
|
+
};
|
|
1791
|
+
});
|
|
1792
|
+
if (measuredFromDom)
|
|
1793
|
+
return measuredFromDom;
|
|
1794
|
+
if (!element.pageBox)
|
|
1795
|
+
return null;
|
|
1796
|
+
const viewport = page.viewportSize();
|
|
1797
|
+
if (!viewport)
|
|
1798
|
+
return null;
|
|
1799
|
+
const scroll = await page.evaluate(() => ({ x: window.scrollX, y: window.scrollY }));
|
|
1800
|
+
const fallbackBox = {
|
|
1801
|
+
x: Math.round(element.pageBox.x - scroll.x),
|
|
1802
|
+
y: Math.round(element.pageBox.y - scroll.y),
|
|
1803
|
+
width: element.pageBox.width,
|
|
1804
|
+
height: element.pageBox.height,
|
|
1805
|
+
};
|
|
1806
|
+
return {
|
|
1807
|
+
boundingBox: fallbackBox,
|
|
1808
|
+
pageBox: element.pageBox,
|
|
1809
|
+
visibilityState: this.computeVisibilityState(fallbackBox, viewport),
|
|
1810
|
+
};
|
|
1811
|
+
}
|
|
1812
|
+
async ensureElementInView(index, margin = 24) {
|
|
1813
|
+
return this.alignElementInView(index, { align: 'center', margin });
|
|
1814
|
+
}
|
|
1815
|
+
async alignElementInView(index, options = {}) {
|
|
1816
|
+
const page = this.ensurePage();
|
|
1817
|
+
const element = this.elementMap.get(index);
|
|
1818
|
+
if (!element)
|
|
1819
|
+
throw new Error(`Element index ${index} not found`);
|
|
1820
|
+
const align = options.align ?? 'center';
|
|
1821
|
+
const margin = options.margin ?? 24;
|
|
1822
|
+
await this.withResolvedLocator(element, async (locator) => {
|
|
1823
|
+
await locator.scrollIntoViewIfNeeded({ timeout: 5000 });
|
|
1824
|
+
});
|
|
1825
|
+
let measured = await this.measureElement(index);
|
|
1826
|
+
if (!measured)
|
|
1827
|
+
throw new Error(`Could not measure element index ${index}`);
|
|
1828
|
+
const viewport = page.viewportSize();
|
|
1829
|
+
if (!viewport)
|
|
1830
|
+
return measured;
|
|
1831
|
+
const maxScroll = await page.evaluate(() => {
|
|
1832
|
+
const root = document.scrollingElement || document.documentElement;
|
|
1833
|
+
return {
|
|
1834
|
+
x: Math.max(0, root.scrollWidth - window.innerWidth),
|
|
1835
|
+
y: Math.max(0, root.scrollHeight - window.innerHeight),
|
|
1836
|
+
};
|
|
1837
|
+
});
|
|
1838
|
+
const pageBox = measured.pageBox;
|
|
1839
|
+
const computeAxisTarget = (coord, size, viewportSize, maxAxis) => {
|
|
1840
|
+
if (align === 'start') {
|
|
1841
|
+
return Math.max(0, Math.min(Math.round(coord - margin), maxAxis));
|
|
1842
|
+
}
|
|
1843
|
+
if (align === 'end') {
|
|
1844
|
+
return Math.max(0, Math.min(Math.round(coord + size - viewportSize + margin), maxAxis));
|
|
1845
|
+
}
|
|
1846
|
+
const centered = size + margin * 2 <= viewportSize
|
|
1847
|
+
? coord - Math.max(0, (viewportSize - size) / 2)
|
|
1848
|
+
: coord - margin;
|
|
1849
|
+
return Math.max(0, Math.min(Math.round(centered), maxAxis));
|
|
1850
|
+
};
|
|
1851
|
+
const targetScrollX = computeAxisTarget(pageBox.x, pageBox.width, viewport.width, maxScroll.x);
|
|
1852
|
+
const targetScrollY = computeAxisTarget(pageBox.y, pageBox.height, viewport.height, maxScroll.y);
|
|
1853
|
+
await page.evaluate(({ x, y }) => window.scrollTo(x, y), {
|
|
1854
|
+
x: targetScrollX,
|
|
1855
|
+
y: targetScrollY,
|
|
1856
|
+
});
|
|
1857
|
+
await page.waitForTimeout(250);
|
|
1858
|
+
measured = await this.measureElement(index);
|
|
1859
|
+
if (!measured)
|
|
1860
|
+
throw new Error(`Could not re-measure element index ${index}`);
|
|
1861
|
+
return measured;
|
|
1862
|
+
}
|
|
1863
|
+
async scrollElementIntoView(index, options = {}) {
|
|
1864
|
+
await this.alignElementInView(index, options);
|
|
1865
|
+
}
|
|
1866
|
+
async dismissOverlays() {
|
|
1867
|
+
return dismissCookiesAndWidgets(this.ensurePage());
|
|
1868
|
+
}
|
|
1869
|
+
async hoverByIndex(index) {
|
|
1870
|
+
const page = this.ensurePage();
|
|
1871
|
+
const element = this.elementMap.get(index);
|
|
1872
|
+
if (!element)
|
|
1873
|
+
throw new Error(`Element index ${index} not found`);
|
|
1874
|
+
const measured = await this.ensureElementInView(index, 32);
|
|
1875
|
+
const viewport = page.viewportSize();
|
|
1876
|
+
if (!viewport)
|
|
1877
|
+
throw new Error('Viewport unavailable during hover');
|
|
1878
|
+
const point = this.getVisiblePoint(measured.boundingBox, viewport);
|
|
1879
|
+
await page.mouse.move(point.x, point.y);
|
|
1880
|
+
}
|
|
1881
|
+
async hoverBySelector(selector) {
|
|
1882
|
+
const page = this.ensurePage();
|
|
1883
|
+
await page.hover(selector, { timeout: 5000 });
|
|
1884
|
+
}
|
|
1885
|
+
async hoverByCoordinates(x, y) {
|
|
1886
|
+
const page = this.ensurePage();
|
|
1887
|
+
await page.mouse.move(x, y);
|
|
1888
|
+
}
|
|
1889
|
+
async safeExpand(options) {
|
|
1890
|
+
const page = this.ensurePage();
|
|
1891
|
+
const waitForExpansionSignal = async (locator, stateBefore) => {
|
|
1892
|
+
if (options.expectedSelector) {
|
|
1893
|
+
await page.locator(options.expectedSelector).first().waitFor({ state: 'visible', timeout: 3000 });
|
|
1894
|
+
return;
|
|
1895
|
+
}
|
|
1896
|
+
if (options.expectedText) {
|
|
1897
|
+
await page.getByText(options.expectedText, { exact: false }).first().waitFor({ state: 'visible', timeout: 3000 });
|
|
1898
|
+
return;
|
|
1899
|
+
}
|
|
1900
|
+
if (stateBefore.ariaExpanded !== null) {
|
|
1901
|
+
for (let attempt = 0; attempt < 8; attempt += 1) {
|
|
1902
|
+
const expanded = await locator.evaluate(el => el.getAttribute('aria-expanded'));
|
|
1903
|
+
if (expanded === 'true')
|
|
1904
|
+
return;
|
|
1905
|
+
await page.waitForTimeout(120);
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
if (stateBefore.ariaControls) {
|
|
1909
|
+
await page.waitForFunction((id) => {
|
|
1910
|
+
const controlled = document.getElementById(id);
|
|
1911
|
+
if (!controlled)
|
|
1912
|
+
return false;
|
|
1913
|
+
const style = window.getComputedStyle(controlled);
|
|
1914
|
+
const rect = controlled.getBoundingClientRect();
|
|
1915
|
+
return style.display !== 'none'
|
|
1916
|
+
&& style.visibility !== 'hidden'
|
|
1917
|
+
&& parseFloat(style.opacity || '1') > 0
|
|
1918
|
+
&& rect.width > 0
|
|
1919
|
+
&& rect.height > 0;
|
|
1920
|
+
}, stateBefore.ariaControls, { timeout: 2500 }).catch(() => { });
|
|
1921
|
+
return;
|
|
1922
|
+
}
|
|
1923
|
+
if (stateBefore.detailsOpen) {
|
|
1924
|
+
return;
|
|
1925
|
+
}
|
|
1926
|
+
await page.waitForTimeout(350);
|
|
1927
|
+
};
|
|
1928
|
+
const expandWithLocator = async (locator) => {
|
|
1929
|
+
await locator.waitFor({ state: 'visible', timeout: 5000 });
|
|
1930
|
+
const stateBefore = await locator.evaluate((el) => ({
|
|
1931
|
+
ariaExpanded: el.getAttribute('aria-expanded'),
|
|
1932
|
+
ariaControls: el.getAttribute('aria-controls'),
|
|
1933
|
+
detailsOpen: el.tagName === 'SUMMARY'
|
|
1934
|
+
? !!(el.parentElement && el.parentElement.tagName === 'DETAILS' && el.parentElement.open)
|
|
1935
|
+
: el.open === true,
|
|
1936
|
+
}));
|
|
1937
|
+
if (stateBefore.ariaExpanded === 'true' || stateBefore.detailsOpen) {
|
|
1938
|
+
await waitForExpansionSignal(locator, stateBefore);
|
|
1939
|
+
return;
|
|
1940
|
+
}
|
|
1941
|
+
await locator.click({ timeout: 5000 });
|
|
1942
|
+
await waitForExpansionSignal(locator, stateBefore);
|
|
1943
|
+
};
|
|
1944
|
+
if (options.index !== undefined) {
|
|
1945
|
+
const element = this.elementMap.get(options.index);
|
|
1946
|
+
if (!element)
|
|
1947
|
+
throw new Error(`Element index ${options.index} not found`);
|
|
1948
|
+
await this.ensureElementInView(options.index, 32);
|
|
1949
|
+
const expanded = await this.withResolvedLocator(element, async (locator) => {
|
|
1950
|
+
await expandWithLocator(locator);
|
|
1951
|
+
return true;
|
|
1952
|
+
});
|
|
1953
|
+
if (!expanded) {
|
|
1954
|
+
throw new Error(`Could not resolve expandable element index ${options.index}`);
|
|
1955
|
+
}
|
|
1956
|
+
return;
|
|
1957
|
+
}
|
|
1958
|
+
if (options.selector) {
|
|
1959
|
+
await expandWithLocator(page.locator(options.selector).first());
|
|
1960
|
+
return;
|
|
1961
|
+
}
|
|
1962
|
+
throw new Error('safeExpand requires index or selector');
|
|
1963
|
+
}
|
|
1964
|
+
async clickByIndex(index) {
|
|
1965
|
+
const page = this.ensurePage();
|
|
1966
|
+
const element = this.elementMap.get(index);
|
|
1967
|
+
if (!element)
|
|
1968
|
+
throw new Error(`Element index ${index} not found`);
|
|
1969
|
+
const measured = await this.ensureElementInView(index, 32);
|
|
1970
|
+
const viewport = page.viewportSize();
|
|
1971
|
+
if (!viewport)
|
|
1972
|
+
throw new Error('Viewport unavailable during click');
|
|
1973
|
+
// Prefer coordinate click — it's always precise after re-centering and
|
|
1974
|
+
// avoids ambiguous selectors when multiple elements share the same classes.
|
|
1975
|
+
const point = this.getVisiblePoint(measured.boundingBox, viewport);
|
|
1976
|
+
await page.mouse.click(point.x, point.y);
|
|
1977
|
+
}
|
|
1978
|
+
async clickBySelector(selector, options) {
|
|
1979
|
+
const page = this.ensurePage();
|
|
1980
|
+
await page.click(selector, { timeout: 5000, force: options?.force });
|
|
1981
|
+
}
|
|
1982
|
+
async clickByCoordinates(x, y) {
|
|
1983
|
+
const page = this.ensurePage();
|
|
1984
|
+
await page.mouse.click(x, y);
|
|
1985
|
+
}
|
|
1986
|
+
async typeText(text, options) {
|
|
1987
|
+
const page = this.ensurePage();
|
|
1988
|
+
if (options?.index !== undefined) {
|
|
1989
|
+
const measured = await this.ensureElementInView(options.index, 32);
|
|
1990
|
+
const viewport = page.viewportSize();
|
|
1991
|
+
if (!viewport)
|
|
1992
|
+
throw new Error('Viewport unavailable during typing');
|
|
1993
|
+
const point = this.getVisiblePoint(measured.boundingBox, viewport);
|
|
1994
|
+
await page.mouse.click(point.x, point.y);
|
|
1995
|
+
if (options.clearFirst) {
|
|
1996
|
+
await page.keyboard.press('Control+A');
|
|
1997
|
+
}
|
|
1998
|
+
await page.keyboard.type(text);
|
|
1999
|
+
}
|
|
2000
|
+
else if (options?.selector) {
|
|
2001
|
+
if (options.clearFirst) {
|
|
2002
|
+
await page.fill(options.selector, text);
|
|
2003
|
+
}
|
|
2004
|
+
else {
|
|
2005
|
+
await page.click(options.selector, { timeout: 5000 });
|
|
2006
|
+
await page.keyboard.type(text);
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
else {
|
|
2010
|
+
await page.keyboard.type(text);
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
async selectOption(options) {
|
|
2014
|
+
const page = this.ensurePage();
|
|
2015
|
+
const option = typeof options.optionLabel === 'string' ? { label: options.optionLabel }
|
|
2016
|
+
: typeof options.optionValue === 'string' ? { value: options.optionValue }
|
|
2017
|
+
: typeof options.optionIndex === 'number' ? { index: options.optionIndex }
|
|
2018
|
+
: null;
|
|
2019
|
+
if (!option) {
|
|
2020
|
+
throw new Error('selectOption requires optionLabel, optionValue, or optionIndex');
|
|
2021
|
+
}
|
|
2022
|
+
if (options.index !== undefined) {
|
|
2023
|
+
const element = this.elementMap.get(options.index);
|
|
2024
|
+
if (!element)
|
|
2025
|
+
throw new Error(`Element index ${options.index} not found`);
|
|
2026
|
+
await this.ensureElementInView(options.index, 32);
|
|
2027
|
+
const selected = await this.withResolvedLocator(element, async (locator) => {
|
|
2028
|
+
await locator.selectOption(option, { timeout: 5000 });
|
|
2029
|
+
return true;
|
|
2030
|
+
});
|
|
2031
|
+
if (!selected) {
|
|
2032
|
+
throw new Error(`Could not resolve select element index ${options.index}`);
|
|
2033
|
+
}
|
|
2034
|
+
return;
|
|
2035
|
+
}
|
|
2036
|
+
if (options.selector) {
|
|
2037
|
+
const locator = page.locator(options.selector).first();
|
|
2038
|
+
await locator.waitFor({ state: 'visible', timeout: 5000 });
|
|
2039
|
+
await locator.selectOption(option, { timeout: 5000 });
|
|
2040
|
+
return;
|
|
2041
|
+
}
|
|
2042
|
+
throw new Error('selectOption requires index or selector');
|
|
2043
|
+
}
|
|
2044
|
+
async scroll(direction, amount = 500, selector) {
|
|
2045
|
+
const page = this.ensurePage();
|
|
2046
|
+
const dx = direction === 'right' ? amount : direction === 'left' ? -amount : 0;
|
|
2047
|
+
const dy = direction === 'down' ? amount : direction === 'up' ? -amount : 0;
|
|
2048
|
+
if (selector) {
|
|
2049
|
+
await page.evaluate(({ sel, deltaX, deltaY }) => {
|
|
2050
|
+
const el = document.querySelector(sel);
|
|
2051
|
+
if (el)
|
|
2052
|
+
el.scrollBy(deltaX, deltaY);
|
|
2053
|
+
}, { sel: selector, deltaX: dx, deltaY: dy });
|
|
2054
|
+
}
|
|
2055
|
+
else {
|
|
2056
|
+
await page.evaluate(({ deltaX, deltaY }) => window.scrollBy(deltaX, deltaY), { deltaX: dx, deltaY: dy });
|
|
2057
|
+
}
|
|
2058
|
+
}
|
|
2059
|
+
async pressKey(key) {
|
|
2060
|
+
const page = this.ensurePage();
|
|
2061
|
+
await page.keyboard.press(key);
|
|
2062
|
+
}
|
|
2063
|
+
async searchText(query) {
|
|
2064
|
+
const page = this.ensurePage();
|
|
2065
|
+
return page.evaluate((q) => {
|
|
2066
|
+
const results = [];
|
|
2067
|
+
// Use a page-global counter so data-ak-search-index values are stable across
|
|
2068
|
+
// multiple searchText calls within the same element capture session.
|
|
2069
|
+
const w = window;
|
|
2070
|
+
if (typeof w.__akSearchIndexCounter === 'undefined')
|
|
2071
|
+
w.__akSearchIndexCounter = 0;
|
|
2072
|
+
let searchIndexCounter = w.__akSearchIndexCounter;
|
|
2073
|
+
const lowerQ = q.toLowerCase();
|
|
2074
|
+
const vw = window.innerWidth;
|
|
2075
|
+
const vh = window.innerHeight;
|
|
2076
|
+
// Helper: get direct/own text of an element
|
|
2077
|
+
function getOwnText(el) {
|
|
2078
|
+
let t = '';
|
|
2079
|
+
for (const child of el.childNodes) {
|
|
2080
|
+
if (child.nodeType === Node.TEXT_NODE) {
|
|
2081
|
+
t += (child.textContent || '').trim() + ' ';
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
return t.trim();
|
|
2085
|
+
}
|
|
2086
|
+
const quoteForSelector = (value) => value
|
|
2087
|
+
.replace(/\\/g, '\\\\')
|
|
2088
|
+
.replace(/"/g, '\\"');
|
|
2089
|
+
const isUniqueSelector = (selector, expected) => {
|
|
2090
|
+
try {
|
|
2091
|
+
const matches = document.querySelectorAll(selector);
|
|
2092
|
+
return matches.length === 1 && matches[0] === expected;
|
|
2093
|
+
}
|
|
2094
|
+
catch {
|
|
2095
|
+
return false;
|
|
2096
|
+
}
|
|
2097
|
+
};
|
|
2098
|
+
const buildStableCssSelector = (node) => {
|
|
2099
|
+
const tag = node.tagName.toLowerCase();
|
|
2100
|
+
const candidates = [];
|
|
2101
|
+
const push = (selector) => {
|
|
2102
|
+
if (selector && !candidates.includes(selector)) {
|
|
2103
|
+
candidates.push(selector);
|
|
2104
|
+
}
|
|
2105
|
+
};
|
|
2106
|
+
const role = node.getAttribute('role');
|
|
2107
|
+
const type = node.getAttribute('type');
|
|
2108
|
+
const id = node.getAttribute('id');
|
|
2109
|
+
if (id) {
|
|
2110
|
+
push(`#${CSS.escape(id)}`);
|
|
2111
|
+
}
|
|
2112
|
+
for (const attr of ['data-testid', 'data-test', 'data-qa', 'data-cy']) {
|
|
2113
|
+
const value = node.getAttribute(attr);
|
|
2114
|
+
if (!value)
|
|
2115
|
+
continue;
|
|
2116
|
+
push(`[${attr}="${quoteForSelector(value)}"]`);
|
|
2117
|
+
push(`${tag}[${attr}="${quoteForSelector(value)}"]`);
|
|
2118
|
+
}
|
|
2119
|
+
const name = node.getAttribute('name');
|
|
2120
|
+
if (name) {
|
|
2121
|
+
push(`${tag}[name="${quoteForSelector(name)}"]`);
|
|
2122
|
+
if (type) {
|
|
2123
|
+
push(`${tag}[name="${quoteForSelector(name)}"][type="${quoteForSelector(type)}"]`);
|
|
2124
|
+
}
|
|
2125
|
+
}
|
|
2126
|
+
const rawHref = node.getAttribute('href');
|
|
2127
|
+
if (tag === 'a' && rawHref) {
|
|
2128
|
+
push(`a[href="${quoteForSelector(rawHref)}"]`);
|
|
2129
|
+
const hreflang = node.getAttribute('hreflang');
|
|
2130
|
+
if (hreflang) {
|
|
2131
|
+
push(`a[href="${quoteForSelector(rawHref)}"][hreflang="${quoteForSelector(hreflang)}"]`);
|
|
2132
|
+
}
|
|
2133
|
+
}
|
|
2134
|
+
for (const attr of ['aria-label', 'title', 'placeholder', 'data-lang', 'data-language', 'data-locale', 'data-theme', 'data-color-scheme', 'hreflang']) {
|
|
2135
|
+
const value = node.getAttribute(attr);
|
|
2136
|
+
if (!value)
|
|
2137
|
+
continue;
|
|
2138
|
+
push(`${tag}[${attr}="${quoteForSelector(value)}"]`);
|
|
2139
|
+
if (role) {
|
|
2140
|
+
push(`${tag}[role="${quoteForSelector(role)}"][${attr}="${quoteForSelector(value)}"]`);
|
|
2141
|
+
}
|
|
2142
|
+
if (type) {
|
|
2143
|
+
push(`${tag}[type="${quoteForSelector(type)}"][${attr}="${quoteForSelector(value)}"]`);
|
|
2144
|
+
}
|
|
2145
|
+
}
|
|
2146
|
+
if (type) {
|
|
2147
|
+
push(`${tag}[type="${quoteForSelector(type)}"]`);
|
|
2148
|
+
}
|
|
2149
|
+
for (const selector of candidates) {
|
|
2150
|
+
if (isUniqueSelector(selector, node)) {
|
|
2151
|
+
return selector;
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
return null;
|
|
2155
|
+
};
|
|
2156
|
+
// Walk all elements, find those containing the query text
|
|
2157
|
+
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
|
|
2158
|
+
const seen = new Set();
|
|
2159
|
+
let node = walker.currentNode;
|
|
2160
|
+
while (node) {
|
|
2161
|
+
const el = node;
|
|
2162
|
+
const fullText = (el.textContent || '').trim();
|
|
2163
|
+
const ownText = getOwnText(el);
|
|
2164
|
+
const ariaLabel = el.getAttribute('aria-label') || '';
|
|
2165
|
+
const placeholder = el.placeholder || '';
|
|
2166
|
+
const title = el.getAttribute('title') || '';
|
|
2167
|
+
// Match on the element's OWN text using word-boundary regex (e.g. "pro" should
|
|
2168
|
+
// match the heading <h3>pro</h3> but NOT "The problem" or "3 projects").
|
|
2169
|
+
// Fall back to substring matching on aria/placeholder/title (these are always specific).
|
|
2170
|
+
const escapedQ = q.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
2171
|
+
const wordBoundaryRe = new RegExp(`\\b${escapedQ}\\b`, 'i');
|
|
2172
|
+
const matchOwn = wordBoundaryRe.test(ownText) || wordBoundaryRe.test(ariaLabel)
|
|
2173
|
+
|| ariaLabel.toLowerCase().includes(lowerQ)
|
|
2174
|
+
|| placeholder.toLowerCase().includes(lowerQ)
|
|
2175
|
+
|| title.toLowerCase().includes(lowerQ);
|
|
2176
|
+
// For leaf elements (no element children) also allow full-text word-boundary match.
|
|
2177
|
+
const isLeaf = el.children.length === 0;
|
|
2178
|
+
const matchText = matchOwn || (isLeaf && wordBoundaryRe.test(fullText));
|
|
2179
|
+
if (matchText && !seen.has(el)) {
|
|
2180
|
+
const style = window.getComputedStyle(el);
|
|
2181
|
+
if (style.display !== 'none' && style.visibility !== 'hidden' && parseFloat(style.opacity) !== 0) {
|
|
2182
|
+
const rect = el.getBoundingClientRect();
|
|
2183
|
+
if (rect.width > 0 && rect.height > 0) {
|
|
2184
|
+
seen.add(el);
|
|
2185
|
+
// Determine if clickable
|
|
2186
|
+
const isInteractive = ['A', 'BUTTON', 'INPUT', 'SELECT', 'TEXTAREA'].includes(el.tagName);
|
|
2187
|
+
const hasRole = ['button', 'link', 'tab', 'menuitem', 'option'].includes(el.getAttribute('role') || '');
|
|
2188
|
+
const hasCursor = style.cursor === 'pointer';
|
|
2189
|
+
const hasTabindex = el.hasAttribute('tabindex') && el.getAttribute('tabindex') !== '-1';
|
|
2190
|
+
const hasOnclick = el.hasAttribute('onclick');
|
|
2191
|
+
// Display text: prefer own text or aria-label (most specific)
|
|
2192
|
+
const displayText = ownText || ariaLabel || placeholder || title || fullText;
|
|
2193
|
+
// Get role
|
|
2194
|
+
const role = el.getAttribute('role')
|
|
2195
|
+
|| (el.tagName === 'A' ? 'link' : '')
|
|
2196
|
+
|| (el.tagName === 'BUTTON' ? 'button' : '')
|
|
2197
|
+
|| el.tagName.toLowerCase();
|
|
2198
|
+
// Get href
|
|
2199
|
+
const href = el.tagName === 'A' ? el.href : null;
|
|
2200
|
+
const visibleWidth = Math.max(0, Math.min(rect.right, vw) - Math.max(rect.left, 0));
|
|
2201
|
+
const visibleHeight = Math.max(0, Math.min(rect.bottom, vh) - Math.max(rect.top, 0));
|
|
2202
|
+
const inViewport = visibleWidth > 0 && visibleHeight > 0;
|
|
2203
|
+
const fullyVisible = rect.top >= 0 && rect.left >= 0 && rect.bottom <= vh && rect.right <= vw;
|
|
2204
|
+
const visibilityState = !inViewport ? 'offscreen' : fullyVisible ? 'full' : 'partial';
|
|
2205
|
+
// Prefer a stable CSS selector; fall back to temporary automation selectors
|
|
2206
|
+
// only when the DOM does not expose a unique attribute-based handle.
|
|
2207
|
+
let selector;
|
|
2208
|
+
const stableSelector = buildStableCssSelector(el);
|
|
2209
|
+
const existingIdx = el.getAttribute('data-ak-interactive-index');
|
|
2210
|
+
const existingSearchIdx = el.getAttribute('data-ak-search-index');
|
|
2211
|
+
if (stableSelector) {
|
|
2212
|
+
selector = stableSelector;
|
|
2213
|
+
}
|
|
2214
|
+
else if (existingIdx !== null) {
|
|
2215
|
+
selector = `[data-ak-interactive-index="${existingIdx}"]`;
|
|
2216
|
+
}
|
|
2217
|
+
else if (existingSearchIdx !== null) {
|
|
2218
|
+
// Reuse the existing search index from a previous search call
|
|
2219
|
+
// instead of assigning a new one — keeps selectors stable.
|
|
2220
|
+
selector = `[data-ak-search-index="${existingSearchIdx}"]`;
|
|
2221
|
+
}
|
|
2222
|
+
else {
|
|
2223
|
+
el.setAttribute('data-ak-search-index', String(searchIndexCounter));
|
|
2224
|
+
selector = `[data-ak-search-index="${searchIndexCounter}"]`;
|
|
2225
|
+
searchIndexCounter++;
|
|
2226
|
+
}
|
|
2227
|
+
// Find nearest meaningful container (card, list item, section, etc.)
|
|
2228
|
+
// Walk up the DOM and look for a component-like ancestor.
|
|
2229
|
+
const SEMANTIC_CONTAINERS = new Set([
|
|
2230
|
+
'ARTICLE', 'SECTION', 'LI', 'ASIDE', 'DIALOG', 'DETAILS', 'FIGURE', 'FORM', 'NAV',
|
|
2231
|
+
]);
|
|
2232
|
+
const CONTAINER_ROLES = new Set([
|
|
2233
|
+
'listitem', 'article', 'dialog', 'region', 'group', 'tabpanel', 'card',
|
|
2234
|
+
]);
|
|
2235
|
+
let containerInfo;
|
|
2236
|
+
let bestContainer = null;
|
|
2237
|
+
let ancestor = el.parentElement;
|
|
2238
|
+
const maxWalkUp = 8;
|
|
2239
|
+
let walkCount = 0;
|
|
2240
|
+
const viewportArea = Math.max(1, vw * vh);
|
|
2241
|
+
while (ancestor && ancestor !== document.body && walkCount < maxWalkUp) {
|
|
2242
|
+
walkCount++;
|
|
2243
|
+
const ancTag = ancestor.tagName;
|
|
2244
|
+
const ancRole = ancestor.getAttribute('role') || '';
|
|
2245
|
+
const ancRect = ancestor.getBoundingClientRect();
|
|
2246
|
+
// Skip invisible or zero-size ancestors
|
|
2247
|
+
if (ancRect.width <= 0 || ancRect.height <= 0) {
|
|
2248
|
+
ancestor = ancestor.parentElement;
|
|
2249
|
+
continue;
|
|
2250
|
+
}
|
|
2251
|
+
// A container must be meaningfully larger than the matched element
|
|
2252
|
+
const areaRatio = (ancRect.width * ancRect.height) / Math.max(1, rect.width * rect.height);
|
|
2253
|
+
const viewportRatio = (ancRect.width * ancRect.height) / viewportArea;
|
|
2254
|
+
const ancestorClass = ancestor.className && typeof ancestor.className === 'string'
|
|
2255
|
+
? ancestor.className
|
|
2256
|
+
: '';
|
|
2257
|
+
const looksLikeCardClass = /\b(card|tile|item|row|preset|mockup|template|gallery|capture|thumb|thumbnail)\b/i
|
|
2258
|
+
.test(`${ancestor.id || ''} ${ancestorClass}`);
|
|
2259
|
+
let reason = '';
|
|
2260
|
+
let score = 0;
|
|
2261
|
+
if (SEMANTIC_CONTAINERS.has(ancTag)) {
|
|
2262
|
+
reason = `semantic <${ancTag.toLowerCase()}>`;
|
|
2263
|
+
score += 6;
|
|
2264
|
+
}
|
|
2265
|
+
else if (CONTAINER_ROLES.has(ancRole)) {
|
|
2266
|
+
reason = `role="${ancRole}"`;
|
|
2267
|
+
score += 6;
|
|
2268
|
+
}
|
|
2269
|
+
else if (ancestor.id && !/^(root|app|__next)$/i.test(ancestor.id)) {
|
|
2270
|
+
reason = `#${ancestor.id}`;
|
|
2271
|
+
score += 4;
|
|
2272
|
+
}
|
|
2273
|
+
else if (looksLikeCardClass) {
|
|
2274
|
+
reason = `class/id match`;
|
|
2275
|
+
score += 5;
|
|
2276
|
+
}
|
|
2277
|
+
else if (areaRatio >= 3) {
|
|
2278
|
+
// Check if this element is a grid/flex child (sibling of similar elements)
|
|
2279
|
+
const parentDisplay = ancestor.parentElement
|
|
2280
|
+
? window.getComputedStyle(ancestor.parentElement).display
|
|
2281
|
+
: '';
|
|
2282
|
+
const isGridFlexChild = parentDisplay.includes('flex') || parentDisplay.includes('grid');
|
|
2283
|
+
if (isGridFlexChild && ancestor.parentElement && ancestor.parentElement.children.length >= 2) {
|
|
2284
|
+
reason = `flex/grid child (${ancestor.parentElement.children.length} siblings)`;
|
|
2285
|
+
score += 5;
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
// Skip giant layout wrappers such as <main> that are usually too broad to be a component crop.
|
|
2289
|
+
const isLayoutWrapper = /^(MAIN|BODY|HTML)$/i.test(ancTag);
|
|
2290
|
+
if (isLayoutWrapper && viewportRatio >= 0.75) {
|
|
2291
|
+
ancestor = ancestor.parentElement;
|
|
2292
|
+
continue;
|
|
2293
|
+
}
|
|
2294
|
+
if (reason && areaRatio >= 2) {
|
|
2295
|
+
if (viewportRatio <= 0.9)
|
|
2296
|
+
score += 2;
|
|
2297
|
+
if (areaRatio <= 40)
|
|
2298
|
+
score += 2;
|
|
2299
|
+
if (walkCount <= 3)
|
|
2300
|
+
score += 1;
|
|
2301
|
+
score -= Math.max(0, viewportRatio - 0.65) * 8;
|
|
2302
|
+
score -= Math.max(0, areaRatio - 60) / 20;
|
|
2303
|
+
if (!bestContainer || score > bestContainer.score) {
|
|
2304
|
+
bestContainer = {
|
|
2305
|
+
ancestor,
|
|
2306
|
+
rect: ancRect,
|
|
2307
|
+
reason,
|
|
2308
|
+
score,
|
|
2309
|
+
};
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
ancestor = ancestor.parentElement;
|
|
2313
|
+
}
|
|
2314
|
+
if (bestContainer) {
|
|
2315
|
+
const { ancestor: containerAncestor, rect: containerRect, reason } = bestContainer;
|
|
2316
|
+
// Prefer a stable CSS selector for the container as well.
|
|
2317
|
+
let containerSel;
|
|
2318
|
+
const stableContainerSelector = buildStableCssSelector(containerAncestor);
|
|
2319
|
+
const existingAkIdx = containerAncestor.getAttribute('data-ak-interactive-index');
|
|
2320
|
+
const existingSearchIdx = containerAncestor.getAttribute('data-ak-search-index');
|
|
2321
|
+
const existingContainerIdx = containerAncestor.getAttribute('data-ak-container-index');
|
|
2322
|
+
if (stableContainerSelector) {
|
|
2323
|
+
containerSel = stableContainerSelector;
|
|
2324
|
+
}
|
|
2325
|
+
else if (existingAkIdx !== null) {
|
|
2326
|
+
containerSel = `[data-ak-interactive-index="${existingAkIdx}"]`;
|
|
2327
|
+
}
|
|
2328
|
+
else if (existingSearchIdx !== null) {
|
|
2329
|
+
containerSel = `[data-ak-search-index="${existingSearchIdx}"]`;
|
|
2330
|
+
}
|
|
2331
|
+
else if (existingContainerIdx !== null) {
|
|
2332
|
+
containerSel = `[data-ak-container-index="${existingContainerIdx}"]`;
|
|
2333
|
+
}
|
|
2334
|
+
else {
|
|
2335
|
+
containerAncestor.setAttribute('data-ak-container-index', String(searchIndexCounter));
|
|
2336
|
+
containerSel = `[data-ak-container-index="${searchIndexCounter}"]`;
|
|
2337
|
+
searchIndexCounter++;
|
|
2338
|
+
}
|
|
2339
|
+
containerInfo = {
|
|
2340
|
+
tag: containerAncestor.tagName.toLowerCase(),
|
|
2341
|
+
selector: containerSel,
|
|
2342
|
+
boundingBox: {
|
|
2343
|
+
x: Math.round(containerRect.x),
|
|
2344
|
+
y: Math.round(containerRect.y),
|
|
2345
|
+
width: Math.round(containerRect.width),
|
|
2346
|
+
height: Math.round(containerRect.height),
|
|
2347
|
+
},
|
|
2348
|
+
reason,
|
|
2349
|
+
};
|
|
2350
|
+
}
|
|
2351
|
+
results.push({
|
|
2352
|
+
tag: el.tagName.toLowerCase(),
|
|
2353
|
+
text: displayText.slice(0, 100),
|
|
2354
|
+
href,
|
|
2355
|
+
role,
|
|
2356
|
+
clickable: isInteractive || hasRole || hasCursor || hasTabindex || hasOnclick,
|
|
2357
|
+
visible: inViewport,
|
|
2358
|
+
visibilityState,
|
|
2359
|
+
boundingBox: {
|
|
2360
|
+
x: Math.round(rect.x),
|
|
2361
|
+
y: Math.round(rect.y),
|
|
2362
|
+
width: Math.round(rect.width),
|
|
2363
|
+
height: Math.round(rect.height),
|
|
2364
|
+
},
|
|
2365
|
+
selector,
|
|
2366
|
+
container: containerInfo,
|
|
2367
|
+
});
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
node = walker.nextNode();
|
|
2372
|
+
}
|
|
2373
|
+
// Persist the counter so subsequent searchText calls don't reuse the same indices.
|
|
2374
|
+
w.__akSearchIndexCounter = searchIndexCounter;
|
|
2375
|
+
const visibilityRank = (state) => state === 'full' ? 0 : state === 'partial' ? 1 : 2;
|
|
2376
|
+
// Prefer actionable, visible matches before falling back to smaller fuzzy matches.
|
|
2377
|
+
return results
|
|
2378
|
+
.sort((a, b) => {
|
|
2379
|
+
if (a.clickable !== b.clickable)
|
|
2380
|
+
return a.clickable ? -1 : 1;
|
|
2381
|
+
const visibilityDelta = visibilityRank(a.visibilityState) - visibilityRank(b.visibilityState);
|
|
2382
|
+
if (visibilityDelta !== 0)
|
|
2383
|
+
return visibilityDelta;
|
|
2384
|
+
return (a.boundingBox.width * a.boundingBox.height) - (b.boundingBox.width * b.boundingBox.height);
|
|
2385
|
+
})
|
|
2386
|
+
.slice(0, 15);
|
|
2387
|
+
}, query);
|
|
2388
|
+
}
|
|
2389
|
+
/** Temporarily hide all fixed/sticky overlays (navbars, banners, FABs, etc.) */
|
|
2390
|
+
async hideFixedOverlays() {
|
|
2391
|
+
const page = this.ensurePage();
|
|
2392
|
+
await page.evaluate(() => {
|
|
2393
|
+
document.querySelectorAll('*').forEach(el => {
|
|
2394
|
+
const pos = getComputedStyle(el).position;
|
|
2395
|
+
if (pos === 'fixed' || pos === 'sticky') {
|
|
2396
|
+
const h = el;
|
|
2397
|
+
h.setAttribute('data-ak-vis', h.style.visibility);
|
|
2398
|
+
h.style.setProperty('visibility', 'hidden', 'important');
|
|
2399
|
+
}
|
|
2400
|
+
});
|
|
2401
|
+
});
|
|
2402
|
+
}
|
|
2403
|
+
/** Restore overlays hidden by hideFixedOverlays */
|
|
2404
|
+
async restoreFixedOverlays() {
|
|
2405
|
+
const page = this.ensurePage();
|
|
2406
|
+
await page.evaluate(() => {
|
|
2407
|
+
document.querySelectorAll('[data-ak-vis]').forEach(el => {
|
|
2408
|
+
const h = el;
|
|
2409
|
+
h.style.visibility = h.getAttribute('data-ak-vis') || '';
|
|
2410
|
+
h.removeAttribute('data-ak-vis');
|
|
2411
|
+
});
|
|
2412
|
+
});
|
|
2413
|
+
}
|
|
2414
|
+
async screenshotElement(index, padding = 0) {
|
|
2415
|
+
const page = this.ensurePage();
|
|
2416
|
+
const element = this.elementMap.get(index);
|
|
2417
|
+
if (!element)
|
|
2418
|
+
throw new Error(`Element index ${index} not found in elementMap`);
|
|
2419
|
+
// Move cursor off-screen to avoid hover artifacts
|
|
2420
|
+
await page.mouse.move(0, 0);
|
|
2421
|
+
// Hide fixed/sticky overlays (navbars, banners) that could cover the element
|
|
2422
|
+
await this.hideFixedOverlays();
|
|
2423
|
+
const originalViewport = page.viewportSize();
|
|
2424
|
+
let resizedForCapture = false;
|
|
2425
|
+
try {
|
|
2426
|
+
let measured = await this.ensureElementInView(index, Math.max(24, padding + 16));
|
|
2427
|
+
let viewport = page.viewportSize();
|
|
2428
|
+
if (viewport && originalViewport) {
|
|
2429
|
+
const requiredWidth = measured.boundingBox.width + padding * 2 + 48;
|
|
2430
|
+
const requiredHeight = measured.boundingBox.height + padding * 2 + 48;
|
|
2431
|
+
const needsResize = requiredWidth > viewport.width || requiredHeight > viewport.height;
|
|
2432
|
+
if (needsResize) {
|
|
2433
|
+
await page.setViewportSize({
|
|
2434
|
+
width: Math.max(originalViewport.width, Math.ceil(requiredWidth)),
|
|
2435
|
+
height: Math.max(originalViewport.height, Math.ceil(requiredHeight)),
|
|
2436
|
+
});
|
|
2437
|
+
resizedForCapture = true;
|
|
2438
|
+
await page.waitForTimeout(250);
|
|
2439
|
+
measured = await this.ensureElementInView(index, Math.max(24, padding + 16));
|
|
2440
|
+
viewport = page.viewportSize();
|
|
2441
|
+
}
|
|
2442
|
+
}
|
|
2443
|
+
if (!viewport)
|
|
2444
|
+
throw new Error('Viewport unavailable during element screenshot');
|
|
2445
|
+
const bb = measured.boundingBox;
|
|
2446
|
+
const clip = {
|
|
2447
|
+
x: Math.max(0, bb.x - padding),
|
|
2448
|
+
y: Math.max(0, bb.y - padding),
|
|
2449
|
+
width: Math.max(1, bb.width + padding * 2),
|
|
2450
|
+
height: Math.max(1, bb.height + padding * 2),
|
|
2451
|
+
};
|
|
2452
|
+
clip.width = Math.min(clip.width, viewport.width - clip.x);
|
|
2453
|
+
clip.height = Math.min(clip.height, viewport.height - clip.y);
|
|
2454
|
+
if (clip.width <= 0 || clip.height <= 0) {
|
|
2455
|
+
throw new Error(`Element index ${index} is still outside the viewport after alignment`);
|
|
2456
|
+
}
|
|
2457
|
+
return Buffer.from(await page.screenshot({ type: 'png', clip }));
|
|
2458
|
+
}
|
|
2459
|
+
finally {
|
|
2460
|
+
if (resizedForCapture && originalViewport) {
|
|
2461
|
+
await page.setViewportSize(originalViewport).catch(() => { });
|
|
2462
|
+
await page.waitForTimeout(150).catch(() => { });
|
|
2463
|
+
}
|
|
2464
|
+
await this.restoreFixedOverlays();
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2467
|
+
async screenshotRegion(x, y, width, height, padding = 0) {
|
|
2468
|
+
const page = this.ensurePage();
|
|
2469
|
+
// Move cursor off-screen to avoid hover artifacts
|
|
2470
|
+
await page.mouse.move(0, 0);
|
|
2471
|
+
const scroll = await page.evaluate(() => ({ x: window.scrollX, y: window.scrollY }));
|
|
2472
|
+
const deviceScaleFactor = await page.evaluate(() => window.devicePixelRatio || 1);
|
|
2473
|
+
const clip = {
|
|
2474
|
+
x: Math.max(0, Math.round(scroll.x + x - padding)),
|
|
2475
|
+
y: Math.max(0, Math.round(scroll.y + y - padding)),
|
|
2476
|
+
width: Math.max(1, Math.round(width + padding * 2)),
|
|
2477
|
+
height: Math.max(1, Math.round(height + padding * 2)),
|
|
2478
|
+
};
|
|
2479
|
+
// Hide fixed/sticky overlays (navbars, banners) that could cover the region
|
|
2480
|
+
await this.hideFixedOverlays();
|
|
2481
|
+
try {
|
|
2482
|
+
const fullPage = Buffer.from(await page.screenshot({ type: 'png', fullPage: true }));
|
|
2483
|
+
const image = sharp(fullPage);
|
|
2484
|
+
const meta = await image.metadata();
|
|
2485
|
+
const imageWidth = meta.width ?? 0;
|
|
2486
|
+
const imageHeight = meta.height ?? 0;
|
|
2487
|
+
const left = Math.max(0, Math.min(Math.round(clip.x * deviceScaleFactor), Math.max(0, imageWidth - 1)));
|
|
2488
|
+
const top = Math.max(0, Math.min(Math.round(clip.y * deviceScaleFactor), Math.max(0, imageHeight - 1)));
|
|
2489
|
+
const extractWidth = Math.max(1, Math.min(Math.round(clip.width * deviceScaleFactor), Math.max(1, imageWidth - left)));
|
|
2490
|
+
const extractHeight = Math.max(1, Math.min(Math.round(clip.height * deviceScaleFactor), Math.max(1, imageHeight - top)));
|
|
2491
|
+
return await image
|
|
2492
|
+
.extract({ left, top, width: extractWidth, height: extractHeight })
|
|
2493
|
+
.png()
|
|
2494
|
+
.toBuffer();
|
|
2495
|
+
}
|
|
2496
|
+
finally {
|
|
2497
|
+
await this.restoreFixedOverlays();
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
async screenshotBySelector(selector, outscale = 0) {
|
|
2501
|
+
const page = this.ensurePage();
|
|
2502
|
+
const config = typeof outscale === 'number' ? { padding: outscale } : outscale;
|
|
2503
|
+
const shouldClamp = config.clampToViewport !== false;
|
|
2504
|
+
// 1. Atomic DOM validation: uniqueness + visibility
|
|
2505
|
+
const domCheck = await page.evaluate((sel) => {
|
|
2506
|
+
const nodes = Array.from(document.querySelectorAll(sel));
|
|
2507
|
+
if (nodes.length !== 1) {
|
|
2508
|
+
return { matchCount: nodes.length, visible: false, boundingBox: null, isFixed: false };
|
|
2509
|
+
}
|
|
2510
|
+
const el = nodes[0];
|
|
2511
|
+
const rect = el.getBoundingClientRect();
|
|
2512
|
+
const style = getComputedStyle(el);
|
|
2513
|
+
const isFixed = style.position === 'fixed' || style.position === 'sticky';
|
|
2514
|
+
const visible = (el.offsetParent !== null || isFixed) && rect.width > 0 && rect.height > 0;
|
|
2515
|
+
return {
|
|
2516
|
+
matchCount: 1,
|
|
2517
|
+
visible,
|
|
2518
|
+
isFixed,
|
|
2519
|
+
boundingBox: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
|
|
2520
|
+
};
|
|
2521
|
+
}, selector);
|
|
2522
|
+
if (domCheck.matchCount === 0) {
|
|
2523
|
+
const err = {
|
|
2524
|
+
error: 'no_match',
|
|
2525
|
+
errorMessage: `No element matches selector "${selector}"`,
|
|
2526
|
+
};
|
|
2527
|
+
throw err;
|
|
2528
|
+
}
|
|
2529
|
+
if (domCheck.matchCount > 1) {
|
|
2530
|
+
const err = {
|
|
2531
|
+
error: 'ambiguous',
|
|
2532
|
+
errorMessage: `${domCheck.matchCount} elements match selector "${selector}" — selector must be unique`,
|
|
2533
|
+
};
|
|
2534
|
+
throw err;
|
|
2535
|
+
}
|
|
2536
|
+
if (!domCheck.visible) {
|
|
2537
|
+
const bb = domCheck.boundingBox;
|
|
2538
|
+
const isZeroSize = bb !== null && bb.width === 0 && bb.height === 0;
|
|
2539
|
+
const err = {
|
|
2540
|
+
error: isZeroSize ? 'zero_size' : 'invisible',
|
|
2541
|
+
errorMessage: isZeroSize
|
|
2542
|
+
? `Element matches but has zero dimensions (${bb?.width ?? 0}x${bb?.height ?? 0}) — element may be hidden`
|
|
2543
|
+
: `Element matches but is not visible (offsetParent null and not fixed/sticky) — element may be display:none`,
|
|
2544
|
+
};
|
|
2545
|
+
throw err;
|
|
2546
|
+
}
|
|
2547
|
+
// 2. Scroll element into view and wait for layout to settle
|
|
2548
|
+
const locator = page.locator(selector);
|
|
2549
|
+
if (!domCheck.isFixed) {
|
|
2550
|
+
await locator.scrollIntoViewIfNeeded({ timeout: 5000 });
|
|
2551
|
+
await page.waitForTimeout(200);
|
|
2552
|
+
}
|
|
2553
|
+
// 3. Re-measure post-scroll bounding box (viewport-relative) + scroll position
|
|
2554
|
+
const [rect, pageInfo] = await Promise.all([
|
|
2555
|
+
locator.boundingBox(),
|
|
2556
|
+
page.evaluate(() => ({
|
|
2557
|
+
scrollX: window.scrollX,
|
|
2558
|
+
scrollY: window.scrollY,
|
|
2559
|
+
docWidth: document.documentElement.scrollWidth,
|
|
2560
|
+
docHeight: document.documentElement.scrollHeight,
|
|
2561
|
+
dpr: window.devicePixelRatio || 1,
|
|
2562
|
+
isFixed: false, // already know from domCheck
|
|
2563
|
+
})),
|
|
2564
|
+
]);
|
|
2565
|
+
if (!rect) {
|
|
2566
|
+
const err = {
|
|
2567
|
+
error: 'invisible',
|
|
2568
|
+
errorMessage: `Element "${selector}" became unavailable after scroll`,
|
|
2569
|
+
};
|
|
2570
|
+
throw err;
|
|
2571
|
+
}
|
|
2572
|
+
// 4. Resolve directional padding
|
|
2573
|
+
const pad = resolveEffectivePadding(config, rect);
|
|
2574
|
+
// 5. Convert viewport-relative bbox to document coordinates and apply padding
|
|
2575
|
+
// For fixed/sticky elements: don't add scroll offset (already viewport-absolute)
|
|
2576
|
+
const docX = domCheck.isFixed ? rect.x : rect.x + pageInfo.scrollX;
|
|
2577
|
+
const docY = domCheck.isFixed ? rect.y : rect.y + pageInfo.scrollY;
|
|
2578
|
+
let clipDoc = {
|
|
2579
|
+
x: docX - pad.left,
|
|
2580
|
+
y: docY - pad.top,
|
|
2581
|
+
width: rect.width + pad.left + pad.right,
|
|
2582
|
+
height: rect.height + pad.top + pad.bottom,
|
|
2583
|
+
};
|
|
2584
|
+
// 6. Clamp to document bounds
|
|
2585
|
+
if (shouldClamp) {
|
|
2586
|
+
clipDoc.x = Math.max(0, clipDoc.x);
|
|
2587
|
+
clipDoc.y = Math.max(0, clipDoc.y);
|
|
2588
|
+
clipDoc.width = Math.min(clipDoc.width, pageInfo.docWidth - clipDoc.x);
|
|
2589
|
+
clipDoc.height = Math.min(clipDoc.height, pageInfo.docHeight - clipDoc.y);
|
|
2590
|
+
}
|
|
2591
|
+
if (clipDoc.width <= 0 || clipDoc.height <= 0) {
|
|
2592
|
+
const err = {
|
|
2593
|
+
error: 'invisible',
|
|
2594
|
+
errorMessage: `Element "${selector}" capture zone has zero dimensions after clamping`,
|
|
2595
|
+
};
|
|
2596
|
+
throw err;
|
|
2597
|
+
}
|
|
2598
|
+
// 7. Hide overlays, take full-page screenshot, extract clip via sharp
|
|
2599
|
+
await page.mouse.move(0, 0);
|
|
2600
|
+
await this.hideFixedOverlays();
|
|
2601
|
+
try {
|
|
2602
|
+
const dpr = pageInfo.dpr;
|
|
2603
|
+
const fullPage = Buffer.from(await page.screenshot({ type: 'png', fullPage: true }));
|
|
2604
|
+
const image = sharp(fullPage);
|
|
2605
|
+
const meta = await image.metadata();
|
|
2606
|
+
const imgW = meta.width ?? 0;
|
|
2607
|
+
const imgH = meta.height ?? 0;
|
|
2608
|
+
const left = Math.max(0, Math.min(Math.round(clipDoc.x * dpr), Math.max(0, imgW - 1)));
|
|
2609
|
+
const top = Math.max(0, Math.min(Math.round(clipDoc.y * dpr), Math.max(0, imgH - 1)));
|
|
2610
|
+
const extractW = Math.max(1, Math.min(Math.round(clipDoc.width * dpr), Math.max(1, imgW - left)));
|
|
2611
|
+
const extractH = Math.max(1, Math.min(Math.round(clipDoc.height * dpr), Math.max(1, imgH - top)));
|
|
2612
|
+
const buffer = await image
|
|
2613
|
+
.extract({ left, top, width: extractW, height: extractH })
|
|
2614
|
+
.png()
|
|
2615
|
+
.toBuffer();
|
|
2616
|
+
const validation = {
|
|
2617
|
+
matchCount: 1,
|
|
2618
|
+
visible: true,
|
|
2619
|
+
boundingBox: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
|
|
2620
|
+
};
|
|
2621
|
+
return { buffer, validation };
|
|
2622
|
+
}
|
|
2623
|
+
finally {
|
|
2624
|
+
await this.restoreFixedOverlays();
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
/**
|
|
2628
|
+
* Capture a page region by bounding box coordinates (document-relative).
|
|
2629
|
+
* Used as a fallback when no CSS selector can be resolved for the target element.
|
|
2630
|
+
*/
|
|
2631
|
+
async screenshotByRegion(region, outscale = 0) {
|
|
2632
|
+
const page = this.ensurePage();
|
|
2633
|
+
const config = typeof outscale === 'number' ? { padding: outscale } : outscale;
|
|
2634
|
+
const shouldClamp = config.clampToViewport !== false;
|
|
2635
|
+
const pageInfo = await page.evaluate(() => ({
|
|
2636
|
+
scrollX: window.scrollX,
|
|
2637
|
+
scrollY: window.scrollY,
|
|
2638
|
+
docWidth: document.documentElement.scrollWidth,
|
|
2639
|
+
docHeight: document.documentElement.scrollHeight,
|
|
2640
|
+
dpr: window.devicePixelRatio || 1,
|
|
2641
|
+
}));
|
|
2642
|
+
const pad = resolveEffectivePadding(config, region);
|
|
2643
|
+
let clipDoc = {
|
|
2644
|
+
x: region.x - pad.left,
|
|
2645
|
+
y: region.y - pad.top,
|
|
2646
|
+
width: region.width + pad.left + pad.right,
|
|
2647
|
+
height: region.height + pad.top + pad.bottom,
|
|
2648
|
+
};
|
|
2649
|
+
if (shouldClamp) {
|
|
2650
|
+
clipDoc.x = Math.max(0, clipDoc.x);
|
|
2651
|
+
clipDoc.y = Math.max(0, clipDoc.y);
|
|
2652
|
+
clipDoc.width = Math.min(clipDoc.width, pageInfo.docWidth - clipDoc.x);
|
|
2653
|
+
clipDoc.height = Math.min(clipDoc.height, pageInfo.docHeight - clipDoc.y);
|
|
2654
|
+
}
|
|
2655
|
+
if (clipDoc.width <= 0 || clipDoc.height <= 0) {
|
|
2656
|
+
throw new Error(`Region capture zone has zero dimensions after clamping`);
|
|
2657
|
+
}
|
|
2658
|
+
// Scroll region into view
|
|
2659
|
+
await page.evaluate(({ x, y }) => window.scrollTo(x, y - 100), { x: clipDoc.x, y: clipDoc.y });
|
|
2660
|
+
await page.waitForTimeout(200);
|
|
2661
|
+
await page.mouse.move(0, 0);
|
|
2662
|
+
await this.hideFixedOverlays();
|
|
2663
|
+
try {
|
|
2664
|
+
const dpr = pageInfo.dpr;
|
|
2665
|
+
const fullPage = Buffer.from(await page.screenshot({ type: 'png', fullPage: true }));
|
|
2666
|
+
const image = sharp(fullPage);
|
|
2667
|
+
const meta = await image.metadata();
|
|
2668
|
+
const imgW = meta.width ?? 0;
|
|
2669
|
+
const imgH = meta.height ?? 0;
|
|
2670
|
+
const left = Math.max(0, Math.min(Math.round(clipDoc.x * dpr), Math.max(0, imgW - 1)));
|
|
2671
|
+
const top = Math.max(0, Math.min(Math.round(clipDoc.y * dpr), Math.max(0, imgH - 1)));
|
|
2672
|
+
const extractW = Math.max(1, Math.min(Math.round(clipDoc.width * dpr), Math.max(1, imgW - left)));
|
|
2673
|
+
const extractH = Math.max(1, Math.min(Math.round(clipDoc.height * dpr), Math.max(1, imgH - top)));
|
|
2674
|
+
return image
|
|
2675
|
+
.extract({ left, top, width: extractW, height: extractH })
|
|
2676
|
+
.png()
|
|
2677
|
+
.toBuffer();
|
|
2678
|
+
}
|
|
2679
|
+
finally {
|
|
2680
|
+
await this.restoreFixedOverlays();
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
async wait(ms) {
|
|
2684
|
+
const page = this.ensurePage();
|
|
2685
|
+
await page.waitForTimeout(Math.min(ms, 5000));
|
|
2686
|
+
}
|
|
2687
|
+
/**
|
|
2688
|
+
* Force-load all lazy-loaded images on the page before element capture.
|
|
2689
|
+
*
|
|
2690
|
+
* 1. Strip loading="lazy" and promote data-src/data-srcset attributes
|
|
2691
|
+
* 2. Scroll through the full page to trigger IntersectionObserver callbacks
|
|
2692
|
+
* 3. Wait for all visible images to finish loading
|
|
2693
|
+
*/
|
|
2694
|
+
async forceLoadLazyImages(options) {
|
|
2695
|
+
const page = this.ensurePage();
|
|
2696
|
+
const timeout = options?.timeout ?? 10_000;
|
|
2697
|
+
const startTime = Date.now();
|
|
2698
|
+
// Phase 1: Strip lazy attributes and promote data-src
|
|
2699
|
+
await page.evaluate(() => {
|
|
2700
|
+
for (const el of document.querySelectorAll('img[loading="lazy"], iframe[loading="lazy"]')) {
|
|
2701
|
+
el.removeAttribute('loading');
|
|
2702
|
+
if (el.tagName === 'IMG') {
|
|
2703
|
+
const img = el;
|
|
2704
|
+
const src = img.src;
|
|
2705
|
+
if (src) {
|
|
2706
|
+
img.src = '';
|
|
2707
|
+
img.src = src;
|
|
2708
|
+
}
|
|
2709
|
+
}
|
|
2710
|
+
}
|
|
2711
|
+
for (const img of document.querySelectorAll('img[data-src]')) {
|
|
2712
|
+
const ds = img.getAttribute('data-src');
|
|
2713
|
+
if (!img.src || img.src === '' || img.src === window.location.href)
|
|
2714
|
+
img.src = ds;
|
|
2715
|
+
}
|
|
2716
|
+
for (const img of document.querySelectorAll('img[data-srcset]')) {
|
|
2717
|
+
const dss = img.getAttribute('data-srcset');
|
|
2718
|
+
if (!img.getAttribute('srcset'))
|
|
2719
|
+
img.setAttribute('srcset', dss);
|
|
2720
|
+
}
|
|
2721
|
+
});
|
|
2722
|
+
// Phase 2: Scroll sweep to trigger IntersectionObserver callbacks
|
|
2723
|
+
const dims = await page.evaluate(() => ({
|
|
2724
|
+
scrollHeight: document.documentElement.scrollHeight,
|
|
2725
|
+
viewportHeight: window.innerHeight,
|
|
2726
|
+
scrollX: window.scrollX,
|
|
2727
|
+
scrollY: window.scrollY,
|
|
2728
|
+
}));
|
|
2729
|
+
if (dims.scrollHeight > dims.viewportHeight) {
|
|
2730
|
+
const step = Math.floor(dims.viewportHeight * 0.8);
|
|
2731
|
+
let y = 0;
|
|
2732
|
+
while (y < dims.scrollHeight) {
|
|
2733
|
+
if (Date.now() - startTime > timeout * 0.6)
|
|
2734
|
+
break;
|
|
2735
|
+
await page.evaluate((scrollY) => window.scrollTo(0, scrollY), y);
|
|
2736
|
+
await page.waitForTimeout(150);
|
|
2737
|
+
y += step;
|
|
2738
|
+
}
|
|
2739
|
+
// Scroll to very bottom to catch remaining elements
|
|
2740
|
+
await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
|
|
2741
|
+
await page.waitForTimeout(150);
|
|
2742
|
+
// Restore original scroll position
|
|
2743
|
+
await page.evaluate(({ x, y }) => window.scrollTo(x, y), { x: dims.scrollX, y: dims.scrollY });
|
|
2744
|
+
}
|
|
2745
|
+
// Phase 3: Wait for all images to finish loading
|
|
2746
|
+
const remainingMs = Math.max(timeout - (Date.now() - startTime), 2000);
|
|
2747
|
+
try {
|
|
2748
|
+
await page.evaluate((maxWait) => {
|
|
2749
|
+
return new Promise((resolve) => {
|
|
2750
|
+
const deadline = Date.now() + maxWait;
|
|
2751
|
+
const check = () => {
|
|
2752
|
+
const imgs = document.querySelectorAll('img');
|
|
2753
|
+
for (const img of imgs) {
|
|
2754
|
+
if (!img.src || img.src.startsWith('data:'))
|
|
2755
|
+
continue;
|
|
2756
|
+
if (img.width <= 1 && img.height <= 1)
|
|
2757
|
+
continue;
|
|
2758
|
+
if (!img.complete)
|
|
2759
|
+
return false;
|
|
2760
|
+
}
|
|
2761
|
+
return true;
|
|
2762
|
+
};
|
|
2763
|
+
if (check()) {
|
|
2764
|
+
resolve();
|
|
2765
|
+
return;
|
|
2766
|
+
}
|
|
2767
|
+
const iv = setInterval(() => {
|
|
2768
|
+
if (check() || Date.now() > deadline) {
|
|
2769
|
+
clearInterval(iv);
|
|
2770
|
+
resolve();
|
|
2771
|
+
}
|
|
2772
|
+
}, 200);
|
|
2773
|
+
});
|
|
2774
|
+
}, remainingMs);
|
|
2775
|
+
}
|
|
2776
|
+
catch {
|
|
2777
|
+
// Timeout — proceed with what we have
|
|
2778
|
+
}
|
|
2779
|
+
await page.waitForTimeout(200);
|
|
2780
|
+
}
|
|
2781
|
+
async setColorScheme(scheme) {
|
|
2782
|
+
const page = this.ensurePage();
|
|
2783
|
+
await page.emulateMedia({ colorScheme: scheme });
|
|
2784
|
+
// Grace period: wait for CSS transitions triggered by theme switch to settle.
|
|
2785
|
+
// Without this, elements may have pointer-events intercepted by the <html> element
|
|
2786
|
+
// during the transition, causing Playwright clicks to timeout.
|
|
2787
|
+
await page.waitForTimeout(600);
|
|
2788
|
+
}
|
|
2789
|
+
async setLanguage(lang) {
|
|
2790
|
+
const context = this.ensureContext();
|
|
2791
|
+
const page = this.ensurePage();
|
|
2792
|
+
await context.setExtraHTTPHeaders({ 'Accept-Language': lang });
|
|
2793
|
+
await page.addInitScript((locale) => {
|
|
2794
|
+
Object.defineProperty(navigator, 'language', { get: () => locale, configurable: true });
|
|
2795
|
+
Object.defineProperty(navigator, 'languages', { get: () => [locale], configurable: true });
|
|
2796
|
+
}, lang);
|
|
2797
|
+
}
|
|
2798
|
+
async resizeViewport(width, height) {
|
|
2799
|
+
const page = this.ensurePage();
|
|
2800
|
+
await page.setViewportSize({
|
|
2801
|
+
width: normalizeViewportDimension(width),
|
|
2802
|
+
height: normalizeViewportDimension(height),
|
|
2803
|
+
});
|
|
2804
|
+
}
|
|
2805
|
+
get currentPage() {
|
|
2806
|
+
return this.ensurePage();
|
|
2807
|
+
}
|
|
2808
|
+
get browserContext() {
|
|
2809
|
+
return this.ensureContext();
|
|
2810
|
+
}
|
|
2811
|
+
/**
|
|
2812
|
+
* Observation pass for mock data generation.
|
|
2813
|
+
* Navigates to the given URL, waits for network idle, and records all JSON API responses.
|
|
2814
|
+
* Auth/analytics/asset endpoints are filtered out automatically.
|
|
2815
|
+
*/
|
|
2816
|
+
async observeNetworkRequests(url, waitMs = 2000) {
|
|
2817
|
+
const page = this.ensurePage();
|
|
2818
|
+
const observed = [];
|
|
2819
|
+
const SKIP_PATTERNS = [
|
|
2820
|
+
/\/auth\b/, /\/oauth\b/, /\/token\b/, /\/session\b/,
|
|
2821
|
+
/\/login\b/, /\/logout\b/, /\/refresh\b/, /\/verify\b/,
|
|
2822
|
+
/\/_next\//, /\/analytics\b/, /posthog\./, /sentry\./,
|
|
2823
|
+
/\.(js|css|woff2?|ttf|svg|png|jpg|jpeg|gif|ico|webp)(\?|$)/i,
|
|
2824
|
+
/favicon/i,
|
|
2825
|
+
];
|
|
2826
|
+
const handler = async (response) => {
|
|
2827
|
+
try {
|
|
2828
|
+
const reqUrl = response.url();
|
|
2829
|
+
if (SKIP_PATTERNS.some((re) => re.test(reqUrl)))
|
|
2830
|
+
return;
|
|
2831
|
+
const ct = response.headers()['content-type'] ?? '';
|
|
2832
|
+
if (!ct.includes('application/json'))
|
|
2833
|
+
return;
|
|
2834
|
+
const body = await response.json().catch(() => null);
|
|
2835
|
+
if (body === null)
|
|
2836
|
+
return;
|
|
2837
|
+
observed.push({
|
|
2838
|
+
url: reqUrl,
|
|
2839
|
+
method: response.request().method(),
|
|
2840
|
+
status: response.status(),
|
|
2841
|
+
responseBody: body,
|
|
2842
|
+
});
|
|
2843
|
+
}
|
|
2844
|
+
catch {
|
|
2845
|
+
// ignore errors from individual responses
|
|
2846
|
+
}
|
|
2847
|
+
};
|
|
2848
|
+
page.on('response', handler);
|
|
2849
|
+
try {
|
|
2850
|
+
await page.goto(url, { waitUntil: 'networkidle', timeout: 15000 }).catch(() => { });
|
|
2851
|
+
if (waitMs > 0)
|
|
2852
|
+
await page.waitForTimeout(waitMs);
|
|
2853
|
+
}
|
|
2854
|
+
finally {
|
|
2855
|
+
page.off('response', handler);
|
|
2856
|
+
}
|
|
2857
|
+
return observed;
|
|
2858
|
+
}
|
|
2859
|
+
/**
|
|
2860
|
+
* Sets up Playwright route interceptors for each resolved mock.
|
|
2861
|
+
* Must be called before navigating to the page for the actual capture.
|
|
2862
|
+
*/
|
|
2863
|
+
async setupRouteInterception(mocks) {
|
|
2864
|
+
const page = this.ensurePage();
|
|
2865
|
+
for (const mock of mocks) {
|
|
2866
|
+
await page.route(mock.urlPattern, (route) => {
|
|
2867
|
+
if (mock.method && route.request().method() !== mock.method) {
|
|
2868
|
+
return route.continue();
|
|
2869
|
+
}
|
|
2870
|
+
const body = mock.contentType.includes('application/json')
|
|
2871
|
+
? JSON.stringify(mock.responseBody)
|
|
2872
|
+
: typeof mock.responseBody === 'string'
|
|
2873
|
+
? mock.responseBody
|
|
2874
|
+
: JSON.stringify(mock.responseBody);
|
|
2875
|
+
return route.fulfill({
|
|
2876
|
+
status: mock.status,
|
|
2877
|
+
contentType: mock.contentType,
|
|
2878
|
+
body,
|
|
2879
|
+
});
|
|
2880
|
+
});
|
|
2881
|
+
}
|
|
2882
|
+
}
|
|
2883
|
+
/**
|
|
2884
|
+
* Removes all route interceptors set by setupRouteInterception.
|
|
2885
|
+
* Must be called before setting up new mocks to prevent stacking.
|
|
2886
|
+
*/
|
|
2887
|
+
async clearRouteInterception() {
|
|
2888
|
+
const page = this.ensurePage();
|
|
2889
|
+
await page.unrouteAll({ behavior: 'wait' });
|
|
2890
|
+
}
|
|
2891
|
+
ensurePage() {
|
|
2892
|
+
if (!this.page)
|
|
2893
|
+
throw new Error('Browser not launched. Call launch() first.');
|
|
2894
|
+
return this.page;
|
|
2895
|
+
}
|
|
2896
|
+
ensureContext() {
|
|
2897
|
+
if (!this.context)
|
|
2898
|
+
throw new Error('Browser not launched. Call launch() first.');
|
|
2899
|
+
return this.context;
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
//# sourceMappingURL=browser.js.map
|