@testmuai/playwright-bindings 0.1.26 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actionEngine.d.ts +37 -0
- package/dist/actionEngine.d.ts.map +1 -0
- package/dist/actionEngine.js +194 -0
- package/dist/actionEngine.js.map +1 -0
- package/dist/actionSpecs.d.ts +35 -0
- package/dist/actionSpecs.d.ts.map +1 -0
- package/dist/actionSpecs.js +354 -0
- package/dist/actionSpecs.js.map +1 -0
- package/dist/capability.d.ts +3 -5
- package/dist/capability.d.ts.map +1 -1
- package/dist/capability.js +66 -6
- package/dist/capability.js.map +1 -1
- package/dist/configure.d.ts +16 -0
- package/dist/configure.d.ts.map +1 -1
- package/dist/configure.js +14 -0
- package/dist/configure.js.map +1 -1
- package/dist/errors.d.ts +19 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +29 -0
- package/dist/errors.js.map +1 -1
- package/dist/healCache.d.ts +28 -0
- package/dist/healCache.d.ts.map +1 -0
- package/dist/healCache.js +17 -0
- package/dist/healCache.js.map +1 -0
- package/dist/healPatch.d.ts.map +1 -1
- package/dist/healPatch.js +45 -3
- package/dist/healPatch.js.map +1 -1
- package/dist/helpers/assertion.d.ts.map +1 -1
- package/dist/helpers/assertion.js +18 -0
- package/dist/helpers/assertion.js.map +1 -1
- package/dist/helpers/assertionV3.d.ts +67 -0
- package/dist/helpers/assertionV3.d.ts.map +1 -0
- package/dist/helpers/assertionV3.js +384 -0
- package/dist/helpers/assertionV3.js.map +1 -0
- package/dist/helpers/condition.d.ts +84 -0
- package/dist/helpers/condition.d.ts.map +1 -0
- package/dist/helpers/condition.js +292 -0
- package/dist/helpers/condition.js.map +1 -0
- package/dist/helpers/cookies.d.ts +12 -0
- package/dist/helpers/cookies.d.ts.map +1 -0
- package/dist/helpers/cookies.js +66 -0
- package/dist/helpers/cookies.js.map +1 -0
- package/dist/helpers/coordinateResolver.d.ts +29 -0
- package/dist/helpers/coordinateResolver.d.ts.map +1 -0
- package/dist/helpers/coordinateResolver.js +118 -0
- package/dist/helpers/coordinateResolver.js.map +1 -0
- package/dist/helpers/coordinateResolverRead.d.ts +36 -0
- package/dist/helpers/coordinateResolverRead.d.ts.map +1 -0
- package/dist/helpers/coordinateResolverRead.js +107 -0
- package/dist/helpers/coordinateResolverRead.js.map +1 -0
- package/dist/helpers/dialog.d.ts +29 -0
- package/dist/helpers/dialog.d.ts.map +1 -0
- package/dist/helpers/dialog.js +35 -0
- package/dist/helpers/dialog.js.map +1 -0
- package/dist/helpers/drag.d.ts +36 -1
- package/dist/helpers/drag.d.ts.map +1 -1
- package/dist/helpers/drag.js +267 -0
- package/dist/helpers/drag.js.map +1 -1
- package/dist/helpers/executeApi.d.ts +2 -0
- package/dist/helpers/executeApi.d.ts.map +1 -1
- package/dist/helpers/executeApi.js +30 -10
- package/dist/helpers/executeApi.js.map +1 -1
- package/dist/helpers/executeJs.d.ts.map +1 -1
- package/dist/helpers/executeJs.js +46 -3
- package/dist/helpers/executeJs.js.map +1 -1
- package/dist/helpers/gesture.d.ts +45 -0
- package/dist/helpers/gesture.d.ts.map +1 -0
- package/dist/helpers/gesture.js +131 -0
- package/dist/helpers/gesture.js.map +1 -0
- package/dist/helpers/healCascade.d.ts +57 -0
- package/dist/helpers/healCascade.d.ts.map +1 -0
- package/dist/helpers/healCascade.js +363 -0
- package/dist/helpers/healCascade.js.map +1 -0
- package/dist/helpers/png.d.ts +3 -0
- package/dist/helpers/png.d.ts.map +1 -0
- package/dist/helpers/png.js +18 -0
- package/dist/helpers/png.js.map +1 -0
- package/dist/helpers/screenshot.d.ts +27 -0
- package/dist/helpers/screenshot.d.ts.map +1 -1
- package/dist/helpers/screenshot.js +56 -0
- package/dist/helpers/screenshot.js.map +1 -1
- package/dist/helpers/scrollUntil.d.ts +13 -0
- package/dist/helpers/scrollUntil.d.ts.map +1 -0
- package/dist/helpers/scrollUntil.js +88 -0
- package/dist/helpers/scrollUntil.js.map +1 -0
- package/dist/helpers/tabs.d.ts +42 -1
- package/dist/helpers/tabs.d.ts.map +1 -1
- package/dist/helpers/tabs.js +259 -5
- package/dist/helpers/tabs.js.map +1 -1
- package/dist/helpers/text.d.ts +15 -0
- package/dist/helpers/text.d.ts.map +1 -0
- package/dist/helpers/text.js +24 -0
- package/dist/helpers/text.js.map +1 -0
- package/dist/helpers/textNormalize.d.ts +18 -0
- package/dist/helpers/textNormalize.d.ts.map +1 -0
- package/dist/helpers/textNormalize.js +39 -0
- package/dist/helpers/textNormalize.js.map +1 -0
- package/dist/helpers/textualRead.d.ts +37 -0
- package/dist/helpers/textualRead.d.ts.map +1 -0
- package/dist/helpers/textualRead.js +211 -0
- package/dist/helpers/textualRead.js.map +1 -0
- package/dist/helpers/vision.d.ts +2 -0
- package/dist/helpers/vision.d.ts.map +1 -1
- package/dist/helpers/vision.js +11 -0
- package/dist/helpers/vision.js.map +1 -1
- package/dist/helpers/visionV3.d.ts +38 -0
- package/dist/helpers/visionV3.d.ts.map +1 -0
- package/dist/helpers/visionV3.js +291 -0
- package/dist/helpers/visionV3.js.map +1 -0
- package/dist/helpers/wait.d.ts +14 -1
- package/dist/helpers/wait.d.ts.map +1 -1
- package/dist/helpers/wait.js +25 -0
- package/dist/helpers/wait.js.map +1 -1
- package/dist/index.d.ts +29 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +23 -4
- package/dist/index.js.map +1 -1
- package/dist/log.d.ts +4 -1
- package/dist/log.d.ts.map +1 -1
- package/dist/log.js +8 -3
- package/dist/log.js.map +1 -1
- package/dist/rankedLocator.d.ts +27 -0
- package/dist/rankedLocator.d.ts.map +1 -0
- package/dist/rankedLocator.js +85 -0
- package/dist/rankedLocator.js.map +1 -0
- package/dist/reporters/index.d.ts +2 -0
- package/dist/reporters/index.d.ts.map +1 -1
- package/dist/reporters/index.js.map +1 -1
- package/dist/reporters/lt.d.ts.map +1 -1
- package/dist/reporters/lt.js +7 -1
- package/dist/reporters/lt.js.map +1 -1
- package/dist/routeFailure.d.ts +9 -0
- package/dist/routeFailure.d.ts.map +1 -0
- package/dist/routeFailure.js +34 -0
- package/dist/routeFailure.js.map +1 -0
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +21 -0
- package/dist/session.js.map +1 -1
- package/dist/step.d.ts +10 -0
- package/dist/step.d.ts.map +1 -1
- package/dist/step.js +14 -1
- package/dist/step.js.map +1 -1
- package/dist/testConfig.d.ts.map +1 -1
- package/dist/testConfig.js +8 -1
- package/dist/testConfig.js.map +1 -1
- package/dist/vars.d.ts +10 -0
- package/dist/vars.d.ts.map +1 -1
- package/dist/vars.js +30 -3
- package/dist/vars.js.map +1 -1
- package/dist/visionLocator.d.ts +53 -0
- package/dist/visionLocator.d.ts.map +1 -0
- package/dist/visionLocator.js +117 -0
- package/dist/visionLocator.js.map +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
import { AutohealExhausted } from '../errors.js';
|
|
2
|
+
import { markAutoHeal } from '../step.js';
|
|
3
|
+
import { log } from '../log.js';
|
|
4
|
+
import { get as configGet } from '../configure.js';
|
|
5
|
+
import { pngDimensions } from './png.js';
|
|
6
|
+
import { resolveCoordinate } from './coordinateResolver.js';
|
|
7
|
+
import { captureScreenshotB64, captureFullPageScreenshotB64 } from './screenshot.js';
|
|
8
|
+
// Locate is on AUTOMIND_URL (NOT the TESTMU_AI_API_HOST vision stack).
|
|
9
|
+
const AUTOMIND_FALLBACK = 'https://kaneai-api.lambdatest.com';
|
|
10
|
+
const LOCATE_TIMEOUT_MS = 120_000;
|
|
11
|
+
// Real function, NOT a string: the grid CDP relay returns undefined for
|
|
12
|
+
// string page-functions (probed 2026-07-30; corner-case for the catalog).
|
|
13
|
+
const VIEWPORT_FN = () => [window.innerWidth, window.innerHeight];
|
|
14
|
+
const DEFAULT_HEAL_TIERS = ['DESKTOP_LOCATE'];
|
|
15
|
+
const MAX_LOCATE_ATTEMPTS = 3;
|
|
16
|
+
const ASPECT_RATIO_TOLERANCE = 0.02;
|
|
17
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
18
|
+
function resolveAutomindUrl() {
|
|
19
|
+
return process.env['AUTOMIND_URL'] || AUTOMIND_FALLBACK;
|
|
20
|
+
}
|
|
21
|
+
/** LT Basic auth from env (PY create_session parity). */
|
|
22
|
+
function basicAuthHeaders() {
|
|
23
|
+
const headers = {
|
|
24
|
+
'content-type': 'application/json',
|
|
25
|
+
'accept-encoding': 'gzip, deflate',
|
|
26
|
+
};
|
|
27
|
+
const user = process.env['LT_USERNAME'] ?? '';
|
|
28
|
+
const key = process.env['LT_ACCESS_KEY'] ?? '';
|
|
29
|
+
if (user && key) {
|
|
30
|
+
headers['authorization'] = `Basic ${Buffer.from(`${user}:${key}`).toString('base64')}`;
|
|
31
|
+
}
|
|
32
|
+
const sessionId = process.env['TESTMUAI_SESSION_ID'] ?? '';
|
|
33
|
+
if (sessionId)
|
|
34
|
+
headers['x-session-id'] = sessionId;
|
|
35
|
+
return headers;
|
|
36
|
+
}
|
|
37
|
+
function requestId() {
|
|
38
|
+
// 16 hex chars; Math.random is fine for a tracing id.
|
|
39
|
+
return [...Array(16)].map(() => Math.floor(Math.random() * 16).toString(16)).join('');
|
|
40
|
+
}
|
|
41
|
+
/** Shared locate POST with the cascade's retry contract. Returns the parsed
|
|
42
|
+
* body on 200, null on miss-worthy outcomes; 4xx returns 'fatal'. */
|
|
43
|
+
async function postLocate(payload, label, description, attempt) {
|
|
44
|
+
const endpoint = `${resolveAutomindUrl()}/v2/locate/desktop`;
|
|
45
|
+
const resp = await fetch(endpoint, {
|
|
46
|
+
method: 'POST',
|
|
47
|
+
headers: basicAuthHeaders(),
|
|
48
|
+
body: JSON.stringify(payload),
|
|
49
|
+
signal: AbortSignal.timeout(LOCATE_TIMEOUT_MS),
|
|
50
|
+
});
|
|
51
|
+
if (resp.status >= 400 && resp.status < 500) {
|
|
52
|
+
log.helper(`[heal ${label}] 4xx status=${resp.status} for '${description.slice(0, 80)}' — not retrying`);
|
|
53
|
+
return 'fatal';
|
|
54
|
+
}
|
|
55
|
+
if (resp.status !== 200) {
|
|
56
|
+
log.helper(`[heal ${label}] non-200 status=${resp.status} for '${description.slice(0, 80)}' (attempt ${attempt + 1}/${MAX_LOCATE_ATTEMPTS})`);
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
return (await resp.json());
|
|
60
|
+
}
|
|
61
|
+
/** Miss detection: /v2/locate/desktop returns HTTP 200 with coordinate [0,0]
|
|
62
|
+
* on not-found (NOT 404). Missing/null/[0,0]/len<2 are all miss. */
|
|
63
|
+
function extractCoordinate(result) {
|
|
64
|
+
const c = result.coordinate;
|
|
65
|
+
if (!c || c.length < 2 || (c[0] === 0 && c[1] === 0))
|
|
66
|
+
return null;
|
|
67
|
+
return [Number(c[0]), Number(c[1])];
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* DESKTOP_LOCATE client — viewport screenshot → {AUTOMIND_URL}/v2/locate/desktop.
|
|
71
|
+
* Returns clamped (cssX, cssY) on hit, null on miss. Never throws.
|
|
72
|
+
* dropAware: the drag-drop heal sets it true when locating a drop TARGET.
|
|
73
|
+
*/
|
|
74
|
+
export async function getV3DesktopLocateTarget(page, description, _methodName = 'click', opts = {}) {
|
|
75
|
+
if (!description) {
|
|
76
|
+
log.helper('[heal desktop_locate] empty description — skipping locate call');
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
for (let attempt = 0; attempt < MAX_LOCATE_ATTEMPTS; attempt++) {
|
|
80
|
+
try {
|
|
81
|
+
// Fresh screenshot + viewport dims each attempt (page mutations between
|
|
82
|
+
// retries are reflected). CDP capture — page.screenshot() hangs on
|
|
83
|
+
// document.fonts.ready behind the grid proxy.
|
|
84
|
+
const imageB64 = await captureScreenshotB64(page);
|
|
85
|
+
const bytes = Buffer.from(imageB64, 'base64');
|
|
86
|
+
let imageWidth, imageHeight;
|
|
87
|
+
try {
|
|
88
|
+
[imageWidth, imageHeight] = pngDimensions(bytes);
|
|
89
|
+
}
|
|
90
|
+
catch (e) {
|
|
91
|
+
// Invalid PNG is deterministic — never retried, never posted.
|
|
92
|
+
log.helper(`[heal desktop_locate] invalid PNG screenshot for '${description.slice(0, 80)}': ${String(e)}`);
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
const dims = (await page.evaluate(VIEWPORT_FN));
|
|
96
|
+
const viewportWidth = Math.trunc(dims[0]);
|
|
97
|
+
const viewportHeight = Math.trunc(dims[1]);
|
|
98
|
+
if (viewportWidth > 0 && viewportHeight > 0 && imageHeight > 0) {
|
|
99
|
+
const imgRatio = imageWidth / imageHeight;
|
|
100
|
+
const vpRatio = viewportWidth / viewportHeight;
|
|
101
|
+
if (Math.abs(imgRatio - vpRatio) / vpRatio > ASPECT_RATIO_TOLERANCE) {
|
|
102
|
+
log.helper(`[heal desktop_locate] aspect ratio mismatch for '${description.slice(0, 80)}': image=${imgRatio.toFixed(4)} viewport=${vpRatio.toFixed(4)}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const result = await postLocate({
|
|
106
|
+
intent: description,
|
|
107
|
+
image: imageB64,
|
|
108
|
+
os: 'desktop',
|
|
109
|
+
platform: 'web',
|
|
110
|
+
screen_width: viewportWidth,
|
|
111
|
+
screen_height: viewportHeight,
|
|
112
|
+
drop_aware: opts.dropAware ?? false,
|
|
113
|
+
request_id: requestId(),
|
|
114
|
+
a11y_flatten: [],
|
|
115
|
+
}, 'desktop_locate', description, attempt);
|
|
116
|
+
if (result === 'fatal')
|
|
117
|
+
return null;
|
|
118
|
+
if (result === null) {
|
|
119
|
+
if (attempt < MAX_LOCATE_ATTEMPTS - 1)
|
|
120
|
+
await sleep(1000);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const coord = extractCoordinate(result);
|
|
124
|
+
if (coord === null) {
|
|
125
|
+
log.helper(`[heal desktop_locate] MISS for '${description.slice(0, 80)}'`);
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
const [rawX, rawY] = coord;
|
|
129
|
+
// Scale image-space → viewport CSS pixels, then clamp.
|
|
130
|
+
let cssX = Math.trunc((rawX * viewportWidth) / Math.max(imageWidth, 1));
|
|
131
|
+
let cssY = Math.trunc((rawY * viewportHeight) / Math.max(imageHeight, 1));
|
|
132
|
+
cssX = Math.max(0, Math.min(cssX, viewportWidth - 1));
|
|
133
|
+
cssY = Math.max(0, Math.min(cssY, viewportHeight - 1));
|
|
134
|
+
log.helper(`[heal desktop_locate] HIT image=(${rawX},${rawY}) image_dims=(${imageWidth},${imageHeight}) viewport=(${viewportWidth},${viewportHeight}) css=(${cssX},${cssY}) confidence=${JSON.stringify(result.confidence)}`);
|
|
135
|
+
return [cssX, cssY];
|
|
136
|
+
}
|
|
137
|
+
catch (e) {
|
|
138
|
+
const name = e?.name ?? '';
|
|
139
|
+
// Transport-level failures are retried; anything else (page closed,
|
|
140
|
+
// evaluate failure) is not.
|
|
141
|
+
if (name === 'TimeoutError' || name === 'AbortError' || name === 'TypeError') {
|
|
142
|
+
log.helper(`[heal desktop_locate] transport error for '${description.slice(0, 80)}' (attempt ${attempt + 1}/${MAX_LOCATE_ATTEMPTS}): ${String(e)}`);
|
|
143
|
+
if (attempt < MAX_LOCATE_ATTEMPTS - 1)
|
|
144
|
+
await sleep(1000);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
log.helper(`[heal desktop_locate] unexpected error for '${description.slice(0, 80)}': ${String(e)}`);
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
log.helper(`[heal desktop_locate] all ${MAX_LOCATE_ATTEMPTS} attempts exhausted for '${description.slice(0, 80)}'`);
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
async function runDesktopLocateTier(page, description, verbName) {
|
|
156
|
+
const coords = await getV3DesktopLocateTarget(page, description, verbName);
|
|
157
|
+
if (coords === null)
|
|
158
|
+
return null;
|
|
159
|
+
// Derive a verified XPath so the engine can act with full element semantics
|
|
160
|
+
// (spec §3.2); coordinates stay populated as the in-round fallback.
|
|
161
|
+
const resolved = await resolveCoordinate(page, coords[0], coords[1]);
|
|
162
|
+
log.helper(` [heal] DESKTOP_LOCATE derived_xpath=${resolved?.xpath ?? 'None'}`);
|
|
163
|
+
return {
|
|
164
|
+
tierUsed: 'DESKTOP_LOCATE',
|
|
165
|
+
selectorXpath: resolved?.xpath ?? null,
|
|
166
|
+
coordinates: coords,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* DESKTOP_LOCATE_FULL_PAGE client — full-page screenshot ONCE (reused across
|
|
171
|
+
* retries), screen_width/height = 0 per the full-page contract; returns
|
|
172
|
+
* page-space CSS coordinates.
|
|
173
|
+
*/
|
|
174
|
+
export async function getV3DesktopLocateFullPageTarget(page, description) {
|
|
175
|
+
if (!description) {
|
|
176
|
+
log.helper('[heal desktop_locate_full_page] empty description — skipping');
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
const imageB64 = await captureFullPageScreenshotB64(page);
|
|
180
|
+
const bytes = Buffer.from(imageB64, 'base64');
|
|
181
|
+
let imageWidth, imageHeight;
|
|
182
|
+
try {
|
|
183
|
+
[imageWidth, imageHeight] = pngDimensions(bytes);
|
|
184
|
+
}
|
|
185
|
+
catch (e) {
|
|
186
|
+
log.helper(`[heal desktop_locate_full_page] invalid PNG screenshot for '${description.slice(0, 80)}': ${String(e)}`);
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
const dims = (await page.evaluate(() => [document.documentElement.scrollWidth, document.documentElement.scrollHeight]));
|
|
190
|
+
const pageWidth = Math.trunc(dims[0]);
|
|
191
|
+
const pageHeight = Math.trunc(dims[1]);
|
|
192
|
+
for (let attempt = 0; attempt < MAX_LOCATE_ATTEMPTS; attempt++) {
|
|
193
|
+
try {
|
|
194
|
+
const result = await postLocate({
|
|
195
|
+
intent: description,
|
|
196
|
+
image: imageB64,
|
|
197
|
+
os: 'desktop',
|
|
198
|
+
platform: 'web',
|
|
199
|
+
screen_width: 0,
|
|
200
|
+
screen_height: 0,
|
|
201
|
+
drop_aware: false,
|
|
202
|
+
request_id: requestId(),
|
|
203
|
+
a11y_flatten: [],
|
|
204
|
+
}, 'desktop_locate_full_page', description, attempt);
|
|
205
|
+
if (result === 'fatal')
|
|
206
|
+
return null;
|
|
207
|
+
if (result === null) {
|
|
208
|
+
if (attempt < MAX_LOCATE_ATTEMPTS - 1)
|
|
209
|
+
await sleep(1000);
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
const coord = extractCoordinate(result);
|
|
213
|
+
if (coord === null) {
|
|
214
|
+
log.helper(`[heal desktop_locate_full_page] MISS for '${description.slice(0, 80)}'`);
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
217
|
+
const [rawX, rawY] = coord;
|
|
218
|
+
let pageX = Math.trunc((rawX * pageWidth) / Math.max(imageWidth, 1));
|
|
219
|
+
let pageY = Math.trunc((rawY * pageHeight) / Math.max(imageHeight, 1));
|
|
220
|
+
pageX = Math.max(0, Math.min(pageX, pageWidth - 1));
|
|
221
|
+
pageY = Math.max(0, Math.min(pageY, pageHeight - 1));
|
|
222
|
+
log.helper(`[heal desktop_locate_full_page] HIT image=(${rawX},${rawY}) image_dims=(${imageWidth},${imageHeight}) page_dims=(${pageWidth},${pageHeight}) page_css=(${pageX},${pageY})`);
|
|
223
|
+
return [pageX, pageY];
|
|
224
|
+
}
|
|
225
|
+
catch (e) {
|
|
226
|
+
const name = e?.name ?? '';
|
|
227
|
+
if (name === 'TimeoutError' || name === 'AbortError' || name === 'TypeError') {
|
|
228
|
+
log.helper(`[heal desktop_locate_full_page] transport error for '${description.slice(0, 80)}' (attempt ${attempt + 1}/${MAX_LOCATE_ATTEMPTS}): ${String(e)}`);
|
|
229
|
+
if (attempt < MAX_LOCATE_ATTEMPTS - 1)
|
|
230
|
+
await sleep(1000);
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
log.helper(`[heal desktop_locate_full_page] unexpected error for '${description.slice(0, 80)}': ${String(e)}`);
|
|
234
|
+
return null;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
log.helper(`[heal desktop_locate_full_page] all ${MAX_LOCATE_ATTEMPTS} attempts exhausted for '${description.slice(0, 80)}'`);
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
async function runDesktopLocateFullPageTier(page, description, _verbName) {
|
|
242
|
+
const coords = await getV3DesktopLocateFullPageTarget(page, description);
|
|
243
|
+
if (coords === null)
|
|
244
|
+
return null;
|
|
245
|
+
const [pageX, pageY] = coords;
|
|
246
|
+
const vp = (await page.evaluate(VIEWPORT_FN));
|
|
247
|
+
const vw = Math.trunc(vp[0]);
|
|
248
|
+
const vh = Math.trunc(vp[1]);
|
|
249
|
+
// Scroll to center the target, settle, read back offsets.
|
|
250
|
+
await page.evaluate(([px, py, w, h]) => window.scrollTo(Math.max(0, px - Math.floor(w / 2)), Math.max(0, py - Math.floor(h / 2))), [pageX, pageY, vw, vh]);
|
|
251
|
+
await sleep(150);
|
|
252
|
+
const scroll = (await page.evaluate(() => [window.scrollX, window.scrollY]));
|
|
253
|
+
const vx = Math.max(0, Math.min(pageX - Math.trunc(scroll[0]), vw - 1));
|
|
254
|
+
const vy = Math.max(0, Math.min(pageY - Math.trunc(scroll[1]), vh - 1));
|
|
255
|
+
const resolved = await resolveCoordinate(page, vx, vy);
|
|
256
|
+
log.helper(` [heal] DESKTOP_LOCATE_FULL_PAGE page=(${pageX},${pageY}) viewport=(${vx},${vy}) derived_xpath=${resolved?.xpath ?? 'None'}`);
|
|
257
|
+
return {
|
|
258
|
+
tierUsed: 'DESKTOP_LOCATE_FULL_PAGE',
|
|
259
|
+
selectorXpath: resolved?.xpath ?? null,
|
|
260
|
+
coordinates: [vx, vy],
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
/** VISION_QUERY tier — POST {automind}/v1/heal/vision with the V3 payload;
|
|
264
|
+
* reads `value` or `xpath`. Auth = Basic from configure creds (V3 contract). */
|
|
265
|
+
async function runVisionQueryTier(page, description, verbName) {
|
|
266
|
+
const baseUrl = String(configGet('automindUrl') ?? '') || resolveAutomindUrl();
|
|
267
|
+
if (!baseUrl) {
|
|
268
|
+
log.helper('[v3 vision] no automind_url configured');
|
|
269
|
+
return null;
|
|
270
|
+
}
|
|
271
|
+
try {
|
|
272
|
+
const snapshotB64 = await captureScreenshotB64(page);
|
|
273
|
+
const cfg = (k) => String(configGet(k) ?? '');
|
|
274
|
+
const payload = {
|
|
275
|
+
code_export_id: cfg('codeExportId'),
|
|
276
|
+
current_action: {
|
|
277
|
+
operation_type: verbName,
|
|
278
|
+
operation_intent: description,
|
|
279
|
+
instruction_id: '',
|
|
280
|
+
operation_id: '',
|
|
281
|
+
use_query_v2: true,
|
|
282
|
+
version: 'v3',
|
|
283
|
+
selector: [],
|
|
284
|
+
frame_info: [],
|
|
285
|
+
sub_instruction_obj: { operation_dict: { queried_value: description } },
|
|
286
|
+
},
|
|
287
|
+
commit_id: cfg('commitId'),
|
|
288
|
+
test_id: cfg('testId'),
|
|
289
|
+
username: cfg('username') || (process.env['LT_USERNAME'] ?? ''),
|
|
290
|
+
accesskey: cfg('accesskey') || (process.env['LT_ACCESS_KEY'] ?? ''),
|
|
291
|
+
version: 'v3',
|
|
292
|
+
org_id: cfg('orgId'),
|
|
293
|
+
session_id: cfg('sessionId'),
|
|
294
|
+
untagged_image_base64: snapshotB64,
|
|
295
|
+
};
|
|
296
|
+
const user = payload.username;
|
|
297
|
+
const key = payload.accesskey;
|
|
298
|
+
const resp = await fetch(`${baseUrl}/v1/heal/vision`, {
|
|
299
|
+
method: 'POST',
|
|
300
|
+
headers: {
|
|
301
|
+
'content-type': 'application/json',
|
|
302
|
+
authorization: `Basic ${Buffer.from(`${user}:${key}`).toString('base64')}`,
|
|
303
|
+
},
|
|
304
|
+
body: JSON.stringify(payload),
|
|
305
|
+
signal: AbortSignal.timeout(LOCATE_TIMEOUT_MS),
|
|
306
|
+
});
|
|
307
|
+
if (!resp.ok)
|
|
308
|
+
return null;
|
|
309
|
+
const body = (await resp.json());
|
|
310
|
+
if (body['error'])
|
|
311
|
+
return null;
|
|
312
|
+
const raw = body['value'] ?? body['xpath'];
|
|
313
|
+
if (typeof raw !== 'string' || !raw.trim())
|
|
314
|
+
return null;
|
|
315
|
+
return { tierUsed: 'VISION_QUERY', selectorXpath: raw.trim() };
|
|
316
|
+
}
|
|
317
|
+
catch (e) {
|
|
318
|
+
log.helper(`[v3 vision] '${description.slice(0, 80)}': ${String(e)}`);
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
const TIER_DISPATCHERS = {
|
|
323
|
+
DESKTOP_LOCATE: runDesktopLocateTier,
|
|
324
|
+
DESKTOP_LOCATE_FULL_PAGE: runDesktopLocateFullPageTier,
|
|
325
|
+
VISION_QUERY: runVisionQueryTier,
|
|
326
|
+
};
|
|
327
|
+
/**
|
|
328
|
+
* Run heal tiers in order until one returns an actionable result.
|
|
329
|
+
* Throws AutohealExhausted when every tier misses (`cause` carries the
|
|
330
|
+
* original locator failure so it stays on the trace).
|
|
331
|
+
*/
|
|
332
|
+
export async function healCascade(opts) {
|
|
333
|
+
const { page, description, verbName } = opts;
|
|
334
|
+
const tiers = opts.tiers ?? DEFAULT_HEAL_TIERS;
|
|
335
|
+
// Entering the cascade means the authored locator didn't resolve — flag the
|
|
336
|
+
// step as auto-healing NOW (even if the cascade later exhausts).
|
|
337
|
+
markAutoHeal();
|
|
338
|
+
for (const tier of tiers) {
|
|
339
|
+
const dispatcher = TIER_DISPATCHERS[tier];
|
|
340
|
+
if (!dispatcher) {
|
|
341
|
+
log.helper(`[heal] unknown tier '${tier}' — skipping`);
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
log.helper(`[heal] tier=${tier} for '${description.slice(0, 80)}'`);
|
|
345
|
+
let result = null;
|
|
346
|
+
try {
|
|
347
|
+
result = await dispatcher(page, description, verbName);
|
|
348
|
+
}
|
|
349
|
+
catch (e) {
|
|
350
|
+
log.helper(`[heal] tier=${tier} raised: ${String(e)}`);
|
|
351
|
+
result = null;
|
|
352
|
+
}
|
|
353
|
+
if (result !== null) {
|
|
354
|
+
log.helper(`[heal] tier=${tier} HIT (selector=${JSON.stringify(result.selectorXpath)}, coords=${JSON.stringify(result.coordinates)})`);
|
|
355
|
+
return result;
|
|
356
|
+
}
|
|
357
|
+
log.helper(`[heal] tier=${tier} MISS`);
|
|
358
|
+
}
|
|
359
|
+
const err = new AutohealExhausted(`heal cascade exhausted for '${description}' (${verbName})`);
|
|
360
|
+
err.cause = opts.lastException;
|
|
361
|
+
throw err;
|
|
362
|
+
}
|
|
363
|
+
//# sourceMappingURL=healCascade.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"healCascade.js","sourceRoot":"","sources":["../../src/helpers/healCascade.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChC,OAAO,EAAE,GAAG,IAAI,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAErF,uEAAuE;AACvE,MAAM,iBAAiB,GAAG,mCAAmC,CAAC;AAC9D,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAClC,wEAAwE;AACxE,0EAA0E;AAC1E,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;AAClE,MAAM,kBAAkB,GAAsB,CAAC,gBAAgB,CAAC,CAAC;AACjE,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAapE,SAAS,kBAAkB;IACzB,OAAO,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,iBAAiB,CAAC;AAC1D,CAAC;AAED,yDAAyD;AACzD,SAAS,gBAAgB;IACvB,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;QAClC,iBAAiB,EAAE,eAAe;KACnC,CAAC;IACF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;IAC/C,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACzF,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAC;IAC3D,IAAI,SAAS;QAAE,OAAO,CAAC,cAAc,CAAC,GAAG,SAAS,CAAC;IACnD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS;IAChB,sDAAsD;IACtD,OAAO,CAAC,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxF,CAAC;AAOD;qEACqE;AACrE,KAAK,UAAU,UAAU,CACvB,OAAgC,EAChC,KAAa,EACb,WAAmB,EACnB,OAAe;IAEf,MAAM,QAAQ,GAAG,GAAG,kBAAkB,EAAE,oBAAoB,CAAC;IAC7D,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,gBAAgB,EAAE;QAC3B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,iBAAiB,CAAC;KAC/C,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC5C,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,gBAAgB,IAAI,CAAC,MAAM,SAAS,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC;QACzG,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACxB,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,oBAAoB,IAAI,CAAC,MAAM,SAAS,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,OAAO,GAAG,CAAC,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC9I,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAmB,CAAC;AAC/C,CAAC;AAED;oEACoE;AACpE,SAAS,iBAAiB,CAAC,MAAsB;IAC/C,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC;IAC5B,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAClE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,IAAU,EACV,WAAmB,EACnB,WAAW,GAAG,OAAO,EACrB,OAAgC,EAAE;IAElC,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,GAAG,CAAC,MAAM,CAAC,gEAAgE,CAAC,CAAC;QAC7E,OAAO,IAAI,CAAC;IACd,CAAC;IACD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,mBAAmB,EAAE,OAAO,EAAE,EAAE,CAAC;QAC/D,IAAI,CAAC;YACH,wEAAwE;YACxE,mEAAmE;YACnE,8CAA8C;YAC9C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE9C,IAAI,UAAkB,EAAE,WAAmB,CAAC;YAC5C,IAAI,CAAC;gBACH,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACnD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,8DAA8D;gBAC9D,GAAG,CAAC,MAAM,CAAC,qDAAqD,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC3G,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAqB,CAAC;YACpE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAE3C,IAAI,aAAa,GAAG,CAAC,IAAI,cAAc,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;gBAC/D,MAAM,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAC;gBAC1C,MAAM,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;gBAC/C,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,OAAO,GAAG,sBAAsB,EAAE,CAAC;oBACpE,GAAG,CAAC,MAAM,CAAC,oDAAoD,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC3J,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;gBAC9B,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,QAAQ;gBACf,EAAE,EAAE,SAAS;gBACb,QAAQ,EAAE,KAAK;gBACf,YAAY,EAAE,aAAa;gBAC3B,aAAa,EAAE,cAAc;gBAC7B,UAAU,EAAE,IAAI,CAAC,SAAS,IAAI,KAAK;gBACnC,UAAU,EAAE,SAAS,EAAE;gBACvB,YAAY,EAAE,EAAE;aACjB,EAAE,gBAAgB,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAE3C,IAAI,MAAM,KAAK,OAAO;gBAAE,OAAO,IAAI,CAAC;YACpC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,IAAI,OAAO,GAAG,mBAAmB,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzD,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,GAAG,CAAC,MAAM,CAAC,mCAAmC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC3E,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC;YAE3B,uDAAuD;YACvD,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;YACxE,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,cAAc,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;YAC1E,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC;YACtD,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC;YAEvD,GAAG,CAAC,MAAM,CAAC,oCAAoC,IAAI,IAAI,IAAI,iBAAiB,UAAU,IAAI,WAAW,eAAe,aAAa,IAAI,cAAc,UAAU,IAAI,IAAI,IAAI,gBAAgB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC9N,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACtB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,GAAI,CAAW,EAAE,IAAI,IAAI,EAAE,CAAC;YACtC,oEAAoE;YACpE,4BAA4B;YAC5B,IAAI,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC7E,GAAG,CAAC,MAAM,CAAC,8CAA8C,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,OAAO,GAAG,CAAC,IAAI,mBAAmB,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACpJ,IAAI,OAAO,GAAG,mBAAmB,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,MAAM,CAAC,+CAA+C,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrG,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IACD,GAAG,CAAC,MAAM,CAAC,6BAA6B,mBAAmB,4BAA4B,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IACpH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,IAAU,EAAE,WAAmB,EAAE,QAAgB;IAEjD,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC3E,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACjC,4EAA4E;IAC5E,oEAAoE;IACpE,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,GAAG,CAAC,MAAM,CAAC,2CAA2C,QAAQ,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC,CAAC;IACnF,OAAO;QACL,QAAQ,EAAE,gBAAgB;QAC1B,aAAa,EAAE,QAAQ,EAAE,KAAK,IAAI,IAAI;QACtC,WAAW,EAAE,MAAM;KACpB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,IAAU,EACV,WAAmB;IAEnB,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,GAAG,CAAC,MAAM,CAAC,8DAA8D,CAAC,CAAC;QAC3E,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC9C,IAAI,UAAkB,EAAE,WAAmB,CAAC;IAC5C,IAAI,CAAC;QACH,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,GAAG,CAAC,MAAM,CAAC,+DAA+D,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrH,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAC/B,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,EAAE,QAAQ,CAAC,eAAe,CAAC,YAAY,CAAC,CACpF,CAAqB,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,mBAAmB,EAAE,OAAO,EAAE,EAAE,CAAC;QAC/D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;gBAC9B,MAAM,EAAE,WAAW;gBACnB,KAAK,EAAE,QAAQ;gBACf,EAAE,EAAE,SAAS;gBACb,QAAQ,EAAE,KAAK;gBACf,YAAY,EAAE,CAAC;gBACf,aAAa,EAAE,CAAC;gBAChB,UAAU,EAAE,KAAK;gBACjB,UAAU,EAAE,SAAS,EAAE;gBACvB,YAAY,EAAE,EAAE;aACjB,EAAE,0BAA0B,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YAErD,IAAI,MAAM,KAAK,OAAO;gBAAE,OAAO,IAAI,CAAC;YACpC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,IAAI,OAAO,GAAG,mBAAmB,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,GAAG,CAAC,MAAM,CAAC,6CAA6C,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;gBACrF,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC;YAC3B,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;YACrE,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;YACvE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;YACpD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC;YACrD,GAAG,CAAC,MAAM,CAAC,8CAA8C,IAAI,IAAI,IAAI,iBAAiB,UAAU,IAAI,WAAW,gBAAgB,SAAS,IAAI,UAAU,eAAe,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC;YACxL,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,GAAI,CAAW,EAAE,IAAI,IAAI,EAAE,CAAC;YACtC,IAAI,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBAC7E,GAAG,CAAC,MAAM,CAAC,wDAAwD,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,OAAO,GAAG,CAAC,IAAI,mBAAmB,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9J,IAAI,OAAO,GAAG,mBAAmB,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,MAAM,CAAC,yDAAyD,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC/G,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IACD,GAAG,CAAC,MAAM,CAAC,uCAAuC,mBAAmB,4BAA4B,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAC9H,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,IAAU,EAAE,WAAmB,EAAE,SAAiB;IAElD,MAAM,MAAM,GAAG,MAAM,gCAAgC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACzE,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,CAAC;IAE9B,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAqB,CAAC;IAClE,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7B,0DAA0D;IAC1D,MAAM,IAAI,CAAC,QAAQ,CACjB,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAW,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAC3C,IAAI,CAAC,GAAG,CAAC,CAAC,EAAG,EAAa,GAAG,IAAI,CAAC,KAAK,CAAE,CAAY,GAAG,CAAC,CAAC,CAAC,EAC3D,IAAI,CAAC,GAAG,CAAC,CAAC,EAAG,EAAa,GAAG,IAAI,CAAC,KAAK,CAAE,CAAY,GAAG,CAAC,CAAC,CAAC,CAC5D,EACD,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CACvB,CAAC;IACF,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IACjB,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAqB,CAAC;IACjG,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAExE,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACvD,GAAG,CAAC,MAAM,CAAC,6CAA6C,KAAK,IAAI,KAAK,eAAe,EAAE,IAAI,EAAE,mBAAmB,QAAQ,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC,CAAC;IAC7I,OAAO;QACL,QAAQ,EAAE,0BAA0B;QACpC,aAAa,EAAE,QAAQ,EAAE,KAAK,IAAI,IAAI;QACtC,WAAW,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;KACtB,CAAC;AACJ,CAAC;AAED;gFACgF;AAChF,KAAK,UAAU,kBAAkB,CAC/B,IAAU,EAAE,WAAmB,EAAE,QAAgB;IAEjD,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,kBAAkB,EAAE,CAAC;IAC/E,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,GAAG,CAAC,MAAM,CAAC,wCAAwC,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG;YACd,cAAc,EAAE,GAAG,CAAC,cAAc,CAAC;YACnC,cAAc,EAAE;gBACd,cAAc,EAAE,QAAQ;gBACxB,gBAAgB,EAAE,WAAW;gBAC7B,cAAc,EAAE,EAAE;gBAClB,YAAY,EAAE,EAAE;gBAChB,YAAY,EAAE,IAAI;gBAClB,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,EAAE;gBACZ,UAAU,EAAE,EAAE;gBACd,mBAAmB,EAAE,EAAE,cAAc,EAAE,EAAE,aAAa,EAAE,WAAW,EAAE,EAAE;aACxE;YACD,SAAS,EAAE,GAAG,CAAC,UAAU,CAAC;YAC1B,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC;YACtB,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAC/D,SAAS,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;YACnE,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC;YACpB,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC;YAC5B,qBAAqB,EAAE,WAAW;SACnC,CAAC;QACF,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC;QAC9B,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,iBAAiB,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;aAC3E;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,iBAAiB,CAAC;SAC/C,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAA4B,CAAC;QAC5D,IAAI,IAAI,CAAC,OAAO,CAAC;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QACxD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;IACjE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,GAAG,CAAC,MAAM,CAAC,gBAAgB,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAID,MAAM,gBAAgB,GAAmC;IACvD,cAAc,EAAE,oBAAoB;IACpC,wBAAwB,EAAE,4BAA4B;IACtD,YAAY,EAAE,kBAAkB;CACjC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAOjC;IACC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,kBAAkB,CAAC;IAE/C,4EAA4E;IAC5E,iEAAiE;IACjE,YAAY,EAAE,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,GAAG,CAAC,MAAM,CAAC,wBAAwB,IAAI,cAAc,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,SAAS,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QACpE,IAAI,MAAM,GAAsB,IAAI,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,YAAY,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACvD,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,kBAAkB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACvI,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,iBAAiB,CAC/B,+BAA+B,WAAW,MAAM,QAAQ,GAAG,CAC5D,CAAC;IACD,GAAmC,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;IAChE,MAAM,GAAG,CAAC;AACZ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"png.d.ts","sourceRoot":"","sources":["../../src/helpers/png.ts"],"names":[],"mappings":"AASA,8EAA8E;AAC9E,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAS5D"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PNG IHDR dimension reader — no image library dependency.
|
|
3
|
+
*
|
|
4
|
+
* The DESKTOP_LOCATE heal tier (and scroll_until later) need PNG dims from
|
|
5
|
+
* raw bytes. Reads the 8-byte signature + IHDR chunk only; width/height are
|
|
6
|
+
* big-endian uint32 at offsets 16/20. Mirrors PY _helpers/_png.py.
|
|
7
|
+
*/
|
|
8
|
+
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
9
|
+
/** Return [width, height]; throws on anything that isn't a valid PNG+IHDR. */
|
|
10
|
+
export function pngDimensions(data) {
|
|
11
|
+
if (data.length < 24 ||
|
|
12
|
+
!data.subarray(0, 8).equals(PNG_SIGNATURE) ||
|
|
13
|
+
data.subarray(12, 16).toString('latin1') !== 'IHDR') {
|
|
14
|
+
throw new Error('screenshot is not a valid PNG');
|
|
15
|
+
}
|
|
16
|
+
return [data.readUInt32BE(16), data.readUInt32BE(20)];
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=png.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"png.js","sourceRoot":"","sources":["../../src/helpers/png.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAEpF,8EAA8E;AAC9E,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,IACE,IAAI,CAAC,MAAM,GAAG,EAAE;QAChB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM,EACnD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC;AACxD,CAAC"}
|
|
@@ -15,4 +15,31 @@ import type { Page, PageScreenshotOptions } from 'playwright';
|
|
|
15
15
|
* across HiDPI and SD runners)
|
|
16
16
|
*/
|
|
17
17
|
export declare function takeScreenshot(page: Page, options?: PageScreenshotOptions): Promise<Buffer>;
|
|
18
|
+
/**
|
|
19
|
+
* Capture a viewport screenshot as a base64 PNG via CDP, bypassing
|
|
20
|
+
* `page.screenshot()`.
|
|
21
|
+
*
|
|
22
|
+
* `page.screenshot()` blocks on `document.fonts.ready` (and animation-frame
|
|
23
|
+
* stability) with a 30s default timeout, so one web font that never finishes
|
|
24
|
+
* loading makes it throw — which is what happens behind the LambdaTest grid
|
|
25
|
+
* proxy. `Page.captureScreenshot` is the CDP analogue and returns base64 PNG
|
|
26
|
+
* directly. Chromium-only; the runtime connects over the Chromium CDP path.
|
|
27
|
+
*/
|
|
28
|
+
export declare function captureScreenshotB64(page: Page): Promise<string>;
|
|
29
|
+
/**
|
|
30
|
+
* Capture a FULL-PAGE screenshot as base64 PNG via CDP.
|
|
31
|
+
*
|
|
32
|
+
* Same motivation as captureScreenshotB64. `captureBeyondViewport` with a clip
|
|
33
|
+
* sized to the document's cssContentSize grabs the whole scrollable page in one
|
|
34
|
+
* shot. Coordinate consumers normalise by the returned image's own dimensions,
|
|
35
|
+
* so the absolute capture scale cancels out.
|
|
36
|
+
*/
|
|
37
|
+
export declare function captureFullPageScreenshotB64(page: Page): Promise<string>;
|
|
38
|
+
/**
|
|
39
|
+
* Viewport screenshot as base64 PNG, gated on kaneVersion.
|
|
40
|
+
*
|
|
41
|
+
* V3 grabs via CDP, dodging the fonts.ready timeout behind the grid proxy;
|
|
42
|
+
* otherwise the high-level binding screenshot is unchanged.
|
|
43
|
+
*/
|
|
44
|
+
export declare function captureScreenshotVersionedB64(page: Page): Promise<string>;
|
|
18
45
|
//# sourceMappingURL=screenshot.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/helpers/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/helpers/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAG9D;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,MAAM,CAAC,CAgBjB;AAED;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAUtE;AAED;;;;;;;GAOG;AACH,wBAAsB,4BAA4B,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAiB9E;AAED;;;;;GAKG;AACH,wBAAsB,6BAA6B,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAG/E"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isV3 } from '../configure.js';
|
|
1
2
|
/**
|
|
2
3
|
* Internal helper: wait briefly then take a page screenshot.
|
|
3
4
|
*
|
|
@@ -31,4 +32,59 @@ export async function takeScreenshot(page, options = {}) {
|
|
|
31
32
|
}
|
|
32
33
|
return page.screenshot(merged);
|
|
33
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* Capture a viewport screenshot as a base64 PNG via CDP, bypassing
|
|
37
|
+
* `page.screenshot()`.
|
|
38
|
+
*
|
|
39
|
+
* `page.screenshot()` blocks on `document.fonts.ready` (and animation-frame
|
|
40
|
+
* stability) with a 30s default timeout, so one web font that never finishes
|
|
41
|
+
* loading makes it throw — which is what happens behind the LambdaTest grid
|
|
42
|
+
* proxy. `Page.captureScreenshot` is the CDP analogue and returns base64 PNG
|
|
43
|
+
* directly. Chromium-only; the runtime connects over the Chromium CDP path.
|
|
44
|
+
*/
|
|
45
|
+
export async function captureScreenshotB64(page) {
|
|
46
|
+
const cdp = await page.context().newCDPSession(page);
|
|
47
|
+
try {
|
|
48
|
+
const result = (await cdp.send('Page.captureScreenshot', { format: 'png' }));
|
|
49
|
+
return result.data;
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
await cdp.detach().catch(() => { });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Capture a FULL-PAGE screenshot as base64 PNG via CDP.
|
|
57
|
+
*
|
|
58
|
+
* Same motivation as captureScreenshotB64. `captureBeyondViewport` with a clip
|
|
59
|
+
* sized to the document's cssContentSize grabs the whole scrollable page in one
|
|
60
|
+
* shot. Coordinate consumers normalise by the returned image's own dimensions,
|
|
61
|
+
* so the absolute capture scale cancels out.
|
|
62
|
+
*/
|
|
63
|
+
export async function captureFullPageScreenshotB64(page) {
|
|
64
|
+
const cdp = await page.context().newCDPSession(page);
|
|
65
|
+
try {
|
|
66
|
+
const metrics = (await cdp.send('Page.getLayoutMetrics'));
|
|
67
|
+
const size = metrics.cssContentSize ?? metrics.contentSize ?? undefined;
|
|
68
|
+
const params = { format: 'png', captureBeyondViewport: true };
|
|
69
|
+
if (size?.width && size?.height) {
|
|
70
|
+
params['clip'] = { x: 0, y: 0, width: size.width, height: size.height, scale: 1 };
|
|
71
|
+
}
|
|
72
|
+
const result = (await cdp.send('Page.captureScreenshot', params));
|
|
73
|
+
return result.data;
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
await cdp.detach().catch(() => { });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Viewport screenshot as base64 PNG, gated on kaneVersion.
|
|
81
|
+
*
|
|
82
|
+
* V3 grabs via CDP, dodging the fonts.ready timeout behind the grid proxy;
|
|
83
|
+
* otherwise the high-level binding screenshot is unchanged.
|
|
84
|
+
*/
|
|
85
|
+
export async function captureScreenshotVersionedB64(page) {
|
|
86
|
+
if (isV3())
|
|
87
|
+
return captureScreenshotB64(page);
|
|
88
|
+
return (await takeScreenshot(page)).toString('base64');
|
|
89
|
+
}
|
|
34
90
|
//# sourceMappingURL=screenshot.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/helpers/screenshot.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../src/helpers/screenshot.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAEvC;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAU,EACV,UAAiC,EAAE;IAEnC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC;IACpC,MAAM,MAAM,GAA0B;QACpC,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,KAAK;QACZ,GAAG,OAAO;QACV,IAAI;KACL,CAAC;IACF,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACvC,OAAO,MAAM,CAAC,cAAc,CAAC;IAC/B,CAAC;SAAM,CAAC;QACN,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAU;IACnD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAE1E,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAA8B,CAAC,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,IAAU;IAC3D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAGvD,CAAC;QACF,MAAM,IAAI,GAAG,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,IAAI,SAAS,CAAC;QACxE,MAAM,MAAM,GAA4B,EAAE,MAAM,EAAE,KAAK,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;QACvF,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACpF,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAqB,CAAC;QACtF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,GAAG,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAA8B,CAAC,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,IAAU;IAC5D,IAAI,IAAI,EAAE;QAAE,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC9C,OAAO,CAAC,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACzD,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scrollUntilElement — vision-agent SCROLL_UNTIL_ELEMENT verb (no selector).
|
|
3
|
+
* PY _helpers/scroll_until.py port: full-page screenshot → locate POST →
|
|
4
|
+
* image→page-space scale → scroll target to centre.
|
|
5
|
+
*
|
|
6
|
+
* NOT smart-gated — a locate/AUTOMIND_URL verb (execute_db sibling), not a
|
|
7
|
+
* TESTMU_AI_API_HOST vision verb. Always works or throws; never silently
|
|
8
|
+
* passes. Step-2 DOM-probe → xpath intentionally skipped (a scroll needs no
|
|
9
|
+
* persisted selector).
|
|
10
|
+
*/
|
|
11
|
+
import type { Page } from 'playwright';
|
|
12
|
+
export declare function scrollUntilElement(page: Page, descriptionIn: string): Promise<void>;
|
|
13
|
+
//# sourceMappingURL=scrollUntil.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrollUntil.d.ts","sourceRoot":"","sources":["../../src/helpers/scrollUntil.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AA0BvC,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAsEzF"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { resolveVar } from '../vars.js';
|
|
2
|
+
import { pngDimensions } from './png.js';
|
|
3
|
+
import { captureFullPageScreenshotB64 } from './screenshot.js';
|
|
4
|
+
import { log } from '../log.js';
|
|
5
|
+
const AUTOMIND_FALLBACK = 'https://kaneai-api.lambdatest.com';
|
|
6
|
+
const LOCATE_TIMEOUT_MS = 120_000;
|
|
7
|
+
// Page + viewport dims in one round-trip. Real function, not a string —
|
|
8
|
+
// the grid CDP relay returns undefined for string page-functions.
|
|
9
|
+
const DIMS_FN = () => [
|
|
10
|
+
document.documentElement.scrollWidth, document.documentElement.scrollHeight,
|
|
11
|
+
window.innerWidth, window.innerHeight,
|
|
12
|
+
];
|
|
13
|
+
function basicAuthHeaders() {
|
|
14
|
+
const headers = { 'content-type': 'application/json' };
|
|
15
|
+
const user = process.env['LT_USERNAME'] ?? '';
|
|
16
|
+
const key = process.env['LT_ACCESS_KEY'] ?? '';
|
|
17
|
+
if (user && key) {
|
|
18
|
+
headers['authorization'] = `Basic ${Buffer.from(`${user}:${key}`).toString('base64')}`;
|
|
19
|
+
}
|
|
20
|
+
return headers;
|
|
21
|
+
}
|
|
22
|
+
export async function scrollUntilElement(page, descriptionIn) {
|
|
23
|
+
const resolved = await resolveVar(descriptionIn);
|
|
24
|
+
const description = resolved === null || resolved === undefined ? '' : String(resolved);
|
|
25
|
+
log.helper(`[scroll_until_element] target=${description.slice(0, 80)}`);
|
|
26
|
+
// CDP full-page capture — page.screenshot(fullPage) hangs on fonts.ready
|
|
27
|
+
// behind the grid proxy.
|
|
28
|
+
const imageB64 = await captureFullPageScreenshotB64(page);
|
|
29
|
+
const screenshotBytes = Buffer.from(imageB64, 'base64');
|
|
30
|
+
const dims = (await page.evaluate(DIMS_FN));
|
|
31
|
+
const pageWidth = Math.trunc(dims[0] ?? 0);
|
|
32
|
+
const pageHeight = Math.trunc(dims[1] ?? 0);
|
|
33
|
+
const viewportWidth = Math.trunc(dims[2] ?? 0);
|
|
34
|
+
const viewportHeight = Math.trunc(dims[3] ?? 0);
|
|
35
|
+
const endpoint = `${process.env['AUTOMIND_URL'] || AUTOMIND_FALLBACK}/v2/locate/desktop`;
|
|
36
|
+
let result;
|
|
37
|
+
try {
|
|
38
|
+
const resp = await fetch(endpoint, {
|
|
39
|
+
method: 'POST',
|
|
40
|
+
headers: basicAuthHeaders(),
|
|
41
|
+
body: JSON.stringify({
|
|
42
|
+
intent: description,
|
|
43
|
+
image: imageB64,
|
|
44
|
+
os: 'desktop',
|
|
45
|
+
platform: 'web',
|
|
46
|
+
screen_height: 0,
|
|
47
|
+
screen_width: 0,
|
|
48
|
+
drop_aware: false,
|
|
49
|
+
}),
|
|
50
|
+
signal: AbortSignal.timeout(LOCATE_TIMEOUT_MS),
|
|
51
|
+
});
|
|
52
|
+
if (resp.status !== 200) {
|
|
53
|
+
const text = await resp.text().catch(() => '');
|
|
54
|
+
throw new Error(`locate API error ${resp.status} for '${description}': ${text.slice(0, 500)}`);
|
|
55
|
+
}
|
|
56
|
+
result = (await resp.json());
|
|
57
|
+
}
|
|
58
|
+
catch (e) {
|
|
59
|
+
if (e?.message?.includes('locate API error'))
|
|
60
|
+
throw e;
|
|
61
|
+
throw new Error(`scroll_until_element locate failed for '${description}': ${String(e)}`);
|
|
62
|
+
}
|
|
63
|
+
// Missing key and explicit [0,0] both mean "not found".
|
|
64
|
+
const coordinate = result.coordinate ?? [0, 0];
|
|
65
|
+
if (!coordinate.length || (coordinate[0] === 0 && coordinate[1] === 0)) {
|
|
66
|
+
throw new Error(`scroll_until_element: element not found for '${description}' ` +
|
|
67
|
+
`(coordinate=${JSON.stringify(coordinate)}, reason=${result.reason ?? ''})`);
|
|
68
|
+
}
|
|
69
|
+
const rawX = Math.trunc(Number(coordinate[0]));
|
|
70
|
+
const rawY = Math.trunc(Number(coordinate[1]));
|
|
71
|
+
let imageWidth, imageHeight;
|
|
72
|
+
try {
|
|
73
|
+
[imageWidth, imageHeight] = pngDimensions(screenshotBytes);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
throw new Error('scroll_until_element: screenshot is not a valid PNG');
|
|
77
|
+
}
|
|
78
|
+
const scaleX = pageWidth / Math.max(imageWidth, 1);
|
|
79
|
+
const scaleY = pageHeight / Math.max(imageHeight, 1);
|
|
80
|
+
const pageX = Math.trunc(rawX * scaleX);
|
|
81
|
+
const pageY = Math.trunc(rawY * scaleY);
|
|
82
|
+
log.helper(`[scroll_until_element] image=(${rawX},${rawY}) page=(${pageX},${pageY}) scale=(${scaleX.toFixed(3)},${scaleY.toFixed(3)})`);
|
|
83
|
+
const targetX = Math.max(0, pageX - Math.floor(viewportWidth / 2));
|
|
84
|
+
const targetY = Math.max(0, pageY - Math.floor(viewportHeight / 2));
|
|
85
|
+
await page.evaluate(([x, y]) => window.scrollTo(x, y), [targetX, targetY]);
|
|
86
|
+
log.helper(`[scroll_until_element] scrolled to (${targetX},${targetY})`);
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=scrollUntil.js.map
|