@keboola/validate-ui 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +116 -0
- package/dist/cli.cjs +118 -1152
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.cts +1 -1
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +118 -1143
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +40 -1101
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +177 -159
- package/dist/index.d.ts +177 -159
- package/dist/index.js +10 -1064
- package/dist/index.js.map +1 -1
- package/dist/runner-CM9sF-C8.cjs +1518 -0
- package/dist/runner-CM9sF-C8.cjs.map +1 -0
- package/dist/runner-ZOonToZg.js +1308 -0
- package/dist/runner-ZOonToZg.js.map +1 -0
- package/package.json +8 -8
|
@@ -0,0 +1,1518 @@
|
|
|
1
|
+
//#region \0rolldown/runtime.js
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __copyProps = (to, from, except, desc) => {
|
|
9
|
+
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
|
|
10
|
+
key = keys[i];
|
|
11
|
+
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
|
|
12
|
+
get: ((k) => from[k]).bind(null, key),
|
|
13
|
+
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule || !__hasOwnProp.call(mod, "default") ? __defProp(target, "default", {
|
|
19
|
+
value: mod,
|
|
20
|
+
enumerable: true
|
|
21
|
+
}) : target, mod));
|
|
22
|
+
//#endregion
|
|
23
|
+
let playwright = require("playwright");
|
|
24
|
+
let node_fs = require("node:fs");
|
|
25
|
+
let node_fs_promises = require("node:fs/promises");
|
|
26
|
+
let node_http = require("node:http");
|
|
27
|
+
let node_path = require("node:path");
|
|
28
|
+
let axe_core = require("axe-core");
|
|
29
|
+
axe_core = __toESM(axe_core, 1);
|
|
30
|
+
let zod = require("zod");
|
|
31
|
+
let _anthropic_ai_sdk = require("@anthropic-ai/sdk");
|
|
32
|
+
_anthropic_ai_sdk = __toESM(_anthropic_ai_sdk, 1);
|
|
33
|
+
let pixelmatch = require("pixelmatch");
|
|
34
|
+
pixelmatch = __toESM(pixelmatch, 1);
|
|
35
|
+
let pngjs = require("pngjs");
|
|
36
|
+
let _keboola_brand_registry = require("@keboola/brand-registry");
|
|
37
|
+
//#region src/prepare-screenshot.ts
|
|
38
|
+
const disableAnimations = async (page) => {
|
|
39
|
+
await page.addStyleTag({ content: `*, *::before, *::after {
|
|
40
|
+
animation-duration: 0s !important;
|
|
41
|
+
animation-delay: 0s !important;
|
|
42
|
+
transition-duration: 0s !important;
|
|
43
|
+
transition-delay: 0s !important;
|
|
44
|
+
caret-color: transparent !important;
|
|
45
|
+
}` });
|
|
46
|
+
};
|
|
47
|
+
const parkMouseOffElements = async (page) => {
|
|
48
|
+
await page.mouse.move(0, 0);
|
|
49
|
+
};
|
|
50
|
+
const waitForFonts = async (page) => {
|
|
51
|
+
await page.evaluate(() => document.fonts.ready.then(() => void 0));
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Settle the page before a screenshot: freeze animations first (so parking the
|
|
55
|
+
* mouse can't trigger a transition), park the cursor off any element, and wait
|
|
56
|
+
* for web fonts. Deterministic output makes visual diffs meaningful.
|
|
57
|
+
*/
|
|
58
|
+
const prepareForScreenshot = async (page) => {
|
|
59
|
+
await disableAnimations(page);
|
|
60
|
+
await parkMouseOffElements(page);
|
|
61
|
+
await waitForFonts(page);
|
|
62
|
+
};
|
|
63
|
+
//#endregion
|
|
64
|
+
//#region src/capture.ts
|
|
65
|
+
const DESKTOP_VIEWPORT = {
|
|
66
|
+
label: "desktop",
|
|
67
|
+
width: 1280,
|
|
68
|
+
height: 800
|
|
69
|
+
};
|
|
70
|
+
const MOBILE_VIEWPORT = {
|
|
71
|
+
label: "mobile",
|
|
72
|
+
width: 390,
|
|
73
|
+
height: 844
|
|
74
|
+
};
|
|
75
|
+
const DEFAULT_VIEWPORTS = [DESKTOP_VIEWPORT, MOBILE_VIEWPORT];
|
|
76
|
+
const DEFAULT_TIMEOUT_MS$1 = 3e4;
|
|
77
|
+
const attachRecorders = (page) => {
|
|
78
|
+
const consoleEntries = [];
|
|
79
|
+
const networkEntries = [];
|
|
80
|
+
page.on("console", (message) => {
|
|
81
|
+
consoleEntries.push({
|
|
82
|
+
type: message.type(),
|
|
83
|
+
text: message.text(),
|
|
84
|
+
location: message.location().url || void 0
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
page.on("pageerror", (error) => {
|
|
88
|
+
consoleEntries.push({
|
|
89
|
+
type: "pageerror",
|
|
90
|
+
text: error.message
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
page.on("requestfailed", (request) => {
|
|
94
|
+
networkEntries.push({
|
|
95
|
+
url: request.url(),
|
|
96
|
+
method: request.method(),
|
|
97
|
+
status: 0,
|
|
98
|
+
failure: request.failure()?.errorText
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
page.on("response", (response) => {
|
|
102
|
+
networkEntries.push({
|
|
103
|
+
url: response.url(),
|
|
104
|
+
method: response.request().method(),
|
|
105
|
+
status: response.status()
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
return {
|
|
109
|
+
console: consoleEntries,
|
|
110
|
+
network: networkEntries
|
|
111
|
+
};
|
|
112
|
+
};
|
|
113
|
+
/**
|
|
114
|
+
* Render `url` in an already-open page at `viewport` and capture ground truth.
|
|
115
|
+
* Recorders are attached before navigation so nothing is missed. The caller
|
|
116
|
+
* owns the page lifecycle — this is the building block {@link capture} and the
|
|
117
|
+
* orchestrator reuse.
|
|
118
|
+
*/
|
|
119
|
+
const capturePage = async (page, url, viewport, timeoutMs = DEFAULT_TIMEOUT_MS$1) => {
|
|
120
|
+
const recorders = attachRecorders(page);
|
|
121
|
+
await page.setViewportSize({
|
|
122
|
+
width: viewport.width,
|
|
123
|
+
height: viewport.height
|
|
124
|
+
});
|
|
125
|
+
await page.goto(url, {
|
|
126
|
+
waitUntil: "networkidle",
|
|
127
|
+
timeout: timeoutMs
|
|
128
|
+
});
|
|
129
|
+
await prepareForScreenshot(page);
|
|
130
|
+
return {
|
|
131
|
+
url,
|
|
132
|
+
viewport,
|
|
133
|
+
screenshot: await page.screenshot({ fullPage: true }),
|
|
134
|
+
dom: await page.content(),
|
|
135
|
+
renderedText: (await page.evaluate(() => document.body.innerText)).trim(),
|
|
136
|
+
console: recorders.console,
|
|
137
|
+
network: recorders.network
|
|
138
|
+
};
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
141
|
+
* Launch a headless browser (unless one is supplied), render `url` at each
|
|
142
|
+
* viewport in a fresh context, and return one artifact per viewport.
|
|
143
|
+
*/
|
|
144
|
+
const capture = async (url, options = {}) => {
|
|
145
|
+
const viewports = options.viewports ?? DEFAULT_VIEWPORTS;
|
|
146
|
+
const browser = options.browser ?? await playwright.chromium.launch();
|
|
147
|
+
const ownsBrowser = !options.browser;
|
|
148
|
+
try {
|
|
149
|
+
const artifacts = [];
|
|
150
|
+
for (const viewport of viewports) {
|
|
151
|
+
const context = await browser.newContext({ viewport: {
|
|
152
|
+
width: viewport.width,
|
|
153
|
+
height: viewport.height
|
|
154
|
+
} });
|
|
155
|
+
const page = await context.newPage();
|
|
156
|
+
try {
|
|
157
|
+
artifacts.push(await capturePage(page, url, viewport, options.timeoutMs));
|
|
158
|
+
} finally {
|
|
159
|
+
await context.close();
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return artifacts;
|
|
163
|
+
} finally {
|
|
164
|
+
if (ownsBrowser) await browser.close();
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
//#endregion
|
|
168
|
+
//#region src/serve.ts
|
|
169
|
+
const CONTENT_TYPES = {
|
|
170
|
+
".html": "text/html; charset=utf-8",
|
|
171
|
+
".js": "text/javascript; charset=utf-8",
|
|
172
|
+
".css": "text/css; charset=utf-8",
|
|
173
|
+
".json": "application/json; charset=utf-8",
|
|
174
|
+
".svg": "image/svg+xml",
|
|
175
|
+
".png": "image/png",
|
|
176
|
+
".jpg": "image/jpeg",
|
|
177
|
+
".woff2": "font/woff2",
|
|
178
|
+
".woff": "font/woff"
|
|
179
|
+
};
|
|
180
|
+
/**
|
|
181
|
+
* Serve a built SPA directory (e.g. `apps/boilerplate/dist`) on an ephemeral
|
|
182
|
+
* port with client-side-routing fallback to `index.html`, so a validator can
|
|
183
|
+
* render any client route without a dev server. Node stdlib only — no runtime
|
|
184
|
+
* dependency added.
|
|
185
|
+
*/
|
|
186
|
+
const serveStatic = async (rootDir) => {
|
|
187
|
+
const indexPath = (0, node_path.join)(rootDir, "index.html");
|
|
188
|
+
const rootResolved = (0, node_path.resolve)(rootDir);
|
|
189
|
+
const server = (0, node_http.createServer)((request, response) => {
|
|
190
|
+
const send = (filePath, status = 200) => {
|
|
191
|
+
response.writeHead(status, { "content-type": CONTENT_TYPES[(0, node_path.extname)(filePath)] ?? "application/octet-stream" });
|
|
192
|
+
const stream = (0, node_fs.createReadStream)(filePath);
|
|
193
|
+
stream.on("error", (error) => {
|
|
194
|
+
console.error(`serveStatic: failed to read ${filePath}:`, error);
|
|
195
|
+
if (!response.headersSent) response.writeHead(500);
|
|
196
|
+
response.end();
|
|
197
|
+
});
|
|
198
|
+
stream.pipe(response);
|
|
199
|
+
};
|
|
200
|
+
const notFound = () => {
|
|
201
|
+
response.writeHead(404);
|
|
202
|
+
response.end();
|
|
203
|
+
};
|
|
204
|
+
let rawPath;
|
|
205
|
+
try {
|
|
206
|
+
rawPath = decodeURIComponent((request.url ?? "/").split("?")[0] ?? "/");
|
|
207
|
+
} catch {
|
|
208
|
+
response.writeHead(400);
|
|
209
|
+
response.end();
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
const relative = (0, node_path.normalize)(rawPath).replace(/^(\.\.[/\\])+/, "");
|
|
213
|
+
const candidate = (0, node_path.join)(rootDir, relative);
|
|
214
|
+
const candidateResolved = (0, node_path.resolve)(candidate);
|
|
215
|
+
if (candidateResolved !== rootResolved && !candidateResolved.startsWith(rootResolved + node_path.sep)) {
|
|
216
|
+
notFound();
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
(0, node_fs_promises.stat)(candidate).then((info) => {
|
|
220
|
+
send(info.isDirectory() ? indexPath : candidate);
|
|
221
|
+
}).catch(() => {
|
|
222
|
+
if ((0, node_path.extname)(candidate) === "") {
|
|
223
|
+
send(indexPath);
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
notFound();
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
230
|
+
const address = server.address();
|
|
231
|
+
if (address === null || typeof address === "string") throw new Error("serveStatic: failed to bind an ephemeral port");
|
|
232
|
+
return {
|
|
233
|
+
url: `http://127.0.0.1:${address.port}`,
|
|
234
|
+
close: () => new Promise((resolve, reject) => {
|
|
235
|
+
server.close((error) => error ? reject(error) : resolve());
|
|
236
|
+
})
|
|
237
|
+
};
|
|
238
|
+
};
|
|
239
|
+
//#endregion
|
|
240
|
+
//#region src/axes/clipped-text.ts
|
|
241
|
+
/**
|
|
242
|
+
* Elements whose content is cut off with no way for the user to reveal it.
|
|
243
|
+
*
|
|
244
|
+
* This exists because a page can be free of console errors, axe violations and layout
|
|
245
|
+
* overflow while still showing the user the wrong thing. The case that prompted it: an
|
|
246
|
+
* editable financial grid whose cells were narrower than their own padding, so a value of
|
|
247
|
+
* `800` rendered as `8`. Every other axis scored it 100.
|
|
248
|
+
*
|
|
249
|
+
* Deliberate truncation is not a finding. Text clipped with `text-overflow: ellipsis`
|
|
250
|
+
* announces itself, and a scrollable box (`overflow: auto | scroll`) can be scrolled to
|
|
251
|
+
* the rest. Only silent loss is reported — `overflow: hidden | clip` with no ellipsis, and
|
|
252
|
+
* `<input>`s whose value is wider than the box that holds it (a single-line input is never
|
|
253
|
+
* scrollable by eye; the user has to select into it to discover the rest). A `<textarea>`
|
|
254
|
+
* is judged like any other box: it renders a real scrollbar when its content overflows
|
|
255
|
+
* horizontally, so it is only a finding once something hides the overflow.
|
|
256
|
+
*/
|
|
257
|
+
/** Ignore sub-pixel and rounding noise — real clipping is wider than a stray pixel. */
|
|
258
|
+
const CLIP_TOLERANCE_PX = 2;
|
|
259
|
+
/** Cap the reported list so one systematically broken component can't flood the report. */
|
|
260
|
+
const MAX_REPORTED = 10;
|
|
261
|
+
/**
|
|
262
|
+
* Runs in the page. Kept self-contained (no imports, no outer-scope references) because it
|
|
263
|
+
* is serialised into the browser context.
|
|
264
|
+
*/
|
|
265
|
+
/* c8 ignore start -- executes in the browser, not under node coverage */
|
|
266
|
+
const findClippedElements = ({ tolerance, max }) => {
|
|
267
|
+
const describe = (el) => {
|
|
268
|
+
const tag = el.tagName.toLowerCase();
|
|
269
|
+
const label = el.getAttribute("aria-label");
|
|
270
|
+
if (label) return `${tag}[aria-label="${label}"]`;
|
|
271
|
+
if (el.id) return `${tag}#${el.id}`;
|
|
272
|
+
const name = el.getAttribute("name");
|
|
273
|
+
if (name) return `${tag}[name="${name}"]`;
|
|
274
|
+
return `${tag}${typeof el.className === "string" && el.className.trim() ? `.${el.className.trim().split(/\s+/).slice(0, 2).join(".")}` : ""}`;
|
|
275
|
+
};
|
|
276
|
+
const results = [];
|
|
277
|
+
const reported = /* @__PURE__ */ new Set();
|
|
278
|
+
const hasReportedAncestor = (el) => {
|
|
279
|
+
let parent = el.parentElement;
|
|
280
|
+
while (parent) {
|
|
281
|
+
if (reported.has(parent)) return true;
|
|
282
|
+
parent = parent.parentElement;
|
|
283
|
+
}
|
|
284
|
+
return false;
|
|
285
|
+
};
|
|
286
|
+
for (const el of document.querySelectorAll("*")) {
|
|
287
|
+
if (results.length >= max) break;
|
|
288
|
+
if (hasReportedAncestor(el)) continue;
|
|
289
|
+
const style = window.getComputedStyle(el);
|
|
290
|
+
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue;
|
|
291
|
+
if (el.clientWidth <= 1 || el.clientHeight <= 1) continue;
|
|
292
|
+
const overflowX = style.overflowX;
|
|
293
|
+
const isFormValue = el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement;
|
|
294
|
+
const isSingleLineInput = el instanceof HTMLInputElement;
|
|
295
|
+
if (!isSingleLineInput && overflowX !== "hidden" && overflowX !== "clip") continue;
|
|
296
|
+
if (!isSingleLineInput && style.textOverflow === "ellipsis") continue;
|
|
297
|
+
if (el.scrollWidth - el.clientWidth <= tolerance) continue;
|
|
298
|
+
const text = isFormValue ? el.value : (el.textContent ?? "").trim();
|
|
299
|
+
if (!text) continue;
|
|
300
|
+
reported.add(el);
|
|
301
|
+
results.push({
|
|
302
|
+
selector: describe(el),
|
|
303
|
+
text: text.length > 40 ? `${text.slice(0, 40)}…` : text,
|
|
304
|
+
visibleWidth: Math.round(el.clientWidth),
|
|
305
|
+
contentWidth: Math.round(el.scrollWidth),
|
|
306
|
+
isInput: isFormValue
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
return results;
|
|
310
|
+
};
|
|
311
|
+
/* c8 ignore stop */
|
|
312
|
+
/** Format one clipped element as a finding. */
|
|
313
|
+
const toFinding$1 = (clipped) => ({
|
|
314
|
+
message: `Content is cut off with no way to reveal it (${clipped.selector})`,
|
|
315
|
+
severity: "serious",
|
|
316
|
+
detail: `${clipped.isInput ? "Input value" : "Text"} "${clipped.text}" needs ${clipped.contentWidth}px but has ${clipped.visibleWidth}px. Widen the element, or make the truncation explicit with \`text-overflow: ellipsis\`.`
|
|
317
|
+
});
|
|
318
|
+
/**
|
|
319
|
+
* Report every element whose content is silently cut off. Returns no findings when the
|
|
320
|
+
* page cannot be inspected, so a caller can always spread the result.
|
|
321
|
+
*/
|
|
322
|
+
const clippedTextFindings = async (page) => {
|
|
323
|
+
try {
|
|
324
|
+
return (await page.evaluate(findClippedElements, {
|
|
325
|
+
tolerance: CLIP_TOLERANCE_PX,
|
|
326
|
+
max: MAX_REPORTED
|
|
327
|
+
})).map(toFinding$1);
|
|
328
|
+
} catch (error) {
|
|
329
|
+
console.error("clipped-text: could not inspect the page", error);
|
|
330
|
+
return [];
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
//#endregion
|
|
334
|
+
//#region src/axes/accessibility.ts
|
|
335
|
+
const IMPACTS = [
|
|
336
|
+
"critical",
|
|
337
|
+
"serious",
|
|
338
|
+
"moderate",
|
|
339
|
+
"minor"
|
|
340
|
+
];
|
|
341
|
+
const toSeverity = (impact) => impact && IMPACTS.includes(impact) ? impact : "minor";
|
|
342
|
+
/** First `wcag*` tag if present, else the first tag, else undefined. */
|
|
343
|
+
const wcagTag = (tags) => tags.find((tag) => tag.startsWith("wcag")) ?? tags[0];
|
|
344
|
+
const toFinding = (violation) => {
|
|
345
|
+
const targets = violation.nodes.map((node) => node.target.join(" ")).filter((target) => target.length > 0).join(", ");
|
|
346
|
+
const wcag = wcagTag(violation.tags);
|
|
347
|
+
return {
|
|
348
|
+
message: `${violation.help || violation.description}${targets ? ` (${targets})` : ""}`,
|
|
349
|
+
severity: toSeverity(violation.impact),
|
|
350
|
+
ref: [wcag, violation.id].filter(Boolean).join(" · ")
|
|
351
|
+
};
|
|
352
|
+
};
|
|
353
|
+
/**
|
|
354
|
+
* A3 · Accessibility verdict — UT-4494.
|
|
355
|
+
*
|
|
356
|
+
* Injects axe-core into `context.page` and maps each violation to a Finding
|
|
357
|
+
* (WCAG tag + rule id as `ref`, axe impact as severity). Pass rule: a page
|
|
358
|
+
* passes when no violation has impact `critical` or `serious` — moderate/minor
|
|
359
|
+
* violations are reported but don't fail the axis. Falls back to a clear
|
|
360
|
+
* finding rather than throwing when no live page is provided.
|
|
361
|
+
*/
|
|
362
|
+
/** Not-assessed outcome — a11y can't fail if it never ran. */
|
|
363
|
+
const notAssessed = (message) => ({
|
|
364
|
+
axis: "a11y",
|
|
365
|
+
pass: true,
|
|
366
|
+
findings: [{
|
|
367
|
+
message,
|
|
368
|
+
severity: "minor"
|
|
369
|
+
}]
|
|
370
|
+
});
|
|
371
|
+
const accessibilityAxis = {
|
|
372
|
+
name: "a11y",
|
|
373
|
+
run: async (context) => {
|
|
374
|
+
const { page } = context;
|
|
375
|
+
if (!page) return notAssessed("a11y skipped: no live page provided");
|
|
376
|
+
await page.addScriptTag({ content: axe_core.default.source });
|
|
377
|
+
const results = await page.evaluate(() => {
|
|
378
|
+
const injected = window.axe;
|
|
379
|
+
return injected ? injected.run(document) : null;
|
|
380
|
+
});
|
|
381
|
+
if (results === null) return notAssessed("a11y not assessed: axe-core failed to inject (page CSP?)");
|
|
382
|
+
const findings = [...results.violations.map(toFinding), ...await clippedTextFindings(page)];
|
|
383
|
+
return {
|
|
384
|
+
axis: "a11y",
|
|
385
|
+
pass: !findings.some((finding) => finding.severity === "critical" || finding.severity === "serious"),
|
|
386
|
+
findings
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
};
|
|
390
|
+
//#endregion
|
|
391
|
+
//#region src/axes/vlm-provider.ts
|
|
392
|
+
/** Concatenate a message's text blocks. */
|
|
393
|
+
const textFromMessage = (message) => message.content.filter((block) => block.type === "text").map((block) => block.text).join("");
|
|
394
|
+
const makeProvider = (backend, clientOptions) => {
|
|
395
|
+
const client = new _anthropic_ai_sdk.default(clientOptions);
|
|
396
|
+
return {
|
|
397
|
+
backend,
|
|
398
|
+
judge: async ({ screenshotBase64, prompt, model }) => {
|
|
399
|
+
const message = await client.messages.create({
|
|
400
|
+
model,
|
|
401
|
+
max_tokens: 1024,
|
|
402
|
+
messages: [{
|
|
403
|
+
role: "user",
|
|
404
|
+
content: [{
|
|
405
|
+
type: "image",
|
|
406
|
+
source: {
|
|
407
|
+
type: "base64",
|
|
408
|
+
media_type: "image/png",
|
|
409
|
+
data: screenshotBase64
|
|
410
|
+
}
|
|
411
|
+
}, {
|
|
412
|
+
type: "text",
|
|
413
|
+
text: prompt
|
|
414
|
+
}]
|
|
415
|
+
}]
|
|
416
|
+
});
|
|
417
|
+
return textFromMessage(message);
|
|
418
|
+
}
|
|
419
|
+
};
|
|
420
|
+
};
|
|
421
|
+
const nonEmpty = (value) => value !== void 0 && value !== "";
|
|
422
|
+
/**
|
|
423
|
+
* Pick a VLM backend from the environment, or `undefined` when none is
|
|
424
|
+
* configured (the axis then skips gracefully).
|
|
425
|
+
*
|
|
426
|
+
* Preference order mirrors how kai-agent routes Claude for internal/team use
|
|
427
|
+
* (`apps/kai-agent/src/services/entrypoint-builder.ts`): the Keboola LLM path
|
|
428
|
+
* points the Anthropic SDK at a Keboola-hosted proxy via `baseURL` and
|
|
429
|
+
* authenticates with a Keboola-minted token sent as `x-api-key` (the SDK's
|
|
430
|
+
* `apiKey`). kai-agent injects that pair through the SDK-native
|
|
431
|
+
* `ANTHROPIC_BASE_URL` + `ANTHROPIC_API_KEY` env vars; we accept those, and
|
|
432
|
+
* prefer the explicit `VALIDATE_UI_LLM_BASE_URL` / `VALIDATE_UI_LLM_TOKEN`
|
|
433
|
+
* (with `KBC_TOKEN` as a Keboola-convention alias) when set. The proxy path is
|
|
434
|
+
* preferred because the team plan has no dedicated raw key.
|
|
435
|
+
*
|
|
436
|
+
* When no proxy `baseURL` is configured, a bare `ANTHROPIC_API_KEY` selects the
|
|
437
|
+
* raw Anthropic backend — the original behavior and the fallback.
|
|
438
|
+
*
|
|
439
|
+
* Both backends pass the resolved key (and, for the proxy, `baseURL`) into the
|
|
440
|
+
* `Anthropic` constructor explicitly, so resolution never reads ambient
|
|
441
|
+
* `process.env` behind the injected `env` arg.
|
|
442
|
+
*/
|
|
443
|
+
const resolveVlmProvider = (env = process.env) => {
|
|
444
|
+
const baseUrl = env.VALIDATE_UI_LLM_BASE_URL ?? env.ANTHROPIC_BASE_URL;
|
|
445
|
+
const proxyToken = env.VALIDATE_UI_LLM_TOKEN ?? env.KBC_TOKEN ?? env.ANTHROPIC_API_KEY;
|
|
446
|
+
if (nonEmpty(baseUrl) && nonEmpty(proxyToken)) return makeProvider("keboola-llm", {
|
|
447
|
+
baseURL: baseUrl,
|
|
448
|
+
apiKey: proxyToken
|
|
449
|
+
});
|
|
450
|
+
if (nonEmpty(env.ANTHROPIC_API_KEY)) return makeProvider("raw-anthropic", { apiKey: env.ANTHROPIC_API_KEY });
|
|
451
|
+
};
|
|
452
|
+
//#endregion
|
|
453
|
+
//#region src/axes/brief-conformance.ts
|
|
454
|
+
/** Structured judgement the VLM must return, validated before interpretation. */
|
|
455
|
+
const JudgeResult = zod.z.object({
|
|
456
|
+
matchesBrief: zod.z.boolean(),
|
|
457
|
+
missingStates: zod.z.array(zod.z.string()),
|
|
458
|
+
reasons: zod.z.array(zod.z.string())
|
|
459
|
+
});
|
|
460
|
+
const MODEL = process.env.VALIDATE_UI_MODEL ?? "claude-sonnet-4-6";
|
|
461
|
+
const DOM_EXCERPT_LIMIT = 12e3;
|
|
462
|
+
/**
|
|
463
|
+
* Pure interpreter: turn a validated {@link JudgeResult} into a {@link Verdict}.
|
|
464
|
+
* A brief mismatch is serious; each missing state is a moderate gap.
|
|
465
|
+
*/
|
|
466
|
+
const interpretJudgeResult = (result) => {
|
|
467
|
+
const findings = [];
|
|
468
|
+
if (!result.matchesBrief) findings.push({
|
|
469
|
+
message: `UI does not match the brief: ${result.reasons.join("; ")}`,
|
|
470
|
+
severity: "serious"
|
|
471
|
+
});
|
|
472
|
+
for (const state of result.missingStates) findings.push({
|
|
473
|
+
message: `missing state: ${state}`,
|
|
474
|
+
severity: "moderate"
|
|
475
|
+
});
|
|
476
|
+
return {
|
|
477
|
+
axis: "brief-conformance",
|
|
478
|
+
pass: result.matchesBrief && result.missingStates.length === 0,
|
|
479
|
+
findings
|
|
480
|
+
};
|
|
481
|
+
};
|
|
482
|
+
const skip$1 = (message) => ({
|
|
483
|
+
axis: "brief-conformance",
|
|
484
|
+
pass: true,
|
|
485
|
+
findings: [{
|
|
486
|
+
message,
|
|
487
|
+
severity: "minor"
|
|
488
|
+
}]
|
|
489
|
+
});
|
|
490
|
+
const RUBRIC = [
|
|
491
|
+
"You are judging whether a rendered UI conforms to the brief it was generated from.",
|
|
492
|
+
"Assess two things:",
|
|
493
|
+
"1. matchesBrief — does the screenshot + DOM implement what the brief describes?",
|
|
494
|
+
"2. missingStates — which UI states the brief implies are NOT represented.",
|
|
495
|
+
" Use short lowercase labels like \"empty\", \"loading\", \"error\".",
|
|
496
|
+
"Reply with ONLY a JSON object, no prose, matching exactly:",
|
|
497
|
+
"{ \"matchesBrief\": boolean, \"missingStates\": string[], \"reasons\": string[] }",
|
|
498
|
+
"reasons: brief explanations for any mismatch (empty array if it matches)."
|
|
499
|
+
].join("\n");
|
|
500
|
+
const briefConformanceAxis = {
|
|
501
|
+
name: "brief-conformance",
|
|
502
|
+
run: async (context) => {
|
|
503
|
+
const brief = context.brief?.trim();
|
|
504
|
+
if (brief === void 0 || brief === "") return skip$1("no brief provided; conformance not assessed");
|
|
505
|
+
const provider = resolveVlmProvider();
|
|
506
|
+
if (provider === void 0) return skip$1("brief-conformance skipped: no VLM backend configured (set VALIDATE_UI_LLM_BASE_URL + VALIDATE_UI_LLM_TOKEN for the Keboola LLM path, or ANTHROPIC_API_KEY for a raw key)");
|
|
507
|
+
const { artifact } = context;
|
|
508
|
+
const screenshotBase64 = Buffer.from(artifact.screenshot).toString("base64");
|
|
509
|
+
const domExcerpt = artifact.dom.slice(0, DOM_EXCERPT_LIMIT);
|
|
510
|
+
const reply = await provider.judge({
|
|
511
|
+
model: MODEL,
|
|
512
|
+
screenshotBase64,
|
|
513
|
+
prompt: `${RUBRIC}\n\nBrief:\n${brief}\n\nDOM excerpt:\n${domExcerpt}`
|
|
514
|
+
});
|
|
515
|
+
try {
|
|
516
|
+
const parsed = JudgeResult.parse(JSON.parse(reply));
|
|
517
|
+
return interpretJudgeResult(parsed);
|
|
518
|
+
} catch (error) {
|
|
519
|
+
console.error("brief-conformance: failed to parse judge response", error);
|
|
520
|
+
return {
|
|
521
|
+
axis: "brief-conformance",
|
|
522
|
+
pass: false,
|
|
523
|
+
findings: [{
|
|
524
|
+
message: "could not parse judge response",
|
|
525
|
+
severity: "moderate"
|
|
526
|
+
}]
|
|
527
|
+
};
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
};
|
|
531
|
+
//#endregion
|
|
532
|
+
//#region src/compare/diff.ts
|
|
533
|
+
/** Match a `*`-glob (or exact string) against a region key. */
|
|
534
|
+
const matchesGlob = (pattern, value) => {
|
|
535
|
+
if (!pattern.includes("*")) return pattern === value;
|
|
536
|
+
const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
537
|
+
return new RegExp(`^${escaped}$`).test(value);
|
|
538
|
+
};
|
|
539
|
+
/** Multiset difference: tokens present more often in `from` than in `to`. */
|
|
540
|
+
const multisetMissing = (from, to) => {
|
|
541
|
+
const remaining = /* @__PURE__ */ new Map();
|
|
542
|
+
for (const value of to) remaining.set(value, (remaining.get(value) ?? 0) + 1);
|
|
543
|
+
const missing = [];
|
|
544
|
+
for (const value of from) {
|
|
545
|
+
const count = remaining.get(value) ?? 0;
|
|
546
|
+
if (count > 0) remaining.set(value, count - 1);
|
|
547
|
+
else missing.push(value);
|
|
548
|
+
}
|
|
549
|
+
return missing;
|
|
550
|
+
};
|
|
551
|
+
/** Stable signature of a delta — identical across routes for the same shared element. */
|
|
552
|
+
const signature = (regionKey, tokens) => `${regionKey}::${[...tokens].sort().join("|")}`;
|
|
553
|
+
/** The rule that suppresses removing a whole region (region match, no value scope). */
|
|
554
|
+
const regionRemovalRule = (rules, key) => rules.find((rule) => rule.value === void 0 && rule.region !== void 0 && matchesGlob(rule.region, key));
|
|
555
|
+
/** Does a value rule apply to this (region, value) pair? */
|
|
556
|
+
const valueRuleMatches = (rule, key, value) => rule.value !== void 0 && matchesGlob(rule.value, value) && (rule.region === void 0 || matchesGlob(rule.region, key));
|
|
557
|
+
const removedFinding = (region, rule) => ({
|
|
558
|
+
regionKey: region.key,
|
|
559
|
+
kind: "region-removed",
|
|
560
|
+
valueSignature: signature(region.key, region.values),
|
|
561
|
+
message: rule === void 0 ? `region "${region.label}" was removed` : `region "${region.label}" removed as expected`,
|
|
562
|
+
severity: rule === void 0 ? "serious" : "minor",
|
|
563
|
+
...rule !== void 0 && { suppressed: { reason: rule.reason } }
|
|
564
|
+
});
|
|
565
|
+
const degradedFinding = (region) => ({
|
|
566
|
+
regionKey: region.key,
|
|
567
|
+
kind: "region-degraded",
|
|
568
|
+
valueSignature: signature(region.key, [region.state]),
|
|
569
|
+
message: `region "${region.label}" degraded to ${region.state} state — not compared as value loss`,
|
|
570
|
+
severity: "minor"
|
|
571
|
+
});
|
|
572
|
+
const addedFinding = (region) => ({
|
|
573
|
+
regionKey: region.key,
|
|
574
|
+
kind: "region-added",
|
|
575
|
+
valueSignature: signature(region.key, []),
|
|
576
|
+
message: `region "${region.label}" is new`,
|
|
577
|
+
severity: "minor"
|
|
578
|
+
});
|
|
579
|
+
/**
|
|
580
|
+
* Diff one region present on both sides. Returns the value-missing findings
|
|
581
|
+
* (active and/or suppressed), or none when values are unchanged.
|
|
582
|
+
*/
|
|
583
|
+
const diffRegionValues = (oldRegion, newRegion, rules) => {
|
|
584
|
+
const missing = multisetMissing(oldRegion.values, newRegion.values);
|
|
585
|
+
if (missing.length === 0) return [];
|
|
586
|
+
const active = [];
|
|
587
|
+
const suppressed = [];
|
|
588
|
+
const reasons = /* @__PURE__ */ new Set();
|
|
589
|
+
for (const value of missing) {
|
|
590
|
+
const rule = rules.find((r) => valueRuleMatches(r, newRegion.key, value));
|
|
591
|
+
if (rule === void 0) active.push(value);
|
|
592
|
+
else {
|
|
593
|
+
suppressed.push(value);
|
|
594
|
+
reasons.add(rule.reason);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
const findings = [];
|
|
598
|
+
if (active.length > 0) findings.push({
|
|
599
|
+
regionKey: newRegion.key,
|
|
600
|
+
kind: "value-missing",
|
|
601
|
+
valueSignature: signature(newRegion.key, active),
|
|
602
|
+
message: `region "${newRegion.label}" is missing ${active.length} value(s)`,
|
|
603
|
+
severity: "moderate",
|
|
604
|
+
detail: active.join(", ")
|
|
605
|
+
});
|
|
606
|
+
if (suppressed.length > 0) findings.push({
|
|
607
|
+
regionKey: newRegion.key,
|
|
608
|
+
kind: "value-missing",
|
|
609
|
+
valueSignature: signature(newRegion.key, suppressed),
|
|
610
|
+
message: `region "${newRegion.label}" is missing ${suppressed.length} expected value(s)`,
|
|
611
|
+
severity: "minor",
|
|
612
|
+
detail: suppressed.join(", "),
|
|
613
|
+
suppressed: { reason: [...reasons].join("; ") }
|
|
614
|
+
});
|
|
615
|
+
return findings;
|
|
616
|
+
};
|
|
617
|
+
/**
|
|
618
|
+
* Semantic old-vs-new diff of two snapshots. Walks regions by key and localizes
|
|
619
|
+
* every delta to a region: a removed region is `serious` unless allowlisted; a
|
|
620
|
+
* region that degraded to an empty/error state is reported informationally and
|
|
621
|
+
* NOT diffed as value loss; surviving regions get a per-region value-multiset
|
|
622
|
+
* diff with allowlist suppression; new regions are non-regression notes. Pure —
|
|
623
|
+
* no browser, no I/O.
|
|
624
|
+
*/
|
|
625
|
+
const diffSnapshots = (oldSnap, newSnap, config = {}) => {
|
|
626
|
+
const rules = config.expectedAbsent ?? [];
|
|
627
|
+
const newByKey = new Map(newSnap.regions.map((region) => [region.key, region]));
|
|
628
|
+
const oldKeys = new Set(oldSnap.regions.map((region) => region.key));
|
|
629
|
+
const findings = [];
|
|
630
|
+
for (const oldRegion of oldSnap.regions) {
|
|
631
|
+
const newRegion = newByKey.get(oldRegion.key);
|
|
632
|
+
if (newRegion === void 0) {
|
|
633
|
+
findings.push(removedFinding(oldRegion, regionRemovalRule(rules, oldRegion.key)));
|
|
634
|
+
continue;
|
|
635
|
+
}
|
|
636
|
+
if (oldRegion.state === "ok" && newRegion.state !== "ok") {
|
|
637
|
+
findings.push(degradedFinding(newRegion));
|
|
638
|
+
continue;
|
|
639
|
+
}
|
|
640
|
+
findings.push(...diffRegionValues(oldRegion, newRegion, rules));
|
|
641
|
+
}
|
|
642
|
+
for (const newRegion of newSnap.regions) if (!oldKeys.has(newRegion.key)) findings.push(addedFinding(newRegion));
|
|
643
|
+
return findings;
|
|
644
|
+
};
|
|
645
|
+
/** A compare run passes when no active (non-suppressed) delta is serious/critical. */
|
|
646
|
+
const comparePass = (findings) => !findings.some((f) => f.suppressed === void 0 && (f.severity === "serious" || f.severity === "critical"));
|
|
647
|
+
//#endregion
|
|
648
|
+
//#region src/compare/regions.ts
|
|
649
|
+
const CANDIDATE_SELECTOR = [
|
|
650
|
+
"[data-region]",
|
|
651
|
+
"main",
|
|
652
|
+
"nav",
|
|
653
|
+
"header",
|
|
654
|
+
"aside",
|
|
655
|
+
"footer",
|
|
656
|
+
"section[aria-label]",
|
|
657
|
+
"[role=main]",
|
|
658
|
+
"[role=navigation]",
|
|
659
|
+
"[role=banner]",
|
|
660
|
+
"[role=complementary]",
|
|
661
|
+
"[role=contentinfo]",
|
|
662
|
+
"[role=region]"
|
|
663
|
+
].join(",");
|
|
664
|
+
const DEFAULT_EMPTY_MARKERS = [
|
|
665
|
+
"no data",
|
|
666
|
+
"no results",
|
|
667
|
+
"nothing to show",
|
|
668
|
+
"nothing here",
|
|
669
|
+
"get started"
|
|
670
|
+
];
|
|
671
|
+
const DEFAULT_ERROR_MARKERS = [
|
|
672
|
+
"something went wrong",
|
|
673
|
+
"failed to load",
|
|
674
|
+
"could not load",
|
|
675
|
+
"unavailable",
|
|
676
|
+
"try again"
|
|
677
|
+
];
|
|
678
|
+
/**
|
|
679
|
+
* Reduce the live page to its diffable regions, in the browser. An explicit
|
|
680
|
+
* `[data-region]` is an authoritative boundary (its subtree is one region);
|
|
681
|
+
* elsewhere boundaries are the outermost-empty ("leaf") landmarks/sections, so
|
|
682
|
+
* regions never overlap or double-count values. Each region's key is resolved by
|
|
683
|
+
* precedence
|
|
684
|
+
* (`data-region` → `data-testid` → role → `aria-label` → nearest heading), its
|
|
685
|
+
* state from `data-state`/text markers, and its values as normalized numeric
|
|
686
|
+
* tokens. Symmetric across old and new so the two sides diff cleanly.
|
|
687
|
+
*/
|
|
688
|
+
const extractRegions = async (page, config = {}) => {
|
|
689
|
+
const emptyMarkers = [...DEFAULT_EMPTY_MARKERS, ...config.emptyStateMarkers ?? []];
|
|
690
|
+
const errorMarkers = [...DEFAULT_ERROR_MARKERS, ...config.errorStateMarkers ?? []];
|
|
691
|
+
return page.evaluate(({ selector, emptyMarkers, errorMarkers }) => {
|
|
692
|
+
const STATES = [
|
|
693
|
+
"ok",
|
|
694
|
+
"empty",
|
|
695
|
+
"error",
|
|
696
|
+
"loading"
|
|
697
|
+
];
|
|
698
|
+
const isState = (value) => STATES.includes(value);
|
|
699
|
+
const slugify = (raw) => raw.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "region";
|
|
700
|
+
const roleOf = (el) => {
|
|
701
|
+
const explicit = el.getAttribute("role");
|
|
702
|
+
if (explicit !== null && explicit !== "") return explicit;
|
|
703
|
+
const tag = el.tagName.toLowerCase();
|
|
704
|
+
if (tag === "main") return "main";
|
|
705
|
+
if (tag === "nav") return "navigation";
|
|
706
|
+
if (tag === "aside") return "complementary";
|
|
707
|
+
if (tag === "header" || tag === "footer") {
|
|
708
|
+
if (el.closest("main,nav,section,article,aside") !== null) return void 0;
|
|
709
|
+
return tag === "header" ? "banner" : "contentinfo";
|
|
710
|
+
}
|
|
711
|
+
};
|
|
712
|
+
const headingOf = (el) => {
|
|
713
|
+
const text = el.querySelector("h1,h2,h3,h4,h5,h6")?.textContent?.trim();
|
|
714
|
+
return text !== void 0 && text !== "" ? text : void 0;
|
|
715
|
+
};
|
|
716
|
+
const keyOf = (el) => {
|
|
717
|
+
const region = el.getAttribute("data-region");
|
|
718
|
+
if (region !== null && region !== "") return slugify(region);
|
|
719
|
+
const testid = el.getAttribute("data-testid");
|
|
720
|
+
if (testid !== null && testid !== "") return slugify(testid);
|
|
721
|
+
const role = roleOf(el);
|
|
722
|
+
if (role !== void 0) return role;
|
|
723
|
+
const label = el.getAttribute("aria-label");
|
|
724
|
+
if (label !== null && label !== "") return slugify(label);
|
|
725
|
+
const heading = headingOf(el);
|
|
726
|
+
if (heading !== void 0) return slugify(heading);
|
|
727
|
+
return slugify(el.tagName);
|
|
728
|
+
};
|
|
729
|
+
const labelOf = (el, key) => {
|
|
730
|
+
const label = el.getAttribute("aria-label");
|
|
731
|
+
if (label !== null && label.trim() !== "") return label.trim();
|
|
732
|
+
const heading = headingOf(el);
|
|
733
|
+
if (heading !== void 0) return heading;
|
|
734
|
+
const testid = el.getAttribute("data-testid");
|
|
735
|
+
if (testid !== null && testid !== "") return testid;
|
|
736
|
+
return key;
|
|
737
|
+
};
|
|
738
|
+
const textOf = (el) => (el.innerText || el.textContent || "").trim();
|
|
739
|
+
const stateOf = (el, text) => {
|
|
740
|
+
const explicit = (el.getAttribute("data-state") ?? "").toLowerCase();
|
|
741
|
+
if (isState(explicit)) return explicit;
|
|
742
|
+
const lower = text.toLowerCase();
|
|
743
|
+
if (errorMarkers.some((marker) => lower.includes(marker))) return "error";
|
|
744
|
+
if (lower.includes("loading")) return "loading";
|
|
745
|
+
if (emptyMarkers.some((marker) => lower.includes(marker))) return "empty";
|
|
746
|
+
return "ok";
|
|
747
|
+
};
|
|
748
|
+
const valuesOf = (text) => (text.match(/-?\$?\s?\d[\d,]*(?:\.\d+)?%?/g) ?? []).map((token) => token.replace(/[$\s,]/g, "")).filter((token) => token !== "" && token !== "-");
|
|
749
|
+
const qualifies = (el) => {
|
|
750
|
+
const dataRegion = el.getAttribute("data-region");
|
|
751
|
+
if (dataRegion !== null && dataRegion !== "") {
|
|
752
|
+
const parent = el.parentElement;
|
|
753
|
+
return parent === null || parent.closest("[data-region]") === null;
|
|
754
|
+
}
|
|
755
|
+
return el.querySelector(selector) === null && el.closest("[data-region]") === null;
|
|
756
|
+
};
|
|
757
|
+
const regions = Array.from(document.querySelectorAll(selector)).filter(qualifies);
|
|
758
|
+
const elements = regions.length > 0 ? regions : [document.body];
|
|
759
|
+
const used = {};
|
|
760
|
+
const uniqueKey = (key) => {
|
|
761
|
+
const count = (used[key] ?? 0) + 1;
|
|
762
|
+
used[key] = count;
|
|
763
|
+
return count === 1 ? key : `${key}-${count}`;
|
|
764
|
+
};
|
|
765
|
+
return elements.map((el) => {
|
|
766
|
+
const text = textOf(el);
|
|
767
|
+
const key = uniqueKey(keyOf(el));
|
|
768
|
+
return {
|
|
769
|
+
key,
|
|
770
|
+
label: labelOf(el, key),
|
|
771
|
+
state: stateOf(el, text),
|
|
772
|
+
values: valuesOf(text)
|
|
773
|
+
};
|
|
774
|
+
});
|
|
775
|
+
}, {
|
|
776
|
+
selector: CANDIDATE_SELECTOR,
|
|
777
|
+
emptyMarkers,
|
|
778
|
+
errorMarkers
|
|
779
|
+
});
|
|
780
|
+
};
|
|
781
|
+
//#endregion
|
|
782
|
+
//#region src/compare/types.ts
|
|
783
|
+
/**
|
|
784
|
+
* Backend/render state of a region. A region that rendered its data is `ok`;
|
|
785
|
+
* `empty`/`error`/`loading` are non-data states — an old→new transition into one
|
|
786
|
+
* of them is a degrade, not a value regression (a backend was unavailable, or an
|
|
787
|
+
* intentional empty state is showing).
|
|
788
|
+
*/
|
|
789
|
+
const REGION_STATES = [
|
|
790
|
+
"ok",
|
|
791
|
+
"empty",
|
|
792
|
+
"error",
|
|
793
|
+
"loading"
|
|
794
|
+
];
|
|
795
|
+
/** One diffable region of a page — a landmark, labelled section, or component. */
|
|
796
|
+
const Region = zod.z.object({
|
|
797
|
+
/** Stable id: data-region → data-testid → role → aria-label → heading slug. */
|
|
798
|
+
key: zod.z.string(),
|
|
799
|
+
/** Human label for the report. */
|
|
800
|
+
label: zod.z.string(),
|
|
801
|
+
state: zod.z.enum(REGION_STATES),
|
|
802
|
+
/** Normalized numeric / currency / percent tokens found in the region. */
|
|
803
|
+
values: zod.z.array(zod.z.string())
|
|
804
|
+
});
|
|
805
|
+
/**
|
|
806
|
+
* A page reduced to its diffable regions — the currency both input paths (a live
|
|
807
|
+
* boot or a recorded JSON file) produce and the differ consumes. Pure JSON, so
|
|
808
|
+
* it is safe to serialize to disk and re-load; {@link loadSnapshot} validates it.
|
|
809
|
+
*/
|
|
810
|
+
const CompareSnapshot = zod.z.object({
|
|
811
|
+
/** Route the snapshot was captured at, e.g. `/financials`. */
|
|
812
|
+
route: zod.z.string(),
|
|
813
|
+
regions: zod.z.array(Region)
|
|
814
|
+
}).refine((snap) => new Set(snap.regions.map((region) => region.key)).size === snap.regions.length, { message: "region keys must be unique" });
|
|
815
|
+
/**
|
|
816
|
+
* One expected-absent rule. `region` (exact or `*`-glob against {@link Region.key})
|
|
817
|
+
* alone suppresses a whole `region-removed`; add `value` to suppress a specific
|
|
818
|
+
* missing value token (optionally scoped to a region). `reason` is required and
|
|
819
|
+
* surfaced in the report so suppression is auditable, never silent.
|
|
820
|
+
*/
|
|
821
|
+
const ExpectedAbsentRule = zod.z.object({
|
|
822
|
+
region: zod.z.string().optional(),
|
|
823
|
+
value: zod.z.string().optional(),
|
|
824
|
+
reason: zod.z.string().min(1)
|
|
825
|
+
}).refine((rule) => rule.region !== void 0 || rule.value !== void 0, { message: "expectedAbsent rule needs at least one of `region` or `value`" });
|
|
826
|
+
/** Semantic-compare configuration — the allowlist plus extra state markers. */
|
|
827
|
+
const CompareConfig = zod.z.object({
|
|
828
|
+
expectedAbsent: zod.z.array(ExpectedAbsentRule).optional(),
|
|
829
|
+
emptyStateMarkers: zod.z.array(zod.z.string()).optional(),
|
|
830
|
+
errorStateMarkers: zod.z.array(zod.z.string()).optional()
|
|
831
|
+
});
|
|
832
|
+
//#endregion
|
|
833
|
+
//#region src/compare/snapshot.ts
|
|
834
|
+
/** Reduce a loaded live page to a {@link CompareSnapshot} for the given route. */
|
|
835
|
+
const captureSnapshot = async (page, route, config) => ({
|
|
836
|
+
route,
|
|
837
|
+
regions: await extractRegions(page, config)
|
|
838
|
+
});
|
|
839
|
+
/** Persist a snapshot as pretty JSON so an "old" side can be recorded once and re-diffed. */
|
|
840
|
+
const saveSnapshot = async (snapshot, path) => {
|
|
841
|
+
await (0, node_fs_promises.writeFile)(path, JSON.stringify(snapshot, null, 2));
|
|
842
|
+
};
|
|
843
|
+
/** Read and validate a recorded snapshot from disk. */
|
|
844
|
+
const loadSnapshot = async (path) => {
|
|
845
|
+
const raw = await (0, node_fs_promises.readFile)(path, "utf8");
|
|
846
|
+
return CompareSnapshot.parse(JSON.parse(raw));
|
|
847
|
+
};
|
|
848
|
+
//#endregion
|
|
849
|
+
//#region src/axes/compare.ts
|
|
850
|
+
const skip = (message) => ({
|
|
851
|
+
axis: "compare",
|
|
852
|
+
pass: true,
|
|
853
|
+
findings: [{
|
|
854
|
+
message,
|
|
855
|
+
severity: "minor"
|
|
856
|
+
}]
|
|
857
|
+
});
|
|
858
|
+
/**
|
|
859
|
+
* A5 · Semantic old-vs-new compare verdict — UT-4572.
|
|
860
|
+
*
|
|
861
|
+
* Activates only when `context.comparison` (an "old" {@link CompareSnapshot}) is
|
|
862
|
+
* supplied; otherwise it degrades to a passing skip, so ordinary single-page runs
|
|
863
|
+
* are unaffected. Extracts the "new" snapshot from the live `context.page`, then
|
|
864
|
+
* runs the pure {@link diffSnapshots}: deltas are localized per region, declared
|
|
865
|
+
* expected-absent deltas are suppressed (not dropped), and a region that degraded
|
|
866
|
+
* to an empty/error state is reported informationally rather than as value loss.
|
|
867
|
+
*/
|
|
868
|
+
const compareAxis = {
|
|
869
|
+
name: "compare",
|
|
870
|
+
run: async (context) => {
|
|
871
|
+
const { comparison, compareConfig, page } = context;
|
|
872
|
+
if (comparison === void 0) return skip("no comparison snapshot provided; old-vs-new not assessed");
|
|
873
|
+
if (page === void 0) return skip("compare needs a live page to extract the new snapshot");
|
|
874
|
+
const newSnapshot = await captureSnapshot(page, comparison.route, compareConfig);
|
|
875
|
+
const findings = diffSnapshots(comparison, newSnapshot, compareConfig);
|
|
876
|
+
return {
|
|
877
|
+
axis: "compare",
|
|
878
|
+
pass: comparePass(findings),
|
|
879
|
+
findings
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
};
|
|
883
|
+
//#endregion
|
|
884
|
+
//#region src/axes/runtime-health.ts
|
|
885
|
+
const ERROR_MARKERS = [
|
|
886
|
+
"something went wrong",
|
|
887
|
+
"vite-error-overlay",
|
|
888
|
+
"vite-plugin-checker-error-overlay",
|
|
889
|
+
"react-error-overlay"
|
|
890
|
+
];
|
|
891
|
+
const pageErrorFindings = (console) => console.filter((entry) => entry.type === "pageerror").map((entry) => ({
|
|
892
|
+
message: `Uncaught page error: ${entry.text}`,
|
|
893
|
+
severity: "critical",
|
|
894
|
+
detail: entry.location
|
|
895
|
+
}));
|
|
896
|
+
const RESOURCE_LOAD_ECHO = "failed to load resource";
|
|
897
|
+
const isResourceLoadEcho = (entry) => entry.text.toLowerCase().includes(RESOURCE_LOAD_ECHO);
|
|
898
|
+
/**
|
|
899
|
+
* Real app console errors only — the generic "Failed to load resource" echoes
|
|
900
|
+
* are dropped (the network finding for the same URL is more useful). Identical
|
|
901
|
+
* texts are deduped since one noisy error often logs on every render.
|
|
902
|
+
*/
|
|
903
|
+
const consoleErrorFindings = (console) => {
|
|
904
|
+
const seen = /* @__PURE__ */ new Set();
|
|
905
|
+
const findings = [];
|
|
906
|
+
for (const entry of console) {
|
|
907
|
+
if (entry.type !== "error" || isResourceLoadEcho(entry) || seen.has(entry.text)) continue;
|
|
908
|
+
seen.add(entry.text);
|
|
909
|
+
findings.push({
|
|
910
|
+
message: `Console error: ${entry.text}`,
|
|
911
|
+
severity: "serious",
|
|
912
|
+
detail: entry.location
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
return findings;
|
|
916
|
+
};
|
|
917
|
+
/** Turn one network entry into a finding, or null for a healthy (<400) response. */
|
|
918
|
+
const toNetworkFinding = (entry) => {
|
|
919
|
+
if (entry.status === 0) return {
|
|
920
|
+
message: `Request failed: ${entry.method} ${entry.url}`,
|
|
921
|
+
severity: "serious",
|
|
922
|
+
detail: entry.failure,
|
|
923
|
+
ref: entry.url
|
|
924
|
+
};
|
|
925
|
+
if (entry.status >= 500) return {
|
|
926
|
+
message: `Server error ${entry.status}: ${entry.method} ${entry.url}`,
|
|
927
|
+
severity: "serious",
|
|
928
|
+
ref: entry.url
|
|
929
|
+
};
|
|
930
|
+
if (entry.status >= 400) return {
|
|
931
|
+
message: `Client error ${entry.status}: ${entry.method} ${entry.url}`,
|
|
932
|
+
severity: "moderate",
|
|
933
|
+
ref: entry.url
|
|
934
|
+
};
|
|
935
|
+
return null;
|
|
936
|
+
};
|
|
937
|
+
const SEVERITY_RANK = {
|
|
938
|
+
critical: 3,
|
|
939
|
+
serious: 2,
|
|
940
|
+
moderate: 1,
|
|
941
|
+
minor: 0
|
|
942
|
+
};
|
|
943
|
+
/**
|
|
944
|
+
* One finding per URL. A single failing resource surfaces as a `requestfailed`,
|
|
945
|
+
* a 4xx/5xx response, and a console echo — all the same root cause. Collapse
|
|
946
|
+
* them by URL, keeping the most severe (a request-failure or explicit HTTP
|
|
947
|
+
* status over the generic browser echo).
|
|
948
|
+
*/
|
|
949
|
+
const networkFindings = (network) => {
|
|
950
|
+
const byUrl = /* @__PURE__ */ new Map();
|
|
951
|
+
for (const entry of network) {
|
|
952
|
+
const finding = toNetworkFinding(entry);
|
|
953
|
+
if (finding === null) continue;
|
|
954
|
+
const existing = byUrl.get(entry.url);
|
|
955
|
+
if (existing === void 0 || SEVERITY_RANK[finding.severity] > SEVERITY_RANK[existing.severity]) byUrl.set(entry.url, finding);
|
|
956
|
+
}
|
|
957
|
+
return [...byUrl.values()];
|
|
958
|
+
};
|
|
959
|
+
/** An error overlay / boundary in the DOM means a crashed render, not real content. */
|
|
960
|
+
const errorMarkerFinding = (dom) => {
|
|
961
|
+
const lower = dom.toLowerCase();
|
|
962
|
+
const marker = ERROR_MARKERS.find((m) => lower.includes(m));
|
|
963
|
+
return marker === void 0 ? null : {
|
|
964
|
+
message: "Error boundary / error overlay rendered instead of the app",
|
|
965
|
+
severity: "critical",
|
|
966
|
+
detail: marker
|
|
967
|
+
};
|
|
968
|
+
};
|
|
969
|
+
/**
|
|
970
|
+
* White-screen check on the reliable live-DOM signal (`renderedText`). Skipped
|
|
971
|
+
* when absent — parsing the HTML string for "emptiness" gives false positives.
|
|
972
|
+
*/
|
|
973
|
+
const whiteScreenFinding = (renderedText) => renderedText !== void 0 && renderedText.length === 0 ? {
|
|
974
|
+
message: "White screen: page rendered no visible text",
|
|
975
|
+
severity: "critical"
|
|
976
|
+
} : null;
|
|
977
|
+
/**
|
|
978
|
+
* A2 · Runtime-health verdict — UT-4493.
|
|
979
|
+
*
|
|
980
|
+
* Pure over `context.artifact` (no live page). Collects uncaught page errors,
|
|
981
|
+
* console errors, failed/4xx/5xx requests, and white-screen / error-overlay
|
|
982
|
+
* signals into `findings`.
|
|
983
|
+
*/
|
|
984
|
+
const runtimeHealthAxis = {
|
|
985
|
+
name: "runtime-health",
|
|
986
|
+
run: (context) => {
|
|
987
|
+
const { artifact } = context;
|
|
988
|
+
const findings = [
|
|
989
|
+
...pageErrorFindings(artifact.console),
|
|
990
|
+
...consoleErrorFindings(artifact.console),
|
|
991
|
+
...networkFindings(artifact.network)
|
|
992
|
+
];
|
|
993
|
+
const errorMarker = errorMarkerFinding(artifact.dom);
|
|
994
|
+
if (errorMarker !== null) findings.push(errorMarker);
|
|
995
|
+
const whiteScreen = whiteScreenFinding(artifact.renderedText);
|
|
996
|
+
if (whiteScreen !== null) findings.push(whiteScreen);
|
|
997
|
+
const fatal = ["critical", "serious"];
|
|
998
|
+
const pass = !findings.some((f) => fatal.includes(f.severity));
|
|
999
|
+
return Promise.resolve({
|
|
1000
|
+
axis: "runtime-health",
|
|
1001
|
+
pass,
|
|
1002
|
+
findings
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
};
|
|
1006
|
+
//#endregion
|
|
1007
|
+
//#region src/axes/visual-brand.ts
|
|
1008
|
+
const DRIFT_THRESHOLD = .01;
|
|
1009
|
+
const RESKIN_THRESHOLD = .005;
|
|
1010
|
+
const DRIFT_PIXEL_THRESHOLD = .1;
|
|
1011
|
+
const RESKIN_PIXEL_THRESHOLD = .02;
|
|
1012
|
+
const DEFAULT_ALT_BRAND = "example-customer";
|
|
1013
|
+
/** Decode two PNGs and return their pixel mismatch ratio (0 = identical, 1 = every pixel differs). */
|
|
1014
|
+
const pngMismatchRatio = (a, b, pixelThreshold = DRIFT_PIXEL_THRESHOLD) => {
|
|
1015
|
+
const imgA = pngjs.PNG.sync.read(Buffer.from(a));
|
|
1016
|
+
const imgB = pngjs.PNG.sync.read(Buffer.from(b));
|
|
1017
|
+
if (imgA.width !== imgB.width || imgA.height !== imgB.height) return {
|
|
1018
|
+
ratio: 1,
|
|
1019
|
+
sizeMismatch: true
|
|
1020
|
+
};
|
|
1021
|
+
const total = imgA.width * imgA.height;
|
|
1022
|
+
if (total === 0) return {
|
|
1023
|
+
ratio: 0,
|
|
1024
|
+
sizeMismatch: false
|
|
1025
|
+
};
|
|
1026
|
+
return {
|
|
1027
|
+
ratio: (0, pixelmatch.default)(imgA.data, imgB.data, void 0, imgA.width, imgA.height, { threshold: pixelThreshold }) / total,
|
|
1028
|
+
sizeMismatch: false
|
|
1029
|
+
};
|
|
1030
|
+
};
|
|
1031
|
+
const baselineFinding = (screenshot, baseline) => {
|
|
1032
|
+
const { ratio, sizeMismatch } = pngMismatchRatio(screenshot, baseline);
|
|
1033
|
+
if (sizeMismatch) return {
|
|
1034
|
+
message: "screenshot dimensions differ from baseline",
|
|
1035
|
+
severity: "moderate"
|
|
1036
|
+
};
|
|
1037
|
+
if (ratio > DRIFT_THRESHOLD) return {
|
|
1038
|
+
message: `visual drift ${(ratio * 100).toFixed(1)}% vs baseline`,
|
|
1039
|
+
severity: "moderate",
|
|
1040
|
+
detail: `${(ratio * 100).toFixed(2)}% of pixels changed`
|
|
1041
|
+
};
|
|
1042
|
+
return null;
|
|
1043
|
+
};
|
|
1044
|
+
/**
|
|
1045
|
+
* Whether the alt-brand capture actually activated that brand.
|
|
1046
|
+
*
|
|
1047
|
+
* `captureUnderBrand` requests a brand with `?brand=<id>`, which only works if the app
|
|
1048
|
+
* reads that param and re-resolves its `ThemeProvider`. Most apps deliberately do not:
|
|
1049
|
+
* the brand is build configuration, not a runtime switch, and shipping a query-param
|
|
1050
|
+
* brand switcher into every generated app is not something a scaffold should impose.
|
|
1051
|
+
*
|
|
1052
|
+
* Without this check, such an app captures the same brand twice, and the identical
|
|
1053
|
+
* screenshots are reported as "chrome did not re-skin" — a false failure for code that is
|
|
1054
|
+
* entirely correct. The brand is nominated via `data-brand` on the root element (written
|
|
1055
|
+
* by `@keboola/design`'s ThemeProvider), so absence of the requested id means the request
|
|
1056
|
+
* was ignored, not that the chrome failed to re-skin.
|
|
1057
|
+
*/
|
|
1058
|
+
const altBrandWasApplied = (altDom, brand) => altDom.includes(`data-brand="${brand}"`);
|
|
1059
|
+
const altBrandFinding = (screenshot, altScreenshot, brand, altDom) => {
|
|
1060
|
+
if (!altBrandWasApplied(altDom, brand)) return {
|
|
1061
|
+
message: `alt-brand re-skin not assessed: the app did not apply '${brand}' from ?brand=`,
|
|
1062
|
+
severity: "minor",
|
|
1063
|
+
detail: "Brand is usually build configuration rather than a runtime switch. To have this axis assess re-skinning, register the alternate brand and resolve the active id from the `brand` query parameter."
|
|
1064
|
+
};
|
|
1065
|
+
const { ratio } = pngMismatchRatio(screenshot, altScreenshot, RESKIN_PIXEL_THRESHOLD);
|
|
1066
|
+
if (ratio < RESKIN_THRESHOLD) return {
|
|
1067
|
+
message: `chrome did not re-skin under alternate brand '${brand}'`,
|
|
1068
|
+
severity: "serious",
|
|
1069
|
+
detail: `${(ratio * 100).toFixed(2)}% of pixels changed`
|
|
1070
|
+
};
|
|
1071
|
+
return null;
|
|
1072
|
+
};
|
|
1073
|
+
/**
|
|
1074
|
+
* Non-fatal `moderate` findings for a brand's interactive foreground↔background
|
|
1075
|
+
* pairings that fall below WCAG-AA (button/link labels on branded fills). Surfaces
|
|
1076
|
+
* the guarantee `@keboola/brand-registry` computes so an AA-failing token is a
|
|
1077
|
+
* build-time warning here, not a runtime axe surprise. Empty for an AA-safe brand.
|
|
1078
|
+
*/
|
|
1079
|
+
const interactiveContrastFindings = (brand) => (0, _keboola_brand_registry.checkInteractiveContrast)(brand).map((warning) => ({
|
|
1080
|
+
message: `brand '${brand.id}' ${warning.message}`,
|
|
1081
|
+
severity: "moderate",
|
|
1082
|
+
detail: `${warning.foreground} on ${warning.background} (${warning.palette})`
|
|
1083
|
+
}));
|
|
1084
|
+
const overflowFinding = async (page) => {
|
|
1085
|
+
if (await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth)) return {
|
|
1086
|
+
message: "horizontal layout overflow",
|
|
1087
|
+
severity: "moderate"
|
|
1088
|
+
};
|
|
1089
|
+
return null;
|
|
1090
|
+
};
|
|
1091
|
+
/**
|
|
1092
|
+
* A4 · Visual & brand-correctness verdict — UT-4495.
|
|
1093
|
+
*
|
|
1094
|
+
* Baseline diff of `context.artifact.screenshot` vs `context.baseline`
|
|
1095
|
+
* (pixelmatch + pngjs), plus the alt-brand fitness function via
|
|
1096
|
+
* `context.captureUnderBrand`: the chrome must visibly re-skin under an
|
|
1097
|
+
* alternate brand. Flags layout overflow. Each check degrades to no finding when
|
|
1098
|
+
* its input is absent. (Asserting categorical colors — Badge/Alert/ModalIcon —
|
|
1099
|
+
* stay fixed pixel-wise is future work; see `altBrandFinding`.)
|
|
1100
|
+
*/
|
|
1101
|
+
const visualBrandAxis = {
|
|
1102
|
+
name: "visual-brand",
|
|
1103
|
+
run: async (context) => {
|
|
1104
|
+
const { artifact, baseline, captureUnderBrand, page } = context;
|
|
1105
|
+
const findings = [];
|
|
1106
|
+
if (baseline !== void 0) {
|
|
1107
|
+
const finding = baselineFinding(artifact.screenshot, baseline);
|
|
1108
|
+
if (finding !== null) findings.push(finding);
|
|
1109
|
+
}
|
|
1110
|
+
if (captureUnderBrand !== void 0) {
|
|
1111
|
+
const altArtifact = await captureUnderBrand(DEFAULT_ALT_BRAND);
|
|
1112
|
+
const finding = altBrandFinding(artifact.screenshot, altArtifact.screenshot, DEFAULT_ALT_BRAND, altArtifact.dom);
|
|
1113
|
+
if (finding !== null) findings.push(finding);
|
|
1114
|
+
const altBrand = _keboola_brand_registry.brands.find((brand) => brand.id === DEFAULT_ALT_BRAND);
|
|
1115
|
+
if (altBrand !== void 0) findings.push(...interactiveContrastFindings(altBrand));
|
|
1116
|
+
}
|
|
1117
|
+
if (page !== void 0) {
|
|
1118
|
+
const finding = await overflowFinding(page);
|
|
1119
|
+
if (finding !== null) findings.push(finding);
|
|
1120
|
+
}
|
|
1121
|
+
return {
|
|
1122
|
+
axis: "visual-brand",
|
|
1123
|
+
pass: !findings.some((f) => f.severity === "critical" || f.severity === "serious"),
|
|
1124
|
+
findings
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
};
|
|
1128
|
+
//#endregion
|
|
1129
|
+
//#region src/axes/index.ts
|
|
1130
|
+
/**
|
|
1131
|
+
* The registered axes, in report order. `compare` runs last and only activates
|
|
1132
|
+
* when a comparison snapshot is supplied, so ordinary single-page runs are
|
|
1133
|
+
* unaffected.
|
|
1134
|
+
*/
|
|
1135
|
+
const ALL_AXES = [
|
|
1136
|
+
runtimeHealthAxis,
|
|
1137
|
+
accessibilityAxis,
|
|
1138
|
+
visualBrandAxis,
|
|
1139
|
+
briefConformanceAxis,
|
|
1140
|
+
compareAxis
|
|
1141
|
+
];
|
|
1142
|
+
//#endregion
|
|
1143
|
+
//#region src/aggregate.ts
|
|
1144
|
+
/**
|
|
1145
|
+
* Run each axis against the shared context, isolating failures: an axis that
|
|
1146
|
+
* throws becomes a failing verdict instead of aborting the whole run. The error
|
|
1147
|
+
* is logged (never swallowed) and surfaced as a critical finding.
|
|
1148
|
+
*/
|
|
1149
|
+
const runAxes = async (axes, context) => {
|
|
1150
|
+
const verdicts = [];
|
|
1151
|
+
for (const axis of axes) try {
|
|
1152
|
+
verdicts.push(await axis.run(context));
|
|
1153
|
+
} catch (error) {
|
|
1154
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1155
|
+
console.error(`validate-ui: axis "${axis.name}" threw:`, error);
|
|
1156
|
+
verdicts.push({
|
|
1157
|
+
axis: axis.name,
|
|
1158
|
+
pass: false,
|
|
1159
|
+
findings: [{
|
|
1160
|
+
message: `Axis crashed: ${message}`,
|
|
1161
|
+
severity: "critical"
|
|
1162
|
+
}]
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
return verdicts;
|
|
1166
|
+
};
|
|
1167
|
+
/** A run passes only when every axis passed. */
|
|
1168
|
+
const aggregatePass = (verdicts) => verdicts.every((v) => v.pass);
|
|
1169
|
+
const withBrandParam = (url, brand) => {
|
|
1170
|
+
const parsed = new URL(url);
|
|
1171
|
+
parsed.searchParams.set("brand", brand);
|
|
1172
|
+
return parsed.toString();
|
|
1173
|
+
};
|
|
1174
|
+
/**
|
|
1175
|
+
* The closed loop: render `url`, capture ground truth, and run every axis over
|
|
1176
|
+
* it, returning one machine-readable {@link AggregateVerdict}. Keeps the page
|
|
1177
|
+
* open for axes that need a live DOM (a11y, visual) and provides
|
|
1178
|
+
* `captureUnderBrand` for the alt-brand fitness function.
|
|
1179
|
+
*/
|
|
1180
|
+
const validate = async (options) => {
|
|
1181
|
+
const viewport = options.viewport ?? DESKTOP_VIEWPORT;
|
|
1182
|
+
const axes = options.axes ?? ALL_AXES;
|
|
1183
|
+
const browser = options.browser ?? await playwright.chromium.launch();
|
|
1184
|
+
const ownsBrowser = !options.browser;
|
|
1185
|
+
const context = await browser.newContext({ viewport: {
|
|
1186
|
+
width: viewport.width,
|
|
1187
|
+
height: viewport.height
|
|
1188
|
+
} });
|
|
1189
|
+
const page = await context.newPage();
|
|
1190
|
+
try {
|
|
1191
|
+
const axisContext = {
|
|
1192
|
+
artifact: await capturePage(page, options.url, viewport, options.timeoutMs),
|
|
1193
|
+
page,
|
|
1194
|
+
browser,
|
|
1195
|
+
brief: options.brief,
|
|
1196
|
+
baseline: options.baseline,
|
|
1197
|
+
comparison: options.comparison,
|
|
1198
|
+
compareConfig: options.compareConfig,
|
|
1199
|
+
captureUnderBrand: async (brand) => {
|
|
1200
|
+
const brandContext = await browser.newContext({ viewport: {
|
|
1201
|
+
width: viewport.width,
|
|
1202
|
+
height: viewport.height
|
|
1203
|
+
} });
|
|
1204
|
+
const brandPage = await brandContext.newPage();
|
|
1205
|
+
try {
|
|
1206
|
+
return await capturePage(brandPage, withBrandParam(options.url, brand), viewport, options.timeoutMs);
|
|
1207
|
+
} finally {
|
|
1208
|
+
await brandContext.close();
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
};
|
|
1212
|
+
const verdicts = await runAxes(axes, axisContext);
|
|
1213
|
+
return {
|
|
1214
|
+
url: options.url,
|
|
1215
|
+
pass: aggregatePass(verdicts),
|
|
1216
|
+
verdicts
|
|
1217
|
+
};
|
|
1218
|
+
} finally {
|
|
1219
|
+
await context.close();
|
|
1220
|
+
if (ownsBrowser) await browser.close();
|
|
1221
|
+
}
|
|
1222
|
+
};
|
|
1223
|
+
//#endregion
|
|
1224
|
+
//#region src/compare/dedup.ts
|
|
1225
|
+
const groupKey = (finding) => `${finding.kind}::${finding.valueSignature}`;
|
|
1226
|
+
/**
|
|
1227
|
+
* Collapse deltas that recur across routes into a single `shared-chrome` finding.
|
|
1228
|
+
* A value flagged on a shared header/daily-brief element would otherwise be
|
|
1229
|
+
* multiplied across every route (the prototype's `8.4/7.7/-8%/67%` on all 10
|
|
1230
|
+
* pages); here it is reported once, tagged with the routes it spans, and dropped
|
|
1231
|
+
* from each route's own list. Pure — no browser, no I/O.
|
|
1232
|
+
*/
|
|
1233
|
+
const dedupeSharedChrome = (perRoute) => {
|
|
1234
|
+
const routesByGroup = /* @__PURE__ */ new Map();
|
|
1235
|
+
for (const { route, findings } of perRoute) for (const finding of findings) {
|
|
1236
|
+
const key = groupKey(finding);
|
|
1237
|
+
const routes = routesByGroup.get(key) ?? /* @__PURE__ */ new Set();
|
|
1238
|
+
routes.add(route);
|
|
1239
|
+
routesByGroup.set(key, routes);
|
|
1240
|
+
}
|
|
1241
|
+
const sharedGroups = new Set([...routesByGroup].filter(([, routes]) => routes.size >= 2).map(([key]) => key));
|
|
1242
|
+
const sharedChrome = [];
|
|
1243
|
+
const seenShared = /* @__PURE__ */ new Set();
|
|
1244
|
+
const trimmedPerRoute = perRoute.map(({ route, findings }) => ({
|
|
1245
|
+
route,
|
|
1246
|
+
findings: findings.filter((finding) => !sharedGroups.has(groupKey(finding)))
|
|
1247
|
+
}));
|
|
1248
|
+
for (const { findings } of perRoute) for (const finding of findings) {
|
|
1249
|
+
const key = groupKey(finding);
|
|
1250
|
+
if (!sharedGroups.has(key) || seenShared.has(key)) continue;
|
|
1251
|
+
seenShared.add(key);
|
|
1252
|
+
const routes = [...routesByGroup.get(key) ?? []].sort();
|
|
1253
|
+
sharedChrome.push({
|
|
1254
|
+
...finding,
|
|
1255
|
+
message: `${finding.message} (shared across ${routes.length} routes: ${routes.join(", ")})`
|
|
1256
|
+
});
|
|
1257
|
+
}
|
|
1258
|
+
return {
|
|
1259
|
+
sharedChrome,
|
|
1260
|
+
perRoute: trimmedPerRoute
|
|
1261
|
+
};
|
|
1262
|
+
};
|
|
1263
|
+
//#endregion
|
|
1264
|
+
//#region src/compare/runner.ts
|
|
1265
|
+
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
1266
|
+
const joinUrl = (base, route) => `${base.replace(/\/$/, "")}/${route.replace(/^\//, "")}`;
|
|
1267
|
+
/** Recorded old snapshot filename for a route: `/a/b` → `a_b.json`, `/` → `index.json`. */
|
|
1268
|
+
const routeSnapshotFile = (route) => `${route.replace(/^\//, "").replace(/\/+$/, "").replace(/\//g, "_") || "index"}.json`;
|
|
1269
|
+
/**
|
|
1270
|
+
* Multi-route semantic compare. Boots each route on the new side (and the old
|
|
1271
|
+
* side too, unless recorded snapshots are supplied), diffs per route, then
|
|
1272
|
+
* collapses deltas that recur across routes into a single shared-chrome entry so
|
|
1273
|
+
* one shared element is not reported as N regressions. Exactly one of
|
|
1274
|
+
* `oldBaseUrl` / `oldSnapshotDir` must be given.
|
|
1275
|
+
*/
|
|
1276
|
+
const compareRoutes = async (options) => {
|
|
1277
|
+
if (options.oldBaseUrl === void 0 === (options.oldSnapshotDir === void 0)) throw new Error("compareRoutes: provide exactly one of oldBaseUrl or oldSnapshotDir");
|
|
1278
|
+
const viewport = options.viewport ?? DESKTOP_VIEWPORT;
|
|
1279
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
1280
|
+
const browser = options.browser ?? await playwright.chromium.launch();
|
|
1281
|
+
const ownsBrowser = options.browser === void 0;
|
|
1282
|
+
const { oldBaseUrl, oldSnapshotDir } = options;
|
|
1283
|
+
const snapshotUrl = async (url, route) => {
|
|
1284
|
+
const context = await browser.newContext({ viewport: {
|
|
1285
|
+
width: viewport.width,
|
|
1286
|
+
height: viewport.height
|
|
1287
|
+
} });
|
|
1288
|
+
const page = await context.newPage();
|
|
1289
|
+
try {
|
|
1290
|
+
await page.goto(url, {
|
|
1291
|
+
waitUntil: "networkidle",
|
|
1292
|
+
timeout: timeoutMs
|
|
1293
|
+
});
|
|
1294
|
+
await prepareForScreenshot(page);
|
|
1295
|
+
return await captureSnapshot(page, route, options.config);
|
|
1296
|
+
} finally {
|
|
1297
|
+
await context.close();
|
|
1298
|
+
}
|
|
1299
|
+
};
|
|
1300
|
+
const oldSnapshotFor = (route) => {
|
|
1301
|
+
if (oldBaseUrl !== void 0) return snapshotUrl(joinUrl(oldBaseUrl, route), route);
|
|
1302
|
+
if (oldSnapshotDir !== void 0) return loadSnapshot((0, node_path.join)(oldSnapshotDir, routeSnapshotFile(route)));
|
|
1303
|
+
throw new Error("compareRoutes: provide exactly one of oldBaseUrl or oldSnapshotDir");
|
|
1304
|
+
};
|
|
1305
|
+
try {
|
|
1306
|
+
const perRoute = [];
|
|
1307
|
+
for (const route of options.routes) {
|
|
1308
|
+
const oldSnapshot = await oldSnapshotFor(route);
|
|
1309
|
+
const newSnapshot = await snapshotUrl(joinUrl(options.newBaseUrl, route), route);
|
|
1310
|
+
perRoute.push({
|
|
1311
|
+
route,
|
|
1312
|
+
findings: diffSnapshots(oldSnapshot, newSnapshot, options.config)
|
|
1313
|
+
});
|
|
1314
|
+
}
|
|
1315
|
+
const { sharedChrome, perRoute: trimmed } = dedupeSharedChrome(perRoute);
|
|
1316
|
+
const routes = trimmed.map(({ route, findings }) => ({
|
|
1317
|
+
route,
|
|
1318
|
+
findings,
|
|
1319
|
+
pass: comparePass(findings)
|
|
1320
|
+
}));
|
|
1321
|
+
return {
|
|
1322
|
+
routes,
|
|
1323
|
+
sharedChrome,
|
|
1324
|
+
pass: comparePass(sharedChrome) && routes.every((route) => route.pass)
|
|
1325
|
+
};
|
|
1326
|
+
} finally {
|
|
1327
|
+
if (ownsBrowser) await browser.close();
|
|
1328
|
+
}
|
|
1329
|
+
};
|
|
1330
|
+
//#endregion
|
|
1331
|
+
Object.defineProperty(exports, "ALL_AXES", {
|
|
1332
|
+
enumerable: true,
|
|
1333
|
+
get: function() {
|
|
1334
|
+
return ALL_AXES;
|
|
1335
|
+
}
|
|
1336
|
+
});
|
|
1337
|
+
Object.defineProperty(exports, "CompareConfig", {
|
|
1338
|
+
enumerable: true,
|
|
1339
|
+
get: function() {
|
|
1340
|
+
return CompareConfig;
|
|
1341
|
+
}
|
|
1342
|
+
});
|
|
1343
|
+
Object.defineProperty(exports, "CompareSnapshot", {
|
|
1344
|
+
enumerable: true,
|
|
1345
|
+
get: function() {
|
|
1346
|
+
return CompareSnapshot;
|
|
1347
|
+
}
|
|
1348
|
+
});
|
|
1349
|
+
Object.defineProperty(exports, "DEFAULT_VIEWPORTS", {
|
|
1350
|
+
enumerable: true,
|
|
1351
|
+
get: function() {
|
|
1352
|
+
return DEFAULT_VIEWPORTS;
|
|
1353
|
+
}
|
|
1354
|
+
});
|
|
1355
|
+
Object.defineProperty(exports, "DESKTOP_VIEWPORT", {
|
|
1356
|
+
enumerable: true,
|
|
1357
|
+
get: function() {
|
|
1358
|
+
return DESKTOP_VIEWPORT;
|
|
1359
|
+
}
|
|
1360
|
+
});
|
|
1361
|
+
Object.defineProperty(exports, "ExpectedAbsentRule", {
|
|
1362
|
+
enumerable: true,
|
|
1363
|
+
get: function() {
|
|
1364
|
+
return ExpectedAbsentRule;
|
|
1365
|
+
}
|
|
1366
|
+
});
|
|
1367
|
+
Object.defineProperty(exports, "MOBILE_VIEWPORT", {
|
|
1368
|
+
enumerable: true,
|
|
1369
|
+
get: function() {
|
|
1370
|
+
return MOBILE_VIEWPORT;
|
|
1371
|
+
}
|
|
1372
|
+
});
|
|
1373
|
+
Object.defineProperty(exports, "REGION_STATES", {
|
|
1374
|
+
enumerable: true,
|
|
1375
|
+
get: function() {
|
|
1376
|
+
return REGION_STATES;
|
|
1377
|
+
}
|
|
1378
|
+
});
|
|
1379
|
+
Object.defineProperty(exports, "Region", {
|
|
1380
|
+
enumerable: true,
|
|
1381
|
+
get: function() {
|
|
1382
|
+
return Region;
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
Object.defineProperty(exports, "__toESM", {
|
|
1386
|
+
enumerable: true,
|
|
1387
|
+
get: function() {
|
|
1388
|
+
return __toESM;
|
|
1389
|
+
}
|
|
1390
|
+
});
|
|
1391
|
+
Object.defineProperty(exports, "accessibilityAxis", {
|
|
1392
|
+
enumerable: true,
|
|
1393
|
+
get: function() {
|
|
1394
|
+
return accessibilityAxis;
|
|
1395
|
+
}
|
|
1396
|
+
});
|
|
1397
|
+
Object.defineProperty(exports, "aggregatePass", {
|
|
1398
|
+
enumerable: true,
|
|
1399
|
+
get: function() {
|
|
1400
|
+
return aggregatePass;
|
|
1401
|
+
}
|
|
1402
|
+
});
|
|
1403
|
+
Object.defineProperty(exports, "briefConformanceAxis", {
|
|
1404
|
+
enumerable: true,
|
|
1405
|
+
get: function() {
|
|
1406
|
+
return briefConformanceAxis;
|
|
1407
|
+
}
|
|
1408
|
+
});
|
|
1409
|
+
Object.defineProperty(exports, "capture", {
|
|
1410
|
+
enumerable: true,
|
|
1411
|
+
get: function() {
|
|
1412
|
+
return capture;
|
|
1413
|
+
}
|
|
1414
|
+
});
|
|
1415
|
+
Object.defineProperty(exports, "capturePage", {
|
|
1416
|
+
enumerable: true,
|
|
1417
|
+
get: function() {
|
|
1418
|
+
return capturePage;
|
|
1419
|
+
}
|
|
1420
|
+
});
|
|
1421
|
+
Object.defineProperty(exports, "captureSnapshot", {
|
|
1422
|
+
enumerable: true,
|
|
1423
|
+
get: function() {
|
|
1424
|
+
return captureSnapshot;
|
|
1425
|
+
}
|
|
1426
|
+
});
|
|
1427
|
+
Object.defineProperty(exports, "compareAxis", {
|
|
1428
|
+
enumerable: true,
|
|
1429
|
+
get: function() {
|
|
1430
|
+
return compareAxis;
|
|
1431
|
+
}
|
|
1432
|
+
});
|
|
1433
|
+
Object.defineProperty(exports, "comparePass", {
|
|
1434
|
+
enumerable: true,
|
|
1435
|
+
get: function() {
|
|
1436
|
+
return comparePass;
|
|
1437
|
+
}
|
|
1438
|
+
});
|
|
1439
|
+
Object.defineProperty(exports, "compareRoutes", {
|
|
1440
|
+
enumerable: true,
|
|
1441
|
+
get: function() {
|
|
1442
|
+
return compareRoutes;
|
|
1443
|
+
}
|
|
1444
|
+
});
|
|
1445
|
+
Object.defineProperty(exports, "dedupeSharedChrome", {
|
|
1446
|
+
enumerable: true,
|
|
1447
|
+
get: function() {
|
|
1448
|
+
return dedupeSharedChrome;
|
|
1449
|
+
}
|
|
1450
|
+
});
|
|
1451
|
+
Object.defineProperty(exports, "diffSnapshots", {
|
|
1452
|
+
enumerable: true,
|
|
1453
|
+
get: function() {
|
|
1454
|
+
return diffSnapshots;
|
|
1455
|
+
}
|
|
1456
|
+
});
|
|
1457
|
+
Object.defineProperty(exports, "extractRegions", {
|
|
1458
|
+
enumerable: true,
|
|
1459
|
+
get: function() {
|
|
1460
|
+
return extractRegions;
|
|
1461
|
+
}
|
|
1462
|
+
});
|
|
1463
|
+
Object.defineProperty(exports, "loadSnapshot", {
|
|
1464
|
+
enumerable: true,
|
|
1465
|
+
get: function() {
|
|
1466
|
+
return loadSnapshot;
|
|
1467
|
+
}
|
|
1468
|
+
});
|
|
1469
|
+
Object.defineProperty(exports, "prepareForScreenshot", {
|
|
1470
|
+
enumerable: true,
|
|
1471
|
+
get: function() {
|
|
1472
|
+
return prepareForScreenshot;
|
|
1473
|
+
}
|
|
1474
|
+
});
|
|
1475
|
+
Object.defineProperty(exports, "routeSnapshotFile", {
|
|
1476
|
+
enumerable: true,
|
|
1477
|
+
get: function() {
|
|
1478
|
+
return routeSnapshotFile;
|
|
1479
|
+
}
|
|
1480
|
+
});
|
|
1481
|
+
Object.defineProperty(exports, "runAxes", {
|
|
1482
|
+
enumerable: true,
|
|
1483
|
+
get: function() {
|
|
1484
|
+
return runAxes;
|
|
1485
|
+
}
|
|
1486
|
+
});
|
|
1487
|
+
Object.defineProperty(exports, "runtimeHealthAxis", {
|
|
1488
|
+
enumerable: true,
|
|
1489
|
+
get: function() {
|
|
1490
|
+
return runtimeHealthAxis;
|
|
1491
|
+
}
|
|
1492
|
+
});
|
|
1493
|
+
Object.defineProperty(exports, "saveSnapshot", {
|
|
1494
|
+
enumerable: true,
|
|
1495
|
+
get: function() {
|
|
1496
|
+
return saveSnapshot;
|
|
1497
|
+
}
|
|
1498
|
+
});
|
|
1499
|
+
Object.defineProperty(exports, "serveStatic", {
|
|
1500
|
+
enumerable: true,
|
|
1501
|
+
get: function() {
|
|
1502
|
+
return serveStatic;
|
|
1503
|
+
}
|
|
1504
|
+
});
|
|
1505
|
+
Object.defineProperty(exports, "validate", {
|
|
1506
|
+
enumerable: true,
|
|
1507
|
+
get: function() {
|
|
1508
|
+
return validate;
|
|
1509
|
+
}
|
|
1510
|
+
});
|
|
1511
|
+
Object.defineProperty(exports, "visualBrandAxis", {
|
|
1512
|
+
enumerable: true,
|
|
1513
|
+
get: function() {
|
|
1514
|
+
return visualBrandAxis;
|
|
1515
|
+
}
|
|
1516
|
+
});
|
|
1517
|
+
|
|
1518
|
+
//# sourceMappingURL=runner-CM9sF-C8.cjs.map
|