@arclabs561/ai-visual-test 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.secretsignore.example +20 -0
- package/CHANGELOG.md +360 -0
- package/CONTRIBUTING.md +63 -0
- package/DEPLOYMENT.md +80 -0
- package/LICENSE +22 -0
- package/README.md +142 -0
- package/SECURITY.md +108 -0
- package/api/health.js +34 -0
- package/api/validate.js +252 -0
- package/index.d.ts +1221 -0
- package/package.json +112 -0
- package/public/index.html +149 -0
- package/src/batch-optimizer.mjs +451 -0
- package/src/bias-detector.mjs +370 -0
- package/src/bias-mitigation.mjs +233 -0
- package/src/cache.mjs +433 -0
- package/src/config.mjs +268 -0
- package/src/constants.mjs +80 -0
- package/src/context-compressor.mjs +350 -0
- package/src/convenience.mjs +617 -0
- package/src/cost-tracker.mjs +257 -0
- package/src/cross-modal-consistency.mjs +170 -0
- package/src/data-extractor.mjs +232 -0
- package/src/dynamic-few-shot.mjs +140 -0
- package/src/dynamic-prompts.mjs +361 -0
- package/src/ensemble/index.mjs +53 -0
- package/src/ensemble-judge.mjs +366 -0
- package/src/error-handler.mjs +67 -0
- package/src/errors.mjs +167 -0
- package/src/experience-propagation.mjs +128 -0
- package/src/experience-tracer.mjs +487 -0
- package/src/explanation-manager.mjs +299 -0
- package/src/feedback-aggregator.mjs +248 -0
- package/src/game-goal-prompts.mjs +478 -0
- package/src/game-player.mjs +548 -0
- package/src/hallucination-detector.mjs +155 -0
- package/src/helpers/playwright.mjs +80 -0
- package/src/human-validation-manager.mjs +516 -0
- package/src/index.mjs +364 -0
- package/src/judge.mjs +929 -0
- package/src/latency-aware-batch-optimizer.mjs +192 -0
- package/src/load-env.mjs +159 -0
- package/src/logger.mjs +55 -0
- package/src/metrics.mjs +187 -0
- package/src/model-tier-selector.mjs +221 -0
- package/src/multi-modal/index.mjs +36 -0
- package/src/multi-modal-fusion.mjs +190 -0
- package/src/multi-modal.mjs +524 -0
- package/src/natural-language-specs.mjs +1071 -0
- package/src/pair-comparison.mjs +277 -0
- package/src/persona/index.mjs +42 -0
- package/src/persona-enhanced.mjs +200 -0
- package/src/persona-experience.mjs +572 -0
- package/src/position-counterbalance.mjs +140 -0
- package/src/prompt-composer.mjs +375 -0
- package/src/render-change-detector.mjs +583 -0
- package/src/research-enhanced-validation.mjs +436 -0
- package/src/retry.mjs +152 -0
- package/src/rubrics.mjs +231 -0
- package/src/score-tracker.mjs +277 -0
- package/src/smart-validator.mjs +447 -0
- package/src/spec-config.mjs +106 -0
- package/src/spec-templates.mjs +347 -0
- package/src/specs/index.mjs +38 -0
- package/src/temporal/index.mjs +102 -0
- package/src/temporal-adaptive.mjs +163 -0
- package/src/temporal-batch-optimizer.mjs +222 -0
- package/src/temporal-constants.mjs +69 -0
- package/src/temporal-context.mjs +49 -0
- package/src/temporal-decision-manager.mjs +271 -0
- package/src/temporal-decision.mjs +669 -0
- package/src/temporal-errors.mjs +58 -0
- package/src/temporal-note-pruner.mjs +173 -0
- package/src/temporal-preprocessor.mjs +543 -0
- package/src/temporal-prompt-formatter.mjs +219 -0
- package/src/temporal-validation.mjs +159 -0
- package/src/temporal.mjs +415 -0
- package/src/type-guards.mjs +311 -0
- package/src/uncertainty-reducer.mjs +470 -0
- package/src/utils/index.mjs +175 -0
- package/src/validation-framework.mjs +321 -0
- package/src/validation-result-normalizer.mjs +64 -0
- package/src/validation.mjs +243 -0
- package/src/validators/accessibility-programmatic.mjs +345 -0
- package/src/validators/accessibility-validator.mjs +223 -0
- package/src/validators/batch-validator.mjs +143 -0
- package/src/validators/hybrid-validator.mjs +268 -0
- package/src/validators/index.mjs +34 -0
- package/src/validators/prompt-builder.mjs +218 -0
- package/src/validators/rubric.mjs +85 -0
- package/src/validators/state-programmatic.mjs +260 -0
- package/src/validators/state-validator.mjs +291 -0
- package/vercel.json +27 -0
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Programmatic Accessibility Validator
|
|
3
|
+
*
|
|
4
|
+
* Fast, deterministic accessibility checks using DOM inspection.
|
|
5
|
+
* Use this when you have Playwright page access and need fast feedback (<100ms).
|
|
6
|
+
*
|
|
7
|
+
* For semantic evaluation (design principles, context-aware checks), use AccessibilityValidator (VLLM-based).
|
|
8
|
+
*
|
|
9
|
+
* Based on WCAG 2.1 contrast ratio algorithm: https://www.w3.org/WAI/WCAG21/Understanding/contrast-minimum.html
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { ValidationError } from '../errors.mjs';
|
|
13
|
+
import { assertString, assertNumber } from '../type-guards.mjs';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Parse RGB color string to [r, g, b] array
|
|
17
|
+
* Supports rgb(r, g, b), rgba(r, g, b, a), and hex (#rrggbb or #rgb) formats
|
|
18
|
+
*
|
|
19
|
+
* @param {string} rgb - Color string
|
|
20
|
+
* @returns {number[]} [r, g, b] array (0-255)
|
|
21
|
+
*/
|
|
22
|
+
function parseRgb(rgb) {
|
|
23
|
+
if (!rgb || typeof rgb !== 'string') {
|
|
24
|
+
return [255, 255, 255]; // Default to white
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Handle rgb(r, g, b) or rgba(r, g, b, a) format
|
|
28
|
+
const match = rgb.match(/\d+/g);
|
|
29
|
+
if (match && match.length >= 3) {
|
|
30
|
+
return match.slice(0, 3).map(Number);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Handle hex format (#rrggbb or #rgb)
|
|
34
|
+
if (rgb.startsWith('#')) {
|
|
35
|
+
const hex = rgb.slice(1);
|
|
36
|
+
if (hex.length === 3) {
|
|
37
|
+
return hex.split('').map(c => parseInt(c + c, 16));
|
|
38
|
+
}
|
|
39
|
+
if (hex.length === 6) {
|
|
40
|
+
return [
|
|
41
|
+
parseInt(hex.slice(0, 2), 16),
|
|
42
|
+
parseInt(hex.slice(2, 4), 16),
|
|
43
|
+
parseInt(hex.slice(4, 6), 16)
|
|
44
|
+
];
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return [255, 255, 255]; // Default to white
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Calculate relative luminance (WCAG algorithm)
|
|
53
|
+
*
|
|
54
|
+
* @param {number[]} rgb - [r, g, b] array (0-255)
|
|
55
|
+
* @returns {number} Relative luminance (0-1)
|
|
56
|
+
*/
|
|
57
|
+
function getLuminance(rgb) {
|
|
58
|
+
const [r, g, b] = rgb.map(val => {
|
|
59
|
+
val = val / 255;
|
|
60
|
+
return val <= 0.03928 ? val / 12.92 : Math.pow((val + 0.055) / 1.055, 2.4);
|
|
61
|
+
});
|
|
62
|
+
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Calculate contrast ratio between two colors (WCAG algorithm)
|
|
67
|
+
*
|
|
68
|
+
* @param {string} color1 - First color (rgb, rgba, or hex)
|
|
69
|
+
* @param {string} color2 - Second color (rgb, rgba, or hex)
|
|
70
|
+
* @returns {number} Contrast ratio (1.0 to 21.0+)
|
|
71
|
+
*/
|
|
72
|
+
export function getContrastRatio(color1, color2) {
|
|
73
|
+
const rgb1 = parseRgb(color1);
|
|
74
|
+
const rgb2 = parseRgb(color2);
|
|
75
|
+
|
|
76
|
+
const l1 = getLuminance(rgb1);
|
|
77
|
+
const l2 = getLuminance(rgb2);
|
|
78
|
+
|
|
79
|
+
const lighter = Math.max(l1, l2);
|
|
80
|
+
const darker = Math.min(l1, l2);
|
|
81
|
+
|
|
82
|
+
return (lighter + 0.05) / (darker + 0.05);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Check contrast ratio for an element
|
|
87
|
+
*
|
|
88
|
+
* @param {any} page - Playwright page object
|
|
89
|
+
* @param {string} selector - CSS selector for element
|
|
90
|
+
* @param {number} minRatio - Minimum required contrast ratio (default: 4.5 for WCAG-AA)
|
|
91
|
+
* @returns {Promise<{ratio: number, passes: boolean, foreground: string, background: string, foregroundRgb?: number[], backgroundRgb?: number[], error?: string}>}
|
|
92
|
+
* @throws {ValidationError} If page is not a valid Playwright Page object
|
|
93
|
+
*/
|
|
94
|
+
export async function checkElementContrast(page, selector, minRatio = 4.5) {
|
|
95
|
+
// Validate inputs
|
|
96
|
+
if (!page || typeof page.evaluate !== 'function') {
|
|
97
|
+
throw new ValidationError('checkElementContrast requires a Playwright Page object', {
|
|
98
|
+
received: typeof page,
|
|
99
|
+
hasEvaluate: typeof page?.evaluate === 'function'
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
assertString(selector, 'selector');
|
|
104
|
+
assertNumber(minRatio, 'minRatio');
|
|
105
|
+
|
|
106
|
+
if (minRatio < 1 || minRatio > 21) {
|
|
107
|
+
throw new ValidationError('minRatio must be between 1 and 21', { received: minRatio });
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const result = await page.evaluate(({ sel, min }) => {
|
|
111
|
+
const element = document.querySelector(sel);
|
|
112
|
+
if (!element) {
|
|
113
|
+
return { error: 'Element not found', selector: sel };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const style = window.getComputedStyle(element);
|
|
117
|
+
const color = style.color;
|
|
118
|
+
const bgColor = style.backgroundColor;
|
|
119
|
+
|
|
120
|
+
// Get effective background color from parent if element has transparent background
|
|
121
|
+
let effectiveBg = bgColor;
|
|
122
|
+
if (bgColor === 'rgba(0, 0, 0, 0)' || bgColor === 'transparent') {
|
|
123
|
+
let parent = element.parentElement;
|
|
124
|
+
while (parent && parent !== document.body) {
|
|
125
|
+
const parentStyle = window.getComputedStyle(parent);
|
|
126
|
+
const parentBg = parentStyle.backgroundColor;
|
|
127
|
+
if (parentBg !== 'rgba(0, 0, 0, 0)' && parentBg !== 'transparent') {
|
|
128
|
+
effectiveBg = parentBg;
|
|
129
|
+
break;
|
|
130
|
+
}
|
|
131
|
+
parent = parent.parentElement;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// If still transparent, check document.body
|
|
135
|
+
if ((effectiveBg === 'rgba(0, 0, 0, 0)' || effectiveBg === 'transparent') && document.body) {
|
|
136
|
+
const bodyStyle = window.getComputedStyle(document.body);
|
|
137
|
+
const bodyBg = bodyStyle.backgroundColor;
|
|
138
|
+
if (bodyBg !== 'rgba(0, 0, 0, 0)' && bodyBg !== 'transparent') {
|
|
139
|
+
effectiveBg = bodyBg;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Parse RGB values
|
|
145
|
+
const parseRgb = (rgb) => {
|
|
146
|
+
if (!rgb || typeof rgb !== 'string') return [255, 255, 255];
|
|
147
|
+
const match = rgb.match(/\d+/g);
|
|
148
|
+
return match && match.length >= 3 ? match.slice(0, 3).map(Number) : [255, 255, 255];
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
const fg = parseRgb(color);
|
|
152
|
+
const bg = parseRgb(effectiveBg);
|
|
153
|
+
|
|
154
|
+
// Calculate relative luminance (WCAG algorithm)
|
|
155
|
+
const getLuminance = (rgb) => {
|
|
156
|
+
const [r, g, b] = rgb.map(val => {
|
|
157
|
+
val = val / 255;
|
|
158
|
+
return val <= 0.03928 ? val / 12.92 : Math.pow((val + 0.055) / 1.055, 2.4);
|
|
159
|
+
});
|
|
160
|
+
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
const l1 = getLuminance(fg);
|
|
164
|
+
const l2 = getLuminance(bg);
|
|
165
|
+
const ratio = (Math.max(l1, l2) + 0.05) / (Math.min(l1, l2) + 0.05);
|
|
166
|
+
|
|
167
|
+
return {
|
|
168
|
+
ratio,
|
|
169
|
+
passes: ratio >= min,
|
|
170
|
+
foreground: color,
|
|
171
|
+
background: effectiveBg,
|
|
172
|
+
foregroundRgb: fg,
|
|
173
|
+
backgroundRgb: bg
|
|
174
|
+
};
|
|
175
|
+
}, { sel: selector, min: minRatio });
|
|
176
|
+
|
|
177
|
+
return result;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Check contrast for all text elements on page
|
|
182
|
+
*
|
|
183
|
+
* @param {any} page - Playwright page object
|
|
184
|
+
* @param {number} minRatio - Minimum required contrast ratio (default: 4.5 for WCAG-AA)
|
|
185
|
+
* @returns {Promise<{total: number, passing: number, failing: number, violations: Array<{element: string, ratio: string, required: number, foreground: string, background: string}>, elements?: Array}>}
|
|
186
|
+
* @throws {ValidationError} If page is not a valid Playwright Page object
|
|
187
|
+
*/
|
|
188
|
+
export async function checkAllTextContrast(page, minRatio = 4.5) {
|
|
189
|
+
// Validate inputs
|
|
190
|
+
if (!page || typeof page.evaluate !== 'function') {
|
|
191
|
+
throw new ValidationError('checkAllTextContrast requires a Playwright Page object', {
|
|
192
|
+
received: typeof page,
|
|
193
|
+
hasEvaluate: typeof page?.evaluate === 'function'
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
assertNumber(minRatio, 'minRatio');
|
|
198
|
+
|
|
199
|
+
if (minRatio < 1 || minRatio > 21) {
|
|
200
|
+
throw new ValidationError('minRatio must be between 1 and 21', { received: minRatio });
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const result = await page.evaluate((min) => {
|
|
204
|
+
const all = document.querySelectorAll('*');
|
|
205
|
+
const textElements = [];
|
|
206
|
+
const violations = [];
|
|
207
|
+
|
|
208
|
+
const parseRgb = (rgb) => {
|
|
209
|
+
if (!rgb || typeof rgb !== 'string') return [255, 255, 255];
|
|
210
|
+
const match = rgb.match(/\d+/g);
|
|
211
|
+
return match && match.length >= 3 ? match.slice(0, 3).map(Number) : [255, 255, 255];
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
const getLuminance = (rgb) => {
|
|
215
|
+
const [r, g, b] = rgb.map(val => {
|
|
216
|
+
val = val / 255;
|
|
217
|
+
return val <= 0.03928 ? val / 12.92 : Math.pow((val + 0.055) / 1.055, 2.4);
|
|
218
|
+
});
|
|
219
|
+
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
|
220
|
+
};
|
|
221
|
+
|
|
222
|
+
for (const el of all) {
|
|
223
|
+
const style = window.getComputedStyle(el);
|
|
224
|
+
const color = style.color;
|
|
225
|
+
const bgColor = style.backgroundColor;
|
|
226
|
+
|
|
227
|
+
// Check if element has text content
|
|
228
|
+
if (el.textContent && el.textContent.trim().length > 0 &&
|
|
229
|
+
color && color !== 'rgba(0, 0, 0, 0)' &&
|
|
230
|
+
bgColor && bgColor !== 'rgba(0, 0, 0, 0)') {
|
|
231
|
+
|
|
232
|
+
// Get effective background (handle transparent)
|
|
233
|
+
let effectiveBg = bgColor;
|
|
234
|
+
if (bgColor === 'rgba(0, 0, 0, 0)' || bgColor === 'transparent') {
|
|
235
|
+
let parent = el.parentElement;
|
|
236
|
+
while (parent && parent !== document.body) {
|
|
237
|
+
const parentStyle = window.getComputedStyle(parent);
|
|
238
|
+
const parentBg = parentStyle.backgroundColor;
|
|
239
|
+
if (parentBg !== 'rgba(0, 0, 0, 0)' && parentBg !== 'transparent') {
|
|
240
|
+
effectiveBg = parentBg;
|
|
241
|
+
break;
|
|
242
|
+
}
|
|
243
|
+
parent = parent.parentElement;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// If still transparent, check document.body
|
|
247
|
+
if ((effectiveBg === 'rgba(0, 0, 0, 0)' || effectiveBg === 'transparent') && document.body) {
|
|
248
|
+
const bodyStyle = window.getComputedStyle(document.body);
|
|
249
|
+
const bodyBg = bodyStyle.backgroundColor;
|
|
250
|
+
if (bodyBg !== 'rgba(0, 0, 0, 0)' && bodyBg !== 'transparent') {
|
|
251
|
+
effectiveBg = bodyBg;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const fg = parseRgb(color);
|
|
257
|
+
const bg = parseRgb(effectiveBg);
|
|
258
|
+
|
|
259
|
+
const l1 = getLuminance(fg);
|
|
260
|
+
const l2 = getLuminance(bg);
|
|
261
|
+
const ratio = (Math.max(l1, l2) + 0.05) / (Math.min(l1, l2) + 0.05);
|
|
262
|
+
|
|
263
|
+
const elementInfo = {
|
|
264
|
+
tag: el.tagName,
|
|
265
|
+
id: el.id,
|
|
266
|
+
className: el.className,
|
|
267
|
+
ratio,
|
|
268
|
+
passes: ratio >= min,
|
|
269
|
+
foreground: color,
|
|
270
|
+
background: effectiveBg
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
textElements.push(elementInfo);
|
|
274
|
+
|
|
275
|
+
if (!elementInfo.passes) {
|
|
276
|
+
violations.push({
|
|
277
|
+
element: `${el.tagName}${el.id ? '#' + el.id : ''}${el.className ? '.' + el.className.split(' ')[0] : ''}`,
|
|
278
|
+
ratio: ratio.toFixed(2),
|
|
279
|
+
required: min,
|
|
280
|
+
foreground: color,
|
|
281
|
+
background: effectiveBg
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
return {
|
|
288
|
+
total: textElements.length,
|
|
289
|
+
passing: textElements.filter(e => e.passes).length,
|
|
290
|
+
failing: textElements.filter(e => !e.passes).length,
|
|
291
|
+
violations: violations,
|
|
292
|
+
elements: textElements.slice(0, 20) // First 20 for debugging
|
|
293
|
+
};
|
|
294
|
+
}, minRatio);
|
|
295
|
+
|
|
296
|
+
return result;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Check keyboard navigation accessibility
|
|
301
|
+
*
|
|
302
|
+
* @param {any} page - Playwright page object
|
|
303
|
+
* @returns {Promise<{keyboardAccessible: boolean, focusableElements: number, violations: Array<{element: string, issue: string}>, focusableSelectors: string[]}>}
|
|
304
|
+
* @throws {ValidationError} If page is not a valid Playwright Page object
|
|
305
|
+
*/
|
|
306
|
+
export async function checkKeyboardNavigation(page) {
|
|
307
|
+
// Validate inputs
|
|
308
|
+
if (!page || typeof page.evaluate !== 'function') {
|
|
309
|
+
throw new ValidationError('checkKeyboardNavigation requires a Playwright Page object', {
|
|
310
|
+
received: typeof page,
|
|
311
|
+
hasEvaluate: typeof page?.evaluate === 'function'
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const result = await page.evaluate(() => {
|
|
316
|
+
const focusableSelectors = [
|
|
317
|
+
'a[href]',
|
|
318
|
+
'button:not([disabled])',
|
|
319
|
+
'input:not([disabled])',
|
|
320
|
+
'select:not([disabled])',
|
|
321
|
+
'textarea:not([disabled])',
|
|
322
|
+
'[tabindex]:not([tabindex="-1"])'
|
|
323
|
+
];
|
|
324
|
+
|
|
325
|
+
const focusableElements = Array.from(document.querySelectorAll(focusableSelectors.join(', ')));
|
|
326
|
+
const violations = [];
|
|
327
|
+
|
|
328
|
+
// Check for missing focus indicators
|
|
329
|
+
focusableElements.forEach(el => {
|
|
330
|
+
const style = window.getComputedStyle(el, ':focus');
|
|
331
|
+
// Note: :focus pseudo-class can't be checked directly in evaluate
|
|
332
|
+
// This is a basic check - full focus indicator check would require interaction
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
return {
|
|
336
|
+
keyboardAccessible: focusableElements.length > 0,
|
|
337
|
+
focusableElements: focusableElements.length,
|
|
338
|
+
violations: violations,
|
|
339
|
+
focusableSelectors: focusableSelectors
|
|
340
|
+
};
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
return result;
|
|
344
|
+
}
|
|
345
|
+
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Accessibility Validator
|
|
3
|
+
*
|
|
4
|
+
* Configurable accessibility validation with WCAG standards support
|
|
5
|
+
*
|
|
6
|
+
* Provides:
|
|
7
|
+
* - Configurable contrast requirements (WCAG-AA, WCAG-AAA, custom)
|
|
8
|
+
* - Zero tolerance enforcement
|
|
9
|
+
* - Violation detection and reporting
|
|
10
|
+
* - Extensible via plugins
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { validateScreenshot } from '../judge.mjs';
|
|
14
|
+
import { ValidationError } from '../errors.mjs';
|
|
15
|
+
import { assertString } from '../type-guards.mjs';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Accessibility validator with configurable standards
|
|
19
|
+
*/
|
|
20
|
+
export class AccessibilityValidator {
|
|
21
|
+
constructor(options = {}) {
|
|
22
|
+
// Validate minContrast
|
|
23
|
+
if (options.minContrast !== undefined) {
|
|
24
|
+
if (typeof options.minContrast !== 'number' || options.minContrast < 1 || isNaN(options.minContrast)) {
|
|
25
|
+
throw new ValidationError(
|
|
26
|
+
'minContrast must be a number >= 1',
|
|
27
|
+
{ received: options.minContrast }
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
this.minContrast = options.minContrast;
|
|
31
|
+
} else {
|
|
32
|
+
this.minContrast = 4.5; // WCAG AA default, configurable
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Validate standards
|
|
36
|
+
if (options.standards !== undefined) {
|
|
37
|
+
if (!Array.isArray(options.standards)) {
|
|
38
|
+
throw new ValidationError(
|
|
39
|
+
'standards must be an array',
|
|
40
|
+
{ received: typeof options.standards }
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
this.standards = options.standards;
|
|
44
|
+
} else {
|
|
45
|
+
this.standards = ['WCAG-AA']; // Can be WCAG-AA, WCAG-AAA, custom
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
this.zeroTolerance = options.zeroTolerance || false; // Whether violations cause instant fail
|
|
49
|
+
this.validateScreenshot = options.validateScreenshot || validateScreenshot;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Static method for quick validation without instantiation
|
|
54
|
+
*
|
|
55
|
+
* @param {string} screenshotPath - Path to screenshot
|
|
56
|
+
* @param {object} options - Validation options (minContrast, standards, etc.)
|
|
57
|
+
* @returns {Promise<object>} Validation result with violations and contrast info
|
|
58
|
+
*/
|
|
59
|
+
static async validate(screenshotPath, options = {}) {
|
|
60
|
+
const validator = new AccessibilityValidator(options);
|
|
61
|
+
return validator.validateAccessibility(screenshotPath, options);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Validate accessibility with configurable standards
|
|
66
|
+
*
|
|
67
|
+
* @param {string | string[]} screenshotPath - Path to screenshot(s) - supports multi-image for comparison
|
|
68
|
+
* @param {object} options - Validation options
|
|
69
|
+
*/
|
|
70
|
+
async validateAccessibility(screenshotPath, options = {}) {
|
|
71
|
+
// Input validation - support both single and array
|
|
72
|
+
const isArray = Array.isArray(screenshotPath);
|
|
73
|
+
if (!isArray) {
|
|
74
|
+
assertString(screenshotPath, 'screenshotPath');
|
|
75
|
+
} else {
|
|
76
|
+
screenshotPath.forEach((path, i) => {
|
|
77
|
+
assertString(path, `screenshotPath[${i}]`);
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (options.minContrast !== undefined && (typeof options.minContrast !== 'number' || options.minContrast < 1)) {
|
|
82
|
+
throw new ValidationError(
|
|
83
|
+
'minContrast must be a number >= 1',
|
|
84
|
+
{ received: options.minContrast }
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Allow custom prompt override
|
|
89
|
+
const prompt = options.customPrompt || this.buildAccessibilityPrompt(options);
|
|
90
|
+
|
|
91
|
+
try {
|
|
92
|
+
// Pass through all validateScreenshot options
|
|
93
|
+
const screenshotOptions = {
|
|
94
|
+
testType: options.testType || 'accessibility',
|
|
95
|
+
minContrast: this.minContrast,
|
|
96
|
+
standards: this.standards,
|
|
97
|
+
...options,
|
|
98
|
+
// Explicitly pass through common options
|
|
99
|
+
useCache: options.useCache !== undefined ? options.useCache : undefined,
|
|
100
|
+
timeout: options.timeout,
|
|
101
|
+
provider: options.provider,
|
|
102
|
+
viewport: options.viewport
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const result = await this.validateScreenshot(screenshotPath, prompt, screenshotOptions);
|
|
106
|
+
|
|
107
|
+
// Check for violations
|
|
108
|
+
const violations = this.detectViolations(result);
|
|
109
|
+
|
|
110
|
+
// Auto-fail if zero tolerance enabled
|
|
111
|
+
const passes = this.zeroTolerance
|
|
112
|
+
? violations.zeroTolerance.length === 0 && violations.critical.length === 0
|
|
113
|
+
: true; // Don't auto-fail if zero tolerance disabled
|
|
114
|
+
|
|
115
|
+
return {
|
|
116
|
+
...result,
|
|
117
|
+
violations,
|
|
118
|
+
passes,
|
|
119
|
+
contrastCheck: this.extractContrastInfo(result),
|
|
120
|
+
standards: this.standards
|
|
121
|
+
};
|
|
122
|
+
} catch (error) {
|
|
123
|
+
// Re-throw ValidationError as-is, wrap others
|
|
124
|
+
if (error instanceof ValidationError) {
|
|
125
|
+
throw error;
|
|
126
|
+
}
|
|
127
|
+
throw new ValidationError(
|
|
128
|
+
`Accessibility validation failed: ${error.message}`,
|
|
129
|
+
{ screenshotPath, standards: this.standards, originalError: error.message }
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Build accessibility validation prompt
|
|
136
|
+
*/
|
|
137
|
+
buildAccessibilityPrompt(options = {}) {
|
|
138
|
+
const failText = this.zeroTolerance
|
|
139
|
+
? `ZERO TOLERANCE (AUTOMATIC FAIL):
|
|
140
|
+
- Contrast <${this.minContrast}:1 for ANY critical text = INSTANT FAIL
|
|
141
|
+
- Contrast <${this.minContrast}:1 for interactive elements = INSTANT FAIL
|
|
142
|
+
- No keyboard navigation = INSTANT FAIL
|
|
143
|
+
- No screen reader support = INSTANT FAIL`
|
|
144
|
+
: `REQUIREMENTS:
|
|
145
|
+
- Contrast should be ≥${this.minContrast}:1 for all text
|
|
146
|
+
- All interactive elements should be keyboard accessible
|
|
147
|
+
- Screen reader compatibility required`;
|
|
148
|
+
|
|
149
|
+
return `Accessibility validation (${this.standards.join(', ')}):
|
|
150
|
+
|
|
151
|
+
${failText}
|
|
152
|
+
|
|
153
|
+
STANDARDS:
|
|
154
|
+
${this.standards.map(s => `- ${s}`).join('\n')}
|
|
155
|
+
|
|
156
|
+
REQUIREMENTS:
|
|
157
|
+
1. All text must have ≥${this.minContrast}:1 contrast ratio
|
|
158
|
+
2. All interactive elements must be keyboard accessible
|
|
159
|
+
3. All images must have alt text
|
|
160
|
+
4. Semantic HTML structure required
|
|
161
|
+
5. Focus indicators must be visible
|
|
162
|
+
|
|
163
|
+
Return detailed assessment with:
|
|
164
|
+
- Contrast ratios for all text elements
|
|
165
|
+
- Keyboard navigation status
|
|
166
|
+
- Screen reader compatibility
|
|
167
|
+
- WCAG compliance level
|
|
168
|
+
- List of violations`;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Detect accessibility violations
|
|
173
|
+
*/
|
|
174
|
+
detectViolations(result) {
|
|
175
|
+
const zeroTolerance = [];
|
|
176
|
+
const critical = [];
|
|
177
|
+
const warnings = [];
|
|
178
|
+
|
|
179
|
+
const text = (result.reasoning || result.assessment || '').toLowerCase();
|
|
180
|
+
|
|
181
|
+
// Check for contrast violations
|
|
182
|
+
if (text.includes(`contrast <${this.minContrast}`) || text.includes('contrast too low')) {
|
|
183
|
+
(this.zeroTolerance ? zeroTolerance : critical).push('Contrast below minimum requirement');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// Check for keyboard navigation
|
|
187
|
+
if (text.includes('no keyboard') || text.includes('keyboard inaccessible')) {
|
|
188
|
+
(this.zeroTolerance ? zeroTolerance : critical).push('Keyboard navigation missing');
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Check for screen reader
|
|
192
|
+
if (text.includes('no screen reader') || text.includes('screen reader incompatible')) {
|
|
193
|
+
(this.zeroTolerance ? zeroTolerance : critical).push('Screen reader support missing');
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return { zeroTolerance, critical, warnings };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Extract contrast information from result
|
|
201
|
+
*/
|
|
202
|
+
extractContrastInfo(result) {
|
|
203
|
+
const text = result.reasoning || result.assessment || '';
|
|
204
|
+
const contrastMatches = text.match(/(\d+(?:\.\d+)?):1/g);
|
|
205
|
+
|
|
206
|
+
if (!contrastMatches || contrastMatches.length === 0) {
|
|
207
|
+
return {
|
|
208
|
+
ratios: [],
|
|
209
|
+
minRatio: null,
|
|
210
|
+
meetsRequirement: null
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const ratios = contrastMatches.map(m => parseFloat(m)).filter(n => !isNaN(n) && isFinite(n));
|
|
215
|
+
|
|
216
|
+
return {
|
|
217
|
+
ratios: contrastMatches,
|
|
218
|
+
minRatio: ratios.length > 0 ? Math.min(...ratios) : null,
|
|
219
|
+
meetsRequirement: ratios.length > 0 ? ratios.every(r => r >= this.minContrast) : null
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch Validator
|
|
3
|
+
*
|
|
4
|
+
* Enhanced BatchOptimizer with cost tracking and statistics
|
|
5
|
+
*
|
|
6
|
+
* Provides:
|
|
7
|
+
* - All BatchOptimizer functionality
|
|
8
|
+
* - Cost tracking integration
|
|
9
|
+
* - Performance statistics
|
|
10
|
+
* - Success rate tracking
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { BatchOptimizer } from '../batch-optimizer.mjs';
|
|
14
|
+
import { getCostTracker } from '../cost-tracker.mjs';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Batch validator with cost tracking
|
|
18
|
+
*/
|
|
19
|
+
export class BatchValidator extends BatchOptimizer {
|
|
20
|
+
constructor(options = {}) {
|
|
21
|
+
super({
|
|
22
|
+
maxConcurrency: options.maxConcurrency || 5,
|
|
23
|
+
batchSize: options.batchSize || 3,
|
|
24
|
+
cacheEnabled: options.cacheEnabled !== false,
|
|
25
|
+
...options
|
|
26
|
+
});
|
|
27
|
+
this.costTracker = getCostTracker();
|
|
28
|
+
this.trackCosts = options.trackCosts !== false;
|
|
29
|
+
this.trackStats = options.trackStats !== false;
|
|
30
|
+
this.stats = {
|
|
31
|
+
totalRequests: 0,
|
|
32
|
+
totalDuration: 0,
|
|
33
|
+
successfulRequests: 0,
|
|
34
|
+
failedRequests: 0,
|
|
35
|
+
minDuration: Infinity,
|
|
36
|
+
maxDuration: 0
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Validate multiple screenshots with cost tracking
|
|
42
|
+
*/
|
|
43
|
+
async batchValidate(screenshots, prompt, context = {}) {
|
|
44
|
+
const startTime = Date.now();
|
|
45
|
+
|
|
46
|
+
const results = await super.batchValidate(screenshots, prompt, context);
|
|
47
|
+
|
|
48
|
+
const duration = Date.now() - startTime;
|
|
49
|
+
|
|
50
|
+
// Track costs
|
|
51
|
+
if (this.trackCosts && this.costTracker) {
|
|
52
|
+
const screenshotsArray = Array.isArray(screenshots) ? screenshots : [screenshots];
|
|
53
|
+
results.forEach((result, index) => {
|
|
54
|
+
if (result.estimatedCost) {
|
|
55
|
+
try {
|
|
56
|
+
this.costTracker.recordCost({
|
|
57
|
+
provider: result.provider,
|
|
58
|
+
cost: result.estimatedCost.total || 0,
|
|
59
|
+
tokens: result.estimatedCost.tokens || 0,
|
|
60
|
+
testType: context.testType || 'batch',
|
|
61
|
+
screenshot: screenshotsArray[index]
|
|
62
|
+
});
|
|
63
|
+
} catch (error) {
|
|
64
|
+
// Silently fail cost tracking to avoid breaking validation
|
|
65
|
+
// Could log warning in production
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Track stats
|
|
72
|
+
if (this.trackStats) {
|
|
73
|
+
this.stats.totalRequests += results.length;
|
|
74
|
+
this.stats.totalDuration += duration;
|
|
75
|
+
this.stats.minDuration = Math.min(this.stats.minDuration, duration);
|
|
76
|
+
this.stats.maxDuration = Math.max(this.stats.maxDuration, duration);
|
|
77
|
+
|
|
78
|
+
results.forEach(result => {
|
|
79
|
+
if (result.error) {
|
|
80
|
+
this.stats.failedRequests++;
|
|
81
|
+
} else {
|
|
82
|
+
this.stats.successfulRequests++;
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
results,
|
|
89
|
+
stats: this.trackStats ? {
|
|
90
|
+
total: screenshots.length,
|
|
91
|
+
passed: results.filter(r => (r.score || 0) >= (context.passingScore || 7)).length,
|
|
92
|
+
failed: results.filter(r => (r.score || 0) < (context.passingScore || 7)).length,
|
|
93
|
+
duration,
|
|
94
|
+
costStats: this.trackCosts && this.costTracker ? this.costTracker.getStats() : null,
|
|
95
|
+
performance: this.trackStats ? {
|
|
96
|
+
totalRequests: this.stats.totalRequests,
|
|
97
|
+
avgDuration: this.stats.totalRequests > 0 ? this.stats.totalDuration / this.stats.totalRequests : 0,
|
|
98
|
+
minDuration: this.stats.minDuration === Infinity ? 0 : this.stats.minDuration,
|
|
99
|
+
maxDuration: this.stats.maxDuration,
|
|
100
|
+
successRate: this.stats.totalRequests > 0 ? this.stats.successfulRequests / this.stats.totalRequests : 0
|
|
101
|
+
} : null
|
|
102
|
+
} : null
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Get cost statistics
|
|
108
|
+
*/
|
|
109
|
+
getCostStats() {
|
|
110
|
+
if (!this.costTracker) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
return this.costTracker.getStats();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Get performance statistics
|
|
118
|
+
*/
|
|
119
|
+
getPerformanceStats() {
|
|
120
|
+
return {
|
|
121
|
+
totalRequests: this.stats.totalRequests,
|
|
122
|
+
avgDuration: this.stats.totalRequests > 0 ? this.stats.totalDuration / this.stats.totalRequests : 0,
|
|
123
|
+
minDuration: this.stats.minDuration === Infinity ? 0 : this.stats.minDuration,
|
|
124
|
+
maxDuration: this.stats.maxDuration,
|
|
125
|
+
successRate: this.stats.totalRequests > 0 ? this.stats.successfulRequests / this.stats.totalRequests : 0
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Reset statistics
|
|
131
|
+
*/
|
|
132
|
+
resetStats() {
|
|
133
|
+
this.stats = {
|
|
134
|
+
totalRequests: 0,
|
|
135
|
+
totalDuration: 0,
|
|
136
|
+
successfulRequests: 0,
|
|
137
|
+
failedRequests: 0,
|
|
138
|
+
minDuration: Infinity,
|
|
139
|
+
maxDuration: 0
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|