@arclabs561/ai-visual-test 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.secretsignore.example +20 -0
- package/CHANGELOG.md +360 -0
- package/CONTRIBUTING.md +63 -0
- package/DEPLOYMENT.md +80 -0
- package/LICENSE +22 -0
- package/README.md +142 -0
- package/SECURITY.md +108 -0
- package/api/health.js +34 -0
- package/api/validate.js +252 -0
- package/index.d.ts +1221 -0
- package/package.json +112 -0
- package/public/index.html +149 -0
- package/src/batch-optimizer.mjs +451 -0
- package/src/bias-detector.mjs +370 -0
- package/src/bias-mitigation.mjs +233 -0
- package/src/cache.mjs +433 -0
- package/src/config.mjs +268 -0
- package/src/constants.mjs +80 -0
- package/src/context-compressor.mjs +350 -0
- package/src/convenience.mjs +617 -0
- package/src/cost-tracker.mjs +257 -0
- package/src/cross-modal-consistency.mjs +170 -0
- package/src/data-extractor.mjs +232 -0
- package/src/dynamic-few-shot.mjs +140 -0
- package/src/dynamic-prompts.mjs +361 -0
- package/src/ensemble/index.mjs +53 -0
- package/src/ensemble-judge.mjs +366 -0
- package/src/error-handler.mjs +67 -0
- package/src/errors.mjs +167 -0
- package/src/experience-propagation.mjs +128 -0
- package/src/experience-tracer.mjs +487 -0
- package/src/explanation-manager.mjs +299 -0
- package/src/feedback-aggregator.mjs +248 -0
- package/src/game-goal-prompts.mjs +478 -0
- package/src/game-player.mjs +548 -0
- package/src/hallucination-detector.mjs +155 -0
- package/src/helpers/playwright.mjs +80 -0
- package/src/human-validation-manager.mjs +516 -0
- package/src/index.mjs +364 -0
- package/src/judge.mjs +929 -0
- package/src/latency-aware-batch-optimizer.mjs +192 -0
- package/src/load-env.mjs +159 -0
- package/src/logger.mjs +55 -0
- package/src/metrics.mjs +187 -0
- package/src/model-tier-selector.mjs +221 -0
- package/src/multi-modal/index.mjs +36 -0
- package/src/multi-modal-fusion.mjs +190 -0
- package/src/multi-modal.mjs +524 -0
- package/src/natural-language-specs.mjs +1071 -0
- package/src/pair-comparison.mjs +277 -0
- package/src/persona/index.mjs +42 -0
- package/src/persona-enhanced.mjs +200 -0
- package/src/persona-experience.mjs +572 -0
- package/src/position-counterbalance.mjs +140 -0
- package/src/prompt-composer.mjs +375 -0
- package/src/render-change-detector.mjs +583 -0
- package/src/research-enhanced-validation.mjs +436 -0
- package/src/retry.mjs +152 -0
- package/src/rubrics.mjs +231 -0
- package/src/score-tracker.mjs +277 -0
- package/src/smart-validator.mjs +447 -0
- package/src/spec-config.mjs +106 -0
- package/src/spec-templates.mjs +347 -0
- package/src/specs/index.mjs +38 -0
- package/src/temporal/index.mjs +102 -0
- package/src/temporal-adaptive.mjs +163 -0
- package/src/temporal-batch-optimizer.mjs +222 -0
- package/src/temporal-constants.mjs +69 -0
- package/src/temporal-context.mjs +49 -0
- package/src/temporal-decision-manager.mjs +271 -0
- package/src/temporal-decision.mjs +669 -0
- package/src/temporal-errors.mjs +58 -0
- package/src/temporal-note-pruner.mjs +173 -0
- package/src/temporal-preprocessor.mjs +543 -0
- package/src/temporal-prompt-formatter.mjs +219 -0
- package/src/temporal-validation.mjs +159 -0
- package/src/temporal.mjs +415 -0
- package/src/type-guards.mjs +311 -0
- package/src/uncertainty-reducer.mjs +470 -0
- package/src/utils/index.mjs +175 -0
- package/src/validation-framework.mjs +321 -0
- package/src/validation-result-normalizer.mjs +64 -0
- package/src/validation.mjs +243 -0
- package/src/validators/accessibility-programmatic.mjs +345 -0
- package/src/validators/accessibility-validator.mjs +223 -0
- package/src/validators/batch-validator.mjs +143 -0
- package/src/validators/hybrid-validator.mjs +268 -0
- package/src/validators/index.mjs +34 -0
- package/src/validators/prompt-builder.mjs +218 -0
- package/src/validators/rubric.mjs +85 -0
- package/src/validators/state-programmatic.mjs +260 -0
- package/src/validators/state-validator.mjs +291 -0
- package/vercel.json +27 -0
package/package.json
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@arclabs561/ai-visual-test",
|
|
3
|
+
"version": "0.5.1",
|
|
4
|
+
"description": "AI-powered visual testing framework for web applications using Vision Language Models",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "src/index.mjs",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./src/index.mjs",
|
|
9
|
+
"./validators": "./src/validators/index.mjs",
|
|
10
|
+
"./temporal": "./src/temporal/index.mjs",
|
|
11
|
+
"./multi-modal": "./src/multi-modal/index.mjs",
|
|
12
|
+
"./ensemble": "./src/ensemble/index.mjs",
|
|
13
|
+
"./persona": "./src/persona/index.mjs",
|
|
14
|
+
"./specs": "./src/specs/index.mjs",
|
|
15
|
+
"./utils": "./src/utils/index.mjs",
|
|
16
|
+
"./package.json": "./package.json"
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
"src/**/*.mjs",
|
|
20
|
+
"index.d.ts",
|
|
21
|
+
"api/**/*.js",
|
|
22
|
+
"public/**/*.html",
|
|
23
|
+
"vercel.json",
|
|
24
|
+
"README.md",
|
|
25
|
+
"CHANGELOG.md",
|
|
26
|
+
"CONTRIBUTING.md",
|
|
27
|
+
"DEPLOYMENT.md",
|
|
28
|
+
"SECURITY.md",
|
|
29
|
+
"LICENSE",
|
|
30
|
+
".secretsignore.example"
|
|
31
|
+
],
|
|
32
|
+
"scripts": {
|
|
33
|
+
"test": "node --test test/*.test.mjs",
|
|
34
|
+
"test:validation": "node --test test/validation-*.test.mjs",
|
|
35
|
+
"test:unit": "node --test test/unit/*.test.mjs",
|
|
36
|
+
"test:integration": "node --test test/integration/*.test.mjs",
|
|
37
|
+
"test:e2e": "node --test test/e2e/*.test.mjs",
|
|
38
|
+
"test:datasets": "node --test test/dataset-*.test.mjs",
|
|
39
|
+
"playwright:check": "node scripts/ensure-playwright.mjs",
|
|
40
|
+
"playwright:install": "node scripts/ensure-playwright.mjs --install",
|
|
41
|
+
"playwright:setup": "node scripts/ensure-playwright.mjs --install --install-browsers",
|
|
42
|
+
"annotate": "node evaluation/utils/invoke-human-annotation.mjs",
|
|
43
|
+
"annotate:quick": "node evaluation/utils/quick-start-annotation.mjs",
|
|
44
|
+
"annotate:full": "node evaluation/utils/start-human-annotation.mjs",
|
|
45
|
+
"validate:annotations": "node evaluation/utils/validate-annotation-quality.mjs",
|
|
46
|
+
"validate:dataset": "node evaluation/utils/validate-dataset-quality.mjs",
|
|
47
|
+
"match:vllm": "node evaluation/utils/match-annotations-with-vllm.mjs",
|
|
48
|
+
"datasets:download": "node evaluation/utils/download-all-datasets.mjs",
|
|
49
|
+
"datasets:parse": "node evaluation/utils/parse-all-datasets.mjs",
|
|
50
|
+
"datasets:setup": "npm run datasets:download && npm run datasets:parse",
|
|
51
|
+
"docs": "node scripts/generate-docs.mjs",
|
|
52
|
+
"lint": "echo 'No linter configured'",
|
|
53
|
+
"prepublishOnly": "npm test",
|
|
54
|
+
"check:secrets": "node scripts/detect-secrets.mjs",
|
|
55
|
+
"check:quality": "node node_modules/@arclabs561/hookwise/src/hooks/code-quality.mjs || echo 'Hookwise not available'",
|
|
56
|
+
"check:docs": "node node_modules/@arclabs561/hookwise/src/hooks/doc-bloat.mjs || echo 'Hookwise not available'",
|
|
57
|
+
"check:security": "npm run check:secrets",
|
|
58
|
+
"check:test-performance": "node node_modules/@arclabs561/hookwise/src/hooks/test-performance.mjs || node scripts/analyze-test-performance.mjs",
|
|
59
|
+
"check:commit": "node node_modules/@arclabs561/hookwise/src/hooks/commit-msg.mjs .git/COMMIT_EDITMSG || echo 'Hookwise not available'",
|
|
60
|
+
"check:all": "npm run check:secrets && npx hookwise garden && npm run check:test-performance",
|
|
61
|
+
"garden": "npx hookwise garden",
|
|
62
|
+
"garden:enhanced": "node scripts/enhanced-garden.mjs",
|
|
63
|
+
"deprecate:old": "node scripts/deprecate-old-package.mjs",
|
|
64
|
+
"garden:watch": "node scripts/watch-garden.mjs",
|
|
65
|
+
"test:performance": "npm run check:test-performance",
|
|
66
|
+
"test:slow": "npm test 2>&1 | grep -E '✔.*\\([0-9]+\\.[0-9]+ms\\)' | sort -t'(' -k2 -nr | head -20"
|
|
67
|
+
},
|
|
68
|
+
"keywords": [
|
|
69
|
+
"visual-testing",
|
|
70
|
+
"ai",
|
|
71
|
+
"vllm",
|
|
72
|
+
"vision-language-models",
|
|
73
|
+
"playwright",
|
|
74
|
+
"accessibility",
|
|
75
|
+
"ui-testing",
|
|
76
|
+
"screenshot-validation"
|
|
77
|
+
],
|
|
78
|
+
"author": "arclabs561 <henry@henrywallace.io>",
|
|
79
|
+
"license": "MIT",
|
|
80
|
+
"dependencies": {
|
|
81
|
+
"dotenv": "^16.4.5"
|
|
82
|
+
},
|
|
83
|
+
"peerDependencies": {
|
|
84
|
+
"@arclabs561/llm-utils": "*",
|
|
85
|
+
"@playwright/test": "^1.48.0"
|
|
86
|
+
},
|
|
87
|
+
"peerDependenciesMeta": {
|
|
88
|
+
"@arclabs561/llm-utils": {
|
|
89
|
+
"optional": true
|
|
90
|
+
},
|
|
91
|
+
"@playwright/test": {
|
|
92
|
+
"optional": true
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
"devDependencies": {
|
|
96
|
+
"@types/node": "^22.10.1",
|
|
97
|
+
"async-mutex": "0.5.0",
|
|
98
|
+
"fast-check": "4.3.0",
|
|
99
|
+
"proper-lockfile": "4.1.2"
|
|
100
|
+
},
|
|
101
|
+
"engines": {
|
|
102
|
+
"node": ">=18.0.0"
|
|
103
|
+
},
|
|
104
|
+
"repository": {
|
|
105
|
+
"type": "git",
|
|
106
|
+
"url": "git+https://github.com/arclabs561/ai-visual-test.git"
|
|
107
|
+
},
|
|
108
|
+
"homepage": "https://github.com/arclabs561/ai-visual-test#readme",
|
|
109
|
+
"bugs": {
|
|
110
|
+
"url": "https://github.com/arclabs561/ai-visual-test/issues"
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>VLLM Testing - Visual Validation API</title>
|
|
7
|
+
<style>
|
|
8
|
+
* {
|
|
9
|
+
margin: 0;
|
|
10
|
+
padding: 0;
|
|
11
|
+
box-sizing: border-box;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
body {
|
|
15
|
+
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
|
|
16
|
+
background: #0a0a0a;
|
|
17
|
+
color: #ffffff;
|
|
18
|
+
line-height: 1.6;
|
|
19
|
+
padding: 2rem;
|
|
20
|
+
max-width: 1200px;
|
|
21
|
+
margin: 0 auto;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
h1 {
|
|
25
|
+
font-size: 2.5rem;
|
|
26
|
+
margin-bottom: 1rem;
|
|
27
|
+
color: #ffffff;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
h2 {
|
|
31
|
+
font-size: 1.5rem;
|
|
32
|
+
margin-top: 2rem;
|
|
33
|
+
margin-bottom: 1rem;
|
|
34
|
+
color: #ffffff;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
.status {
|
|
38
|
+
display: inline-block;
|
|
39
|
+
padding: 0.5rem 1rem;
|
|
40
|
+
border-radius: 4px;
|
|
41
|
+
font-weight: bold;
|
|
42
|
+
margin-bottom: 2rem;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
.status.ok {
|
|
46
|
+
background: #00ff00;
|
|
47
|
+
color: #000000;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
.status.error {
|
|
51
|
+
background: #ff0000;
|
|
52
|
+
color: #ffffff;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
code {
|
|
56
|
+
background: #1a1a1a;
|
|
57
|
+
padding: 0.2rem 0.4rem;
|
|
58
|
+
border-radius: 3px;
|
|
59
|
+
font-family: 'Monaco', 'Courier New', monospace;
|
|
60
|
+
font-size: 0.9em;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
pre {
|
|
64
|
+
background: #1a1a1a;
|
|
65
|
+
padding: 1rem;
|
|
66
|
+
border-radius: 4px;
|
|
67
|
+
overflow-x: auto;
|
|
68
|
+
margin: 1rem 0;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
.endpoint {
|
|
72
|
+
background: #1a1a1a;
|
|
73
|
+
padding: 1.5rem;
|
|
74
|
+
border-radius: 4px;
|
|
75
|
+
margin: 1rem 0;
|
|
76
|
+
border-left: 4px solid #00ff00;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
.method {
|
|
80
|
+
display: inline-block;
|
|
81
|
+
padding: 0.25rem 0.5rem;
|
|
82
|
+
border-radius: 3px;
|
|
83
|
+
font-weight: bold;
|
|
84
|
+
margin-right: 0.5rem;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
.method.post {
|
|
88
|
+
background: #00ff00;
|
|
89
|
+
color: #000000;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
.method.get {
|
|
93
|
+
background: #0066ff;
|
|
94
|
+
color: #ffffff;
|
|
95
|
+
}
|
|
96
|
+
</style>
|
|
97
|
+
</head>
|
|
98
|
+
<body>
|
|
99
|
+
<h1>VLLM Testing API</h1>
|
|
100
|
+
<div id="status" class="status">Checking...</div>
|
|
101
|
+
|
|
102
|
+
<h2>API Endpoints</h2>
|
|
103
|
+
|
|
104
|
+
<div class="endpoint">
|
|
105
|
+
<span class="method post">POST</span>
|
|
106
|
+
<code>/api/validate</code>
|
|
107
|
+
<p style="margin-top: 1rem;">Validate a screenshot using Vision Language Models.</p>
|
|
108
|
+
<pre>{
|
|
109
|
+
"image": "base64-encoded-image",
|
|
110
|
+
"prompt": "Evaluate this screenshot...",
|
|
111
|
+
"context": {
|
|
112
|
+
"testType": "payment-screen",
|
|
113
|
+
"viewport": { "width": 1280, "height": 720 }
|
|
114
|
+
}
|
|
115
|
+
}</pre>
|
|
116
|
+
</div>
|
|
117
|
+
|
|
118
|
+
<div class="endpoint">
|
|
119
|
+
<span class="method get">GET</span>
|
|
120
|
+
<code>/api/health</code>
|
|
121
|
+
<p style="margin-top: 1rem;">Health check endpoint.</p>
|
|
122
|
+
</div>
|
|
123
|
+
|
|
124
|
+
<h2>Documentation</h2>
|
|
125
|
+
<p>See <a href="https://github.com/arclabs561/ai-visual-test" style="color: #00ff00;">GitHub repository</a> for full documentation.</p>
|
|
126
|
+
|
|
127
|
+
<script>
|
|
128
|
+
// Check health on load
|
|
129
|
+
fetch('/api/health')
|
|
130
|
+
.then(res => res.json())
|
|
131
|
+
.then(data => {
|
|
132
|
+
const statusEl = document.getElementById('status');
|
|
133
|
+
if (data.status === 'ok') {
|
|
134
|
+
statusEl.className = 'status ok';
|
|
135
|
+
statusEl.textContent = `✓ API Online - Provider: ${data.provider || 'none'}`;
|
|
136
|
+
} else {
|
|
137
|
+
statusEl.className = 'status error';
|
|
138
|
+
statusEl.textContent = '✗ API Error';
|
|
139
|
+
}
|
|
140
|
+
})
|
|
141
|
+
.catch(err => {
|
|
142
|
+
const statusEl = document.getElementById('status');
|
|
143
|
+
statusEl.className = 'status error';
|
|
144
|
+
statusEl.textContent = '✗ API Offline';
|
|
145
|
+
});
|
|
146
|
+
</script>
|
|
147
|
+
</body>
|
|
148
|
+
</html>
|
|
149
|
+
|