@veraxhq/verax 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +123 -88
- package/bin/verax.js +11 -452
- package/package.json +14 -36
- package/src/cli/commands/default.js +523 -0
- package/src/cli/commands/doctor.js +165 -0
- package/src/cli/commands/inspect.js +109 -0
- package/src/cli/commands/run.js +402 -0
- package/src/cli/entry.js +196 -0
- package/src/cli/util/atomic-write.js +37 -0
- package/src/cli/util/detection-engine.js +296 -0
- package/src/cli/util/env-url.js +33 -0
- package/src/cli/util/errors.js +44 -0
- package/src/cli/util/events.js +34 -0
- package/src/cli/util/expectation-extractor.js +378 -0
- package/src/cli/util/findings-writer.js +31 -0
- package/src/cli/util/idgen.js +87 -0
- package/src/cli/util/learn-writer.js +39 -0
- package/src/cli/util/observation-engine.js +366 -0
- package/src/cli/util/observe-writer.js +25 -0
- package/src/cli/util/paths.js +29 -0
- package/src/cli/util/project-discovery.js +277 -0
- package/src/cli/util/project-writer.js +26 -0
- package/src/cli/util/redact.js +128 -0
- package/src/cli/util/run-id.js +30 -0
- package/src/cli/util/summary-writer.js +32 -0
- package/src/verax/cli/ci-summary.js +35 -0
- package/src/verax/cli/context-explanation.js +89 -0
- package/src/verax/cli/doctor.js +277 -0
- package/src/verax/cli/error-normalizer.js +154 -0
- package/src/verax/cli/explain-output.js +105 -0
- package/src/verax/cli/finding-explainer.js +130 -0
- package/src/verax/cli/init.js +237 -0
- package/src/verax/cli/run-overview.js +163 -0
- package/src/verax/cli/url-safety.js +101 -0
- package/src/verax/cli/wizard.js +98 -0
- package/src/verax/cli/zero-findings-explainer.js +57 -0
- package/src/verax/cli/zero-interaction-explainer.js +127 -0
- package/src/verax/core/action-classifier.js +86 -0
- package/src/verax/core/budget-engine.js +218 -0
- package/src/verax/core/canonical-outcomes.js +157 -0
- package/src/verax/core/decision-snapshot.js +335 -0
- package/src/verax/core/determinism-model.js +403 -0
- package/src/verax/core/incremental-store.js +237 -0
- package/src/verax/core/invariants.js +356 -0
- package/src/verax/core/promise-model.js +230 -0
- package/src/verax/core/replay-validator.js +350 -0
- package/src/verax/core/replay.js +222 -0
- package/src/verax/core/run-id.js +175 -0
- package/src/verax/core/run-manifest.js +99 -0
- package/src/verax/core/silence-impact.js +369 -0
- package/src/verax/core/silence-model.js +521 -0
- package/src/verax/detect/comparison.js +2 -34
- package/src/verax/detect/confidence-engine.js +764 -329
- package/src/verax/detect/detection-engine.js +293 -0
- package/src/verax/detect/evidence-index.js +177 -0
- package/src/verax/detect/expectation-model.js +194 -172
- package/src/verax/detect/explanation-helpers.js +187 -0
- package/src/verax/detect/finding-detector.js +450 -0
- package/src/verax/detect/findings-writer.js +44 -8
- package/src/verax/detect/flow-detector.js +366 -0
- package/src/verax/detect/index.js +172 -286
- package/src/verax/detect/interactive-findings.js +613 -0
- package/src/verax/detect/signal-mapper.js +308 -0
- package/src/verax/detect/verdict-engine.js +563 -0
- package/src/verax/evidence-index-writer.js +61 -0
- package/src/verax/index.js +90 -14
- package/src/verax/intel/effect-detector.js +368 -0
- package/src/verax/intel/handler-mapper.js +249 -0
- package/src/verax/intel/index.js +281 -0
- package/src/verax/intel/route-extractor.js +280 -0
- package/src/verax/intel/ts-program.js +256 -0
- package/src/verax/intel/vue-navigation-extractor.js +579 -0
- package/src/verax/intel/vue-router-extractor.js +323 -0
- package/src/verax/learn/action-contract-extractor.js +335 -101
- package/src/verax/learn/ast-contract-extractor.js +95 -5
- package/src/verax/learn/flow-extractor.js +172 -0
- package/src/verax/learn/manifest-writer.js +97 -47
- package/src/verax/learn/project-detector.js +40 -0
- package/src/verax/learn/route-extractor.js +27 -96
- package/src/verax/learn/state-extractor.js +212 -0
- package/src/verax/learn/static-extractor-navigation.js +114 -0
- package/src/verax/learn/static-extractor-validation.js +88 -0
- package/src/verax/learn/static-extractor.js +112 -4
- package/src/verax/learn/truth-assessor.js +24 -21
- package/src/verax/observe/aria-sensor.js +211 -0
- package/src/verax/observe/browser.js +10 -5
- package/src/verax/observe/console-sensor.js +1 -17
- package/src/verax/observe/domain-boundary.js +10 -1
- package/src/verax/observe/expectation-executor.js +512 -0
- package/src/verax/observe/flow-matcher.js +143 -0
- package/src/verax/observe/focus-sensor.js +196 -0
- package/src/verax/observe/human-driver.js +643 -275
- package/src/verax/observe/index.js +908 -27
- package/src/verax/observe/index.js.backup +1 -0
- package/src/verax/observe/interaction-discovery.js +365 -14
- package/src/verax/observe/interaction-runner.js +563 -198
- package/src/verax/observe/loading-sensor.js +139 -0
- package/src/verax/observe/navigation-sensor.js +255 -0
- package/src/verax/observe/network-sensor.js +55 -7
- package/src/verax/observe/observed-expectation-deriver.js +186 -0
- package/src/verax/observe/observed-expectation.js +305 -0
- package/src/verax/observe/page-frontier.js +234 -0
- package/src/verax/observe/settle.js +37 -17
- package/src/verax/observe/state-sensor.js +389 -0
- package/src/verax/observe/timing-sensor.js +228 -0
- package/src/verax/observe/traces-writer.js +61 -20
- package/src/verax/observe/ui-signal-sensor.js +136 -17
- package/src/verax/scan-summary-writer.js +77 -15
- package/src/verax/shared/artifact-manager.js +110 -8
- package/src/verax/shared/budget-profiles.js +136 -0
- package/src/verax/shared/ci-detection.js +39 -0
- package/src/verax/shared/config-loader.js +170 -0
- package/src/verax/shared/dynamic-route-utils.js +218 -0
- package/src/verax/shared/expectation-coverage.js +44 -0
- package/src/verax/shared/expectation-prover.js +81 -0
- package/src/verax/shared/expectation-tracker.js +201 -0
- package/src/verax/shared/expectations-writer.js +60 -0
- package/src/verax/shared/first-run.js +44 -0
- package/src/verax/shared/progress-reporter.js +171 -0
- package/src/verax/shared/retry-policy.js +14 -1
- package/src/verax/shared/root-artifacts.js +49 -0
- package/src/verax/shared/scan-budget.js +86 -0
- package/src/verax/shared/url-normalizer.js +162 -0
- package/src/verax/shared/zip-artifacts.js +65 -0
- package/src/verax/validate/context-validator.js +244 -0
- package/src/verax/validate/context-validator.js.bak +0 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Budget Engine
|
|
3
|
+
* Allocates adaptive interaction budgets per route/page based on:
|
|
4
|
+
* - Route criticality (has expectations or not)
|
|
5
|
+
* - Number of routes in project
|
|
6
|
+
* - Number of expectations per route
|
|
7
|
+
*
|
|
8
|
+
* Deterministic and reproducible.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export class BudgetEngine {
|
|
12
|
+
constructor(options = {}) {
|
|
13
|
+
this.baseBudgetPerRoute = options.baseBudgetPerRoute || 30;
|
|
14
|
+
this.criticalRouteMultiplier = options.criticalRouteMultiplier || 2.0;
|
|
15
|
+
this.nonCriticalRouteMultiplier = options.nonCriticalRouteMultiplier || 0.5;
|
|
16
|
+
this.expectationMultiplier = options.expectationMultiplier || 1.5;
|
|
17
|
+
this.minBudget = options.minBudget || 5;
|
|
18
|
+
this.maxBudget = options.maxBudget || 100;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Allocate budget for a single route
|
|
23
|
+
* @param {Object} route - Route object with url, interactions
|
|
24
|
+
* @param {Array} expectations - Expectations for this route
|
|
25
|
+
* @return {Object} Budget allocation { budget, isCritical, reason }
|
|
26
|
+
*/
|
|
27
|
+
allocateBudgetForRoute(route, expectations = []) {
|
|
28
|
+
const isCritical = expectations.length > 0;
|
|
29
|
+
let budget = this.baseBudgetPerRoute;
|
|
30
|
+
|
|
31
|
+
if (isCritical) {
|
|
32
|
+
// Critical routes with expectations get higher budget
|
|
33
|
+
budget = Math.floor(budget * this.criticalRouteMultiplier);
|
|
34
|
+
|
|
35
|
+
// Additional budget boost for routes with many expectations
|
|
36
|
+
if (expectations.length > 3) {
|
|
37
|
+
budget = Math.floor(budget * this.expectationMultiplier);
|
|
38
|
+
}
|
|
39
|
+
} else {
|
|
40
|
+
// Non-critical routes get reduced budget
|
|
41
|
+
budget = Math.floor(budget * this.nonCriticalRouteMultiplier);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Clamp to min/max
|
|
45
|
+
budget = Math.max(this.minBudget, Math.min(this.maxBudget, budget));
|
|
46
|
+
|
|
47
|
+
return {
|
|
48
|
+
budget,
|
|
49
|
+
isCritical,
|
|
50
|
+
reason: isCritical
|
|
51
|
+
? `critical_route_${expectations.length}_expectations`
|
|
52
|
+
: 'non_critical_route',
|
|
53
|
+
routeUrl: route.url || route.path || 'unknown'
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Allocate budgets for all routes deterministically
|
|
59
|
+
* @param {Array} routes - Array of route objects
|
|
60
|
+
* @param {Array} allExpectations - All expectations across routes
|
|
61
|
+
* @return {Array} Array of budget allocations sorted by URL for stability
|
|
62
|
+
*/
|
|
63
|
+
allocateBudgets(routes, allExpectations = []) {
|
|
64
|
+
// Group expectations by route
|
|
65
|
+
const expectationsByRoute = new Map();
|
|
66
|
+
for (const exp of allExpectations) {
|
|
67
|
+
const routeUrl = exp.expectedRoute || exp.fromPath || '/';
|
|
68
|
+
if (!expectationsByRoute.has(routeUrl)) {
|
|
69
|
+
expectationsByRoute.set(routeUrl, []);
|
|
70
|
+
}
|
|
71
|
+
expectationsByRoute.get(routeUrl).push(exp);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Allocate budget for each route
|
|
75
|
+
const allocations = [];
|
|
76
|
+
for (const route of routes) {
|
|
77
|
+
const routeUrl = route.url || route.path || '/';
|
|
78
|
+
const expectations = expectationsByRoute.get(routeUrl) || [];
|
|
79
|
+
const allocation = this.allocateBudgetForRoute(route, expectations);
|
|
80
|
+
allocations.push({
|
|
81
|
+
...allocation,
|
|
82
|
+
routeUrl: routeUrl
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Sort deterministically by routeUrl for stable ordering
|
|
87
|
+
allocations.sort((a, b) => {
|
|
88
|
+
return (a.routeUrl || '').localeCompare(b.routeUrl || '');
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
return allocations;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Compute total budget across all routes
|
|
96
|
+
* @param {Array} routes - Array of route objects
|
|
97
|
+
* @param {Array} allExpectations - All expectations across routes
|
|
98
|
+
* @return {Object} Total budget stats
|
|
99
|
+
*/
|
|
100
|
+
computeTotalBudget(routes, allExpectations = []) {
|
|
101
|
+
const allocations = this.allocateBudgets(routes, allExpectations);
|
|
102
|
+
|
|
103
|
+
const totalBudget = allocations.reduce((sum, alloc) => sum + alloc.budget, 0);
|
|
104
|
+
const criticalRoutes = allocations.filter(a => a.isCritical).length;
|
|
105
|
+
const nonCriticalRoutes = allocations.filter(a => !a.isCritical).length;
|
|
106
|
+
|
|
107
|
+
return {
|
|
108
|
+
totalBudget,
|
|
109
|
+
totalRoutes: routes.length,
|
|
110
|
+
criticalRoutes,
|
|
111
|
+
nonCriticalRoutes,
|
|
112
|
+
averageBudgetPerRoute: routes.length > 0 ? Math.round(totalBudget / routes.length) : 0,
|
|
113
|
+
allocations
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Get budget for a specific route URL
|
|
119
|
+
* @param {string} routeUrl - Route URL to query
|
|
120
|
+
* @param {Array} allocations - Pre-computed allocations
|
|
121
|
+
* @return {number|null} Budget or null if not found
|
|
122
|
+
*/
|
|
123
|
+
getBudgetForRoute(routeUrl, allocations) {
|
|
124
|
+
const allocation = allocations.find(a => a.routeUrl === routeUrl);
|
|
125
|
+
return allocation ? allocation.budget : null;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Legacy functional API for backwards compatibility
|
|
130
|
+
export function computeRouteBudget(manifest, currentUrl, baseBudget) {
|
|
131
|
+
const routes = manifest.routes || [];
|
|
132
|
+
const expectations = manifest.staticExpectations || [];
|
|
133
|
+
const totalRoutes = routes.length;
|
|
134
|
+
const totalExpectations = expectations.length;
|
|
135
|
+
|
|
136
|
+
// Count expectations per route
|
|
137
|
+
const routeExpectationCount = new Map();
|
|
138
|
+
for (const exp of expectations) {
|
|
139
|
+
const routePath = exp.fromPath || '*';
|
|
140
|
+
routeExpectationCount.set(routePath, (routeExpectationCount.get(routePath) || 0) + 1);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Find matching route for current URL
|
|
144
|
+
const urlPath = extractPathFromUrl(currentUrl);
|
|
145
|
+
const urlPathNormalized = normalizePath(urlPath);
|
|
146
|
+
|
|
147
|
+
// Try exact match first, then prefix match (but not root '/')
|
|
148
|
+
const matchingRoute = routes.find(r => {
|
|
149
|
+
const routePath = normalizePath(r.path);
|
|
150
|
+
if (routePath === '*' || routePath === urlPathNormalized) {
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
// Prefix match: only if routePath is not '/' and urlPath starts with routePath + '/'
|
|
154
|
+
if (routePath !== '/' && urlPathNormalized.startsWith(routePath + '/')) {
|
|
155
|
+
return true;
|
|
156
|
+
}
|
|
157
|
+
return false;
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
const routePath = matchingRoute?.path || urlPath || '*';
|
|
161
|
+
const expectationsForRoute = routeExpectationCount.get(routePath) || 0;
|
|
162
|
+
|
|
163
|
+
// Deterministic budget allocation:
|
|
164
|
+
// - Base: baseBudget.maxInteractionsPerPage
|
|
165
|
+
// - Critical routes (with expectations) get 1.5x budget
|
|
166
|
+
// - Non-critical routes get 0.7x budget
|
|
167
|
+
// - If total routes > 50, reduce all budgets proportionally
|
|
168
|
+
|
|
169
|
+
let interactionBudget = baseBudget.maxInteractionsPerPage || 30;
|
|
170
|
+
|
|
171
|
+
if (expectationsForRoute > 0) {
|
|
172
|
+
// Critical route: has expectations
|
|
173
|
+
interactionBudget = Math.floor(interactionBudget * 1.5);
|
|
174
|
+
} else if (totalRoutes > 10) {
|
|
175
|
+
// Non-critical route in large project: reduce budget
|
|
176
|
+
interactionBudget = Math.floor(interactionBudget * 0.7);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Scale down if project is very large
|
|
180
|
+
if (totalRoutes > 50) {
|
|
181
|
+
const scaleFactor = Math.max(0.6, 50 / totalRoutes);
|
|
182
|
+
interactionBudget = Math.floor(interactionBudget * scaleFactor);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Ensure minimum budget
|
|
186
|
+
interactionBudget = Math.max(5, interactionBudget);
|
|
187
|
+
|
|
188
|
+
// Ensure maximum budget cap
|
|
189
|
+
interactionBudget = Math.min(100, interactionBudget);
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
...baseBudget,
|
|
193
|
+
maxInteractionsPerPage: interactionBudget,
|
|
194
|
+
routePath: routePath,
|
|
195
|
+
expectationsForRoute: expectationsForRoute,
|
|
196
|
+
budgetReason: expectationsForRoute > 0 ? 'critical_route' : (totalRoutes > 10 ? 'non_critical_large_project' : 'default')
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Extract path from URL
|
|
202
|
+
*/
|
|
203
|
+
function extractPathFromUrl(url) {
|
|
204
|
+
try {
|
|
205
|
+
const urlObj = new URL(url);
|
|
206
|
+
return urlObj.pathname;
|
|
207
|
+
} catch {
|
|
208
|
+
return url;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Normalize path for comparison
|
|
214
|
+
*/
|
|
215
|
+
function normalizePath(path) {
|
|
216
|
+
if (!path) return '/';
|
|
217
|
+
return path.replace(/\/$/, '') || '/';
|
|
218
|
+
}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CANONICAL OUTCOME TAXONOMY
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth for all outcome classifications in VERAX.
|
|
5
|
+
* Every interaction outcome, silence, gap, and finding must map to exactly ONE of these types.
|
|
6
|
+
*
|
|
7
|
+
* NO free-form or implicit outcome types allowed.
|
|
8
|
+
* NO semantic drift between outputs.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Canonical Outcome Types - Mutually Exclusive Categories
|
|
13
|
+
*/
|
|
14
|
+
export const CANONICAL_OUTCOMES = {
|
|
15
|
+
// Code/user interaction broke (promise unmet, expectation failed)
|
|
16
|
+
SILENT_FAILURE: 'SILENT_FAILURE',
|
|
17
|
+
|
|
18
|
+
// Evaluation was not completed due to time/interaction/data limits (not a failure, just incomplete coverage)
|
|
19
|
+
COVERAGE_GAP: 'COVERAGE_GAP',
|
|
20
|
+
|
|
21
|
+
// Interaction executed but outcome cannot be asserted (ambiguity: SPA fallback, no DOM change signal, missing sensor)
|
|
22
|
+
UNPROVEN_INTERACTION: 'UNPROVEN_INTERACTION',
|
|
23
|
+
|
|
24
|
+
// Intentionally prevented (destructive action, external navigation, safety policy)
|
|
25
|
+
SAFETY_BLOCK: 'SAFETY_BLOCK',
|
|
26
|
+
|
|
27
|
+
// No problem detected; for informational artifacts (e.g., expectations without matching interactions)
|
|
28
|
+
INFORMATIONAL: 'INFORMATIONAL'
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Outcome Definitions - For documentation and validation
|
|
33
|
+
*/
|
|
34
|
+
export const OUTCOME_DEFINITIONS = {
|
|
35
|
+
SILENT_FAILURE: {
|
|
36
|
+
title: 'Silent Failure',
|
|
37
|
+
description: 'User action executed, code ran, but expected observable change did not occur and no error was presented to user',
|
|
38
|
+
example: 'User clicks "Save" → page loads without spinning wheel → data is not saved but user receives no error message',
|
|
39
|
+
implication: 'Gap between user expectation and actual system behavior'
|
|
40
|
+
},
|
|
41
|
+
|
|
42
|
+
COVERAGE_GAP: {
|
|
43
|
+
title: 'Coverage Gap',
|
|
44
|
+
description: 'Interaction or expectation discovered but not evaluated due to scan budget limit (time, page count, interaction count)',
|
|
45
|
+
example: '500 pages discovered but scan budget allows only 100 pages → 400 pages not evaluated',
|
|
46
|
+
implication: 'Incomplete scan; behaviors on skipped interactions are unknown'
|
|
47
|
+
},
|
|
48
|
+
|
|
49
|
+
UNPROVEN_INTERACTION: {
|
|
50
|
+
title: 'Unproven Interaction',
|
|
51
|
+
description: 'Interaction executed but outcome ambiguous: no expectation matched, or observable change is ambiguous, or expectation-driven path failed',
|
|
52
|
+
example: 'User clicks SPA link → no observable change (SPA routing ambiguous) → cannot assert if routing worked or if feature is broken',
|
|
53
|
+
implication: 'Behavior is unknown; cannot determine if code matches reality'
|
|
54
|
+
},
|
|
55
|
+
|
|
56
|
+
SAFETY_BLOCK: {
|
|
57
|
+
title: 'Safety Block',
|
|
58
|
+
description: 'Interaction prevented intentionally: destructive text, external navigation, or safety policy blocks execution',
|
|
59
|
+
example: 'User clicks "Delete Account" → blocked by safety policy → behavior untested',
|
|
60
|
+
implication: 'Intentional limitation; not a failure, just a boundary'
|
|
61
|
+
},
|
|
62
|
+
|
|
63
|
+
INFORMATIONAL: {
|
|
64
|
+
title: 'Informational',
|
|
65
|
+
description: 'Metadata or artifact without direct outcome status (e.g., expectation defined but no matching interaction found)',
|
|
66
|
+
example: 'Manifest defines logout expectation → but no logout interaction discovered on site',
|
|
67
|
+
implication: 'Context for interpretation but not a test failure or success'
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Map silence reason codes to canonical outcomes
|
|
73
|
+
*
|
|
74
|
+
* Used by SilenceTracker and verdict-engine to classify silence entries
|
|
75
|
+
* with explicit outcome types.
|
|
76
|
+
*/
|
|
77
|
+
export function mapSilenceReasonToOutcome(reason) {
|
|
78
|
+
// Timeouts, failures → SILENT_FAILURE (promise unmet)
|
|
79
|
+
if (reason.includes('timeout') || reason.includes('failed') || reason.includes('error')) {
|
|
80
|
+
return CANONICAL_OUTCOMES.SILENT_FAILURE;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Budget/limit exceeded → COVERAGE_GAP (incomplete scan)
|
|
84
|
+
if (reason.includes('budget') || reason.includes('limit') || reason.includes('exceeded')) {
|
|
85
|
+
return CANONICAL_OUTCOMES.COVERAGE_GAP;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Safety/destructive/external → SAFETY_BLOCK (intentional)
|
|
89
|
+
if (reason.includes('destructive') || reason.includes('external') || reason.includes('unsafe') || reason.includes('safety')) {
|
|
90
|
+
return CANONICAL_OUTCOMES.SAFETY_BLOCK;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Incremental reuse → COVERAGE_GAP (unchanged, so gap in new scan)
|
|
94
|
+
if (reason.includes('incremental')) {
|
|
95
|
+
return CANONICAL_OUTCOMES.COVERAGE_GAP;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// No expectation found → INFORMATIONAL (metadata, not a test result)
|
|
99
|
+
if (reason.includes('no_expectation')) {
|
|
100
|
+
return CANONICAL_OUTCOMES.INFORMATIONAL;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Discovery error (selector mismatch, link not found) → COVERAGE_GAP (could not evaluate)
|
|
104
|
+
if (reason.includes('discovery') || reason.includes('no_matching_selector')) {
|
|
105
|
+
return CANONICAL_OUTCOMES.COVERAGE_GAP;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Sensor unavailable → COVERAGE_GAP (incomplete data)
|
|
109
|
+
if (reason.includes('sensor')) {
|
|
110
|
+
return CANONICAL_OUTCOMES.COVERAGE_GAP;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Default: treat as coverage gap if unknown
|
|
114
|
+
return CANONICAL_OUTCOMES.COVERAGE_GAP;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Map finding type to canonical outcome
|
|
119
|
+
*
|
|
120
|
+
* Used by finding-detector.js to classify findings explicitly
|
|
121
|
+
*/
|
|
122
|
+
export function mapFindingTypeToOutcome(findingType) {
|
|
123
|
+
// All forms of silent failure → SILENT_FAILURE
|
|
124
|
+
if (findingType.includes('silent_failure') ||
|
|
125
|
+
findingType.includes('unobserved') ||
|
|
126
|
+
findingType.includes('observed_break')) {
|
|
127
|
+
return CANONICAL_OUTCOMES.SILENT_FAILURE;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Default to SILENT_FAILURE for findings (they represent observed problems)
|
|
131
|
+
return CANONICAL_OUTCOMES.SILENT_FAILURE;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Validate that an outcome is canonical
|
|
136
|
+
*/
|
|
137
|
+
export function isValidOutcome(outcome) {
|
|
138
|
+
return Object.values(CANONICAL_OUTCOMES).includes(outcome);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Format outcome for human display
|
|
143
|
+
*/
|
|
144
|
+
export function formatOutcomeForDisplay(outcome) {
|
|
145
|
+
if (!isValidOutcome(outcome)) {
|
|
146
|
+
throw new Error(`Invalid outcome type: ${outcome}`);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const definition = OUTCOME_DEFINITIONS[outcome];
|
|
150
|
+
return {
|
|
151
|
+
type: outcome,
|
|
152
|
+
title: definition.title,
|
|
153
|
+
description: definition.description
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export default CANONICAL_OUTCOMES;
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PHASE 7 — DECISION SNAPSHOT
|
|
3
|
+
*
|
|
4
|
+
* Computes a top-level decision snapshot that answers 6 mandatory questions:
|
|
5
|
+
*
|
|
6
|
+
* 1. Do we have confirmed SILENT FAILURES?
|
|
7
|
+
* 2. Where exactly are they?
|
|
8
|
+
* 3. How severe are they (user-impact-wise)?
|
|
9
|
+
* 4. What did VERAX NOT verify?
|
|
10
|
+
* 5. Why was it not verified?
|
|
11
|
+
* 6. How much confidence do we have in the findings?
|
|
12
|
+
*
|
|
13
|
+
* Rules:
|
|
14
|
+
* - Derived ONLY from existing data (findings, silences, coverage)
|
|
15
|
+
* - No new detection logic
|
|
16
|
+
* - No subjective language
|
|
17
|
+
* - No recommendations or advice
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Severity levels (user-impact-based, not technical)
|
|
22
|
+
*/
|
|
23
|
+
export const SEVERITY = {
|
|
24
|
+
CRITICAL_USER_BLOCKER: 'critical_user_blocker', // Prevents user from completing core task
|
|
25
|
+
FLOW_BREAKING: 'flow_breaking', // Breaks expected navigation/state flow
|
|
26
|
+
DEGRADING: 'degrading', // Reduces functionality but doesn't block
|
|
27
|
+
INFORMATIONAL: 'informational' // Observable but no clear user impact
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Classify finding severity based on promise type and outcome
|
|
32
|
+
* @param {Object} finding - Finding object
|
|
33
|
+
* @returns {string} - Severity level from SEVERITY enum
|
|
34
|
+
*/
|
|
35
|
+
function classifyFindingSeverity(finding) {
|
|
36
|
+
const promiseType = finding.promiseType || finding.type;
|
|
37
|
+
const outcome = finding.outcome;
|
|
38
|
+
|
|
39
|
+
// Navigation promises that fail are flow-breaking
|
|
40
|
+
if (promiseType === 'navigation' && outcome === 'broken') {
|
|
41
|
+
return SEVERITY.FLOW_BREAKING;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Network actions that fail are critical (forms, submissions)
|
|
45
|
+
if (promiseType === 'networkAction' && outcome === 'broken') {
|
|
46
|
+
return SEVERITY.CRITICAL_USER_BLOCKER;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// State actions that fail are degrading
|
|
50
|
+
if (promiseType === 'stateAction' && outcome === 'broken') {
|
|
51
|
+
return SEVERITY.DEGRADING;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Auth failures are critical user blockers
|
|
55
|
+
if (promiseType === 'authentication' && outcome === 'broken') {
|
|
56
|
+
return SEVERITY.CRITICAL_USER_BLOCKER;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Timeout/unknown outcomes are degrading (cannot confirm impact)
|
|
60
|
+
if (outcome === 'timeout' || outcome === 'unknown') {
|
|
61
|
+
return SEVERITY.DEGRADING;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Default to informational if unclear
|
|
65
|
+
return SEVERITY.INFORMATIONAL;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Compute confidence score (0.0 - 1.0) based on silences and coverage
|
|
70
|
+
* @param {Object} detectTruth - Detect phase truth
|
|
71
|
+
* @param {Object} observeTruth - Observe phase truth
|
|
72
|
+
* @returns {Object} - Confidence assessment
|
|
73
|
+
*/
|
|
74
|
+
function computeConfidence(detectTruth, observeTruth) {
|
|
75
|
+
const totalInteractions = observeTruth?.interactionsObserved || 0;
|
|
76
|
+
const analyzed = detectTruth?.interactionsAnalyzed || 0;
|
|
77
|
+
const skipped = detectTruth?.skips?.total || 0;
|
|
78
|
+
const timeouts = observeTruth?.timeoutsCount || 0;
|
|
79
|
+
const coverageGaps = detectTruth?.coverageGapsCount || 0;
|
|
80
|
+
|
|
81
|
+
// Coverage ratio: how much was actually verified
|
|
82
|
+
const coverageRatio = totalInteractions > 0 ? analyzed / totalInteractions : 0;
|
|
83
|
+
|
|
84
|
+
// Silence penalty: reduce confidence for unknown outcomes
|
|
85
|
+
const silencePenalty = (skipped + timeouts + coverageGaps) / Math.max(totalInteractions, 1);
|
|
86
|
+
|
|
87
|
+
// Base confidence from coverage
|
|
88
|
+
let confidenceScore = coverageRatio;
|
|
89
|
+
|
|
90
|
+
// Apply silence penalty (max 50% reduction)
|
|
91
|
+
confidenceScore = confidenceScore * (1 - Math.min(silencePenalty * 0.5, 0.5));
|
|
92
|
+
|
|
93
|
+
// Clamp to 0-1 range
|
|
94
|
+
confidenceScore = Math.max(0, Math.min(1, confidenceScore));
|
|
95
|
+
|
|
96
|
+
// Classify confidence level
|
|
97
|
+
let confidenceLevel = 'very_low';
|
|
98
|
+
if (confidenceScore >= 0.9) {
|
|
99
|
+
confidenceLevel = 'high';
|
|
100
|
+
} else if (confidenceScore >= 0.7) {
|
|
101
|
+
confidenceLevel = 'medium';
|
|
102
|
+
} else if (confidenceScore >= 0.5) {
|
|
103
|
+
confidenceLevel = 'low';
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return {
|
|
107
|
+
score: confidenceScore,
|
|
108
|
+
level: confidenceLevel,
|
|
109
|
+
coverageRatio,
|
|
110
|
+
silencePenalty,
|
|
111
|
+
factors: {
|
|
112
|
+
totalInteractions,
|
|
113
|
+
analyzed,
|
|
114
|
+
skipped,
|
|
115
|
+
timeouts,
|
|
116
|
+
coverageGaps
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Extract unverified items with reasons
|
|
123
|
+
* @param {Object} detectTruth - Detect phase truth
|
|
124
|
+
* @param {Object} observeTruth - Observe phase truth
|
|
125
|
+
* @returns {Array} - List of unverified items with reasons
|
|
126
|
+
*/
|
|
127
|
+
function extractUnverified(detectTruth, observeTruth) {
|
|
128
|
+
const unverified = [];
|
|
129
|
+
|
|
130
|
+
// Coverage gaps (expectations not evaluated)
|
|
131
|
+
const coverageGaps = detectTruth?.coverageGapsCount || 0;
|
|
132
|
+
if (coverageGaps > 0) {
|
|
133
|
+
unverified.push({
|
|
134
|
+
category: 'expectations',
|
|
135
|
+
count: coverageGaps,
|
|
136
|
+
reason: 'budget_exceeded'
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Skipped interactions by reason
|
|
141
|
+
const skips = detectTruth?.skips?.reasons || [];
|
|
142
|
+
for (const skipReason of skips) {
|
|
143
|
+
unverified.push({
|
|
144
|
+
category: 'interactions',
|
|
145
|
+
count: skipReason.count,
|
|
146
|
+
reason: skipReason.code
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Timeouts
|
|
151
|
+
const timeouts = observeTruth?.timeoutsCount || 0;
|
|
152
|
+
if (timeouts > 0) {
|
|
153
|
+
unverified.push({
|
|
154
|
+
category: 'interactions',
|
|
155
|
+
count: timeouts,
|
|
156
|
+
reason: 'timeout'
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// External navigations blocked
|
|
161
|
+
const externalBlocked = observeTruth?.externalNavigationBlockedCount || 0;
|
|
162
|
+
if (externalBlocked > 0) {
|
|
163
|
+
unverified.push({
|
|
164
|
+
category: 'navigations',
|
|
165
|
+
count: externalBlocked,
|
|
166
|
+
reason: 'external_blocked'
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return unverified;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Compute decision snapshot from scan results
|
|
175
|
+
* @param {Array} findings - Array of findings
|
|
176
|
+
* @param {Object} detectTruth - Detect phase truth
|
|
177
|
+
* @param {Object} observeTruth - Observe phase truth
|
|
178
|
+
* @param {Object} silences - Silence data
|
|
179
|
+
* @returns {Object} - Decision snapshot answering 6 mandatory questions
|
|
180
|
+
*/
|
|
181
|
+
export function computeDecisionSnapshot(findings, detectTruth, observeTruth, silences) {
|
|
182
|
+
// Question 1: Do we have confirmed SILENT FAILURES?
|
|
183
|
+
const confirmedFailures = findings.filter(f =>
|
|
184
|
+
f.outcome === 'broken' || f.type === 'silent_failure'
|
|
185
|
+
);
|
|
186
|
+
const hasConfirmedFailures = confirmedFailures.length > 0;
|
|
187
|
+
|
|
188
|
+
// Question 2: Where exactly are they?
|
|
189
|
+
const failureLocations = confirmedFailures.map(f => ({
|
|
190
|
+
type: f.promiseType || f.type,
|
|
191
|
+
fromPath: f.fromPath,
|
|
192
|
+
toPath: f.toPath,
|
|
193
|
+
selector: f.interaction?.selector,
|
|
194
|
+
description: f.description || f.reason
|
|
195
|
+
}));
|
|
196
|
+
|
|
197
|
+
// Question 3: How severe are they (user-impact-wise)?
|
|
198
|
+
const severityCounts = {
|
|
199
|
+
[SEVERITY.CRITICAL_USER_BLOCKER]: 0,
|
|
200
|
+
[SEVERITY.FLOW_BREAKING]: 0,
|
|
201
|
+
[SEVERITY.DEGRADING]: 0,
|
|
202
|
+
[SEVERITY.INFORMATIONAL]: 0
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
const failuresBySeverity = confirmedFailures.map(f => {
|
|
206
|
+
const severity = classifyFindingSeverity(f);
|
|
207
|
+
severityCounts[severity]++;
|
|
208
|
+
return {
|
|
209
|
+
severity,
|
|
210
|
+
finding: f
|
|
211
|
+
};
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
// Question 4: What did VERAX NOT verify?
|
|
215
|
+
const unverified = extractUnverified(detectTruth, observeTruth);
|
|
216
|
+
const totalUnverified = unverified.reduce((sum, u) => sum + u.count, 0);
|
|
217
|
+
|
|
218
|
+
// Question 5: Why was it not verified?
|
|
219
|
+
const unverifiedReasons = {};
|
|
220
|
+
for (const item of unverified) {
|
|
221
|
+
if (!unverifiedReasons[item.reason]) {
|
|
222
|
+
unverifiedReasons[item.reason] = 0;
|
|
223
|
+
}
|
|
224
|
+
unverifiedReasons[item.reason] += item.count;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// Question 6: How much confidence do we have in the findings?
|
|
228
|
+
const confidence = computeConfidence(detectTruth, observeTruth);
|
|
229
|
+
|
|
230
|
+
return {
|
|
231
|
+
// Question 1
|
|
232
|
+
hasConfirmedFailures,
|
|
233
|
+
confirmedFailureCount: confirmedFailures.length,
|
|
234
|
+
|
|
235
|
+
// Question 2
|
|
236
|
+
failureLocations,
|
|
237
|
+
|
|
238
|
+
// Question 3
|
|
239
|
+
severityCounts,
|
|
240
|
+
failuresBySeverity: failuresBySeverity.map(f => ({
|
|
241
|
+
severity: f.severity,
|
|
242
|
+
type: f.finding.promiseType || f.finding.type,
|
|
243
|
+
description: f.finding.description || f.finding.reason,
|
|
244
|
+
fromPath: f.finding.fromPath
|
|
245
|
+
})),
|
|
246
|
+
|
|
247
|
+
// Question 4
|
|
248
|
+
totalUnverified,
|
|
249
|
+
unverifiedByCategory: unverified.reduce((acc, u) => {
|
|
250
|
+
if (!acc[u.category]) {
|
|
251
|
+
acc[u.category] = 0;
|
|
252
|
+
}
|
|
253
|
+
acc[u.category] += u.count;
|
|
254
|
+
return acc;
|
|
255
|
+
}, {}),
|
|
256
|
+
|
|
257
|
+
// Question 5
|
|
258
|
+
unverifiedReasons,
|
|
259
|
+
unverifiedDetails: unverified,
|
|
260
|
+
|
|
261
|
+
// Question 6
|
|
262
|
+
confidence: {
|
|
263
|
+
level: confidence.level,
|
|
264
|
+
score: confidence.score,
|
|
265
|
+
coverageRatio: confidence.coverageRatio,
|
|
266
|
+
factors: confidence.factors
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Format decision snapshot for human reading
|
|
273
|
+
* @param {Object} snapshot - Decision snapshot
|
|
274
|
+
* @returns {string} - Formatted text
|
|
275
|
+
*/
|
|
276
|
+
export function formatDecisionSnapshot(snapshot) {
|
|
277
|
+
const lines = [];
|
|
278
|
+
|
|
279
|
+
lines.push('=== DECISION SNAPSHOT ===');
|
|
280
|
+
lines.push('');
|
|
281
|
+
|
|
282
|
+
// Question 1: Confirmed failures?
|
|
283
|
+
lines.push(`1. CONFIRMED SILENT FAILURES: ${snapshot.hasConfirmedFailures ? 'YES' : 'NO'}`);
|
|
284
|
+
lines.push(` Count: ${snapshot.confirmedFailureCount}`);
|
|
285
|
+
lines.push('');
|
|
286
|
+
|
|
287
|
+
// Question 2: Where?
|
|
288
|
+
if (snapshot.failureLocations.length > 0) {
|
|
289
|
+
lines.push('2. FAILURE LOCATIONS:');
|
|
290
|
+
for (const loc of snapshot.failureLocations.slice(0, 5)) {
|
|
291
|
+
lines.push(` - ${loc.type}: ${loc.fromPath} → ${loc.toPath || 'unknown'}`);
|
|
292
|
+
if (loc.selector) {
|
|
293
|
+
lines.push(` Selector: ${loc.selector}`);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
if (snapshot.failureLocations.length > 5) {
|
|
297
|
+
lines.push(` ... and ${snapshot.failureLocations.length - 5} more`);
|
|
298
|
+
}
|
|
299
|
+
} else {
|
|
300
|
+
lines.push('2. FAILURE LOCATIONS: None');
|
|
301
|
+
}
|
|
302
|
+
lines.push('');
|
|
303
|
+
|
|
304
|
+
// Question 3: How severe?
|
|
305
|
+
lines.push('3. SEVERITY BREAKDOWN:');
|
|
306
|
+
lines.push(` Critical user blockers: ${snapshot.severityCounts.critical_user_blocker}`);
|
|
307
|
+
lines.push(` Flow-breaking issues: ${snapshot.severityCounts.flow_breaking}`);
|
|
308
|
+
lines.push(` Degrading issues: ${snapshot.severityCounts.degrading}`);
|
|
309
|
+
lines.push(` Informational: ${snapshot.severityCounts.informational}`);
|
|
310
|
+
lines.push('');
|
|
311
|
+
|
|
312
|
+
// Question 4: What NOT verified?
|
|
313
|
+
lines.push('4. NOT VERIFIED:');
|
|
314
|
+
lines.push(` Total: ${snapshot.totalUnverified}`);
|
|
315
|
+
for (const [category, count] of Object.entries(snapshot.unverifiedByCategory)) {
|
|
316
|
+
lines.push(` - ${category}: ${count}`);
|
|
317
|
+
}
|
|
318
|
+
lines.push('');
|
|
319
|
+
|
|
320
|
+
// Question 5: Why not verified?
|
|
321
|
+
lines.push('5. REASONS NOT VERIFIED:');
|
|
322
|
+
for (const [reason, count] of Object.entries(snapshot.unverifiedReasons)) {
|
|
323
|
+
lines.push(` - ${reason}: ${count}`);
|
|
324
|
+
}
|
|
325
|
+
lines.push('');
|
|
326
|
+
|
|
327
|
+
// Question 6: Confidence?
|
|
328
|
+
lines.push('6. CONFIDENCE IN FINDINGS:');
|
|
329
|
+
lines.push(` Level: ${snapshot.confidence.level.toUpperCase()}`);
|
|
330
|
+
lines.push(` Score: ${(snapshot.confidence.score * 100).toFixed(1)}%`);
|
|
331
|
+
lines.push(` Coverage: ${(snapshot.confidence.coverageRatio * 100).toFixed(1)}% of interactions verified`);
|
|
332
|
+
lines.push('');
|
|
333
|
+
|
|
334
|
+
return lines.join('\n');
|
|
335
|
+
}
|