@veraxhq/verax 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +237 -0
- package/bin/verax.js +452 -0
- package/package.json +57 -0
- package/src/verax/detect/comparison.js +69 -0
- package/src/verax/detect/confidence-engine.js +498 -0
- package/src/verax/detect/evidence-validator.js +33 -0
- package/src/verax/detect/expectation-model.js +204 -0
- package/src/verax/detect/findings-writer.js +31 -0
- package/src/verax/detect/index.js +397 -0
- package/src/verax/detect/skip-classifier.js +202 -0
- package/src/verax/flow/flow-engine.js +265 -0
- package/src/verax/flow/flow-spec.js +145 -0
- package/src/verax/flow/redaction.js +74 -0
- package/src/verax/index.js +97 -0
- package/src/verax/learn/action-contract-extractor.js +281 -0
- package/src/verax/learn/ast-contract-extractor.js +255 -0
- package/src/verax/learn/index.js +18 -0
- package/src/verax/learn/manifest-writer.js +97 -0
- package/src/verax/learn/project-detector.js +87 -0
- package/src/verax/learn/react-router-extractor.js +73 -0
- package/src/verax/learn/route-extractor.js +122 -0
- package/src/verax/learn/route-validator.js +215 -0
- package/src/verax/learn/source-instrumenter.js +214 -0
- package/src/verax/learn/static-extractor.js +222 -0
- package/src/verax/learn/truth-assessor.js +96 -0
- package/src/verax/learn/ts-contract-resolver.js +395 -0
- package/src/verax/observe/browser.js +22 -0
- package/src/verax/observe/console-sensor.js +166 -0
- package/src/verax/observe/dom-signature.js +23 -0
- package/src/verax/observe/domain-boundary.js +38 -0
- package/src/verax/observe/evidence-capture.js +5 -0
- package/src/verax/observe/human-driver.js +376 -0
- package/src/verax/observe/index.js +67 -0
- package/src/verax/observe/interaction-discovery.js +269 -0
- package/src/verax/observe/interaction-runner.js +410 -0
- package/src/verax/observe/network-sensor.js +173 -0
- package/src/verax/observe/selector-generator.js +74 -0
- package/src/verax/observe/settle.js +155 -0
- package/src/verax/observe/state-ui-sensor.js +200 -0
- package/src/verax/observe/traces-writer.js +82 -0
- package/src/verax/observe/ui-signal-sensor.js +197 -0
- package/src/verax/resolve-workspace-root.js +173 -0
- package/src/verax/scan-summary-writer.js +41 -0
- package/src/verax/shared/artifact-manager.js +139 -0
- package/src/verax/shared/caching.js +104 -0
- package/src/verax/shared/expectation-proof.js +4 -0
- package/src/verax/shared/redaction.js +227 -0
- package/src/verax/shared/retry-policy.js +89 -0
- package/src/verax/shared/timing-metrics.js +44 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { getUrlPath } from './evidence-validator.js';
|
|
2
|
+
import { ExpectationProof } from '../shared/expectation-proof.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Extracts href value from interaction selector.
|
|
6
|
+
* Used to match SPA expectations by href attribute.
|
|
7
|
+
*/
|
|
8
|
+
function extractHrefFromSelector(selector) {
|
|
9
|
+
if (!selector) return null;
|
|
10
|
+
|
|
11
|
+
// Match href="/about" or href='/about' in selector
|
|
12
|
+
const hrefMatch = selector.match(/href=["']([^"']+)["']/);
|
|
13
|
+
if (hrefMatch) {
|
|
14
|
+
return hrefMatch[1];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Returns expectation info for an interaction at a given URL.
|
|
22
|
+
* Wave 0: PROVEN expectations from staticExpectations (HTML-derived)
|
|
23
|
+
* Wave 1: PROVEN expectations from spaExpectations (AST-derived JSX contracts)
|
|
24
|
+
* Wave 5: PROVEN expectations from actionContracts (AST-derived network actions)
|
|
25
|
+
* Wave 8: PROVEN expectations from actionContracts (AST-derived state actions)
|
|
26
|
+
*
|
|
27
|
+
* @returns {{ hasExpectation: boolean, proof: string, expectedTargetPath?: string, expectationType?: string, method?: string, urlPath?: string, stateKind?: string }}
|
|
28
|
+
*/
|
|
29
|
+
export function getExpectation(manifest, interaction, beforeUrl, attemptMeta = {}) {
|
|
30
|
+
// Wave 5/6/8 - ACTION CONTRACTS: Check network and state action expectations
|
|
31
|
+
if (manifest.actionContracts && manifest.actionContracts.length > 0) {
|
|
32
|
+
// Priority: handlerRef (cross-file) → sourceRef (inline)
|
|
33
|
+
const handlerMatch = attemptMeta.handlerRef
|
|
34
|
+
? manifest.actionContracts.find(
|
|
35
|
+
(contract) => contract.handlerRef === attemptMeta.handlerRef
|
|
36
|
+
)
|
|
37
|
+
: null;
|
|
38
|
+
if (handlerMatch) {
|
|
39
|
+
if (handlerMatch.kind === 'NETWORK_ACTION') {
|
|
40
|
+
return {
|
|
41
|
+
hasExpectation: true,
|
|
42
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
43
|
+
expectationType: 'network_action',
|
|
44
|
+
method: handlerMatch.method,
|
|
45
|
+
urlPath: handlerMatch.urlPath
|
|
46
|
+
};
|
|
47
|
+
} else if (handlerMatch.kind === 'STATE_ACTION') {
|
|
48
|
+
return {
|
|
49
|
+
hasExpectation: true,
|
|
50
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
51
|
+
expectationType: 'state_action',
|
|
52
|
+
stateKind: handlerMatch.stateKind
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const sourceMatch = attemptMeta.sourceRef
|
|
58
|
+
? manifest.actionContracts.find(
|
|
59
|
+
(contract) => contract.source === attemptMeta.sourceRef
|
|
60
|
+
)
|
|
61
|
+
: null;
|
|
62
|
+
if (sourceMatch) {
|
|
63
|
+
if (sourceMatch.kind === 'NETWORK_ACTION') {
|
|
64
|
+
return {
|
|
65
|
+
hasExpectation: true,
|
|
66
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
67
|
+
expectationType: 'network_action',
|
|
68
|
+
method: sourceMatch.method,
|
|
69
|
+
urlPath: sourceMatch.urlPath
|
|
70
|
+
};
|
|
71
|
+
} else if (sourceMatch.kind === 'STATE_ACTION') {
|
|
72
|
+
return {
|
|
73
|
+
hasExpectation: true,
|
|
74
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
75
|
+
expectationType: 'state_action',
|
|
76
|
+
stateKind: sourceMatch.stateKind
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Wave 1 - CODE TRUTH ENGINE: Check SPA expectations (AST-derived contracts)
|
|
83
|
+
if (manifest.spaExpectations && manifest.spaExpectations.length > 0) {
|
|
84
|
+
// For link interactions, extract href and match against expectations
|
|
85
|
+
if (interaction.type === 'link') {
|
|
86
|
+
const href = extractHrefFromSelector(interaction.selector);
|
|
87
|
+
|
|
88
|
+
if (href) {
|
|
89
|
+
// Normalize href (remove trailing slash, ensure leading slash)
|
|
90
|
+
const normalizedHref = (href.startsWith('/') ? href : '/' + href).replace(/\/$/, '') || '/';
|
|
91
|
+
|
|
92
|
+
for (const expectation of manifest.spaExpectations) {
|
|
93
|
+
if (expectation.proof !== ExpectationProof.PROVEN_EXPECTATION) {
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const normalizedTarget = expectation.targetPath.replace(/\/$/, '') || '/';
|
|
98
|
+
|
|
99
|
+
// Match by href attribute value
|
|
100
|
+
if (expectation.matchAttribute === 'href' && normalizedHref === normalizedTarget) {
|
|
101
|
+
return {
|
|
102
|
+
hasExpectation: true,
|
|
103
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
104
|
+
expectedTargetPath: expectation.targetPath,
|
|
105
|
+
expectationType: 'spa_navigation'
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// Also match 'to' attribute for React Router
|
|
110
|
+
if (expectation.matchAttribute === 'to' && normalizedHref === normalizedTarget) {
|
|
111
|
+
return {
|
|
112
|
+
hasExpectation: true,
|
|
113
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
114
|
+
expectedTargetPath: expectation.targetPath,
|
|
115
|
+
expectationType: 'spa_navigation'
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Wave 0: Static HTML expectations
|
|
124
|
+
if (manifest.staticExpectations && manifest.staticExpectations.length > 0) {
|
|
125
|
+
const beforePath = getUrlPath(beforeUrl);
|
|
126
|
+
if (beforePath) {
|
|
127
|
+
const normalizedBefore = beforePath.replace(/\/$/, '') || '/';
|
|
128
|
+
|
|
129
|
+
for (const expectation of manifest.staticExpectations) {
|
|
130
|
+
if (expectation.proof !== ExpectationProof.PROVEN_EXPECTATION) {
|
|
131
|
+
continue; // Only consider proven expectations
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (expectation.type === 'navigation') {
|
|
135
|
+
const normalizedFrom = expectation.fromPath.replace(/\/$/, '') || '/';
|
|
136
|
+
|
|
137
|
+
if (normalizedFrom === normalizedBefore) {
|
|
138
|
+
if (interaction.type === 'link' || interaction.type === 'button') {
|
|
139
|
+
const selectorHint = expectation.evidence.selectorHint || '';
|
|
140
|
+
const interactionSelector = interaction.selector || '';
|
|
141
|
+
|
|
142
|
+
if (selectorHint && interactionSelector) {
|
|
143
|
+
const normalizedSelectorHint = selectorHint.replace(/[\[\]()]/g, '');
|
|
144
|
+
const normalizedInteractionSelector = interactionSelector.replace(/[\[\]()]/g, '');
|
|
145
|
+
|
|
146
|
+
if (selectorHint === interactionSelector ||
|
|
147
|
+
selectorHint.includes(interactionSelector) ||
|
|
148
|
+
interactionSelector.includes(normalizedSelectorHint) ||
|
|
149
|
+
normalizedSelectorHint === normalizedInteractionSelector) {
|
|
150
|
+
return {
|
|
151
|
+
hasExpectation: true,
|
|
152
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
153
|
+
expectedTargetPath: expectation.targetPath,
|
|
154
|
+
expectationType: 'navigation'
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
} else if (expectation.type === 'form_submission') {
|
|
161
|
+
const normalizedFrom = expectation.fromPath.replace(/\/$/, '') || '/';
|
|
162
|
+
|
|
163
|
+
if (normalizedFrom === normalizedBefore && interaction.type === 'form') {
|
|
164
|
+
const selectorHint = expectation.evidence.selectorHint || '';
|
|
165
|
+
const interactionSelector = interaction.selector || '';
|
|
166
|
+
|
|
167
|
+
if (selectorHint && interactionSelector) {
|
|
168
|
+
const normalizedSelectorHint = selectorHint.replace(/[\[\]()]/g, '');
|
|
169
|
+
const normalizedInteractionSelector = interactionSelector.replace(/[\[\]()]/g, '');
|
|
170
|
+
|
|
171
|
+
if (selectorHint === interactionSelector ||
|
|
172
|
+
selectorHint.includes(interactionSelector) ||
|
|
173
|
+
interactionSelector.includes(normalizedSelectorHint) ||
|
|
174
|
+
normalizedSelectorHint === normalizedInteractionSelector) {
|
|
175
|
+
return {
|
|
176
|
+
hasExpectation: true,
|
|
177
|
+
proof: ExpectationProof.PROVEN_EXPECTATION,
|
|
178
|
+
expectedTargetPath: expectation.targetPath,
|
|
179
|
+
expectationType: 'form_submission'
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// No proven expectation found - all heuristic matching removed
|
|
190
|
+
return {
|
|
191
|
+
hasExpectation: false,
|
|
192
|
+
proof: ExpectationProof.UNKNOWN_EXPECTATION
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Legacy function for backward compatibility.
|
|
198
|
+
* Now delegates to getExpectation and returns boolean.
|
|
199
|
+
*/
|
|
200
|
+
export function expectsNavigation(manifest, interaction, beforeUrl) {
|
|
201
|
+
const expectation = getExpectation(manifest, interaction, beforeUrl);
|
|
202
|
+
return expectation.hasExpectation && expectation.proof === ExpectationProof.PROVEN_EXPECTATION;
|
|
203
|
+
}
|
|
204
|
+
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { resolve } from 'path';
|
|
2
|
+
import { mkdirSync, writeFileSync } from 'fs';
|
|
3
|
+
|
|
4
|
+
export function writeFindings(projectDir, url, findings, artifactPaths = null) {
|
|
5
|
+
let findingsPath;
|
|
6
|
+
if (artifactPaths) {
|
|
7
|
+
// Use new artifact structure
|
|
8
|
+
findingsPath = artifactPaths.findings;
|
|
9
|
+
} else {
|
|
10
|
+
// Legacy structure
|
|
11
|
+
const detectDir = resolve(projectDir, '.veraxverax', 'detect');
|
|
12
|
+
mkdirSync(detectDir, { recursive: true });
|
|
13
|
+
findingsPath = resolve(detectDir, 'findings.json');
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const findingsReport = {
|
|
17
|
+
version: 1,
|
|
18
|
+
detectedAt: new Date().toISOString(),
|
|
19
|
+
url: url,
|
|
20
|
+
findings: findings,
|
|
21
|
+
notes: []
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
writeFileSync(findingsPath, JSON.stringify(findingsReport, null, 2) + '\n');
|
|
25
|
+
|
|
26
|
+
return {
|
|
27
|
+
...findingsReport,
|
|
28
|
+
findingsPath: findingsPath
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from 'fs';
|
|
2
|
+
import { getExpectation } from './expectation-model.js';
|
|
3
|
+
import { hasMeaningfulUrlChange, hasVisibleChange, hasDomChange } from './comparison.js';
|
|
4
|
+
import { writeFindings } from './findings-writer.js';
|
|
5
|
+
import { getUrlPath } from './evidence-validator.js';
|
|
6
|
+
import { classifySkipReason, collectSkipReasons } from './skip-classifier.js';
|
|
7
|
+
import { ExpectationProof } from '../shared/expectation-proof.js';
|
|
8
|
+
import { computeConfidence } from './confidence-engine.js';
|
|
9
|
+
|
|
10
|
+
// Helper to create finding with confidence score
|
|
11
|
+
function createFindingWithConfidence(baseFinding, expectationInfo, sensors, comparisons) {
|
|
12
|
+
const confidence = computeConfidence({
|
|
13
|
+
findingType: baseFinding.type,
|
|
14
|
+
expectation: expectationInfo,
|
|
15
|
+
sensors: sensors,
|
|
16
|
+
comparisons: comparisons,
|
|
17
|
+
attemptMeta: {}
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
return {
|
|
21
|
+
...baseFinding,
|
|
22
|
+
confidence
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export async function detect(manifestPath, tracesPath, validation = null, artifactPaths = null) {
|
|
27
|
+
if (!existsSync(manifestPath)) {
|
|
28
|
+
throw new Error(`Manifest not found: ${manifestPath}`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (!existsSync(tracesPath)) {
|
|
32
|
+
throw new Error(`Observation traces not found: ${tracesPath}`);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const manifestContent = readFileSync(manifestPath, 'utf-8');
|
|
36
|
+
const tracesContent = readFileSync(tracesPath, 'utf-8');
|
|
37
|
+
|
|
38
|
+
const manifest = JSON.parse(manifestContent);
|
|
39
|
+
const observation = JSON.parse(tracesContent);
|
|
40
|
+
|
|
41
|
+
const projectDir = manifest.projectDir;
|
|
42
|
+
const findings = [];
|
|
43
|
+
|
|
44
|
+
let interactionsAnalyzed = 0;
|
|
45
|
+
let interactionsSkippedNoExpectation = 0;
|
|
46
|
+
const skips = [];
|
|
47
|
+
|
|
48
|
+
for (const trace of observation.traces) {
|
|
49
|
+
const interaction = trace.interaction;
|
|
50
|
+
const beforeUrl = trace.before.url;
|
|
51
|
+
const afterUrl = trace.after.url;
|
|
52
|
+
const beforeScreenshot = trace.before.screenshot;
|
|
53
|
+
const afterScreenshot = trace.after.screenshot;
|
|
54
|
+
|
|
55
|
+
// Get expectation with proof status (Wave 0 - TRUTH LOCK, Wave 5 - ACTION CONTRACTS)
|
|
56
|
+
const attemptMeta = trace.meta || {};
|
|
57
|
+
const expectationInfo = getExpectation(manifest, interaction, beforeUrl, attemptMeta);
|
|
58
|
+
|
|
59
|
+
// Only analyze interactions with PROVEN expectations
|
|
60
|
+
if (!expectationInfo.hasExpectation || expectationInfo.proof !== ExpectationProof.PROVEN_EXPECTATION) {
|
|
61
|
+
interactionsSkippedNoExpectation++;
|
|
62
|
+
const skipReason = {
|
|
63
|
+
code: 'UNPROVEN_EXPECTATION',
|
|
64
|
+
message: 'No proven code-derived expectation for this interaction',
|
|
65
|
+
interaction: {
|
|
66
|
+
type: interaction.type,
|
|
67
|
+
selector: interaction.selector,
|
|
68
|
+
label: interaction.label
|
|
69
|
+
},
|
|
70
|
+
expectationProof: expectationInfo.proof
|
|
71
|
+
};
|
|
72
|
+
skips.push(skipReason);
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const expectedTargetPath = expectationInfo.expectedTargetPath;
|
|
77
|
+
const expectationType = expectationInfo.expectationType;
|
|
78
|
+
|
|
79
|
+
// Expectation is PROVEN, proceed with analysis
|
|
80
|
+
interactionsAnalyzed++;
|
|
81
|
+
|
|
82
|
+
const hasUrlChange = hasMeaningfulUrlChange(beforeUrl, afterUrl);
|
|
83
|
+
const hasVisibleChangeResult = hasVisibleChange(beforeScreenshot, afterScreenshot, projectDir);
|
|
84
|
+
const hasDomChangeResult = hasDomChange(trace);
|
|
85
|
+
|
|
86
|
+
// Wave 3: Analyze sensor data for silent failures
|
|
87
|
+
const sensorData = trace.sensors || {};
|
|
88
|
+
const networkSummary = sensorData.network || {};
|
|
89
|
+
const consoleSummary = sensorData.console || {};
|
|
90
|
+
const uiSignalChanges = sensorData.uiSignals?.changes || {};
|
|
91
|
+
|
|
92
|
+
// All expectations reaching here are PROVEN with explicit target paths
|
|
93
|
+
const afterPath = getUrlPath(afterUrl);
|
|
94
|
+
const normalizedTarget = expectedTargetPath.replace(/\/$/, '') || '/';
|
|
95
|
+
const normalizedAfter = afterPath ? afterPath.replace(/\/$/, '') || '/' : '';
|
|
96
|
+
|
|
97
|
+
if (expectationType === 'form_submission') {
|
|
98
|
+
if (normalizedAfter !== normalizedTarget && !hasUrlChange && !hasDomChangeResult) {
|
|
99
|
+
// Check for network errors during form submission
|
|
100
|
+
if (networkSummary.failedRequests > 0) {
|
|
101
|
+
const finding = createFindingWithConfidence({
|
|
102
|
+
type: 'network_silent_failure',
|
|
103
|
+
interaction: {
|
|
104
|
+
type: interaction.type,
|
|
105
|
+
selector: interaction.selector,
|
|
106
|
+
label: interaction.label
|
|
107
|
+
},
|
|
108
|
+
reason: 'Form submission triggered network errors with no UI feedback',
|
|
109
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
110
|
+
evidence: {
|
|
111
|
+
before: beforeScreenshot,
|
|
112
|
+
after: afterScreenshot,
|
|
113
|
+
beforeUrl: beforeUrl,
|
|
114
|
+
afterUrl: afterUrl,
|
|
115
|
+
networkErrors: networkSummary.failedRequests,
|
|
116
|
+
failedByStatus: networkSummary.failedByStatus,
|
|
117
|
+
topFailedUrls: networkSummary.topFailedUrls
|
|
118
|
+
}
|
|
119
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
120
|
+
findings.push(finding);
|
|
121
|
+
}
|
|
122
|
+
// Check for validation errors (console errors with no feedback)
|
|
123
|
+
else if (consoleSummary.hasErrors && !uiSignalChanges.changed) {
|
|
124
|
+
const finding = createFindingWithConfidence({
|
|
125
|
+
type: 'validation_silent_failure',
|
|
126
|
+
interaction: {
|
|
127
|
+
type: interaction.type,
|
|
128
|
+
selector: interaction.selector,
|
|
129
|
+
label: interaction.label
|
|
130
|
+
},
|
|
131
|
+
reason: 'Form validation errors logged to console with no visible feedback',
|
|
132
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
133
|
+
evidence: {
|
|
134
|
+
before: beforeScreenshot,
|
|
135
|
+
after: afterScreenshot,
|
|
136
|
+
beforeUrl: beforeUrl,
|
|
137
|
+
afterUrl: afterUrl,
|
|
138
|
+
consoleErrors: consoleSummary.consoleErrorCount,
|
|
139
|
+
errorMessages: consoleSummary.lastErrors?.slice(0, 3)
|
|
140
|
+
}
|
|
141
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
142
|
+
findings.push(finding);
|
|
143
|
+
}
|
|
144
|
+
// Check for missing loading/error feedback
|
|
145
|
+
else if (!uiSignalChanges.changed) {
|
|
146
|
+
const finding = createFindingWithConfidence({
|
|
147
|
+
type: 'missing_feedback_failure',
|
|
148
|
+
interaction: {
|
|
149
|
+
type: interaction.type,
|
|
150
|
+
selector: interaction.selector,
|
|
151
|
+
label: interaction.label
|
|
152
|
+
},
|
|
153
|
+
reason: 'Form submission occurred with no loading or error feedback',
|
|
154
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
155
|
+
evidence: {
|
|
156
|
+
before: beforeScreenshot,
|
|
157
|
+
after: afterScreenshot,
|
|
158
|
+
beforeUrl: beforeUrl,
|
|
159
|
+
afterUrl: afterUrl,
|
|
160
|
+
networkActivity: networkSummary.totalRequests || 0,
|
|
161
|
+
hadFeedback: uiSignalChanges.changed || false
|
|
162
|
+
}
|
|
163
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
164
|
+
findings.push(finding);
|
|
165
|
+
}
|
|
166
|
+
// Generic no-effect failure (no network, no console errors, no visible changes)
|
|
167
|
+
else {
|
|
168
|
+
const finding = createFindingWithConfidence({
|
|
169
|
+
type: 'no_effect_silent_failure',
|
|
170
|
+
interaction: {
|
|
171
|
+
type: interaction.type,
|
|
172
|
+
selector: interaction.selector,
|
|
173
|
+
label: interaction.label
|
|
174
|
+
},
|
|
175
|
+
reason: 'Expected form submission did not occur',
|
|
176
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
177
|
+
evidence: {
|
|
178
|
+
before: beforeScreenshot,
|
|
179
|
+
after: afterScreenshot,
|
|
180
|
+
beforeUrl: beforeUrl,
|
|
181
|
+
afterUrl: afterUrl
|
|
182
|
+
}
|
|
183
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
184
|
+
findings.push(finding);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
} else if (expectationType === 'navigation' || expectationType === 'spa_navigation') {
|
|
188
|
+
// Wave 1: spa_navigation expectations from AST contracts
|
|
189
|
+
const urlMatchesTarget = normalizedAfter === normalizedTarget;
|
|
190
|
+
const hasEffect = urlMatchesTarget || hasVisibleChangeResult || hasDomChangeResult;
|
|
191
|
+
|
|
192
|
+
if (!hasEffect) {
|
|
193
|
+
// Check what prevented the navigation
|
|
194
|
+
if (networkSummary.failedRequests > 0) {
|
|
195
|
+
const finding = createFindingWithConfidence({
|
|
196
|
+
type: 'network_silent_failure',
|
|
197
|
+
interaction: {
|
|
198
|
+
type: interaction.type,
|
|
199
|
+
selector: interaction.selector,
|
|
200
|
+
label: interaction.label
|
|
201
|
+
},
|
|
202
|
+
reason: 'Navigation attempt resulted in network errors with no UI feedback',
|
|
203
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
204
|
+
evidence: {
|
|
205
|
+
before: beforeScreenshot,
|
|
206
|
+
after: afterScreenshot,
|
|
207
|
+
beforeUrl: beforeUrl,
|
|
208
|
+
afterUrl: afterUrl,
|
|
209
|
+
networkErrors: networkSummary.failedRequests,
|
|
210
|
+
topFailedUrls: networkSummary.topFailedUrls
|
|
211
|
+
}
|
|
212
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
213
|
+
findings.push(finding);
|
|
214
|
+
} else if (consoleSummary.hasErrors) {
|
|
215
|
+
const finding = createFindingWithConfidence({
|
|
216
|
+
type: 'validation_silent_failure',
|
|
217
|
+
interaction: {
|
|
218
|
+
type: interaction.type,
|
|
219
|
+
selector: interaction.selector,
|
|
220
|
+
label: interaction.label
|
|
221
|
+
},
|
|
222
|
+
reason: 'Navigation blocked by client-side validation errors',
|
|
223
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
224
|
+
evidence: {
|
|
225
|
+
before: beforeScreenshot,
|
|
226
|
+
after: afterScreenshot,
|
|
227
|
+
beforeUrl: beforeUrl,
|
|
228
|
+
afterUrl: afterUrl,
|
|
229
|
+
consoleErrors: consoleSummary.consoleErrorCount,
|
|
230
|
+
errorMessages: consoleSummary.lastErrors?.slice(0, 3)
|
|
231
|
+
}
|
|
232
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
233
|
+
findings.push(finding);
|
|
234
|
+
} else if (networkSummary.slowRequestsCount > 0 && !uiSignalChanges.changed) {
|
|
235
|
+
const finding = createFindingWithConfidence({
|
|
236
|
+
type: 'missing_feedback_failure',
|
|
237
|
+
interaction: {
|
|
238
|
+
type: interaction.type,
|
|
239
|
+
selector: interaction.selector,
|
|
240
|
+
label: interaction.label
|
|
241
|
+
},
|
|
242
|
+
reason: 'Slow navigation request with no loading indicator',
|
|
243
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
244
|
+
evidence: {
|
|
245
|
+
before: beforeScreenshot,
|
|
246
|
+
after: afterScreenshot,
|
|
247
|
+
beforeUrl: beforeUrl,
|
|
248
|
+
afterUrl: afterUrl,
|
|
249
|
+
slowRequests: networkSummary.slowRequestsCount,
|
|
250
|
+
slowRequestDurations: networkSummary.slowRequests?.map(r => r.duration)
|
|
251
|
+
}
|
|
252
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
253
|
+
findings.push(finding);
|
|
254
|
+
} else {
|
|
255
|
+
const finding = createFindingWithConfidence({
|
|
256
|
+
type: 'no_effect_silent_failure',
|
|
257
|
+
interaction: {
|
|
258
|
+
type: interaction.type,
|
|
259
|
+
selector: interaction.selector,
|
|
260
|
+
label: interaction.label
|
|
261
|
+
},
|
|
262
|
+
reason: 'Expected user-visible outcome did not occur',
|
|
263
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
264
|
+
evidence: {
|
|
265
|
+
before: beforeScreenshot,
|
|
266
|
+
after: afterScreenshot,
|
|
267
|
+
beforeUrl: beforeUrl,
|
|
268
|
+
afterUrl: afterUrl
|
|
269
|
+
}
|
|
270
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
271
|
+
findings.push(finding);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
} else if (expectationType === 'network_action') {
|
|
275
|
+
// Wave 5: Network action expectations from AST action contracts
|
|
276
|
+
// Check if promised network request actually occurred
|
|
277
|
+
if (networkSummary.totalRequests === 0) {
|
|
278
|
+
// Code promised a fetch/axios call but no network activity detected
|
|
279
|
+
const finding = createFindingWithConfidence({
|
|
280
|
+
type: 'missing_network_action',
|
|
281
|
+
interaction: {
|
|
282
|
+
type: interaction.type,
|
|
283
|
+
selector: interaction.selector,
|
|
284
|
+
label: interaction.label
|
|
285
|
+
},
|
|
286
|
+
reason: 'Code contract promises network request but none was made',
|
|
287
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
288
|
+
evidence: {
|
|
289
|
+
before: beforeScreenshot,
|
|
290
|
+
after: afterScreenshot,
|
|
291
|
+
beforeUrl: beforeUrl,
|
|
292
|
+
afterUrl: afterUrl,
|
|
293
|
+
expectedMethod: expectationInfo.method,
|
|
294
|
+
expectedUrl: expectationInfo.urlPath,
|
|
295
|
+
sourceRef: attemptMeta.sourceRef
|
|
296
|
+
}
|
|
297
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
298
|
+
findings.push(finding);
|
|
299
|
+
}
|
|
300
|
+
// Check for network errors without feedback
|
|
301
|
+
else if (networkSummary.failedRequests > 0 && !uiSignalChanges.changed) {
|
|
302
|
+
const finding = createFindingWithConfidence({
|
|
303
|
+
type: 'network_silent_failure',
|
|
304
|
+
interaction: {
|
|
305
|
+
type: interaction.type,
|
|
306
|
+
selector: interaction.selector,
|
|
307
|
+
label: interaction.label
|
|
308
|
+
},
|
|
309
|
+
reason: 'Network action failed with no UI feedback',
|
|
310
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
311
|
+
evidence: {
|
|
312
|
+
before: beforeScreenshot,
|
|
313
|
+
after: afterScreenshot,
|
|
314
|
+
beforeUrl: beforeUrl,
|
|
315
|
+
afterUrl: afterUrl,
|
|
316
|
+
networkErrors: networkSummary.failedRequests,
|
|
317
|
+
failedByStatus: networkSummary.failedByStatus,
|
|
318
|
+
topFailedUrls: networkSummary.topFailedUrls,
|
|
319
|
+
expectedMethod: expectationInfo.method,
|
|
320
|
+
expectedUrl: expectationInfo.urlPath,
|
|
321
|
+
sourceRef: attemptMeta.sourceRef
|
|
322
|
+
}
|
|
323
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
324
|
+
findings.push(finding);
|
|
325
|
+
}
|
|
326
|
+
// Check for missing feedback on slow requests
|
|
327
|
+
else if (networkSummary.slowRequestsCount > 0 && !uiSignalChanges.changed) {
|
|
328
|
+
const finding = createFindingWithConfidence({
|
|
329
|
+
type: 'missing_feedback_failure',
|
|
330
|
+
interaction: {
|
|
331
|
+
type: interaction.type,
|
|
332
|
+
selector: interaction.selector,
|
|
333
|
+
label: interaction.label
|
|
334
|
+
},
|
|
335
|
+
reason: 'Slow network action with no loading feedback',
|
|
336
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
337
|
+
evidence: {
|
|
338
|
+
before: beforeScreenshot,
|
|
339
|
+
after: afterScreenshot,
|
|
340
|
+
beforeUrl: beforeUrl,
|
|
341
|
+
afterUrl: afterUrl,
|
|
342
|
+
slowRequests: networkSummary.slowRequestsCount,
|
|
343
|
+
expectedMethod: expectationInfo.method,
|
|
344
|
+
expectedUrl: expectationInfo.urlPath,
|
|
345
|
+
sourceRef: attemptMeta.sourceRef
|
|
346
|
+
}
|
|
347
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
348
|
+
findings.push(finding);
|
|
349
|
+
}
|
|
350
|
+
} else if (expectationType === 'state_action') {
|
|
351
|
+
// Wave 8: State action expectations from AST state contracts
|
|
352
|
+
const stateUIData = sensorData.stateUI || {};
|
|
353
|
+
const stateChanged = stateUIData.changed === true;
|
|
354
|
+
|
|
355
|
+
// Check if promised state mutation actually occurred
|
|
356
|
+
if (!stateChanged && !hasUrlChange && !hasDomChangeResult && networkSummary.totalRequests === 0) {
|
|
357
|
+
const finding = createFindingWithConfidence({
|
|
358
|
+
type: 'missing_state_action',
|
|
359
|
+
interaction: {
|
|
360
|
+
type: interaction.type,
|
|
361
|
+
selector: interaction.selector,
|
|
362
|
+
label: interaction.label
|
|
363
|
+
},
|
|
364
|
+
reason: `Code contract promises state mutation (${expectationInfo.stateKind}) but state UI did not change`,
|
|
365
|
+
expectationProof: ExpectationProof.PROVEN_EXPECTATION,
|
|
366
|
+
evidence: {
|
|
367
|
+
before: beforeScreenshot,
|
|
368
|
+
after: afterScreenshot,
|
|
369
|
+
beforeUrl: beforeUrl,
|
|
370
|
+
afterUrl: afterUrl,
|
|
371
|
+
expectedStateKind: expectationInfo.stateKind,
|
|
372
|
+
sourceRef: attemptMeta.sourceRef,
|
|
373
|
+
handlerRef: attemptMeta.handlerRef,
|
|
374
|
+
stateUIReasons: stateUIData.reasons || []
|
|
375
|
+
}
|
|
376
|
+
}, expectationInfo, sensorData, { hasUrlChange, hasDomChange: hasDomChangeResult, hasVisibleChange: hasVisibleChangeResult });
|
|
377
|
+
findings.push(finding);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
const findingsResult = writeFindings(projectDir, observation.url, findings, artifactPaths);
|
|
383
|
+
|
|
384
|
+
const skipSummary = collectSkipReasons(skips);
|
|
385
|
+
|
|
386
|
+
const detectTruth = {
|
|
387
|
+
interactionsAnalyzed: interactionsAnalyzed,
|
|
388
|
+
interactionsSkippedNoExpectation: skipSummary.total,
|
|
389
|
+
findingsCount: findings.length,
|
|
390
|
+
skips: skipSummary
|
|
391
|
+
};
|
|
392
|
+
|
|
393
|
+
return {
|
|
394
|
+
...findingsResult,
|
|
395
|
+
detectTruth: detectTruth
|
|
396
|
+
};
|
|
397
|
+
}
|