@veraxhq/verax 0.2.1 → 0.4.0

package/src/verax/detect/test-failure-cause-inference.js

@@ -0,0 +1,314 @@
+#!/usr/bin/env node
+
+/**
+ * Test: Failure Cause Inference (FCI)
+ * 
+ * Tests:
+ * 1. No evidence -> no causes (Evidence Law enforcement)
+ * 2. Determinism: same input -> same causes, same ordering, same wording
+ * 3. Cause inference for C2 (state mutation no UI)
+ * 4. Cause inference for C3 (dead click)
+ * 5. Cause inference for C7 (network silent)
+ */
+
+import {
+  inferCauses,
+  inferCausesForFindings,
+  findingsWithCauses,
+  attachCausesToFinding
+} from './failure-cause-inference.js';
+
+let testCount = 0;
+let passCount = 0;
+
+function test(name, fn) {
+  testCount++;
+  try {
+    fn();
+    console.log(`✓ Test ${testCount}: ${name}`);
+    passCount++;
+  } catch (e) {
+    console.error(`✗ Test ${testCount}: ${name}`);
+    console.error(`  ${e.message}`);
+  }
+}
+
+function assert(condition, message) {
+  if (!condition) {
+    throw new Error(message);
+  }
+}
+
+// Test 1: No evidence -> no causes
+test('No evidence => no causes (Evidence Law)', () => {
+  const finding = {
+    id: 'test-1',
+    type: 'silent_failure',
+    evidence: {}
+  };
+  const causes = inferCauses(finding);
+  assert(Array.isArray(causes), 'Should return array');
+  assert(causes.length === 0, `Expected 0 causes, got ${causes.length}`);
+});
+
+// Test 2: Null/undefined finding
+test('Null finding => no causes', () => {
+  const causes = inferCauses(null);
+  assert(causes.length === 0, 'Should return empty array for null');
+});
+
+// Test 3: Missing evidence field
+test('Missing evidence field => no causes', () => {
+  const finding = {
+    id: 'test-2',
+    type: 'silent_failure'
+  };
+  const causes = inferCauses(finding);
+  assert(causes.length === 0, 'Should return empty array');
+});
+
+// Test 4: Determinism - run twice, get identical output
+test('Determinism: same finding => same causes', () => {
+  const finding = {
+    id: 'test-det-1',
+    type: 'silent_failure',
+    evidence: {
+      stateMutation: true,
+      domChanged: false,
+      navigationOccurred: false,
+      uiFeedback: false
+    }
+  };
+  const run1 = inferCauses(finding);
+  const run2 = inferCauses(finding);
+  
+  assert(JSON.stringify(run1) === JSON.stringify(run2), 'Runs should be identical');
+  assert(run1.length > 0, 'Should detect C2 cause');
+  assert(run1[0].id === 'C2_STATE_MUTATION_NO_UI', 'Should detect C2');
+});
+
+// Test 5: Determinism - ordering
+test('Determinism: causes ordered by id', () => {
+  const finding = {
+    id: 'test-ord-1',
+    type: 'silent_failure',
+    evidence: {
+      stateMutation: true,
+      domChanged: false,
+      navigationOccurred: false,
+      uiFeedback: false,
+      interactionPerformed: true,
+      networkActivity: false,
+      userFeedback: false
+    }
+  };
+  const causes = inferCauses(finding);
+  // Should have both C2 and C3
+  assert(causes.length >= 1, 'Should detect at least 1 cause');
+  
+  // Check ordering
+  for (let i = 1; i < causes.length; i++) {
+    assert(
+      causes[i].id.localeCompare(causes[i-1].id) > 0,
+      `Causes not ordered: ${causes[i].id} should come after ${causes[i-1].id}`
+    );
+  }
+});
+
+// Test 6: C2 inference - state mutation no UI
+test('C2: State mutation + no DOM change + no feedback', () => {
+  const finding = {
+    id: 'c2-test',
+    type: 'state_action',
+    evidence: {
+      stateMutation: true,
+      domChanged: false,
+      navigationOccurred: false,
+      uiFeedback: false
+    }
+  };
+  const causes = inferCauses(finding);
+  assert(causes.length >= 1, 'Should find at least one cause');
+  assert(causes.some(c => c.id === 'C2_STATE_MUTATION_NO_UI'), 'Should find C2');
+  
+  const c2 = causes.find(c => c.id === 'C2_STATE_MUTATION_NO_UI');
+  assert(c2.statement.includes('Likely cause:'), 'Should start with "Likely cause:"');
+  assert(c2.confidence === 'MEDIUM', 'Should be MEDIUM confidence');
+  assert(Array.isArray(c2.evidence_refs), 'Should have evidence_refs array');
+  assert(c2.evidence_refs.length > 0, 'Should have evidence references');
+});
+
+// Test 7: C3 inference - dead click
+test('C3: Interaction but no network/nav/DOM/feedback', () => {
+  const finding = {
+    id: 'c3-test',
+    type: 'silent_failure',
+    evidence: {
+      interactionPerformed: true,
+      networkActivity: false,
+      navigationOccurred: false,
+      domChanged: false,
+      userFeedback: false
+    }
+  };
+  const causes = inferCauses(finding);
+  assert(causes.some(c => c.id === 'C3_DEAD_CLICK'), 'Should find C3');
+  
+  const c3 = causes.find(c => c.id === 'C3_DEAD_CLICK');
+  assert(c3.title.includes('dead'), 'Title should mention dead');
+  assert(c3.confidence === 'MEDIUM', 'Should be MEDIUM confidence');
+});
+
+// Test 8: C7 inference - network silent
+test('C7: Network failure + no feedback', () => {
+  const finding = {
+    id: 'c7-test',
+    type: 'network_silent_failure',
+    evidence: {
+      networkFailure: true,
+      uiFeedback: false,
+      domChanged: false
+    }
+  };
+  const causes = inferCauses(finding);
+  assert(causes.some(c => c.id === 'C7_NETWORK_SILENT'), 'Should find C7');
+  
+  const c7 = causes.find(c => c.id === 'C7_NETWORK_SILENT');
+  assert(c7.statement.includes('network'), 'Should mention network');
+  assert(c7.confidence === 'MEDIUM', 'Should be MEDIUM confidence');
+});
+
+// Test 9: Batch inference
+test('Batch inference on multiple findings', () => {
+  const findings = [
+    {
+      id: 'batch-1',
+      type: 'silent_failure',
+      evidence: { }
+    },
+    {
+      id: 'batch-2',
+      type: 'state_action',
+      evidence: {
+        stateMutation: true,
+        domChanged: false,
+        navigationOccurred: false,
+        uiFeedback: false
+      }
+    }
+  ];
+  const causesMap = inferCausesForFindings(findings);
+  assert('batch-1' in causesMap === false, 'batch-1 has no evidence, should not be in map');
+  assert('batch-2' in causesMap, 'batch-2 should be in map');
+  assert(causesMap['batch-2'].some(c => c.id === 'C2_STATE_MUTATION_NO_UI'), 'batch-2 should have C2');
+});
+
+// Test 10: findingsWithCauses filter
+test('findingsWithCauses filters correctly', () => {
+  const findings = [
+    {
+      id: 'with-evidence',
+      type: 'state_action',
+      evidence: {
+        stateMutation: true,
+        domChanged: false,
+        navigationOccurred: false,
+        uiFeedback: false
+      }
+    },
+    {
+      id: 'no-evidence',
+      type: 'silent_failure',
+      evidence: { }
+    }
+  ];
+  const filtered = findingsWithCauses(findings);
+  assert(filtered.length === 1, `Expected 1 finding with causes, got ${filtered.length}`);
+  assert(filtered[0].id === 'with-evidence', 'Should only include finding with causes');
+});
+
+// Test 11: attachCausesToFinding mutation
+test('attachCausesToFinding mutates finding', () => {
+  const finding = {
+    id: 'mutate-test',
+    type: 'state_action',
+    evidence: {
+      stateMutation: true,
+      domChanged: false,
+      navigationOccurred: false,
+      uiFeedback: false
+    }
+  };
+  assert(!('causes' in finding), 'Should not have causes before');
+  attachCausesToFinding(finding);
+  assert('causes' in finding, 'Should have causes after');
+  assert(finding.causes.length > 0, 'Should have non-empty causes array');
+});
+
+// Test 12: Cause statement format
+test('Cause statements start with "Likely cause:"', () => {
+  const finding = {
+    id: 'format-test',
+    type: 'state_action',
+    evidence: {
+      stateMutation: true,
+      domChanged: false,
+      navigationOccurred: false,
+      uiFeedback: false
+    }
+  };
+  const causes = inferCauses(finding);
+  causes.forEach(cause => {
+    assert(
+      cause.statement.startsWith('Likely cause:'),
+      `Statement should start with "Likely cause:", got: ${cause.statement}`
+    );
+  });
+});
+
+// Test 13: Confidence never HIGH
+test('Confidence is never HIGH', () => {
+  const findings = [
+    {
+      id: 't1',
+      type: 'state_action',
+      evidence: {
+        stateMutation: true,
+        domChanged: false,
+        navigationOccurred: false,
+        uiFeedback: false
+      }
+    },
+    {
+      id: 't2',
+      type: 'silent_failure',
+      evidence: {
+        interactionPerformed: true,
+        networkActivity: false,
+        navigationOccurred: false,
+        domChanged: false,
+        userFeedback: false
+      }
+    }
+  ];
+  
+  findings.forEach(finding => {
+    const causes = inferCauses(finding);
+    causes.forEach(cause => {
+      assert(
+        cause.confidence === 'LOW' || cause.confidence === 'MEDIUM',
+        `Confidence should be LOW or MEDIUM, got ${cause.confidence}`
+      );
+    });
+  });
+});
+
+// Results
+console.log(`\n${passCount}/${testCount} tests passed`);
+if (passCount === testCount) {
+  console.log('✓ All tests passed');
+  process.exit(0);
+} else {
+  console.log(`✗ ${testCount - passCount} test(s) failed`);
+  process.exit(1);
+}

package/src/verax/detect/ui-feedback-findings.js

@@ -0,0 +1,207 @@
+/**
+ * PHASE 13 — UI Feedback Findings Detector
+ * 
+ * Detects UI feedback-related silent failures by correlating promises
+ * with feedback signals and evaluating outcomes.
+ */
+
+import {
+  detectUIFeedbackSignals,
+  scoreUIFeedback,
+  correlatePromiseWithFeedback,
+  buildUIFeedbackEvidence,
+  FEEDBACK_SCORE,
+} from '../core/ui-feedback-intelligence.js';
+import { computeConfidence } from './confidence-engine.js';
+import { computeConfidenceForFinding } from '../core/confidence-engine.js';
+import { buildAndEnforceEvidencePackage } from '../core/evidence-builder.js';
+import { applyGuardrails } from '../core/guardrails-engine.js';
+
+/**
+ * PHASE 13: Detect UI feedback-related findings
+ * 
+ * @param {Array} traces - Interaction traces
+ * @param {Object} manifest - Project manifest with expectations
+ * @param {Array} _findings - Findings array to append to
+ * @ts-expect-error - JSDoc param documented but unused
+ * @returns {Array} UI feedback-related findings
+ */
+export function detectUIFeedbackFindings(traces, manifest, _findings) {
+  const feedbackFindings = [];
+  
+  // Process each trace
+  for (const trace of traces) {
+    const interaction = trace.interaction || {};
+    
+    // Find expectations for this interaction
+    const expectations = findExpectationsForInteraction(manifest, interaction, trace);
+    
+    for (const expectation of expectations) {
+      // Detect UI feedback signals
+      const signals = detectUIFeedbackSignals(trace);
+      
+      // Score feedback presence/absence
+      const feedbackScore = scoreUIFeedback(signals, expectation, trace);
+      
+      // Correlate promise with feedback
+      const correlation = correlatePromiseWithFeedback(expectation, feedbackScore, trace);
+      
+      // Generate finding if correlation indicates silent failure
+      if (correlation.outcome === 'CONFIRMED' || correlation.outcome === 'SUSPECTED') {
+        // Build evidence
+        const evidence = buildUIFeedbackEvidence(feedbackScore, correlation, trace, expectation);
+        
+        // PHASE 13: Evidence Law - require sufficient evidence for CONFIRMED
+        const hasSufficientEvidence = evidence.beforeAfter.beforeScreenshot &&
+                                      evidence.beforeAfter.afterScreenshot &&
+                                      (evidence.feedback.signals.length > 0 || 
+                                       evidence.feedback.score === FEEDBACK_SCORE.MISSING);
+        
+        // Determine finding type early (before use in confidence call)
+        let findingType = 'ui_feedback_silent_failure';
+        if (expectation.type === 'network_action' || expectation.type === 'network') {
+          findingType = 'network_feedback_missing';
+        } else if (expectation.type === 'navigation' || expectation.type === 'spa_navigation') {
+          findingType = 'navigation_feedback_missing';
+        } else if (expectation.type === 'validation' || expectation.type === 'form_submission') {
+          findingType = 'validation_feedback_missing';
+        }
+        
+        // PHASE 15: Compute unified confidence
+        const unifiedConfidence = computeConfidenceForFinding({
+          findingType: findingType,
+          expectation,
+          sensors: trace.sensors || {},
+          comparisons: {},
+          evidence,
+          options: {}
+        });
+        
+        // Legacy confidence for backward compatibility
+        const _confidence = computeConfidence({
+          findingType: 'ui_feedback_silent_failure',
+          expectation,
+          sensors: trace.sensors || {},
+          comparisons: {},
+          attemptMeta: {},
+        });
+        
+        // Determine severity based on evidence
+        const severity = hasSufficientEvidence && correlation.outcome === 'CONFIRMED' && (unifiedConfidence.score01 || unifiedConfidence.score || 0) >= 0.8
+          ? 'CONFIRMED'
+          : 'SUSPECTED';
+        
+        const finding = {
+          type: findingType,
+          severity,
+          confidence: unifiedConfidence.score01 || unifiedConfidence.score || 0, // Contract v1: score01 canonical
+          confidenceLevel: unifiedConfidence.level, // PHASE 15: Add confidence level
+          confidenceReasons: unifiedConfidence.topReasons || unifiedConfidence.reasons || [], // Contract v1: topReasons
+          interaction: {
+            type: interaction.type,
+            selector: interaction.selector,
+            label: interaction.label,
+          },
+          reason: correlation.reason || feedbackScore.explanation,
+          evidence,
+          source: {
+            file: expectation.source?.file || null,
+            line: expectation.source?.line || null,
+            column: expectation.source?.column || null,
+            context: expectation.source?.context || null,
+            astSource: expectation.source?.astSource || null,
+          },
+        };
+        
+        // PHASE 16: Build and enforce evidence package
+        const findingWithEvidence = buildAndEnforceEvidencePackage(finding, {
+          expectation,
+          trace,
+          evidence,
+          confidence: unifiedConfidence,
+        });
+        
+        // PHASE 17: Apply guardrails (AFTER evidence builder)
+        const context = {
+          evidencePackage: findingWithEvidence.evidencePackage,
+          signals: findingWithEvidence.evidencePackage?.signals || evidence.signals || {},
+          confidenceReasons: unifiedConfidence.reasons || [],
+          promiseType: expectation?.type || null,
+        };
+        const { finding: findingWithGuardrails } = applyGuardrails(findingWithEvidence, context);
+        
+        feedbackFindings.push(findingWithGuardrails);
+      }
+    }
+  }
+  
+  return feedbackFindings;
+}
+
+/**
+ * Find expectations matching the interaction
+ */
+function findExpectationsForInteraction(manifest, interaction, trace) {
+  const expectations = [];
+  
+  // Check static expectations
+  if (manifest.staticExpectations) {
+    const beforeUrl = trace.before?.url || trace.sensors?.navigation?.beforeUrl || '';
+    const beforePath = extractPathFromUrl(beforeUrl);
+    
+    if (beforePath) {
+      const normalizedBefore = beforePath.replace(/\/$/, '') || '/';
+      
+      for (const expectation of manifest.staticExpectations) {
+        const normalizedFrom = (expectation.fromPath || '').replace(/\/$/, '') || '/';
+        if (normalizedFrom === normalizedBefore) {
+          // Check selector match
+          const selectorHint = expectation.selectorHint || '';
+          const interactionSelector = interaction.selector || '';
+          
+          if (!selectorHint || !interactionSelector || 
+              selectorHint === interactionSelector ||
+              selectorHint.includes(interactionSelector) ||
+              interactionSelector.includes(selectorHint)) {
+            expectations.push(expectation);
+          }
+        }
+      }
+    }
+  }
+  
+  // Check expectations from intel (AST-based)
+  if (manifest.expectations) {
+    for (const expectation of manifest.expectations) {
+      // Match by selector or label
+      const selectorHint = expectation.selectorHint || '';
+      const interactionSelector = interaction.selector || '';
+      const interactionLabel = (interaction.label || '').toLowerCase();
+      const expectationLabel = (expectation.promise?.value || '').toLowerCase();
+      
+      if (selectorHint === interactionSelector ||
+          (expectationLabel && interactionLabel && expectationLabel.includes(interactionLabel))) {
+        expectations.push(expectation);
+      }
+    }
+  }
+  
+  return expectations;
+}
+
+/**
+ * Extract path from URL
+ */
+function extractPathFromUrl(url) {
+  if (!url || typeof url !== 'string') return '';
+  
+  try {
+    const urlObj = new URL(url);
+    return urlObj.pathname;
+  } catch {
+    // Relative URL
+    const pathMatch = url.match(/^([^?#]+)/);
+    return pathMatch ? pathMatch[1] : url;
+  }
+}
+