@veraxhq/verax 0.2.1 → 0.4.0

package/src/cli/util/console-reporter.js

@@ -0,0 +1,72 @@
+/**
+ * 
+ * Formats and prints standardized CLI output for VERAX runs.
+ * 
+ * Output format (decision-oriented, minimal):
+ * - ✔ VERAX finished
+ * - ✔ 14 interactions tested
+ * - ✖ 3 user problems found (2 HIGH, 1 MEDIUM)
+ * - → Open .verax/SUMMARY.md
+ */
+
+/**
+ * Format console output for a completed run
+ * 
+ * @param {Object} stats - Run statistics
+ * @param {number} stats.flowsScanned - Public flows scanned
+ * @param {number} stats.silentFailures - Silent failures detected
+ * @param {string} stats.outDir - Output directory path
+ * @returns {string} Formatted output (multiple lines)
+ */
+export function formatConsoleOutput(stats) {
+  const lines = [];
+  
+  const flowsScanned = stats?.flowsScanned || 0;
+  const silentFailures = stats?.silentFailures || 0;
+  const outDir = stats?.outDir || '.verax';
+  
+  // Line 1: Scan complete summary
+  const flowsStr = `${flowsScanned} flow${flowsScanned !== 1 ? 's' : ''}`;
+  const failuresStr = `${silentFailures} silent failure${silentFailures !== 1 ? 's' : ''}`;
+  lines.push(`✓ Scan complete — ${flowsStr} checked, ${failuresStr} found`);
+  
+  // Line 2: Point to SUMMARY.md
+  lines.push(`→ See ${outDir}/SUMMARY.md for details`);
+  
+  // Line 3: Mention output directory
+  lines.push('');
+  lines.push(`Output saved to ${outDir}/`);
+  lines.push(`  • SUMMARY.md — Human-readable findings`);
+  lines.push(`  • REPORT.json — Machine-readable results`);
+  
+  return lines.join('\n');
+}
+
+/**
+ * Print formatted console output to stdout
+ */
+export function printConsoleOutput(stats) {
+  const output = formatConsoleOutput(stats);
+  console.log(output);
+  return output;
+}
+
+/**
+ * Format and print error output when fatal error occurs
+ * 
+ * @param {string} message - Error message
+ * @param {boolean} [_debug] - If true, include more details; if false, minimal
+ */
+export function formatErrorOutput(message, _debug = false) {
+  // Single-line error message with [ERROR] prefix
+  return `[ERROR] ${message}`;
+}
+
+/**
+ * Print error output
+ */
+export function printErrorOutput(message, _debug = false) {
+  const output = formatErrorOutput(message, _debug);
+  console.error(output);
+  return output;
+}

package/src/cli/util/detection-engine.js

@@ -75,6 +75,14 @@ export async function detectFindings(learnData, observeData, projectPath, onProg
     findings,
     stats,
     detectedAt: new Date().toISOString(),
+    enforcement: {
+      evidenceLawEnforced: true,
+      contractVersion: 1,
+      timestamp: new Date().toISOString(),
+      droppedCount: 0,
+      downgradedCount: 0,
+      downgrades: []
+    }
   };
 }
 
@@ -100,7 +108,9 @@ function getObservationForExpectation(expectation, observationMap) {
 }
 
 function classificationIcon(classification) {
-  switch (classification) {
+  // Handle both old format and new taxonomy format
+  const baseClassification = classification.split(':')[0];
+  switch (baseClassification) {
     case 'observed':
       return '✓';
     case 'silent-failure':
@@ -115,7 +125,9 @@ function classificationIcon(classification) {
 }
 
 function findingStatKey(classification) {
-  switch (classification) {
+  // Handle both old format and new taxonomy format
+  const baseClassification = classification.split(':')[0];
+  switch (baseClassification) {
     case 'silent-failure':
       return 'silentFailures';
     case 'observed':
@@ -131,6 +143,8 @@ function findingStatKey(classification) {
 
 /**
  * Classify a single expectation according to deterministic rules.
+ * EVIDENCE GATE: silent-failure REQUIRES evidence.
+ * OUTCOME BINDING: Use attempt.cause to provide precise taxonomy.
  */
 function classifyExpectation(expectation, observation) {
   const finding = {
@@ -148,13 +162,15 @@ function classifyExpectation(expectation, observation) {
   const attempted = Boolean(observation?.attempted);
   const observed = observation?.observed === true;
   const reason = observation?.reason || null;
+  const cause = observation?.cause || null; // NEW: precise cause from planner
 
   const evidence = normalizeEvidence(observation?.evidenceFiles || []);
   finding.evidence = evidence;
 
   const evidenceSignals = analyzeEvidenceSignals(observation, evidence);
+  const hasAnyEvidence = evidence.length > 0 || evidenceSignals.hasDomChange;
 
-  // 1) observed
+  // 1) observed (success)
   if (observed) {
     finding.classification = 'observed';
     finding.reason = 'Expectation observed at runtime';
@@ -163,8 +179,8 @@ function classifyExpectation(expectation, observation) {
     return finding;
   }
 
-  // 2) coverage-gap (not attempted or explicitly blocked)
-  if (!attempted || isSafetySkip(reason)) {
+  // 2) coverage-gap (not attempted or safety skip)
+  if (!attempted) {
     finding.classification = 'coverage-gap';
     finding.reason = reason || 'No observation attempt recorded';
     finding.impact = 'LOW';
@@ -172,29 +188,48 @@ function classifyExpectation(expectation, observation) {
     return finding;
   }
 
-  // 3) silent-failure (attempted, observed === false, no safety skip)
-  if (attempted && observation?.observed === false && !isSafetySkip(reason)) {
-    finding.classification = 'silent-failure';
-    finding.reason = reason || 'Expected behavior not observed';
-    finding.impact = getImpact(expectation);
-    finding.confidence = calculateConfidence(expectation, evidenceSignals, 'silent-failure');
-    return finding;
-  }
-
-  // 4) unproven (attempted, ambiguous evidence)
+  // 3) Attempted but not observed - apply EVIDENCE GATE + OUTCOME BINDING
   if (attempted && !observed) {
-    finding.classification = 'unproven';
-    finding.reason = reason || 'Attempted but evidence insufficient';
-    finding.impact = 'MEDIUM';
-    finding.confidence = calculateConfidence(expectation, evidenceSignals, 'unproven');
+    // CRITICAL: Evidence Gate - silent-failure REQUIRES evidence
+    if (!hasAnyEvidence) {
+      // NO EVIDENCE → cannot prove silence → coverage-gap or unproven
+      if (isSafetySkip(reason)) {
+        finding.classification = 'coverage-gap';
+        finding.reason = reason || 'Blocked or skipped for safety';
+        finding.impact = 'LOW';
+        finding.confidence = 0;
+      } else {
+        finding.classification = 'unproven';
+        finding.reason = reason || 'Attempted but no evidence captured';
+        finding.impact = 'MEDIUM';
+        finding.confidence = 0;
+      }
+      return finding;
+    }
+
+    // HAS EVIDENCE → can classify as silent-failure with PRECISE taxonomy
+    let taxonomy = 'no-change'; // default
+    
+    if (cause) {
+      // Use the cause from interaction planner (most precise)
+      taxonomy = cause; // 'not-found' | 'blocked' | 'prevented-submit' | 'timeout' | 'no-change' | 'error'
+    } else {
+      // Fallback to signal-based detection (legacy)
+      taxonomy = determineSilenceTaxonomy(reason, evidenceSignals);
+    }
+    
+    finding.classification = `silent-failure:${taxonomy}`;
+    finding.reason = reason || `Expected behavior not observed (${taxonomy})`;
+    finding.impact = getImpact(expectation);
+    finding.confidence = calculateConfidenceFromEvidence(evidenceSignals);
     return finding;
   }
 
-  // 5) informational fallback
+  // 4) Fallback
   finding.classification = 'informational';
   finding.reason = reason || 'No classification rule matched';
   finding.impact = 'LOW';
-  finding.confidence = calculateConfidence(expectation, evidenceSignals, 'informational');
+  finding.confidence = 0;
   return finding;
 }
 
@@ -206,32 +241,61 @@ function isSafetySkip(reason) {
 }
 
 /**
- * Deterministic confidence calculation.
- * Screenshots + network + DOM change => >=0.8
- * Screenshots only => ~0.6
- * Weak signals => <0.5
+ * Determine silence taxonomy based on reason and evidence signals.
+ * Returns: no-change | blocked | not-found | timeout | prevented-submit
  */
-function calculateConfidence(expectation, evidenceSignals, classification) {
-  if (classification === 'observed') return 1.0;
-  if (classification === 'coverage-gap') return 0;
+function determineSilenceTaxonomy(reason, evidenceSignals) {
+  if (!reason) {
+    // No explicit reason - check evidence
+    if (evidenceSignals.hasScreenshots || evidenceSignals.hasDomChange) {
+      return 'no-change'; // Evidence exists but no observed change
+    }
+    return 'no-change';
+  }
 
-  const { hasScreenshots, hasNetworkLogs, hasDomChange } = evidenceSignals;
+  const lower = reason.toLowerCase();
 
-  if (classification === 'silent-failure') {
-    if (hasScreenshots && hasNetworkLogs && hasDomChange) return 0.85;
-    if (hasScreenshots && hasNetworkLogs) return 0.75;
-    if (hasScreenshots && hasDomChange) return 0.7;
-    if (hasScreenshots) return 0.6;
-    if (hasNetworkLogs || hasDomChange) return 0.5;
-    return 0.4; // weak signals, attempted but no evidence
+  // Check for specific conditions
+  if (lower.includes('timeout') || lower.includes('timed out')) {
+    return 'timeout';
   }
-
-  if (classification === 'unproven') {
-    if (hasScreenshots || hasNetworkLogs || hasDomChange) return 0.45;
-    return 0.3;
+  if (lower.includes('not found') || lower.includes('element not found') || lower.includes('selector not found') || lower.includes('not-found')) {
+    return 'not-found';
+  }
+  if (lower.includes('blocked') || lower.includes('not-interactable') || lower.includes('interactable')) {
+    return 'blocked';
   }
+  if (lower.includes('prevented') || lower.includes('prevented-submit') || lower.includes('submit-prevented')) {
+    return 'prevented-submit';
+  }
+  if (lower.includes('no matching event') || lower.includes('no change') || lower.includes('no-change')) {
+    return 'no-change';
+  }
+
+  // Default to no-change if evidence exists
+  return 'no-change';
+}
+
+/**
+ * Calculate confidence from evidence ONLY.
+ * No evidence = 0 confidence.
+ */
+function calculateConfidenceFromEvidence(evidenceSignals) {
+  const { hasScreenshots, hasNetworkLogs, hasDomChange } = evidenceSignals;
 
-  return 0.3;
+  // Multiple strong signals
+  if (hasScreenshots && hasNetworkLogs && hasDomChange) return 0.85;
+  if (hasScreenshots && hasNetworkLogs) return 0.75;
+  if (hasScreenshots && hasDomChange) return 0.7;
+  
+  // Single strong signal
+  if (hasScreenshots) return 0.6;
+  
+  // Weak signals
+  if (hasNetworkLogs || hasDomChange) return 0.5;
+  
+  // No evidence
+  return 0;
 }
 
 function analyzeEvidenceSignals(observation, evidence) {

package/src/cli/util/determinism-runner.js

@@ -0,0 +1,124 @@
+/**
+ * PHASE 18 — Determinism Runner
+ * 
+ * Wraps a scan execution to run it multiple times and compare results.
+ */
+
+import { resolve } from 'path';
+import { readFileSync, existsSync } from 'fs';
+import { runDeterminismCheck } from '../../verax/core/determinism/engine.js';
+import { verifyRun } from '../../verax/core/artifacts/verifier.js';
+import { writeDeterminismReport } from './determinism-writer.js';
+
+/**
+ * PHASE 18: Run scan with determinism checking
+ * 
+ * @param {Function} scanFn - Function that executes a scan and returns { runId }
+ * @param {Object} options - Options
+ * @param {number} options.runs - Number of runs (default: 2)
+ * @param {string} options.out - Output directory
+ * @returns {Promise<Object>} Determinism check results
+ */
+export async function runWithDeterminism(scanFn, options = { runs: 2, out: '.verax' }) {
+  const { runs = 2, out = '.verax' } = options;
+  
+  // Wrap scan function to return artifact paths
+  const runFn = async (runConfig) => {
+    const result = await scanFn({ ...options, ...runConfig });
+    
+    // Extract artifact paths from result
+    const artifactPaths = {};
+    if (result.runId) {
+      const runDir = resolve(out, 'runs', result.runId);
+      
+      // Map artifact keys to paths
+      const artifactMap = {
+        findings: resolve(runDir, 'findings.json'),
+        runStatus: resolve(runDir, 'run.status.json'),
+        summary: resolve(runDir, 'summary.json'),
+        learn: resolve(runDir, 'learn.json'),
+        observe: resolve(runDir, 'observe.json'),
+      };
+      
+      for (const [key, path] of Object.entries(artifactMap)) {
+        if (existsSync(path)) {
+          artifactPaths[key] = path;
+        }
+      }
+    }
+    
+    return {
+      runId: result.runId,
+      artifactPaths,
+    };
+  };
+  
+  // PHASE 25: Load run fingerprints from run.meta.json
+  const loadRunFingerprints = async (runMeta) => {
+    if (runMeta.runId) {
+      const runDir = resolve(out, 'runs', runMeta.runId);
+      const metaPath = resolve(runDir, 'run.meta.json');
+      if (existsSync(metaPath)) {
+        try {
+          const metaContent = readFileSync(metaPath, 'utf-8');
+  // @ts-expect-error - readFileSync with encoding returns string
+          const meta = JSON.parse(metaContent);
+          return meta.runFingerprint || null;
+        } catch {
+          return null;
+        }
+      }
+    }
+    return null;
+  };
+  
+  // Run determinism check
+  const determinismResult = await runDeterminismCheck(runFn, {
+    runs,
+    config: options,
+    normalize: true,
+  });
+  
+  // PHASE 25: Load run fingerprints for each run
+  for (const runMeta of determinismResult.runsMeta) {
+    runMeta.runFingerprint = await loadRunFingerprints(runMeta);
+  }
+  
+  // Verify each run
+  const verificationResults = [];
+  for (const runMeta of determinismResult.runsMeta) {
+    if (runMeta.runId) {
+      const runDir = resolve(out, 'runs', runMeta.runId);
+      if (existsSync(runDir)) {
+        try {
+          const { getArtifactVersions } = await import('../../verax/core/artifacts/registry.js');
+          const verification = verifyRun(runDir, getArtifactVersions());
+          verificationResults.push({
+            runId: runMeta.runId,
+            verification,
+          });
+        } catch (error) {
+          // Verification failed
+          verificationResults.push({
+            runId: runMeta.runId,
+            verification: { ok: false, errors: [error.message] },
+          });
+        }
+      }
+    }
+  }
+  
+  // Write determinism report
+  const reportPath = await writeDeterminismReport(
+    determinismResult,
+    verificationResults,
+    out
+  );
+  
+  return {
+    ...determinismResult,
+    verificationResults,
+    reportPath,
+  };
+}
+

package/src/cli/util/determinism-writer.js

@@ -0,0 +1,129 @@
+/**
+ * PHASE 18 — Determinism Report Writer
+ * 
+ * Writes determinism check results to artifact.
+ */
+
+import { resolve } from 'path';
+import { mkdirSync, writeFileSync } from 'fs';
+import { ARTIFACT_REGISTRY, getArtifactVersions } from '../../verax/core/artifacts/registry.js'; // eslint-disable-line no-unused-vars
+
+/**
+ * PHASE 18: Write determinism report
+ * PHASE 25: Enhanced with runFingerprint, contract validation, and structured verdicts
+ * 
+ * @param {Object} determinismResult - Result from runDeterminismCheck
+ * @param {Array} verificationResults - Verification results for each run
+ * @param {string} outDir - Output directory
+ * @returns {Promise<string>} Path to determinism report
+ */
+export async function writeDeterminismReport(determinismResult, verificationResults, outDir) {
+  const reportsDir = resolve(outDir, 'determinism');
+  mkdirSync(reportsDir, { recursive: true });
+  
+  const reportId = `determinism-${Date.now()}`;
+  const reportPath = resolve(reportsDir, `${reportId}.json`);
+  
+  // PHASE 25: Check run fingerprints
+  const runFingerprints = [];
+  const runFingerprintMismatches = [];
+  for (const runMeta of determinismResult.runsMeta) {
+    if (runMeta.runFingerprint) {
+      runFingerprints.push(runMeta.runFingerprint);
+    }
+  }
+  if (runFingerprints.length > 1) {
+    const firstFingerprint = runFingerprints[0];
+    for (let i = 1; i < runFingerprints.length; i++) {
+      if (runFingerprints[i] !== firstFingerprint) {
+        runFingerprintMismatches.push({
+          runIndex: i + 1,
+          expected: firstFingerprint,
+          actual: runFingerprints[i]
+        });
+      }
+    }
+  }
+  
+  // PHASE 25: Check verifier errors
+  const verifierErrors = [];
+  for (const verification of verificationResults) {
+    if (verification.verification && !verification.verification.ok) {
+      verifierErrors.push({
+        runId: verification.runId,
+        errors: verification.verification.errors || [],
+        verdictStatus: verification.verification.verdictStatus
+      });
+    }
+  }
+  
+  // PHASE 25: Determine final verdict with expected/unexpected distinction
+  let finalVerdict = determinismResult.verdict;
+  if (runFingerprintMismatches.length > 0) {
+    finalVerdict = 'NON_DETERMINISTIC_UNEXPECTED';
+  } else if (verifierErrors.length > 0) {
+    finalVerdict = 'NON_DETERMINISTIC_UNEXPECTED';
+  } else if (determinismResult.adaptiveEvents && determinismResult.adaptiveEvents.length > 0) {
+    finalVerdict = 'NON_DETERMINISTIC_EXPECTED';
+  } else if (determinismResult.diffs && determinismResult.diffs.length > 0) {
+    const blockerDiffs = determinismResult.diffs.filter(d => d.severity === 'BLOCKER');
+    if (blockerDiffs.length > 0) {
+      finalVerdict = 'NON_DETERMINISTIC_UNEXPECTED';
+    } else {
+      finalVerdict = 'NON_DETERMINISTIC_EXPECTED';
+    }
+  }
+  
+  // PHASE 25: Build top reasons
+  const topReasons = [];
+  if (runFingerprintMismatches.length > 0) {
+    topReasons.push({ code: 'RUN_FINGERPRINT_MISMATCH', count: runFingerprintMismatches.length });
+  }
+  if (verifierErrors.length > 0) {
+    topReasons.push({ code: 'VERIFIER_ERRORS_DETECTED', count: verifierErrors.length });
+  }
+  if (determinismResult.adaptiveEvents && determinismResult.adaptiveEvents.length > 0) {
+    topReasons.push({ code: 'EXPECTED_ADAPTIVE_BEHAVIOR', count: determinismResult.adaptiveEvents.length });
+  }
+  if (determinismResult.diffs && determinismResult.diffs.length > 0) {
+    const blockerCount = determinismResult.diffs.filter(d => d.severity === 'BLOCKER').length;
+    if (blockerCount > 0) {
+      topReasons.push({ code: 'ARTIFACT_DIFF_DETECTED', count: blockerCount });
+    }
+  }
+  
+  const report = {
+    version: 2, // PHASE 25: Bump version
+    contractVersion: 1,
+    artifactVersions: getArtifactVersions(),
+    generatedAt: new Date().toISOString(),
+    verdict: finalVerdict,
+    summary: {
+      ...determinismResult.summary,
+      runFingerprintMismatches: runFingerprintMismatches.length,
+      verifierErrors: verifierErrors.length,
+      topReasons: topReasons.slice(0, 10)
+    },
+    runsMeta: determinismResult.runsMeta,
+    runFingerprints,
+    runFingerprintMismatches,
+    verificationResults,
+    verifierErrors,
+    normalizationRulesApplied: [
+      'Stripped timestamps (startedAt, completedAt, detectedAt, etc.)',
+      'Stripped runId fields',
+      'Normalized absolute paths to relative',
+      'Normalized floating scores to 3 decimals',
+      'Sorted arrays by stable keys',
+      'Normalized evidence paths but preserved presence/absence',
+    ],
+    diffs: determinismResult.diffs,
+    adaptiveEvents: determinismResult.adaptiveEvents || [],
+    stabilityScore: determinismResult.summary.stabilityScore,
+  };
+  
+  writeFileSync(reportPath, JSON.stringify(report, null, 2) + '\n');
+  
+  return reportPath;
+}
+