@veraxhq/verax 0.3.0 → 0.4.0

package/src/cli/util/detection-engine.js

@@ -75,6 +75,14 @@ export async function detectFindings(learnData, observeData, projectPath, onProg
     findings,
     stats,
     detectedAt: new Date().toISOString(),
+    enforcement: {
+      evidenceLawEnforced: true,
+      contractVersion: 1,
+      timestamp: new Date().toISOString(),
+      droppedCount: 0,
+      downgradedCount: 0,
+      downgrades: []
+    }
   };
 }
 
@@ -100,7 +108,9 @@ function getObservationForExpectation(expectation, observationMap) {
 }
 
 function classificationIcon(classification) {
-  switch (classification) {
+  // Handle both old format and new taxonomy format
+  const baseClassification = classification.split(':')[0];
+  switch (baseClassification) {
     case 'observed':
       return '✓';
     case 'silent-failure':
@@ -115,7 +125,9 @@ function classificationIcon(classification) {
 }
 
 function findingStatKey(classification) {
-  switch (classification) {
+  // Handle both old format and new taxonomy format
+  const baseClassification = classification.split(':')[0];
+  switch (baseClassification) {
     case 'silent-failure':
       return 'silentFailures';
     case 'observed':
@@ -131,6 +143,8 @@ function findingStatKey(classification) {
 
 /**
  * Classify a single expectation according to deterministic rules.
+ * EVIDENCE GATE: silent-failure REQUIRES evidence.
+ * OUTCOME BINDING: Use attempt.cause to provide precise taxonomy.
  */
 function classifyExpectation(expectation, observation) {
   const finding = {
@@ -148,13 +162,15 @@ function classifyExpectation(expectation, observation) {
   const attempted = Boolean(observation?.attempted);
   const observed = observation?.observed === true;
   const reason = observation?.reason || null;
+  const cause = observation?.cause || null; // NEW: precise cause from planner
 
   const evidence = normalizeEvidence(observation?.evidenceFiles || []);
   finding.evidence = evidence;
 
   const evidenceSignals = analyzeEvidenceSignals(observation, evidence);
+  const hasAnyEvidence = evidence.length > 0 || evidenceSignals.hasDomChange;
 
-  // 1) observed
+  // 1) observed (success)
   if (observed) {
     finding.classification = 'observed';
     finding.reason = 'Expectation observed at runtime';
@@ -163,8 +179,8 @@ function classifyExpectation(expectation, observation) {
     return finding;
   }
 
-  // 2) coverage-gap (not attempted or explicitly blocked)
-  if (!attempted || isSafetySkip(reason)) {
+  // 2) coverage-gap (not attempted or safety skip)
+  if (!attempted) {
     finding.classification = 'coverage-gap';
     finding.reason = reason || 'No observation attempt recorded';
     finding.impact = 'LOW';
@@ -172,29 +188,48 @@ function classifyExpectation(expectation, observation) {
     return finding;
   }
 
-  // 3) silent-failure (attempted, observed === false, no safety skip)
-  if (attempted && observation?.observed === false && !isSafetySkip(reason)) {
-    finding.classification = 'silent-failure';
-    finding.reason = reason || 'Expected behavior not observed';
-    finding.impact = getImpact(expectation);
-    finding.confidence = calculateConfidence(expectation, evidenceSignals, 'silent-failure');
-    return finding;
-  }
-
-  // 4) unproven (attempted, ambiguous evidence)
+  // 3) Attempted but not observed - apply EVIDENCE GATE + OUTCOME BINDING
   if (attempted && !observed) {
-    finding.classification = 'unproven';
-    finding.reason = reason || 'Attempted but evidence insufficient';
-    finding.impact = 'MEDIUM';
-    finding.confidence = calculateConfidence(expectation, evidenceSignals, 'unproven');
+    // CRITICAL: Evidence Gate - silent-failure REQUIRES evidence
+    if (!hasAnyEvidence) {
+      // NO EVIDENCE → cannot prove silence → coverage-gap or unproven
+      if (isSafetySkip(reason)) {
+        finding.classification = 'coverage-gap';
+        finding.reason = reason || 'Blocked or skipped for safety';
+        finding.impact = 'LOW';
+        finding.confidence = 0;
+      } else {
+        finding.classification = 'unproven';
+        finding.reason = reason || 'Attempted but no evidence captured';
+        finding.impact = 'MEDIUM';
+        finding.confidence = 0;
+      }
+      return finding;
+    }
+
+    // HAS EVIDENCE → can classify as silent-failure with PRECISE taxonomy
+    let taxonomy = 'no-change'; // default
+    
+    if (cause) {
+      // Use the cause from interaction planner (most precise)
+      taxonomy = cause; // 'not-found' | 'blocked' | 'prevented-submit' | 'timeout' | 'no-change' | 'error'
+    } else {
+      // Fallback to signal-based detection (legacy)
+      taxonomy = determineSilenceTaxonomy(reason, evidenceSignals);
+    }
+    
+    finding.classification = `silent-failure:${taxonomy}`;
+    finding.reason = reason || `Expected behavior not observed (${taxonomy})`;
+    finding.impact = getImpact(expectation);
+    finding.confidence = calculateConfidenceFromEvidence(evidenceSignals);
     return finding;
   }
 
-  // 5) informational fallback
+  // 4) Fallback
   finding.classification = 'informational';
   finding.reason = reason || 'No classification rule matched';
   finding.impact = 'LOW';
-  finding.confidence = calculateConfidence(expectation, evidenceSignals, 'informational');
+  finding.confidence = 0;
   return finding;
 }
 
@@ -206,32 +241,61 @@ function isSafetySkip(reason) {
 }
 
 /**
- * Deterministic confidence calculation.
- * Screenshots + network + DOM change => >=0.8
- * Screenshots only => ~0.6
- * Weak signals => <0.5
+ * Determine silence taxonomy based on reason and evidence signals.
+ * Returns: no-change | blocked | not-found | timeout | prevented-submit
  */
-function calculateConfidence(expectation, evidenceSignals, classification) {
-  if (classification === 'observed') return 1.0;
-  if (classification === 'coverage-gap') return 0;
+function determineSilenceTaxonomy(reason, evidenceSignals) {
+  if (!reason) {
+    // No explicit reason - check evidence
+    if (evidenceSignals.hasScreenshots || evidenceSignals.hasDomChange) {
+      return 'no-change'; // Evidence exists but no observed change
+    }
+    return 'no-change';
+  }
 
-  const { hasScreenshots, hasNetworkLogs, hasDomChange } = evidenceSignals;
+  const lower = reason.toLowerCase();
 
-  if (classification === 'silent-failure') {
-    if (hasScreenshots && hasNetworkLogs && hasDomChange) return 0.85;
-    if (hasScreenshots && hasNetworkLogs) return 0.75;
-    if (hasScreenshots && hasDomChange) return 0.7;
-    if (hasScreenshots) return 0.6;
-    if (hasNetworkLogs || hasDomChange) return 0.5;
-    return 0.4; // weak signals, attempted but no evidence
+  // Check for specific conditions
+  if (lower.includes('timeout') || lower.includes('timed out')) {
+    return 'timeout';
   }
-
-  if (classification === 'unproven') {
-    if (hasScreenshots || hasNetworkLogs || hasDomChange) return 0.45;
-    return 0.3;
+  if (lower.includes('not found') || lower.includes('element not found') || lower.includes('selector not found') || lower.includes('not-found')) {
+    return 'not-found';
+  }
+  if (lower.includes('blocked') || lower.includes('not-interactable') || lower.includes('interactable')) {
+    return 'blocked';
   }
+  if (lower.includes('prevented') || lower.includes('prevented-submit') || lower.includes('submit-prevented')) {
+    return 'prevented-submit';
+  }
+  if (lower.includes('no matching event') || lower.includes('no change') || lower.includes('no-change')) {
+    return 'no-change';
+  }
+
+  // Default to no-change if evidence exists
+  return 'no-change';
+}
+
+/**
+ * Calculate confidence from evidence ONLY.
+ * No evidence = 0 confidence.
+ */
+function calculateConfidenceFromEvidence(evidenceSignals) {
+  const { hasScreenshots, hasNetworkLogs, hasDomChange } = evidenceSignals;
 
-  return 0.3;
+  // Multiple strong signals
+  if (hasScreenshots && hasNetworkLogs && hasDomChange) return 0.85;
+  if (hasScreenshots && hasNetworkLogs) return 0.75;
+  if (hasScreenshots && hasDomChange) return 0.7;
+  
+  // Single strong signal
+  if (hasScreenshots) return 0.6;
+  
+  // Weak signals
+  if (hasNetworkLogs || hasDomChange) return 0.5;
+  
+  // No evidence
+  return 0;
 }
 
 function analyzeEvidenceSignals(observation, evidence) {

package/src/cli/util/determinism-runner.js

@@ -19,7 +19,7 @@ import { writeDeterminismReport } from './determinism-writer.js';
  * @param {string} options.out - Output directory
  * @returns {Promise<Object>} Determinism check results
  */
-export async function runWithDeterminism(scanFn, options = {}) {
+export async function runWithDeterminism(scanFn, options = { runs: 2, out: '.verax' }) {
   const { runs = 2, out = '.verax' } = options;
   
   // Wrap scan function to return artifact paths
@@ -61,6 +61,7 @@ export async function runWithDeterminism(scanFn, options = {}) {
       if (existsSync(metaPath)) {
         try {
           const metaContent = readFileSync(metaPath, 'utf-8');
+  // @ts-expect-error - readFileSync with encoding returns string
           const meta = JSON.parse(metaContent);
           return meta.runFingerprint || null;
         } catch {

package/src/cli/util/determinism-writer.js

@@ -6,7 +6,7 @@
 
 import { resolve } from 'path';
 import { mkdirSync, writeFileSync } from 'fs';
-import { ARTIFACT_REGISTRY, getArtifactVersions } from '../../verax/core/artifacts/registry.js';
+import { ARTIFACT_REGISTRY, getArtifactVersions } from '../../verax/core/artifacts/registry.js'; // eslint-disable-line no-unused-vars
 
 /**
  * PHASE 18: Write determinism report

package/src/cli/util/digest-engine.js

@@ -0,0 +1,359 @@
+/**
+ * Digest Engine
+ * Produces cryptographic hashes of run artifacts for determinism proof
+ * Ensures two identical runs produce identical digests
+ */
+
+import { createHash } from 'crypto';
+import { readFileSync, writeFileSync } from 'fs';
+import { resolve } from 'path';
+
+/**
+ * Normalize JSON for hashing (canonicalize)
+ * Removes volatile fields, sorts keys consistently
+ */
+function normalizeJSON(obj, volatileFields = []) {
+  if (typeof obj !== 'object' || obj === null) {
+    return obj;
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(item => normalizeJSON(item, volatileFields));
+  }
+
+  const normalized = {};
+  const keys = Object.keys(obj).sort();
+
+  for (const key of keys) {
+    // Skip volatile fields
+    if (volatileFields.some(field => key.includes(field))) {
+      continue;
+    }
+
+    const value = obj[key];
+    if (typeof value === 'object' && value !== null) {
+      normalized[key] = normalizeJSON(value, volatileFields);
+    } else {
+      normalized[key] = value;
+    }
+  }
+
+  return normalized;
+}
+
+/**
+ * Strip timestamps and volatile data from string content
+ */
+function stripVolatile(content) {
+  if (typeof content !== 'string') {
+    return content;
+  }
+
+  let cleaned = content;
+
+  // Strip ISO timestamps
+  cleaned = cleaned.replace(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[.Z0-9]*/g, '[TIMESTAMP]');
+
+  // Strip UUIDs
+  cleaned = cleaned.replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '[UUID]');
+
+  // Strip run IDs (format: YYYY-MM-DDTHH-MM-SSZ_XXXXXX)
+  cleaned = cleaned.replace(/\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z_[a-z0-9]+/gi, '[RUN_ID]');
+
+  // Strip execution times (durations)
+  cleaned = cleaned.replace(/"endedAt":\s*"[^"]*"/g, '"endedAt":"[TIMESTAMP]"');
+  cleaned = cleaned.replace(/"startedAt":\s*"[^"]*"/g, '"startedAt":"[TIMESTAMP]"');
+  cleaned = cleaned.replace(/"observedAt":\s*"[^"]*"/g, '"observedAt":"[TIMESTAMP]"');
+
+  // Strip finding IDs
+  cleaned = cleaned.replace(/"findingId":\s*"[^"]*"/g, '"findingId":"[FINDING_ID]"');
+
+  return cleaned;
+}
+
+/**
+ * Compute SHA256 hash of content
+ */
+function hashContent(content) {
+  const hash = createHash('sha256');
+  hash.update(content, 'utf-8');
+  return hash.digest('hex');
+}
+
+/**
+ * Normalize and hash a JSON file
+ */
+function hashJSONFile(filePath, volatileFields = []) {
+  try {
+    const content = readFileSync(filePath, 'utf-8');
+  // @ts-expect-error - readFileSync with encoding returns string
+    const parsed = JSON.parse(content);
+    const normalized = normalizeJSON(parsed, volatileFields);
+    const serialized = JSON.stringify(normalized);
+    const stripped = stripVolatile(serialized);
+    return hashContent(stripped);
+  } catch (e) {
+    return null;
+  }
+}
+
+/**
+ * Hash a raw file (e.g., screenshot PNG)
+ */
+function _hashFile(filePath) {
+  try {
+    const content = readFileSync(filePath);
+    return hashContent(content.toString('utf-8'));
+  } catch (e) {
+    return null;
+  }
+}
+
+/**
+ * H5: Compute deterministic digest for observations
+ * Ensures identical inputs produce identical digests
+ * Used for reproducibility proof
+ */
+export function computeDigest(expectations, observations, metadata = {}) {
+  const digest = {
+    version: '1.0',
+    deterministicSeed: 'verax-h5-determinism-proof',
+    contentHashes: {},
+    normalized: {},
+  };
+
+  // Normalize expectations
+  const normalizedExpectations = (expectations || []).map(exp => ({
+    id: exp.id,
+    type: exp.type,
+    category: exp.category,
+    promise: exp.promise,
+  }));
+
+  const expString = JSON.stringify(normalizedExpectations);
+  digest.contentHashes.expectations = hashContent(expString);
+  digest.normalized.expectations = normalizedExpectations;
+
+  // Normalize observations (remove timing)
+  const normalizedObservations = (observations || []).map(obs => ({
+    id: obs.id,
+    category: obs.category,
+    observed: obs.observed,
+    reason: obs.reason,
+  }));
+
+  const obsString = JSON.stringify(normalizedObservations);
+  digest.contentHashes.observations = hashContent(obsString);
+  digest.normalized.observations = normalizedObservations;
+
+  // Normalize metadata
+  const normalizedMetadata = {
+    framework: metadata.framework,
+    url: metadata.url,
+    version: metadata.version,
+  };
+
+  const metaString = JSON.stringify(normalizedMetadata);
+  digest.contentHashes.metadata = hashContent(metaString);
+  digest.normalized.metadata = normalizedMetadata;
+
+  // Compute final digest
+  const digestInput = [
+    digest.contentHashes.expectations,
+    digest.contentHashes.observations,
+    digest.contentHashes.metadata,
+    digest.deterministicSeed,
+  ].join(':');
+
+  digest.deterministicDigest = hashContent(digestInput);
+
+  return digest;
+}
+
+/**
+ * H5: Validate determinism across multiple runs
+ */
+export function validateDeterminism(digests) {
+  if (!digests || digests.length === 0) {
+    return {
+      isDeterministic: true,
+      reason: 'No runs to compare',
+    };
+  }
+
+  const firstDigest = digests[0].deterministicDigest;
+  const allMatch = digests.every(d => d.deterministicDigest === firstDigest);
+
+  return {
+    isDeterministic: allMatch,
+    firstDigest,
+    mismatchedRuns: !allMatch ? digests.map((d, i) => ({
+      runIndex: i,
+      digest: d.deterministicDigest,
+    })).filter(d => d.digest !== firstDigest) : [],
+  };
+}
+
+/**
+ * Produce a complete run digest
+ */
+export async function produceRunDigest(runPath, runData) {
+  const digest = {
+    format: 'run-digest-v1',
+    timestamp: new Date().toISOString(),
+    digests: {
+      learn: null,
+      observe: null,
+      findings: null,
+      evidence: {},
+    },
+    metadata: {
+      runPath,
+      isReproducible: false,
+    },
+  };
+
+  // Hash learn.json (should be deterministic)
+  const learnPath = resolve(runPath, 'learn.json');
+  const learnHash = hashJSONFile(learnPath, ['extractedAt', 'duration']);
+  if (learnHash) {
+    digest.digests.learn = learnHash;
+  }
+
+  // Hash observe.json (normalized)
+  const observePath = resolve(runPath, 'observe.json');
+  const observeHash = hashJSONFile(observePath, [
+    'observedAt',
+    'startedAt',
+    'endedAt',
+    'timing',
+    'duration',
+    'findingId',
+  ]);
+  if (observeHash) {
+    digest.digests.observe = observeHash;
+  }
+
+  // Hash findings.json (normalized)
+  const findingsPath = resolve(runPath, 'findings.json');
+  const findingsHash = hashJSONFile(findingsPath, ['findingId', 'confidence']);
+  if (findingsHash) {
+    digest.digests.findings = findingsHash;
+  }
+
+  // Hash evidence directory
+  if (runData?.evidence && Array.isArray(runData.evidence)) {
+    const evidenceDigests = {};
+    for (const evidenceFile of runData.evidence) {
+      if (evidenceFile.endsWith('.png') || evidenceFile.endsWith('.jpg')) {
+        // Skip images (they may have subtle compression differences)
+        evidenceDigests[evidenceFile] = '[IMAGE_SKIPPED]';
+      } else if (evidenceFile.endsWith('.json')) {
+        const filePath = resolve(runPath, 'evidence', evidenceFile);
+        const hash = hashJSONFile(filePath, ['timestamp', 'duration']);
+        if (hash) {
+          evidenceDigests[evidenceFile] = hash;
+        }
+      }
+    }
+    digest.digests.evidence = evidenceDigests;
+  }
+
+  // Determine if reproducible (all core files match if run again)
+  const hasAllHashes =
+    digest.digests.learn &&
+    digest.digests.observe &&
+    digest.digests.findings &&
+    Object.keys(digest.digests.evidence).length > 0;
+
+  digest.metadata.isReproducible = hasAllHashes;
+
+  return digest;
+}
+
+/**
+ * Compare two digests for determinism
+ */
+export function compareDigests(digest1, digest2) {
+  const comparison = {
+    match: false,
+    diffs: [],
+  };
+
+  // Learn must match (deterministic)
+  if (digest1.digests.learn !== digest2.digests.learn) {
+    comparison.diffs.push('learn.json hash differs');
+  }
+
+  // Observe must match (deterministic execution)
+  if (digest1.digests.observe !== digest2.digests.observe) {
+    comparison.diffs.push('observe.json hash differs');
+  }
+
+  // Findings must match (deterministic classification)
+  if (digest1.digests.findings !== digest2.digests.findings) {
+    comparison.diffs.push('findings.json hash differs');
+  }
+
+  // Evidence files should match
+  const evidenceKeys1 = Object.keys(digest1.digests.evidence || {});
+  const evidenceKeys2 = Object.keys(digest2.digests.evidence || {});
+
+  if (evidenceKeys1.length !== evidenceKeys2.length) {
+    comparison.diffs.push(
+      `Evidence count differs: ${evidenceKeys1.length} vs ${evidenceKeys2.length}`
+    );
+  }
+
+  for (const key of evidenceKeys1) {
+    if (
+      digest1.digests.evidence[key] &&
+      digest2.digests.evidence[key] &&
+      digest1.digests.evidence[key] !== '[IMAGE_SKIPPED]' &&
+      digest1.digests.evidence[key] !== digest2.digests.evidence[key]
+    ) {
+      comparison.diffs.push(`Evidence file '${key}' hash differs`);
+    }
+  }
+
+  comparison.match = comparison.diffs.length === 0;
+
+  return comparison;
+}
+
+/**
+ * Save digest to file
+ */
+export function saveDigest(digestPath, digest) {
+  try {
+    writeFileSync(digestPath, JSON.stringify(digest, null, 2), 'utf-8');
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Load digest from file
+ */
+export function loadDigest(digestPath) {
+  try {
+    const content = readFileSync(digestPath, 'utf-8');
+  // @ts-expect-error - readFileSync with encoding returns string
+    return JSON.parse(content);
+  } catch (e) {
+    return null;
+  }
+}
+
+/**
+ * Check if a run is deterministically reproducible
+ */
+export function isRunDeterministic(digest) {
+  if (!digest) return false;
+  return (
+    Boolean(digest.digests.learn) &&
+    Boolean(digest.digests.observe) &&
+    Boolean(digest.digests.findings)
+  );
+}