@odavl/guardian 0.1.0-rc1 → 1.0.0

package/src/guardian/smoke.js

@@ -0,0 +1,258 @@
+const { AttemptEngine } = require('./attempt-engine');
+const { getAttemptDefinition } = require('./attempt-registry');
+const { BrowserPool } = require('./browser-pool');
+const { checkPrerequisites } = require('./prerequisite-checker');
+const { validateParallel, executeParallel } = require('./parallel-executor');
+const { getTimeoutProfile } = require('./timeout-profiles');
+const { isCiMode } = require('./ci-mode');
+
+const SMOKE_ATTEMPTS = ['universal_reality', 'login', 'signup', 'contact_form'];
+const DEFAULT_PARALLEL = 2;
+const DEFAULT_BUDGET_MS = 30000;
+const DEFAULT_PREREQ_TIMEOUT = 2000;
+const SMOKE_BROWSER_ARGS = ['--no-sandbox', '--disable-setuid-sandbox', '--proxy-bypass-list=*'];
+
+function validateUrl(url) {
+  try {
+    // eslint-disable-next-line no-new
+    new URL(url);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+function summarizeResults(results) {
+  const success = results.filter(r => r.outcome === 'SUCCESS').length;
+  const friction = results.filter(r => r.outcome === 'FRICTION').length;
+  const failure = results.filter(r => r.outcome === 'FAILURE').length;
+  const skipped = results.filter(r => r.outcome === 'SKIPPED').length;
+  return { success, friction, failure, skipped };
+}
+
+function authPathStatus(results) {
+  const authResults = results.filter(r => r.attemptId === 'login' || r.attemptId === 'signup');
+  const hasAuthSuccess = authResults.some(r => r.outcome === 'SUCCESS' || r.outcome === 'FRICTION');
+  const authFailures = authResults.filter(r => r.outcome === 'FAILURE').length;
+  return { hasAuthSuccess, authFailures };
+}
+
+function chooseExitCode({ failure, friction }, timedOut, authMissing, authFailuresToIgnore = 0) {
+  const effectiveFailures = Math.max(0, failure - authFailuresToIgnore);
+  if (timedOut || authMissing || effectiveFailures > 0) return 2;
+  if (friction > 0) return 1;
+  return 0;
+}
+
+async function executeSmoke(config) {
+  const baseUrl = config.baseUrl;
+  if (!validateUrl(baseUrl)) {
+    throw new Error(`Invalid URL: ${baseUrl}`);
+  }
+
+  const savedNoProxy = { NO_PROXY: process.env.NO_PROXY, no_proxy: process.env.no_proxy };
+  const forcedNoProxy = (process.env.NO_PROXY || process.env.no_proxy)
+    ? `${process.env.NO_PROXY || process.env.no_proxy},127.0.0.1,localhost`
+    : '127.0.0.1,localhost';
+  process.env.NO_PROXY = forcedNoProxy;
+  process.env.no_proxy = forcedNoProxy;
+
+  const ciMode = isCiMode();
+  const timeoutProfile = getTimeoutProfile('fast');
+  const resolvedTimeout = timeoutProfile.default;
+  const budgetMs = Number(process.env.GUARDIAN_SMOKE_BUDGET_MS || config.timeBudgetMs || DEFAULT_BUDGET_MS);
+
+  const parallelValidation = validateParallel(DEFAULT_PARALLEL);
+  if (!parallelValidation.valid) {
+    throw new Error(parallelValidation.error || 'Invalid parallel value');
+  }
+  const parallel = parallelValidation.parallel || DEFAULT_PARALLEL;
+
+  if (!ciMode) {
+    console.log('\nSMOKE MODE: Fast market sanity check (<30s)');
+    console.log(`Target: ${baseUrl}`);
+    console.log(`Attempts: ${SMOKE_ATTEMPTS.join(', ')}`);
+  } else {
+    console.log('SMOKE MODE: Fast market sanity check (<30s)');
+    console.log(`Target: ${baseUrl}`);
+    console.log(`Attempts: ${SMOKE_ATTEMPTS.join(', ')}`);
+  }
+
+  const browserPool = new BrowserPool();
+  await browserPool.launch({ headless: !config.headful, timeout: resolvedTimeout, args: SMOKE_BROWSER_ARGS });
+
+  const startedAt = Date.now();
+  let timedOut = false;
+  let shouldStop = false;
+  const attemptResults = [];
+
+  const budgetTimer = setTimeout(() => {
+    timedOut = true;
+    shouldStop = true;
+  }, budgetMs);
+
+  const attemptRunner = async (attemptId) => {
+    if (timedOut) {
+      return null;
+    }
+
+    const attemptDef = getAttemptDefinition(attemptId);
+    if (!attemptDef) {
+      return {
+        attemptId,
+        attemptName: attemptId,
+        outcome: 'FAILURE',
+        error: `Attempt ${attemptId} not found`,
+        friction: null
+      };
+    }
+
+    let context = null;
+    let page = null;
+    let result;
+
+    try {
+      const ctx = await browserPool.createContext({
+        timeout: resolvedTimeout,
+        ignoreHTTPSErrors: true
+      });
+      context = ctx.context;
+      page = ctx.page;
+      await page.goto(baseUrl, { waitUntil: 'domcontentloaded', timeout: resolvedTimeout });
+      const prereq = await checkPrerequisites(page, attemptId, DEFAULT_PREREQ_TIMEOUT);
+      if (!prereq.canProceed) {
+        result = {
+          attemptId,
+          attemptName: attemptDef.name,
+          outcome: 'SKIPPED',
+          skipReason: prereq.reason,
+          friction: null,
+          error: null
+        };
+      } else {
+        const engine = new AttemptEngine({
+          attemptId,
+          timeout: resolvedTimeout,
+          frictionThresholds: {
+            totalDurationMs: 5000,
+            stepDurationMs: 2500,
+            retryCount: 0
+          },
+          maxStepRetries: 1
+        });
+
+        const attemptResult = await engine.executeAttempt(page, attemptId, baseUrl, null, attemptDef.validators || []);
+        result = {
+          attemptId,
+          attemptName: attemptDef.name,
+          outcome: attemptResult.outcome,
+          friction: attemptResult.friction,
+          error: attemptResult.error,
+          successReason: attemptResult.successReason,
+          skipReason: null
+        };
+      }
+    } catch (err) {
+      result = {
+        attemptId,
+        attemptName: attemptDef?.name || attemptId,
+        outcome: 'FAILURE',
+        friction: null,
+        error: err.message,
+        skipReason: null
+      };
+    } finally {
+      if (context) {
+        await browserPool.closeContext(context);
+      }
+    }
+
+    // Enforce fail-fast
+    if (result.outcome === 'FAILURE') {
+      shouldStop = true;
+    }
+
+    // Enforce budget after attempt completes
+    if (Date.now() - startedAt >= budgetMs) {
+      timedOut = true;
+      shouldStop = true;
+    }
+
+    return result;
+  };
+
+  const parallelResults = await executeParallel(
+    SMOKE_ATTEMPTS,
+    attemptRunner,
+    parallel,
+    { shouldStop: () => shouldStop }
+  );
+
+  clearTimeout(budgetTimer);
+  for (const r of parallelResults) {
+    if (r) {
+      attemptResults.push(r);
+    }
+  }
+
+  if (process.env.GUARDIAN_SMOKE_DEBUG) {
+    console.log('DEBUG attempt results:', JSON.stringify(attemptResults, null, 2));
+  }
+
+  const summary = summarizeResults(attemptResults);
+  const authStatus = authPathStatus(attemptResults);
+  const effectiveFailures = Math.max(0, summary.failure - (authStatus.hasAuthSuccess ? authStatus.authFailures : 0));
+  const exitCode = chooseExitCode({ ...summary, failure: effectiveFailures }, timedOut, !authStatus.hasAuthSuccess, 0);
+
+  const elapsed = Date.now() - startedAt;
+
+  const lines = [];
+  lines.push(`Summary: success=${summary.success}, friction=${summary.friction}, failure=${effectiveFailures}, skipped=${summary.skipped}`);
+  if (timedOut) {
+    lines.push(`Result: FAILURE (time budget exceeded at ${elapsed}ms)`);
+  } else if (!authStatus.hasAuthSuccess) {
+    lines.push('Result: FAILURE (auth path unreachable)');
+  } else if (exitCode === 2) {
+    lines.push('Result: FAILURE');
+  } else if (exitCode === 1) {
+    lines.push('Result: FRICTION');
+  } else {
+    lines.push('Result: PASS');
+  }
+
+  for (const line of lines) {
+    console.log(line);
+  }
+
+  await browserPool.close();
+
+  // Restore proxy env vars
+  if (savedNoProxy.NO_PROXY !== undefined) {
+    process.env.NO_PROXY = savedNoProxy.NO_PROXY;
+  } else {
+    delete process.env.NO_PROXY;
+  }
+  if (savedNoProxy.no_proxy !== undefined) {
+    process.env.no_proxy = savedNoProxy.no_proxy;
+  } else {
+    delete process.env.no_proxy;
+  }
+
+  return { exitCode, attemptResults, timedOut, authAvailable: authStatus.hasAuthSuccess, elapsed };
+}
+
+async function runSmokeCLI(config) {
+  try {
+    const result = await executeSmoke(config);
+    process.exit(result.exitCode);
+  } catch (err) {
+    console.error(`Error: ${err.message}`);
+    process.exit(2);
+  }
+}
+
+module.exports = {
+  executeSmoke,
+  runSmokeCLI,
+  SMOKE_ATTEMPTS
+};

package/src/guardian/snapshot-schema.js

@@ -45,7 +45,7 @@ const SNAPSHOT_SCHEMA_VERSION = 'v1';
  * @property {string} attemptId - unique attempt identifier
  * @property {string} attemptName - human-readable name
  * @property {string} goal - what the user tried to achieve
- * @property {string} outcome - 'SUCCESS', 'FAILURE', or 'FRICTION'
+ * @property {string} outcome - 'SUCCESS', 'FAILURE', 'FRICTION', 'NOT_APPLICABLE', 'DISCOVERY_FAILED', 'SKIPPED'
  * @property {number} totalDurationMs - elapsed time
  * @property {number} stepCount - how many steps executed
  * @property {number} failedStepIndex - index of first failed step, or -1 if all succeeded
@@ -53,6 +53,10 @@ const SNAPSHOT_SCHEMA_VERSION = 'v1';
  * @property {ValidatorResult[]} [validators] - soft failure detectors (Phase 2)
  * @property {number} [softFailureCount] - count of failed validators
  * @property {string} [riskCategory] - 'LEAD', 'REVENUE', 'TRUST/UX' (Phase 2)
+ * @property {string} [skipReason] - reason if SKIPPED, NOT_APPLICABLE, or DISCOVERY_FAILED
+ * @property {string[]} [selectorChainTried] - selectors attempted during discovery
+ * @property {Object} [discoverySignals] - element discovery signals and heuristics
+ * @property {string} [finalSelection] - which selector/strategy successfully matched element
  */
 
 /**
@@ -133,6 +137,13 @@ const SNAPSHOT_SCHEMA_VERSION = 'v1';
  * @typedef {Object} MarketRealitySnapshot
  * @property {string} schemaVersion - always 'v1'
  * @property {SnapshotMeta} meta
+ * @property {Object} [verdict] - unified run-level verdict
+ * @property {('READY'|'DO_NOT_LAUNCH'|'FRICTION')} verdict.verdict
+ * @property {{ level: ('low'|'medium'|'high'), score: number, reasons: string[] }} verdict.confidence
+ * @property {string} verdict.why
+ * @property {string[]} verdict.keyFindings
+ * @property {{ screenshots?: string[], traces?: string[], reportPaths?: string[], affectedPages?: string[] }} verdict.evidence
+ * @property {string[]} verdict.limits
  * @property {CrawlResult} [crawl]
  * @property {AttemptResult[]} attempts
  * @property {Array} flows
@@ -166,6 +177,7 @@ function createEmptySnapshot(baseUrl, runId, toolVersion) {
     attempts: [],
     flows: [],
     signals: [],
+    verdict: null,
     riskSummary: {
       totalSoftFailures: 0,
       totalFriction: 0,
@@ -253,6 +265,18 @@ function validateSnapshot(snapshot) {
     errors.push('Missing baseline section');
   }
 
+  // Basic verdict validation (if present)
+  if (snapshot.verdict) {
+    const v = snapshot.verdict;
+    const allowed = ['READY', 'DO_NOT_LAUNCH', 'FRICTION'];
+    if (!v.verdict || !allowed.includes(v.verdict)) {
+      errors.push('Invalid verdict.verdict');
+    }
+    if (!v.confidence || typeof v.confidence.score !== 'number' || v.confidence.score < 0 || v.confidence.score > 1) {
+      errors.push('Invalid verdict.confidence.score');
+    }
+  }
+
   return {
     valid: errors.length === 0,
     errors

package/src/guardian/snapshot.js

@@ -34,10 +34,62 @@ class SnapshotBuilder {
     };
   }
 
+  /**
+   * Set unified verdict object
+   */
+  setVerdict(verdict) {
+    if (!verdict) return;
+    this.snapshot.verdict = {
+      verdict: verdict.verdict,
+      confidence: verdict.confidence,
+      why: verdict.why || '',
+      keyFindings: Array.isArray(verdict.keyFindings) ? verdict.keyFindings.slice(0, 7) : [],
+      evidence: verdict.evidence || {},
+      limits: Array.isArray(verdict.limits) ? verdict.limits.slice(0, 6) : []
+    };
+  }
+
   /**
    * Add attempt result to snapshot
    */
   addAttempt(attemptResult, artifactDir) {
+    // Handle NOT_APPLICABLE and DISCOVERY_FAILED attempts
+    if (attemptResult.outcome === 'NOT_APPLICABLE' || attemptResult.outcome === 'DISCOVERY_FAILED') {
+      this.snapshot.attempts.push({
+        attemptId: attemptResult.attemptId,
+        attemptName: attemptResult.attemptName,
+        goal: attemptResult.goal,
+        outcome: attemptResult.outcome,
+        executed: false,
+        skipReason: attemptResult.skipReason || (attemptResult.outcome === 'NOT_APPLICABLE' ? 'Feature not present' : 'Element discovery failed'),
+        skipReasonCode: attemptResult.skipReasonCode,
+        discoverySignals: attemptResult.discoverySignals || {},
+        totalDurationMs: attemptResult.totalDurationMs || 0,
+        stepCount: attemptResult.stepCount || 0,
+        failedStepIndex: -1,
+        friction: null
+      });
+      return; // Don't create signals for non-applicable attempts
+    }
+
+    // Phase 7.4: Handle SKIPPED attempts (don't add as signal)
+    if (attemptResult.outcome === 'SKIPPED') {
+      this.snapshot.attempts.push({
+        attemptId: attemptResult.attemptId,
+        attemptName: attemptResult.attemptName,
+        goal: attemptResult.goal,
+        outcome: 'SKIPPED',
+        executed: false,
+        skipReason: attemptResult.skipReason || 'Prerequisites not met',
+        skipReasonCode: attemptResult.skipReasonCode,
+        totalDurationMs: 0,
+        stepCount: 0,
+        failedStepIndex: -1,
+        friction: null
+      });
+      return; // Don't create signals for skipped attempts
+    }
+
     const signal = {
       id: `attempt_${attemptResult.attemptId}`,
       severity: attemptResult.outcome === 'FAILURE' ? 'high' : 'medium',
@@ -55,10 +107,17 @@ class SnapshotBuilder {
       attemptName: attemptResult.attemptName,
       goal: attemptResult.goal,
       outcome: attemptResult.outcome,
+      executed: true,
+      discoverySignals: attemptResult.discoverySignals || {},
       totalDurationMs: attemptResult.attemptResult?.totalDurationMs || 0,
       stepCount: (attemptResult.steps || []).length,
       failedStepIndex: (attemptResult.steps || []).findIndex(s => s.status === 'failed'),
-      friction: attemptResult.friction || null
+      friction: attemptResult.friction || null,
+      evidenceSummary: {
+        screenshots: (attemptResult.steps || []).reduce((sum, s) => sum + (Array.isArray(s.screenshots) ? s.screenshots.length : 0), 0),
+        validators: Array.isArray(attemptResult.validators) ? attemptResult.validators.length : 0,
+        tracesCaptured: attemptResult.tracePath ? 1 : 0
+      }
     });
 
     // Track artifacts
@@ -66,7 +125,8 @@ class SnapshotBuilder {
       this.snapshot.evidence.attemptArtifacts[attemptResult.attemptId] = {
         reportJson: path.join(attemptResult.attemptId, 'attempt-report.json'),
         reportHtml: path.join(attemptResult.attemptId, 'attempt-report.html'),
-        screenshotDir: path.join(attemptResult.attemptId, 'attempt-screenshots')
+        screenshotDir: path.join(attemptResult.attemptId, 'attempt-screenshots'),
+        attemptJson: attemptResult.attemptJsonPath ? path.relative(artifactDir, attemptResult.attemptJsonPath) : undefined
       };
     }
 
@@ -123,7 +183,13 @@ class SnapshotBuilder {
       stepsTotal: flowResult.stepsTotal || 0,
       durationMs: flowResult.durationMs || 0,
       failedStep: flowResult.failedStep || null,
-      error: flowResult.error || null
+      error: flowResult.error || null,
+      successEval: flowResult.successEval ? {
+        status: flowResult.successEval.status,
+        confidence: flowResult.successEval.confidence,
+        reasons: (flowResult.successEval.reasons || []).slice(0, 3),
+        evidence: flowResult.successEval.evidence || {}
+      } : null
     });
 
     if (runDir) {

package/src/guardian/stability-scorer.js

@@ -0,0 +1,169 @@
+/**
+ * Stability Scorer - Real-world reliability metrics
+ * 
+ * Measures how stable a journey run was:
+ * - Per-step stability (transient vs deterministic failures)
+ * - Overall run stability score (0-100)
+ */
+
+/**
+ * Classify error type for determining if it's transient or deterministic
+ * @param {string} errorMessage - Error message from step
+ * @returns {object} - { isTransient: boolean, classification: string }
+ */
+function classifyErrorType(errorMessage) {
+  if (!errorMessage) return { isTransient: false, classification: 'UNKNOWN' };
+
+  const msg = errorMessage.toLowerCase();
+
+  // Transient errors (safe to retry)
+  if (msg.includes('timeout') || msg.includes('timed out')) {
+    return { isTransient: true, classification: 'TIMEOUT' };
+  }
+  if (msg.includes('navigation') && (msg.includes('timeout') || msg.includes('closed'))) {
+    return { isTransient: true, classification: 'NAVIGATION_TIMEOUT' };
+  }
+  if (msg.includes('detached') || msg.includes('frame')) {
+    return { isTransient: true, classification: 'DETACHED_FRAME' };
+  }
+  if (msg.includes('econnrefused') || msg.includes('network') || msg.includes('socket')) {
+    return { isTransient: true, classification: 'NETWORK_ERROR' };
+  }
+  if (msg.includes('connection') && (msg.includes('reset') || msg.includes('closed'))) {
+    return { isTransient: true, classification: 'CONNECTION_ERROR' };
+  }
+
+  // Deterministic errors (don't retry)
+  if (msg.includes('not found') && (msg.includes('cta') || msg.includes('element'))) {
+    return { isTransient: false, classification: 'ELEMENT_NOT_FOUND' };
+  }
+  if (msg.includes('not visible')) {
+    return { isTransient: false, classification: 'ELEMENT_NOT_VISIBLE' };
+  }
+  if (msg.includes('cta') && msg.includes('found')) {
+    return { isTransient: false, classification: 'CTA_NOT_FOUND' };
+  }
+
+  // Default: assume transient to be safe
+  return { isTransient: true, classification: 'UNKNOWN' };
+}
+
+/**
+ * Compute stability score for a single step
+ * @param {object} step - Executed step result { id, name, success, attemptNumber, error }
+ * @returns {object} - { attempts, finalStatus, stable, confidence, errorType }
+ */
+function scoreStepStability(step) {
+  const attempts = step.attemptNumber || 1;
+  const finalStatus = step.success ? 'SUCCESS' : 'FAILED';
+  const errorType = classifyErrorType(step.error);
+
+  // Determine stability
+  let stable = true;
+  let confidence = 100;
+
+  if (finalStatus === 'SUCCESS') {
+    if (attempts > 1) {
+      // Success after retries = transient failure
+      stable = true; // The step ultimately worked
+      confidence = Math.max(30, 100 - (attempts - 1) * 20);
+    }
+  } else {
+    // Step failed all retries
+    stable = false;
+    confidence = 10; // Very low confidence in a consistently failing step
+  }
+
+  return {
+    stepId: step.id,
+    attempts,
+    finalStatus,
+    stable,
+    confidence,
+    errorType: errorType.classification,
+    isTransient: errorType.isTransient
+  };
+}
+
+/**
+ * Compute overall run stability score (0-100)
+ * @param {object} result - Journey result with executedSteps array
+ * @returns {number} - Stability score 0-100
+ */
+function computeRunStabilityScore(result) {
+  const steps = result.executedSteps || [];
+
+  if (steps.length === 0) return 0;
+
+  // Calculate step-level stability
+  const stepScores = steps.map(scoreStepStability);
+
+  // Count how many steps needed retries
+  const stepsWithRetries = stepScores.filter(s => s.attempts > 1).length;
+  const failedSteps = stepScores.filter(s => s.finalStatus === 'FAILED').length;
+
+  // Scoring algorithm:
+  // - Start at 100
+  // - Deduct 10 points per step that needed retries
+  // - Deduct 30 points per failed step
+  // - Floor at 0
+  let score = 100;
+  score -= stepsWithRetries * 10;
+  score -= failedSteps * 30;
+  score = Math.max(0, score);
+
+  // Consistency check: if goalReached varies, reduce score
+  // (This is a simple heuristic; more complex consistency checks could be added)
+  const hasInconsistency = false; // Would need multiple runs to detect
+  if (hasInconsistency) {
+    score = Math.max(0, score - 20);
+  }
+
+  return Math.round(score);
+}
+
+/**
+ * Build stability report from journey result
+ * @param {object} result - Journey scan result
+ * @returns {object} - Stability report with scores and metrics
+ */
+function buildStabilityReport(result) {
+  const steps = result.executedSteps || [];
+  const stepStability = steps.map(scoreStepStability);
+  const runScore = computeRunStabilityScore(result);
+
+  const metrics = {
+    totalSteps: steps.length,
+    succeededSteps: stepStability.filter(s => s.finalStatus === 'SUCCESS').length,
+    failedSteps: stepStability.filter(s => s.finalStatus === 'FAILED').length,
+    stepsWithRetries: stepStability.filter(s => s.attempts > 1).length,
+    totalAttempts: stepStability.reduce((sum, s) => sum + s.attempts, 0)
+  };
+
+  return {
+    runStabilityScore: runScore,
+    metrics,
+    stepStability,
+    assessment: assessStability(runScore)
+  };
+}
+
+/**
+ * Assess stability level based on score
+ * @param {number} score - Stability score 0-100
+ * @returns {string} - Assessment: 'excellent' | 'good' | 'fair' | 'poor'
+ */
+function assessStability(score) {
+  if (score >= 80) return 'excellent';
+  if (score >= 60) return 'good';
+  if (score >= 40) return 'fair';
+  return 'poor';
+}
+
+module.exports = {
+  classifyErrorType,
+  scoreStepStability,
+  computeRunStabilityScore,
+  buildStabilityReport,
+  assessStability
+};