@odavl/guardian 0.2.0 → 1.0.0

package/src/guardian/snapshot-schema.js

@@ -45,7 +45,7 @@ const SNAPSHOT_SCHEMA_VERSION = 'v1';
  * @property {string} attemptId - unique attempt identifier
  * @property {string} attemptName - human-readable name
  * @property {string} goal - what the user tried to achieve
- * @property {string} outcome - 'SUCCESS', 'FAILURE', or 'FRICTION'
+ * @property {string} outcome - 'SUCCESS', 'FAILURE', 'FRICTION', 'NOT_APPLICABLE', 'DISCOVERY_FAILED', 'SKIPPED'
  * @property {number} totalDurationMs - elapsed time
  * @property {number} stepCount - how many steps executed
  * @property {number} failedStepIndex - index of first failed step, or -1 if all succeeded
@@ -53,6 +53,10 @@ const SNAPSHOT_SCHEMA_VERSION = 'v1';
  * @property {ValidatorResult[]} [validators] - soft failure detectors (Phase 2)
  * @property {number} [softFailureCount] - count of failed validators
  * @property {string} [riskCategory] - 'LEAD', 'REVENUE', 'TRUST/UX' (Phase 2)
+ * @property {string} [skipReason] - reason if SKIPPED, NOT_APPLICABLE, or DISCOVERY_FAILED
+ * @property {string[]} [selectorChainTried] - selectors attempted during discovery
+ * @property {Object} [discoverySignals] - element discovery signals and heuristics
+ * @property {string} [finalSelection] - which selector/strategy successfully matched element
  */
 
 /**
@@ -133,6 +137,13 @@ const SNAPSHOT_SCHEMA_VERSION = 'v1';
  * @typedef {Object} MarketRealitySnapshot
  * @property {string} schemaVersion - always 'v1'
  * @property {SnapshotMeta} meta
+ * @property {Object} [verdict] - unified run-level verdict
+ * @property {('READY'|'DO_NOT_LAUNCH'|'FRICTION')} verdict.verdict
+ * @property {{ level: ('low'|'medium'|'high'), score: number, reasons: string[] }} verdict.confidence
+ * @property {string} verdict.why
+ * @property {string[]} verdict.keyFindings
+ * @property {{ screenshots?: string[], traces?: string[], reportPaths?: string[], affectedPages?: string[] }} verdict.evidence
+ * @property {string[]} verdict.limits
  * @property {CrawlResult} [crawl]
  * @property {AttemptResult[]} attempts
  * @property {Array} flows
@@ -166,6 +177,7 @@ function createEmptySnapshot(baseUrl, runId, toolVersion) {
     attempts: [],
     flows: [],
     signals: [],
+    verdict: null,
     riskSummary: {
       totalSoftFailures: 0,
       totalFriction: 0,
@@ -253,6 +265,18 @@ function validateSnapshot(snapshot) {
     errors.push('Missing baseline section');
   }
 
+  // Basic verdict validation (if present)
+  if (snapshot.verdict) {
+    const v = snapshot.verdict;
+    const allowed = ['READY', 'DO_NOT_LAUNCH', 'FRICTION'];
+    if (!v.verdict || !allowed.includes(v.verdict)) {
+      errors.push('Invalid verdict.verdict');
+    }
+    if (!v.confidence || typeof v.confidence.score !== 'number' || v.confidence.score < 0 || v.confidence.score > 1) {
+      errors.push('Invalid verdict.confidence.score');
+    }
+  }
+
   return {
     valid: errors.length === 0,
     errors

package/src/guardian/snapshot.js

@@ -34,10 +34,44 @@ class SnapshotBuilder {
     };
   }
 
+  /**
+   * Set unified verdict object
+   */
+  setVerdict(verdict) {
+    if (!verdict) return;
+    this.snapshot.verdict = {
+      verdict: verdict.verdict,
+      confidence: verdict.confidence,
+      why: verdict.why || '',
+      keyFindings: Array.isArray(verdict.keyFindings) ? verdict.keyFindings.slice(0, 7) : [],
+      evidence: verdict.evidence || {},
+      limits: Array.isArray(verdict.limits) ? verdict.limits.slice(0, 6) : []
+    };
+  }
+
   /**
    * Add attempt result to snapshot
    */
   addAttempt(attemptResult, artifactDir) {
+    // Handle NOT_APPLICABLE and DISCOVERY_FAILED attempts
+    if (attemptResult.outcome === 'NOT_APPLICABLE' || attemptResult.outcome === 'DISCOVERY_FAILED') {
+      this.snapshot.attempts.push({
+        attemptId: attemptResult.attemptId,
+        attemptName: attemptResult.attemptName,
+        goal: attemptResult.goal,
+        outcome: attemptResult.outcome,
+        executed: false,
+        skipReason: attemptResult.skipReason || (attemptResult.outcome === 'NOT_APPLICABLE' ? 'Feature not present' : 'Element discovery failed'),
+        skipReasonCode: attemptResult.skipReasonCode,
+        discoverySignals: attemptResult.discoverySignals || {},
+        totalDurationMs: attemptResult.totalDurationMs || 0,
+        stepCount: attemptResult.stepCount || 0,
+        failedStepIndex: -1,
+        friction: null
+      });
+      return; // Don't create signals for non-applicable attempts
+    }
+
     // Phase 7.4: Handle SKIPPED attempts (don't add as signal)
     if (attemptResult.outcome === 'SKIPPED') {
       this.snapshot.attempts.push({
@@ -45,7 +79,9 @@ class SnapshotBuilder {
         attemptName: attemptResult.attemptName,
         goal: attemptResult.goal,
         outcome: 'SKIPPED',
+        executed: false,
         skipReason: attemptResult.skipReason || 'Prerequisites not met',
+        skipReasonCode: attemptResult.skipReasonCode,
         totalDurationMs: 0,
         stepCount: 0,
         failedStepIndex: -1,
@@ -71,10 +107,17 @@ class SnapshotBuilder {
       attemptName: attemptResult.attemptName,
       goal: attemptResult.goal,
       outcome: attemptResult.outcome,
+      executed: true,
+      discoverySignals: attemptResult.discoverySignals || {},
       totalDurationMs: attemptResult.attemptResult?.totalDurationMs || 0,
       stepCount: (attemptResult.steps || []).length,
       failedStepIndex: (attemptResult.steps || []).findIndex(s => s.status === 'failed'),
-      friction: attemptResult.friction || null
+      friction: attemptResult.friction || null,
+      evidenceSummary: {
+        screenshots: (attemptResult.steps || []).reduce((sum, s) => sum + (Array.isArray(s.screenshots) ? s.screenshots.length : 0), 0),
+        validators: Array.isArray(attemptResult.validators) ? attemptResult.validators.length : 0,
+        tracesCaptured: attemptResult.tracePath ? 1 : 0
+      }
     });
 
     // Track artifacts
@@ -82,7 +125,8 @@ class SnapshotBuilder {
       this.snapshot.evidence.attemptArtifacts[attemptResult.attemptId] = {
         reportJson: path.join(attemptResult.attemptId, 'attempt-report.json'),
         reportHtml: path.join(attemptResult.attemptId, 'attempt-report.html'),
-        screenshotDir: path.join(attemptResult.attemptId, 'attempt-screenshots')
+        screenshotDir: path.join(attemptResult.attemptId, 'attempt-screenshots'),
+        attemptJson: attemptResult.attemptJsonPath ? path.relative(artifactDir, attemptResult.attemptJsonPath) : undefined
       };
     }
 

package/src/guardian/stability-scorer.js

@@ -0,0 +1,169 @@
+/**
+ * Stability Scorer - Real-world reliability metrics
+ * 
+ * Measures how stable a journey run was:
+ * - Per-step stability (transient vs deterministic failures)
+ * - Overall run stability score (0-100)
+ */
+
+/**
+ * Classify error type for determining if it's transient or deterministic
+ * @param {string} errorMessage - Error message from step
+ * @returns {object} - { isTransient: boolean, classification: string }
+ */
+function classifyErrorType(errorMessage) {
+  if (!errorMessage) return { isTransient: false, classification: 'UNKNOWN' };
+
+  const msg = errorMessage.toLowerCase();
+
+  // Transient errors (safe to retry)
+  if (msg.includes('timeout') || msg.includes('timed out')) {
+    return { isTransient: true, classification: 'TIMEOUT' };
+  }
+  if (msg.includes('navigation') && (msg.includes('timeout') || msg.includes('closed'))) {
+    return { isTransient: true, classification: 'NAVIGATION_TIMEOUT' };
+  }
+  if (msg.includes('detached') || msg.includes('frame')) {
+    return { isTransient: true, classification: 'DETACHED_FRAME' };
+  }
+  if (msg.includes('econnrefused') || msg.includes('network') || msg.includes('socket')) {
+    return { isTransient: true, classification: 'NETWORK_ERROR' };
+  }
+  if (msg.includes('connection') && (msg.includes('reset') || msg.includes('closed'))) {
+    return { isTransient: true, classification: 'CONNECTION_ERROR' };
+  }
+
+  // Deterministic errors (don't retry)
+  if (msg.includes('not found') && (msg.includes('cta') || msg.includes('element'))) {
+    return { isTransient: false, classification: 'ELEMENT_NOT_FOUND' };
+  }
+  if (msg.includes('not visible')) {
+    return { isTransient: false, classification: 'ELEMENT_NOT_VISIBLE' };
+  }
+  if (msg.includes('cta') && msg.includes('found')) {
+    return { isTransient: false, classification: 'CTA_NOT_FOUND' };
+  }
+
+  // Default: assume transient to be safe
+  return { isTransient: true, classification: 'UNKNOWN' };
+}
+
+/**
+ * Compute stability score for a single step
+ * @param {object} step - Executed step result { id, name, success, attemptNumber, error }
+ * @returns {object} - { attempts, finalStatus, stable, confidence, errorType }
+ */
+function scoreStepStability(step) {
+  const attempts = step.attemptNumber || 1;
+  const finalStatus = step.success ? 'SUCCESS' : 'FAILED';
+  const errorType = classifyErrorType(step.error);
+
+  // Determine stability
+  let stable = true;
+  let confidence = 100;
+
+  if (finalStatus === 'SUCCESS') {
+    if (attempts > 1) {
+      // Success after retries = transient failure
+      stable = true; // The step ultimately worked
+      confidence = Math.max(30, 100 - (attempts - 1) * 20);
+    }
+  } else {
+    // Step failed all retries
+    stable = false;
+    confidence = 10; // Very low confidence in a consistently failing step
+  }
+
+  return {
+    stepId: step.id,
+    attempts,
+    finalStatus,
+    stable,
+    confidence,
+    errorType: errorType.classification,
+    isTransient: errorType.isTransient
+  };
+}
+
+/**
+ * Compute overall run stability score (0-100)
+ * @param {object} result - Journey result with executedSteps array
+ * @returns {number} - Stability score 0-100
+ */
+function computeRunStabilityScore(result) {
+  const steps = result.executedSteps || [];
+
+  if (steps.length === 0) return 0;
+
+  // Calculate step-level stability
+  const stepScores = steps.map(scoreStepStability);
+
+  // Count how many steps needed retries
+  const stepsWithRetries = stepScores.filter(s => s.attempts > 1).length;
+  const failedSteps = stepScores.filter(s => s.finalStatus === 'FAILED').length;
+
+  // Scoring algorithm:
+  // - Start at 100
+  // - Deduct 10 points per step that needed retries
+  // - Deduct 30 points per failed step
+  // - Floor at 0
+  let score = 100;
+  score -= stepsWithRetries * 10;
+  score -= failedSteps * 30;
+  score = Math.max(0, score);
+
+  // Consistency check: if goalReached varies, reduce score
+  // (This is a simple heuristic; more complex consistency checks could be added)
+  const hasInconsistency = false; // Would need multiple runs to detect
+  if (hasInconsistency) {
+    score = Math.max(0, score - 20);
+  }
+
+  return Math.round(score);
+}
+
+/**
+ * Build stability report from journey result
+ * @param {object} result - Journey scan result
+ * @returns {object} - Stability report with scores and metrics
+ */
+function buildStabilityReport(result) {
+  const steps = result.executedSteps || [];
+  const stepStability = steps.map(scoreStepStability);
+  const runScore = computeRunStabilityScore(result);
+
+  const metrics = {
+    totalSteps: steps.length,
+    succeededSteps: stepStability.filter(s => s.finalStatus === 'SUCCESS').length,
+    failedSteps: stepStability.filter(s => s.finalStatus === 'FAILED').length,
+    stepsWithRetries: stepStability.filter(s => s.attempts > 1).length,
+    totalAttempts: stepStability.reduce((sum, s) => sum + s.attempts, 0)
+  };
+
+  return {
+    runStabilityScore: runScore,
+    metrics,
+    stepStability,
+    assessment: assessStability(runScore)
+  };
+}
+
+/**
+ * Assess stability level based on score
+ * @param {number} score - Stability score 0-100
+ * @returns {string} - Assessment: 'excellent' | 'good' | 'fair' | 'poor'
+ */
+function assessStability(score) {
+  if (score >= 80) return 'excellent';
+  if (score >= 60) return 'good';
+  if (score >= 40) return 'fair';
+  return 'poor';
+}
+
+module.exports = {
+  classifyErrorType,
+  scoreStepStability,
+  computeRunStabilityScore,
+  buildStabilityReport,
+  assessStability
+};

package/src/guardian/template-command.js

@@ -0,0 +1,184 @@
+/**
+ * Template Command
+ * 
+ * Generate minimal config templates:
+ *   guardian template saas
+ *   guardian template shop
+ *   guardian template landing
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const TEMPLATES = {
+  saas: {
+    name: 'SaaS Startup Check',
+    description: 'Verify core SaaS flows: signup, login, dashboard',
+    intent: {
+      category: 'product_type',
+      value: 'saas',
+      confidence: 1.0
+    },
+    journeys: [
+      {
+        id: 'signup_flow',
+        name: 'User Signup',
+        description: 'Verify signup journey completes',
+        steps: [
+          { action: 'navigate', target: '/', name: 'Home' },
+          { action: 'click', target: 'a[href*="/signup"], button:has-text("Sign up")', name: 'Click signup' },
+          { action: 'wait', ms: 2000 },
+          { action: 'screenshot', name: 'Signup form loaded' }
+        ],
+        criticality: 'CRITICAL',
+        onFailure: 'FAIL'
+      },
+      {
+        id: 'landing_page',
+        name: 'Landing Page Load',
+        description: 'Verify homepage loads without errors',
+        steps: [
+          { action: 'navigate', target: '/', name: 'Home' },
+          { action: 'screenshot', name: 'Homepage loaded' }
+        ],
+        criticality: 'CRITICAL',
+        onFailure: 'WARN'
+      }
+    ],
+    policy: {
+      failOnSeverity: 'CRITICAL',
+      maxWarnings: 10,
+      requireBaseline: false
+    }
+  },
+
+  shop: {
+    name: 'E-Commerce Shop Check',
+    description: 'Verify core shop flows: browse, add to cart, checkout',
+    intent: {
+      category: 'product_type',
+      value: 'ecommerce',
+      confidence: 1.0
+    },
+    journeys: [
+      {
+        id: 'browse_products',
+        name: 'Browse Products',
+        description: 'Verify product catalog loads',
+        steps: [
+          { action: 'navigate', target: '/', name: 'Home' },
+          { action: 'click', target: 'a[href*="/shop"], a[href*="/products"], button:has-text("Shop")', name: 'Go to shop' },
+          { action: 'wait', ms: 2000 },
+          { action: 'screenshot', name: 'Products page' }
+        ],
+        criticality: 'CRITICAL',
+        onFailure: 'FAIL'
+      },
+      {
+        id: 'add_to_cart',
+        name: 'Add to Cart',
+        description: 'Verify add-to-cart flow',
+        steps: [
+          { action: 'navigate', target: '/', name: 'Home' },
+          { action: 'click', target: 'a[href*="/shop"], a[href*="/products"]', name: 'Go to shop' },
+          { action: 'wait', ms: 2000 },
+          { action: 'click', target: 'button:has-text("Add"), button:has-text("Cart")', name: 'Add item' },
+          { action: 'screenshot', name: 'Cart updated' }
+        ],
+        criticality: 'CRITICAL',
+        onFailure: 'WARN'
+      }
+    ],
+    policy: {
+      failOnSeverity: 'CRITICAL',
+      maxWarnings: 10,
+      requireBaseline: false
+    }
+  },
+
+  landing: {
+    name: 'Landing Page Check',
+    description: 'Verify landing page loads and core CTAs work',
+    intent: {
+      category: 'product_type',
+      value: 'landing',
+      confidence: 1.0
+    },
+    journeys: [
+      {
+        id: 'page_load',
+        name: 'Page Load',
+        description: 'Verify landing page loads',
+        steps: [
+          { action: 'navigate', target: '/', name: 'Home' },
+          { action: 'screenshot', name: 'Landing page' }
+        ],
+        criticality: 'CRITICAL',
+        onFailure: 'FAIL'
+      },
+      {
+        id: 'cta_click',
+        name: 'Call-to-Action',
+        description: 'Verify main CTA is clickable',
+        steps: [
+          { action: 'navigate', target: '/', name: 'Home' },
+          { action: 'click', target: 'button:has-text("Get started"), button:has-text("Start"), button:has-text("Sign up"), a[href*="/signup"]', name: 'Click CTA' },
+          { action: 'screenshot', name: 'CTA works' }
+        ],
+        criticality: 'CRITICAL',
+        onFailure: 'WARN'
+      }
+    ],
+    policy: {
+      failOnSeverity: 'CRITICAL',
+      maxWarnings: 5,
+      requireBaseline: false
+    }
+  }
+};
+
+/**
+ * Generate config file from template
+ * @param {string} templateName - saas, shop, or landing
+ * @param {object} options - Generation options
+ * @returns {object} Result with generated file path
+ */
+function generateTemplate(templateName, options = {}) {
+  const cwd = options.cwd || process.cwd();
+  const outputFile = options.output || `guardian-${templateName}.json`;
+  const outputPath = path.join(cwd, outputFile);
+
+  if (!TEMPLATES[templateName]) {
+    throw new Error(`Unknown template: ${templateName}. Available: ${Object.keys(TEMPLATES).join(', ')}`);
+  }
+
+  const template = TEMPLATES[templateName];
+  const content = JSON.stringify(template, null, 2);
+
+  fs.writeFileSync(outputPath, content, 'utf-8');
+
+  return {
+    success: true,
+    template: templateName,
+    outputPath,
+    message: `Generated ${templateName} template: ${outputFile}`
+  };
+}
+
+/**
+ * List available templates
+ * @returns {array} Array of template names with descriptions
+ */
+function listTemplates() {
+  return Object.entries(TEMPLATES).map(([name, config]) => ({
+    name,
+    description: config.description,
+    journeys: config.journeys.length
+  }));
+}
+
+module.exports = {
+  generateTemplate,
+  listTemplates,
+  TEMPLATES
+};