@sanity/ailf 0.1.0 → 0.1.2

package/dist/scripts/pipeline.js

@@ -1,1031 +0,0 @@
-/**
- * pipeline.ts
- *
- * CLI orchestrator for the modular evaluation pipeline.
- * Runs steps in sequence with validation between each.
- *
- * This is the single entry point for both local and CI evaluation.
- * The CI workflow (eval.yml) calls this script, then layers on
- * CI-specific post-steps (PR comment posting, artifact upload).
- *
- * Usage:
- *   pnpm pipeline                          # full baseline pipeline
- *   pnpm pipeline --dry-run                # validate only, no execution
- *   pnpm pipeline --skip-fetch             # reuse cached doc contexts
- *   pnpm pipeline --skip-eval              # recalculate from existing results
- *   pnpm pipeline --mode agentic           # run agentic pipeline
- *   pnpm pipeline --mode observed          # run observed pipeline
- *   pnpm pipeline --source staging         # use staging doc source
- *   pnpm pipeline --debug                  # run first 2 tests only (fast)
- *   pnpm pipeline --debug-n 5              # run first 5 tests
- *   pnpm pipeline --debug-pattern "Blog"   # filter by description
- *   pnpm pipeline --debug-sample 3         # random sample of 3 tests
- *   pnpm pipeline --no-cache               # bypass caching, force re-run
- *   pnpm pipeline --concurrency 64         # override max parallel API calls
- *   pnpm pipeline --area groq,frameworks   # only evaluate these areas
- *   pnpm pipeline --task groq-blog-queries # only evaluate this task
- *   pnpm pipeline --changed-docs groq-introduction,how-queries-work
- *                                          # auto-scope to affected tasks
- *   pnpm pipeline --url https://...        # override docs base URL
- *   pnpm pipeline --sanity-dataset staging # override Sanity dataset
- *   pnpm pipeline --sanity-project abc123  # override Sanity project ID
- *   pnpm pipeline --sanity-perspective agent-c7OKTk
- *                                          # evaluate a Sanity release
- *   pnpm pipeline --sanity-document <uuid>
- *                                          # evaluate specific document(s)
- *   pnpm pipeline --sanity-document <uuid> --sanity-documents <uuid>
- *                                          # singular and plural aliases work
- *   pnpm pipeline --header "X-Vercel-Protection-Bypass: <secret>"
- *                                          # custom HTTP header (repeatable)
- *   pnpm pipeline --allowed-origin my-branch.sanity.build
- *                                          # sandbox agent to this origin
- *   pnpm pipeline --before published         # run before/after impact evaluation
- *   pnpm pipeline --before production        # before = production source
- *   pnpm pipeline --before results/baselines/20260310.json  # use existing scores
- *   pnpm pipeline --before latest-baseline   # use most recent baseline
- *   pnpm pipeline --compare                # compare scores against latest baseline
- *   pnpm pipeline --compare --compare-baseline <path>  # compare against specific file
- *   pnpm pipeline --compare --threshold 5  # noise threshold for unchanged (default: 2)
- *   pnpm pipeline --output /tmp/report.md  # write report to specific path
- *   pnpm pipeline --promptfoo-url <url>    # include Promptfoo URL in report
- *   pnpm pipeline --gap-analysis            # run failure mode + impact analysis
- *   pnpm pipeline --publish                # write report to Sanity + fan out to sinks
- *   pnpm pipeline --publish --publish-tag "daily-2026-03-11"  # tag the report
- *   pnpm pipeline --publish --report-dataset ailf-reports    # report store dataset
- *   pnpm pipeline --publish --report-project abc123          # report store project
- *
- * Override precedence (highest wins):
- *   CLI flag (--url, --sanity-dataset, --sanity-project, --allowed-origin)
- *   → Environment variable (DOC_BASE_URL, SANITY_DATASET, SANITY_PROJECT_ID, DOC_ALLOWED_ORIGIN)
- *   → config/sources.yaml default value
- *
- *   --header flags are additive and do not override env vars — they are
- *   always merged with any headers defined in DOC_HEADERS env var.
- *
- * Environment variable fallbacks (for CI):
- *   DEBUG_EVAL=1              → --debug
- *   DEBUG_EVAL_N=2            → --debug-n 2
- *   DEBUG_EVAL_PATTERN        → --debug-pattern
- *   DEBUG_EVAL_SAMPLE         → --debug-sample
- *   EVAL_FILTER_AREAS         → --area
- *   EVAL_FILTER_TASKS         → --task
- *   EVAL_CHANGED_DOCS         → --changed-docs
- *   AILF_REPORT_DATASET       → --report-dataset (report store, not eval)
- *   AILF_REPORT_PROJECT_ID    → --report-project (report store, not eval)
- */
-import { config as dotenvConfig } from "dotenv";
-import { existsSync, readFileSync, writeFileSync } from "fs";
-import { load } from "js-yaml";
-import { dirname, join, resolve } from "path";
-import { fileURLToPath } from "url";
-import { createCacheStats } from "../pipeline/cache.js";
-import { checkEnvironment } from "../pipeline/checks.js";
-import { runCalculateScores } from "../pipeline/steps/calculate-scores-step.js";
-import { runCompare } from "../pipeline/steps/compare-step.js";
-import { runGraderConsistency } from "../pipeline/steps/grader-consistency-step.js";
-import { extractShareUrl, runEval } from "../pipeline/steps/eval-step.js";
-import { runFetchDocs } from "../pipeline/steps/fetch-docs-step.js";
-import { runGenerateConfigs } from "../pipeline/steps/generate-configs-step.js";
-import { runPublishReport } from "../pipeline/steps/publish-report-step.js";
-import { runReport } from "../pipeline/steps/report-step.js";
-import { assessImpact, buildReverseMapping, } from "../pipeline/reverse-mapping.js";
-import { classifyUrls } from "../pipeline/classify-url.js";
-import { validateConfiguration } from "../pipeline/validate.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..");
-// ---------------------------------------------------------------------------
-// Load root .env — override shell env vars for consistency with individual
-// scripts that use `dotenv -e ../../.env -o --`. The `-o` (override) flag in
-// those scripts means .env always wins, so we replicate that here.
-// In CI, env vars are set via workflow `env:` blocks, so this is a no-op
-// (the .env file doesn't exist in CI).
-// ---------------------------------------------------------------------------
-const envPath = resolve(ROOT, "..", "..", ".env");
-if (existsSync(envPath)) {
-    dotenvConfig({ override: true, path: envPath });
-}
-// ---------------------------------------------------------------------------
-// CLI argument parsing
-// ---------------------------------------------------------------------------
-const args = process.argv.slice(2);
-function getAllOptions(...names) {
-    const results = [];
-    for (const name of names) {
-        const flag = `--${name}`;
-        for (let i = 0; i < args.length; i++) {
-            if (args[i] === flag && i + 1 < args.length) {
-                results.push(args[i + 1]);
-            }
-        }
-    }
-    return results;
-}
-function getFlag(name) {
-    return args.includes(`--${name}`);
-}
-function getOption(...names) {
-    for (const name of names) {
-        const idx = args.indexOf(`--${name}`);
-        if (idx !== -1 && idx + 1 < args.length)
-            return args[idx + 1];
-    }
-    return undefined;
-}
-const dryRun = getFlag("dry-run");
-const skipFetch = getFlag("skip-fetch");
-const skipEval = getFlag("skip-eval");
-const noCache = getFlag("no-cache");
-const mode = (getOption("mode") ?? "baseline");
-const source = getOption("source");
-const outputPath = getOption("output");
-const promptfooUrl = getOption("promptfoo-url");
-// Debug: CLI flags take precedence, then env vars.
-// Any sub-flag (--debug-n, --debug-pattern, --debug-sample) implies --debug.
-const debugN = getOption("debug-n") ?? process.env.DEBUG_EVAL_N ?? undefined;
-const debugPattern = getOption("debug-pattern") ?? process.env.DEBUG_EVAL_PATTERN ?? undefined;
-const debugSample = getOption("debug-sample") ?? process.env.DEBUG_EVAL_SAMPLE ?? undefined;
-const debugEnabled = getFlag("debug") ||
-    process.env.DEBUG_EVAL === "1" ||
-    debugN !== undefined ||
-    debugPattern !== undefined ||
-    debugSample !== undefined;
-const debug = debugEnabled
-    ? {
-        enabled: true,
-        firstN: debugN ? parseInt(debugN, 10) : undefined,
-        pattern: debugPattern,
-        sample: debugSample ? parseInt(debugSample, 10) : undefined,
-    }
-    : undefined;
-const concurrencyStr = getOption("concurrency");
-const concurrency = concurrencyStr ? parseInt(concurrencyStr, 10) : undefined;
-// Scoping: filter to specific areas or tasks
-const areaOption = getOption("area");
-const taskOption = getOption("task");
-// Document-driven scoping: --changed-docs slug1,slug2 or env var
-const changedDocsOption = getOption("changed-docs") ?? process.env.EVAL_CHANGED_DOCS ?? undefined;
-// Grader consistency: measure grading variance
-const graderReplicationsStr = getOption("grader-replications");
-const graderReplications = graderReplicationsStr
-    ? parseInt(graderReplicationsStr, 10)
-    : undefined;
-// Before/after impact evaluation: --before <source>
-// When provided, runs the evaluation twice (before + after) and auto-compares.
-// Accepts: "published" (Sanity published state), "production" (prod source),
-// a path to an existing score-summary.json, or "latest-baseline".
-const beforeOption = getOption("before");
-// Comparison: compare current scores against a baseline
-const compareEnabled = getFlag("compare") || beforeOption !== undefined;
-const compareBaseline = getOption("compare-baseline");
-const compareThresholdStr = getOption("threshold");
-const compareThreshold = compareThresholdStr
-    ? parseFloat(compareThresholdStr)
-    : undefined;
-// Gap analysis: run failure mode classification + impact estimation (Phase 3)
-const gapAnalysisEnabled = getFlag("gap-analysis");
-// Readiness report: launch readiness checklist for a specific area (Phase 5b)
-const readinessEnabled = getFlag("readiness");
-// Discovery report: agent discoverability from agentic retrieval metrics (Phase 5c)
-const discoveryReportEnabled = getFlag("discovery-report");
-// Publish: write report to Sanity Content Lake + fan out to sinks
-const publishEnabled = getFlag("publish") || process.env.AILF_PUBLISH === "1";
-const publishTag = getOption("publish-tag");
-// Report store: target dataset/project for report persistence
-// Independent of SANITY_DATASET/SANITY_PROJECT_ID (which control doc evaluation)
-const reportDataset = getOption("report-dataset") ?? process.env.AILF_REPORT_DATASET ?? undefined;
-const reportProjectId = getOption("report-project") ?? process.env.AILF_REPORT_PROJECT_ID ?? undefined;
-// Search mode: controls web_search tool in agentic mode
-// --search open (default), --search origin-only, --search off
-const searchMode = getOption("search") ?? process.env.EVAL_SEARCH_MODE ?? "open";
-const VALID_SEARCH_MODES = ["open", "origin-only", "off"];
-if (!VALID_SEARCH_MODES.includes(searchMode)) {
-    console.error(`❌ Invalid --search mode "${searchMode}". Must be one of: ${VALID_SEARCH_MODES.join(", ")}`);
-    process.exit(1);
-}
-// Pass through to generate-configs via env var
-if (searchMode !== "open") {
-    process.env.EVAL_SEARCH_MODE = searchMode;
-}
-// Source overrides: --url/--urls, --sanity-dataset, --sanity-project, --sanity-perspective
-// CLI flag → env var → config/sources.yaml default
-// These set process.env so the interpolation layer in sources.ts picks them up.
-const urlArgs = getAllOptions("url", "urls");
-const datasetOverride = getOption("sanity-dataset");
-const projectIdOverride = getOption("sanity-project");
-const perspectiveOverride = getOption("sanity-perspective");
-const studioOriginOverride = getOption("sanity-studio-origin");
-const sanityDocumentArgs = getAllOptions("sanity-document", "sanity-documents");
-// Classify URLs: detect Sanity releases, documents, and direct URLs
-if (urlArgs.length > 0) {
-    const classification = classifyUrls(urlArgs);
-    // First URL becomes the base URL (used for agentic mode, llms.txt, etc.)
-    process.env.DOC_BASE_URL = urlArgs[0];
-    // Merge classified document IDs with explicit --sanity-document args
-    if (classification.documentIds.length > 0) {
-        const existing = sanityDocumentArgs.length > 0 ? sanityDocumentArgs : [];
-        const merged = [...new Set([...existing, ...classification.documentIds])];
-        process.env.SANITY_DOCUMENT_IDS = merged.join(",");
-    }
-    // Use inferred perspective if no explicit one was provided
-    if (classification.inferredPerspective && !perspectiveOverride) {
-        process.env.SANITY_PERSPECTIVE = classification.inferredPerspective;
-    }
-    // Use inferred studio origin if no explicit one was provided
-    if (classification.inferredStudioOrigin && !studioOriginOverride) {
-        process.env.SANITY_STUDIO_ORIGIN = classification.inferredStudioOrigin;
-    }
-    // Direct URLs: pass through for baseline mode doc fetching
-    if (classification.directUrls.length > 0) {
-        process.env.DOC_DIRECT_URLS = classification.directUrls.join(",");
-    }
-}
-if (datasetOverride)
-    process.env.SANITY_DATASET = datasetOverride;
-if (projectIdOverride)
-    process.env.SANITY_PROJECT_ID = projectIdOverride;
-if (perspectiveOverride)
-    process.env.SANITY_PERSPECTIVE = perspectiveOverride;
-if (studioOriginOverride)
-    process.env.SANITY_STUDIO_ORIGIN = studioOriginOverride;
-if (sanityDocumentArgs.length > 0)
-    process.env.SANITY_DOCUMENT_IDS = sanityDocumentArgs.join(",");
-// Custom HTTP headers: --header "Key: Value" (repeatable, like curl -H)
-// Parsed into DOC_HEADERS env var as JSON for sources.ts to pick up.
-const headerArgs = getAllOptions("header", "headers");
-if (headerArgs.length > 0) {
-    const headers = {};
-    for (const h of headerArgs) {
-        const colonIdx = h.indexOf(":");
-        if (colonIdx === -1) {
-            console.error(`❌ Invalid header format: "${h}". Expected "Key: Value".`);
-            process.exit(1);
-        }
-        const key = h.slice(0, colonIdx).trim();
-        const value = h.slice(colonIdx + 1).trim();
-        if (!key) {
-            console.error(`❌ Invalid header: empty key in "${h}"`);
-            process.exit(1);
-        }
-        headers[key] = value;
-    }
-    process.env.DOC_HEADERS = JSON.stringify(headers);
-}
-// Allowed origin sandboxing: --allowed-origin my-branch.sanity.build (repeatable)
-// Restricts agentic providers to only fetch pages from these origins.
-// Supports glob patterns: --allowed-origin "*.sanity.build"
-// Plural alias: --allowed-origins
-const allowedOriginArgs = getAllOptions("allowed-origin", "allowed-origins");
-// Auto-infer allowed origin from --url when no explicit origins are provided.
-// This makes agentic isolation the default when a --url is passed.
-if (urlArgs.length > 0 && allowedOriginArgs.length === 0) {
-    try {
-        const hostname = new URL(urlArgs[0]).hostname.replace(/^www\./, "");
-        allowedOriginArgs.push(hostname);
-    }
-    catch {
-        // Invalid URL — will be caught later in validation
-    }
-}
-// Set env var for downstream consumption (comma-separated)
-// DOC_ALLOWED_ORIGINS (plural) is the canonical env var.
-// DOC_ALLOWED_ORIGIN (singular) is kept for backward compatibility.
-if (allowedOriginArgs.length > 0) {
-    process.env.DOC_ALLOWED_ORIGINS = allowedOriginArgs.join(",");
-}
-// Document-driven scoping: resolve changed docs to areas and tasks
-// This runs before area/task env var assignment so it can set defaults
-// that are then overridden or intersected by explicit --area/--task flags.
-let impactSummary;
-if (changedDocsOption) {
-    const changedSlugs = changedDocsOption
-        .split(",")
-        .map((s) => s.trim())
-        .filter(Boolean);
-    if (changedSlugs.length > 0) {
-        const reverseMapping = buildReverseMapping(ROOT);
-        impactSummary = assessImpact(changedSlugs, reverseMapping);
-        if (impactSummary.areas.length === 0) {
-            console.warn(`\n⚠️  No evaluation tasks reference any of the changed documents:`);
-            for (const slug of changedSlugs) {
-                console.warn(`    - ${slug}`);
-            }
-            console.warn(`\n    Score impact cannot be measured for these documents.\n`);
-        }
-        else {
-            // Apply document-driven scoping: set areas and tasks from impact
-            // If --area is also provided, intersect (both filters must match)
-            if (areaOption) {
-                const explicitAreas = new Set(areaOption.split(",").map((s) => s.trim()));
-                const intersected = impactSummary.areas.filter((a) => explicitAreas.has(a));
-                if (intersected.length > 0) {
-                    process.env.EVAL_FILTER_AREAS = intersected.join(",");
-                }
-                else {
-                    console.warn(`\n⚠️  No overlap between --area (${areaOption}) and affected areas (${impactSummary.areas.join(", ")})`);
-                    console.warn(`    Running with --area filter only (${areaOption})\n`);
-                }
-            }
-            else {
-                process.env.EVAL_FILTER_AREAS = impactSummary.areas.join(",");
-            }
-            // If --task is also provided, intersect
-            if (taskOption) {
-                const explicitTasks = new Set(taskOption.split(",").map((s) => s.trim()));
-                const intersected = impactSummary.taskIds.filter((t) => explicitTasks.has(t));
-                if (intersected.length > 0) {
-                    process.env.EVAL_FILTER_TASKS = intersected.join(",");
-                }
-                // If no overlap, let the explicit --task filter win
-            }
-            else {
-                process.env.EVAL_FILTER_TASKS = impactSummary.taskIds.join(",");
-            }
-        }
-    }
-}
-// Area/task scoping: passed via env vars to generate-configs subprocess
-// Explicit --area/--task flags override document-driven scoping (if not already set above)
-if (areaOption && !process.env.EVAL_FILTER_AREAS)
-    process.env.EVAL_FILTER_AREAS = areaOption;
-if (taskOption && !process.env.EVAL_FILTER_TASKS)
-    process.env.EVAL_FILTER_TASKS = taskOption;
-// Validate mode
-const VALID_MODES = ["baseline", "observed", "agentic"];
-if (!VALID_MODES.includes(mode)) {
-    console.error(`❌ Invalid mode "${mode}". Must be one of: ${VALID_MODES.join(", ")}`);
-    process.exit(1);
-}
-// Export mode for downstream consumption (calculate-scores source verification)
-process.env.EVAL_MODE = mode;
-// ---------------------------------------------------------------------------
-// Promptfoo share URL extraction
-// ---------------------------------------------------------------------------
-function buildResult(pipelineStart, steps, validation, cacheStats, resolvedPromptfooUrl) {
-    const durationMs = Date.now() - pipelineStart;
-    const success = Object.values(steps).every((s) => s.status === "success" || s.status === "skipped");
-    return {
-        durationMs,
-        promptfooUrl: resolvedPromptfooUrl,
-        steps,
-        success,
-        validation,
-        ...(cacheStats && {
-            cache: {
-                hits: cacheStats.hits,
-                misses: cacheStats.misses,
-                skipped: cacheStats.skipped,
-                total: cacheStats.total,
-            },
-        }),
-    };
-}
-// ---------------------------------------------------------------------------
-// Formatting helpers
-// ---------------------------------------------------------------------------
-function formatDuration(ms) {
-    if (ms < 1000)
-        return `${ms}ms`;
-    if (ms < 60000)
-        return `${(ms / 1000).toFixed(1)}s`;
-    const min = Math.floor(ms / 60000);
-    const sec = Math.round((ms % 60000) / 1000);
-    return `${min}m ${sec}s`;
-}
-function printStepResult(name, result) {
-    const icon = stepIcon(result);
-    switch (result.status) {
-        case "failed":
-            console.log(`  ${icon} ${name} (${formatDuration(result.durationMs)}) — ${result.error}`);
-            break;
-        case "skipped":
-            console.log(`  ${icon} ${name} — ${result.reason}`);
-            break;
-        case "success":
-            console.log(`  ${icon} ${name} (${formatDuration(result.durationMs)}) — ${result.summary}`);
-            break;
-    }
-}
-// ---------------------------------------------------------------------------
-// Pipeline execution
-// ---------------------------------------------------------------------------
-async function runPipeline() {
-    const pipelineStart = Date.now();
-    const steps = {};
-    console.log("=== ai-literacy-framework — Evaluation Pipeline ===\n");
-    const cacheStats = createCacheStats();
-    console.log(`  Mode:       ${mode}`);
-    console.log(`  Source:     ${source ?? "default (production)"}`);
-    console.log(`  Dry run:    ${dryRun}`);
-    console.log(`  Skip fetch: ${skipFetch}`);
-    console.log(`  Skip eval:  ${skipEval}`);
-    console.log(`  Cache:      ${noCache ? "disabled (--no-cache)" : "enabled"}`);
-    if (urlArgs.length > 0) {
-        console.log(`  URLs:       ${urlArgs.length} URL(s)`);
-        for (const u of urlArgs)
-            console.log(`              ${u}`);
-    }
-    if (datasetOverride) {
-        console.log(`  Dataset:    ${datasetOverride}`);
-    }
-    if (projectIdOverride) {
-        console.log(`  Project ID: ${projectIdOverride}`);
-    }
-    if (perspectiveOverride) {
-        console.log(`  Perspective: ${perspectiveOverride}`);
-    }
-    if (studioOriginOverride) {
-        console.log(`  Studio:     ${studioOriginOverride}`);
-    }
-    if (sanityDocumentArgs.length > 0) {
-        console.log(`  Documents:  ${sanityDocumentArgs.length} document(s)`);
-        for (const d of sanityDocumentArgs)
-            console.log(`              ${d}`);
-    }
-    if (allowedOriginArgs.length > 0) {
-        const originDisplay = allowedOriginArgs.join(", ");
-        const autoInferred = urlArgs.length > 0 &&
-            !args.some((a) => a === "--allowed-origin" || a === "--allowed-origins");
-        console.log(`  Origin:     ${originDisplay}${autoInferred ? " (auto-inferred)" : " (sandbox)"}`);
-    }
-    if (searchMode !== "open") {
-        const searchDesc = searchMode === "off" ? "disabled" : "results filtered to allowed origins";
-        console.log(`  Search:     ${searchMode} (${searchDesc})`);
-    }
-    if (headerArgs.length > 0) {
-        console.log(`  Headers:    ${headerArgs.length} custom header(s)`);
-    }
-    if (outputPath) {
-        console.log(`  Output:     ${outputPath}`);
-    }
-    if (debug) {
-        const filters = [
-            debug.firstN ? `first ${debug.firstN}` : null,
-            debug.pattern ? `pattern "${debug.pattern}"` : null,
-            debug.sample ? `sample ${debug.sample}` : null,
-        ].filter(Boolean);
-        console.log(`  Debug:      ${filters.length > 0 ? filters.join(", ") : "first 2 (default)"}`);
-    }
-    if (concurrency) {
-        console.log(`  Concurrency: ${concurrency}`);
-    }
-    if (changedDocsOption) {
-        const slugs = changedDocsOption
-            .split(",")
-            .map((s) => s.trim())
-            .filter(Boolean);
-        console.log(`  Changed:    ${slugs.length} document(s)`);
-        for (const s of slugs)
-            console.log(`              ${s}`);
-        if (impactSummary) {
-            if (impactSummary.areas.length > 0) {
-                console.log(`  Affected:   ${impactSummary.taskIds.length} task(s) in ${impactSummary.areas.length} area(s)`);
-                console.log(`              areas: ${impactSummary.areas.join(", ")}`);
-                console.log(`              tasks: ${impactSummary.taskIds.join(", ")}`);
-            }
-            if (impactSummary.unmatchedSlugs.length > 0) {
-                console.log(`  Unmatched:  ${impactSummary.unmatchedSlugs.length} document(s) not in any task`);
-                for (const s of impactSummary.unmatchedSlugs)
-                    console.log(`              ${s}`);
-            }
-        }
-    }
-    if (areaOption) {
-        console.log(`  Areas:      ${process.env.EVAL_FILTER_AREAS ?? areaOption}`);
-    }
-    else if (impactSummary?.areas.length) {
-        console.log(`  Areas:      ${impactSummary.areas.join(", ")} (from changed docs)`);
-    }
-    if (taskOption) {
-        console.log(`  Tasks:      ${process.env.EVAL_FILTER_TASKS ?? taskOption}`);
-    }
-    else if (impactSummary?.taskIds.length) {
-        console.log(`  Tasks:      ${impactSummary.taskIds.join(", ")} (from changed docs)`);
-    }
-    if (graderReplications) {
-        console.log(`  Grader:     ${graderReplications} replications`);
-    }
-    if (beforeOption) {
-        console.log(`  Before:     ${beforeOption}`);
-    }
-    if (compareEnabled) {
-        const vsTarget = compareBaseline
-            ? ` (vs ${compareBaseline})`
-            : beforeOption
-                ? ` (vs --before ${beforeOption})`
-                : " (vs latest baseline)";
-        console.log(`  Compare:    enabled${vsTarget}`);
-        if (compareThreshold !== undefined) {
-            console.log(`  Threshold:  ±${compareThreshold}`);
-        }
-    }
-    if (readinessEnabled) {
-        const readinessArea = areaOption ?? process.env.EVAL_FILTER_AREAS;
-        console.log(`  Readiness:  enabled${readinessArea ? ` (area: ${readinessArea})` : " (all areas)"}`);
-    }
-    if (discoveryReportEnabled) {
-        console.log("  Discovery:  enabled");
-    }
-    if (publishEnabled) {
-        console.log(`  Publish:    enabled${publishTag ? ` (tag: "${publishTag}")` : ""}`);
-    }
-    console.log();
-    // -----------------------------------------------------------------------
-    // Step 0: Validate configuration
-    // -----------------------------------------------------------------------
-    console.log("─── Step 0: Validate Configuration ────────────────────────\n");
-    const validation = validateConfiguration(ROOT);
-    const envIssues = checkEnvironment(ROOT);
-    validation.issues.push(...envIssues);
-    validation.valid =
-        validation.valid && envIssues.every((i) => i.severity !== "error");
-    const errors = validation.issues.filter((i) => i.severity === "error");
-    const warnings = validation.issues.filter((i) => i.severity === "warning");
-    if (warnings.length > 0) {
-        for (const w of warnings) {
-            console.log(`  ⚠️  [${w.source}] ${w.message}`);
-        }
-        console.log();
-    }
-    if (errors.length > 0) {
-        console.log("❌ Configuration validation failed:\n");
-        for (const e of errors) {
-            console.log(`  ERROR [${e.source}] ${e.message}`);
-            if (e.path)
-                console.log(`        at ${e.path}`);
-        }
-        return {
-            durationMs: Date.now() - pipelineStart,
-            steps,
-            success: false,
-            validation,
-        };
-    }
-    console.log("  ✅ Configuration is valid\n");
-    if (dryRun) {
-        console.log("─── Dry run complete ────────────────────────────────────────────\n");
-        console.log("  Pipeline configuration is valid. No steps were executed.");
-        console.log("  Remove --dry-run to execute the full pipeline.\n");
-        return {
-            durationMs: Date.now() - pipelineStart,
-            steps,
-            success: true,
-            validation,
-        };
-    }
-    // -----------------------------------------------------------------------
-    // Step 1: Fetch documentation
-    // -----------------------------------------------------------------------
-    console.log("─── Step 1: Fetch Documentation ───────────────────────────\n");
-    if (skipFetch) {
-        steps["fetch-docs"] = { reason: "--skip-fetch", status: "skipped" };
-        cacheStats.skipped++;
-    }
-    else {
-        steps["fetch-docs"] = await runFetchDocs(source, noCache);
-    }
-    trackCacheStats(cacheStats, "fetch-docs", steps["fetch-docs"], noCache);
-    printStepResult("fetch-docs", steps["fetch-docs"]);
-    console.log();
-    if (steps["fetch-docs"].status === "failed") {
-        return buildResult(pipelineStart, steps, validation, cacheStats);
-    }
-    // -----------------------------------------------------------------------
-    // Step 2: Generate configs
-    // -----------------------------------------------------------------------
-    console.log("─── Step 2: Generate Configs ──────────────────────────────\n");
-    steps["generate-configs"] = runGenerateConfigs(source, noCache);
-    trackCacheStats(cacheStats, "generate-configs", steps["generate-configs"], noCache);
-    printStepResult("generate-configs", steps["generate-configs"]);
-    console.log();
-    if (steps["generate-configs"].status === "failed") {
-        return buildResult(pipelineStart, steps, validation, cacheStats);
-    }
-    // -----------------------------------------------------------------------
-    // Step 3: Run evaluation
-    // -----------------------------------------------------------------------
-    console.log("─── Step 3: Run Evaluation ────────────────────────────────\n");
-    if (skipEval) {
-        steps["eval"] = { reason: "--skip-eval", status: "skipped" };
-        cacheStats.skipped++;
-    }
-    else {
-        const evalResult = await runEval(mode, debug, concurrency, noCache);
-        steps["eval"] = evalResult.stepResult;
-    }
-    trackCacheStats(cacheStats, "eval", steps["eval"], noCache);
-    printStepResult("eval", steps["eval"]);
-    console.log();
-    if (steps["eval"].status === "failed") {
-        return buildResult(pipelineStart, steps, validation, cacheStats);
-    }
-    // -----------------------------------------------------------------------
-    // Step 3b: Extract Promptfoo share URL (non-blocking)
-    // -----------------------------------------------------------------------
-    // Promptfoo auto-shares when PROMPTFOO_API_KEY is set during eval.
-    // Extract the URL from the eval results JSON for inclusion in the report.
-    let resolvedPromptfooUrl = promptfooUrl;
-    if (!resolvedPromptfooUrl && !skipEval) {
-        resolvedPromptfooUrl = extractShareUrl(mode);
-        if (resolvedPromptfooUrl) {
-            console.log(`  📤 Promptfoo results: ${resolvedPromptfooUrl}\n`);
-        }
-    }
-    // -----------------------------------------------------------------------
-    // Step 3c: Grader consistency analysis (optional)
-    // -----------------------------------------------------------------------
-    if (graderReplications) {
-        console.log("─── Step 3c: Grader Consistency Analysis ────────────────────\n");
-        steps["grader-consistency"] = runGraderConsistency(graderReplications, mode);
-        printStepResult("grader-consistency", steps["grader-consistency"]);
-        console.log();
-        if (steps["grader-consistency"].status === "failed") {
-            return buildResult(pipelineStart, steps, validation, cacheStats);
-        }
-    }
-    // -----------------------------------------------------------------------
-    // Step 4: Calculate scores
-    // -----------------------------------------------------------------------
-    console.log("─── Step 4: Calculate Scores ──────────────────────────────\n");
-    steps["calculate-scores"] = runCalculateScores(source, mode, noCache);
-    trackCacheStats(cacheStats, "calculate-scores", steps["calculate-scores"], noCache);
-    printStepResult("calculate-scores", steps["calculate-scores"]);
-    console.log();
-    if (steps["calculate-scores"].status === "failed") {
-        return buildResult(pipelineStart, steps, validation, cacheStats);
-    }
-    // -----------------------------------------------------------------------
-    // Step 4b: Gap analysis (optional, Phase 3)
-    // -----------------------------------------------------------------------
-    if (gapAnalysisEnabled) {
-        console.log("─── Step 4b: Gap Analysis ─────────────────────────────────\n");
-        const judgmentsPath = resolve(ROOT, "results", "latest", "grader-judgments.json");
-        const scoreSummaryPath = resolve(ROOT, "results", "latest", "score-summary.json");
-        if (!existsSync(judgmentsPath)) {
-            steps["gap-analysis"] = {
-                reason: "No grader-judgments.json — run a full evaluation first",
-                status: "skipped",
-            };
-        }
-        else if (!existsSync(scoreSummaryPath)) {
-            steps["gap-analysis"] = {
-                reason: "No score-summary.json",
-                status: "skipped",
-            };
-        }
-        else {
-            const gaStart = Date.now();
-            try {
-                const { buildFailureModeReport, formatFailureModesConsole } = await import("../pipeline/failure-modes.js");
-                const { buildGapAnalysisReport, formatGapAnalysisConsole } = await import("../pipeline/gap-analysis.js");
-                const judgments = JSON.parse(readFileSync(judgmentsPath, "utf-8"));
-                const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-                // Phase 3a: Failure mode classification
-                const failureModeReport = buildFailureModeReport(judgments, scoreSummary.scores);
-                console.log(formatFailureModesConsole(failureModeReport));
-                // Phase 3b: Gap analysis (impact estimation)
-                const gapReport = buildGapAnalysisReport(failureModeReport, scoreSummary.scores);
-                console.log(formatGapAnalysisConsole(gapReport));
-                // Persist reports
-                const outDir = resolve(ROOT, "results", "latest");
-                writeFileSync(join(outDir, "failure-modes.json"), JSON.stringify(failureModeReport, null, 2));
-                writeFileSync(join(outDir, "gap-analysis.json"), JSON.stringify(gapReport, null, 2));
-                // Enrich score-summary.json with failure mode data
-                const enrichedSummary = {
-                    ...scoreSummary,
-                    failureModes: failureModeReport,
-                };
-                writeFileSync(scoreSummaryPath, JSON.stringify(enrichedSummary, null, 2));
-                const gapCount = gapReport.gaps.length;
-                const classRate = failureModeReport.classificationRate.toFixed(0);
-                steps["gap-analysis"] = {
-                    durationMs: Date.now() - gaStart,
-                    status: "success",
-                    summary: `${failureModeReport.totalJudgments} judgments analyzed (${classRate}% classified), ${gapCount} actionable gaps identified`,
-                };
-            }
-            catch (err) {
-                steps["gap-analysis"] = {
-                    durationMs: Date.now() - gaStart,
-                    error: err instanceof Error ? err.message : String(err),
-                    status: "failed",
-                };
-            }
-        }
-        printStepResult("gap-analysis", steps["gap-analysis"]);
-        console.log();
-        // Gap analysis failures are non-fatal — it's diagnostic, not blocking
-        if (steps["gap-analysis"].status === "failed") {
-            console.warn("  ⚠️  Gap analysis failed — continuing without diagnostic data\n");
-        }
-    }
-    // -----------------------------------------------------------------------
-    // Step 4c: Publish report to store + sinks (optional)
-    // -----------------------------------------------------------------------
-    if (publishEnabled) {
-        console.log("─── Step 4b: Publish Report ───────────────────────────────\n");
-        steps["publish-report"] = await runPublishReport(pipelineStart, {
-            promptfooUrl: resolvedPromptfooUrl,
-            reportDataset,
-            reportProjectId,
-            tag: publishTag,
-        });
-        printStepResult("publish-report", steps["publish-report"]);
-        console.log();
-        // Publish failures are non-fatal (P5: local-first)
-        if (steps["publish-report"].status === "failed") {
-            console.warn("  ⚠️  Publish failed — continuing with local results only\n");
-        }
-    }
-    // -----------------------------------------------------------------------
-    // Step 5: Generate report
-    // -----------------------------------------------------------------------
-    console.log("─── Step 5: Generate Report ───────────────────────────────\n");
-    steps["report"] = runReport(outputPath, resolvedPromptfooUrl);
-    printStepResult("report", steps["report"]);
-    console.log();
-    // -----------------------------------------------------------------------
-    // Step 5b: Compare against baseline (optional)
-    // -----------------------------------------------------------------------
-    if (compareEnabled) {
-        console.log("─── Step 5b: Compare Against Baseline ─────────────────────\n");
-        // Resolve --before option to a baseline file path
-        let resolvedBaseline = compareBaseline;
-        if (beforeOption && !resolvedBaseline) {
-            if (beforeOption === "latest-baseline") {
-                // Use the most recent baseline file (compare step handles this by default)
-                resolvedBaseline = undefined;
-            }
-            else if (beforeOption === "published" ||
-                beforeOption === "production") {
-                // TODO (Phase 2b full): Run before-evaluation with the specified source.
-                // For now, fall back to latest baseline and log a message.
-                console.log(`  ℹ️  --before ${beforeOption}: full before/after orchestration not yet implemented.`);
-                console.log("     Falling back to comparison against latest baseline.\n");
-                resolvedBaseline = undefined;
-            }
-            else if (existsSync(beforeOption)) {
-                // Treat as a path to a score-summary.json file
-                resolvedBaseline = beforeOption;
-                console.log(`  📂 Using before-state from file: ${resolvedBaseline}\n`);
-            }
-            else {
-                console.warn(`  ⚠️  --before "${beforeOption}" is not a recognized option or file path.`);
-                console.warn("     Expected: published, production, latest-baseline, or a file path.\n");
-            }
-        }
-        steps["compare"] = runCompare(ROOT, resolvedBaseline, {
-            noiseThreshold: compareThreshold,
-        });
-        printStepResult("compare", steps["compare"]);
-        // If --changed-docs was provided and comparison succeeded, run attribution
-        if (steps["compare"].status === "success" &&
-            changedDocsOption &&
-            impactSummary) {
-            const comparisonReportPath = resolve(ROOT, "results", "latest", "comparison-report.json");
-            if (existsSync(comparisonReportPath)) {
-                try {
-                    const { attributeChanges, formatAttributionConsole } = await import("../pipeline/attribution.js");
-                    const { resolveMappings } = await import("../pipeline/resolve-mappings.js");
-                    const comparisonReport = JSON.parse(readFileSync(comparisonReportPath, "utf-8"));
-                    const mappings = resolveMappings(ROOT);
-                    const changedSlugs = changedDocsOption
-                        .split(",")
-                        .map((s) => s.trim())
-                        .filter(Boolean);
-                    const attribution = attributeChanges(comparisonReport, changedSlugs, mappings, comparisonReport.noiseThreshold);
-                    // Write attribution to comparison report
-                    const enrichedReport = { ...comparisonReport, attribution };
-                    writeFileSync(comparisonReportPath, JSON.stringify(enrichedReport, null, 2));
-                    // Print attribution to console
-                    console.log();
-                    console.log(formatAttributionConsole(attribution));
-                }
-                catch (err) {
-                    console.warn(`  ⚠️  Attribution failed: ${err instanceof Error ? err.message : String(err)}`);
-                }
-            }
-        }
-        console.log();
-    }
-    // -----------------------------------------------------------------------
-    // Step 6: Readiness report (optional, Phase 5b)
-    // -----------------------------------------------------------------------
-    if (readinessEnabled) {
-        console.log("─── Step 6b: Readiness Report ─────────────────────────────────\n");
-        const readinessStart = Date.now();
-        try {
-            const { formatReadinessMarkdown, generateReadinessReport } = await import("../scripts/readiness-report.js");
-            const { ThresholdConfigSchema } = await import("../pipeline/schemas.js");
-            const scoreSummaryPath = resolve(ROOT, "results", "latest", "score-summary.json");
-            const thresholdsPath = resolve(ROOT, "config", "thresholds.yaml");
-            if (!existsSync(scoreSummaryPath)) {
-                steps["readiness"] = {
-                    durationMs: Date.now() - readinessStart,
-                    error: "score-summary.json not found",
-                    status: "failed",
-                };
-            }
-            else if (!existsSync(thresholdsPath)) {
-                steps["readiness"] = {
-                    durationMs: Date.now() - readinessStart,
-                    error: "config/thresholds.yaml not found — required for readiness report",
-                    status: "failed",
-                };
-            }
-            else {
-                const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-                const rawThresholds = load(readFileSync(thresholdsPath, "utf-8"));
-                const thresholdConfig = ThresholdConfigSchema.parse(rawThresholds);
-                // Load gap analysis if available
-                const gapPath = resolve(ROOT, "results", "latest", "gap-analysis.json");
-                const gapAnalysis = existsSync(gapPath)
-                    ? JSON.parse(readFileSync(gapPath, "utf-8"))
-                    : undefined;
-                // Determine which areas to generate readiness for
-                const readinessAreas = areaOption
-                    ? areaOption.split(",").map((s) => s.trim())
-                    : process.env.EVAL_FILTER_AREAS
-                        ? process.env.EVAL_FILTER_AREAS.split(",").map((s) => s.trim())
-                        : scoreSummary.scores.map((s) => s.feature);
-                const readinessLines = [];
-                for (const area of readinessAreas) {
-                    const areaScore = scoreSummary.scores.find((s) => s.feature === area);
-                    if (!areaScore) {
-                        console.warn(`  ⚠️  Area "${area}" not found in scores — skipping`);
-                        continue;
-                    }
-                    const report = generateReadinessReport({
-                        area,
-                        gapAnalysis,
-                        scoreSummary,
-                        thresholdConfig,
-                    });
-                    const md = formatReadinessMarkdown(report);
-                    readinessLines.push(md);
-                    console.log(md);
-                }
-                // Write combined readiness output
-                if (readinessLines.length > 0) {
-                    const readinessOutPath = resolve(ROOT, "results", "latest", "readiness-report.md");
-                    writeFileSync(readinessOutPath, readinessLines.join("\n---\n\n"));
-                }
-                const passCount = readinessAreas.filter((area) => {
-                    const areaScore = scoreSummary.scores.find((s) => s.feature === area);
-                    if (!areaScore)
-                        return false;
-                    const report = generateReadinessReport({
-                        area,
-                        scoreSummary,
-                        thresholdConfig,
-                    });
-                    return report.pass;
-                }).length;
-                steps["readiness"] = {
-                    durationMs: Date.now() - readinessStart,
-                    status: "success",
-                    summary: `${passCount}/${readinessAreas.length} areas ready`,
-                };
-            }
-        }
-        catch (err) {
-            steps["readiness"] = {
-                durationMs: Date.now() - readinessStart,
-                error: err instanceof Error ? err.message : String(err),
-                status: "failed",
-            };
-        }
-        printStepResult("readiness", steps["readiness"]);
-        console.log();
-    }
-    // -----------------------------------------------------------------------
-    // Step 6c: Discovery report (optional, Phase 5c)
-    // -----------------------------------------------------------------------
-    if (discoveryReportEnabled) {
-        console.log("─── Step 6c: Discovery Report ─────────────────────────────────\n");
-        const discoveryStart = Date.now();
-        try {
-            const { formatDiscoveryMarkdown, generateDiscoveryReport } = await import("../scripts/discovery-report.js");
-            const scoreSummaryPath = resolve(ROOT, "results", "latest", "score-summary.json");
-            if (!existsSync(scoreSummaryPath)) {
-                steps["discovery-report"] = {
-                    durationMs: Date.now() - discoveryStart,
-                    error: "score-summary.json not found",
-                    status: "failed",
-                };
-            }
-            else {
-                const scoreSummary = JSON.parse(readFileSync(scoreSummaryPath, "utf-8"));
-                if (!scoreSummary.retrievalMetrics) {
-                    steps["discovery-report"] = {
-                        reason: "No retrieval metrics in score summary — run an agentic evaluation first",
-                        status: "skipped",
-                    };
-                }
-                else {
-                    const areaFilter = areaOption
-                        ? areaOption.split(",").map((s) => s.trim())
-                        : undefined;
-                    const report = generateDiscoveryReport(scoreSummary, areaFilter);
-                    const md = formatDiscoveryMarkdown(report);
-                    // Write to file
-                    const discoveryOutPath = resolve(ROOT, "results", "latest", "discovery-report.md");
-                    writeFileSync(discoveryOutPath, md);
-                    console.log(md);
-                    const invisible = report.invisibleDocs.length;
-                    const f1 = report.overall.avgF1.toFixed(2);
-                    steps["discovery-report"] = {
-                        durationMs: Date.now() - discoveryStart,
-                        status: "success",
-                        summary: `F1=${f1}, ${invisible} invisible doc${invisible === 1 ? "" : "s"}, ${report.recommendations.length} recommendation${report.recommendations.length === 1 ? "" : "s"}`,
-                    };
-                }
-            }
-        }
-        catch (err) {
-            steps["discovery-report"] = {
-                durationMs: Date.now() - discoveryStart,
-                error: err instanceof Error ? err.message : String(err),
-                status: "failed",
-            };
-        }
-        printStepResult("discovery-report", steps["discovery-report"]);
-        console.log();
-    }
-    return buildResult(pipelineStart, steps, validation, cacheStats, resolvedPromptfooUrl);
-}
-function stepIcon(result) {
-    switch (result.status) {
-        case "failed":
-            return "❌";
-        case "skipped":
-            return "⏭️";
-        case "success":
-            return "✅";
-    }
-}
-/**
- * Track cache hit/miss stats for a step based on its result summary.
- * Steps that return "Skipped (cached)" in their summary are cache hits.
- */
-function trackCacheStats(stats, stepName, result, noCacheFlag) {
-    if (result.status === "skipped") {
-        stats.steps[stepName] = "skipped";
-        // skipped count already incremented at skip site
-        return;
-    }
-    if (noCacheFlag) {
-        stats.total++;
-        stats.misses++;
-        stats.steps[stepName] = "disabled";
-        return;
-    }
-    stats.total++;
-    if (result.status === "success" &&
-        result.summary.startsWith("Skipped (cached)")) {
-        stats.hits++;
-        stats.steps[stepName] = "hit";
-    }
-    else {
-        stats.misses++;
-        stats.steps[stepName] = "miss";
-    }
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-const result = await runPipeline();
-// Print summary
-console.log("═══════════════════════════════════════════════════════════════\n");
-if (result.success) {
-    console.log(`  ✅ Pipeline completed successfully (${formatDuration(result.durationMs)})`);
-}
-else {
-    console.log(`  ❌ Pipeline failed (${formatDuration(result.durationMs)})`);
-}
-console.log();
-console.log("  Steps:");
-for (const [name, step] of Object.entries(result.steps)) {
-    printStepResult(name, step);
-}
-if (result.cache && result.cache.total > 0) {
-    const c = result.cache;
-    const parts = [];
-    if (c.hits > 0)
-        parts.push(`${c.hits} cached`);
-    if (c.misses > 0)
-        parts.push(`${c.misses} evaluated`);
-    if (c.skipped > 0)
-        parts.push(`${c.skipped} skipped`);
-    console.log(`\n  📦 Cache: ${parts.join(", ")} (${c.total} total steps)`);
-}
-console.log();
-// Write machine-readable result for CI consumption
-const resultFile = resolve(ROOT, "results", "latest", "pipeline-result.json");
-try {
-    writeFileSync(resultFile, JSON.stringify(result, null, 2));
-    console.log(`  📄 Pipeline result: ${resultFile}\n`);
-}
-catch {
-    // results/latest/ may not exist yet on first run — not critical
-}
-process.exit(result.success ? 0 : 1);