@sanity/ailf 0.1.0

package/dist/pipeline/calculate-scores.js

@@ -0,0 +1,1128 @@
+/**
+ * pipeline/calculate-scores.ts
+ *
+ * Reads Promptfoo evaluation output and computes the AI Literacy Score
+ * for each feature area. Each dimension is scored on a uniform 0–100 scale:
+ *
+ *   Task Completion    (0–100)  — Can the LLM implement the feature?
+ *   Code Correctness   (0–100)  — Is the code idiomatic and correct?
+ *   Doc Coverage       (0–100)  — Did docs provide the needed info?
+ *
+ * Dimensions are combined into a weighted composite (0–100) using weights
+ * from config/rubrics.yaml (default: Task×0.50 + Code×0.25 + Docs×0.25).
+ *
+ * Additionally compares with-docs vs without-docs scores to calculate
+ * the "Doc Lift" — how much documentation helps vs parametric knowledge.
+ *
+ * When tests are run with the InstrumentedProvider (agent-observer),
+ * this script also aggregates and reports agent behavior data: which
+ * documentation pages were visited, what searches were performed, and
+ * overall network activity patterns.
+ *
+ * All functions accept rootDir as a parameter — no module-level constants.
+ * No process.argv parsing. No env var fallbacks.
+ *
+ * @see docs/exec-plans/active/eliminate-lib-layer.md
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { join } from "path";
+import { calculateCost } from "../agent-observer/pricing.js";
+import { checkResultsExist } from "./checks.js";
+import { loadRubricTemplates } from "./expand-tasks.js";
+import { loadSource } from "../sources.js";
+import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
+import { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
+// Re-export pure functions from core for backward compatibility.
+// Existing imports from this file continue to work unchanged.
+export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, } from "../_vendor/ailf-core/index.js";
+/**
+ * Calculate scores grouped by model. Each model gets its own FeatureScore[]
+ * and model-level aggregates.
+ *
+ * Uses the provider.id from Promptfoo results to identify models.
+ * Falls back to provider.label, then "unknown" if neither is available.
+ *
+ * @returns Record keyed by model ID, or null if only one model was used
+ *          (per-model breakdown is redundant when there's only one model).
+ */
+export function calculateScoresPerModel(resultsPath, weights) {
+    const results = readAndNormalizeResults(resultsPath);
+    // Group results by provider
+    const byModel = {};
+    for (const result of results) {
+        const modelId = result.providerId ?? result.providerLabel ?? "unknown";
+        const label = result.providerLabel ?? result.providerId ?? "unknown";
+        if (!byModel[modelId]) {
+            byModel[modelId] = { label, results: [] };
+        }
+        byModel[modelId].results.push(result);
+    }
+    const modelIds = Object.keys(byModel);
+    // Skip per-model breakdown if there's only one model — it would be
+    // Identical to the overall scores and adds no information.
+    if (modelIds.length <= 1) {
+        return null;
+    }
+    const perModel = [];
+    for (const [modelId, { label, results: modelResults }] of Object.entries(byModel)) {
+        const scores = scoreResults(modelResults, weights, modelId);
+        const totalTests = scores.reduce((s, sc) => s + sc.testCount, 0);
+        const totalCost = scores.reduce((s, sc) => s + sc.totalCost, 0);
+        const avgScore = scores.length > 0
+            ? scores.reduce((s, sc) => s + sc.totalScore, 0) / scores.length
+            : 0;
+        const avgDocLift = scores.length > 0
+            ? scores.reduce((s, sc) => s + sc.docLift, 0) / scores.length
+            : 0;
+        perModel.push({
+            label,
+            modelId,
+            overall: {
+                avgDocLift,
+                avgScore,
+                cost: totalCost > 0 ? totalCost : undefined,
+                testCount: totalTests,
+            },
+            scores,
+        });
+    }
+    return perModel;
+}
+// classifyRubric, detectFeatureArea — imported from @sanity/ailf-core above
+// ---------------------------------------------------------------------------
+// URL extraction from assertion metadata
+// ---------------------------------------------------------------------------
+/**
+ * Extract grader judgments (reason text + scores) from evaluation results.
+ *
+ * This preserves the grader's natural language reasoning for downstream
+ * analysis (failure mode classification, gap analysis). Each llm-rubric
+ * assertion produces one GraderJudgment entry.
+ *
+ * Phase 3a prerequisite: structured judgment data for failure mode extraction.
+ */
+export function extractGraderJudgments(resultsPath) {
+    const results = readAndNormalizeResults(resultsPath);
+    const judgments = [];
+    for (const result of results) {
+        const taskId = result.description;
+        const modelId = result.providerId ?? result.providerLabel ?? "unknown";
+        for (const comp of result.gradingResult.componentResults) {
+            if (comp.assertion?.type !== "llm-rubric") {
+                continue;
+            }
+            const kind = classifyRubric(comp);
+            if (!kind) {
+                continue;
+            }
+            const score = parseRubricScore(comp);
+            // Extract the reason text — the grader's reasoning
+            let reason = comp.reason ?? "";
+            if (reason) {
+                // Try to parse JSON reason to extract the reason field
+                try {
+                    const parsed = JSON.parse(reason);
+                    const obj = parsed;
+                    if (typeof obj.reason === "string") {
+                        ;
+                        ({ reason } = obj);
+                    }
+                }
+                catch {
+                    // Not JSON — use raw reason string
+                }
+            }
+            // Map internal dimension names to hyphenated form
+            const dimensionMap = {
+                codeCorrectness: "code-correctness",
+                docCoverage: "doc-coverage",
+                taskCompletion: "task-completion",
+            };
+            judgments.push({
+                dimension: dimensionMap[kind] ?? kind,
+                modelId,
+                reason,
+                score,
+                taskId,
+            });
+        }
+    }
+    return judgments;
+}
+/**
+ * Finds the URL-extraction assertion result in a test's componentResults
+ * and parses the structured JSON from its `reason` field.
+ */
+// extractUrlMetadata, parseRubricScore — imported from @sanity/ailf-core above
+/**
+ * Aggregates agent behavior data across all test results, grouped by
+ * feature area. Returns null if no behavior data is present.
+ */
+function aggregateAgentBehavior(resultsPath) {
+    const results = readAndNormalizeResults(resultsPath);
+    const byFeature = {};
+    let hasBehaviorData = false;
+    for (const result of results) {
+        const feature = detectFeatureArea(result.description);
+        const behavior = extractAgentBehavior(result);
+        if (!behavior) {
+            continue;
+        }
+        hasBehaviorData = true;
+        if (!byFeature[feature]) {
+            byFeature[feature] = [];
+        }
+        byFeature[feature].push(behavior);
+    }
+    if (!hasBehaviorData) {
+        return null;
+    }
+    return Object.entries(byFeature)
+        .map(([feature, behaviors]) => {
+        const count = behaviors.length || 1;
+        return {
+            avgDocPagesVisited: behaviors.reduce((s, b) => s + b.docPagesVisited, 0) / count,
+            avgNetworkTimeMs: behaviors.reduce((s, b) => s + b.totalNetworkMs, 0) / count,
+            avgSearchesPerformed: behaviors.reduce((s, b) => s + b.searchesPerformed, 0) / count,
+            docSlugsVisited: [
+                ...new Set(behaviors.flatMap((b) => b.docSlugsVisited)),
+            ],
+            externalDomains: [
+                ...new Set(behaviors.flatMap((b) => b.externalDomains)),
+            ],
+            feature,
+            searchQueries: [
+                ...new Set(behaviors.flatMap((b) => b.uniqueSearchQueries)),
+            ],
+            tasksWithBehaviorData: behaviors.length,
+        };
+    })
+        .sort((a, b) => a.feature.localeCompare(b.feature));
+}
+/**
+ * Computes aggregate source isolation metrics from agentic eval results.
+ *
+ * Reads DOC_ALLOWED_ORIGINS from the environment (set by pipeline.ts)
+ * and analyzes all doc page visits across all test results.
+ *
+ * Returns null if no origin sandboxing was configured or no agent behavior
+ * data is present.
+ */
+function aggregateSourceIsolation(resultsPath, allowedOriginsParam) {
+    const allowedOrigins = allowedOriginsParam;
+    if (!allowedOrigins || allowedOrigins.length === 0) {
+        return null;
+    }
+    const results = readAndNormalizeResults(resultsPath);
+    // Collect all doc page visits from all test results
+    const allDocVisits = [];
+    for (const result of results) {
+        const behavior = result.metadata?.agentBehavior;
+        if (!behavior?.docPageVisits) {
+            continue;
+        }
+        for (const visit of behavior.docPageVisits) {
+            allDocVisits.push({ url: visit.url });
+        }
+    }
+    if (allDocVisits.length === 0) {
+        return null;
+    }
+    return analyzeSourceIsolation(allDocVisits, allowedOrigins);
+}
+// ---------------------------------------------------------------------------
+// Feature area detection
+// ---------------------------------------------------------------------------
+/**
+ * Aggregates URL references across all test results, grouped by feature
+ * area and gold/baseline variant.
+ */
+function aggregateUrlReferences(resultsPath) {
+    const results = readAndNormalizeResults(resultsPath);
+    const byFeature = {};
+    for (const result of results) {
+        const feature = detectFeatureArea(result.description);
+        if (!byFeature[feature]) {
+            byFeature[feature] = {
+                baseline: { testCount: 0, urls: {} },
+                gold: { testCount: 0, urls: {} },
+            };
+        }
+        const hasDocs = result.vars.docs && result.vars.docs.trim().length > 0;
+        const bucket = hasDocs
+            ? byFeature[feature].gold
+            : byFeature[feature].baseline;
+        const meta = extractUrlMetadata(result);
+        if (!meta) {
+            continue;
+        }
+        bucket.testCount++;
+        for (const url of meta.sanityUrls) {
+            bucket.urls[url] = (bucket.urls[url] || 0) + 1;
+        }
+    }
+    return Object.entries(byFeature)
+        .map(([feature, data]) => ({ feature, ...data }))
+        .sort((a, b) => a.feature.localeCompare(b.feature));
+}
+// ---------------------------------------------------------------------------
+// Score calculation
+// ---------------------------------------------------------------------------
+/**
+ * Build source verification data for the score summary.
+ *
+ * Combines pipeline configuration (mode, source, sandbox) with runtime
+ * metadata (URL fetch results, isolation scores) to produce a unified
+ * verification report.
+ */
+function buildSourceVerification(root, source, verificationCtx) {
+    const mode = verificationCtx?.mode ?? "baseline";
+    const sourceUrl = source?.baseUrl ?? "default";
+    const searchMode = verificationCtx?.searchMode;
+    const allowedOrigins = verificationCtx?.allowedOrigins;
+    // Read URL fetch metadata if it exists (written by fetch-docs.ts)
+    let urlFetch;
+    const urlFetchPath = join(root, "contexts", "url-fetch.json");
+    if (existsSync(urlFetchPath)) {
+        try {
+            urlFetch = JSON.parse(readFileSync(urlFetchPath, "utf-8"));
+        }
+        catch {
+            // Malformed JSON — skip
+        }
+    }
+    return {
+        ...(allowedOrigins && { allowedOrigins }),
+        mode,
+        ...(searchMode && { searchMode }),
+        source: sourceUrl,
+        ...(urlFetch && {
+            urlFetch: {
+                failures: urlFetch.failures,
+                fetchedUrls: urlFetch.fetchedUrls.map((f) => ({
+                    method: f.method,
+                    status: f.status,
+                    url: f.url,
+                })),
+                totalFailed: urlFetch.totalFailed,
+                totalFetched: urlFetch.totalFetched,
+            },
+        }),
+    };
+}
+// ---------------------------------------------------------------------------
+// Agent behavior aggregation
+// ---------------------------------------------------------------------------
+/**
+ * Calculate overall scores (all models combined).
+ * This is the original scoring path — backward compatible.
+ */
+function calculateScores(resultsPath, weights) {
+    const results = readAndNormalizeResults(resultsPath);
+    return scoreResults(results, weights);
+}
+/**
+ * Extracts agent behavior summary from a test result's metadata.
+ * Returns null if the test was not run with the instrumented provider.
+ */
+function extractAgentBehavior(test) {
+    const { metadata } = test;
+    if (!metadata?.agentBehaviorSummary) {
+        return null;
+    }
+    return metadata.agentBehaviorSummary;
+}
+/**
+ * Extracts grader (assertion) cost from the raw Promptfoo results file.
+ * Promptfoo tracks assertion token usage separately in stats.tokenUsage.assertions.
+ * The grader model is found in config.defaultTest.options.provider.
+ */
+function extractGraderCost(resultsPath) {
+    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    const stats = file.results?.stats;
+    const assertions = stats?.tokenUsage?.assertions;
+    if (!assertions || assertions.total === 0) {
+        return null;
+    }
+    // Extract grader model from config
+    const graderModel = file.config?.defaultTest?.options?.rubricProvider ??
+        file.config?.defaultTest?.options?.provider;
+    // Extract just the model name from "openai:gpt-5-2025-08-07" format
+    const modelName = graderModel?.replace(/^[^:]+:/, "");
+    const cost = modelName
+        ? calculateCost(modelName, assertions.prompt, assertions.completion)
+        : 0;
+    return {
+        completionTokens: assertions.completion,
+        cost,
+        model: graderModel,
+        promptTokens: assertions.prompt,
+        totalTokens: assertions.total,
+    };
+}
+/**
+ * Prints a formatted report of agent behavior observations.
+ */
+function printAgentBehaviorReport(agentBehavior) {
+    console.log("-".repeat(80));
+    console.log("AGENT BEHAVIOR OBSERVATION");
+    console.log("-".repeat(80));
+    console.log();
+    // Summary table
+    const h = "| Feature Area        | Tests | Doc Pages | Searches | Net (ms) |";
+    const sep = "|---------------------|-------|-----------|----------|----------|";
+    console.log(h);
+    console.log(sep);
+    for (const ab of agentBehavior) {
+        console.log(`| ${ab.feature.padEnd(19)} | ` +
+            `${ab.tasksWithBehaviorData.toString().padStart(5)} | ` +
+            `${ab.avgDocPagesVisited.toFixed(1).padStart(9)} | ` +
+            `${ab.avgSearchesPerformed.toFixed(1).padStart(8)} | ` +
+            `${Math.round(ab.avgNetworkTimeMs).toString().padStart(8)} |`);
+    }
+    console.log();
+    // Doc pages visited
+    console.log("  Doc pages visited:");
+    for (const ab of agentBehavior) {
+        if (ab.docSlugsVisited.length === 0) {
+            console.log(`    ${ab.feature}: (none)`);
+        }
+        else {
+            console.log(`    ${ab.feature}:`);
+            for (const slug of ab.docSlugsVisited) {
+                console.log(`      - /docs/${slug}`);
+            }
+        }
+    }
+    console.log();
+    // Search queries
+    const hasSearches = agentBehavior.some((ab) => ab.searchQueries.length > 0);
+    if (hasSearches) {
+        console.log("  Search queries:");
+        for (const ab of agentBehavior) {
+            if (ab.searchQueries.length === 0) {
+                continue;
+            }
+            console.log(`    ${ab.feature}:`);
+            for (const q of ab.searchQueries) {
+                console.log(`      - "${q}"`);
+            }
+        }
+        console.log();
+    }
+    // External domains
+    const allExternalDomains = [
+        ...new Set(agentBehavior.flatMap((ab) => ab.externalDomains)),
+    ];
+    if (allExternalDomains.length > 0) {
+        console.log("  External domains contacted:");
+        for (const d of allExternalDomains) {
+            console.log(`    - ${d}`);
+        }
+        console.log();
+    }
+}
+// ---------------------------------------------------------------------------
+// Report
+// ---------------------------------------------------------------------------
+// ---------------------------------------------------------------------------
+// Grader cost extraction
+// ---------------------------------------------------------------------------
+/**
+ * Reads the raw Promptfoo output file and normalizes each result so that
+ * `description` is always a top-level field (pulled from `testCase` if needed).
+ */
+function readAndNormalizeResults(resultsPath) {
+    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
+    const wrapper = file.results ?? file;
+    const all = wrapper.results.map((r) => ({
+        cost: r.cost ?? 0,
+        description: r.testCase?.description ?? "unknown",
+        error: r.error,
+        gradingResult: r.gradingResult,
+        metadata: r.metadata,
+        provider: r.provider?.label ?? r.provider?.id,
+        providerId: r.provider?.id,
+        providerLabel: r.provider?.label,
+        response: r.response,
+        vars: r.vars ?? r.testCase?.vars ?? {},
+    }));
+    // Filter out results where gradingResult is null (errored/timed-out tests).
+    // Promptfoo sets gradingResult to null when a test errors before grading.
+    const valid = all.filter((r) => r.gradingResult !== null);
+    const skipped = all.length - valid.length;
+    if (skipped > 0) {
+        console.warn(`  ⚠ Skipping ${skipped} of ${all.length} result(s) with null gradingResult (errored tests):`);
+        for (const r of all) {
+            if (r.gradingResult === null) {
+                const providerLabel = r.provider ? `[${r.provider}] ` : "";
+                const errorMsg = r.error
+                    ? r.error.slice(0, 150)
+                    : "unknown error (no error field in result)";
+                console.warn(`    ✗ ${providerLabel}"${r.description}" — ${errorMsg}`);
+            }
+        }
+    }
+    return valid;
+}
+/**
+ * Core scoring logic: takes a pre-filtered array of TestResult and produces
+ * FeatureScore[] grouped by feature area. This is the shared implementation
+ * used by both the overall scoring and per-model scoring paths.
+ *
+ * @param results  Pre-filtered (valid) test results
+ * @param weights  Dimension weights from rubrics.yaml
+ * @param modelId  Optional model identifier to tag each FeatureScore
+ */
+function scoreResults(results, weights, modelId) {
+    const wTask = weights["task-completion"] ?? 0.5;
+    const wCode = weights["code-correctness"] ?? 0.25;
+    const wDoc = weights["doc-coverage"] ?? 0.25;
+    // Group by feature + docs/no-docs
+    const byFeature = {};
+    for (const result of results) {
+        const feature = detectFeatureArea(result.description);
+        if (!byFeature[feature]) {
+            byFeature[feature] = { withDocs: [], withoutDocs: [] };
+        }
+        const hasDocs = result.vars.docs && result.vars.docs.trim().length > 0;
+        if (hasDocs) {
+            byFeature[feature].withDocs.push(result);
+        }
+        else {
+            byFeature[feature].withoutDocs.push(result);
+        }
+    }
+    const scores = [];
+    for (const [feature, data] of Object.entries(byFeature)) {
+        // --- With docs ---
+        let totalTask = 0;
+        let totalCode = 0;
+        let totalDoc = 0;
+        let featureCost = 0;
+        const countWithDocs = data.withDocs.length || 1;
+        for (const test of data.withDocs) {
+            featureCost += test.cost;
+            for (const comp of test.gradingResult.componentResults) {
+                if (comp.assertion?.type !== "llm-rubric") {
+                    continue;
+                }
+                const score = parseRubricScore(comp);
+                const kind = classifyRubric(comp);
+                if (kind === "taskCompletion") {
+                    totalTask += score;
+                }
+                else if (kind === "codeCorrectness") {
+                    totalCode += score;
+                }
+                else if (kind === "docCoverage") {
+                    totalDoc += score;
+                }
+            }
+        }
+        // Per-dimension averages (each 0–100)
+        const avgTask = totalTask / countWithDocs;
+        const avgCode = totalCode / countWithDocs;
+        const avgDoc = totalDoc / countWithDocs;
+        // Weighted composite (0–100)
+        const withDocsTotal = avgTask * wTask + avgCode * wCode + avgDoc * wDoc;
+        // --- Without docs (baseline) ---
+        let baselineTotal = 0;
+        let baselineCount = 0;
+        for (const test of data.withoutDocs) {
+            featureCost += test.cost;
+            for (const comp of test.gradingResult.componentResults) {
+                if (comp.assertion?.type !== "llm-rubric") {
+                    continue;
+                }
+                baselineTotal += parseRubricScore(comp);
+                baselineCount++;
+            }
+        }
+        const withoutDocsScore = baselineCount > 0 ? baselineTotal / baselineCount : 0;
+        const ceilingScore = Math.round(withDocsTotal);
+        const floorScore = Math.round(withoutDocsScore);
+        const docLift = ceilingScore - floorScore;
+        scores.push({
+            ceilingScore,
+            codeCorrectness: Math.round(avgCode),
+            docCoverage: Math.round(avgDoc),
+            docLift,
+            docQualityGap: 100 - ceilingScore,
+            feature,
+            floorScore,
+            ...(modelId && { modelId }),
+            negativeDocLift: docLift < 0,
+            taskCompletion: Math.round(avgTask),
+            testCount: data.withDocs.length,
+            totalCost: featureCost,
+            totalScore: ceilingScore,
+        });
+    }
+    return scores.sort((a, b) => a.feature.localeCompare(b.feature));
+}
+// ---------------------------------------------------------------------------
+// Agentic scoring — all results are "actual" (agent retrieves docs via tools)
+// ---------------------------------------------------------------------------
+/**
+ * Score agentic evaluation results. In agentic mode, all test entries are
+ * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).
+ * The model retrieves docs via tools, so all results map to "actual" scores.
+ *
+ * Returns a record keyed by feature area with the composite actual score.
+ */
+// ActualScoreEntry — imported from @sanity/ailf-core via pipeline/types.js
+export function scoreAgenticResults(resultsPath, weights) {
+    const results = readAndNormalizeResults(resultsPath);
+    const wTask = weights["task-completion"] ?? 0.5;
+    const wCode = weights["code-correctness"] ?? 0.25;
+    const wDoc = weights["doc-coverage"] ?? 0.25;
+    // Group by feature area
+    const byFeature = {};
+    for (const result of results) {
+        const feature = detectFeatureArea(result.description);
+        if (!byFeature[feature]) {
+            byFeature[feature] = [];
+        }
+        byFeature[feature].push(result);
+    }
+    const entries = {};
+    for (const [feature, featureResults] of Object.entries(byFeature)) {
+        let totalTask = 0;
+        let totalCode = 0;
+        let totalDoc = 0;
+        let featureCost = 0;
+        const count = featureResults.length || 1;
+        for (const test of featureResults) {
+            featureCost += test.cost;
+            for (const comp of test.gradingResult.componentResults) {
+                if (comp.assertion?.type !== "llm-rubric")
+                    continue;
+                const score = parseRubricScore(comp);
+                const kind = classifyRubric(comp);
+                if (kind === "taskCompletion")
+                    totalTask += score;
+                else if (kind === "codeCorrectness")
+                    totalCode += score;
+                else if (kind === "docCoverage")
+                    totalDoc += score;
+            }
+        }
+        const avgTask = totalTask / count;
+        const avgCode = totalCode / count;
+        const avgDoc = totalDoc / count;
+        const actualScore = Math.round(avgTask * wTask + avgCode * wCode + avgDoc * wDoc);
+        entries[feature] = {
+            actualScore,
+            codeCorrectness: Math.round(avgCode),
+            docCoverage: Math.round(avgDoc),
+            taskCompletion: Math.round(avgTask),
+            testCount: featureResults.length,
+            totalCost: featureCost,
+        };
+    }
+    return entries;
+}
+// ---------------------------------------------------------------------------
+// Score merging — combine baseline floor/ceiling with agentic actual
+// ---------------------------------------------------------------------------
+/**
+ * Merge baseline FeatureScore[] with agentic actual scores to produce
+ * the full three-layer decomposition.
+ *
+ * The merge is per feature area. For each area:
+ * - If baseline data exists: floor, ceiling, docLift, docQualityGap are populated
+ * - If agentic data exists: actualScore is populated
+ * - If both exist: retrievalGap and infrastructureEfficiency are computed
+ *
+ * @param baselineScores  Floor/ceiling scores from baseline evaluation (may be empty)
+ * @param agenticScores   Actual scores from agentic evaluation (may be empty)
+ */
+// mergeScores — imported from @sanity/ailf-core above
+const CRITICAL_THRESHOLD = 40;
+export function calculateAndWriteScores(options) {
+    const ROOT = options.rootDir;
+    const sourceName = options.source;
+    // Pre-resolved source wins over name-based lookup
+    let source = options.resolvedSource;
+    if (!source) {
+        try {
+            source = loadSource(sourceName);
+        }
+        catch {
+            console.warn(`  [warn] Could not load source "${sourceName}", proceeding without source metadata`);
+        }
+    }
+    // Determine mode — controls which result files are read
+    const mode = options.mode ?? "baseline";
+    const baselineResultsPath = options.resultsPath ?? join(ROOT, "results", "latest", "eval-results.json");
+    // Agentic results path (only used in full mode)
+    const agenticResultsPath = join(ROOT, "results", "latest", "eval-results-agentic.json");
+    // Validate baseline results file
+    const resultsIssues = checkResultsExist(ROOT, baselineResultsPath);
+    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
+    if (resultsErrors.length > 0) {
+        console.error("❌ Results validation failed:");
+        for (const e of resultsErrors) {
+            console.error(`  ERROR: ${e.message}`);
+            if (e.path) {
+                console.error(`         at ${e.path}`);
+            }
+        }
+        console.error("\nRun 'pnpm eval' first to generate results, then 'pnpm calculate-scores'.");
+        process.exit(1);
+    }
+    console.log(`Reading results from: ${baselineResultsPath}`);
+    if (source) {
+        console.log(`Source: ${sourceName} (${source.baseUrl})`);
+    }
+    // Load dimension weights from rubrics.yaml
+    const rubricConfig = loadRubricTemplates(ROOT);
+    const baselineScores = calculateScores(baselineResultsPath, rubricConfig.weights);
+    const perModel = calculateScoresPerModel(baselineResultsPath, rubricConfig.weights);
+    const urlRefs = aggregateUrlReferences(baselineResultsPath);
+    const sourceVerification = buildSourceVerification(ROOT, source, {
+        allowedOrigins: options.allowedOrigins,
+        mode: options.mode ?? mode,
+        searchMode: options.searchMode,
+    });
+    const graderCost = extractGraderCost(baselineResultsPath);
+    // Full mode: merge baseline floor/ceiling with agentic actual scores
+    let scores;
+    let agentBehavior = null;
+    let sourceIsolation = null;
+    let evaluationMode;
+    if (mode === "full" && existsSync(agenticResultsPath)) {
+        console.log(`\nReading agentic results from: ${agenticResultsPath}`);
+        const agenticScores = scoreAgenticResults(agenticResultsPath, rubricConfig.weights);
+        scores = mergeScores(baselineScores, agenticScores);
+        evaluationMode = "full";
+        // Aggregate agent behavior and source isolation from agentic results
+        agentBehavior = aggregateAgentBehavior(agenticResultsPath);
+        sourceIsolation = aggregateSourceIsolation(agenticResultsPath, options?.allowedOrigins);
+        // Merge grader costs from both files
+        const agenticGraderCost = extractGraderCost(agenticResultsPath);
+        if (graderCost && agenticGraderCost) {
+            graderCost.cost += agenticGraderCost.cost;
+            graderCost.totalTokens += agenticGraderCost.totalTokens;
+            graderCost.promptTokens += agenticGraderCost.promptTokens;
+            graderCost.completionTokens += agenticGraderCost.completionTokens;
+        }
+    }
+    else if (mode === "agentic") {
+        scores = baselineScores;
+        agentBehavior = aggregateAgentBehavior(baselineResultsPath);
+        sourceIsolation = aggregateSourceIsolation(baselineResultsPath, options?.allowedOrigins);
+        evaluationMode = "agentic";
+    }
+    else {
+        scores = baselineScores;
+        agentBehavior = aggregateAgentBehavior(baselineResultsPath);
+        sourceIsolation = aggregateSourceIsolation(baselineResultsPath, options?.allowedOrigins);
+        evaluationMode = mode === "observed" ? "observed" : "baseline";
+    }
+    const summary = printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode);
+    // Persist
+    const outDir = join(ROOT, "results", "latest");
+    mkdirSync(outDir, { recursive: true });
+    writeFileSync(join(outDir, "score-summary.json"), JSON.stringify(summary, null, 2));
+    console.log("Score summary written to results/latest/score-summary.json");
+    // Extract and persist grader judgments (Phase 3a: failure mode extraction)
+    const judgments = extractGraderJudgments(baselineResultsPath);
+    // In full mode, also extract judgments from agentic results
+    if (mode === "full" && existsSync(agenticResultsPath)) {
+        const agenticJudgments = extractGraderJudgments(agenticResultsPath);
+        judgments.push(...agenticJudgments);
+    }
+    if (judgments.length > 0) {
+        writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
+        console.log(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
+    }
+    // Exit with non-zero if any area below critical threshold
+    if (summary.belowCritical.length > 0) {
+        process.exit(1);
+    }
+}
+function printPerModelReport(perModel) {
+    console.log("-".repeat(80));
+    console.log("PER-MODEL BREAKDOWN");
+    console.log("-".repeat(80));
+    console.log();
+    // Model summary table
+    const h = "| Model                          | Avg Score | Avg Lift | Tests |     Cost |";
+    const sep = "|--------------------------------|-----------|----------|-------|----------|";
+    console.log(h);
+    console.log(sep);
+    const sorted = [...perModel].sort((a, b) => b.overall.avgScore - a.overall.avgScore);
+    for (const entry of sorted) {
+        const displayName = entry.label || entry.modelId;
+        const costStr = entry.overall.cost
+            ? `$${entry.overall.cost.toFixed(4)}`
+            : "—";
+        const liftStr = entry.overall.avgDocLift >= 0
+            ? `+${entry.overall.avgDocLift.toFixed(1)}`
+            : entry.overall.avgDocLift.toFixed(1);
+        console.log(`| ${displayName.padEnd(30)} | ` +
+            `${entry.overall.avgScore.toFixed(1).padStart(9)} | ` +
+            `${liftStr.padStart(8)} | ` +
+            `${entry.overall.testCount.toString().padStart(5)} | ` +
+            `${costStr.padStart(8)} |`);
+    }
+    console.log();
+    // Per-model × per-area breakdown
+    for (const entry of sorted) {
+        const displayName = entry.label || entry.modelId;
+        console.log(`  ${displayName} (${entry.modelId}):`);
+        const areaH = "    | Feature Area        | Task | Code | Docs | Total | Lift |";
+        const areaSep = "    |---------------------|------|------|------|-------|------|";
+        console.log(areaH);
+        console.log(areaSep);
+        for (const s of entry.scores) {
+            const lift = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
+            console.log(`    | ${s.feature.padEnd(19)} | ` +
+                `${s.taskCompletion.toString().padStart(4)} | ` +
+                `${s.codeCorrectness.toString().padStart(4)} | ` +
+                `${s.docCoverage.toString().padStart(4)} | ` +
+                `${s.totalScore.toString().padStart(5)} | ` +
+                `${lift.padStart(4)} |`);
+        }
+        console.log();
+    }
+    // Cost-per-quality-point
+    const modelsWithCost = sorted.filter((e) => e.overall.cost && e.overall.cost > 0);
+    if (modelsWithCost.length > 0) {
+        console.log("  Cost per quality point:");
+        for (const entry of modelsWithCost) {
+            const displayName = entry.label;
+            const costPerPoint = entry.overall.avgScore > 0
+                ? (entry.overall.cost ?? 0) / entry.overall.avgScore
+                : 0;
+            console.log(`    ${displayName}: $${costPerPoint.toFixed(6)}/point (score: ${entry.overall.avgScore.toFixed(1)}, cost: $${(entry.overall.cost ?? 0).toFixed(4)})`);
+        }
+        console.log();
+    }
+}
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+function printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode) {
+    console.log("\n" + "=".repeat(80));
+    console.log("                    SANITY AI LITERACY SCORE REPORT");
+    console.log("=".repeat(80));
+    console.log();
+    // Table header
+    const h = "| Feature Area        | Task | Code | Docs | Total | w/o Docs | Doc Lift |";
+    const sep = "|---------------------|------|------|------|-------|----------|----------|";
+    console.log(h);
+    console.log(sep);
+    for (const s of scores) {
+        const status = s.totalScore < CRITICAL_THRESHOLD ? "!!" : "ok";
+        const lift = s.docLift > 0 ? `+${s.docLift}` : `${s.docLift}`;
+        console.log(`| ${status} ${s.feature.padEnd(17)} | ` +
+            `${s.taskCompletion.toString().padStart(4)} | ` +
+            `${s.codeCorrectness.toString().padStart(4)} | ` +
+            `${s.docCoverage.toString().padStart(4)} | ` +
+            `${s.totalScore.toString().padStart(5)} | ` +
+            `${s.floorScore.toString().padStart(8)} | ` +
+            `${lift.padStart(8)} |`);
+    }
+    console.log();
+    // OKR status
+    const belowCritical = scores.filter((s) => s.totalScore < CRITICAL_THRESHOLD);
+    const lowestScore = scores.reduce((min, s) => s.totalScore < min.totalScore ? s : min);
+    const avgScore = scores.reduce((sum, s) => sum + s.totalScore, 0) / scores.length;
+    const avgLift = scores.reduce((sum, s) => sum + s.docLift, 0) / scores.length;
+    const avgCeilingScore = scores.reduce((sum, s) => sum + s.ceilingScore, 0) / scores.length;
+    const avgFloorScore = scores.reduce((sum, s) => sum + s.floorScore, 0) / scores.length;
+    const avgDocQualityGap = scores.reduce((sum, s) => sum + s.docQualityGap, 0) / scores.length;
+    const negativeDocLiftScores = scores.filter((s) => s.negativeDocLift);
+    const negativeDocLiftAreas = negativeDocLiftScores.map((s) => ({
+        area: s.feature,
+        docLift: s.docLift,
+    }));
+    console.log("-".repeat(80));
+    console.log("OKR STATUS");
+    console.log("-".repeat(80));
+    console.log();
+    if (belowCritical.length === 0) {
+        console.log("  KR1: PASS -- All areas above critical threshold (>=40)");
+    }
+    else {
+        console.log("  KR1: FAIL -- Areas below critical threshold:");
+        belowCritical.forEach((s) => console.log(`        - ${s.feature}: ${s.totalScore}`));
+    }
+    console.log();
+    console.log(`  Lowest area: ${lowestScore.feature} (${lowestScore.totalScore})`);
+    console.log(`  Target: +15 points improvement`);
+    console.log();
+    console.log(`  Avg score:     ${avgScore.toFixed(1)}`);
+    console.log(`  Avg doc lift:  +${avgLift.toFixed(1)} points`);
+    console.log(`  (Doc lift = how much docs help vs parametric knowledge alone)`);
+    console.log();
+    // Ceiling decomposition
+    console.log("-".repeat(80));
+    console.log("CEILING DECOMPOSITION");
+    console.log("-".repeat(80));
+    console.log();
+    const ceilH = "| Feature Area        | Floor | Ceiling | Doc Lift | Quality Gap |";
+    const ceilSep = "|---------------------|-------|---------|----------|-------------|";
+    console.log(ceilH);
+    console.log(ceilSep);
+    for (const s of scores) {
+        const liftStr = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
+        const liftFlag = s.negativeDocLift ? " 🚨" : "";
+        console.log(`| ${s.feature.padEnd(19)} | ` +
+            `${s.floorScore.toString().padStart(5)} | ` +
+            `${s.ceilingScore.toString().padStart(7)} | ` +
+            `${liftStr.padStart(8)}${liftFlag} | ` +
+            `${s.docQualityGap.toString().padStart(11)} |`);
+    }
+    console.log();
+    if (negativeDocLiftAreas.length > 0) {
+        console.log("  🚨 NEGATIVE DOC LIFT DETECTED:");
+        for (const { area, docLift } of negativeDocLiftAreas) {
+            const s = scores.find((sc) => sc.feature === area);
+            console.log(`     ${area}: Doc Lift = ${docLift} (floor: ${s.floorScore}, ceiling: ${s.ceilingScore})`);
+        }
+        console.log("     Documentation is HURTING model performance for these areas.");
+        console.log("     See docs/design-docs/scenario-matrix/evaluation-ceiling.md");
+        console.log();
+    }
+    else {
+        console.log("  ✅ No areas with negative Doc Lift detected.");
+        console.log();
+    }
+    // Three-layer decomposition (only when actual scores are present)
+    const hasActualScores = scores.some((s) => s.actualScore !== undefined);
+    if (hasActualScores) {
+        console.log("-".repeat(80));
+        console.log("THREE-LAYER DECOMPOSITION (floor → ceiling → actual)");
+        console.log("-".repeat(80));
+        console.log();
+        const decompH = "| Feature Area        | Floor | Ceiling | Actual | Doc Lift | Ret. Gap | Infra % |";
+        const decompSep = "|---------------------|-------|---------|--------|----------|----------|---------|";
+        console.log(decompH);
+        console.log(decompSep);
+        for (const s of scores) {
+            const liftStr = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
+            const actualStr = s.actualScore !== undefined ? s.actualScore.toString() : "—";
+            const gapStr = s.retrievalGap !== undefined
+                ? s.retrievalGap >= 0
+                    ? `+${s.retrievalGap}`
+                    : `${s.retrievalGap}`
+                : "—";
+            const infraStr = s.infrastructureEfficiency != null
+                ? `${Math.round(s.infrastructureEfficiency * 100)}%`
+                : "—";
+            const flag = s.invertedRetrievalGap ? " 🔄" : "";
+            console.log(`| ${s.feature.padEnd(19)} | ` +
+                `${s.floorScore.toString().padStart(5)} | ` +
+                `${s.ceilingScore.toString().padStart(7)} | ` +
+                `${actualStr.padStart(6)} | ` +
+                `${liftStr.padStart(8)} | ` +
+                `${(gapStr + flag).padStart(8)} | ` +
+                `${infraStr.padStart(7)} |`);
+        }
+        console.log();
+        console.log("  Doc Lift = ceiling − floor  |  Ret. Gap = ceiling − actual  |  Infra = actual / ceiling");
+        console.log("  🔄 = inverted retrieval gap (agents avoid bad docs → higher actual than ceiling)");
+        console.log();
+    }
+    // Cost summary
+    const totalCost = scores.reduce((sum, s) => sum + s.totalCost, 0);
+    const totalTests = scores.reduce((sum, s) => sum + s.testCount, 0);
+    const graderCostTotal = graderCost?.cost ?? 0;
+    const combinedCost = totalCost + graderCostTotal;
+    if (totalCost > 0 || graderCostTotal > 0) {
+        console.log("-".repeat(80));
+        console.log("COST SUMMARY");
+        console.log("-".repeat(80));
+        console.log();
+        console.log(`  Provider cost:        $${totalCost.toFixed(4)}`);
+        if (graderCostTotal > 0) {
+            const graderLabel = graderCost?.model ?? "unknown";
+            console.log(`  Grader cost:          $${graderCostTotal.toFixed(4)}  (${graderLabel}, ${(graderCost?.totalTokens ?? 0).toLocaleString()} tokens)`);
+        }
+        console.log(`  Total cost:           $${combinedCost.toFixed(4)}`);
+        console.log(`  Avg cost per test:    $${(combinedCost / (totalTests || 1)).toFixed(4)}`);
+        console.log();
+        const costHeader = "| Feature Area        | Tests |     Cost | Avg/Test |";
+        const costSep = "|---------------------|-------|----------|----------|";
+        console.log(costHeader);
+        console.log(costSep);
+        for (const s of scores) {
+            const avgCost = s.testCount > 0 ? s.totalCost / s.testCount : 0;
+            console.log(`| ${s.feature.padEnd(19)} | ` +
+                `${s.testCount.toString().padStart(5)} | ` +
+                `$${s.totalCost.toFixed(4).padStart(7)} | ` +
+                `$${avgCost.toFixed(4).padStart(7)} |`);
+        }
+        console.log();
+    }
+    // Per-model breakdown
+    if (perModel) {
+        printPerModelReport(perModel);
+    }
+    // URL References
+    printUrlReport(urlRefs);
+    // Agent Behavior (only present when run with instrumented provider)
+    if (agentBehavior && agentBehavior.length > 0) {
+        printAgentBehaviorReport(agentBehavior);
+    }
+    // Source verification (unified report for all modes)
+    if (sourceVerification || sourceIsolation) {
+        console.log("-".repeat(80));
+        console.log("📋 SOURCE VERIFICATION");
+        console.log("-".repeat(80));
+        if (sourceVerification) {
+            console.log(`  Source:  ${sourceVerification.source}`);
+            console.log(`  Mode:    ${sourceVerification.mode}`);
+            if (sourceVerification.allowedOrigins) {
+                console.log(`  Sandbox: ${sourceVerification.allowedOrigins.join(", ")}`);
+            }
+            if (sourceVerification.searchMode) {
+                console.log(`  Search:  ${sourceVerification.searchMode}`);
+            }
+            // URL fetch results (baseline mode with direct URLs)
+            if (sourceVerification.urlFetch) {
+                const uf = sourceVerification.urlFetch;
+                console.log();
+                console.log(`  URL fetch: ${uf.totalFetched} fetched, ${uf.totalFailed} failed`);
+                for (const f of uf.fetchedUrls) {
+                    console.log(`    ✅ ${f.url} (via ${f.method})`);
+                }
+                for (const f of uf.failures) {
+                    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means no error info
+                    console.log(`    ⚠️  ${f.url}: ${f.error || "unknown error"}`);
+                }
+            }
+        }
+        // Agentic isolation score
+        if (sourceIsolation) {
+            const pct = Math.round(sourceIsolation.isolationScore * 100);
+            const icon = sourceIsolation.offOrigin === 0 ? "✅" : "⚠️";
+            console.log();
+            console.log(`  Agent isolation: ${icon} ${pct}% (${sourceIsolation.onOrigin}/${sourceIsolation.total} on-origin)`);
+            if (sourceIsolation.offOrigin > 0) {
+                console.log(`  Off-origin fetches: ${sourceIsolation.offOrigin}`);
+                for (const url of sourceIsolation.offOriginUrls.slice(0, 10)) {
+                    console.log(`    • ${url}`);
+                }
+            }
+            if (Object.keys(sourceIsolation.originBreakdown).length > 0) {
+                console.log("  Origin breakdown:");
+                for (const [origin, count] of Object.entries(sourceIsolation.originBreakdown).sort((a, b) => b[1] - a[1])) {
+                    console.log(`    ${origin}: ${count}`);
+                }
+            }
+        }
+        console.log();
+    }
+    // Build overall agent behavior stats for summary
+    const overallAgentBehavior = agentBehavior && agentBehavior.length > 0
+        ? {
+            avgDocPagesVisited: agentBehavior.reduce((s, ab) => s + ab.avgDocPagesVisited, 0) /
+                agentBehavior.length,
+            avgNetworkTimeMs: agentBehavior.reduce((s, ab) => s + ab.avgNetworkTimeMs, 0) /
+                agentBehavior.length,
+            avgSearchesPerformed: agentBehavior.reduce((s, ab) => s + ab.avgSearchesPerformed, 0) /
+                agentBehavior.length,
+            testsWithBehaviorData: agentBehavior.reduce((s, ab) => s + ab.tasksWithBehaviorData, 0),
+            totalUniqueDocSlugs: [
+                ...new Set(agentBehavior.flatMap((ab) => ab.docSlugsVisited)),
+            ].length,
+            totalUniqueSearchQueries: [
+                ...new Set(agentBehavior.flatMap((ab) => ab.searchQueries)),
+            ].length,
+        }
+        : undefined;
+    // Compute aggregate metrics from actual scores (when agentic data present)
+    const scoresWithActual = scores.filter((s) => s.actualScore !== undefined);
+    const avgActualScore = scoresWithActual.length > 0
+        ? scoresWithActual.reduce((sum, s) => sum + (s.actualScore ?? 0), 0) /
+            scoresWithActual.length
+        : undefined;
+    const scoresWithGap = scores.filter((s) => s.retrievalGap !== undefined);
+    const avgRetrievalGap = scoresWithGap.length > 0
+        ? scoresWithGap.reduce((sum, s) => sum + (s.retrievalGap ?? 0), 0) /
+            scoresWithGap.length
+        : undefined;
+    const scoresWithInfra = scores.filter((s) => s.infrastructureEfficiency != null);
+    const avgInfrastructureEfficiency = scoresWithInfra.length > 0
+        ? scoresWithInfra.reduce((sum, s) => sum + (s.infrastructureEfficiency ?? 0), 0) / scoresWithInfra.length
+        : undefined;
+    return {
+        agentBehavior: agentBehavior ?? undefined,
+        belowCritical: belowCritical.map((s) => s.feature),
+        ...(evaluationMode && { evaluationMode }),
+        lowestArea: lowestScore.feature,
+        lowestScore: lowestScore.totalScore,
+        ...(negativeDocLiftAreas.length > 0 && { negativeDocLiftAreas }),
+        overall: {
+            agentBehavior: overallAgentBehavior,
+            ...(avgActualScore !== undefined && { avgActualScore }),
+            avgCeilingScore: avgCeilingScore,
+            avgDocLift: avgLift,
+            avgDocQualityGap: avgDocQualityGap,
+            avgFloorScore: avgFloorScore,
+            ...(avgInfrastructureEfficiency !== undefined && {
+                avgInfrastructureEfficiency,
+            }),
+            ...(avgRetrievalGap !== undefined && { avgRetrievalGap }),
+            avgScore,
+            cost: totalCost > 0 || graderCostTotal > 0
+                ? {
+                    graderModel: graderCost?.model,
+                    graderTotal: graderCostTotal,
+                    perTest: combinedCost / (totalTests || 1),
+                    total: combinedCost,
+                    totalTokens: graderCost?.totalTokens ?? 0,
+                }
+                : undefined,
+            negativeDocLiftCount: negativeDocLiftAreas.length,
+        },
+        scores,
+        source: source
+            ? {
+                baseUrl: source.baseUrl,
+                dataset: source.dataset,
+                name: source.name ?? "default",
+                perspective: source.perspective,
+                projectId: source.projectId,
+            }
+            : undefined,
+        ...(perModel && { perModel }),
+        ...(sourceIsolation && { sourceIsolation }),
+        ...(sourceVerification && { sourceVerification }),
+        timestamp: new Date().toISOString(),
+        urlReferences: urlRefs,
+    };
+}
+function printUrlReport(urlRefs) {
+    console.log("-".repeat(80));
+    console.log("URL REFERENCES");
+    console.log("-".repeat(80));
+    console.log();
+    for (const ref of urlRefs) {
+        const goldUrls = Object.entries(ref.gold.urls).sort((a, b) => b[1] - a[1]);
+        const baselineUrls = Object.entries(ref.baseline.urls).sort((a, b) => b[1] - a[1]);
+        if (goldUrls.length > 0) {
+            console.log(`  ${ref.feature} (gold):`);
+            for (const [url, count] of goldUrls) {
+                const suffix = count > 1 ? ` (${count} tests)` : "";
+                console.log(`    ${url}${suffix}`);
+            }
+        }
+        if (baselineUrls.length > 0) {
+            console.log(`  ${ref.feature} (baseline):`);
+            for (const [url, count] of baselineUrls) {
+                const suffix = count > 1 ? ` (${count} tests)` : "";
+                console.log(`    ${url}${suffix} [parametric]`);
+            }
+        }
+        if (goldUrls.length === 0 && baselineUrls.length === 0) {
+            console.log(`  ${ref.feature}: no URLs referenced`);
+        }
+        console.log();
+    }
+}