@sanity/ailf 2.0.1 → 2.1.0

package/dist/scripts/calculate-scores.js

@@ -1,1296 +0,0 @@
-/**
- * Calculate-scores.ts
- *
- * Reads Promptfoo evaluation output and computes the AI Literacy Score
- * for each feature area. Each dimension is scored on a uniform 0–100 scale:
- *
- *   Task Completion    (0–100)  — Can the LLM implement the feature?
- *   Code Correctness   (0–100)  — Is the code idiomatic and correct?
- *   Doc Coverage       (0–100)  — Did docs provide the needed info?
- *
- * Dimensions are combined into a weighted composite (0–100) using weights
- * from config/rubrics.yaml (default: Task×0.50 + Code×0.25 + Docs×0.25).
- *
- * Additionally compares with-docs vs without-docs scores to calculate
- * the "Doc Lift" — how much documentation helps vs parametric knowledge.
- *
- * When tests are run with the InstrumentedProvider (agent-observer),
- * this script also aggregates and reports agent behavior data: which
- * documentation pages were visited, what searches were performed, and
- * overall network activity patterns.
- */
-// oxlint-disable-next-line import/no-unassigned-import -- side-effect: loads .env into process.env
-import "dotenv/config";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { dirname, join } from "path";
-import { calculateCost } from "../agent-observer/pricing.js";
-import { checkResultsExist } from "../pipeline/checks.js";
-import { loadRubricTemplates } from "../pipeline/expand-tasks.js";
-import { loadSource } from "../sources.js";
-import { analyzeSourceIsolation, } from "../assertions/source-isolation.js";
-/**
- * Calculate scores grouped by model. Each model gets its own FeatureScore[]
- * and model-level aggregates.
- *
- * Uses the provider.id from Promptfoo results to identify models.
- * Falls back to provider.label, then "unknown" if neither is available.
- *
- * @returns Record keyed by model ID, or null if only one model was used
- *          (per-model breakdown is redundant when there's only one model).
- */
-export function calculateScoresPerModel(resultsPath, weights) {
-    const results = readAndNormalizeResults(resultsPath);
-    // Group results by provider
-    const byModel = {};
-    for (const result of results) {
-        const modelId = result.providerId ?? result.providerLabel ?? "unknown";
-        const label = result.providerLabel ?? result.providerId ?? "unknown";
-        if (!byModel[modelId]) {
-            byModel[modelId] = { label, results: [] };
-        }
-        byModel[modelId].results.push(result);
-    }
-    const modelIds = Object.keys(byModel);
-    // Skip per-model breakdown if there's only one model — it would be
-    // Identical to the overall scores and adds no information.
-    if (modelIds.length <= 1) {
-        return null;
-    }
-    const perModel = [];
-    for (const [modelId, { label, results: modelResults }] of Object.entries(byModel)) {
-        const scores = scoreResults(modelResults, weights, modelId);
-        const totalTests = scores.reduce((s, sc) => s + sc.testCount, 0);
-        const totalCost = scores.reduce((s, sc) => s + sc.totalCost, 0);
-        const avgScore = scores.length > 0
-            ? scores.reduce((s, sc) => s + sc.totalScore, 0) / scores.length
-            : 0;
-        const avgDocLift = scores.length > 0
-            ? scores.reduce((s, sc) => s + sc.docLift, 0) / scores.length
-            : 0;
-        perModel.push({
-            label,
-            modelId,
-            overall: {
-                avgDocLift,
-                avgScore,
-                cost: totalCost > 0 ? totalCost : undefined,
-                testCount: totalTests,
-            },
-            scores,
-        });
-    }
-    return perModel;
-}
-export function classifyRubric(component) {
-    // Prefer structured metadata (Approach 5) over heuristic matching
-    const metadata = component.assertion?.metadata;
-    if (metadata?.dimension) {
-        switch (metadata.dimension) {
-            case "code-correctness":
-                return "codeCorrectness";
-            case "doc-coverage":
-                return "docCoverage";
-            case "task-completion":
-                return "taskCompletion";
-            default:
-                return null;
-        }
-    }
-    // Fallback: heuristic name matching (for backward compatibility)
-    const value = (component.assertion?.value ?? "").toLowerCase();
-    if (value.includes("task completion")) {
-        return "taskCompletion";
-    }
-    if (value.includes("code correctness")) {
-        return "codeCorrectness";
-    }
-    if (value.includes("documentation coverage") ||
-        value.includes("hallucinate")) {
-        return "docCoverage";
-    }
-    return null;
-}
-export function detectFeatureArea(description) {
-    const desc = description.toLowerCase();
-    if (desc.includes("studio")) {
-        return "studio-setup";
-    }
-    if (desc.includes("visual") ||
-        desc.includes("presentation") ||
-        desc.includes("live preview")) {
-        return "visual-editing";
-    }
-    if (desc.includes("function") || desc.includes("webhook")) {
-        return "functions";
-    }
-    if (desc.startsWith("groq")) {
-        return "groq";
-    }
-    if (desc.includes("next") || desc.includes("app router")) {
-        return "nextjs-live";
-    }
-    if (desc.includes("remix") ||
-        desc.includes("nuxt") ||
-        desc.includes("svelte")) {
-        return "frameworks";
-    }
-    return "other";
-}
-// ---------------------------------------------------------------------------
-// URL extraction from assertion metadata
-// ---------------------------------------------------------------------------
-/**
- * Extract grader judgments (reason text + scores) from evaluation results.
- *
- * This preserves the grader's natural language reasoning for downstream
- * analysis (failure mode classification, gap analysis). Each llm-rubric
- * assertion produces one GraderJudgment entry.
- *
- * Phase 3a prerequisite: structured judgment data for failure mode extraction.
- */
-export function extractGraderJudgments(resultsPath) {
-    const results = readAndNormalizeResults(resultsPath);
-    const judgments = [];
-    for (const result of results) {
-        const taskId = result.description;
-        const modelId = result.providerId ?? result.providerLabel ?? "unknown";
-        for (const comp of result.gradingResult.componentResults) {
-            if (comp.assertion?.type !== "llm-rubric") {
-                continue;
-            }
-            const kind = classifyRubric(comp);
-            if (!kind) {
-                continue;
-            }
-            const score = parseRubricScore(comp);
-            // Extract the reason text — the grader's reasoning
-            let reason = comp.reason ?? "";
-            if (reason) {
-                // Try to parse JSON reason to extract the reason field
-                try {
-                    const parsed = JSON.parse(reason);
-                    const obj = parsed;
-                    if (typeof obj.reason === "string") {
-                        ;
-                        ({ reason } = obj);
-                    }
-                }
-                catch {
-                    // Not JSON — use raw reason string
-                }
-            }
-            // Map internal dimension names to hyphenated form
-            const dimensionMap = {
-                codeCorrectness: "code-correctness",
-                docCoverage: "doc-coverage",
-                taskCompletion: "task-completion",
-            };
-            judgments.push({
-                dimension: dimensionMap[kind] ?? kind,
-                modelId,
-                reason,
-                score,
-                taskId,
-            });
-        }
-    }
-    return judgments;
-}
-/**
- * Finds the URL-extraction assertion result in a test's componentResults
- * and parses the structured JSON from its `reason` field.
- */
-export function extractUrlMetadata(test) {
-    for (const comp of test.gradingResult.componentResults) {
-        if (comp.assertion?.type !== "javascript") {
-            continue;
-        }
-        if (!comp.reason) {
-            continue;
-        }
-        try {
-            const parsed = JSON.parse(comp.reason);
-            const obj = parsed;
-            if (Array.isArray(obj.sanityUrls) &&
-                typeof obj.totalUrlCount === "number") {
-                return parsed;
-            }
-        }
-        catch {
-            // Not the URL-extraction assertion - skip
-        }
-    }
-    return null;
-}
-// ---------------------------------------------------------------------------
-// Per-model scoring (Approach 3 from evaluation roadmap)
-// ---------------------------------------------------------------------------
-export function parseRubricScore(component) {
-    // Direct score field
-    if (typeof component.score === "number") {
-        return component.score;
-    }
-    // Try to extract from reason (LLM rubric returns JSON)
-    if (component.reason) {
-        try {
-            const parsed = JSON.parse(component.reason);
-            const obj = parsed;
-            if (typeof obj.score === "number") {
-                return obj.score;
-            }
-        }
-        catch {
-            // Try to find a bare number
-            const match = component.reason.match(/(\d+)/);
-            if (match) {
-                return parseInt(match[1], 10);
-            }
-        }
-    }
-    return 0;
-}
-/**
- * Aggregates agent behavior data across all test results, grouped by
- * feature area. Returns null if no behavior data is present.
- */
-function aggregateAgentBehavior(resultsPath) {
-    const results = readAndNormalizeResults(resultsPath);
-    const byFeature = {};
-    let hasBehaviorData = false;
-    for (const result of results) {
-        const feature = detectFeatureArea(result.description);
-        const behavior = extractAgentBehavior(result);
-        if (!behavior) {
-            continue;
-        }
-        hasBehaviorData = true;
-        if (!byFeature[feature]) {
-            byFeature[feature] = [];
-        }
-        byFeature[feature].push(behavior);
-    }
-    if (!hasBehaviorData) {
-        return null;
-    }
-    return Object.entries(byFeature)
-        .map(([feature, behaviors]) => {
-        const count = behaviors.length || 1;
-        return {
-            avgDocPagesVisited: behaviors.reduce((s, b) => s + b.docPagesVisited, 0) / count,
-            avgNetworkTimeMs: behaviors.reduce((s, b) => s + b.totalNetworkMs, 0) / count,
-            avgSearchesPerformed: behaviors.reduce((s, b) => s + b.searchesPerformed, 0) / count,
-            docSlugsVisited: [
-                ...new Set(behaviors.flatMap((b) => b.docSlugsVisited)),
-            ],
-            externalDomains: [
-                ...new Set(behaviors.flatMap((b) => b.externalDomains)),
-            ],
-            feature,
-            searchQueries: [
-                ...new Set(behaviors.flatMap((b) => b.uniqueSearchQueries)),
-            ],
-            tasksWithBehaviorData: behaviors.length,
-        };
-    })
-        .sort((a, b) => a.feature.localeCompare(b.feature));
-}
-/**
- * Computes aggregate source isolation metrics from agentic eval results.
- *
- * Reads DOC_ALLOWED_ORIGINS from the environment (set by pipeline.ts)
- * and analyzes all doc page visits across all test results.
- *
- * Returns null if no origin sandboxing was configured or no agent behavior
- * data is present.
- */
-function aggregateSourceIsolation(resultsPath) {
-    const originsEnv = process.env.DOC_ALLOWED_ORIGINS;
-    if (!originsEnv) {
-        return null;
-    }
-    const allowedOrigins = originsEnv
-        .split(",")
-        .map((o) => o.trim())
-        .filter(Boolean);
-    if (allowedOrigins.length === 0) {
-        return null;
-    }
-    const results = readAndNormalizeResults(resultsPath);
-    // Collect all doc page visits from all test results
-    const allDocVisits = [];
-    for (const result of results) {
-        const behavior = result.metadata?.agentBehavior;
-        if (!behavior?.docPageVisits) {
-            continue;
-        }
-        for (const visit of behavior.docPageVisits) {
-            allDocVisits.push({ url: visit.url });
-        }
-    }
-    if (allDocVisits.length === 0) {
-        return null;
-    }
-    return analyzeSourceIsolation(allDocVisits, allowedOrigins);
-}
-// ---------------------------------------------------------------------------
-// Feature area detection
-// ---------------------------------------------------------------------------
-/**
- * Aggregates URL references across all test results, grouped by feature
- * area and gold/baseline variant.
- */
-function aggregateUrlReferences(resultsPath) {
-    const results = readAndNormalizeResults(resultsPath);
-    const byFeature = {};
-    for (const result of results) {
-        const feature = detectFeatureArea(result.description);
-        if (!byFeature[feature]) {
-            byFeature[feature] = {
-                baseline: { testCount: 0, urls: {} },
-                gold: { testCount: 0, urls: {} },
-            };
-        }
-        const hasDocs = result.vars.docs && result.vars.docs.trim().length > 0;
-        const bucket = hasDocs
-            ? byFeature[feature].gold
-            : byFeature[feature].baseline;
-        const meta = extractUrlMetadata(result);
-        if (!meta) {
-            continue;
-        }
-        bucket.testCount++;
-        for (const url of meta.sanityUrls) {
-            bucket.urls[url] = (bucket.urls[url] || 0) + 1;
-        }
-    }
-    return Object.entries(byFeature)
-        .map(([feature, data]) => ({ feature, ...data }))
-        .sort((a, b) => a.feature.localeCompare(b.feature));
-}
-// ---------------------------------------------------------------------------
-// Score calculation
-// ---------------------------------------------------------------------------
-/**
- * Build source verification data for the score summary.
- *
- * Combines pipeline configuration (mode, source, sandbox) with runtime
- * metadata (URL fetch results, isolation scores) to produce a unified
- * verification report.
- */
-function buildSourceVerification(root, source) {
-    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means unset
-    const mode = process.env.EVAL_MODE || "baseline";
-    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means unset
-    const sourceUrl = source?.baseUrl || process.env.DOC_BASE_URL || "default";
-    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means unset
-    const searchMode = process.env.EVAL_SEARCH_MODE || undefined;
-    const allowedOriginsEnv = process.env.DOC_ALLOWED_ORIGINS;
-    const allowedOrigins = allowedOriginsEnv
-        ? allowedOriginsEnv
-            .split(",")
-            .map((o) => o.trim())
-            .filter(Boolean)
-        : undefined;
-    // Read URL fetch metadata if it exists (written by fetch-docs.ts)
-    let urlFetch;
-    const urlFetchPath = join(root, "contexts", "url-fetch.json");
-    if (existsSync(urlFetchPath)) {
-        try {
-            urlFetch = JSON.parse(readFileSync(urlFetchPath, "utf-8"));
-        }
-        catch {
-            // Malformed JSON — skip
-        }
-    }
-    return {
-        ...(allowedOrigins && { allowedOrigins }),
-        mode,
-        ...(searchMode && { searchMode }),
-        source: sourceUrl,
-        ...(urlFetch && {
-            urlFetch: {
-                failures: urlFetch.failures,
-                fetchedUrls: urlFetch.fetchedUrls.map((f) => ({
-                    method: f.method,
-                    status: f.status,
-                    url: f.url,
-                })),
-                totalFailed: urlFetch.totalFailed,
-                totalFetched: urlFetch.totalFetched,
-            },
-        }),
-    };
-}
-// ---------------------------------------------------------------------------
-// Agent behavior aggregation
-// ---------------------------------------------------------------------------
-/**
- * Calculate overall scores (all models combined).
- * This is the original scoring path — backward compatible.
- */
-function calculateScores(resultsPath, weights) {
-    const results = readAndNormalizeResults(resultsPath);
-    return scoreResults(results, weights);
-}
-/**
- * Extracts agent behavior summary from a test result's metadata.
- * Returns null if the test was not run with the instrumented provider.
- */
-function extractAgentBehavior(test) {
-    const { metadata } = test;
-    if (!metadata?.agentBehaviorSummary) {
-        return null;
-    }
-    return metadata.agentBehaviorSummary;
-}
-/**
- * Extracts grader (assertion) cost from the raw Promptfoo results file.
- * Promptfoo tracks assertion token usage separately in stats.tokenUsage.assertions.
- * The grader model is found in config.defaultTest.options.provider.
- */
-function extractGraderCost(resultsPath) {
-    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
-    const stats = file.results?.stats;
-    const assertions = stats?.tokenUsage?.assertions;
-    if (!assertions || assertions.total === 0) {
-        return null;
-    }
-    // Extract grader model from config
-    const graderModel = file.config?.defaultTest?.options?.rubricProvider ??
-        file.config?.defaultTest?.options?.provider;
-    // Extract just the model name from "openai:gpt-5-2025-08-07" format
-    const modelName = graderModel?.replace(/^[^:]+:/, "");
-    const cost = modelName
-        ? calculateCost(modelName, assertions.prompt, assertions.completion)
-        : 0;
-    return {
-        completionTokens: assertions.completion,
-        cost,
-        model: graderModel,
-        promptTokens: assertions.prompt,
-        totalTokens: assertions.total,
-    };
-}
-/**
- * Prints a formatted report of agent behavior observations.
- */
-function printAgentBehaviorReport(agentBehavior) {
-    console.log("-".repeat(80));
-    console.log("AGENT BEHAVIOR OBSERVATION");
-    console.log("-".repeat(80));
-    console.log();
-    // Summary table
-    const h = "| Feature Area        | Tests | Doc Pages | Searches | Net (ms) |";
-    const sep = "|---------------------|-------|-----------|----------|----------|";
-    console.log(h);
-    console.log(sep);
-    for (const ab of agentBehavior) {
-        console.log(`| ${ab.feature.padEnd(19)} | ` +
-            `${ab.tasksWithBehaviorData.toString().padStart(5)} | ` +
-            `${ab.avgDocPagesVisited.toFixed(1).padStart(9)} | ` +
-            `${ab.avgSearchesPerformed.toFixed(1).padStart(8)} | ` +
-            `${Math.round(ab.avgNetworkTimeMs).toString().padStart(8)} |`);
-    }
-    console.log();
-    // Doc pages visited
-    console.log("  Doc pages visited:");
-    for (const ab of agentBehavior) {
-        if (ab.docSlugsVisited.length === 0) {
-            console.log(`    ${ab.feature}: (none)`);
-        }
-        else {
-            console.log(`    ${ab.feature}:`);
-            for (const slug of ab.docSlugsVisited) {
-                console.log(`      - /docs/${slug}`);
-            }
-        }
-    }
-    console.log();
-    // Search queries
-    const hasSearches = agentBehavior.some((ab) => ab.searchQueries.length > 0);
-    if (hasSearches) {
-        console.log("  Search queries:");
-        for (const ab of agentBehavior) {
-            if (ab.searchQueries.length === 0) {
-                continue;
-            }
-            console.log(`    ${ab.feature}:`);
-            for (const q of ab.searchQueries) {
-                console.log(`      - "${q}"`);
-            }
-        }
-        console.log();
-    }
-    // External domains
-    const allExternalDomains = [
-        ...new Set(agentBehavior.flatMap((ab) => ab.externalDomains)),
-    ];
-    if (allExternalDomains.length > 0) {
-        console.log("  External domains contacted:");
-        for (const d of allExternalDomains) {
-            console.log(`    - ${d}`);
-        }
-        console.log();
-    }
-}
-// ---------------------------------------------------------------------------
-// Report
-// ---------------------------------------------------------------------------
-// ---------------------------------------------------------------------------
-// Grader cost extraction
-// ---------------------------------------------------------------------------
-/**
- * Reads the raw Promptfoo output file and normalizes each result so that
- * `description` is always a top-level field (pulled from `testCase` if needed).
- */
-function readAndNormalizeResults(resultsPath) {
-    const file = JSON.parse(readFileSync(resultsPath, "utf-8"));
-    const wrapper = file.results ?? file;
-    const all = wrapper.results.map((r) => ({
-        cost: r.cost ?? 0,
-        description: r.testCase?.description ?? "unknown",
-        error: r.error,
-        gradingResult: r.gradingResult,
-        metadata: r.metadata,
-        provider: r.provider?.label ?? r.provider?.id,
-        providerId: r.provider?.id,
-        providerLabel: r.provider?.label,
-        response: r.response,
-        vars: r.vars ?? r.testCase?.vars ?? {},
-    }));
-    // Filter out results where gradingResult is null (errored/timed-out tests).
-    // Promptfoo sets gradingResult to null when a test errors before grading.
-    const valid = all.filter((r) => r.gradingResult !== null);
-    const skipped = all.length - valid.length;
-    if (skipped > 0) {
-        console.warn(`  ⚠ Skipping ${skipped} of ${all.length} result(s) with null gradingResult (errored tests):`);
-        for (const r of all) {
-            if (r.gradingResult === null) {
-                const providerLabel = r.provider ? `[${r.provider}] ` : "";
-                const errorMsg = r.error
-                    ? r.error.slice(0, 150)
-                    : "unknown error (no error field in result)";
-                console.warn(`    ✗ ${providerLabel}"${r.description}" — ${errorMsg}`);
-            }
-        }
-    }
-    return valid;
-}
-/**
- * Core scoring logic: takes a pre-filtered array of TestResult and produces
- * FeatureScore[] grouped by feature area. This is the shared implementation
- * used by both the overall scoring and per-model scoring paths.
- *
- * @param results  Pre-filtered (valid) test results
- * @param weights  Dimension weights from rubrics.yaml
- * @param modelId  Optional model identifier to tag each FeatureScore
- */
-function scoreResults(results, weights, modelId) {
-    const wTask = weights["task-completion"] ?? 0.5;
-    const wCode = weights["code-correctness"] ?? 0.25;
-    const wDoc = weights["doc-coverage"] ?? 0.25;
-    // Group by feature + docs/no-docs
-    const byFeature = {};
-    for (const result of results) {
-        const feature = detectFeatureArea(result.description);
-        if (!byFeature[feature]) {
-            byFeature[feature] = { withDocs: [], withoutDocs: [] };
-        }
-        const hasDocs = result.vars.docs && result.vars.docs.trim().length > 0;
-        if (hasDocs) {
-            byFeature[feature].withDocs.push(result);
-        }
-        else {
-            byFeature[feature].withoutDocs.push(result);
-        }
-    }
-    const scores = [];
-    for (const [feature, data] of Object.entries(byFeature)) {
-        // --- With docs ---
-        let totalTask = 0;
-        let totalCode = 0;
-        let totalDoc = 0;
-        let featureCost = 0;
-        const countWithDocs = data.withDocs.length || 1;
-        for (const test of data.withDocs) {
-            featureCost += test.cost;
-            for (const comp of test.gradingResult.componentResults) {
-                if (comp.assertion?.type !== "llm-rubric") {
-                    continue;
-                }
-                const score = parseRubricScore(comp);
-                const kind = classifyRubric(comp);
-                if (kind === "taskCompletion") {
-                    totalTask += score;
-                }
-                else if (kind === "codeCorrectness") {
-                    totalCode += score;
-                }
-                else if (kind === "docCoverage") {
-                    totalDoc += score;
-                }
-            }
-        }
-        // Per-dimension averages (each 0–100)
-        const avgTask = totalTask / countWithDocs;
-        const avgCode = totalCode / countWithDocs;
-        const avgDoc = totalDoc / countWithDocs;
-        // Weighted composite (0–100)
-        const withDocsTotal = avgTask * wTask + avgCode * wCode + avgDoc * wDoc;
-        // --- Without docs (baseline) ---
-        let baselineTotal = 0;
-        let baselineCount = 0;
-        for (const test of data.withoutDocs) {
-            featureCost += test.cost;
-            for (const comp of test.gradingResult.componentResults) {
-                if (comp.assertion?.type !== "llm-rubric") {
-                    continue;
-                }
-                baselineTotal += parseRubricScore(comp);
-                baselineCount++;
-            }
-        }
-        const withoutDocsScore = baselineCount > 0 ? baselineTotal / baselineCount : 0;
-        const ceilingScore = Math.round(withDocsTotal);
-        const floorScore = Math.round(withoutDocsScore);
-        const docLift = ceilingScore - floorScore;
-        scores.push({
-            ceilingScore,
-            codeCorrectness: Math.round(avgCode),
-            docCoverage: Math.round(avgDoc),
-            docLift,
-            docQualityGap: 100 - ceilingScore,
-            feature,
-            floorScore,
-            ...(modelId && { modelId }),
-            negativeDocLift: docLift < 0,
-            taskCompletion: Math.round(avgTask),
-            testCount: data.withDocs.length,
-            totalCost: featureCost,
-            totalScore: ceilingScore,
-        });
-    }
-    return scores.sort((a, b) => a.feature.localeCompare(b.feature));
-}
-export function scoreAgenticResults(resultsPath, weights) {
-    const results = readAndNormalizeResults(resultsPath);
-    const wTask = weights["task-completion"] ?? 0.5;
-    const wCode = weights["code-correctness"] ?? 0.25;
-    const wDoc = weights["doc-coverage"] ?? 0.25;
-    // Group by feature area
-    const byFeature = {};
-    for (const result of results) {
-        const feature = detectFeatureArea(result.description);
-        if (!byFeature[feature]) {
-            byFeature[feature] = [];
-        }
-        byFeature[feature].push(result);
-    }
-    const entries = {};
-    for (const [feature, featureResults] of Object.entries(byFeature)) {
-        let totalTask = 0;
-        let totalCode = 0;
-        let totalDoc = 0;
-        let featureCost = 0;
-        const count = featureResults.length || 1;
-        for (const test of featureResults) {
-            featureCost += test.cost;
-            for (const comp of test.gradingResult.componentResults) {
-                if (comp.assertion?.type !== "llm-rubric")
-                    continue;
-                const score = parseRubricScore(comp);
-                const kind = classifyRubric(comp);
-                if (kind === "taskCompletion")
-                    totalTask += score;
-                else if (kind === "codeCorrectness")
-                    totalCode += score;
-                else if (kind === "docCoverage")
-                    totalDoc += score;
-            }
-        }
-        const avgTask = totalTask / count;
-        const avgCode = totalCode / count;
-        const avgDoc = totalDoc / count;
-        const actualScore = Math.round(avgTask * wTask + avgCode * wCode + avgDoc * wDoc);
-        entries[feature] = {
-            actualScore,
-            codeCorrectness: Math.round(avgCode),
-            docCoverage: Math.round(avgDoc),
-            taskCompletion: Math.round(avgTask),
-            testCount: featureResults.length,
-            totalCost: featureCost,
-        };
-    }
-    return entries;
-}
-// ---------------------------------------------------------------------------
-// Score merging — combine baseline floor/ceiling with agentic actual
-// ---------------------------------------------------------------------------
-/**
- * Merge baseline FeatureScore[] with agentic actual scores to produce
- * the full three-layer decomposition.
- *
- * The merge is per feature area. For each area:
- * - If baseline data exists: floor, ceiling, docLift, docQualityGap are populated
- * - If agentic data exists: actualScore is populated
- * - If both exist: retrievalGap and infrastructureEfficiency are computed
- *
- * @param baselineScores  Floor/ceiling scores from baseline evaluation (may be empty)
- * @param agenticScores   Actual scores from agentic evaluation (may be empty)
- */
-export function mergeScores(baselineScores, agenticScores) {
-    // Build a map of baseline scores by feature
-    const baselineMap = new Map();
-    for (const score of baselineScores) {
-        baselineMap.set(score.feature, score);
-    }
-    // Collect all feature areas from both sources
-    const allFeatures = new Set([
-        ...baselineScores.map((s) => s.feature),
-        ...Object.keys(agenticScores),
-    ]);
-    const merged = [];
-    for (const feature of allFeatures) {
-        const baseline = baselineMap.get(feature);
-        const agentic = agenticScores[feature];
-        if (baseline && agentic) {
-            // Both data sources — full decomposition
-            const retrievalGap = baseline.ceilingScore - agentic.actualScore;
-            const negativeDocLift = baseline.docLift < 0;
-            // Infrastructure efficiency: actual / ceiling
-            // Null when ceiling ≤ 0 or negative Doc Lift (the metric is meaningless
-            // when docs hurt performance — see evaluation-ceiling.md)
-            let infrastructureEfficiency = null;
-            if (!negativeDocLift && baseline.ceilingScore > 0) {
-                infrastructureEfficiency = agentic.actualScore / baseline.ceilingScore;
-            }
-            // Inverted retrieval gap: agents outperform injected docs.
-            // This happens when Doc Lift is negative AND actual > ceiling.
-            // It means retrieval failure is masking a doc quality problem.
-            const invertedRetrievalGap = negativeDocLift && retrievalGap < 0;
-            merged.push({
-                ...baseline,
-                actualScore: agentic.actualScore,
-                infrastructureEfficiency,
-                invertedRetrievalGap: invertedRetrievalGap || undefined,
-                retrievalGap,
-                totalCost: baseline.totalCost + agentic.totalCost,
-            });
-        }
-        else if (baseline) {
-            // Baseline only — no agentic data (partial summary)
-            merged.push({ ...baseline });
-        }
-        else if (agentic) {
-            // Agentic only — no baseline data (partial summary)
-            merged.push({
-                actualScore: agentic.actualScore,
-                ceilingScore: 0,
-                codeCorrectness: agentic.codeCorrectness,
-                docCoverage: agentic.docCoverage,
-                docLift: 0,
-                docQualityGap: 100,
-                feature,
-                floorScore: 0,
-                negativeDocLift: false,
-                taskCompletion: agentic.taskCompletion,
-                testCount: agentic.testCount,
-                totalCost: agentic.totalCost,
-                totalScore: agentic.actualScore,
-            });
-        }
-    }
-    return merged.sort((a, b) => a.feature.localeCompare(b.feature));
-}
-const CRITICAL_THRESHOLD = 40;
-function main() {
-    const ROOT = join(dirname(new URL(import.meta.url).pathname), "..", "..");
-    const args = process.argv.slice(2);
-    // Parse --source <name> argument
-    const sourceIdx = args.indexOf("--source");
-    const sourceName = sourceIdx !== -1 ? args[sourceIdx + 1] : undefined;
-    // Always load source config so environment info is included in score summary.
-    // When no --source is specified, defaults to production.
-    let source;
-    try {
-        source = loadSource(sourceName);
-    }
-    catch {
-        console.warn(`  [warn] Could not load source "${sourceName}", proceeding without source metadata`);
-    }
-    // Determine mode — controls which result files are read
-    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means unset
-    const mode = process.env.EVAL_MODE || "baseline";
-    // First positional arg (not a flag) is the results path (baseline results)
-    const baselineResultsPath = args.find((a) => !a.startsWith("--") && args[args.indexOf(a) - 1] !== "--source") ?? join(ROOT, "results", "latest", "eval-results.json");
-    // Agentic results path (only used in full mode)
-    const agenticResultsPath = join(ROOT, "results", "latest", "eval-results-agentic.json");
-    // Validate baseline results file
-    const resultsIssues = checkResultsExist(ROOT, baselineResultsPath);
-    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
-    if (resultsErrors.length > 0) {
-        console.error("❌ Results validation failed:");
-        for (const e of resultsErrors) {
-            console.error(`  ERROR: ${e.message}`);
-            if (e.path) {
-                console.error(`         at ${e.path}`);
-            }
-        }
-        console.error("\nRun 'pnpm eval' first to generate results, then 'pnpm calculate-scores'.");
-        process.exit(1);
-    }
-    console.log(`Reading results from: ${baselineResultsPath}`);
-    if (source) {
-        console.log(`Source: ${sourceName} (${source.baseUrl})`);
-    }
-    // Load dimension weights from rubrics.yaml
-    const rubricConfig = loadRubricTemplates(ROOT);
-    const baselineScores = calculateScores(baselineResultsPath, rubricConfig.weights);
-    const perModel = calculateScoresPerModel(baselineResultsPath, rubricConfig.weights);
-    const urlRefs = aggregateUrlReferences(baselineResultsPath);
-    const sourceVerification = buildSourceVerification(ROOT, source);
-    const graderCost = extractGraderCost(baselineResultsPath);
-    // Full mode: merge baseline floor/ceiling with agentic actual scores
-    let scores;
-    let agentBehavior = null;
-    let sourceIsolation = null;
-    let evaluationMode;
-    if (mode === "full" && existsSync(agenticResultsPath)) {
-        console.log(`\nReading agentic results from: ${agenticResultsPath}`);
-        const agenticScores = scoreAgenticResults(agenticResultsPath, rubricConfig.weights);
-        scores = mergeScores(baselineScores, agenticScores);
-        evaluationMode = "full";
-        // Aggregate agent behavior and source isolation from agentic results
-        agentBehavior = aggregateAgentBehavior(agenticResultsPath);
-        sourceIsolation = aggregateSourceIsolation(agenticResultsPath);
-        // Merge grader costs from both files
-        const agenticGraderCost = extractGraderCost(agenticResultsPath);
-        if (graderCost && agenticGraderCost) {
-            graderCost.cost += agenticGraderCost.cost;
-            graderCost.totalTokens += agenticGraderCost.totalTokens;
-            graderCost.promptTokens += agenticGraderCost.promptTokens;
-            graderCost.completionTokens += agenticGraderCost.completionTokens;
-        }
-    }
-    else if (mode === "agentic") {
-        scores = baselineScores;
-        agentBehavior = aggregateAgentBehavior(baselineResultsPath);
-        sourceIsolation = aggregateSourceIsolation(baselineResultsPath);
-        evaluationMode = "agentic";
-    }
-    else {
-        scores = baselineScores;
-        agentBehavior = aggregateAgentBehavior(baselineResultsPath);
-        sourceIsolation = aggregateSourceIsolation(baselineResultsPath);
-        evaluationMode = mode === "observed" ? "observed" : "baseline";
-    }
-    const summary = printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode);
-    // Persist
-    const outDir = join(ROOT, "results", "latest");
-    mkdirSync(outDir, { recursive: true });
-    writeFileSync(join(outDir, "score-summary.json"), JSON.stringify(summary, null, 2));
-    console.log("Score summary written to results/latest/score-summary.json");
-    // Extract and persist grader judgments (Phase 3a: failure mode extraction)
-    const judgments = extractGraderJudgments(baselineResultsPath);
-    // In full mode, also extract judgments from agentic results
-    if (mode === "full" && existsSync(agenticResultsPath)) {
-        const agenticJudgments = extractGraderJudgments(agenticResultsPath);
-        judgments.push(...agenticJudgments);
-    }
-    if (judgments.length > 0) {
-        writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
-        console.log(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
-    }
-    // Exit with non-zero if any area below critical threshold
-    if (summary.belowCritical.length > 0) {
-        process.exit(1);
-    }
-}
-function printPerModelReport(perModel) {
-    console.log("-".repeat(80));
-    console.log("PER-MODEL BREAKDOWN");
-    console.log("-".repeat(80));
-    console.log();
-    // Model summary table
-    const h = "| Model                          | Avg Score | Avg Lift | Tests |     Cost |";
-    const sep = "|--------------------------------|-----------|----------|-------|----------|";
-    console.log(h);
-    console.log(sep);
-    const sorted = [...perModel].sort((a, b) => b.overall.avgScore - a.overall.avgScore);
-    for (const entry of sorted) {
-        const displayName = entry.label || entry.modelId;
-        const costStr = entry.overall.cost
-            ? `$${entry.overall.cost.toFixed(4)}`
-            : "—";
-        const liftStr = entry.overall.avgDocLift >= 0
-            ? `+${entry.overall.avgDocLift.toFixed(1)}`
-            : entry.overall.avgDocLift.toFixed(1);
-        console.log(`| ${displayName.padEnd(30)} | ` +
-            `${entry.overall.avgScore.toFixed(1).padStart(9)} | ` +
-            `${liftStr.padStart(8)} | ` +
-            `${entry.overall.testCount.toString().padStart(5)} | ` +
-            `${costStr.padStart(8)} |`);
-    }
-    console.log();
-    // Per-model × per-area breakdown
-    for (const entry of sorted) {
-        const displayName = entry.label || entry.modelId;
-        console.log(`  ${displayName} (${entry.modelId}):`);
-        const areaH = "    | Feature Area        | Task | Code | Docs | Total | Lift |";
-        const areaSep = "    |---------------------|------|------|------|-------|------|";
-        console.log(areaH);
-        console.log(areaSep);
-        for (const s of entry.scores) {
-            const lift = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
-            console.log(`    | ${s.feature.padEnd(19)} | ` +
-                `${s.taskCompletion.toString().padStart(4)} | ` +
-                `${s.codeCorrectness.toString().padStart(4)} | ` +
-                `${s.docCoverage.toString().padStart(4)} | ` +
-                `${s.totalScore.toString().padStart(5)} | ` +
-                `${lift.padStart(4)} |`);
-        }
-        console.log();
-    }
-    // Cost-per-quality-point
-    const modelsWithCost = sorted.filter((e) => e.overall.cost && e.overall.cost > 0);
-    if (modelsWithCost.length > 0) {
-        console.log("  Cost per quality point:");
-        for (const entry of modelsWithCost) {
-            const displayName = entry.label;
-            const costPerPoint = entry.overall.avgScore > 0
-                ? (entry.overall.cost ?? 0) / entry.overall.avgScore
-                : 0;
-            console.log(`    ${displayName}: $${costPerPoint.toFixed(6)}/point (score: ${entry.overall.avgScore.toFixed(1)}, cost: $${(entry.overall.cost ?? 0).toFixed(4)})`);
-        }
-        console.log();
-    }
-}
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-function printReport(scores, urlRefs, source, agentBehavior, graderCost, perModel, sourceIsolation, sourceVerification, evaluationMode) {
-    console.log("\n" + "=".repeat(80));
-    console.log("                    SANITY AI LITERACY SCORE REPORT");
-    console.log("=".repeat(80));
-    console.log();
-    // Table header
-    const h = "| Feature Area        | Task | Code | Docs | Total | w/o Docs | Doc Lift |";
-    const sep = "|---------------------|------|------|------|-------|----------|----------|";
-    console.log(h);
-    console.log(sep);
-    for (const s of scores) {
-        const status = s.totalScore < CRITICAL_THRESHOLD ? "!!" : "ok";
-        const lift = s.docLift > 0 ? `+${s.docLift}` : `${s.docLift}`;
-        console.log(`| ${status} ${s.feature.padEnd(17)} | ` +
-            `${s.taskCompletion.toString().padStart(4)} | ` +
-            `${s.codeCorrectness.toString().padStart(4)} | ` +
-            `${s.docCoverage.toString().padStart(4)} | ` +
-            `${s.totalScore.toString().padStart(5)} | ` +
-            `${s.floorScore.toString().padStart(8)} | ` +
-            `${lift.padStart(8)} |`);
-    }
-    console.log();
-    // OKR status
-    const belowCritical = scores.filter((s) => s.totalScore < CRITICAL_THRESHOLD);
-    const lowestScore = scores.reduce((min, s) => s.totalScore < min.totalScore ? s : min);
-    const avgScore = scores.reduce((sum, s) => sum + s.totalScore, 0) / scores.length;
-    const avgLift = scores.reduce((sum, s) => sum + s.docLift, 0) / scores.length;
-    const avgCeilingScore = scores.reduce((sum, s) => sum + s.ceilingScore, 0) / scores.length;
-    const avgFloorScore = scores.reduce((sum, s) => sum + s.floorScore, 0) / scores.length;
-    const avgDocQualityGap = scores.reduce((sum, s) => sum + s.docQualityGap, 0) / scores.length;
-    const negativeDocLiftScores = scores.filter((s) => s.negativeDocLift);
-    const negativeDocLiftAreas = negativeDocLiftScores.map((s) => ({
-        area: s.feature,
-        docLift: s.docLift,
-    }));
-    console.log("-".repeat(80));
-    console.log("OKR STATUS");
-    console.log("-".repeat(80));
-    console.log();
-    if (belowCritical.length === 0) {
-        console.log("  KR1: PASS -- All areas above critical threshold (>=40)");
-    }
-    else {
-        console.log("  KR1: FAIL -- Areas below critical threshold:");
-        belowCritical.forEach((s) => console.log(`        - ${s.feature}: ${s.totalScore}`));
-    }
-    console.log();
-    console.log(`  Lowest area: ${lowestScore.feature} (${lowestScore.totalScore})`);
-    console.log(`  Target: +15 points improvement`);
-    console.log();
-    console.log(`  Avg score:     ${avgScore.toFixed(1)}`);
-    console.log(`  Avg doc lift:  +${avgLift.toFixed(1)} points`);
-    console.log(`  (Doc lift = how much docs help vs parametric knowledge alone)`);
-    console.log();
-    // Ceiling decomposition
-    console.log("-".repeat(80));
-    console.log("CEILING DECOMPOSITION");
-    console.log("-".repeat(80));
-    console.log();
-    const ceilH = "| Feature Area        | Floor | Ceiling | Doc Lift | Quality Gap |";
-    const ceilSep = "|---------------------|-------|---------|----------|-------------|";
-    console.log(ceilH);
-    console.log(ceilSep);
-    for (const s of scores) {
-        const liftStr = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
-        const liftFlag = s.negativeDocLift ? " 🚨" : "";
-        console.log(`| ${s.feature.padEnd(19)} | ` +
-            `${s.floorScore.toString().padStart(5)} | ` +
-            `${s.ceilingScore.toString().padStart(7)} | ` +
-            `${liftStr.padStart(8)}${liftFlag} | ` +
-            `${s.docQualityGap.toString().padStart(11)} |`);
-    }
-    console.log();
-    if (negativeDocLiftAreas.length > 0) {
-        console.log("  🚨 NEGATIVE DOC LIFT DETECTED:");
-        for (const { area, docLift } of negativeDocLiftAreas) {
-            const s = scores.find((sc) => sc.feature === area);
-            console.log(`     ${area}: Doc Lift = ${docLift} (floor: ${s.floorScore}, ceiling: ${s.ceilingScore})`);
-        }
-        console.log("     Documentation is HURTING model performance for these areas.");
-        console.log("     See docs/design-docs/scenario-matrix/evaluation-ceiling.md");
-        console.log();
-    }
-    else {
-        console.log("  ✅ No areas with negative Doc Lift detected.");
-        console.log();
-    }
-    // Three-layer decomposition (only when actual scores are present)
-    const hasActualScores = scores.some((s) => s.actualScore !== undefined);
-    if (hasActualScores) {
-        console.log("-".repeat(80));
-        console.log("THREE-LAYER DECOMPOSITION (floor → ceiling → actual)");
-        console.log("-".repeat(80));
-        console.log();
-        const decompH = "| Feature Area        | Floor | Ceiling | Actual | Doc Lift | Ret. Gap | Infra % |";
-        const decompSep = "|---------------------|-------|---------|--------|----------|----------|---------|";
-        console.log(decompH);
-        console.log(decompSep);
-        for (const s of scores) {
-            const liftStr = s.docLift >= 0 ? `+${s.docLift}` : `${s.docLift}`;
-            const actualStr = s.actualScore !== undefined ? s.actualScore.toString() : "—";
-            const gapStr = s.retrievalGap !== undefined
-                ? s.retrievalGap >= 0
-                    ? `+${s.retrievalGap}`
-                    : `${s.retrievalGap}`
-                : "—";
-            const infraStr = s.infrastructureEfficiency != null
-                ? `${Math.round(s.infrastructureEfficiency * 100)}%`
-                : "—";
-            const flag = s.invertedRetrievalGap ? " 🔄" : "";
-            console.log(`| ${s.feature.padEnd(19)} | ` +
-                `${s.floorScore.toString().padStart(5)} | ` +
-                `${s.ceilingScore.toString().padStart(7)} | ` +
-                `${actualStr.padStart(6)} | ` +
-                `${liftStr.padStart(8)} | ` +
-                `${(gapStr + flag).padStart(8)} | ` +
-                `${infraStr.padStart(7)} |`);
-        }
-        console.log();
-        console.log("  Doc Lift = ceiling − floor  |  Ret. Gap = ceiling − actual  |  Infra = actual / ceiling");
-        console.log("  🔄 = inverted retrieval gap (agents avoid bad docs → higher actual than ceiling)");
-        console.log();
-    }
-    // Cost summary
-    const totalCost = scores.reduce((sum, s) => sum + s.totalCost, 0);
-    const totalTests = scores.reduce((sum, s) => sum + s.testCount, 0);
-    const graderCostTotal = graderCost?.cost ?? 0;
-    const combinedCost = totalCost + graderCostTotal;
-    if (totalCost > 0 || graderCostTotal > 0) {
-        console.log("-".repeat(80));
-        console.log("COST SUMMARY");
-        console.log("-".repeat(80));
-        console.log();
-        console.log(`  Provider cost:        $${totalCost.toFixed(4)}`);
-        if (graderCostTotal > 0) {
-            const graderLabel = graderCost?.model ?? "unknown";
-            console.log(`  Grader cost:          $${graderCostTotal.toFixed(4)}  (${graderLabel}, ${(graderCost?.totalTokens ?? 0).toLocaleString()} tokens)`);
-        }
-        console.log(`  Total cost:           $${combinedCost.toFixed(4)}`);
-        console.log(`  Avg cost per test:    $${(combinedCost / (totalTests || 1)).toFixed(4)}`);
-        console.log();
-        const costHeader = "| Feature Area        | Tests |     Cost | Avg/Test |";
-        const costSep = "|---------------------|-------|----------|----------|";
-        console.log(costHeader);
-        console.log(costSep);
-        for (const s of scores) {
-            const avgCost = s.testCount > 0 ? s.totalCost / s.testCount : 0;
-            console.log(`| ${s.feature.padEnd(19)} | ` +
-                `${s.testCount.toString().padStart(5)} | ` +
-                `$${s.totalCost.toFixed(4).padStart(7)} | ` +
-                `$${avgCost.toFixed(4).padStart(7)} |`);
-        }
-        console.log();
-    }
-    // Per-model breakdown
-    if (perModel) {
-        printPerModelReport(perModel);
-    }
-    // URL References
-    printUrlReport(urlRefs);
-    // Agent Behavior (only present when run with instrumented provider)
-    if (agentBehavior && agentBehavior.length > 0) {
-        printAgentBehaviorReport(agentBehavior);
-    }
-    // Source verification (unified report for all modes)
-    if (sourceVerification || sourceIsolation) {
-        console.log("-".repeat(80));
-        console.log("📋 SOURCE VERIFICATION");
-        console.log("-".repeat(80));
-        if (sourceVerification) {
-            console.log(`  Source:  ${sourceVerification.source}`);
-            console.log(`  Mode:    ${sourceVerification.mode}`);
-            if (sourceVerification.allowedOrigins) {
-                console.log(`  Sandbox: ${sourceVerification.allowedOrigins.join(", ")}`);
-            }
-            if (sourceVerification.searchMode) {
-                console.log(`  Search:  ${sourceVerification.searchMode}`);
-            }
-            // URL fetch results (baseline mode with direct URLs)
-            if (sourceVerification.urlFetch) {
-                const uf = sourceVerification.urlFetch;
-                console.log();
-                console.log(`  URL fetch: ${uf.totalFetched} fetched, ${uf.totalFailed} failed`);
-                for (const f of uf.fetchedUrls) {
-                    console.log(`    ✅ ${f.url} (via ${f.method})`);
-                }
-                for (const f of uf.failures) {
-                    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string means no error info
-                    console.log(`    ⚠️  ${f.url}: ${f.error || "unknown error"}`);
-                }
-            }
-        }
-        // Agentic isolation score
-        if (sourceIsolation) {
-            const pct = Math.round(sourceIsolation.isolationScore * 100);
-            const icon = sourceIsolation.offOrigin === 0 ? "✅" : "⚠️";
-            console.log();
-            console.log(`  Agent isolation: ${icon} ${pct}% (${sourceIsolation.onOrigin}/${sourceIsolation.total} on-origin)`);
-            if (sourceIsolation.offOrigin > 0) {
-                console.log(`  Off-origin fetches: ${sourceIsolation.offOrigin}`);
-                for (const url of sourceIsolation.offOriginUrls.slice(0, 10)) {
-                    console.log(`    • ${url}`);
-                }
-            }
-            if (Object.keys(sourceIsolation.originBreakdown).length > 0) {
-                console.log("  Origin breakdown:");
-                for (const [origin, count] of Object.entries(sourceIsolation.originBreakdown).sort((a, b) => b[1] - a[1])) {
-                    console.log(`    ${origin}: ${count}`);
-                }
-            }
-        }
-        console.log();
-    }
-    // Build overall agent behavior stats for summary
-    const overallAgentBehavior = agentBehavior && agentBehavior.length > 0
-        ? {
-            avgDocPagesVisited: agentBehavior.reduce((s, ab) => s + ab.avgDocPagesVisited, 0) /
-                agentBehavior.length,
-            avgNetworkTimeMs: agentBehavior.reduce((s, ab) => s + ab.avgNetworkTimeMs, 0) /
-                agentBehavior.length,
-            avgSearchesPerformed: agentBehavior.reduce((s, ab) => s + ab.avgSearchesPerformed, 0) /
-                agentBehavior.length,
-            testsWithBehaviorData: agentBehavior.reduce((s, ab) => s + ab.tasksWithBehaviorData, 0),
-            totalUniqueDocSlugs: [
-                ...new Set(agentBehavior.flatMap((ab) => ab.docSlugsVisited)),
-            ].length,
-            totalUniqueSearchQueries: [
-                ...new Set(agentBehavior.flatMap((ab) => ab.searchQueries)),
-            ].length,
-        }
-        : undefined;
-    // Compute aggregate metrics from actual scores (when agentic data present)
-    const scoresWithActual = scores.filter((s) => s.actualScore !== undefined);
-    const avgActualScore = scoresWithActual.length > 0
-        ? scoresWithActual.reduce((sum, s) => sum + (s.actualScore ?? 0), 0) /
-            scoresWithActual.length
-        : undefined;
-    const scoresWithGap = scores.filter((s) => s.retrievalGap !== undefined);
-    const avgRetrievalGap = scoresWithGap.length > 0
-        ? scoresWithGap.reduce((sum, s) => sum + (s.retrievalGap ?? 0), 0) /
-            scoresWithGap.length
-        : undefined;
-    const scoresWithInfra = scores.filter((s) => s.infrastructureEfficiency != null);
-    const avgInfrastructureEfficiency = scoresWithInfra.length > 0
-        ? scoresWithInfra.reduce((sum, s) => sum + (s.infrastructureEfficiency ?? 0), 0) / scoresWithInfra.length
-        : undefined;
-    return {
-        agentBehavior: agentBehavior ?? undefined,
-        belowCritical: belowCritical.map((s) => s.feature),
-        ...(evaluationMode && { evaluationMode }),
-        lowestArea: lowestScore.feature,
-        lowestScore: lowestScore.totalScore,
-        ...(negativeDocLiftAreas.length > 0 && { negativeDocLiftAreas }),
-        overall: {
-            agentBehavior: overallAgentBehavior,
-            ...(avgActualScore !== undefined && { avgActualScore }),
-            avgCeilingScore: avgCeilingScore,
-            avgDocLift: avgLift,
-            avgDocQualityGap: avgDocQualityGap,
-            avgFloorScore: avgFloorScore,
-            ...(avgInfrastructureEfficiency !== undefined && {
-                avgInfrastructureEfficiency,
-            }),
-            ...(avgRetrievalGap !== undefined && { avgRetrievalGap }),
-            avgScore,
-            cost: totalCost > 0 || graderCostTotal > 0
-                ? {
-                    graderModel: graderCost?.model,
-                    graderTotal: graderCostTotal,
-                    perTest: combinedCost / (totalTests || 1),
-                    total: combinedCost,
-                    totalTokens: graderCost?.totalTokens ?? 0,
-                }
-                : undefined,
-            negativeDocLiftCount: negativeDocLiftAreas.length,
-        },
-        scores,
-        source: source
-            ? {
-                baseUrl: source.baseUrl,
-                dataset: source.dataset,
-                name: source.name ?? "default",
-                perspective: source.perspective,
-                projectId: source.projectId,
-            }
-            : undefined,
-        ...(perModel && { perModel }),
-        ...(sourceIsolation && { sourceIsolation }),
-        ...(sourceVerification && { sourceVerification }),
-        timestamp: new Date().toISOString(),
-        urlReferences: urlRefs,
-    };
-}
-function printUrlReport(urlRefs) {
-    console.log("-".repeat(80));
-    console.log("URL REFERENCES");
-    console.log("-".repeat(80));
-    console.log();
-    for (const ref of urlRefs) {
-        const goldUrls = Object.entries(ref.gold.urls).sort((a, b) => b[1] - a[1]);
-        const baselineUrls = Object.entries(ref.baseline.urls).sort((a, b) => b[1] - a[1]);
-        if (goldUrls.length > 0) {
-            console.log(`  ${ref.feature} (gold):`);
-            for (const [url, count] of goldUrls) {
-                const suffix = count > 1 ? ` (${count} tests)` : "";
-                console.log(`    ${url}${suffix}`);
-            }
-        }
-        if (baselineUrls.length > 0) {
-            console.log(`  ${ref.feature} (baseline):`);
-            for (const [url, count] of baselineUrls) {
-                const suffix = count > 1 ? ` (${count} tests)` : "";
-                console.log(`    ${url}${suffix} [parametric]`);
-            }
-        }
-        if (goldUrls.length === 0 && baselineUrls.length === 0) {
-            console.log(`  ${ref.feature}: no URLs referenced`);
-        }
-        console.log();
-    }
-}
-// Only run when invoked directly (not when imported for testing)
-if (process.argv[1]?.endsWith("calculate-scores.ts") ||
-    process.argv[1]?.endsWith("calculate-scores.js")) {
-    main();
-}